diff --git a/.ci/docker-compose-file/.env b/.ci/docker-compose-file/.env index 3b00b454f..12bc988bf 100644 --- a/.ci/docker-compose-file/.env +++ b/.ci/docker-compose-file/.env @@ -7,6 +7,7 @@ INFLUXDB_TAG=2.5.0 TDENGINE_TAG=3.0.2.4 DYNAMO_TAG=1.21.0 CASSANDRA_TAG=3.11.6 +MINIO_TAG=RELEASE.2023-03-20T20-16-18Z OPENTS_TAG=9aa7f88 MS_IMAGE_ADDR=mcr.microsoft.com/mssql/server diff --git a/.ci/docker-compose-file/docker-compose-minio-tcp.yaml b/.ci/docker-compose-file/docker-compose-minio-tcp.yaml new file mode 100644 index 000000000..fa78e4426 --- /dev/null +++ b/.ci/docker-compose-file/docker-compose-minio-tcp.yaml @@ -0,0 +1,21 @@ +version: '3.7' + +services: + minio: + hostname: minio + image: quay.io/minio/minio:${MINIO_TAG} + command: server --address ":9000" --console-address ":9001" /minio-data + expose: + - "9000" + - "9001" + ports: + - "9000:9000" + - "9001:9001" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 30s + timeout: 5s + retries: 3 + networks: + emqx_bridge: + diff --git a/.ci/docker-compose-file/docker-compose-minio-tls.yaml b/.ci/docker-compose-file/docker-compose-minio-tls.yaml new file mode 100644 index 000000000..4999cccb5 --- /dev/null +++ b/.ci/docker-compose-file/docker-compose-minio-tls.yaml @@ -0,0 +1,23 @@ +version: '3.7' + +services: + minio_tls: + hostname: minio-tls + image: quay.io/minio/minio:${MINIO_TAG} + command: server --certs-dir /etc/certs --address ":9100" --console-address ":9101" /minio-data + volumes: + - ./certs/server.crt:/etc/certs/public.crt + - ./certs/server.key:/etc/certs/private.key + expose: + - "9100" + - "9101" + ports: + - "9100:9100" + - "9101:9101" + healthcheck: + test: ["CMD", "curl", "-k", "-f", "https://localhost:9100/minio/health/live"] + interval: 30s + timeout: 5s + retries: 3 + networks: + emqx_bridge: diff --git a/.ci/docker-compose-file/docker-compose-toxiproxy.yaml b/.ci/docker-compose-file/docker-compose-toxiproxy.yaml index d91118406..88c2cb61a 100644 --- a/.ci/docker-compose-file/docker-compose-toxiproxy.yaml +++ b/.ci/docker-compose-file/docker-compose-toxiproxy.yaml @@ -13,19 +13,37 @@ services: volumes: - "./toxiproxy.json:/config/toxiproxy.json" ports: + # Toxiproxy management API - 8474:8474 + # InfluxDB - 8086:8086 + # InfluxDB TLS - 8087:8087 + # SQL Server - 11433:1433 + # MySQL - 13306:3306 + # MySQL TLS - 13307:3307 + # PostgreSQL - 15432:5432 + # PostgreSQL TLS - 15433:5433 + # TDEngine - 16041:6041 + # DynamoDB - 18000:8000 + # RocketMQ - 19876:9876 + # Cassandra - 19042:9042 + # Cassandra TLS - 19142:9142 + # S3 + - 19000:19000 + # S3 TLS + - 19100:19100 + # IOTDB - 14242:4242 - 28080:18080 command: diff --git a/.ci/docker-compose-file/toxiproxy.json b/.ci/docker-compose-file/toxiproxy.json index dee3134f5..c266b2792 100644 --- a/.ci/docker-compose-file/toxiproxy.json +++ b/.ci/docker-compose-file/toxiproxy.json @@ -131,5 +131,17 @@ "listen": "0.0.0.0:18080", "upstream": "iotdb:18080", "enabled": true + }, + { + "name": "minio_tcp", + "listen": "0.0.0.0:19000", + "upstream": "minio:9000", + "enabled": true + }, + { + "name": "minio_tls", + "listen": "0.0.0.0:19100", + "upstream": "minio-tls:9100", + "enabled": true } ] diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 5db0f4465..a45d9af59 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -8,6 +8,7 @@ /apps/emqx_connector/ @emqx/emqx-review-board @JimMoen /apps/emqx_dashboard/ @emqx/emqx-review-board @JimMoen @lafirest /apps/emqx_exhook/ @emqx/emqx-review-board @JimMoen @lafirest +/apps/emqx_ft/ @emqx/emqx-review-board @savonarola @keynslug /apps/emqx_gateway/ @emqx/emqx-review-board @lafirest /apps/emqx_management/ @emqx/emqx-review-board @lafirest @sstrigler /apps/emqx_plugin_libs/ @emqx/emqx-review-board @lafirest diff --git a/.github/workflows/run_conf_tests.yaml b/.github/workflows/run_conf_tests.yaml new file mode 100644 index 000000000..93d9fee95 --- /dev/null +++ b/.github/workflows/run_conf_tests.yaml @@ -0,0 +1,53 @@ +name: Run Configuration tests + +concurrency: + group: test-${{ github.event_name }}-${{ github.ref }} + cancel-in-progress: true + +on: + push: + branches: + - master + - 'ci/**' + tags: + - v* + - e* + pull_request: + +env: + IS_CI: "yes" + +jobs: + run_conf_tests: + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + profile: + - emqx + - emqx-enterprise + container: "ghcr.io/emqx/emqx-builder/5.0-34:1.13.4-25.1.2-3-ubuntu22.04" + steps: + - uses: AutoModality/action-clean@v1 + - uses: actions/checkout@v3 + with: + path: source + - name: build_package + working-directory: source + run: | + make ${{ matrix.profile }} + - name: run_tests + working-directory: source + env: + PROFILE: ${{ matrix.profile }} + run: | + ./scripts/conf-test/run.sh + - name: print_erlang_log + if: failure() + run: | + cat source/_build/${{ matrix.profile }}/rel/emqx/logs/erlang.log.* + - uses: actions/upload-artifact@v3 + if: failure() + with: + name: logs-${{ matrix.profile }} + path: source/_build/${{ matrix.profile }}/rel/emqx/logs diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 04a4c0efc..f96798493 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -195,6 +195,7 @@ jobs: INFLUXDB_TAG: "2.5.0" TDENGINE_TAG: "3.0.2.4" OPENTS_TAG: "9aa7f88" + MINIO_TAG: "RELEASE.2023-03-20T20-16-18Z" PROFILE: ${{ matrix.profile }} CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }} run: ./scripts/ct/run.sh --ci --app ${{ matrix.app }} diff --git a/Makefile b/Makefile index 1ad4421aa..785458601 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ endif # Dashbord version # from https://github.com/emqx/emqx-dashboard5 export EMQX_DASHBOARD_VERSION ?= v1.2.4-1 -export EMQX_EE_DASHBOARD_VERSION ?= e1.0.6 +export EMQX_EE_DASHBOARD_VERSION ?= e1.0.7-beta.3 # `:=` should be used here, otherwise the `$(shell ...)` will be executed every time when the variable is used # In make 4.4+, for backward-compatibility the value from the original environment is used. diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index 1b4f8a6b6..b23e59f29 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -35,7 +35,7 @@ -define(EMQX_RELEASE_CE, "5.0.25"). %% Enterprise edition --define(EMQX_RELEASE_EE, "5.0.4-alpha.1"). +-define(EMQX_RELEASE_EE, "5.0.4-alpha.2"). %% the HTTP API version -define(EMQX_API_VERSION, "5.0"). diff --git a/apps/emqx/priv/bpapi.versions b/apps/emqx/priv/bpapi.versions index dceb38c47..aabe80b7d 100644 --- a/apps/emqx/priv/bpapi.versions +++ b/apps/emqx/priv/bpapi.versions @@ -9,12 +9,16 @@ {emqx_bridge,4}. {emqx_broker,1}. {emqx_cm,1}. +{emqx_cm,2}. {emqx_conf,1}. {emqx_conf,2}. {emqx_dashboard,1}. {emqx_delayed,1}. {emqx_eviction_agent,1}. {emqx_exhook,1}. +{emqx_ft_storage_exporter_fs,1}. +{emqx_ft_storage_fs,1}. +{emqx_ft_storage_fs_reader,1}. {emqx_gateway_api_listeners,1}. {emqx_gateway_cm,1}. {emqx_gateway_http,1}. diff --git a/apps/emqx/src/emqx_calendar.erl b/apps/emqx/src/emqx_calendar.erl new file mode 100644 index 000000000..17218c6a5 --- /dev/null +++ b/apps/emqx/src/emqx_calendar.erl @@ -0,0 +1,456 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_calendar). + +-define(SECONDS_PER_MINUTE, 60). +-define(SECONDS_PER_HOUR, 3600). +-define(SECONDS_PER_DAY, 86400). +-define(DAYS_PER_YEAR, 365). +-define(DAYS_PER_LEAP_YEAR, 366). +-define(DAYS_FROM_0_TO_1970, 719528). +-define(SECONDS_FROM_0_TO_1970, ?DAYS_FROM_0_TO_1970 * ?SECONDS_PER_DAY). + +-export([ + formatter/1, + format/3, + format/4, + parse/3, + offset_second/1 +]). + +-define(DATE_PART, [ + year, + month, + day, + hour, + minute, + second, + nanosecond, + millisecond, + microsecond +]). + +-define(DATE_ZONE_NAME, [ + timezone, + timezone1, + timezone2 +]). + +formatter(FormatterStr) when is_list(FormatterStr) -> + formatter(list_to_binary(FormatterStr)); +formatter(FormatterBin) when is_binary(FormatterBin) -> + do_formatter(FormatterBin, []). + +offset_second(Offset) -> + offset_second_(Offset). + +format(Time, Unit, Formatter) -> + format(Time, Unit, undefined, Formatter). + +format(Time, Unit, Offset, FormatterBin) when is_binary(FormatterBin) -> + format(Time, Unit, Offset, formatter(FormatterBin)); +format(Time, Unit, Offset, Formatter) -> + do_format(Time, time_unit(Unit), offset_second(Offset), Formatter). + +parse(DateStr, Unit, FormatterBin) when is_binary(FormatterBin) -> + parse(DateStr, Unit, formatter(FormatterBin)); +parse(DateStr, Unit, Formatter) -> + do_parse(DateStr, Unit, Formatter). +%% ------------------------------------------------------------------------------------------------- +%% internal + +time_unit(second) -> second; +time_unit(millisecond) -> millisecond; +time_unit(microsecond) -> microsecond; +time_unit(nanosecond) -> nanosecond; +time_unit("second") -> second; +time_unit("millisecond") -> millisecond; +time_unit("microsecond") -> microsecond; +time_unit("nanosecond") -> nanosecond; +time_unit(<<"second">>) -> second; +time_unit(<<"millisecond">>) -> millisecond; +time_unit(<<"microsecond">>) -> microsecond; +time_unit(<<"nanosecond">>) -> nanosecond. + +%% ------------------------------------------------------------------------------------------------- +%% internal: format part + +do_formatter(<<>>, Formatter) -> + lists:reverse(Formatter); +do_formatter(<<"%Y", Tail/binary>>, Formatter) -> + do_formatter(Tail, [year | Formatter]); +do_formatter(<<"%m", Tail/binary>>, Formatter) -> + do_formatter(Tail, [month | Formatter]); +do_formatter(<<"%d", Tail/binary>>, Formatter) -> + do_formatter(Tail, [day | Formatter]); +do_formatter(<<"%H", Tail/binary>>, Formatter) -> + do_formatter(Tail, [hour | Formatter]); +do_formatter(<<"%M", Tail/binary>>, Formatter) -> + do_formatter(Tail, [minute | Formatter]); +do_formatter(<<"%S", Tail/binary>>, Formatter) -> + do_formatter(Tail, [second | Formatter]); +do_formatter(<<"%N", Tail/binary>>, Formatter) -> + do_formatter(Tail, [nanosecond | Formatter]); +do_formatter(<<"%3N", Tail/binary>>, Formatter) -> + do_formatter(Tail, [millisecond | Formatter]); +do_formatter(<<"%6N", Tail/binary>>, Formatter) -> + do_formatter(Tail, [microsecond | Formatter]); +do_formatter(<<"%z", Tail/binary>>, Formatter) -> + do_formatter(Tail, [timezone | Formatter]); +do_formatter(<<"%:z", Tail/binary>>, Formatter) -> + do_formatter(Tail, [timezone1 | Formatter]); +do_formatter(<<"%::z", Tail/binary>>, Formatter) -> + do_formatter(Tail, [timezone2 | Formatter]); +do_formatter(<>, [Str | Formatter]) when is_list(Str) -> + do_formatter(Tail, [lists:append(Str, [Char]) | Formatter]); +do_formatter(<>, Formatter) -> + do_formatter(Tail, [[Char] | Formatter]). + +offset_second_(OffsetSecond) when is_integer(OffsetSecond) -> OffsetSecond; +offset_second_(undefined) -> + 0; +offset_second_("local") -> + offset_second_(local); +offset_second_(<<"local">>) -> + offset_second_(local); +offset_second_(local) -> + UniversalTime = calendar:system_time_to_universal_time(erlang:system_time(second), second), + LocalTime = erlang:universaltime_to_localtime(UniversalTime), + LocalSecs = calendar:datetime_to_gregorian_seconds(LocalTime), + UniversalSecs = calendar:datetime_to_gregorian_seconds(UniversalTime), + LocalSecs - UniversalSecs; +offset_second_(Offset) when is_binary(Offset) -> + offset_second_(erlang:binary_to_list(Offset)); +offset_second_("Z") -> + 0; +offset_second_("z") -> + 0; +offset_second_(Offset) when is_list(Offset) -> + Sign = hd(Offset), + ((Sign == $+) orelse (Sign == $-)) orelse + error({bad_time_offset, Offset}), + Signs = #{$+ => 1, $- => -1}, + PosNeg = maps:get(Sign, Signs), + [Sign | HM] = Offset, + {HourStr, MinuteStr, SecondStr} = + case string:tokens(HM, ":") of + [H, M] -> + {H, M, "0"}; + [H, M, S] -> + {H, M, S}; + [HHMM] when erlang:length(HHMM) == 4 -> + {string:sub_string(HHMM, 1, 2), string:sub_string(HHMM, 3, 4), "0"}; + _ -> + error({bad_time_offset, Offset}) + end, + Hour = erlang:list_to_integer(HourStr), + Minute = erlang:list_to_integer(MinuteStr), + Second = erlang:list_to_integer(SecondStr), + (Hour =< 23) orelse error({bad_time_offset_hour, Hour}), + (Minute =< 59) orelse error({bad_time_offset_minute, Minute}), + (Second =< 59) orelse error({bad_time_offset_second, Second}), + PosNeg * (Hour * 3600 + Minute * 60 + Second). + +do_format(Time, Unit, Offset, Formatter) -> + Adjustment = erlang:convert_time_unit(Offset, second, Unit), + AdjustedTime = Time + Adjustment, + Factor = factor(Unit), + Secs = AdjustedTime div Factor, + DateTime = system_time_to_datetime(Secs), + {{Year, Month, Day}, {Hour, Min, Sec}} = DateTime, + Date = #{ + year => padding(Year, 4), + month => padding(Month, 2), + day => padding(Day, 2), + hour => padding(Hour, 2), + minute => padding(Min, 2), + second => padding(Sec, 2), + millisecond => trans_x_second(Unit, millisecond, Time), + microsecond => trans_x_second(Unit, microsecond, Time), + nanosecond => trans_x_second(Unit, nanosecond, Time) + }, + Timezones = formatter_timezones(Offset, Formatter, #{}), + DateWithZone = maps:merge(Date, Timezones), + [maps:get(Key, DateWithZone, Key) || Key <- Formatter]. + +formatter_timezones(_Offset, [], Zones) -> + Zones; +formatter_timezones(Offset, [Timezone | Formatter], Zones) -> + case lists:member(Timezone, [timezone, timezone1, timezone2]) of + true -> + NZones = Zones#{Timezone => offset_to_timezone(Offset, Timezone)}, + formatter_timezones(Offset, Formatter, NZones); + false -> + formatter_timezones(Offset, Formatter, Zones) + end. + +offset_to_timezone(Offset, Timezone) -> + Sign = + case Offset >= 0 of + true -> + $+; + false -> + $- + end, + {H, M, S} = seconds_to_time(abs(Offset)), + %% TODO: Support zone define %:::z + %% Numeric time zone with ":" to necessary precision (e.g., -04, +05:30). + case Timezone of + timezone -> + %% +0800 + io_lib:format("~c~2.10.0B~2.10.0B", [Sign, H, M]); + timezone1 -> + %% +08:00 + io_lib:format("~c~2.10.0B:~2.10.0B", [Sign, H, M]); + timezone2 -> + %% +08:00:00 + io_lib:format("~c~2.10.0B:~2.10.0B:~2.10.0B", [Sign, H, M, S]) + end. + +factor(second) -> 1; +factor(millisecond) -> 1000; +factor(microsecond) -> 1000000; +factor(nanosecond) -> 1000000000. + +system_time_to_datetime(Seconds) -> + gregorian_seconds_to_datetime(Seconds + ?SECONDS_FROM_0_TO_1970). + +gregorian_seconds_to_datetime(Secs) when Secs >= 0 -> + Days = Secs div ?SECONDS_PER_DAY, + Rest = Secs rem ?SECONDS_PER_DAY, + {gregorian_days_to_date(Days), seconds_to_time(Rest)}. + +seconds_to_time(Secs) when Secs >= 0, Secs < ?SECONDS_PER_DAY -> + Secs0 = Secs rem ?SECONDS_PER_DAY, + Hour = Secs0 div ?SECONDS_PER_HOUR, + Secs1 = Secs0 rem ?SECONDS_PER_HOUR, + Minute = Secs1 div ?SECONDS_PER_MINUTE, + Second = Secs1 rem ?SECONDS_PER_MINUTE, + {Hour, Minute, Second}. + +gregorian_days_to_date(Days) -> + {Year, DayOfYear} = day_to_year(Days), + {Month, DayOfMonth} = year_day_to_date(Year, DayOfYear), + {Year, Month, DayOfMonth}. + +day_to_year(DayOfEpoch) when DayOfEpoch >= 0 -> + YMax = DayOfEpoch div ?DAYS_PER_YEAR, + YMin = DayOfEpoch div ?DAYS_PER_LEAP_YEAR, + {Y1, D1} = dty(YMin, YMax, DayOfEpoch, dy(YMin), dy(YMax)), + {Y1, DayOfEpoch - D1}. + +year_day_to_date(Year, DayOfYear) -> + ExtraDay = + case is_leap_year(Year) of + true -> + 1; + false -> + 0 + end, + {Month, Day} = year_day_to_date2(ExtraDay, DayOfYear), + {Month, Day + 1}. + +dty(Min, Max, _D1, DMin, _DMax) when Min == Max -> + {Min, DMin}; +dty(Min, Max, D1, DMin, DMax) -> + Diff = Max - Min, + Mid = Min + Diff * (D1 - DMin) div (DMax - DMin), + MidLength = + case is_leap_year(Mid) of + true -> + ?DAYS_PER_LEAP_YEAR; + false -> + ?DAYS_PER_YEAR + end, + case dy(Mid) of + D2 when D1 < D2 -> + NewMax = Mid - 1, + dty(Min, NewMax, D1, DMin, dy(NewMax)); + D2 when D1 - D2 >= MidLength -> + NewMin = Mid + 1, + dty(NewMin, Max, D1, dy(NewMin), DMax); + D2 -> + {Mid, D2} + end. + +dy(Y) when Y =< 0 -> + 0; +dy(Y) -> + X = Y - 1, + X div 4 - X div 100 + X div 400 + X * ?DAYS_PER_YEAR + ?DAYS_PER_LEAP_YEAR. + +is_leap_year(Y) when is_integer(Y), Y >= 0 -> + is_leap_year1(Y). + +is_leap_year1(Year) when Year rem 4 =:= 0, Year rem 100 > 0 -> + true; +is_leap_year1(Year) when Year rem 400 =:= 0 -> + true; +is_leap_year1(_) -> + false. + +year_day_to_date2(_, Day) when Day < 31 -> + {1, Day}; +year_day_to_date2(E, Day) when 31 =< Day, Day < 59 + E -> + {2, Day - 31}; +year_day_to_date2(E, Day) when 59 + E =< Day, Day < 90 + E -> + {3, Day - (59 + E)}; +year_day_to_date2(E, Day) when 90 + E =< Day, Day < 120 + E -> + {4, Day - (90 + E)}; +year_day_to_date2(E, Day) when 120 + E =< Day, Day < 151 + E -> + {5, Day - (120 + E)}; +year_day_to_date2(E, Day) when 151 + E =< Day, Day < 181 + E -> + {6, Day - (151 + E)}; +year_day_to_date2(E, Day) when 181 + E =< Day, Day < 212 + E -> + {7, Day - (181 + E)}; +year_day_to_date2(E, Day) when 212 + E =< Day, Day < 243 + E -> + {8, Day - (212 + E)}; +year_day_to_date2(E, Day) when 243 + E =< Day, Day < 273 + E -> + {9, Day - (243 + E)}; +year_day_to_date2(E, Day) when 273 + E =< Day, Day < 304 + E -> + {10, Day - (273 + E)}; +year_day_to_date2(E, Day) when 304 + E =< Day, Day < 334 + E -> + {11, Day - (304 + E)}; +year_day_to_date2(E, Day) when 334 + E =< Day -> + {12, Day - (334 + E)}. + +trans_x_second(FromUnit, ToUnit, Time) -> + XSecond = do_trans_x_second(FromUnit, ToUnit, Time), + Len = + case ToUnit of + millisecond -> 3; + microsecond -> 6; + nanosecond -> 9 + end, + padding(XSecond, Len). + +do_trans_x_second(second, _, _Time) -> 0; +do_trans_x_second(millisecond, millisecond, Time) -> Time rem 1000; +do_trans_x_second(millisecond, microsecond, Time) -> (Time rem 1000) * 1000; +do_trans_x_second(millisecond, nanosecond, Time) -> (Time rem 1000) * 1000_000; +do_trans_x_second(microsecond, millisecond, Time) -> Time div 1000 rem 1000; +do_trans_x_second(microsecond, microsecond, Time) -> Time rem 1000000; +do_trans_x_second(microsecond, nanosecond, Time) -> (Time rem 1000000) * 1000; +do_trans_x_second(nanosecond, millisecond, Time) -> Time div 1000000 rem 1000; +do_trans_x_second(nanosecond, microsecond, Time) -> Time div 1000 rem 1000000; +do_trans_x_second(nanosecond, nanosecond, Time) -> Time rem 1000000000. + +padding(Data, Len) when is_integer(Data) -> + padding(integer_to_list(Data), Len); +padding(Data, Len) when Len > 0 andalso erlang:length(Data) < Len -> + [$0 | padding(Data, Len - 1)]; +padding(Data, _Len) -> + Data. + +%% ------------------------------------------------------------------------------------------------- +%% internal +%% parse part + +do_parse(DateStr, Unit, Formatter) -> + DateInfo = do_parse_date_str(DateStr, Formatter, #{}), + {Precise, PrecisionUnit} = precision(DateInfo), + Counter = + fun + (year, V, Res) -> + Res + dy(V) * ?SECONDS_PER_DAY * Precise - (?SECONDS_FROM_0_TO_1970 * Precise); + (month, V, Res) -> + Res + dm(V) * ?SECONDS_PER_DAY * Precise; + (day, V, Res) -> + Res + (V * ?SECONDS_PER_DAY * Precise); + (hour, V, Res) -> + Res + (V * ?SECONDS_PER_HOUR * Precise); + (minute, V, Res) -> + Res + (V * ?SECONDS_PER_MINUTE * Precise); + (second, V, Res) -> + Res + V * Precise; + (millisecond, V, Res) -> + case PrecisionUnit of + millisecond -> + Res + V; + microsecond -> + Res + (V * 1000); + nanosecond -> + Res + (V * 1000000) + end; + (microsecond, V, Res) -> + case PrecisionUnit of + microsecond -> + Res + V; + nanosecond -> + Res + (V * 1000) + end; + (nanosecond, V, Res) -> + Res + V; + (parsed_offset, V, Res) -> + Res - V + end, + Count = maps:fold(Counter, 0, DateInfo) - (?SECONDS_PER_DAY * Precise), + erlang:convert_time_unit(Count, PrecisionUnit, Unit). + +precision(#{nanosecond := _}) -> {1000_000_000, nanosecond}; +precision(#{microsecond := _}) -> {1000_000, microsecond}; +precision(#{millisecond := _}) -> {1000, millisecond}; +precision(#{second := _}) -> {1, second}; +precision(_) -> {1, second}. + +do_parse_date_str(<<>>, _, Result) -> + Result; +do_parse_date_str(_, [], Result) -> + Result; +do_parse_date_str(Date, [Key | Formatter], Result) -> + Size = date_size(Key), + <> = Date, + case lists:member(Key, ?DATE_PART) of + true -> + do_parse_date_str(Tail, Formatter, Result#{Key => erlang:binary_to_integer(DatePart)}); + false -> + case lists:member(Key, ?DATE_ZONE_NAME) of + true -> + do_parse_date_str(Tail, Formatter, Result#{ + parsed_offset => offset_second(DatePart) + }); + false -> + do_parse_date_str(Tail, Formatter, Result) + end + end. + +date_size(Str) when is_list(Str) -> erlang:length(Str); +date_size(year) -> 4; +date_size(month) -> 2; +date_size(day) -> 2; +date_size(hour) -> 2; +date_size(minute) -> 2; +date_size(second) -> 2; +date_size(millisecond) -> 3; +date_size(microsecond) -> 6; +date_size(nanosecond) -> 9; +date_size(timezone) -> 5; +date_size(timezone1) -> 6; +date_size(timezone2) -> 9. + +dm(1) -> 0; +dm(2) -> 31; +dm(3) -> 59; +dm(4) -> 90; +dm(5) -> 120; +dm(6) -> 151; +dm(7) -> 181; +dm(8) -> 212; +dm(9) -> 243; +dm(10) -> 273; +dm(11) -> 304; +dm(12) -> 334. diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index a2d88c647..d583b1691 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -715,9 +715,13 @@ do_publish(_PacketId, Msg = #message{qos = ?QOS_0}, Channel) -> {ok, NChannel}; do_publish(PacketId, Msg = #message{qos = ?QOS_1}, Channel) -> PubRes = emqx_broker:publish(Msg), - RC = puback_reason_code(PubRes), - NChannel = ensure_quota(PubRes, Channel), - handle_out(puback, {PacketId, RC}, NChannel); + RC = puback_reason_code(PacketId, Msg, PubRes), + case RC of + undefined -> + {ok, Channel}; + _Value -> + do_finish_publish(PacketId, PubRes, RC, Channel) + end; do_publish( PacketId, Msg = #message{qos = ?QOS_2}, @@ -725,7 +729,7 @@ do_publish( ) -> case emqx_session:publish(ClientInfo, PacketId, Msg, Session) of {ok, PubRes, NSession} -> - RC = puback_reason_code(PubRes), + RC = pubrec_reason_code(PubRes), NChannel0 = set_session(NSession, Channel), NChannel1 = ensure_timer(await_timer, NChannel0), NChannel2 = ensure_quota(PubRes, NChannel1), @@ -738,6 +742,10 @@ do_publish( handle_out(disconnect, RC, Channel) end. +do_finish_publish(PacketId, PubRes, RC, Channel) -> + NChannel = ensure_quota(PubRes, Channel), + handle_out(puback, {PacketId, RC}, NChannel). + ensure_quota(_, Channel = #channel{quota = infinity}) -> Channel; ensure_quota(PubRes, Channel = #channel{quota = Limiter}) -> @@ -757,9 +765,14 @@ ensure_quota(PubRes, Channel = #channel{quota = Limiter}) -> ensure_timer(quota_timer, Intv, Channel#channel{quota = NLimiter}) end. --compile({inline, [puback_reason_code/1]}). -puback_reason_code([]) -> ?RC_NO_MATCHING_SUBSCRIBERS; -puback_reason_code([_ | _]) -> ?RC_SUCCESS. +-compile({inline, [pubrec_reason_code/1]}). +pubrec_reason_code([]) -> ?RC_NO_MATCHING_SUBSCRIBERS; +pubrec_reason_code([_ | _]) -> ?RC_SUCCESS. + +puback_reason_code(PacketId, Msg, [] = PubRes) -> + emqx_hooks:run_fold('message.puback', [PacketId, Msg, PubRes], ?RC_NO_MATCHING_SUBSCRIBERS); +puback_reason_code(PacketId, Msg, [_ | _] = PubRes) -> + emqx_hooks:run_fold('message.puback', [PacketId, Msg, PubRes], ?RC_SUCCESS). -compile({inline, [after_message_acked/3]}). after_message_acked(ClientInfo, Msg, PubAckProps) -> @@ -1264,6 +1277,8 @@ handle_info(die_if_test = Info, Channel) -> {ok, Channel}; handle_info({disconnect, ReasonCode, ReasonName, Props}, Channel) -> handle_out(disconnect, {ReasonCode, ReasonName, Props}, Channel); +handle_info({puback, PacketId, PubRes, RC}, Channel) -> + do_finish_publish(PacketId, PubRes, RC, Channel); handle_info(Info, Channel) -> ?SLOG(error, #{msg => "unexpected_info", info => Info}), {ok, Channel}. diff --git a/apps/emqx/src/emqx_cm.erl b/apps/emqx/src/emqx_cm.erl index 66c1db36e..9a3b4e39b 100644 --- a/apps/emqx/src/emqx_cm.erl +++ b/apps/emqx/src/emqx_cm.erl @@ -97,6 +97,7 @@ mark_channel_connected/1, mark_channel_disconnected/1, get_connected_client_count/0, + takeover_finish/2, do_kick_session/3, do_get_chan_stats/2, @@ -188,11 +189,13 @@ unregister_channel(ClientId) when is_binary(ClientId) -> ok. %% @private -do_unregister_channel(Chan) -> +do_unregister_channel({_ClientId, ChanPid} = Chan) -> ok = emqx_cm_registry:unregister_channel(Chan), true = ets:delete(?CHAN_CONN_TAB, Chan), true = ets:delete(?CHAN_INFO_TAB, Chan), - ets:delete_object(?CHAN_TAB, Chan). + ets:delete_object(?CHAN_TAB, Chan), + ok = emqx_hooks:run('channel.unregistered', [ChanPid]), + true. -spec connection_closed(emqx_types:clientid()) -> true. connection_closed(ClientId) -> @@ -220,7 +223,7 @@ do_get_chan_info(ClientId, ChanPid) -> -spec get_chan_info(emqx_types:clientid(), chan_pid()) -> maybe(emqx_types:infos()). get_chan_info(ClientId, ChanPid) -> - wrap_rpc(emqx_cm_proto_v1:get_chan_info(ClientId, ChanPid)). + wrap_rpc(emqx_cm_proto_v2:get_chan_info(ClientId, ChanPid)). %% @doc Update infos of the channel. -spec set_chan_info(emqx_types:clientid(), emqx_types:attrs()) -> boolean(). @@ -250,7 +253,7 @@ do_get_chan_stats(ClientId, ChanPid) -> -spec get_chan_stats(emqx_types:clientid(), chan_pid()) -> maybe(emqx_types:stats()). get_chan_stats(ClientId, ChanPid) -> - wrap_rpc(emqx_cm_proto_v1:get_chan_stats(ClientId, ChanPid)). + wrap_rpc(emqx_cm_proto_v2:get_chan_stats(ClientId, ChanPid)). %% @doc Set channel's stats. -spec set_chan_stats(emqx_types:clientid(), emqx_types:stats()) -> boolean(). @@ -312,13 +315,7 @@ open_session(false, ClientInfo = #{clientid := ClientId}, ConnInfo) -> }}; {living, ConnMod, ChanPid, Session} -> ok = emqx_session:resume(ClientInfo, Session), - case - request_stepdown( - {takeover, 'end'}, - ConnMod, - ChanPid - ) - of + case wrap_rpc(emqx_cm_proto_v2:takeover_finish(ConnMod, ChanPid)) of {ok, Pendings} -> Session1 = emqx_persistent_session:persist( ClientInfo, ConnInfo, Session @@ -408,6 +405,13 @@ takeover_session(ClientId) -> takeover_session(ClientId, ChanPid) end. +takeover_finish(ConnMod, ChanPid) -> + request_stepdown( + {takeover, 'end'}, + ConnMod, + ChanPid + ). + takeover_session(ClientId, Pid) -> try do_takeover_session(ClientId, Pid) @@ -437,7 +441,7 @@ do_takeover_session(ClientId, ChanPid) when node(ChanPid) == node() -> end end; do_takeover_session(ClientId, ChanPid) -> - wrap_rpc(emqx_cm_proto_v1:takeover_session(ClientId, ChanPid)). + wrap_rpc(emqx_cm_proto_v2:takeover_session(ClientId, ChanPid)). %% @doc Discard all the sessions identified by the ClientId. -spec discard_session(emqx_types:clientid()) -> ok. @@ -539,7 +543,7 @@ do_kick_session(Action, ClientId, ChanPid) -> %% @private This function is shared for session 'kick' and 'discard' (as the first arg Action). kick_session(Action, ClientId, ChanPid) -> try - wrap_rpc(emqx_cm_proto_v1:kick_session(Action, ClientId, ChanPid)) + wrap_rpc(emqx_cm_proto_v2:kick_session(Action, ClientId, ChanPid)) catch Error:Reason -> %% This should mostly be RPC failures. @@ -759,7 +763,7 @@ do_get_chann_conn_mod(ClientId, ChanPid) -> end. get_chann_conn_mod(ClientId, ChanPid) -> - wrap_rpc(emqx_cm_proto_v1:get_chann_conn_mod(ClientId, ChanPid)). + wrap_rpc(emqx_cm_proto_v2:get_chann_conn_mod(ClientId, ChanPid)). mark_channel_connected(ChanPid) -> ?tp(emqx_cm_connected_client_count_inc, #{chan_pid => ChanPid}), diff --git a/apps/emqx/src/emqx_maybe.erl b/apps/emqx/src/emqx_maybe.erl new file mode 100644 index 000000000..5b5d5e94b --- /dev/null +++ b/apps/emqx/src/emqx_maybe.erl @@ -0,0 +1,90 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_maybe). + +-include_lib("emqx/include/types.hrl"). + +-export([to_list/1]). +-export([from_list/1]). +-export([define/2]). +-export([apply/2]). + +-type t(T) :: maybe(T). +-export_type([t/1]). + +-spec to_list(maybe(A)) -> [A]. +to_list(undefined) -> + []; +to_list(Term) -> + [Term]. + +-spec from_list([A]) -> maybe(A). +from_list([]) -> + undefined; +from_list([Term]) -> + Term. + +-spec define(maybe(A), B) -> A | B. +define(undefined, Term) -> + Term; +define(Term, _) -> + Term. + +%% @doc Apply a function to a maybe argument. +-spec apply(fun((A) -> maybe(A)), maybe(A)) -> + maybe(A). +apply(_Fun, undefined) -> + undefined; +apply(Fun, Term) when is_function(Fun) -> + erlang:apply(Fun, [Term]). + +%% + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +to_list_test_() -> + [ + ?_assertEqual([], to_list(undefined)), + ?_assertEqual([42], to_list(42)) + ]. + +from_list_test_() -> + [ + ?_assertEqual(undefined, from_list([])), + ?_assertEqual(3.1415, from_list([3.1415])), + ?_assertError(_, from_list([1, 2, 3])) + ]. + +define_test_() -> + [ + ?_assertEqual(42, define(42, undefined)), + ?_assertEqual(<<"default">>, define(undefined, <<"default">>)), + ?_assertEqual(undefined, define(undefined, undefined)) + ]. + +apply_test_() -> + [ + ?_assertEqual(<<"42">>, ?MODULE:apply(fun erlang:integer_to_binary/1, 42)), + ?_assertEqual(undefined, ?MODULE:apply(fun erlang:integer_to_binary/1, undefined)), + ?_assertEqual(undefined, ?MODULE:apply(fun crash/1, undefined)) + ]. + +crash(_) -> + erlang:error(crashed). + +-endif. diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 0e9a29f3d..914f87d1f 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -28,6 +28,7 @@ -include("emqx_access_control.hrl"). -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). +-include_lib("logger.hrl"). -type duration() :: integer(). -type duration_s() :: integer(). @@ -3324,6 +3325,11 @@ naive_env_interpolation("$" ++ Maybe = Original) -> {ok, Path} -> filename:join([Path, Tail]); error -> + ?SLOG(warning, #{ + msg => "failed_to_resolve_env_variable", + env => Env, + original => Original + }), Original end; naive_env_interpolation(Other) -> diff --git a/apps/emqx/src/emqx_tls_lib.erl b/apps/emqx/src/emqx_tls_lib.erl index 2683d2a9d..db0996e56 100644 --- a/apps/emqx/src/emqx_tls_lib.erl +++ b/apps/emqx/src/emqx_tls_lib.erl @@ -620,9 +620,11 @@ ensure_bin(A) when is_atom(A) -> atom_to_binary(A, utf8). ensure_ssl_file_key(_SSL, []) -> ok; ensure_ssl_file_key(SSL, RequiredKeyPaths) -> - NotFoundRef = make_ref(), Filter = fun(KeyPath) -> - NotFoundRef =:= emqx_utils_maps:deep_get(KeyPath, SSL, NotFoundRef) + case emqx_utils_maps:deep_find(KeyPath, SSL) of + {not_found, _, _} -> true; + _ -> false + end end, case lists:filter(Filter, RequiredKeyPaths) of [] -> ok; diff --git a/apps/emqx/src/emqx_types.erl b/apps/emqx/src/emqx_types.erl index b1e2c7534..a7ec0cec4 100644 --- a/apps/emqx/src/emqx_types.erl +++ b/apps/emqx/src/emqx_types.erl @@ -101,6 +101,8 @@ -export_type([oom_policy/0]). +-export_type([takeover_data/0]). + -type proto_ver() :: ?MQTT_PROTO_V3 | ?MQTT_PROTO_V4 @@ -242,3 +244,5 @@ max_heap_size => non_neg_integer(), enable => boolean() }. + +-type takeover_data() :: map(). diff --git a/apps/emqx/src/emqx_wdgraph.erl b/apps/emqx/src/emqx_wdgraph.erl new file mode 100644 index 000000000..bd7f58e7c --- /dev/null +++ b/apps/emqx/src/emqx_wdgraph.erl @@ -0,0 +1,208 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% Weighted directed graph. +%% +%% Purely functional, built on top of a single `gb_tree`. +%% Weights are currently assumed to be non-negative numbers, hovewer +%% presumably anything that is ≥ 0 should work (but won't typecheck 🥲). + +-module(emqx_wdgraph). + +-export([new/0]). +-export([insert_edge/5]). +-export([find_edge/3]). +-export([get_edges/2]). + +-export([fold/3]). + +-export([find_shortest_path/3]). + +-export_type([t/0]). +-export_type([t/2]). +-export_type([weight/0]). + +-type gnode() :: term(). +-type weight() :: _NonNegative :: number(). +-type label() :: term(). + +-opaque t() :: t(gnode(), label()). +-opaque t(Node, Label) :: gb_trees:tree({Node}, [{Node, weight(), Label}]). + +%% + +-spec new() -> t(_, _). +new() -> + gb_trees:empty(). + +%% Add an edge. +%% Nodes are not expected to exist beforehand, and created lazily. +%% There could be only one edge between each pair of nodes, this function +%% replaces any existing edge in the graph. +-spec insert_edge(Node, Node, weight(), Label, t(Node, Label)) -> t(Node, Label). +insert_edge(From, To, Weight, EdgeLabel, G) -> + Edges = tree_lookup({From}, G, []), + EdgesNext = lists:keystore(To, 1, Edges, {To, Weight, EdgeLabel}), + tree_update({From}, EdgesNext, G). + +%% Find exising edge between two nodes, if any. +-spec find_edge(Node, Node, t(Node, Label)) -> {weight(), Label} | false. +find_edge(From, To, G) -> + Edges = tree_lookup({From}, G, []), + case lists:keyfind(To, 1, Edges) of + {To, Weight, Label} -> + {Weight, Label}; + false -> + false + end. + +%% Get all edges from the given node. +-spec get_edges(Node, t(Node, Label)) -> [{Node, weight(), Label}]. +get_edges(Node, G) -> + tree_lookup({Node}, G, []). + +-spec fold(FoldFun, Acc, t(Node, Label)) -> Acc when + FoldFun :: fun((Node, _Edge :: {Node, weight(), Label}, Acc) -> Acc). +fold(FoldFun, Acc, G) -> + fold_iterator(FoldFun, Acc, gb_trees:iterator(G)). + +fold_iterator(FoldFun, AccIn, It) -> + case gb_trees:next(It) of + {{Node}, Edges = [_ | _], ItNext} -> + AccNext = lists:foldl( + fun(Edge = {_To, _Weight, _Label}, Acc) -> + FoldFun(Node, Edge, Acc) + end, + AccIn, + Edges + ), + fold_iterator(FoldFun, AccNext, ItNext); + none -> + AccIn + end. + +% Find the shortest path between two nodes, if any. If the path exists, return list +% of edge labels along that path. +% This is a Dijkstra shortest path algorithm. It is one-way right now, for +% simplicity sake. +-spec find_shortest_path(Node, Node, t(Node, Label)) -> [Label] | {false, _StoppedAt :: Node}. +find_shortest_path(From, To, G1) -> + % NOTE + % If `From` and `To` are the same node, then path is `[]` even if this + % node does not exist in the graph. + G2 = set_cost(From, 0, [], G1), + case find_shortest_path(From, 0, To, G2) of + {true, G3} -> + construct_path(From, To, [], G3); + {false, Last} -> + {false, Last} + end. + +find_shortest_path(Node, Cost, Target, G1) -> + Edges = get_edges(Node, G1), + G2 = update_neighbours(Node, Cost, Edges, G1), + case take_queued(G2) of + {Target, _NextCost, G3} -> + {true, G3}; + {Next, NextCost, G3} -> + find_shortest_path(Next, NextCost, Target, G3); + none -> + {false, Node} + end. + +construct_path(From, From, Acc, _) -> + Acc; +construct_path(From, To, Acc, G) -> + {Prev, Label} = get_label(To, G), + construct_path(From, Prev, [Label | Acc], G). + +update_neighbours(Node, NodeCost, Edges, G1) -> + lists:foldl( + fun(Edge, GAcc) -> update_neighbour(Node, NodeCost, Edge, GAcc) end, + G1, + Edges + ). + +update_neighbour(Node, NodeCost, {Neighbour, Weight, Label}, G) -> + case is_visited(G, Neighbour) of + false -> + CurrentCost = get_cost(Neighbour, G), + case NodeCost + Weight of + NeighCost when NeighCost < CurrentCost -> + set_cost(Neighbour, NeighCost, {Node, Label}, G); + _ -> + G + end; + true -> + G + end. + +get_cost(Node, G) -> + case tree_lookup({Node, cost}, G, inf) of + {Cost, _Label} -> + Cost; + inf -> + inf + end. + +get_label(Node, G) -> + {_Cost, Label} = gb_trees:get({Node, cost}, G), + Label. + +set_cost(Node, Cost, Label, G1) -> + G3 = + case tree_lookup({Node, cost}, G1, inf) of + {CostWas, _Label} -> + {true, G2} = gb_trees:take({queued, CostWas, Node}, G1), + gb_trees:insert({queued, Cost, Node}, true, G2); + inf -> + gb_trees:insert({queued, Cost, Node}, true, G1) + end, + G4 = tree_update({Node, cost}, {Cost, Label}, G3), + G4. + +take_queued(G1) -> + It = gb_trees:iterator_from({queued, 0, 0}, G1), + case gb_trees:next(It) of + {{queued, Cost, Node} = Index, true, _It} -> + {Node, Cost, gb_trees:delete(Index, G1)}; + _ -> + none + end. + +is_visited(G, Node) -> + case tree_lookup({Node, cost}, G, inf) of + inf -> + false; + {Cost, _Label} -> + not tree_lookup({queued, Cost, Node}, G, false) + end. + +tree_lookup(Index, Tree, Default) -> + case gb_trees:lookup(Index, Tree) of + {value, V} -> + V; + none -> + Default + end. + +tree_update(Index, Value, Tree) -> + case gb_trees:is_defined(Index, Tree) of + true -> + gb_trees:update(Index, Value, Tree); + false -> + gb_trees:insert(Index, Value, Tree) + end. diff --git a/apps/emqx/src/proto/emqx_cm_proto_v2.erl b/apps/emqx/src/proto/emqx_cm_proto_v2.erl new file mode 100644 index 000000000..4208df97f --- /dev/null +++ b/apps/emqx/src/proto/emqx_cm_proto_v2.erl @@ -0,0 +1,88 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_cm_proto_v2). + +-behaviour(emqx_bpapi). + +-export([ + introduced_in/0, + + lookup_client/2, + kickout_client/2, + + get_chan_stats/2, + get_chan_info/2, + get_chann_conn_mod/2, + + takeover_session/2, + takeover_finish/2, + kick_session/3 +]). + +-include("bpapi.hrl"). +-include("src/emqx_cm.hrl"). + +introduced_in() -> + "5.0.0". + +-spec kickout_client(node(), emqx_types:clientid()) -> ok | {badrpc, _}. +kickout_client(Node, ClientId) -> + rpc:call(Node, emqx_cm, kick_session, [ClientId]). + +-spec lookup_client(node(), {clientid, emqx_types:clientid()} | {username, emqx_types:username()}) -> + [emqx_cm:channel_info()] | {badrpc, _}. +lookup_client(Node, Key) -> + rpc:call(Node, emqx_cm, lookup_client, [Key]). + +-spec get_chan_stats(emqx_types:clientid(), emqx_cm:chan_pid()) -> emqx_types:stats() | {badrpc, _}. +get_chan_stats(ClientId, ChanPid) -> + rpc:call(node(ChanPid), emqx_cm, do_get_chan_stats, [ClientId, ChanPid], ?T_GET_INFO * 2). + +-spec get_chan_info(emqx_types:clientid(), emqx_cm:chan_pid()) -> emqx_types:infos() | {badrpc, _}. +get_chan_info(ClientId, ChanPid) -> + rpc:call(node(ChanPid), emqx_cm, do_get_chan_info, [ClientId, ChanPid], ?T_GET_INFO * 2). + +-spec get_chann_conn_mod(emqx_types:clientid(), emqx_cm:chan_pid()) -> + module() | undefined | {badrpc, _}. +get_chann_conn_mod(ClientId, ChanPid) -> + rpc:call(node(ChanPid), emqx_cm, do_get_chann_conn_mod, [ClientId, ChanPid], ?T_GET_INFO * 2). + +-spec takeover_session(emqx_types:clientid(), emqx_cm:chan_pid()) -> + none + | {expired | persistent, emqx_session:session()} + | {living, _ConnMod :: atom(), emqx_cm:chan_pid(), emqx_session:session()} + | {badrpc, _}. +takeover_session(ClientId, ChanPid) -> + rpc:call(node(ChanPid), emqx_cm, takeover_session, [ClientId, ChanPid], ?T_TAKEOVER * 2). + +-spec takeover_finish(module(), emqx_cm:chan_pid()) -> + {ok, emqx_type:takeover_data()} + | {ok, list(emqx_type:deliver()), emqx_type:takeover_data()} + | {error, term()} + | {badrpc, _}. +takeover_finish(ConnMod, ChanPid) -> + erpc:call( + node(ChanPid), + emqx_cm, + takeover_finish, + [ConnMod, ChanPid], + ?T_TAKEOVER * 2 + ). + +-spec kick_session(kick | discard, emqx_types:clientid(), emqx_cm:chan_pid()) -> ok | {badrpc, _}. +kick_session(Action, ClientId, ChanPid) -> + rpc:call(node(ChanPid), emqx_cm, do_kick_session, [Action, ClientId, ChanPid], ?T_KICK * 2). diff --git a/apps/emqx/test/emqx_channel_SUITE.erl b/apps/emqx/test/emqx_channel_SUITE.erl index e822bf5e7..6f14293ed 100644 --- a/apps/emqx/test/emqx_channel_SUITE.erl +++ b/apps/emqx/test/emqx_channel_SUITE.erl @@ -1133,7 +1133,7 @@ t_ws_cookie_init(_) -> ?assertMatch(#{ws_cookie := WsCookie}, emqx_channel:info(clientinfo, Channel)). %%-------------------------------------------------------------------- -%% Test cases for other mechnisms +%% Test cases for other mechanisms %%-------------------------------------------------------------------- t_flapping_detect(_) -> diff --git a/apps/emqx/test/emqx_channel_delayed_puback_SUITE.erl b/apps/emqx/test/emqx_channel_delayed_puback_SUITE.erl new file mode 100644 index 000000000..4f2938b24 --- /dev/null +++ b/apps/emqx/test/emqx_channel_delayed_puback_SUITE.erl @@ -0,0 +1,70 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2018-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_channel_delayed_puback_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("emqx/include/emqx_hooks.hrl"). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + emqx_common_test_helpers:boot_modules(all), + emqx_common_test_helpers:start_apps([]), + Config. + +end_per_suite(_Config) -> + emqx_common_test_helpers:stop_apps([]). + +init_per_testcase(Case, Config) -> + ?MODULE:Case({init, Config}). + +end_per_testcase(Case, Config) -> + ?MODULE:Case({'end', Config}). + +%%-------------------------------------------------------------------- +%% Test cases +%%-------------------------------------------------------------------- + +t_delayed_puback({init, Config}) -> + emqx_hooks:put('message.puback', {?MODULE, on_message_puback, []}, ?HP_LOWEST), + Config; +t_delayed_puback({'end', _Config}) -> + emqx_hooks:del('message.puback', {?MODULE, on_message_puback}); +t_delayed_puback(_Config) -> + {ok, ConnPid} = emqtt:start_link([{clientid, <<"clientid">>}, {proto_ver, v5}]), + {ok, _} = emqtt:connect(ConnPid), + {ok, #{reason_code := ?RC_UNSPECIFIED_ERROR}} = emqtt:publish( + ConnPid, <<"topic">>, <<"hello">>, 1 + ), + emqtt:disconnect(ConnPid). + +%%-------------------------------------------------------------------- +%% Helpers +%%-------------------------------------------------------------------- + +on_message_puback(PacketId, _Msg, PubRes, _RC) -> + erlang:send(self(), {puback, PacketId, PubRes, ?RC_UNSPECIFIED_ERROR}), + {stop, undefined}. diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index c22944209..44897793b 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -30,6 +30,7 @@ start_apps/1, start_apps/2, start_apps/3, + start_app/2, stop_apps/1, stop_apps/2, reload/2, @@ -246,6 +247,9 @@ do_render_app_config(App, Schema, ConfigFile, Opts) -> copy_certs(App, RenderedConfigFile), ok. +start_app(App, SpecAppConfig) -> + start_app(App, SpecAppConfig, #{}). + start_app(App, SpecAppConfig, Opts) -> render_and_load_app_config(App, Opts), SpecAppConfig(App), @@ -326,12 +330,7 @@ read_schema_configs(no_schema, _ConfigFile) -> ok; read_schema_configs(Schema, ConfigFile) -> NewConfig = generate_config(Schema, ConfigFile), - lists:foreach( - fun({App, Configs}) -> - [application:set_env(App, Par, Value) || {Par, Value} <- Configs] - end, - NewConfig - ). + application:set_env(NewConfig). generate_config(SchemaModule, ConfigFile) when is_atom(SchemaModule) -> {ok, Conf0} = hocon:load(ConfigFile, #{format => richmap}), @@ -687,11 +686,17 @@ emqx_cluster(Specs0, CommonOpts) -> ]), %% Set the default node of the cluster: CoreNodes = [node_name(Name) || {{core, Name, _}, _} <- Specs], - JoinTo = + JoinTo0 = case CoreNodes of [First | _] -> First; _ -> undefined end, + JoinTo = + case maps:find(join_to, CommonOpts) of + {ok, true} -> JoinTo0; + {ok, JT} -> JT; + error -> JoinTo0 + end, [ {Name, merge_opts(Opts, #{ diff --git a/apps/emqx/test/emqx_proper_types.erl b/apps/emqx/test/emqx_proper_types.erl index 2f0f9d494..e1d95227b 100644 --- a/apps/emqx/test/emqx_proper_types.erl +++ b/apps/emqx/test/emqx_proper_types.erl @@ -43,12 +43,21 @@ ip/0, port/0, limited_atom/0, - limited_latin_atom/0 + limited_latin_atom/0, + printable_utf8/0, + printable_codepoint/0 +]). + +%% Generic Types +-export([ + scaled/2 ]). %% Iterators -export([nof/1]). +-type proptype() :: proper_types:raw_type(). + %%-------------------------------------------------------------------- %% Types High level %%-------------------------------------------------------------------- @@ -606,6 +615,20 @@ limited_atom() -> limited_any_term() -> oneof([binary(), number(), string()]). +printable_utf8() -> + ?SUCHTHAT( + String, + ?LET(L, list(printable_codepoint()), unicode:characters_to_binary(L)), + is_binary(String) + ). + +printable_codepoint() -> + frequency([ + {7, range(16#20, 16#7E)}, + {2, range(16#00A0, 16#D7FF)}, + {1, range(16#E000, 16#FFFD)} + ]). + %%-------------------------------------------------------------------- %% Iterators %%-------------------------------------------------------------------- @@ -632,6 +655,14 @@ limited_list(N, T) -> end ). +%%-------------------------------------------------------------------- +%% Generic Types +%%-------------------------------------------------------------------- + +-spec scaled(number(), proptype()) -> proptype(). +scaled(F, T) when F > 0 -> + ?SIZED(S, resize(round(S * F), T)). + %%-------------------------------------------------------------------- %% Internal funcs %%-------------------------------------------------------------------- diff --git a/apps/emqx/test/emqx_wdgraph_tests.erl b/apps/emqx/test/emqx_wdgraph_tests.erl new file mode 100644 index 000000000..c159a0f49 --- /dev/null +++ b/apps/emqx/test/emqx_wdgraph_tests.erl @@ -0,0 +1,104 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_wdgraph_tests). + +-include_lib("eunit/include/eunit.hrl"). + +empty_test_() -> + G = emqx_wdgraph:new(), + [ + ?_assertEqual([], emqx_wdgraph:get_edges(foo, G)), + ?_assertEqual(false, emqx_wdgraph:find_edge(foo, bar, G)) + ]. + +edges_nodes_test_() -> + G1 = emqx_wdgraph:new(), + G2 = emqx_wdgraph:insert_edge(foo, bar, 42, "fancy", G1), + G3 = emqx_wdgraph:insert_edge(bar, baz, 1, "cheapest", G2), + G4 = emqx_wdgraph:insert_edge(bar, foo, 0, "free", G3), + G5 = emqx_wdgraph:insert_edge(foo, bar, 100, "luxury", G4), + [ + ?_assertEqual({42, "fancy"}, emqx_wdgraph:find_edge(foo, bar, G2)), + ?_assertEqual({100, "luxury"}, emqx_wdgraph:find_edge(foo, bar, G5)), + ?_assertEqual([{bar, 100, "luxury"}], emqx_wdgraph:get_edges(foo, G5)), + + ?_assertEqual({1, "cheapest"}, emqx_wdgraph:find_edge(bar, baz, G5)), + ?_assertEqual([{baz, 1, "cheapest"}, {foo, 0, "free"}], emqx_wdgraph:get_edges(bar, G5)) + ]. + +fold_test_() -> + G1 = emqx_wdgraph:new(), + G2 = emqx_wdgraph:insert_edge(foo, bar, 42, "fancy", G1), + G3 = emqx_wdgraph:insert_edge(bar, baz, 1, "cheapest", G2), + G4 = emqx_wdgraph:insert_edge(bar, foo, 0, "free", G3), + G5 = emqx_wdgraph:insert_edge(foo, bar, 100, "luxury", G4), + [ + ?_assertEqual( + % 100 + 0 + 1 + 101, + emqx_wdgraph:fold(fun(_From, {_, Weight, _}, Acc) -> Weight + Acc end, 0, G5) + ), + ?_assertEqual( + [bar, baz, foo], + lists:usort( + emqx_wdgraph:fold(fun(From, {To, _, _}, Acc) -> [From, To | Acc] end, [], G5) + ) + ) + ]. + +nonexistent_nodes_path_test_() -> + G1 = emqx_wdgraph:new(), + G2 = emqx_wdgraph:insert_edge(foo, bar, 42, "fancy", G1), + G3 = emqx_wdgraph:insert_edge(bar, baz, 1, "cheapest", G2), + [ + ?_assertEqual( + {false, nosuchnode}, + emqx_wdgraph:find_shortest_path(nosuchnode, baz, G3) + ), + ?_assertEqual( + [], + emqx_wdgraph:find_shortest_path(nosuchnode, nosuchnode, G3) + ) + ]. + +nonexistent_path_test_() -> + G1 = emqx_wdgraph:new(), + G2 = emqx_wdgraph:insert_edge(foo, bar, 42, "fancy", G1), + G3 = emqx_wdgraph:insert_edge(baz, boo, 1, "cheapest", G2), + G4 = emqx_wdgraph:insert_edge(boo, last, 3.5, "change", G3), + [ + ?_assertEqual( + {false, last}, + emqx_wdgraph:find_shortest_path(baz, foo, G4) + ), + ?_assertEqual( + {false, bar}, + emqx_wdgraph:find_shortest_path(foo, last, G4) + ) + ]. + +shortest_path_test() -> + G1 = emqx_wdgraph:new(), + G2 = emqx_wdgraph:insert_edge(foo, bar, 42, "fancy", G1), + G3 = emqx_wdgraph:insert_edge(bar, baz, 1, "cheapest", G2), + G4 = emqx_wdgraph:insert_edge(baz, last, 0, "free", G3), + G5 = emqx_wdgraph:insert_edge(bar, last, 100, "luxury", G4), + G6 = emqx_wdgraph:insert_edge(bar, foo, 0, "comeback", G5), + ?assertEqual( + ["fancy", "cheapest", "free"], + emqx_wdgraph:find_shortest_path(foo, last, G6) + ). diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index 27157d82f..f39d920fc 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -886,6 +886,29 @@ format_metrics(#{ Rate5m, RateMax, Rcvd + ); +format_metrics(_Metrics) -> + %% Empty metrics: can happen when a node joins another and a + %% bridge is not yet replicated to it, so the counters map is + %% empty. + ?METRICS( + _Dropped = 0, + _DroppedOther = 0, + _DroppedExpired = 0, + _DroppedQueueFull = 0, + _DroppedResourceNotFound = 0, + _DroppedResourceStopped = 0, + _Matched = 0, + _Queued = 0, + _Retried = 0, + _LateReply = 0, + _SentFailed = 0, + _SentInflight = 0, + _SentSucc = 0, + _Rate = 0, + _Rate5m = 0, + _RateMax = 0, + _Rcvd = 0 ). fill_defaults(Type, RawConf) -> diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl index b590f0cd4..90e62764a 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl @@ -51,11 +51,39 @@ post_request() -> api_schema(Method) -> Broker = [ - ref(Mod, Method) - || Mod <- [emqx_bridge_webhook_schema, emqx_bridge_mqtt_schema] + {Type, ref(Mod, Method)} + || {Type, Mod} <- [ + {<<"webhook">>, emqx_bridge_webhook_schema}, + {<<"mqtt">>, emqx_bridge_mqtt_schema} + ] ], EE = ee_api_schemas(Method), - hoconsc:union(Broker ++ EE). + hoconsc:union(bridge_api_union(Broker ++ EE)). + +bridge_api_union(Refs) -> + Index = maps:from_list(Refs), + fun + (all_union_members) -> + maps:values(Index); + ({value, V}) -> + case V of + #{<<"type">> := T} -> + case maps:get(T, Index, undefined) of + undefined -> + throw(#{ + field_name => type, + reason => <<"unknown bridge type">> + }); + Ref -> + [Ref] + end; + _ -> + throw(#{ + field_name => type, + reason => <<"unknown bridge type">> + }) + end + end. -if(?EMQX_RELEASE_EDITION == ee). ee_api_schemas(Method) -> diff --git a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl index 288b1da29..49ddd19bd 100644 --- a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl @@ -70,18 +70,22 @@ all() -> [ {group, single}, + {group, cluster_later_join}, {group, cluster} ]. groups() -> + AllTCs = emqx_common_test_helpers:all(?MODULE), SingleOnlyTests = [ t_broken_bpapi_vsn, t_old_bpapi_vsn, t_bridges_probe ], + ClusterLaterJoinOnlyTCs = [t_cluster_later_join_metrics], [ - {single, [], emqx_common_test_helpers:all(?MODULE)}, - {cluster, [], emqx_common_test_helpers:all(?MODULE) -- SingleOnlyTests} + {single, [], AllTCs -- ClusterLaterJoinOnlyTCs}, + {cluster_later_join, [], ClusterLaterJoinOnlyTCs}, + {cluster, [], (AllTCs -- SingleOnlyTests) -- ClusterLaterJoinOnlyTCs} ]. suite() -> @@ -104,6 +108,17 @@ init_per_group(cluster, Config) -> ok = erpc:call(NodePrimary, fun() -> init_node(primary) end), _ = [ok = erpc:call(Node, fun() -> init_node(regular) end) || Node <- NodesRest], [{group, cluster}, {cluster_nodes, Nodes}, {api_node, NodePrimary} | Config]; +init_per_group(cluster_later_join, Config) -> + Cluster = mk_cluster_specs(Config, #{join_to => undefined}), + ct:pal("Starting ~p", [Cluster]), + Nodes = [ + emqx_common_test_helpers:start_slave(Name, Opts) + || {Name, Opts} <- Cluster + ], + [NodePrimary | NodesRest] = Nodes, + ok = erpc:call(NodePrimary, fun() -> init_node(primary) end), + _ = [ok = erpc:call(Node, fun() -> init_node(regular) end) || Node <- NodesRest], + [{group, cluster_later_join}, {cluster_nodes, Nodes}, {api_node, NodePrimary} | Config]; init_per_group(_, Config) -> ok = emqx_mgmt_api_test_util:init_suite(?SUITE_APPS), ok = load_suite_config(emqx_rule_engine), @@ -111,6 +126,9 @@ init_per_group(_, Config) -> [{group, single}, {api_node, node()} | Config]. mk_cluster_specs(Config) -> + mk_cluster_specs(Config, #{}). + +mk_cluster_specs(Config, Opts) -> Specs = [ {core, emqx_bridge_api_SUITE1, #{}}, {core, emqx_bridge_api_SUITE2, #{}} @@ -132,6 +150,7 @@ mk_cluster_specs(Config) -> load_apps => ?SUITE_APPS ++ [emqx_dashboard], env_handler => fun load_suite_config/1, load_schema => false, + join_to => maps:get(join_to, Opts, true), priv_data_dir => ?config(priv_dir, Config) }, emqx_common_test_helpers:emqx_cluster(Specs, CommonOpts). @@ -142,7 +161,7 @@ init_node(Type) -> primary -> ok = emqx_config:put( [dashboard, listeners], - #{http => #{enable => true, bind => 18083}} + #{http => #{enable => true, bind => 18083}, proxy_header => false} ), ok = emqx_dashboard:start_listeners(), ready = emqx_dashboard_listener:regenerate_minirest_dispatch(), @@ -164,7 +183,10 @@ load_suite_config(emqx_bridge) -> load_suite_config(_) -> ok. -end_per_group(cluster, Config) -> +end_per_group(Group, Config) when + Group =:= cluster; + Group =:= cluster_later_join +-> ok = lists:foreach( fun(Node) -> _ = erpc:call(Node, emqx_common_test_helpers, stop_apps, [?SUITE_APPS]), @@ -1298,6 +1320,44 @@ t_inconsistent_webhook_request_timeouts(Config) -> validate_resource_request_timeout(proplists:get_value(group, Config), 1000, Name), ok. +t_cluster_later_join_metrics(Config) -> + Port = ?config(port, Config), + APINode = ?config(api_node, Config), + ClusterNodes = ?config(cluster_nodes, Config), + [OtherNode | _] = ClusterNodes -- [APINode], + URL1 = ?URL(Port, "path1"), + Name = ?BRIDGE_NAME, + BridgeParams = ?HTTP_BRIDGE(URL1, Name), + BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_HTTP, Name), + ?check_trace( + begin + %% Create a bridge on only one of the nodes. + ?assertMatch({ok, 201, _}, request_json(post, uri(["bridges"]), BridgeParams, Config)), + %% Pre-condition. + ?assertMatch( + {ok, 200, #{ + <<"metrics">> := #{<<"success">> := _}, + <<"node_metrics">> := [_ | _] + }}, + request_json(get, uri(["bridges", BridgeID, "metrics"]), Config) + ), + %% Now join the other node join with the api node. + ok = erpc:call(OtherNode, ekka, join, [APINode]), + %% Check metrics; shouldn't crash even if the bridge is not + %% ready on the node that just joined the cluster. + ?assertMatch( + {ok, 200, #{ + <<"metrics">> := #{<<"success">> := _}, + <<"node_metrics">> := [#{<<"metrics">> := #{}}, #{<<"metrics">> := #{}} | _] + }}, + request_json(get, uri(["bridges", BridgeID, "metrics"]), Config) + ), + ok + end, + [] + ), + ok. + validate_resource_request_timeout(single, Timeout, Name) -> SentData = #{payload => <<"Hello EMQX">>, timestamp => 1668602148000}, BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_HTTP, Name), diff --git a/apps/emqx_bridge_dynamo/rebar.config b/apps/emqx_bridge_dynamo/rebar.config index fbccb5c9a..d3ba1093d 100644 --- a/apps/emqx_bridge_dynamo/rebar.config +++ b/apps/emqx_bridge_dynamo/rebar.config @@ -1,6 +1,6 @@ %% -*- mode: erlang; -*- {erl_opts, [debug_info]}. -{deps, [ {erlcloud, {git, "https://github.com/emqx/erlcloud.git", {tag, "3.5.16-emqx-1"}}} +{deps, [ {erlcloud, {git, "https://github.com/emqx/erlcloud", {tag, "3.6.8-emqx-1"}}} , {emqx_connector, {path, "../../apps/emqx_connector"}} , {emqx_resource, {path, "../../apps/emqx_resource"}} , {emqx_bridge, {path, "../../apps/emqx_bridge"}} diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl index 5a87d8a0c..7d1b20d24 100644 --- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl @@ -45,9 +45,19 @@ fields(config) -> } )}, {authentication, - mk(hoconsc:union([none, ref(auth_basic), ref(auth_token)]), #{ - default => none, desc => ?DESC("authentication") - })} + mk( + hoconsc:union(fun auth_union_member_selector/1), + #{ + default => none, + %% must mark this whole union as sensitive because + %% hocon ignores the `sensitive' metadata in struct + %% fields... Also, when trying to type check a struct + %% that doesn't match the intended type, it won't have + %% sensitivity information from sibling types. + sensitive => true, + desc => ?DESC("authentication") + } + )} ] ++ emqx_connector_schema_lib:ssl_fields(); fields(producer_opts) -> [ @@ -226,3 +236,21 @@ override_default(OriginalFn, NewDefault) -> (default) -> NewDefault; (Field) -> OriginalFn(Field) end. + +auth_union_member_selector(all_union_members) -> + [none, ref(auth_basic), ref(auth_token)]; +auth_union_member_selector({value, V}) -> + case V of + #{<<"password">> := _} -> + [ref(auth_basic)]; + #{<<"jwt">> := _} -> + [ref(auth_token)]; + <<"none">> -> + [none]; + _ -> + Expected = "none | basic | token", + throw(#{ + field_name => authentication, + expected => Expected + }) + end. diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl index 300fe9b2d..59956e1b6 100644 --- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl @@ -100,7 +100,7 @@ on_start(InstanceId, Config) -> msg => "failed_to_start_pulsar_client", instance_id => InstanceId, pulsar_hosts => Servers, - reason => Reason + reason => emqx_utils:redact(Reason, fun is_sensitive_key/1) }), throw(failed_to_start_pulsar_client) end, @@ -332,7 +332,7 @@ start_producer(Config, InstanceId, ClientId, ClientOpts) -> #{ instance_id => InstanceId, kind => Kind, - reason => Error, + reason => emqx_utils:redact(Error, fun is_sensitive_key/1), stacktrace => Stacktrace } ), @@ -419,3 +419,6 @@ get_producer_status(Producers) -> partition_strategy(key_dispatch) -> first_key_dispatch; partition_strategy(Strategy) -> Strategy. + +is_sensitive_key(auth_data) -> true; +is_sensitive_key(_) -> false. diff --git a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src index 141973e1e..321a2b724 100644 --- a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src +++ b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_tdengine, [ {description, "EMQX Enterprise TDEngine Bridge"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {registered, []}, {applications, [kernel, stdlib, tdengine]}, {env, []}, diff --git a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl index 46a70e8b6..5644d0446 100644 --- a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl +++ b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl @@ -25,7 +25,7 @@ on_get_status/2 ]). --export([connect/1, do_get_status/1, execute/3]). +-export([connect/1, do_get_status/1, execute/3, do_batch_insert/4]). -import(hoconsc, [mk/2, enum/1, ref/2]). @@ -124,32 +124,36 @@ on_stop(InstanceId, #{pool_name := PoolName}) -> on_query(InstanceId, {query, SQL}, State) -> do_query(InstanceId, SQL, State); -on_query(InstanceId, Request, State) -> - %% because the `emqx-tdengine` client only supports a single SQL cmd - %% so the `on_query` and `on_batch_query` have the same process, that is: - %% we need to collect all data into one SQL cmd and then call the insert API - on_batch_query(InstanceId, [Request], State). - -on_batch_query( - InstanceId, - BatchReq, - #{batch_inserts := Inserts, batch_params_tokens := ParamsTokens} = State -) -> - case hd(BatchReq) of - {Key, _} -> - case maps:get(Key, Inserts, undefined) of - undefined -> - {error, {unrecoverable_error, batch_prepare_not_implemented}}; - InsertSQL -> - Tokens = maps:get(Key, ParamsTokens), - do_batch_insert(InstanceId, BatchReq, InsertSQL, Tokens, State) - end; - Request -> - LogMeta = #{connector => InstanceId, first_request => Request, state => State}, - ?SLOG(error, LogMeta#{msg => "invalid request"}), +on_query(InstanceId, {Key, Data}, #{insert_tokens := InsertTksMap} = State) -> + case maps:find(Key, InsertTksMap) of + {ok, Tokens} -> + SQL = emqx_plugin_libs_rule:proc_sql_param_str(Tokens, Data), + do_query(InstanceId, SQL, State); + _ -> {error, {unrecoverable_error, invalid_request}} end. +%% aggregate the batch queries to one SQL is a heavy job, we should put it in the worker process +on_batch_query( + InstanceId, + [{Key, _} | _] = BatchReq, + #{batch_tokens := BatchTksMap, query_opts := Opts} = State +) -> + case maps:find(Key, BatchTksMap) of + {ok, Tokens} -> + do_query_job( + InstanceId, + {?MODULE, do_batch_insert, [Tokens, BatchReq, Opts]}, + State + ); + _ -> + {error, {unrecoverable_error, batch_prepare_not_implemented}} + end; +on_batch_query(InstanceId, BatchReq, State) -> + LogMeta = #{connector => InstanceId, request => BatchReq, state => State}, + ?SLOG(error, LogMeta#{msg => "invalid request"}), + {error, {unrecoverable_error, invalid_request}}. + on_get_status(_InstanceId, #{pool_name := PoolName}) -> Health = emqx_resource_pool:health_check_workers(PoolName, fun ?MODULE:do_get_status/1), status_result(Health). @@ -167,17 +171,16 @@ status_result(_Status = false) -> connecting. %% Helper fns %%======================================================================================== -do_batch_insert(InstanceId, BatchReqs, InsertPart, Tokens, State) -> - SQL = emqx_plugin_libs_rule:proc_batch_sql(BatchReqs, InsertPart, Tokens), - do_query(InstanceId, SQL, State). +do_query(InstanceId, Query, #{query_opts := Opts} = State) -> + do_query_job(InstanceId, {?MODULE, execute, [Query, Opts]}, State). -do_query(InstanceId, Query, #{pool_name := PoolName, query_opts := Opts} = State) -> +do_query_job(InstanceId, Job, #{pool_name := PoolName} = State) -> ?TRACE( "QUERY", "tdengine_connector_received", - #{connector => InstanceId, query => Query, state => State} + #{connector => InstanceId, job => Job, state => State} ), - Result = ecpool:pick_and_do(PoolName, {?MODULE, execute, [Query, Opts]}, no_handover), + Result = ecpool:pick_and_do(PoolName, Job, no_handover), case Result of {error, Reason} -> @@ -188,7 +191,7 @@ do_query(InstanceId, Query, #{pool_name := PoolName, query_opts := Opts} = State ?SLOG(error, #{ msg => "tdengine_connector_do_query_failed", connector => InstanceId, - query => Query, + job => Job, reason => Reason }), Result; @@ -203,6 +206,35 @@ do_query(InstanceId, Query, #{pool_name := PoolName, query_opts := Opts} = State execute(Conn, Query, Opts) -> tdengine:insert(Conn, Query, Opts). +do_batch_insert(Conn, Tokens, BatchReqs, Opts) -> + Queries = aggregate_query(Tokens, BatchReqs), + SQL = maps:fold( + fun(InsertPart, Values, Acc) -> + lists:foldl( + fun(ValuePart, IAcc) -> + <> + end, + <>, + Values + ) + end, + <<"INSERT INTO">>, + Queries + ), + execute(Conn, SQL, Opts). + +aggregate_query({InsertPartTks, ParamsPartTks}, BatchReqs) -> + lists:foldl( + fun({_, Data}, Acc) -> + InsertPart = emqx_plugin_libs_rule:proc_sql_param_str(InsertPartTks, Data), + ParamsPart = emqx_plugin_libs_rule:proc_sql_param_str(ParamsPartTks, Data), + Values = maps:get(InsertPart, Acc, []), + maps:put(InsertPart, [ParamsPart | Values], Acc) + end, + #{}, + BatchReqs + ). + connect(Opts) -> tdengine:start_link(Opts). @@ -218,32 +250,49 @@ parse_prepare_sql(Config) -> parse_batch_prepare_sql(maps:to_list(SQL), #{}, #{}). -parse_batch_prepare_sql([{Key, H} | T], BatchInserts, BatchTks) -> +parse_batch_prepare_sql([{Key, H} | T], InsertTksMap, BatchTksMap) -> case emqx_plugin_libs_rule:detect_sql_type(H) of {ok, select} -> - parse_batch_prepare_sql(T, BatchInserts, BatchTks); + parse_batch_prepare_sql(T, InsertTksMap, BatchTksMap); {ok, insert} -> - case emqx_plugin_libs_rule:split_insert_sql(H) of - {ok, {InsertSQL, Params}} -> - ParamsTks = emqx_plugin_libs_rule:preproc_tmpl(Params), + InsertTks = emqx_plugin_libs_rule:preproc_tmpl(H), + H1 = string:trim(H, trailing, ";"), + case split_insert_sql(H1) of + [_InsertStr, InsertPart, _ValuesStr, ParamsPart] -> + InsertPartTks = emqx_plugin_libs_rule:preproc_tmpl(InsertPart), + ParamsPartTks = emqx_plugin_libs_rule:preproc_tmpl(ParamsPart), parse_batch_prepare_sql( T, - BatchInserts#{Key => InsertSQL}, - BatchTks#{Key => ParamsTks} + InsertTksMap#{Key => InsertTks}, + BatchTksMap#{Key => {InsertPartTks, ParamsPartTks}} ); - {error, Reason} -> - ?SLOG(error, #{msg => "split sql failed", sql => H, reason => Reason}), - parse_batch_prepare_sql(T, BatchInserts, BatchTks) + Result -> + ?SLOG(error, #{msg => "split sql failed", sql => H, result => Result}), + parse_batch_prepare_sql(T, InsertTksMap, BatchTksMap) end; {error, Reason} -> ?SLOG(error, #{msg => "detect sql type failed", sql => H, reason => Reason}), - parse_batch_prepare_sql(T, BatchInserts, BatchTks) + parse_batch_prepare_sql(T, InsertTksMap, BatchTksMap) end; -parse_batch_prepare_sql([], BatchInserts, BatchTks) -> +parse_batch_prepare_sql([], InsertTksMap, BatchTksMap) -> #{ - batch_inserts => BatchInserts, - batch_params_tokens => BatchTks + insert_tokens => InsertTksMap, + batch_tokens => BatchTksMap }. to_bin(List) when is_list(List) -> unicode:characters_to_binary(List, utf8). + +split_insert_sql(SQL0) -> + SQL = emqx_plugin_libs_rule:formalize_sql(SQL0), + lists:filtermap( + fun(E) -> + case string:trim(E) of + <<>> -> + false; + E1 -> + {true, E1} + end + end, + re:split(SQL, "(?i)(insert into)|(?i)(values)") + ). diff --git a/apps/emqx_bridge_tdengine/test/emqx_bridge_tdengine_SUITE.erl b/apps/emqx_bridge_tdengine/test/emqx_bridge_tdengine_SUITE.erl index 1b8db1aaa..0b8d20f15 100644 --- a/apps/emqx_bridge_tdengine/test/emqx_bridge_tdengine_SUITE.erl +++ b/apps/emqx_bridge_tdengine/test/emqx_bridge_tdengine_SUITE.erl @@ -24,9 +24,21 @@ ");" ). -define(SQL_DROP_TABLE, "DROP TABLE t_mqtt_msg"). --define(SQL_DELETE, "DELETE from t_mqtt_msg"). +-define(SQL_DROP_STABLE, "DROP STABLE s_tab"). +-define(SQL_DELETE, "DELETE FROM t_mqtt_msg"). -define(SQL_SELECT, "SELECT payload FROM t_mqtt_msg"). +-define(AUTO_CREATE_BRIDGE, + "insert into ${clientid} USING s_tab TAGS (${clientid}) values (${timestamp}, ${payload})" +). + +-define(SQL_CREATE_STABLE, + "CREATE STABLE s_tab (\n" + " ts timestamp,\n" + " payload BINARY(1024)\n" + ") TAGS (clientid BINARY(128));" +). + % DB defaults -define(TD_DATABASE, "mqtt"). -define(TD_USERNAME, "root"). @@ -53,12 +65,13 @@ all() -> groups() -> TCs = emqx_common_test_helpers:all(?MODULE), NonBatchCases = [t_write_timeout], + MustBatchCases = [t_batch_insert, t_auto_create_batch_insert], BatchingGroups = [{group, with_batch}, {group, without_batch}], [ {async, BatchingGroups}, {sync, BatchingGroups}, {with_batch, TCs -- NonBatchCases}, - {without_batch, TCs} + {without_batch, TCs -- MustBatchCases} ]. init_per_group(async, Config) -> @@ -117,7 +130,8 @@ common_init(ConfigT) -> Config0 = [ {td_host, Host}, {td_port, Port}, - {proxy_name, "tdengine_restful"} + {proxy_name, "tdengine_restful"}, + {template, ?SQL_BRIDGE} | ConfigT ], @@ -165,6 +179,7 @@ tdengine_config(BridgeType, Config) -> false -> 1 end, QueryMode = ?config(query_mode, Config), + Template = ?config(template, Config), ConfigString = io_lib:format( "bridges.~s.~s {\n" @@ -187,7 +202,7 @@ tdengine_config(BridgeType, Config) -> ?TD_DATABASE, ?TD_USERNAME, ?TD_PASSWORD, - ?SQL_BRIDGE, + Template, BatchSize, QueryMode ] @@ -272,11 +287,15 @@ connect_direct_tdengine(Config) -> connect_and_create_table(Config) -> ?WITH_CON(begin {ok, _} = directly_query(Con, ?SQL_CREATE_DATABASE, []), - {ok, _} = directly_query(Con, ?SQL_CREATE_TABLE) + {ok, _} = directly_query(Con, ?SQL_CREATE_TABLE), + {ok, _} = directly_query(Con, ?SQL_CREATE_STABLE) end). connect_and_drop_table(Config) -> - ?WITH_CON({ok, _} = directly_query(Con, ?SQL_DROP_TABLE)). + ?WITH_CON(begin + {ok, _} = directly_query(Con, ?SQL_DROP_TABLE), + {ok, _} = directly_query(Con, ?SQL_DROP_STABLE) + end). connect_and_clear_table(Config) -> ?WITH_CON({ok, _} = directly_query(Con, ?SQL_DELETE)). @@ -287,6 +306,15 @@ connect_and_get_payload(Config) -> ), Result. +connect_and_exec(Config, SQL) -> + ?WITH_CON({ok, _} = directly_query(Con, SQL)). + +connect_and_query(Config, SQL) -> + ?WITH_CON( + {ok, #{<<"code">> := 0, <<"data">> := Data}} = directly_query(Con, SQL) + ), + Data. + directly_query(Con, Query) -> directly_query(Con, Query, [{db_name, ?TD_DATABASE}]). @@ -407,7 +435,12 @@ t_write_failure(Config) -> #{?snk_kind := buffer_worker_flush_ack}, 2_000 ), - ?assertMatch({error, econnrefused}, Result), + case Result of + {error, Reason} when Reason =:= econnrefused; Reason =:= closed -> + ok; + _ -> + throw({unexpected, Result}) + end, ok end), ok. @@ -490,26 +523,19 @@ t_missing_data(Config) -> ok. t_bad_sql_parameter(Config) -> - EnableBatch = ?config(enable_batch, Config), ?assertMatch( {ok, _}, create_bridge(Config) ), - Request = {sql, <<"">>, [bad_parameter]}, + Request = {send_message, <<"">>}, {_, {ok, #{result := Result}}} = ?wait_async_action( query_resource(Config, Request), #{?snk_kind := buffer_worker_flush_ack}, 2_000 ), - case EnableBatch of - true -> - ?assertEqual({error, {unrecoverable_error, invalid_request}}, Result); - false -> - ?assertMatch( - {error, {unrecoverable_error, _}}, Result - ) - end, + + ?assertMatch({error, #{<<"code">> := _}}, Result), ok. t_nasty_sql_string(Config) -> @@ -544,7 +570,165 @@ t_nasty_sql_string(Config) -> connect_and_get_payload(Config) ). +t_simple_insert(Config) -> + connect_and_clear_table(Config), + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + + SentData = #{payload => ?PAYLOAD, timestamp => 1668602148000}, + Request = {send_message, SentData}, + {_, {ok, #{result := _Result}}} = + ?wait_async_action( + query_resource(Config, Request), + #{?snk_kind := buffer_worker_flush_ack}, + 2_000 + ), + ?assertMatch( + ?PAYLOAD, + connect_and_get_payload(Config) + ). + +t_batch_insert(Config) -> + connect_and_clear_table(Config), + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + + Size = 5, + Ts = erlang:system_time(millisecond), + {_, {ok, #{result := _Result}}} = + ?wait_async_action( + lists:foreach( + fun(Idx) -> + SentData = #{payload => ?PAYLOAD, timestamp => Ts + Idx}, + Request = {send_message, SentData}, + query_resource(Config, Request) + end, + lists:seq(1, Size) + ), + + #{?snk_kind := buffer_worker_flush_ack}, + 2_000 + ), + + ?retry( + _Sleep = 50, + _Attempts = 30, + ?assertMatch( + [[Size]], + connect_and_query(Config, "SELECT COUNT(1) FROM t_mqtt_msg") + ) + ). + +t_auto_create_simple_insert(Config0) -> + ClientId = to_str(?FUNCTION_NAME), + Config = get_auto_create_config(Config0), + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + + SentData = #{ + payload => ?PAYLOAD, + timestamp => 1668602148000, + clientid => ClientId + }, + Request = {send_message, SentData}, + {_, {ok, #{result := _Result}}} = + ?wait_async_action( + query_resource(Config, Request), + #{?snk_kind := buffer_worker_flush_ack}, + 2_000 + ), + ?assertMatch( + [[?PAYLOAD]], + connect_and_query(Config, "SELECT payload FROM " ++ ClientId) + ), + + ?assertMatch( + [[0]], + connect_and_query(Config, "DROP TABLE " ++ ClientId) + ). + +t_auto_create_batch_insert(Config0) -> + ClientId1 = "client1", + ClientId2 = "client2", + Config = get_auto_create_config(Config0), + + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + + Size1 = 2, + Size2 = 3, + + Ts = erlang:system_time(millisecond), + {_, {ok, #{result := _Result}}} = + ?wait_async_action( + lists:foreach( + fun({Offset, ClientId, Size}) -> + lists:foreach( + fun(Idx) -> + SentData = #{ + payload => ?PAYLOAD, + timestamp => Ts + Idx + Offset, + clientid => ClientId + }, + Request = {send_message, SentData}, + query_resource(Config, Request) + end, + lists:seq(1, Size) + ) + end, + [{0, ClientId1, Size1}, {100, ClientId2, Size2}] + ), + #{?snk_kind := buffer_worker_flush_ack}, + 2_000 + ), + + ?retry( + _Sleep = 50, + _Attempts = 30, + + ?assertMatch( + [[Size1]], + connect_and_query(Config, "SELECT COUNT(1) FROM " ++ ClientId1) + ) + ), + + ?retry( + 50, + 30, + ?assertMatch( + [[Size2]], + connect_and_query(Config, "SELECT COUNT(1) FROM " ++ ClientId2) + ) + ), + + ?assertMatch( + [[0]], + connect_and_query(Config, "DROP TABLE " ++ ClientId1) + ), + + ?assertMatch( + [[0]], + connect_and_query(Config, "DROP TABLE " ++ ClientId2) + ). + to_bin(List) when is_list(List) -> unicode:characters_to_binary(List, utf8); to_bin(Bin) when is_binary(Bin) -> Bin. + +to_str(Atom) when is_atom(Atom) -> + erlang:atom_to_list(Atom). + +get_auto_create_config(Config0) -> + Config = lists:keyreplace(template, 1, Config0, {template, ?AUTO_CREATE_BRIDGE}), + BridgeType = proplists:get_value(bridge_type, Config, <<"tdengine">>), + {_Name, TDConf} = tdengine_config(BridgeType, Config), + lists:keyreplace(tdengine_config, 1, Config, {tdengine_config, TDConf}). diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl index 5cafd2d50..bb8cc00d1 100644 --- a/apps/emqx_connector/src/emqx_connector_mqtt.erl +++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl @@ -112,7 +112,7 @@ bridge_spec(Config) -> id => Name, start => {emqx_connector_mqtt_worker, start_link, [Name, NConfig]}, restart => temporary, - shutdown => 5000 + shutdown => 1000 }. -spec bridges() -> [{_Name, _Status}]. @@ -181,7 +181,7 @@ on_stop(_InstId, #{name := InstanceId}) -> ok; {error, Reason} -> ?SLOG(error, #{ - msg => "stop_mqtt_connector", + msg => "stop_mqtt_connector_error", connector => InstanceId, reason => Reason }) diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl index 880a99313..e49603e51 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl @@ -202,13 +202,13 @@ connect(Name) -> Error end; {error, Reason} = Error -> - ?SLOG(error, #{ + ?SLOG(warning, #{ msg => "client_connect_failed", - reason => Reason + reason => Reason, + name => Name }), Error end. - subscribe_remote_topics(Ref, #{remote := #{topic := FromTopic, qos := QoS}}) -> emqtt:subscribe(ref(Ref), FromTopic, QoS); subscribe_remote_topics(_Ref, undefined) -> diff --git a/apps/emqx_dashboard/src/emqx_dashboard.app.src b/apps/emqx_dashboard/src/emqx_dashboard.app.src index bd022f226..02fbdfb74 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.app.src +++ b/apps/emqx_dashboard/src/emqx_dashboard.app.src @@ -2,7 +2,7 @@ {application, emqx_dashboard, [ {description, "EMQX Web Dashboard"}, % strict semver, bump manually! - {vsn, "5.0.20"}, + {vsn, "5.0.21"}, {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [kernel, stdlib, mnesia, minirest, emqx, emqx_ctl]}, diff --git a/apps/emqx_dashboard/src/emqx_dashboard.erl b/apps/emqx_dashboard/src/emqx_dashboard.erl index 08b7f0142..aec811e5d 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard.erl @@ -32,8 +32,6 @@ -include_lib("emqx/include/http_api.hrl"). -include_lib("emqx/include/emqx_release.hrl"). --define(BASE_PATH, "/api/v5"). - -define(EMQX_MIDDLE, emqx_dashboard_middleware). %%-------------------------------------------------------------------- @@ -52,7 +50,7 @@ start_listeners(Listeners) -> GlobalSpec = #{ openapi => "3.0.0", info => #{title => "EMQX API", version => ?EMQX_API_VERSION}, - servers => [#{url => ?BASE_PATH}], + servers => [#{url => emqx_dashboard_swagger:base_path()}], components => #{ schemas => #{}, 'securitySchemes' => #{ @@ -69,11 +67,11 @@ start_listeners(Listeners) -> {"/", cowboy_static, {priv_file, emqx_dashboard, "www/index.html"}}, {"/static/[...]", cowboy_static, {priv_dir, emqx_dashboard, "www/static"}}, {emqx_mgmt_api_status:path(), emqx_mgmt_api_status, []}, - {?BASE_PATH ++ "/[...]", emqx_dashboard_bad_api, []}, + {emqx_dashboard_swagger:relative_uri("/[...]"), emqx_dashboard_bad_api, []}, {'_', cowboy_static, {priv_file, emqx_dashboard, "www/index.html"}} ], BaseMinirest = #{ - base_path => ?BASE_PATH, + base_path => emqx_dashboard_swagger:base_path(), modules => minirest_api:find_api_modules(apps()), authorization => Authorization, security => [#{'basicAuth' => []}, #{'bearerAuth' => []}], @@ -97,7 +95,7 @@ start_listeners(Listeners) -> end end, {[], []}, - listeners(Listeners) + listeners(ensure_ssl_cert(Listeners)) ), case ErrListeners of [] -> @@ -142,18 +140,18 @@ apps() -> listeners(Listeners) -> lists:filtermap( - fun({Protocol, Conf}) -> - maps:get(enable, Conf) andalso - begin - {Conf1, Bind} = ip_port(Conf), - {true, { - listener_name(Protocol), - Protocol, - Bind, - ranch_opts(Conf1), - proto_opts(Conf1) - }} - end + fun + ({Protocol, Conf = #{enable := true}}) -> + {Conf1, Bind} = ip_port(Conf), + {true, { + listener_name(Protocol), + Protocol, + Bind, + ranch_opts(Conf1), + proto_opts(Conf1) + }}; + ({_Protocol, #{enable := false}}) -> + false end, maps:to_list(Listeners) ). @@ -193,8 +191,8 @@ ranch_opts(Options) -> end, RanchOpts#{socket_opts => InetOpts ++ SocketOpts}. -proto_opts(Options) -> - maps:with([proxy_header], Options). +proto_opts(#{proxy_header := ProxyHeader}) -> + #{proxy_header => ProxyHeader}. filter_false(_K, false, S) -> S; filter_false(K, V, S) -> [{K, V} | S]. @@ -226,7 +224,7 @@ return_unauthorized(Code, Message) -> {401, #{ <<"WWW-Authenticate">> => - <<"Basic Realm=\"minirest-server\"">> + <<"Basic Realm=\"emqx-dashboard\"">> }, #{code => Code, message => Message}}. @@ -249,3 +247,9 @@ api_key_authorize(Req, Key, Secret) -> <<"Check api_key/api_secret">> ) end. + +ensure_ssl_cert(Listeners = #{https := Https0}) -> + Https1 = emqx_tls_lib:to_server_opts(tls, Https0), + Listeners#{https => maps:from_list(Https1)}; +ensure_ssl_cert(Listeners) -> + Listeners. diff --git a/apps/emqx_dashboard/src/emqx_dashboard_error_code_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_error_code_api.erl index 131b08313..47ba02ebf 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_error_code_api.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_error_code_api.erl @@ -20,6 +20,7 @@ -include_lib("emqx/include/http_api.hrl"). -include("emqx_dashboard.hrl"). -include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). -export([ api_spec/0, @@ -50,7 +51,7 @@ schema("/error_codes") -> 'operationId' => error_codes, get => #{ security => [], - description => <<"API Error Codes">>, + description => ?DESC(error_codes), tags => [<<"Error Codes">>], responses => #{ 200 => hoconsc:array(hoconsc:ref(?MODULE, error_code)) @@ -62,7 +63,7 @@ schema("/error_codes/:code") -> 'operationId' => error_code, get => #{ security => [], - description => <<"API Error Codes">>, + description => ?DESC(error_codes_u), tags => [<<"Error Codes">>], parameters => [ {code, diff --git a/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl index c0e162b62..d86dffba0 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl @@ -6,6 +6,7 @@ -include("emqx_dashboard.hrl"). -include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hocon_types.hrl"). -behaviour(minirest_api). @@ -38,7 +39,7 @@ schema("/monitor") -> 'operationId' => monitor, get => #{ tags => [<<"Metrics">>], - desc => <<"List monitor data.">>, + description => ?DESC(list_monitor), parameters => [parameter_latest()], responses => #{ 200 => hoconsc:mk(hoconsc:array(hoconsc:ref(sampler)), #{}), @@ -51,7 +52,7 @@ schema("/monitor/nodes/:node") -> 'operationId' => monitor, get => #{ tags => [<<"Metrics">>], - desc => <<"List the monitor data on the node.">>, + description => ?DESC(list_monitor_node), parameters => [parameter_node(), parameter_latest()], responses => #{ 200 => hoconsc:mk(hoconsc:array(hoconsc:ref(sampler)), #{}), @@ -64,7 +65,7 @@ schema("/monitor_current") -> 'operationId' => monitor_current, get => #{ tags => [<<"Metrics">>], - desc => <<"Current status. Gauge and rate.">>, + description => ?DESC(current_status), responses => #{ 200 => hoconsc:mk(hoconsc:ref(sampler_current), #{}) } @@ -75,7 +76,7 @@ schema("/monitor_current/nodes/:node") -> 'operationId' => monitor_current, get => #{ tags => [<<"Metrics">>], - desc => <<"Node current status. Gauge and rate.">>, + description => ?DESC(current_status_node), parameters => [parameter_node()], responses => #{ 200 => hoconsc:mk(hoconsc:ref(sampler_current), #{}), diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 0344c84c4..a5b826cca 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -19,12 +19,17 @@ -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). +-define(BASE_PATH, "/api/v5"). + %% API -export([spec/1, spec/2]). -export([namespace/0, namespace/1, fields/1]). -export([schema_with_example/2, schema_with_examples/2]). -export([error_codes/1, error_codes/2]). -export([file_schema/1]). +-export([base_path/0]). +-export([relative_uri/1]). +-export([compose_filters/2]). -export([ filter_check_request/2, @@ -84,14 +89,30 @@ -type request() :: #{bindings => map(), query_string => map(), body => map()}. -type request_meta() :: #{module => module(), path => string(), method => atom()}. --type filter_result() :: {ok, request()} | {400, 'BAD_REQUEST', binary()}. --type filter() :: fun((request(), request_meta()) -> filter_result()). +%% More exact types are defined in minirest.hrl, but we don't want to include it +%% because it defines a lot of types and they may clash with the types declared locally. +-type status_code() :: pos_integer(). +-type error_code() :: atom() | binary(). +-type error_message() :: binary(). +-type response_body() :: term(). +-type headers() :: map(). + +-type response() :: + status_code() + | {status_code()} + | {status_code(), response_body()} + | {status_code(), headers(), response_body()} + | {status_code(), error_code(), error_message()}. + +-type filter_result() :: {ok, request()} | response(). +-type filter() :: emqx_maybe:t(fun((request(), request_meta()) -> filter_result())). -type spec_opts() :: #{ check_schema => boolean() | filter(), translate_body => boolean(), schema_converter => fun((hocon_schema:schema(), Module :: atom()) -> map()), - i18n_lang => atom() | string() | binary() + i18n_lang => atom() | string() | binary(), + filter => filter() }. -type route_path() :: string() | binary(). @@ -117,9 +138,9 @@ spec(Module, Options) -> lists:foldl( fun(Path, {AllAcc, AllRefsAcc}) -> {OperationId, Specs, Refs} = parse_spec_ref(Module, Path, Options), - CheckSchema = support_check_schema(Options), + Opts = #{filter => filter(Options)}, { - [{filename:join("/", Path), Specs, OperationId, CheckSchema} | AllAcc], + [{filename:join("/", Path), Specs, OperationId, Opts} | AllAcc], Refs ++ AllRefsAcc } end, @@ -184,6 +205,14 @@ error_codes(Codes = [_ | _], MsgDesc) -> })} ]. +-spec base_path() -> uri_string:uri_string(). +base_path() -> + ?BASE_PATH. + +-spec relative_uri(uri_string:uri_string()) -> uri_string:uri_string(). +relative_uri(Uri) -> + base_path() ++ Uri. + file_schema(FileName) -> #{ content => #{ @@ -242,6 +271,21 @@ gen_api_schema_json_iodata(SchemaMod, SchemaInfo, Converter) -> [pretty, force_utf8] ). +-spec compose_filters(filter(), filter()) -> filter(). +compose_filters(undefined, Filter2) -> + Filter2; +compose_filters(Filter1, undefined) -> + Filter1; +compose_filters(Filter1, Filter2) -> + fun(Request, RequestMeta) -> + case Filter1(Request, RequestMeta) of + {ok, Request1} -> + Filter2(Request1, RequestMeta); + Response -> + Response + end + end. + %%------------------------------------------------------------------------------ %% Private functions %%------------------------------------------------------------------------------ @@ -273,14 +317,22 @@ check_only(Schema, Map, Opts) -> _ = hocon_tconf:check_plain(Schema, Map, Opts), Map. -support_check_schema(#{check_schema := true, translate_body := true}) -> - #{filter => fun ?MODULE:filter_check_request_and_translate_body/2}; -support_check_schema(#{check_schema := true}) -> - #{filter => fun ?MODULE:filter_check_request/2}; -support_check_schema(#{check_schema := Filter}) when is_function(Filter, 2) -> - #{filter => Filter}; -support_check_schema(_) -> - #{filter => undefined}. +filter(Options) -> + CheckSchemaFilter = check_schema_filter(Options), + CustomFilter = custom_filter(Options), + compose_filters(CheckSchemaFilter, CustomFilter). + +custom_filter(Options) -> + maps:get(filter, Options, undefined). + +check_schema_filter(#{check_schema := true, translate_body := true}) -> + fun ?MODULE:filter_check_request_and_translate_body/2; +check_schema_filter(#{check_schema := true}) -> + fun ?MODULE:filter_check_request/2; +check_schema_filter(#{check_schema := Filter}) when is_function(Filter, 2) -> + Filter; +check_schema_filter(_) -> + undefined. parse_spec_ref(Module, Path, Options) -> Schema = diff --git a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl index 1f14b02c0..0d446a7f2 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl @@ -36,8 +36,6 @@ -define(HOST, "http://127.0.0.1:18083"). -%% -define(API_VERSION, "v5"). - -define(BASE_PATH, "/api/v5"). -define(APP_DASHBOARD, emqx_dashboard). @@ -57,6 +55,10 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> + %% Load all applications to ensure swagger.json is fully generated. + Apps = emqx_machine_boot:reboot_apps(), + ct:pal("load apps:~p~n", [Apps]), + lists:foreach(fun(App) -> application:load(App) end, Apps), emqx_mgmt_api_test_util:init_suite([emqx_management]), Config. diff --git a/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl b/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl index 25b4065de..908d7e0ef 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl @@ -26,11 +26,12 @@ request/4, multipart_formdata_request/3, multipart_formdata_request/4, + host/0, uri/0, uri/1 ]). --define(HOST, "http://127.0.0.1:18083/"). +-define(HOST, "http://127.0.0.1:18083"). -define(API_VERSION, "v5"). -define(BASE_PATH, "api"). @@ -98,10 +99,13 @@ request(Username, Method, Url, Body) -> {error, Reason} end. +host() -> + ?HOST. + uri() -> uri([]). uri(Parts) when is_list(Parts) -> NParts = [E || E <- Parts], - ?HOST ++ to_list(filename:join([?BASE_PATH, ?API_VERSION | NParts])). + host() ++ "/" ++ to_list(filename:join([?BASE_PATH, ?API_VERSION | NParts])). auth_header(Username) -> Password = <<"public">>, diff --git a/apps/emqx_dashboard/test/emqx_dashboard_https_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_https_SUITE.erl new file mode 100644 index 000000000..fefaeb7f1 --- /dev/null +++ b/apps/emqx_dashboard/test/emqx_dashboard_https_SUITE.erl @@ -0,0 +1,214 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_dashboard_https_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include("emqx_dashboard.hrl"). + +-define(NAME, 'https:dashboard'). +-define(HOST, "https://127.0.0.1:18084"). +-define(BASE_PATH, "/api/v5"). +-define(OVERVIEWS, [ + "alarms", + "banned", + "stats", + "metrics", + "listeners", + "clients", + "subscriptions" +]). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> Config. +end_per_suite(_Config) -> emqx_mgmt_api_test_util:end_suite([emqx_management]). + +init_per_testcase(_TestCase, Config) -> Config. +end_per_testcase(_TestCase, _Config) -> emqx_mgmt_api_test_util:end_suite([emqx_management]). + +t_default_ssl_cert(_Config) -> + Conf = #{dashboard => #{listeners => #{https => #{bind => 18084, enable => true}}}}, + validate_https(Conf, 512, default_ssl_cert(), verify_none), + ok. + +t_normal_ssl_cert(_Config) -> + MaxConnection = 1000, + Conf = #{ + dashboard => #{ + listeners => #{ + https => #{ + bind => 18084, + enable => true, + cacertfile => naive_env_interpolation(<<"${EMQX_ETC_DIR}/certs/cacert.pem">>), + certfile => naive_env_interpolation(<<"${EMQX_ETC_DIR}/certs/cert.pem">>), + keyfile => naive_env_interpolation(<<"${EMQX_ETC_DIR}/certs/key.pem">>), + max_connections => MaxConnection + } + } + } + }, + validate_https(Conf, MaxConnection, default_ssl_cert(), verify_none), + ok. + +t_verify_cacertfile(_Config) -> + MaxConnection = 1024, + DefaultSSLCert = default_ssl_cert(), + SSLCert = DefaultSSLCert#{cacertfile => <<"">>}, + %% default #{verify => verify_none} + Conf = #{ + dashboard => #{ + listeners => #{ + https => #{ + bind => 18084, + enable => true, + cacertfile => <<"">>, + max_connections => MaxConnection + } + } + } + }, + validate_https(Conf, MaxConnection, SSLCert, verify_none), + %% verify_peer but cacertfile is empty + VerifyPeerConf1 = emqx_utils_maps:deep_put( + [dashboard, listeners, https, verify], + Conf, + verify_peer + ), + emqx_common_test_helpers:load_config(emqx_dashboard_schema, VerifyPeerConf1), + ?assertMatch({error, [?NAME]}, emqx_dashboard:start_listeners()), + %% verify_peer and cacertfile is ok. + VerifyPeerConf2 = emqx_utils_maps:deep_put( + [dashboard, listeners, https, cacertfile], + VerifyPeerConf1, + naive_env_interpolation(<<"${EMQX_ETC_DIR}/certs/cacert.pem">>) + ), + validate_https(VerifyPeerConf2, MaxConnection, DefaultSSLCert, verify_peer), + ok. + +t_bad_certfile(_Config) -> + Conf = #{ + dashboard => #{ + listeners => #{ + https => #{ + bind => 18084, + enable => true, + certfile => <<"${EMQX_ETC_DIR}/certs/not_found_cert.pem">> + } + } + } + }, + emqx_common_test_helpers:load_config(emqx_dashboard_schema, Conf), + ?assertMatch({error, [?NAME]}, emqx_dashboard:start_listeners()), + ok. + +validate_https(Conf, MaxConnection, SSLCert, Verify) -> + emqx_common_test_helpers:load_config(emqx_dashboard_schema, Conf), + emqx_mgmt_api_test_util:init_suite([emqx_management], fun(X) -> X end), + assert_ranch_options(MaxConnection, SSLCert, Verify), + assert_https_request(), + emqx_mgmt_api_test_util:end_suite([emqx_management]). + +assert_ranch_options(MaxConnections0, SSLCert, Verify) -> + Middlewares = [emqx_dashboard_middleware, cowboy_router, cowboy_handler], + [ + ?NAME, + ranch_ssl, + #{ + max_connections := MaxConnections, + num_acceptors := _, + socket_opts := SocketOpts + }, + cowboy_tls, + #{ + env := #{ + dispatch := {persistent_term, ?NAME}, + options := #{ + name := ?NAME, + protocol := https, + protocol_options := #{proxy_header := false}, + security := [#{basicAuth := []}, #{bearerAuth := []}], + swagger_global_spec := _ + } + }, + middlewares := Middlewares, + proxy_header := false + } + ] = ranch_server:get_listener_start_args(?NAME), + ?assertEqual(MaxConnections0, MaxConnections), + ?assert(lists:member(inet, SocketOpts), SocketOpts), + #{ + backlog := 1024, + ciphers := Ciphers, + port := 18084, + send_timeout := 10000, + verify := Verify, + versions := Versions + } = SocketMaps = maps:from_list(SocketOpts -- [inet]), + %% without tlsv1.1 tlsv1 + ?assertMatch(['tlsv1.3', 'tlsv1.2'], Versions), + ?assert(Ciphers =/= []), + maps:foreach( + fun(K, ConfVal) -> + case maps:find(K, SocketMaps) of + {ok, File} -> ?assertEqual(naive_env_interpolation(ConfVal), File); + error -> ?assertEqual(<<"">>, ConfVal) + end + end, + SSLCert + ), + ?assertMatch( + #{ + env := #{dispatch := {persistent_term, ?NAME}}, + middlewares := Middlewares, + proxy_header := false + }, + ranch:get_protocol_options(?NAME) + ), + ok. + +assert_https_request() -> + Headers = emqx_dashboard_SUITE:auth_header_(), + lists:foreach( + fun(Path) -> + ApiPath = api_path([Path]), + ?assertMatch( + {ok, _}, + emqx_dashboard_SUITE:request_dashboard(get, ApiPath, Headers) + ) + end, + ?OVERVIEWS + ). + +api_path(Parts) -> + ?HOST ++ filename:join([?BASE_PATH | Parts]). + +naive_env_interpolation(Str0) -> + Str1 = emqx_schema:naive_env_interpolation(Str0), + %% assert all envs are replaced + ?assertNot(lists:member($$, Str1)), + Str1. + +default_ssl_cert() -> + #{ + cacertfile => <<"${EMQX_ETC_DIR}/certs/cacert.pem">>, + certfile => <<"${EMQX_ETC_DIR}/certs/cert.pem">>, + keyfile => <<"${EMQX_ETC_DIR}/certs/key.pem">> + }. diff --git a/apps/emqx_enterprise/src/emqx_enterprise_schema.erl b/apps/emqx_enterprise/src/emqx_enterprise_schema.erl index 971d1a56f..d128aa844 100644 --- a/apps/emqx_enterprise/src/emqx_enterprise_schema.erl +++ b/apps/emqx_enterprise/src/emqx_enterprise_schema.erl @@ -6,28 +6,24 @@ -behaviour(hocon_schema). --export([namespace/0, roots/0, fields/1, translations/0, translation/1, validations/0, desc/1]). +-export([namespace/0, roots/0, fields/1, translations/0, translation/1, desc/1, validations/0]). --define(EE_SCHEMA_MODULES, [emqx_license_schema, emqx_ee_schema_registry_schema]). +-define(EE_SCHEMA_MODULES, [ + emqx_license_schema, + emqx_ee_schema_registry_schema, + emqx_ft_schema +]). namespace() -> emqx_conf_schema:namespace(). roots() -> - redefine_roots( - lists:foldl( - fun(Module, Roots) -> - Roots ++ apply(Module, roots, []) - end, - emqx_conf_schema:roots(), - ?EE_SCHEMA_MODULES - ) - ). + redefine_roots(emqx_conf_schema:roots()) ++ ee_roots(). fields("node") -> redefine_node(emqx_conf_schema:fields("node")); fields(Name) -> - emqx_conf_schema:fields(Name). + ee_delegate(fields, ?EE_SCHEMA_MODULES, Name). translations() -> emqx_conf_schema:translations(). @@ -35,17 +31,42 @@ translations() -> translation(Name) -> emqx_conf_schema:translation(Name). +desc(Name) -> + ee_delegate(desc, ?EE_SCHEMA_MODULES, Name). + validations() -> emqx_conf_schema:validations(). -redefine_node(Fields) -> - Overrides = [{"applications", #{default => <<"emqx_license">>}}], - override(Fields, Overrides). +%%------------------------------------------------------------------------------ +%% helpers +%%------------------------------------------------------------------------------ + +ee_roots() -> + lists:flatmap( + fun(Module) -> + apply(Module, roots, []) + end, + ?EE_SCHEMA_MODULES + ). + +ee_delegate(Method, [EEMod | EEMods], Name) -> + case lists:member(Name, apply(EEMod, roots, [])) of + true -> + apply(EEMod, Method, [Name]); + false -> + ee_delegate(Method, EEMods, Name) + end; +ee_delegate(Method, [], Name) -> + apply(emqx_conf_schema, Method, [Name]). redefine_roots(Roots) -> Overrides = [{"node", #{type => hoconsc:ref(?MODULE, "node")}}], override(Roots, Overrides). +redefine_node(Fields) -> + Overrides = [{"applications", #{default => <<"emqx_license">>}}], + override(Fields, Overrides). + override(Fields, []) -> Fields; override(Fields, [{Name, Override} | More]) -> @@ -60,6 +81,3 @@ find_schema(Name, Fields) -> replace_schema(Name, Schema, Fields) -> lists:keyreplace(Name, 1, Fields, {Name, Schema}). - -desc(Name) -> - emqx_conf_schema:desc(Name). diff --git a/apps/emqx_eviction_agent/etc/emqx_eviction_agent.conf b/apps/emqx_eviction_agent/etc/emqx_eviction_agent.conf index 011b7fb0f..e69de29bb 100644 --- a/apps/emqx_eviction_agent/etc/emqx_eviction_agent.conf +++ b/apps/emqx_eviction_agent/etc/emqx_eviction_agent.conf @@ -1,3 +0,0 @@ -##-------------------------------------------------------------------- -## EMQX Eviction Agent Plugin -##-------------------------------------------------------------------- diff --git a/apps/emqx_ft/BSL.txt b/apps/emqx_ft/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_ft/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_ft/README.md b/apps/emqx_ft/README.md new file mode 100644 index 000000000..019c63dea --- /dev/null +++ b/apps/emqx_ft/README.md @@ -0,0 +1,86 @@ +# EMQX File Transfer + +EMQX File Transfer application enables the _File Transfer over MQTT_ feature described in [EIP-0021](https://github.com/emqx/eip), and provides support to publish transferred files either to the node-local file system or to the S3 API compatible remote object storage. + +## Usage + +As almost any other EMQX application, `emqx_ft` is configured via the EMQX configuration system. The following snippet is the minimal configuration that will enable File Transfer over MQTT. + +``` +file_transfer { + enable = true +} +``` + +The configuration above will make File Transfer available to all MQTT clients, and will use the default storage backend, which in turn uses node-local file system both for temporary storage and for the final destination of the transferred files. + +## Configuration + +Every configuration parameter is described in the `emqx_ft_schema` module. + +The most important configuration parameter is `storage`, which defines the storage backend to use. Currently, only `local` storage backend is available, which stores all the temporary data accumulating during file transfers in the node-local file system. Those go into `${EMQX_DATA_DIR}/file_transfer` directory by default, but can be configured via `local.storage.segments.root` parameter. The final destination of the transferred files on the other hand is defined by `local.storage.exporter` parameter, and currently can be either `local` or `s3`. + +### Local Exporter + +The `local` exporter is the default one, and it stores the transferred files in the node-local file system. The final destination directory is defined by `local.storage.exporter.local.root` parameter, and defaults to `${EMQX_DATA_DIR}/file_transfer/exports` directory. + +``` +file_transfer { + enable = true + storage { + local { + exporter { + local { root = "/var/lib/emqx/transfers" } + } + } + } +} +``` + +Important to note that even though the transferred files go into the node-local file system, the File Transfer API provides a cluster-wide view of the transferred files, and any file can be downloaded from any node in the cluster. + +### S3 Exporter + +The `s3` exporter stores the transferred files in the S3 API compatible remote object storage. The destination bucket is defined by `local.storage.exporter.s3.bucket` parameter. + +This snippet configures File Transfer to store the transferred files in the `my-bucket` bucket in the `us-east-1` region of the AWS S3 service. + +``` +file_transfer { + enable = true + storage { + local { + exporter { + s3 { + host = "s3.us-east-1.amazonaws.com" + port = "443" + access_key_id = "AKIA27EZDDM9XLINWXFE" + secret_access_key = "..." + bucket = "my-bucket" + } + } + } + } +} + +``` + +## API + +### MQTT + +When enabled, File Transfer application reserves MQTT topics starting with `$file/` prefix for the purpose of serving the File Transfer protocol, as described in [EIP-0021](https://github.com/emqx/eip). + +### REST + +Application publishes a basic set of APIs, to: +* List all the transferred files available for download. +* Configure the application, including the storage backend. +* (When using `local` storage exporter) Download the transferred files. + +Switching to the `s3` storage exporter is possible at any time, but the files transferred before the switch will not be +available for download anymore. Though, the files will still be available in the node-local file system. + +## Contributing + +Please see our [contributing.md](../../CONTRIBUTING.md). diff --git a/apps/emqx_ft/docker-ct b/apps/emqx_ft/docker-ct new file mode 100644 index 000000000..36f9d86d3 --- /dev/null +++ b/apps/emqx_ft/docker-ct @@ -0,0 +1 @@ +minio diff --git a/apps/emqx_ft/etc/emqx_ft.conf b/apps/emqx_ft/etc/emqx_ft.conf new file mode 100644 index 000000000..e69de29bb diff --git a/apps/emqx_ft/include/emqx_ft_storage_fs.hrl b/apps/emqx_ft/include/emqx_ft_storage_fs.hrl new file mode 100644 index 000000000..81ab9cfad --- /dev/null +++ b/apps/emqx_ft/include/emqx_ft_storage_fs.hrl @@ -0,0 +1,29 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-ifndef(EMQX_FT_STORAGE_FS_HRL). +-define(EMQX_FT_STORAGE_FS_HRL, true). + +-record(gcstats, { + started_at :: integer(), + finished_at :: integer() | undefined, + files = 0 :: non_neg_integer(), + directories = 0 :: non_neg_integer(), + space = 0 :: non_neg_integer(), + errors = #{} :: #{_GCSubject => {error, _}} +}). + +-endif. diff --git a/apps/emqx_ft/rebar.config b/apps/emqx_ft/rebar.config new file mode 100644 index 000000000..2c0962035 --- /dev/null +++ b/apps/emqx_ft/rebar.config @@ -0,0 +1,11 @@ +%% -*- mode: erlang -*- + +{erl_opts, [debug_info]}. +{deps, [{emqx, {path, "../emqx"}}]}. + +{shell, [ + % {config, "config/sys.config"}, + {apps, [emqx_ft]} +]}. + +{project_plugins, [erlfmt]}. diff --git a/apps/emqx_ft/src/emqx_ft.app.src b/apps/emqx_ft/src/emqx_ft.app.src new file mode 100644 index 000000000..058fe984a --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft.app.src @@ -0,0 +1,14 @@ +{application, emqx_ft, [ + {description, "EMQX file transfer over MQTT"}, + {vsn, "0.1.0"}, + {registered, []}, + {mod, {emqx_ft_app, []}}, + {applications, [ + kernel, + stdlib, + gproc, + emqx_s3 + ]}, + {env, []}, + {modules, []} +]}. diff --git a/apps/emqx_ft/src/emqx_ft.erl b/apps/emqx_ft/src/emqx_ft.erl new file mode 100644 index 000000000..898203b51 --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft.erl @@ -0,0 +1,425 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft). + +-include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("emqx/include/emqx_hooks.hrl"). +-include_lib("snabbkaffe/include/trace.hrl"). + +-export([ + hook/0, + unhook/0 +]). + +-export([ + on_message_publish/1, + on_message_puback/4 +]). + +-export([ + decode_filemeta/1, + encode_filemeta/1 +]). + +-export([on_complete/4]). + +-export_type([ + clientid/0, + transfer/0, + bytes/0, + offset/0, + filemeta/0, + segment/0, + checksum/0 +]). + +%% Number of bytes +-type bytes() :: non_neg_integer(). + +%% MQTT Client ID +-type clientid() :: binary(). + +-type fileid() :: binary(). +-type transfer() :: {clientid(), fileid()}. +-type offset() :: bytes(). +-type checksum() :: {_Algo :: atom(), _Digest :: binary()}. + +-type filemeta() :: #{ + %% Display name + name := string(), + %% Size in bytes, as advertised by the client. + %% Client is free to specify here whatever it wants, which means we can end + %% up with a file of different size after assembly. It's not clear from + %% specification what that means (e.g. what are clients' expectations), we + %% currently do not condider that an error (or, specifically, a signal that + %% the resulting file is corrupted during transmission). + size => _Bytes :: non_neg_integer(), + checksum => checksum(), + expire_at := emqx_datetime:epoch_second(), + %% TTL of individual segments + %% Somewhat confusing that we won't know it on the nodes where the filemeta + %% is missing. + segments_ttl => _Seconds :: pos_integer(), + user_data => emqx_ft_schema:json_value() +}. + +-type segment() :: {offset(), _Content :: binary()}. + +%%-------------------------------------------------------------------- +%% API for app +%%-------------------------------------------------------------------- + +hook() -> + ok = emqx_hooks:put('message.publish', {?MODULE, on_message_publish, []}, ?HP_LOWEST), + ok = emqx_hooks:put('message.puback', {?MODULE, on_message_puback, []}, ?HP_LOWEST). + +unhook() -> + ok = emqx_hooks:del('message.publish', {?MODULE, on_message_publish}), + ok = emqx_hooks:del('message.puback', {?MODULE, on_message_puback}). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- + +decode_filemeta(Payload) when is_binary(Payload) -> + case emqx_utils_json:safe_decode(Payload, [return_maps]) of + {ok, Map} -> + decode_filemeta(Map); + {error, Error} -> + {error, {invalid_filemeta_json, Error}} + end; +decode_filemeta(Map) when is_map(Map) -> + Schema = emqx_ft_schema:schema(filemeta), + try + Meta = hocon_tconf:check_plain(Schema, Map, #{atom_key => true, required => false}), + {ok, Meta} + catch + throw:{_Schema, Errors} -> + {error, {invalid_filemeta, Errors}} + end. + +encode_filemeta(Meta = #{}) -> + Schema = emqx_ft_schema:schema(filemeta), + hocon_tconf:make_serializable(Schema, emqx_utils_maps:binary_key_map(Meta), #{}). + +%%-------------------------------------------------------------------- +%% Hooks +%%-------------------------------------------------------------------- + +on_message_publish( + Msg = #message{ + id = _Id, + topic = <<"$file/", _/binary>> + } +) -> + Headers = Msg#message.headers, + {stop, Msg#message{headers = Headers#{allow_publish => false}}}; +on_message_publish(Msg) -> + {ok, Msg}. + +on_message_puback(PacketId, #message{topic = Topic} = Msg, _PubRes, _RC) -> + case Topic of + <<"$file/", FileCommand/binary>> -> + {stop, on_file_command(PacketId, Msg, FileCommand)}; + _ -> + ignore + end. + +%%-------------------------------------------------------------------- +%% Handlers for transfer messages +%%-------------------------------------------------------------------- + +%% TODO Move to emqx_ft_mqtt? + +on_file_command(PacketId, Msg, FileCommand) -> + case emqx_topic:tokens(FileCommand) of + [FileIdIn | Rest] -> + validate([{fileid, FileIdIn}], fun([FileId]) -> + on_file_command(PacketId, FileId, Msg, Rest) + end); + [] -> + ?RC_UNSPECIFIED_ERROR + end. + +on_file_command(PacketId, FileId, Msg, FileCommand) -> + Transfer = transfer(Msg, FileId), + case FileCommand of + [<<"init">>] -> + validate( + [{filemeta, Msg#message.payload}], + fun([Meta]) -> + on_init(PacketId, Msg, Transfer, Meta) + end + ); + [<<"fin">>, FinalSizeBin | MaybeChecksum] when length(MaybeChecksum) =< 1 -> + ChecksumBin = emqx_maybe:from_list(MaybeChecksum), + validate( + [{size, FinalSizeBin}, {{maybe, checksum}, ChecksumBin}], + fun([FinalSize, Checksum]) -> + on_fin(PacketId, Msg, Transfer, FinalSize, Checksum) + end + ); + [<<"abort">>] -> + on_abort(Msg, Transfer); + [OffsetBin] -> + validate([{offset, OffsetBin}], fun([Offset]) -> + on_segment(PacketId, Msg, Transfer, Offset, undefined) + end); + [OffsetBin, ChecksumBin] -> + validate( + [{offset, OffsetBin}, {checksum, ChecksumBin}], + fun([Offset, Checksum]) -> + validate( + [{integrity, Msg#message.payload, Checksum}], + fun(_) -> + on_segment(PacketId, Msg, Transfer, Offset, Checksum) + end + ) + end + ); + _ -> + ?RC_UNSPECIFIED_ERROR + end. + +on_init(PacketId, Msg, Transfer, Meta) -> + ?tp(info, "file_transfer_init", #{ + mqtt_msg => Msg, + packet_id => PacketId, + transfer => Transfer, + filemeta => Meta + }), + PacketKey = {self(), PacketId}, + Callback = fun(Result) -> + ?MODULE:on_complete("store_filemeta", PacketKey, Transfer, Result) + end, + with_responder(PacketKey, Callback, emqx_ft_conf:init_timeout(), fun() -> + case store_filemeta(Transfer, Meta) of + % Stored, ack through the responder right away + ok -> + emqx_ft_responder:ack(PacketKey, ok); + % Storage operation started, packet will be acked by the responder + % {async, Pid} -> + % ok = emqx_ft_responder:kickoff(PacketKey, Pid), + % ok; + %% Storage operation failed, ack through the responder + {error, _} = Error -> + emqx_ft_responder:ack(PacketKey, Error) + end + end). + +on_abort(_Msg, _FileId) -> + %% TODO + ?RC_SUCCESS. + +on_segment(PacketId, Msg, Transfer, Offset, Checksum) -> + ?tp(info, "file_transfer_segment", #{ + mqtt_msg => Msg, + packet_id => PacketId, + transfer => Transfer, + offset => Offset, + checksum => Checksum + }), + Segment = {Offset, Msg#message.payload}, + PacketKey = {self(), PacketId}, + Callback = fun(Result) -> + ?MODULE:on_complete("store_segment", PacketKey, Transfer, Result) + end, + with_responder(PacketKey, Callback, emqx_ft_conf:store_segment_timeout(), fun() -> + case store_segment(Transfer, Segment) of + ok -> + emqx_ft_responder:ack(PacketKey, ok); + % {async, Pid} -> + % ok = emqx_ft_responder:kickoff(PacketKey, Pid), + % ok; + {error, _} = Error -> + emqx_ft_responder:ack(PacketKey, Error) + end + end). + +on_fin(PacketId, Msg, Transfer, FinalSize, Checksum) -> + ?tp(info, "file_transfer_fin", #{ + mqtt_msg => Msg, + packet_id => PacketId, + transfer => Transfer, + final_size => FinalSize, + checksum => Checksum + }), + %% TODO: handle checksum? Do we need it? + FinPacketKey = {self(), PacketId}, + Callback = fun(Result) -> + ?MODULE:on_complete("assemble", FinPacketKey, Transfer, Result) + end, + with_responder(FinPacketKey, Callback, emqx_ft_conf:assemble_timeout(), fun() -> + case assemble(Transfer, FinalSize) of + %% Assembling completed, ack through the responder right away + ok -> + emqx_ft_responder:ack(FinPacketKey, ok); + %% Assembling started, packet will be acked by the responder + {async, Pid} -> + ok = emqx_ft_responder:kickoff(FinPacketKey, Pid), + ok; + %% Assembling failed, ack through the responder + {error, _} = Error -> + emqx_ft_responder:ack(FinPacketKey, Error) + end + end). + +with_responder(Key, Callback, Timeout, CriticalSection) -> + case emqx_ft_responder:start(Key, Callback, Timeout) of + %% We have new packet + {ok, _} -> + CriticalSection(); + %% Packet already received. + %% Since we are still handling the previous one, + %% we probably have retransmit here + {error, {already_started, _}} -> + ok + end, + undefined. + +store_filemeta(Transfer, Segment) -> + try + emqx_ft_storage:store_filemeta(Transfer, Segment) + catch + C:E:S -> + ?tp(error, "start_store_filemeta_failed", #{ + class => C, reason => E, stacktrace => S + }), + {error, {internal_error, E}} + end. + +store_segment(Transfer, Segment) -> + try + emqx_ft_storage:store_segment(Transfer, Segment) + catch + C:E:S -> + ?tp(error, "start_store_segment_failed", #{ + class => C, reason => E, stacktrace => S + }), + {error, {internal_error, E}} + end. + +assemble(Transfer, FinalSize) -> + try + emqx_ft_storage:assemble(Transfer, FinalSize) + catch + C:E:S -> + ?tp(error, "start_assemble_failed", #{ + class => C, reason => E, stacktrace => S + }), + {error, {internal_error, E}} + end. + +transfer(Msg, FileId) -> + ClientId = Msg#message.from, + {clientid_to_binary(ClientId), FileId}. + +on_complete(Op, {ChanPid, PacketId}, Transfer, Result) -> + ?tp(debug, "on_complete", #{ + operation => Op, + packet_id => PacketId, + transfer => Transfer + }), + case Result of + {Mode, ok} when Mode == ack orelse Mode == down -> + erlang:send(ChanPid, {puback, PacketId, [], ?RC_SUCCESS}); + {Mode, {error, _} = Reason} when Mode == ack orelse Mode == down -> + ?tp(error, Op ++ "_failed", #{ + transfer => Transfer, + reason => Reason + }), + erlang:send(ChanPid, {puback, PacketId, [], ?RC_UNSPECIFIED_ERROR}); + timeout -> + ?tp(error, Op ++ "_timed_out", #{ + transfer => Transfer + }), + erlang:send(ChanPid, {puback, PacketId, [], ?RC_UNSPECIFIED_ERROR}) + end. + +validate(Validations, Fun) -> + case do_validate(Validations, []) of + {ok, Parsed} -> + Fun(Parsed); + {error, Reason} -> + ?tp(info, "client_violated_protocol", #{reason => Reason}), + ?RC_UNSPECIFIED_ERROR + end. + +do_validate([], Parsed) -> + {ok, lists:reverse(Parsed)}; +do_validate([{fileid, FileId} | Rest], Parsed) -> + case byte_size(FileId) of + S when S > 0 -> + do_validate(Rest, [FileId | Parsed]); + 0 -> + {error, {invalid_fileid, FileId}} + end; +do_validate([{filemeta, Payload} | Rest], Parsed) -> + case decode_filemeta(Payload) of + {ok, Meta} -> + do_validate(Rest, [Meta | Parsed]); + {error, Reason} -> + {error, Reason} + end; +do_validate([{offset, Offset} | Rest], Parsed) -> + case string:to_integer(Offset) of + {Int, <<>>} -> + do_validate(Rest, [Int | Parsed]); + _ -> + {error, {invalid_offset, Offset}} + end; +do_validate([{size, Size} | Rest], Parsed) -> + case string:to_integer(Size) of + {Int, <<>>} -> + do_validate(Rest, [Int | Parsed]); + _ -> + {error, {invalid_size, Size}} + end; +do_validate([{checksum, Checksum} | Rest], Parsed) -> + case parse_checksum(Checksum) of + {ok, Bin} -> + do_validate(Rest, [Bin | Parsed]); + {error, _Reason} -> + {error, {invalid_checksum, Checksum}} + end; +do_validate([{integrity, Payload, Checksum} | Rest], Parsed) -> + case crypto:hash(sha256, Payload) of + Checksum -> + do_validate(Rest, [Payload | Parsed]); + Mismatch -> + {error, {checksum_mismatch, binary:encode_hex(Mismatch)}} + end; +do_validate([{{maybe, _}, undefined} | Rest], Parsed) -> + do_validate(Rest, [undefined | Parsed]); +do_validate([{{maybe, T}, Value} | Rest], Parsed) -> + do_validate([{T, Value} | Rest], Parsed). + +parse_checksum(Checksum) when is_binary(Checksum) andalso byte_size(Checksum) =:= 64 -> + try + {ok, binary:decode_hex(Checksum)} + catch + error:badarg -> + {error, invalid_checksum} + end; +parse_checksum(_Checksum) -> + {error, invalid_checksum}. + +clientid_to_binary(A) when is_atom(A) -> + atom_to_binary(A); +clientid_to_binary(B) when is_binary(B) -> + B. diff --git a/apps/emqx_ft/src/emqx_ft_api.erl b/apps/emqx_ft/src/emqx_ft_api.erl new file mode 100644 index 000000000..3fd279c76 --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_api.erl @@ -0,0 +1,239 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_ft_api). + +-behaviour(minirest_api). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). + +%% Swagger specs from hocon schema +-export([ + api_spec/0, + paths/0, + schema/1, + namespace/0 +]). + +-export([ + roots/0, + fields/1 +]). + +%% Minirest filter for checking if file transfer is enabled +-export([check_ft_enabled/2]). + +%% API callbacks +-export([ + '/file_transfer/files'/2, + '/file_transfer/files/:clientid/:fileid'/2 +]). + +-import(hoconsc, [mk/2, ref/1, ref/2]). + +namespace() -> "file_transfer". + +api_spec() -> + emqx_dashboard_swagger:spec(?MODULE, #{ + check_schema => true, filter => fun ?MODULE:check_ft_enabled/2 + }). + +paths() -> + [ + "/file_transfer/files", + "/file_transfer/files/:clientid/:fileid" + ]. + +schema("/file_transfer/files") -> + #{ + 'operationId' => '/file_transfer/files', + get => #{ + tags => [<<"file_transfer">>], + summary => <<"List all uploaded files">>, + description => ?DESC("file_list"), + parameters => [ + ref(following), + ref(emqx_dashboard_swagger, limit) + ], + responses => #{ + 200 => <<"Operation success">>, + 400 => emqx_dashboard_swagger:error_codes( + ['BAD_REQUEST'], <<"Invalid cursor">> + ), + 503 => emqx_dashboard_swagger:error_codes( + ['SERVICE_UNAVAILABLE'], error_desc('SERVICE_UNAVAILABLE') + ) + } + } + }; +schema("/file_transfer/files/:clientid/:fileid") -> + #{ + 'operationId' => '/file_transfer/files/:clientid/:fileid', + get => #{ + tags => [<<"file_transfer">>], + summary => <<"List files uploaded in a specific transfer">>, + description => ?DESC("file_list_transfer"), + parameters => [ + ref(client_id), + ref(file_id) + ], + responses => #{ + 200 => <<"Operation success">>, + 404 => emqx_dashboard_swagger:error_codes( + ['FILES_NOT_FOUND'], error_desc('FILES_NOT_FOUND') + ), + 503 => emqx_dashboard_swagger:error_codes( + ['SERVICE_UNAVAILABLE'], error_desc('SERVICE_UNAVAILABLE') + ) + } + } + }. + +check_ft_enabled(Params, _Meta) -> + case emqx_ft_conf:enabled() of + true -> + {ok, Params}; + false -> + {503, error_msg('SERVICE_UNAVAILABLE', <<"Service unavailable">>)} + end. + +'/file_transfer/files'(get, #{ + query_string := QueryString +}) -> + try + Limit = limit(QueryString), + Query = + case maps:get(<<"following">>, QueryString, undefined) of + undefined -> + #{limit => Limit}; + Cursor -> + #{limit => Limit, following => Cursor} + end, + case emqx_ft_storage:files(Query) of + {ok, Page} -> + {200, format_page(Page)}; + {error, _} -> + {503, error_msg('SERVICE_UNAVAILABLE')} + end + catch + error:{badarg, cursor} -> + {400, error_msg('BAD_REQUEST', <<"Invalid cursor">>)} + end. + +'/file_transfer/files/:clientid/:fileid'(get, #{ + bindings := #{clientid := ClientId, fileid := FileId} +}) -> + Transfer = {ClientId, FileId}, + case emqx_ft_storage:files(#{transfer => Transfer}) of + {ok, Page} -> + {200, format_page(Page)}; + {error, [{_Node, enoent} | _]} -> + {404, error_msg('FILES_NOT_FOUND')}; + {error, _} -> + {503, error_msg('SERVICE_UNAVAILABLE')} + end. + +format_page(#{items := Files, cursor := Cursor}) -> + #{ + <<"files">> => lists:map(fun format_file_info/1, Files), + <<"cursor">> => Cursor + }; +format_page(#{items := Files}) -> + #{ + <<"files">> => lists:map(fun format_file_info/1, Files) + }. + +error_msg(Code) -> + #{code => Code, message => error_desc(Code)}. + +error_msg(Code, Msg) -> + #{code => Code, message => emqx_utils:readable_error_msg(Msg)}. + +error_desc('FILES_NOT_FOUND') -> + <<"Files requested for this transfer could not be found">>; +error_desc('SERVICE_UNAVAILABLE') -> + <<"Service unavailable">>. + +roots() -> + []. + +-spec fields(hocon_schema:name()) -> [hoconsc:field()]. +fields(client_id) -> + [ + {clientid, + mk(binary(), #{ + in => path, + desc => <<"MQTT Client ID">>, + required => true + })} + ]; +fields(file_id) -> + [ + {fileid, + mk(binary(), #{ + in => path, + desc => <<"File ID">>, + required => true + })} + ]; +fields(following) -> + [ + {following, + mk(binary(), #{ + in => query, + desc => <<"Cursor to start listing files from">>, + required => false + })} + ]. + +%%-------------------------------------------------------------------- +%% Helpers +%%-------------------------------------------------------------------- + +format_file_info( + Info = #{ + name := Name, + size := Size, + uri := URI, + timestamp := Timestamp, + transfer := {ClientId, FileId} + } +) -> + Res = #{ + name => format_name(Name), + size => Size, + timestamp => format_timestamp(Timestamp), + clientid => ClientId, + fileid => FileId, + uri => iolist_to_binary(URI) + }, + case Info of + #{meta := Meta} -> + Res#{metadata => emqx_ft:encode_filemeta(Meta)}; + #{} -> + Res + end. + +format_timestamp(Timestamp) -> + iolist_to_binary(calendar:system_time_to_rfc3339(Timestamp, [{unit, second}])). + +format_name(NameBin) when is_binary(NameBin) -> + NameBin; +format_name(Name) when is_list(Name) -> + iolist_to_binary(Name). + +limit(QueryString) -> + maps:get(<<"limit">>, QueryString, emqx_mgmt:default_row_limit()). diff --git a/apps/emqx_ft/src/emqx_ft_app.erl b/apps/emqx_ft/src/emqx_ft_app.erl new file mode 100644 index 000000000..299683e43 --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_app.erl @@ -0,0 +1,30 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_app). + +-behaviour(application). + +-export([start/2, stop/1]). + +start(_StartType, _StartArgs) -> + {ok, Sup} = emqx_ft_sup:start_link(), + ok = emqx_ft_conf:load(), + {ok, Sup}. + +stop(_State) -> + ok = emqx_ft_conf:unload(), + ok. diff --git a/apps/emqx_ft/src/emqx_ft_assembler.erl b/apps/emqx_ft/src/emqx_ft_assembler.erl new file mode 100644 index 000000000..873efc6ff --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_assembler.erl @@ -0,0 +1,192 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_assembler). + +-export([start_link/3]). + +-behaviour(gen_statem). +-export([callback_mode/0]). +-export([init/1]). +-export([handle_event/4]). +-export([terminate/3]). + +-export([where/1]). + +-type stdata() :: #{ + storage := emqx_ft_storage_fs:storage(), + transfer := emqx_ft:transfer(), + assembly := emqx_ft_assembly:t(), + export => emqx_ft_storage_exporter:export() +}. + +-define(NAME(Transfer), {n, l, {?MODULE, Transfer}}). +-define(REF(Transfer), {via, gproc, ?NAME(Transfer)}). + +%% + +start_link(Storage, Transfer, Size) -> + gen_statem:start_link(?REF(Transfer), ?MODULE, {Storage, Transfer, Size}, []). + +where(Transfer) -> + gproc:where(?NAME(Transfer)). + +%% + +-type state() :: + idle + | list_local_fragments + | {list_remote_fragments, [node()]} + | start_assembling + | {assemble, [{node(), emqx_ft_storage_fs:filefrag()}]} + | complete. + +-define(internal(C), {next_event, internal, C}). + +callback_mode() -> + handle_event_function. + +-spec init(_Args) -> {ok, state(), stdata()}. +init({Storage, Transfer, Size}) -> + _ = erlang:process_flag(trap_exit, true), + St = #{ + storage => Storage, + transfer => Transfer, + assembly => emqx_ft_assembly:new(Size) + }, + {ok, idle, St}. + +-spec handle_event(info | internal, _, state(), stdata()) -> + {next_state, state(), stdata(), {next_event, internal, _}} + | {stop, {shutdown, ok | {error, _}}, stdata()}. +handle_event(info, kickoff, idle, St) -> + % NOTE + % Someone's told us to start the work, which usually means that it has set up a monitor. + % We could wait for this message and handle it at the end of the assembling rather than at + % the beginning, however it would make error handling much more messier. + {next_state, list_local_fragments, St, ?internal([])}; +handle_event(info, kickoff, _, _St) -> + keep_state_and_data; +handle_event( + internal, + _, + list_local_fragments, + St = #{storage := Storage, transfer := Transfer, assembly := Asm} +) -> + % TODO: what we do with non-transients errors here (e.g. `eacces`)? + {ok, Fragments} = emqx_ft_storage_fs:list(Storage, Transfer, fragment), + NAsm = emqx_ft_assembly:update(emqx_ft_assembly:append(Asm, node(), Fragments)), + NSt = St#{assembly := NAsm}, + case emqx_ft_assembly:status(NAsm) of + complete -> + {next_state, start_assembling, NSt, ?internal([])}; + {incomplete, _} -> + Nodes = mria_mnesia:running_nodes() -- [node()], + {next_state, {list_remote_fragments, Nodes}, NSt, ?internal([])}; + % TODO: recovery? + {error, _} = Error -> + {stop, {shutdown, Error}} + end; +handle_event( + internal, + _, + {list_remote_fragments, Nodes}, + St = #{transfer := Transfer, assembly := Asm} +) -> + % TODO + % Async would better because we would not need to wait for some lagging nodes if + % the coverage is already complete. + % TODO: portable "storage" ref + Results = emqx_ft_storage_fs_proto_v1:multilist(Nodes, Transfer, fragment), + NodeResults = lists:zip(Nodes, Results), + NAsm = emqx_ft_assembly:update( + lists:foldl( + fun + ({Node, {ok, {ok, Fragments}}}, Acc) -> + emqx_ft_assembly:append(Acc, Node, Fragments); + ({_Node, _Result}, Acc) -> + % TODO: log? + Acc + end, + Asm, + NodeResults + ) + ), + NSt = St#{assembly := NAsm}, + case emqx_ft_assembly:status(NAsm) of + complete -> + {next_state, start_assembling, NSt, ?internal([])}; + % TODO: retries / recovery? + {incomplete, _} = Status -> + {stop, {shutdown, {error, Status}}}; + {error, _} = Error -> + {stop, {shutdown, Error}} + end; +handle_event( + internal, + _, + start_assembling, + St = #{storage := Storage, transfer := Transfer, assembly := Asm} +) -> + Filemeta = emqx_ft_assembly:filemeta(Asm), + Coverage = emqx_ft_assembly:coverage(Asm), + case emqx_ft_storage_exporter:start_export(Storage, Transfer, Filemeta) of + {ok, Export} -> + {next_state, {assemble, Coverage}, St#{export => Export}, ?internal([])}; + {error, _} = Error -> + {stop, {shutdown, Error}} + end; +handle_event(internal, _, {assemble, [{Node, Segment} | Rest]}, St = #{export := Export}) -> + % TODO + % Currently, race is possible between getting segment info from the remote node and + % this node garbage collecting the segment itself. + % TODO: pipelining + % TODO: better error handling + {ok, Content} = pread(Node, Segment, St), + case emqx_ft_storage_exporter:write(Export, Content) of + {ok, NExport} -> + {next_state, {assemble, Rest}, St#{export := NExport}, ?internal([])}; + {error, _} = Error -> + {stop, {shutdown, Error}, maps:remove(export, St)} + end; +handle_event(internal, _, {assemble, []}, St = #{}) -> + {next_state, complete, St, ?internal([])}; +handle_event(internal, _, complete, St = #{export := Export}) -> + Result = emqx_ft_storage_exporter:complete(Export), + _ = maybe_garbage_collect(Result, St), + {stop, {shutdown, Result}, maps:remove(export, St)}. + +-spec terminate(_Reason, state(), stdata()) -> _. +terminate(_Reason, _StateName, #{export := Export}) -> + emqx_ft_storage_exporter:discard(Export); +terminate(_Reason, _StateName, #{}) -> + ok. + +pread(Node, Segment, #{storage := Storage, transfer := Transfer}) when Node =:= node() -> + emqx_ft_storage_fs:pread(Storage, Transfer, Segment, 0, segsize(Segment)); +pread(Node, Segment, #{transfer := Transfer}) -> + emqx_ft_storage_fs_proto_v1:pread(Node, Transfer, Segment, 0, segsize(Segment)). + +%% + +maybe_garbage_collect(ok, #{storage := Storage, transfer := Transfer, assembly := Asm}) -> + Nodes = emqx_ft_assembly:nodes(Asm), + emqx_ft_storage_fs_gc:collect(Storage, Transfer, Nodes); +maybe_garbage_collect({error, _}, _St) -> + ok. + +segsize(#{fragment := {segment, Info}}) -> + maps:get(size, Info). diff --git a/apps/emqx_ft/src/emqx_ft_assembler_sup.erl b/apps/emqx_ft/src/emqx_ft_assembler_sup.erl new file mode 100644 index 000000000..4ba65c290 --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_assembler_sup.erl @@ -0,0 +1,47 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_assembler_sup). + +-export([start_link/0]). +-export([ensure_child/3]). + +-behaviour(supervisor). +-export([init/1]). + +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +ensure_child(Storage, Transfer, Size) -> + Childspec = #{ + id => Transfer, + start => {emqx_ft_assembler, start_link, [Storage, Transfer, Size]}, + restart => temporary + }, + case supervisor:start_child(?MODULE, Childspec) of + {ok, Pid} -> + {ok, Pid}; + {error, {already_started, Pid}} -> + {ok, Pid} + end. + +init(_) -> + SupFlags = #{ + strategy => one_for_one, + intensity => 10, + period => 1000 + }, + {ok, {SupFlags, []}}. diff --git a/apps/emqx_ft/src/emqx_ft_assembly.erl b/apps/emqx_ft/src/emqx_ft_assembly.erl new file mode 100644 index 000000000..d765a2bd2 --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_assembly.erl @@ -0,0 +1,416 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_assembly). + +-export([new/1]). +-export([append/3]). +-export([update/1]). + +-export([status/1]). +-export([filemeta/1]). +-export([nodes/1]). +-export([coverage/1]). +-export([properties/1]). + +-export_type([t/0]). + +-type filemeta() :: emqx_ft:filemeta(). +-type filefrag() :: emqx_ft_storage_fs:filefrag(). +-type filefrag(T) :: emqx_ft_storage_fs:filefrag(T). +-type segmentinfo() :: emqx_ft_storage_fs:segmentinfo(). + +-record(asm, { + status :: status(), + coverage :: coverage() | undefined, + properties :: properties() | undefined, + meta :: #{filemeta() => {node(), filefrag({filemeta, filemeta()})}}, + segs :: emqx_wdgraph:t(emqx_ft:offset(), {node(), filefrag({segment, segmentinfo()})}), + size :: emqx_ft:bytes() +}). + +-type status() :: + {incomplete, {missing, _}} + | complete + | {error, {inconsistent, _}}. + +-type coverage() :: [{node(), filefrag({segment, segmentinfo()})}]. + +-type properties() :: #{ + %% Node where "most" of the segments are located. + dominant => node() +}. + +-opaque t() :: #asm{}. + +-spec new(emqx_ft:bytes()) -> t(). +new(Size) -> + #asm{ + status = {incomplete, {missing, filemeta}}, + meta = #{}, + segs = emqx_wdgraph:new(), + size = Size + }. + +-spec append(t(), node(), filefrag() | [filefrag()]) -> t(). +append(Asm, Node, Fragments) when is_list(Fragments) -> + lists:foldl(fun(F, AsmIn) -> append(AsmIn, Node, F) end, Asm, Fragments); +append(Asm, Node, Fragment = #{fragment := {filemeta, _}}) -> + append_filemeta(Asm, Node, Fragment); +append(Asm, Node, Segment = #{fragment := {segment, _}}) -> + append_segmentinfo(Asm, Node, Segment). + +-spec update(t()) -> t(). +update(Asm) -> + case status(meta, Asm) of + {complete, _Meta} -> + case status(coverage, Asm) of + {complete, Coverage, Props} -> + Asm#asm{ + status = complete, + coverage = Coverage, + properties = Props + }; + Status -> + Asm#asm{status = Status} + end; + Status -> + Asm#asm{status = Status} + end. + +-spec status(t()) -> status(). +status(#asm{status = Status}) -> + Status. + +-spec filemeta(t()) -> filemeta(). +filemeta(Asm) -> + case status(meta, Asm) of + {complete, Meta} -> Meta; + _Other -> undefined + end. + +-spec coverage(t()) -> coverage() | undefined. +coverage(#asm{coverage = Coverage}) -> + Coverage. + +-spec nodes(t()) -> [node()]. +nodes(#asm{meta = Meta, segs = Segs}) -> + S1 = maps:fold( + fun(_Meta, {Node, _Fragment}, Acc) -> + ordsets:add_element(Node, Acc) + end, + ordsets:new(), + Meta + ), + S2 = emqx_wdgraph:fold( + fun(_Offset, {_End, _, {Node, _Fragment}}, Acc) -> + ordsets:add_element(Node, Acc) + end, + ordsets:new(), + Segs + ), + ordsets:to_list(ordsets:union(S1, S2)). + +properties(#asm{properties = Properties}) -> + Properties. + +status(meta, #asm{meta = Meta}) -> + status(meta, maps:to_list(Meta)); +status(meta, [{Meta, {_Node, _Frag}}]) -> + {complete, Meta}; +status(meta, []) -> + {incomplete, {missing, filemeta}}; +status(meta, [_M1, _M2 | _] = Metas) -> + {error, {inconsistent, [Frag#{node => Node} || {_, {Node, Frag}} <- Metas]}}; +status(coverage, #asm{segs = Segments, size = Size}) -> + case coverage(Segments, Size) of + Coverage when is_list(Coverage) -> + {complete, Coverage, #{ + dominant => dominant(Coverage) + }}; + Missing = {missing, _} -> + {incomplete, Missing} + end. + +append_filemeta(Asm, Node, Fragment = #{fragment := {filemeta, Meta}}) -> + Asm#asm{ + meta = maps:put(Meta, {Node, Fragment}, Asm#asm.meta) + }. + +append_segmentinfo(Asm, _Node, #{fragment := {segment, #{size := 0}}}) -> + % NOTE + % Empty segments are valid but meaningless for coverage. + Asm; +append_segmentinfo(Asm, Node, Fragment = #{fragment := {segment, Info}}) -> + Offset = maps:get(offset, Info), + Size = maps:get(size, Info), + End = Offset + Size, + Segs = add_edge(Asm#asm.segs, Offset, End, locality(Node) * Size, {Node, Fragment}), + Asm#asm{ + % TODO + % In theory it's possible to have two segments with same offset + size on + % different nodes but with differing content. We'd need a checksum to + % be able to disambiguate them though. + segs = Segs + }. + +add_edge(Segs, Offset, End, Weight, Label) -> + % NOTE + % We are expressing coverage problem as a shortest path problem on weighted directed + % graph, where nodes are segments offsets, two nodes are connected with edge if + % there is a segment which "covers" these offsets (i.e. it starts at first node's + % offset and ends at second node's offst) and weights are segments sizes adjusted + % for locality (i.e. weight are always 0 for any local segment). + case emqx_wdgraph:find_edge(Offset, End, Segs) of + {WeightWas, _Label} when WeightWas =< Weight -> + % NOTE + % Discarding any edges with higher weight here. This is fine as long as we + % optimize for locality. + Segs; + _ -> + emqx_wdgraph:insert_edge(Offset, End, Weight, Label, Segs) + end. + +coverage(Segs, Size) -> + case emqx_wdgraph:find_shortest_path(0, Size, Segs) of + Path when is_list(Path) -> + Path; + {false, LastOffset} -> + % NOTE + % This is far from being accurate, but needs no hairy specifics in the + % `emqx_wdgraph` interface. + {missing, {segment, LastOffset, Size}} + end. + +dominant(Coverage) -> + % TODO: needs improvement, better defined _dominance_, maybe some score + Freqs = frequencies(fun({Node, Segment}) -> {Node, segsize(Segment)} end, Coverage), + maxfreq(Freqs, node()). + +frequencies(Fun, List) -> + lists:foldl( + fun(E, Acc) -> + {K, N} = Fun(E), + maps:update_with(K, fun(M) -> M + N end, N, Acc) + end, + #{}, + List + ). + +maxfreq(Freqs, Init) -> + {_, Max} = maps:fold( + fun + (F, N, {M, _MF}) when N > M -> {N, F}; + (_F, _N, {M, MF}) -> {M, MF} + end, + {0, Init}, + Freqs + ), + Max. + +locality(Node) when Node =:= node() -> + % NOTE + % This should prioritize locally available segments over those on remote nodes. + 0; +locality(_RemoteNode) -> + 1. + +segsize(#{fragment := {segment, Info}}) -> + maps:get(size, Info). + +-ifdef(TEST). + +-include_lib("eunit/include/eunit.hrl"). + +incomplete_new_test() -> + ?assertEqual( + {incomplete, {missing, filemeta}}, + status(update(new(42))) + ). + +incomplete_test() -> + ?assertEqual( + {incomplete, {missing, filemeta}}, + status( + update( + append(new(142), node(), [ + segment(p1, 0, 42), + segment(p1, 42, 100) + ]) + ) + ) + ). + +consistent_test() -> + Asm1 = append(new(42), n1, [filemeta(m1, "blarg")]), + Asm2 = append(Asm1, n2, [segment(s2, 0, 42)]), + Asm3 = append(Asm2, n3, [filemeta(m3, "blarg")]), + ?assertMatch({complete, _}, status(meta, Asm3)). + +inconsistent_test() -> + Asm1 = append(new(42), node(), [segment(s1, 0, 42)]), + Asm2 = append(Asm1, n1, [filemeta(m1, "blarg")]), + Asm3 = append(Asm2, n2, [segment(s2, 0, 42), filemeta(m1, "blorg")]), + Asm4 = append(Asm3, n3, [filemeta(m3, "blarg")]), + ?assertMatch( + {error, + {inconsistent, [ + % blarg < blorg + #{node := n3, path := m3, fragment := {filemeta, #{name := "blarg"}}}, + #{node := n2, path := m1, fragment := {filemeta, #{name := "blorg"}}} + ]}}, + status(meta, Asm4) + ). + +simple_coverage_test() -> + Node = node(), + Segs = [ + {node42, segment(n1, 20, 30)}, + {Node, segment(n2, 0, 10)}, + {Node, segment(n3, 50, 50)}, + {Node, segment(n4, 10, 10)} + ], + Asm = append_many(new(100), Segs), + ?assertMatch( + {complete, + [ + {Node, #{path := n2}}, + {Node, #{path := n4}}, + {node42, #{path := n1}}, + {Node, #{path := n3}} + ], + #{dominant := Node}}, + status(coverage, Asm) + ). + +redundant_coverage_test() -> + Node = node(), + Segs = [ + {Node, segment(n1, 0, 20)}, + {node1, segment(n2, 0, 10)}, + {Node, segment(n3, 20, 40)}, + {node2, segment(n4, 10, 10)}, + {node2, segment(n5, 50, 20)}, + {node3, segment(n6, 20, 20)}, + {Node, segment(n7, 50, 10)}, + {node1, segment(n8, 40, 10)} + ], + Asm = append_many(new(70), Segs), + ?assertMatch( + {complete, + [ + {Node, #{path := n1}}, + {node3, #{path := n6}}, + {node1, #{path := n8}}, + {node2, #{path := n5}} + ], + #{dominant := _}}, + status(coverage, Asm) + ). + +redundant_coverage_prefer_local_test() -> + Node = node(), + Segs = [ + {node1, segment(n1, 0, 20)}, + {Node, segment(n2, 0, 10)}, + {Node, segment(n3, 10, 10)}, + {node2, segment(n4, 20, 20)}, + {Node, segment(n5, 30, 10)}, + {Node, segment(n6, 20, 10)} + ], + Asm = append_many(new(40), Segs), + ?assertMatch( + {complete, + [ + {Node, #{path := n2}}, + {Node, #{path := n3}}, + {Node, #{path := n6}}, + {Node, #{path := n5}} + ], + #{dominant := Node}}, + status(coverage, Asm) + ). + +missing_coverage_test() -> + Node = node(), + Segs = [ + {Node, segment(n1, 0, 10)}, + {node1, segment(n3, 10, 20)}, + {Node, segment(n2, 0, 20)}, + {node2, segment(n4, 50, 50)}, + {Node, segment(n5, 40, 60)} + ], + Asm = append_many(new(100), Segs), + ?assertEqual( + % {incomplete, {missing, {segment, 30, 40}}} would be more accurate + {incomplete, {missing, {segment, 30, 100}}}, + status(coverage, Asm) + ). + +missing_end_coverage_test() -> + Node = node(), + Segs = [ + {Node, segment(n1, 0, 15)}, + {node1, segment(n3, 10, 10)} + ], + Asm = append_many(new(20), Segs), + ?assertEqual( + {incomplete, {missing, {segment, 15, 20}}}, + status(coverage, Asm) + ). + +missing_coverage_with_redudancy_test() -> + Segs = [ + {node(), segment(n1, 0, 10)}, + {node(), segment(n2, 0, 20)}, + {node42, segment(n3, 10, 20)}, + {node43, segment(n4, 10, 50)}, + {node(), segment(n5, 40, 60)} + ], + Asm = append_many(new(100), Segs), + ?assertEqual( + % {incomplete, {missing, {segment, 50, 60}}}, ??? + {incomplete, {missing, {segment, 60, 100}}}, + status(coverage, Asm) + ). + +append_many(Asm, List) -> + lists:foldl( + fun({Node, Frag}, Acc) -> append(Acc, Node, Frag) end, + Asm, + List + ). + +filemeta(Path, Name) -> + #{ + path => Path, + fragment => + {filemeta, #{ + name => Name + }} + }. + +segment(Path, Offset, Size) -> + #{ + path => Path, + fragment => + {segment, #{ + offset => Offset, + size => Size + }} + }. + +-endif. diff --git a/apps/emqx_ft/src/emqx_ft_conf.erl b/apps/emqx_ft/src/emqx_ft_conf.erl new file mode 100644 index 000000000..2e994925c --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_conf.erl @@ -0,0 +1,143 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @doc File Transfer configuration management module + +-module(emqx_ft_conf). + +-behaviour(emqx_config_handler). + +-include_lib("emqx/include/logger.hrl"). + +%% Accessors +-export([enabled/0]). +-export([storage/0]). +-export([gc_interval/1]). +-export([segments_ttl/1]). +-export([init_timeout/0]). +-export([store_segment_timeout/0]). +-export([assemble_timeout/0]). + +%% Load/Unload +-export([ + load/0, + unload/0 +]). + +%% callbacks for emqx_config_handler +-export([ + pre_config_update/3, + post_config_update/5 +]). + +-type milliseconds() :: non_neg_integer(). +-type seconds() :: non_neg_integer(). + +%%-------------------------------------------------------------------- +%% Accessors +%%-------------------------------------------------------------------- + +-spec enabled() -> boolean(). +enabled() -> + emqx_config:get([file_transfer, enable], false). + +-spec storage() -> emqx_config:config(). +storage() -> + emqx_config:get([file_transfer, storage]). + +-spec gc_interval(emqx_ft_storage_fs:storage()) -> + emqx_maybe:t(milliseconds()). +gc_interval(Storage) -> + emqx_utils_maps:deep_get([segments, gc, interval], Storage, undefined). + +-spec segments_ttl(emqx_ft_storage_fs:storage()) -> + emqx_maybe:t({_Min :: seconds(), _Max :: seconds()}). +segments_ttl(Storage) -> + Min = emqx_utils_maps:deep_get([segments, gc, minimum_segments_ttl], Storage, undefined), + Max = emqx_utils_maps:deep_get([segments, gc, maximum_segments_ttl], Storage, undefined), + case is_integer(Min) andalso is_integer(Max) of + true -> + {Min, Max}; + false -> + undefined + end. + +init_timeout() -> + emqx_config:get([file_transfer, init_timeout]). + +assemble_timeout() -> + emqx_config:get([file_transfer, assemble_timeout]). + +store_segment_timeout() -> + emqx_config:get([file_transfer, store_segment_timeout]). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- + +-spec load() -> ok. +load() -> + ok = maybe_start(), + emqx_conf:add_handler([file_transfer], ?MODULE). + +-spec unload() -> ok. +unload() -> + ok = stop(), + emqx_conf:remove_handler([file_transfer]). + +%%-------------------------------------------------------------------- +%% emqx_config_handler callbacks +%%-------------------------------------------------------------------- + +-spec pre_config_update(list(atom()), emqx_config:update_request(), emqx_config:raw_config()) -> + {ok, emqx_config:update_request()} | {error, term()}. +pre_config_update(_, Req, _Config) -> + {ok, Req}. + +-spec post_config_update( + list(atom()), + emqx_config:update_request(), + emqx_config:config(), + emqx_config:config(), + emqx_config:app_envs() +) -> + ok | {ok, Result :: any()} | {error, Reason :: term()}. +post_config_update([file_transfer | _], _Req, NewConfig, OldConfig, _AppEnvs) -> + on_config_update(OldConfig, NewConfig). + +on_config_update(#{enable := false}, #{enable := false}) -> + ok; +on_config_update(#{enable := true, storage := OldStorage}, #{enable := false}) -> + ok = emqx_ft_storage:on_config_update(OldStorage, undefined), + ok = emqx_ft:unhook(); +on_config_update(#{enable := false}, #{enable := true, storage := NewStorage}) -> + ok = emqx_ft_storage:on_config_update(undefined, NewStorage), + ok = emqx_ft:hook(); +on_config_update(#{enable := true, storage := OldStorage}, #{enable := true, storage := NewStorage}) -> + ok = emqx_ft_storage:on_config_update(OldStorage, NewStorage). + +maybe_start() -> + case emqx_config:get([file_transfer]) of + #{enable := true, storage := Storage} -> + ok = emqx_ft_storage:on_config_update(undefined, Storage), + ok = emqx_ft:hook(); + _ -> + ok + end. + +stop() -> + ok = emqx_ft:unhook(), + ok = emqx_ft_storage:on_config_update(storage(), undefined). diff --git a/apps/emqx_ft/src/emqx_ft_fs_iterator.erl b/apps/emqx_ft/src/emqx_ft_fs_iterator.erl new file mode 100644 index 000000000..7a58c5b38 --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_fs_iterator.erl @@ -0,0 +1,235 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_fs_iterator). + +-export([new/2]). +-export([next/1]). +-export([next_leaf/1]). + +-export([seek/3]). + +-export([fold/3]). +-export([fold_n/4]). + +-export_type([t/0]). +-export_type([glob/0]). +-export_type([pathstack/0]). + +-type root() :: file:name(). +-type glob() :: ['*' | globfun()]. +-type globfun() :: + fun((_Filename :: file:name()) -> boolean()) + | fun((_Filename :: file:name(), pathstack()) -> boolean()). + +% A path stack is a list of path components, in reverse order. +-type pathstack() :: [file:name(), ...]. + +-opaque t() :: #{ + root := root(), + queue := [_PathStack :: [file:name()]], + head := glob(), + stack := [{[pathstack()], glob()}] +}. + +-type entry() :: entry_leaf() | entry_node(). +-type entry_leaf() :: + {leaf, file:name(), file:file_info() | {error, file:posix()}, pathstack()}. +-type entry_node() :: + {node, file:name(), {error, file:posix()}, pathstack()}. + +-spec new(root(), glob()) -> + t(). +new(Root, Glob) -> + #{ + root => Root, + queue => [[]], + head => Glob, + stack => [] + }. + +-spec next(t()) -> + {entry(), t()} | none. +next(It = #{queue := [PathStack | Rest], head := []}) -> + {emit(PathStack, It), It#{queue => Rest}}; +next(It = #{queue := [PathStack | Rest], head := [Pat | _], root := Root}) -> + Filepath = mk_filepath(PathStack), + case emqx_ft_fs_util:list_dir(filename:join(Root, Filepath)) of + {ok, Filenames} -> + Sorted = lists:sort(Filenames), + Matches = [[Fn | PathStack] || Fn <- Sorted, matches_glob(Pat, Fn, [Fn | PathStack])], + ItNext = windup(It), + next(ItNext#{queue => Matches}); + {error, _} = Error -> + {{node, Filepath, Error, PathStack}, It#{queue => Rest}} + end; +next(It = #{queue := []}) -> + unwind(It). + +windup(It = #{queue := [_ | Rest], head := [Pat | Glob], stack := Stack}) -> + % NOTE + % Preserve unfinished paths and glob in the stack, so that we can resume traversal + % when the lower levels of the tree are exhausted. + It#{ + head => Glob, + stack => [{Rest, [Pat | Glob]} | Stack] + }. + +unwind(It = #{stack := [{Queue, Glob} | StackRest]}) -> + % NOTE + % Resume traversal of unfinished paths from the upper levels of the tree. + next(It#{ + queue => Queue, + head => Glob, + stack => StackRest + }); +unwind(#{stack := []}) -> + none. + +emit(PathStack, #{root := Root}) -> + Filepath = mk_filepath(PathStack), + case emqx_ft_fs_util:read_info(filename:join(Root, Filepath)) of + {ok, Fileinfo} -> + {leaf, Filepath, Fileinfo, PathStack}; + {error, _} = Error -> + {leaf, Filepath, Error, PathStack} + end. + +mk_filepath([]) -> + ""; +mk_filepath(PathStack) -> + filename:join(lists:reverse(PathStack)). + +matches_glob('*', _, _) -> + true; +matches_glob(FilterFun, Filename, _PathStack) when is_function(FilterFun, 1) -> + FilterFun(Filename); +matches_glob(FilterFun, Filename, PathStack) when is_function(FilterFun, 2) -> + FilterFun(Filename, PathStack). + +%% + +-spec next_leaf(t()) -> + {entry_leaf(), t()} | none. +next_leaf(It) -> + case next(It) of + {{leaf, _, _, _} = Leaf, ItNext} -> + {Leaf, ItNext}; + {{node, _Filename, _Error, _PathStack}, ItNext} -> + % NOTE + % Intentionally skipping intermediate traversal errors here, for simplicity. + next_leaf(ItNext); + none -> + none + end. + +%% + +-spec seek([file:name()], root(), glob()) -> + t(). +seek(PathSeek, Root, Glob) -> + SeekGlob = mk_seek_glob(PathSeek, Glob), + SeekStack = lists:reverse(PathSeek), + case next_leaf(new(Root, SeekGlob)) of + {{leaf, _Filepath, _Info, SeekStack}, It} -> + fixup_glob(Glob, It); + {{leaf, _Filepath, _Info, Successor}, It = #{queue := Queue}} -> + fixup_glob(Glob, It#{queue => [Successor | Queue]}); + none -> + none(Root) + end. + +mk_seek_glob(PathSeek, Glob) -> + % NOTE + % The seek glob is a glob that skips all the nodes / leaves that are lexicographically + % smaller than the seek path. For example, if the seek path is ["a", "b", "c"], and + % the glob is ['*', '*', '*', '*'], then the seek glob is: + % [ fun(Path) -> Path >= ["a"] end, + % fun(Path) -> Path >= ["a", "b"] end, + % fun(Path) -> Path >= ["a", "b", "c"] end, + % '*' + % ] + L = min(length(PathSeek), length(Glob)), + merge_glob([mk_seek_pat(lists:sublist(PathSeek, N)) || N <- lists:seq(1, L)], Glob). + +mk_seek_pat(PathSeek) -> + % NOTE + % The `PathStack` and `PathSeek` are of the same length here. + fun(_Filename, PathStack) -> lists:reverse(PathStack) >= PathSeek end. + +merge_glob([Pat | SeekRest], [PatOrig | Rest]) -> + [merge_pat(Pat, PatOrig) | merge_glob(SeekRest, Rest)]; +merge_glob([], [PatOrig | Rest]) -> + [PatOrig | merge_glob([], Rest)]; +merge_glob([], []) -> + []. + +merge_pat(Pat, PatOrig) -> + fun(Filename, PathStack) -> + Pat(Filename, PathStack) andalso matches_glob(PatOrig, Filename, PathStack) + end. + +fixup_glob(Glob, It = #{head := [], stack := Stack}) -> + % NOTE + % Restoring original glob through the stack. Strictly speaking, this is not usually + % necessary, it's a kind of optimization. + fixup_glob(Glob, lists:reverse(Stack), It#{stack => []}). + +fixup_glob(Glob = [_ | Rest], [{Queue, _} | StackRest], It = #{stack := Stack}) -> + fixup_glob(Rest, StackRest, It#{stack => [{Queue, Glob} | Stack]}); +fixup_glob(Rest, [], It) -> + It#{head => Rest}. + +%% + +-spec fold(fun((entry(), Acc) -> Acc), Acc, t()) -> + Acc. +fold(FoldFun, Acc, It) -> + case next(It) of + {Entry, ItNext} -> + fold(FoldFun, FoldFun(Entry, Acc), ItNext); + none -> + Acc + end. + +%% NOTE +%% Passing negative `N` is allowed, in which case the iterator will be exhausted +%% completely, like in `fold/3`. +-spec fold_n(fun((entry(), Acc) -> Acc), Acc, t(), _N :: integer()) -> + {Acc, {more, t()} | none}. +fold_n(_FoldFun, Acc, It, 0) -> + {Acc, {more, It}}; +fold_n(FoldFun, Acc, It, N) -> + case next(It) of + {Entry, ItNext} -> + fold_n(FoldFun, FoldFun(Entry, Acc), ItNext, N - 1); + none -> + {Acc, none} + end. + +%% + +-spec none(root()) -> + t(). +none(Root) -> + % NOTE + % The _none_ iterator is a valid iterator, but it will never yield any entries. + #{ + root => Root, + queue => [], + head => [], + stack => [] + }. diff --git a/apps/emqx_ft/src/emqx_ft_fs_util.erl b/apps/emqx_ft/src/emqx_ft_fs_util.erl new file mode 100644 index 000000000..9028722aa --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_fs_util.erl @@ -0,0 +1,180 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_fs_util). + +-include_lib("snabbkaffe/include/trace.hrl"). +-include_lib("kernel/include/file.hrl"). + +-export([is_filename_safe/1]). +-export([escape_filename/1]). +-export([unescape_filename/1]). + +-export([read_decode_file/2]). +-export([read_info/1]). +-export([list_dir/1]). + +-export([fold/4]). + +-type foldfun(Acc) :: + fun( + ( + _Filepath :: file:name(), + _Info :: file:file_info() | {error, file:posix()}, + _Stack :: emqx_ft_fs_iterator:pathstack(), + Acc + ) -> Acc + ). + +-define(IS_UNSAFE(C), + ((C) =:= $% orelse + (C) =:= $: orelse + (C) =:= $\\ orelse + (C) =:= $/) +). + +-define(IS_PRINTABLE(C), + % NOTE: See `io_lib:printable_unicode_list/1` + (((C) >= 32 andalso (C) =< 126) orelse + ((C) >= 16#A0 andalso (C) < 16#D800) orelse + ((C) > 16#DFFF andalso (C) < 16#FFFE) orelse + ((C) > 16#FFFF andalso (C) =< 16#10FFFF)) +). + +%% + +-spec is_filename_safe(file:filename_all()) -> ok | {error, atom()}. +is_filename_safe(FN) when is_binary(FN) -> + is_filename_safe(unicode:characters_to_list(FN)); +is_filename_safe("") -> + {error, empty}; +is_filename_safe(FN) when FN == "." orelse FN == ".." -> + {error, special}; +is_filename_safe(FN) -> + verify_filename_safe(FN). + +verify_filename_safe([$% | Rest]) -> + verify_filename_safe(Rest); +verify_filename_safe([C | _]) when ?IS_UNSAFE(C) -> + {error, unsafe}; +verify_filename_safe([C | _]) when not ?IS_PRINTABLE(C) -> + {error, nonprintable}; +verify_filename_safe([_ | Rest]) -> + verify_filename_safe(Rest); +verify_filename_safe([]) -> + ok. + +-spec escape_filename(binary()) -> file:name(). +escape_filename(Name) when Name == <<".">> orelse Name == <<"..">> -> + lists:reverse(percent_encode(Name, "")); +escape_filename(Name) -> + escape(Name, ""). + +escape(<>, Acc) when ?IS_UNSAFE(C) -> + escape(Rest, percent_encode(<>, Acc)); +escape(<>, Acc) when not ?IS_PRINTABLE(C) -> + escape(Rest, percent_encode(<>, Acc)); +escape(<>, Acc) -> + escape(Rest, [C | Acc]); +escape(<<>>, Acc) -> + lists:reverse(Acc). + +-spec unescape_filename(file:name()) -> binary(). +unescape_filename(Name) -> + unescape(Name, <<>>). + +unescape([$%, A, B | Rest], Acc) -> + unescape(Rest, percent_decode(A, B, Acc)); +unescape([C | Rest], Acc) -> + unescape(Rest, <>); +unescape([], Acc) -> + Acc. + +percent_encode(<>, Acc) -> + percent_encode(Rest, [dec2hex(B), dec2hex(A), $% | Acc]); +percent_encode(<<>>, Acc) -> + Acc. + +percent_decode(A, B, Acc) -> + <>. + +dec2hex(X) when (X >= 0) andalso (X =< 9) -> X + $0; +dec2hex(X) when (X >= 10) andalso (X =< 15) -> X + $A - 10. + +hex2dec(X) when (X >= $0) andalso (X =< $9) -> X - $0; +hex2dec(X) when (X >= $A) andalso (X =< $F) -> X - $A + 10; +hex2dec(X) when (X >= $a) andalso (X =< $f) -> X - $a + 10; +hex2dec(_) -> error(badarg). + +%% + +-spec read_decode_file(file:name(), fun((binary()) -> Value)) -> + {ok, Value} | {error, _IoError}. +read_decode_file(Filepath, DecodeFun) -> + case file:read_file(Filepath) of + {ok, Content} -> + safe_decode(Content, DecodeFun); + {error, _} = Error -> + Error + end. + +safe_decode(Content, DecodeFun) -> + try + {ok, DecodeFun(Content)} + catch + C:E:Stacktrace -> + ?tp(warning, "safe_decode_failed", #{ + class => C, + exception => E, + stacktrace => Stacktrace + }), + {error, corrupted} + end. + +-spec read_info(file:name_all()) -> + {ok, file:file_info()} | {error, file:posix() | badarg}. +read_info(AbsPath) -> + % NOTE + % Be aware that this function is occasionally mocked in `emqx_ft_fs_util_SUITE`. + file:read_link_info(AbsPath, [{time, posix}, raw]). + +-spec list_dir(file:name_all()) -> + {ok, [file:name()]} | {error, file:posix() | badarg}. +list_dir(AbsPath) -> + case ?MODULE:read_info(AbsPath) of + {ok, #file_info{type = directory}} -> + file:list_dir(AbsPath); + {ok, #file_info{}} -> + {error, enotdir}; + {error, Reason} -> + {error, Reason} + end. + +-spec fold(foldfun(Acc), Acc, _Root :: file:name(), emqx_ft_fs_iterator:glob()) -> + Acc. +fold(FoldFun, Acc, Root, Glob) -> + fold(FoldFun, Acc, emqx_ft_fs_iterator:new(Root, Glob)). + +fold(FoldFun, Acc, It) -> + case emqx_ft_fs_iterator:next(It) of + {{node, _Path, {error, enotdir}, _PathStack}, ItNext} -> + fold(FoldFun, Acc, ItNext); + {{_Type, Path, Info, PathStack}, ItNext} -> + AccNext = FoldFun(Path, Info, PathStack, Acc), + fold(FoldFun, AccNext, ItNext); + none -> + Acc + end. diff --git a/apps/emqx_ft/src/emqx_ft_responder.erl b/apps/emqx_ft/src/emqx_ft_responder.erl new file mode 100644 index 000000000..c2c62e1c2 --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_responder.erl @@ -0,0 +1,116 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_responder). + +-behaviour(gen_server). + +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/types.hrl"). + +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +%% API +-export([start/3]). +-export([kickoff/2]). +-export([ack/2]). + +%% Supervisor API +-export([start_link/3]). + +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2]). + +-define(REF(Key), {via, gproc, {n, l, {?MODULE, Key}}}). + +-type key() :: term(). +-type respfun() :: fun(({ack, _Result} | {down, _Result} | timeout) -> _SideEffect). + +%%-------------------------------------------------------------------- +%% API +%% ------------------------------------------------------------------- + +-spec start(key(), respfun(), timeout()) -> startlink_ret(). +start(Key, RespFun, Timeout) -> + emqx_ft_responder_sup:start_child(Key, RespFun, Timeout). + +-spec kickoff(key(), pid()) -> ok. +kickoff(Key, Pid) -> + gen_server:call(?REF(Key), {kickoff, Pid}). + +-spec ack(key(), _Result) -> _Return. +ack(Key, Result) -> + % TODO: it's possible to avoid term copy + gen_server:call(?REF(Key), {ack, Result}, infinity). + +-spec start_link(key(), timeout(), respfun()) -> startlink_ret(). +start_link(Key, RespFun, Timeout) -> + gen_server:start_link(?REF(Key), ?MODULE, {Key, RespFun, Timeout}, []). + +%%-------------------------------------------------------------------- +%% gen_server callbacks +%% ------------------------------------------------------------------- + +init({Key, RespFun, Timeout}) -> + _ = erlang:process_flag(trap_exit, true), + _TRef = erlang:send_after(Timeout, self(), timeout), + {ok, {Key, RespFun}}. + +handle_call({kickoff, Pid}, _From, St) -> + % TODO: more state? + _MRef = erlang:monitor(process, Pid), + _ = Pid ! kickoff, + {reply, ok, St}; +handle_call({ack, Result}, _From, {Key, RespFun}) -> + Ret = apply(RespFun, [{ack, Result}]), + ?tp(debug, ft_responder_ack, #{key => Key, result => Result, return => Ret}), + {stop, {shutdown, Ret}, Ret, undefined}; +handle_call(Msg, _From, State) -> + ?SLOG(warning, #{msg => "unknown_call", call_msg => Msg}), + {reply, {error, unknown_call}, State}. + +handle_cast(Msg, State) -> + ?SLOG(warning, #{msg => "unknown_cast", cast_msg => Msg}), + {noreply, State}. + +handle_info(timeout, {Key, RespFun}) -> + Ret = apply(RespFun, [timeout]), + ?tp(debug, ft_responder_timeout, #{key => Key, return => Ret}), + {stop, {shutdown, Ret}, undefined}; +handle_info({'DOWN', _MRef, process, _Pid, Reason}, {Key, RespFun}) -> + Ret = apply(RespFun, [{down, map_down_reason(Reason)}]), + ?tp(debug, ft_responder_procdown, #{key => Key, reason => Reason, return => Ret}), + {stop, {shutdown, Ret}, undefined}; +handle_info(Msg, State) -> + ?SLOG(warning, #{msg => "unknown_message", info_msg => Msg}), + {noreply, State}. + +terminate(_Reason, undefined) -> + ok; +terminate(Reason, {Key, RespFun}) -> + Ret = apply(RespFun, [timeout]), + ?tp(debug, ft_responder_shutdown, #{key => Key, reason => Reason, return => Ret}), + ok. + +map_down_reason(normal) -> + ok; +map_down_reason(shutdown) -> + ok; +map_down_reason({shutdown, Result}) -> + Result; +map_down_reason(noproc) -> + {error, noproc}; +map_down_reason(Error) -> + {error, {internal_error, Error}}. diff --git a/apps/emqx_ft/src/emqx_ft_responder_sup.erl b/apps/emqx_ft/src/emqx_ft_responder_sup.erl new file mode 100644 index 000000000..fb3932425 --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_responder_sup.erl @@ -0,0 +1,48 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_responder_sup). + +-export([start_link/0]). +-export([start_child/3]). + +-behaviour(supervisor). +-export([init/1]). + +-define(SUPERVISOR, ?MODULE). + +%% + +-spec start_link() -> {ok, pid()}. +start_link() -> + supervisor:start_link({local, ?SUPERVISOR}, ?MODULE, []). + +start_child(Key, RespFun, Timeout) -> + supervisor:start_child(?SUPERVISOR, [Key, RespFun, Timeout]). + +-spec init(_) -> {ok, {supervisor:sup_flags(), [supervisor:child_spec()]}}. +init(_) -> + Flags = #{ + strategy => simple_one_for_one, + intensity => 100, + period => 100 + }, + ChildSpec = #{ + id => responder, + start => {emqx_ft_responder, start_link, []}, + restart => temporary + }, + {ok, {Flags, [ChildSpec]}}. diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl new file mode 100644 index 000000000..09e9ab0a5 --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_schema.erl @@ -0,0 +1,317 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_schema). + +-behaviour(hocon_schema). + +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("typerefl/include/types.hrl"). + +-export([namespace/0, roots/0, fields/1, tags/0, desc/1]). + +-export([schema/1]). + +-export([translate/1]). + +-type json_value() :: + null + | boolean() + | binary() + | number() + | [json_value()] + | #{binary() => json_value()}. + +-reflect_type([json_value/0]). + +%% NOTE +%% This is rather conservative limit, mostly dictated by the filename limitations +%% on most filesystems. Even though, say, S3 does not have such limitations, it's +%% still useful to have a limit on the filename length, to avoid having to deal with +%% limits in the storage backends. +-define(MAX_FILENAME_BYTELEN, 255). + +-import(hoconsc, [ref/2, mk/2]). + +namespace() -> file_transfer. + +tags() -> + [<<"File Transfer">>]. + +roots() -> [file_transfer]. + +fields(file_transfer) -> + [ + {enable, + mk( + boolean(), + #{ + desc => ?DESC("enable"), + required => false, + default => false + } + )}, + {init_timeout, + mk( + emqx_schema:duration_ms(), + #{ + desc => ?DESC("init_timeout"), + required => false, + default => "10s" + } + )}, + {store_segment_timeout, + mk( + emqx_schema:duration_ms(), + #{ + desc => ?DESC("store_segment_timeout"), + required => false, + default => "5m" + } + )}, + {assemble_timeout, + mk( + emqx_schema:duration_ms(), + #{ + desc => ?DESC("assemble_timeout"), + required => false, + default => "5m" + } + )}, + {storage, + mk( + ref(storage_backend), + #{ + desc => ?DESC("storage_backend"), + required => false, + validator => validator(backend), + default => #{ + <<"local">> => #{} + } + } + )} + ]; +fields(storage_backend) -> + [ + {local, + mk( + ref(local_storage), + #{ + desc => ?DESC("local_storage"), + required => {false, recursively} + } + )} + ]; +fields(local_storage) -> + [ + {segments, + mk( + ref(local_storage_segments), + #{ + desc => ?DESC("local_storage_segments"), + required => false, + default => #{ + <<"gc">> => #{} + } + } + )}, + {exporter, + mk( + ref(local_storage_exporter_backend), + #{ + desc => ?DESC("local_storage_exporter_backend"), + required => false, + validator => validator(backend), + default => #{ + <<"local">> => #{} + } + } + )} + ]; +fields(local_storage_segments) -> + [ + {root, + mk( + binary(), + #{ + desc => ?DESC("local_storage_segments_root"), + required => false + } + )}, + {gc, + mk( + ref(local_storage_segments_gc), #{ + desc => ?DESC("local_storage_segments_gc"), + required => false + } + )} + ]; +fields(local_storage_exporter_backend) -> + [ + {local, + mk( + ref(local_storage_exporter), + #{ + desc => ?DESC("local_storage_exporter"), + required => {false, recursively} + } + )}, + {s3, + mk( + ref(s3_exporter), + #{ + desc => ?DESC("s3_exporter"), + required => {false, recursively} + } + )} + ]; +fields(local_storage_exporter) -> + [ + {root, + mk( + binary(), + #{ + desc => ?DESC("local_storage_exporter_root"), + required => false + } + )} + ]; +fields(s3_exporter) -> + emqx_s3_schema:fields(s3); +fields(local_storage_segments_gc) -> + [ + {interval, + mk( + emqx_schema:duration_ms(), + #{ + desc => ?DESC("storage_gc_interval"), + required => false, + default => "1h" + } + )}, + {maximum_segments_ttl, + mk( + emqx_schema:duration_s(), + #{ + desc => ?DESC("storage_gc_max_segments_ttl"), + required => false, + default => "24h" + } + )}, + {minimum_segments_ttl, + mk( + emqx_schema:duration_s(), + #{ + desc => ?DESC("storage_gc_min_segments_ttl"), + required => false, + default => "5m", + % NOTE + % This setting does not seem to be useful to an end-user. + hidden => true + } + )} + ]. + +desc(file_transfer) -> + "File transfer settings"; +desc(local_storage) -> + "File transfer local storage settings"; +desc(local_storage_segments) -> + "File transfer local segments storage settings"; +desc(local_storage_exporter) -> + "Local Exporter settings for the File transfer local storage backend"; +desc(s3_exporter) -> + "S3 Exporter settings for the File transfer local storage backend"; +desc(local_storage_segments_gc) -> + "Garbage collection settings for the File transfer local segments storage"; +desc(local_storage_exporter_backend) -> + "Exporter for the local file system storage backend"; +desc(storage_backend) -> + "Storage backend settings for file transfer"; +desc(_) -> + undefined. + +schema(filemeta) -> + #{ + roots => [ + {name, + hoconsc:mk(string(), #{ + required => true, + validator => validator(filename), + converter => converter(unicode_string) + })}, + {size, hoconsc:mk(non_neg_integer())}, + {expire_at, hoconsc:mk(non_neg_integer())}, + {checksum, hoconsc:mk({atom(), binary()}, #{converter => converter(checksum)})}, + {segments_ttl, hoconsc:mk(pos_integer())}, + {user_data, hoconsc:mk(json_value())} + ] + }. + +validator(filename) -> + [ + fun(Value) -> + Bin = unicode:characters_to_binary(Value), + byte_size(Bin) =< ?MAX_FILENAME_BYTELEN orelse {error, max_length_exceeded} + end, + fun emqx_ft_fs_util:is_filename_safe/1 + ]; +validator(backend) -> + fun(Config) -> + case maps:keys(Config) of + [_Type] -> + ok; + _Conflicts = [_ | _] -> + {error, multiple_conflicting_backends} + end + end. + +converter(checksum) -> + fun + (undefined, #{}) -> + undefined; + ({sha256, Bin}, #{make_serializable := true}) -> + _ = is_binary(Bin) orelse throw({expected_type, string}), + _ = byte_size(Bin) =:= 32 orelse throw({expected_length, 32}), + binary:encode_hex(Bin); + (Hex, #{}) -> + _ = is_binary(Hex) orelse throw({expected_type, string}), + _ = byte_size(Hex) =:= 64 orelse throw({expected_length, 64}), + {sha256, binary:decode_hex(Hex)} + end; +converter(unicode_string) -> + fun + (undefined, #{}) -> + undefined; + (Str, #{make_serializable := true}) -> + _ = is_list(Str) orelse throw({expected_type, string}), + unicode:characters_to_binary(Str); + (Str, #{}) -> + _ = is_binary(Str) orelse throw({expected_type, string}), + unicode:characters_to_list(Str) + end. + +ref(Ref) -> + ref(?MODULE, Ref). + +translate(Conf) -> + [Root] = roots(), + maps:get( + Root, + hocon_tconf:check_plain( + ?MODULE, #{atom_to_binary(Root) => Conf}, #{atom_key => true}, [Root] + ) + ). diff --git a/apps/emqx_ft/src/emqx_ft_storage.erl b/apps/emqx_ft/src/emqx_ft_storage.erl new file mode 100644 index 000000000..4e1060d88 --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_storage.erl @@ -0,0 +1,195 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_storage). + +-export( + [ + store_filemeta/2, + store_segment/2, + assemble/2, + + files/0, + files/1, + + with_storage_type/2, + with_storage_type/3, + + backend/0, + on_config_update/2 + ] +). + +-type type() :: local. +-type backend() :: {type(), storage()}. +-type storage() :: config(). +-type config() :: emqx_config:config(). + +-export_type([backend/0]). + +-export_type([assemble_callback/0]). + +-export_type([query/1]). +-export_type([page/2]). +-export_type([file_info/0]). +-export_type([export_data/0]). +-export_type([reader/0]). + +-type assemble_callback() :: fun((ok | {error, term()}) -> any()). + +-type query(Cursor) :: + #{transfer => emqx_ft:transfer()} + | #{ + limit => non_neg_integer(), + following => Cursor + }. + +-type page(Item, Cursor) :: #{ + items := [Item], + cursor => Cursor +}. + +-type file_info() :: #{ + transfer := emqx_ft:transfer(), + name := file:name(), + size := _Bytes :: non_neg_integer(), + timestamp := emqx_datetime:epoch_second(), + uri => uri_string:uri_string(), + meta => emqx_ft:filemeta() +}. + +-type export_data() :: binary() | qlc:query_handle(). +-type reader() :: pid(). + +%%-------------------------------------------------------------------- +%% Behaviour +%%-------------------------------------------------------------------- + +%% NOTE +%% An async task will wait for a `kickoff` message to start processing, to give some time +%% to set up monitors, etc. Async task will not explicitly report the processing result, +%% you are expected to receive and handle exit reason of the process, which is +%% -type result() :: `{shutdown, ok | {error, _}}`. + +-callback store_filemeta(storage(), emqx_ft:transfer(), emqx_ft:filemeta()) -> + ok | {async, pid()} | {error, term()}. +-callback store_segment(storage(), emqx_ft:transfer(), emqx_ft:segment()) -> + ok | {async, pid()} | {error, term()}. +-callback assemble(storage(), emqx_ft:transfer(), _Size :: emqx_ft:bytes()) -> + ok | {async, pid()} | {error, term()}. + +-callback files(storage(), query(Cursor)) -> + {ok, page(file_info(), Cursor)} | {error, term()}. + +-callback start(emqx_config:config()) -> any(). +-callback stop(emqx_config:config()) -> any(). + +-callback on_config_update(_OldConfig :: emqx_config:config(), _NewConfig :: emqx_config:config()) -> + any(). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- + +-spec store_filemeta(emqx_ft:transfer(), emqx_ft:filemeta()) -> + ok | {async, pid()} | {error, term()}. +store_filemeta(Transfer, FileMeta) -> + dispatch(store_filemeta, [Transfer, FileMeta]). + +-spec store_segment(emqx_ft:transfer(), emqx_ft:segment()) -> + ok | {async, pid()} | {error, term()}. +store_segment(Transfer, Segment) -> + dispatch(store_segment, [Transfer, Segment]). + +-spec assemble(emqx_ft:transfer(), emqx_ft:bytes()) -> + ok | {async, pid()} | {error, term()}. +assemble(Transfer, Size) -> + dispatch(assemble, [Transfer, Size]). + +-spec files() -> + {ok, page(file_info(), _)} | {error, term()}. +files() -> + files(#{}). + +-spec files(query(Cursor)) -> + {ok, page(file_info(), Cursor)} | {error, term()}. +files(Query) -> + dispatch(files, [Query]). + +-spec dispatch(atom(), list(term())) -> any(). +dispatch(Fun, Args) when is_atom(Fun) -> + {Type, Storage} = backend(), + apply(mod(Type), Fun, [Storage | Args]). + +%% + +-spec with_storage_type(atom(), atom() | function()) -> any(). +with_storage_type(Type, Fun) -> + with_storage_type(Type, Fun, []). + +-spec with_storage_type(atom(), atom() | function(), list(term())) -> any(). +with_storage_type(Type, Fun, Args) -> + case backend() of + {Type, Storage} when is_atom(Fun) -> + apply(mod(Type), Fun, [Storage | Args]); + {Type, Storage} when is_function(Fun) -> + apply(Fun, [Storage | Args]); + {_, _} = Backend -> + {error, {invalid_storage_backend, Backend}} + end. + +%% + +-spec backend() -> backend(). +backend() -> + backend(emqx_ft_conf:storage()). + +-spec on_config_update(_Old :: emqx_maybe:t(config()), _New :: emqx_maybe:t(config())) -> + ok. +on_config_update(ConfigOld, ConfigNew) -> + on_backend_update( + emqx_maybe:apply(fun backend/1, ConfigOld), + emqx_maybe:apply(fun backend/1, ConfigNew) + ). + +on_backend_update({Type, _} = Backend, {Type, _} = Backend) -> + ok; +on_backend_update({Type, StorageOld}, {Type, StorageNew}) -> + ok = (mod(Type)):on_config_update(StorageOld, StorageNew); +on_backend_update(BackendOld, BackendNew) when + (BackendOld =:= undefined orelse is_tuple(BackendOld)) andalso + (BackendNew =:= undefined orelse is_tuple(BackendNew)) +-> + _ = emqx_maybe:apply(fun on_storage_stop/1, BackendOld), + _ = emqx_maybe:apply(fun on_storage_start/1, BackendNew), + ok. + +%%-------------------------------------------------------------------- +%% Local API +%%-------------------------------------------------------------------- + +-spec backend(config()) -> backend(). +backend(#{local := Storage}) -> + {local, Storage}. + +on_storage_start({Type, Storage}) -> + (mod(Type)):start(Storage). + +on_storage_stop({Type, Storage}) -> + (mod(Type)):stop(Storage). + +mod(local) -> + emqx_ft_storage_fs. diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl new file mode 100644 index 000000000..591173615 --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl @@ -0,0 +1,195 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% Filesystem storage exporter +%% +%% This is conceptually a part of the Filesystem storage backend that defines +%% how and where complete transfers are assembled into files and stored. + +-module(emqx_ft_storage_exporter). + +%% Export API +-export([start_export/3]). +-export([write/2]). +-export([complete/1]). +-export([discard/1]). + +%% Listing API +-export([list/2]). + +%% Lifecycle API +-export([on_config_update/2]). + +%% Internal API +-export([exporter/1]). + +-export_type([export/0]). + +-type storage() :: emxt_ft_storage_fs:storage(). +-type transfer() :: emqx_ft:transfer(). +-type filemeta() :: emqx_ft:filemeta(). +-type checksum() :: emqx_ft:checksum(). + +-type exporter_conf() :: map(). +-type export_st() :: term(). +-type hash_state() :: term(). +-opaque export() :: #{ + mod := module(), + st := export_st(), + hash := hash_state(), + filemeta := filemeta() +}. + +%%------------------------------------------------------------------------------ +%% Behaviour +%%------------------------------------------------------------------------------ + +-callback start_export(exporter_conf(), transfer(), filemeta()) -> + {ok, export_st()} | {error, _Reason}. + +%% Exprter must discard the export itself in case of error +-callback write(ExportSt :: export_st(), iodata()) -> + {ok, ExportSt :: export_st()} | {error, _Reason}. + +-callback complete(_ExportSt :: export_st(), _Checksum :: checksum()) -> + ok | {error, _Reason}. + +-callback discard(ExportSt :: export_st()) -> + ok | {error, _Reason}. + +-callback list(exporter_conf(), emqx_ft_storage:query(Cursor)) -> + {ok, emqx_ft_storage:page(emqx_ft_storage:file_info(), Cursor)} | {error, _Reason}. + +%% Lifecycle callbacks + +-callback start(exporter_conf()) -> + ok | {error, _Reason}. + +-callback stop(exporter_conf()) -> + ok. + +-callback update(exporter_conf(), exporter_conf()) -> + ok | {error, _Reason}. + +%%------------------------------------------------------------------------------ +%% API +%%------------------------------------------------------------------------------ + +-spec start_export(storage(), transfer(), filemeta()) -> + {ok, export()} | {error, _Reason}. +start_export(Storage, Transfer, Filemeta) -> + {ExporterMod, ExporterConf} = exporter(Storage), + case ExporterMod:start_export(ExporterConf, Transfer, Filemeta) of + {ok, ExportSt} -> + {ok, #{ + mod => ExporterMod, + st => ExportSt, + hash => init_checksum(Filemeta), + filemeta => Filemeta + }}; + {error, _} = Error -> + Error + end. + +-spec write(export(), iodata()) -> + {ok, export()} | {error, _Reason}. +write(#{mod := ExporterMod, st := ExportSt, hash := Hash} = Export, Content) -> + case ExporterMod:write(ExportSt, Content) of + {ok, ExportStNext} -> + {ok, Export#{ + st := ExportStNext, + hash := update_checksum(Hash, Content) + }}; + {error, _} = Error -> + Error + end. + +-spec complete(export()) -> + ok | {error, _Reason}. +complete(#{mod := ExporterMod, st := ExportSt, hash := Hash, filemeta := Filemeta}) -> + case verify_checksum(Hash, Filemeta) of + {ok, Checksum} -> + ExporterMod:complete(ExportSt, Checksum); + {error, _} = Error -> + _ = ExporterMod:discard(ExportSt), + Error + end. + +-spec discard(export()) -> + ok | {error, _Reason}. +discard(#{mod := ExporterMod, st := ExportSt}) -> + ExporterMod:discard(ExportSt). + +-spec list(storage(), emqx_ft_storage:query(Cursor)) -> + {ok, emqx_ft_storage:page(emqx_ft_storage:file_info(), Cursor)} | {error, _Reason}. +list(Storage, Query) -> + {ExporterMod, ExporterOpts} = exporter(Storage), + ExporterMod:list(ExporterOpts, Query). + +%% Lifecycle + +-spec on_config_update(storage(), storage()) -> ok | {error, term()}. +on_config_update(StorageOld, StorageNew) -> + on_exporter_update( + emqx_maybe:apply(fun exporter/1, StorageOld), + emqx_maybe:apply(fun exporter/1, StorageNew) + ). + +on_exporter_update(Config, Config) -> + ok; +on_exporter_update({ExporterMod, ConfigOld}, {ExporterMod, ConfigNew}) -> + ExporterMod:update(ConfigOld, ConfigNew); +on_exporter_update(ExporterOld, ExporterNew) -> + _ = emqx_maybe:apply(fun stop/1, ExporterOld), + _ = emqx_maybe:apply(fun start/1, ExporterNew), + ok. + +start({ExporterMod, ExporterOpts}) -> + ok = ExporterMod:start(ExporterOpts). + +stop({ExporterMod, ExporterOpts}) -> + ok = ExporterMod:stop(ExporterOpts). + +%%------------------------------------------------------------------------------ +%% Internal functions +%%------------------------------------------------------------------------------ + +exporter(Storage) -> + case maps:get(exporter, Storage) of + #{local := Options} -> + {emqx_ft_storage_exporter_fs, Options}; + #{s3 := Options} -> + {emqx_ft_storage_exporter_s3, Options} + end. + +init_checksum(#{checksum := {Algo, _}}) -> + crypto:hash_init(Algo); +init_checksum(#{}) -> + crypto:hash_init(sha256). + +update_checksum(Ctx, IoData) -> + crypto:hash_update(Ctx, IoData). + +verify_checksum(Ctx, #{checksum := {Algo, Digest} = Checksum}) -> + case crypto:hash_final(Ctx) of + Digest -> + {ok, Checksum}; + Mismatch -> + {error, {checksum, Algo, binary:encode_hex(Mismatch)}} + end; +verify_checksum(Ctx, #{}) -> + Digest = crypto:hash_final(Ctx), + {ok, {sha256, Digest}}. diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl new file mode 100644 index 000000000..6738d6fef --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl @@ -0,0 +1,489 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_storage_exporter_fs). + +-include_lib("kernel/include/file.hrl"). +-include_lib("emqx/include/logger.hrl"). + +%% Exporter API +-behaviour(emqx_ft_storage_exporter). + +-export([start_export/3]). +-export([write/2]). +-export([complete/2]). +-export([discard/1]). +-export([list/1]). + +-export([ + start/1, + stop/1, + update/2 +]). + +%% Internal API for RPC +-export([list_local/1]). +-export([list_local/2]). +-export([list_local_transfer/2]). +-export([start_reader/3]). + +-export([list/2]). + +-export_type([export_st/0]). +-export_type([options/0]). + +-type options() :: #{ + root => file:name(), + _ => _ +}. + +-type query() :: emqx_ft_storage:query(cursor()). +-type page(T) :: emqx_ft_storage:page(T, cursor()). +-type cursor() :: iodata(). + +-type transfer() :: emqx_ft:transfer(). +-type filemeta() :: emqx_ft:filemeta(). +-type exportinfo() :: emqx_ft_storage:file_info(). +-type file_error() :: emqx_ft_storage_fs:file_error(). + +-type export_st() :: #{ + path := file:name(), + handle := io:device(), + result := file:name(), + meta := filemeta() +}. + +-type reader() :: pid(). + +-define(TEMPDIR, "tmp"). +-define(MANIFEST, ".MANIFEST.json"). + +%% NOTE +%% Bucketing of resulting files to accomodate the storage backend for considerably +%% large (e.g. > 10s of millions) amount of files. +-define(BUCKET_HASH, sha). + +%% 2 symbols = at most 256 directories on the upper level +-define(BUCKET1_LEN, 2). +%% 2 symbols = at most 256 directories on the second level +-define(BUCKET2_LEN, 2). + +%%-------------------------------------------------------------------- +%% Exporter behaviour +%%-------------------------------------------------------------------- + +-spec start_export(options(), transfer(), filemeta()) -> + {ok, export_st()} | {error, file_error()}. +start_export(Options, Transfer, Filemeta = #{name := Filename}) -> + TempFilepath = mk_temp_absfilepath(Options, Transfer, Filename), + ResultFilepath = mk_absfilepath(Options, Transfer, result, Filename), + _ = filelib:ensure_dir(TempFilepath), + case file:open(TempFilepath, [write, raw, binary]) of + {ok, Handle} -> + {ok, #{ + path => TempFilepath, + handle => Handle, + result => ResultFilepath, + meta => Filemeta + }}; + {error, _} = Error -> + Error + end. + +-spec write(export_st(), iodata()) -> + {ok, export_st()} | {error, file_error()}. +write(ExportSt = #{handle := Handle}, IoData) -> + case file:write(Handle, IoData) of + ok -> + {ok, ExportSt}; + {error, _} = Error -> + _ = discard(ExportSt), + Error + end. + +-spec complete(export_st(), emqx_ft:checksum()) -> + ok | {error, {checksum, _Algo, _Computed}} | {error, file_error()}. +complete( + #{ + path := Filepath, + handle := Handle, + result := ResultFilepath, + meta := FilemetaIn + }, + Checksum +) -> + Filemeta = FilemetaIn#{checksum => Checksum}, + ok = file:close(Handle), + _ = filelib:ensure_dir(ResultFilepath), + _ = file:write_file(mk_manifest_filename(ResultFilepath), encode_filemeta(Filemeta)), + file:rename(Filepath, ResultFilepath). + +-spec discard(export_st()) -> + ok. +discard(#{path := Filepath, handle := Handle}) -> + ok = file:close(Handle), + file:delete(Filepath). + +%%-------------------------------------------------------------------- +%% Exporter behaviour (lifecycle) +%%-------------------------------------------------------------------- + +%% FS Exporter does not have require any stateful entities, +%% so lifecycle callbacks are no-op. + +-spec start(options()) -> ok. +start(_Options) -> ok. + +-spec stop(options()) -> ok. +stop(_Options) -> ok. + +-spec update(options(), options()) -> ok. +update(_OldOptions, _NewOptions) -> ok. + +%%-------------------------------------------------------------------- +%% Internal API +%%-------------------------------------------------------------------- + +-type local_query() :: emqx_ft_storage:query({transfer(), file:name()}). + +-spec list_local_transfer(options(), transfer()) -> + {ok, [exportinfo()]} | {error, file_error()}. +list_local_transfer(Options, Transfer) -> + It = emqx_ft_fs_iterator:new( + mk_absdir(Options, Transfer, result), + [fun filter_manifest/1] + ), + Result = emqx_ft_fs_iterator:fold( + fun + ({leaf, _Path, Fileinfo = #file_info{type = regular}, [Filename | _]}, Acc) -> + RelFilepath = filename:join(mk_result_reldir(Transfer) ++ [Filename]), + Info = mk_exportinfo(Options, Filename, RelFilepath, Transfer, Fileinfo), + [Info | Acc]; + ({node, _Path, {error, Reason}, []}, []) -> + {error, Reason}; + (Entry, Acc) -> + ok = log_invalid_entry(Options, Entry), + Acc + end, + [], + It + ), + case Result of + Infos = [_ | _] -> + {ok, lists:reverse(Infos)}; + [] -> + {error, enoent}; + {error, Reason} -> + {error, Reason} + end. + +-spec list_local(options()) -> + {ok, [exportinfo()]} | {error, file_error()}. +list_local(Options) -> + list_local(Options, #{}). + +-spec list_local(options(), local_query()) -> + {ok, [exportinfo()]} | {error, file_error()}. +list_local(Options, #{transfer := Transfer}) -> + list_local_transfer(Options, Transfer); +list_local(Options, #{} = Query) -> + Root = get_storage_root(Options), + Glob = [ + _Bucket1 = '*', + _Bucket2 = '*', + _Rest = '*', + _ClientId = '*', + _FileId = '*', + fun filter_manifest/1 + ], + It = + case Query of + #{following := Cursor} -> + emqx_ft_fs_iterator:seek(mk_path_seek(Cursor), Root, Glob); + #{} -> + emqx_ft_fs_iterator:new(Root, Glob) + end, + % NOTE + % In the rare case when some transfer contain more than one file, the paging mechanic + % here may skip over some files, when the cursor is transfer-only. + Limit = maps:get(limit, Query, -1), + {Exports, _} = emqx_ft_fs_iterator:fold_n( + fun(Entry, Acc) -> read_exportinfo(Options, Entry, Acc) end, + [], + It, + Limit + ), + {ok, Exports}. + +mk_path_seek(#{transfer := Transfer, name := Filename}) -> + mk_result_reldir(Transfer) ++ [Filename]; +mk_path_seek(#{transfer := Transfer}) -> + % NOTE: Any bitstring is greater than any list. + mk_result_reldir(Transfer) ++ [<<>>]. + +%%-------------------------------------------------------------------- +%% Helpers +%%-------------------------------------------------------------------- + +filter_manifest(?MANIFEST) -> + % Filename equals `?MANIFEST`, there should also be a manifest for it. + false; +filter_manifest(Filename) -> + ?MANIFEST =/= string:find(Filename, ?MANIFEST, trailing). + +read_exportinfo( + Options, + {leaf, RelFilepath, Fileinfo = #file_info{type = regular}, [Filename, FileId, ClientId | _]}, + Acc +) -> + % NOTE + % There might be more than one file for a single transfer (though + % extremely bad luck is needed for that, e.g. concurrent assemblers with + % different filemetas from different nodes). This might be unexpected for a + % client given the current protocol, yet might be helpful in the future. + Transfer = dirnames_to_transfer(ClientId, FileId), + Info = mk_exportinfo(Options, Filename, RelFilepath, Transfer, Fileinfo), + [Info | Acc]; +read_exportinfo(_Options, {node, _Root = "", {error, enoent}, []}, Acc) -> + % NOTE: Root directory does not exist, this is not an error. + Acc; +read_exportinfo(Options, Entry, Acc) -> + ok = log_invalid_entry(Options, Entry), + Acc. + +mk_exportinfo(Options, Filename, RelFilepath, Transfer, Fileinfo) -> + Root = get_storage_root(Options), + try_read_filemeta( + filename:join(Root, mk_manifest_filename(RelFilepath)), + #{ + transfer => Transfer, + name => Filename, + uri => mk_export_uri(RelFilepath), + timestamp => Fileinfo#file_info.mtime, + size => Fileinfo#file_info.size, + path => filename:join(Root, RelFilepath) + } + ). + +try_read_filemeta(Filepath, Info) -> + case emqx_ft_fs_util:read_decode_file(Filepath, fun decode_filemeta/1) of + {ok, Filemeta} -> + Info#{meta => Filemeta}; + {error, Reason} -> + ?SLOG(warning, "filemeta_inaccessible", #{ + path => Filepath, + reason => Reason + }), + Info + end. + +mk_export_uri(RelFilepath) -> + emqx_ft_storage_exporter_fs_api:mk_export_uri(node(), RelFilepath). + +log_invalid_entry(Options, {_Type, RelFilepath, Fileinfo = #file_info{}, _Stack}) -> + ?SLOG(notice, "filesystem_object_unexpected", #{ + relpath => RelFilepath, + fileinfo => Fileinfo, + options => Options + }); +log_invalid_entry(Options, {_Type, RelFilepath, {error, Reason}, _Stack}) -> + ?SLOG(warning, "filesystem_object_inaccessible", #{ + relpath => RelFilepath, + reason => Reason, + options => Options + }). + +-spec start_reader(options(), file:name(), _Caller :: pid()) -> + {ok, reader()} | {error, enoent}. +start_reader(Options, RelFilepath, CallerPid) -> + Root = get_storage_root(Options), + case filelib:safe_relative_path(RelFilepath, Root) of + SafeFilepath when SafeFilepath /= unsafe -> + AbsFilepath = filename:join(Root, SafeFilepath), + emqx_ft_storage_fs_reader:start_supervised(CallerPid, AbsFilepath); + unsafe -> + {error, enoent} + end. + +%% + +-spec list(options(), query()) -> + {ok, page(exportinfo())} | {error, [{node(), _Reason}]}. +list(_Options, Query = #{transfer := _Transfer}) -> + case list(Query) of + #{items := Exports = [_ | _]} -> + {ok, #{items => Exports}}; + #{items := [], errors := NodeErrors} -> + {error, NodeErrors}; + #{items := []} -> + {ok, #{items => []}} + end; +list(_Options, Query) -> + Result = list(Query), + case Result of + #{errors := NodeErrors} -> + ?SLOG(warning, "list_exports_errors", #{ + query => Query, + errors => NodeErrors + }); + #{} -> + ok + end, + case Result of + #{items := Exports, cursor := Cursor} -> + {ok, #{items => lists:reverse(Exports), cursor => encode_cursor(Cursor)}}; + #{items := Exports} -> + {ok, #{items => lists:reverse(Exports)}} + end. + +list(QueryIn) -> + {Nodes, NodeQuery} = decode_query(QueryIn, lists:sort(mria_mnesia:running_nodes())), + list_nodes(NodeQuery, Nodes, #{items => []}). + +list_nodes(Query, Nodes = [Node | Rest], Acc) -> + case emqx_ft_storage_exporter_fs_proto_v1:list_exports([Node], Query) of + [{ok, Result}] -> + list_accumulate(Result, Query, Nodes, Acc); + [Failure] -> + ?SLOG(warning, #{ + msg => "list_remote_exports_failed", + node => Node, + query => Query, + failure => Failure + }), + list_next(Query, Rest, Acc) + end; +list_nodes(_Query, [], Acc) -> + Acc. + +list_accumulate({ok, Exports}, Query, [Node | Rest], Acc = #{items := EAcc}) -> + NExports = length(Exports), + AccNext = Acc#{items := Exports ++ EAcc}, + case Query of + #{limit := Limit} when NExports < Limit -> + list_next(Query#{limit => Limit - NExports}, Rest, AccNext); + #{limit := _} -> + AccNext#{cursor => mk_cursor(Node, Exports)}; + #{} -> + list_next(Query, Rest, AccNext) + end; +list_accumulate({error, Reason}, Query, [Node | Rest], Acc) -> + EAcc = maps:get(errors, Acc, []), + list_next(Query, Rest, Acc#{errors => [{Node, Reason} | EAcc]}). + +list_next(Query, Nodes, Acc) -> + list_nodes(maps:remove(following, Query), Nodes, Acc). + +decode_query(Query = #{following := Cursor}, Nodes) -> + {Node, NodeCursor} = decode_cursor(Cursor), + {skip_query_nodes(Node, Nodes), Query#{following => NodeCursor}}; +decode_query(Query = #{}, Nodes) -> + {Nodes, Query}. + +skip_query_nodes(CNode, Nodes) -> + lists:dropwhile(fun(N) -> N < CNode end, Nodes). + +mk_cursor(Node, [_Last = #{transfer := Transfer, name := Name} | _]) -> + {Node, #{transfer => Transfer, name => Name}}. + +encode_cursor({Node, #{transfer := {ClientId, FileId}, name := Name}}) -> + emqx_utils_json:encode(#{ + <<"n">> => Node, + <<"cid">> => ClientId, + <<"fid">> => FileId, + <<"fn">> => unicode:characters_to_binary(Name) + }). + +decode_cursor(Cursor) -> + try + #{ + <<"n">> := NodeIn, + <<"cid">> := ClientId, + <<"fid">> := FileId, + <<"fn">> := NameIn + } = emqx_utils_json:decode(Cursor), + true = is_binary(ClientId), + true = is_binary(FileId), + Node = binary_to_existing_atom(NodeIn), + Name = unicode:characters_to_list(NameIn), + true = is_list(Name), + {Node, #{transfer => {ClientId, FileId}, name => Name}} + catch + error:{_, invalid_json} -> + error({badarg, cursor}); + error:{badmatch, _} -> + error({badarg, cursor}); + error:badarg -> + error({badarg, cursor}) + end. + +%% + +-define(PRELUDE(Vsn, Meta), [<<"filemeta">>, Vsn, Meta]). + +encode_filemeta(Meta) -> + emqx_utils_json:encode(?PRELUDE(_Vsn = 1, emqx_ft:encode_filemeta(Meta))). + +decode_filemeta(Binary) when is_binary(Binary) -> + ?PRELUDE(_Vsn = 1, Map) = emqx_utils_json:decode(Binary, [return_maps]), + case emqx_ft:decode_filemeta(Map) of + {ok, Meta} -> + Meta; + {error, Reason} -> + error(Reason) + end. + +mk_manifest_filename(Filename) when is_list(Filename) -> + Filename ++ ?MANIFEST; +mk_manifest_filename(Filename) when is_binary(Filename) -> + <>. + +mk_temp_absfilepath(Options, Transfer, Filename) -> + Unique = erlang:unique_integer([positive]), + TempFilename = integer_to_list(Unique) ++ "." ++ Filename, + filename:join(mk_absdir(Options, Transfer, temporary), TempFilename). + +mk_absdir(Options, _Transfer, temporary) -> + filename:join([get_storage_root(Options), ?TEMPDIR]); +mk_absdir(Options, Transfer, result) -> + filename:join([get_storage_root(Options) | mk_result_reldir(Transfer)]). + +mk_absfilepath(Options, Transfer, What, Filename) -> + filename:join(mk_absdir(Options, Transfer, What), Filename). + +mk_result_reldir(Transfer = {ClientId, FileId}) -> + Hash = mk_transfer_hash(Transfer), + << + Bucket1:?BUCKET1_LEN/binary, + Bucket2:?BUCKET2_LEN/binary, + BucketRest/binary + >> = binary:encode_hex(Hash), + [ + binary_to_list(Bucket1), + binary_to_list(Bucket2), + binary_to_list(BucketRest), + emqx_ft_fs_util:escape_filename(ClientId), + emqx_ft_fs_util:escape_filename(FileId) + ]. + +dirnames_to_transfer(ClientId, FileId) -> + {emqx_ft_fs_util:unescape_filename(ClientId), emqx_ft_fs_util:unescape_filename(FileId)}. + +mk_transfer_hash(Transfer) -> + crypto:hash(?BUCKET_HASH, term_to_binary(Transfer)). + +get_storage_root(Options) -> + maps:get(root, Options, filename:join([emqx:data_dir(), file_transfer, exports])). diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_api.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_api.erl new file mode 100644 index 000000000..abb774f82 --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_api.erl @@ -0,0 +1,182 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_storage_exporter_fs_api). + +-behaviour(minirest_api). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("emqx/include/logger.hrl"). + +%% Swagger specs from hocon schema +-export([ + api_spec/0, + paths/0, + schema/1, + namespace/0 +]). + +-export([ + fields/1, + roots/0 +]). + +%% API callbacks +-export([ + '/file_transfer/file'/2 +]). + +-export([mk_export_uri/2]). + +%% + +namespace() -> "file_transfer". + +api_spec() -> + emqx_dashboard_swagger:spec(?MODULE, #{ + check_schema => true, filter => fun emqx_ft_api:check_ft_enabled/2 + }). + +paths() -> + [ + "/file_transfer/file" + ]. + +schema("/file_transfer/file") -> + #{ + 'operationId' => '/file_transfer/file', + get => #{ + tags => [<<"file_transfer">>], + summary => <<"Download a particular file">>, + description => ?DESC("file_get"), + parameters => [ + hoconsc:ref(file_node), + hoconsc:ref(file_ref) + ], + responses => #{ + 200 => <<"Operation success">>, + 404 => emqx_dashboard_swagger:error_codes(['NOT_FOUND'], <<"Not found">>), + 503 => emqx_dashboard_swagger:error_codes( + ['SERVICE_UNAVAILABLE'], <<"Service unavailable">> + ) + } + } + }. + +roots() -> + [ + file_node, + file_ref + ]. + +-spec fields(hocon_schema:name()) -> hocon_schema:fields(). +fields(file_ref) -> + [ + {fileref, + hoconsc:mk(binary(), #{ + in => query, + desc => <<"File reference">>, + example => <<"file1">>, + required => true + })} + ]; +fields(file_node) -> + [ + {node, + hoconsc:mk(binary(), #{ + in => query, + desc => <<"Node under which the file is located">>, + example => atom_to_list(node()), + required => true + })} + ]. + +'/file_transfer/file'(get, #{query_string := Query}) -> + try + Node = parse_node(maps:get(<<"node">>, Query)), + Filepath = parse_filepath(maps:get(<<"fileref">>, Query)), + case emqx_ft_storage_exporter_fs_proto_v1:read_export_file(Node, Filepath, self()) of + {ok, ReaderPid} -> + FileData = emqx_ft_storage_fs_reader:table(ReaderPid), + {200, + #{ + <<"content-type">> => <<"application/data">>, + <<"content-disposition">> => <<"attachment">> + }, + FileData}; + {error, enoent} -> + {404, error_msg('NOT_FOUND', <<"Not found">>)}; + {error, Error} -> + ?SLOG(warning, #{msg => "get_ready_transfer_fail", error => Error}), + {503, error_msg('SERVICE_UNAVAILABLE', <<"Service unavailable">>)} + end + catch + throw:{invalid, Param} -> + {404, + error_msg( + 'NOT_FOUND', + iolist_to_binary(["Invalid query parameter: ", Param]) + )}; + error:{erpc, noconnection} -> + {503, error_msg('SERVICE_UNAVAILABLE', <<"Service unavailable">>)} + end. + +error_msg(Code, Msg) -> + #{code => Code, message => emqx_utils:readable_error_msg(Msg)}. + +-spec mk_export_uri(node(), file:name()) -> + uri_string:uri_string(). +mk_export_uri(Node, Filepath) -> + emqx_dashboard_swagger:relative_uri([ + "/file_transfer/file?", + uri_string:compose_query([ + {"node", atom_to_list(Node)}, + {"fileref", Filepath} + ]) + ]). + +%% + +parse_node(NodeBin) -> + case emqx_utils:safe_to_existing_atom(NodeBin) of + {ok, Node} -> + Node; + {error, _} -> + throw({invalid, NodeBin}) + end. + +parse_filepath(PathBin) -> + case filename:pathtype(PathBin) of + relative -> + ok; + absolute -> + throw({invalid, PathBin}) + end, + PathComponents = filename:split(PathBin), + case lists:any(fun is_special_component/1, PathComponents) of + false -> + filename:join(PathComponents); + true -> + throw({invalid, PathBin}) + end. + +is_special_component(<<".", _/binary>>) -> + true; +is_special_component([$. | _]) -> + true; +is_special_component(_) -> + false. diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_proxy.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_proxy.erl new file mode 100644 index 000000000..50e02db6f --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs_proxy.erl @@ -0,0 +1,50 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% This methods are called via rpc by `emqx_ft_storage_exporter_fs` +%% They populate the call with actual storage which may be configured differently +%% on a concrete node. + +-module(emqx_ft_storage_exporter_fs_proxy). + +-export([ + list_exports_local/1, + read_export_file_local/2 +]). + +list_exports_local(Query) -> + emqx_ft_storage:with_storage_type(local, fun(Storage) -> + case emqx_ft_storage_exporter:exporter(Storage) of + {emqx_ft_storage_exporter_fs, Options} -> + emqx_ft_storage_exporter_fs:list_local(Options, Query) + % NOTE + % This case clause is currently deemed unreachable by dialyzer. + % InvalidExporter -> + % {error, {invalid_exporter, InvalidExporter}} + end + end). + +read_export_file_local(Filepath, CallerPid) -> + emqx_ft_storage:with_storage_type(local, fun(Storage) -> + case emqx_ft_storage_exporter:exporter(Storage) of + {emqx_ft_storage_exporter_fs, Options} -> + emqx_ft_storage_exporter_fs:start_reader(Options, Filepath, CallerPid) + % NOTE + % This case clause is currently deemed unreachable by dialyzer. + % InvalidExporter -> + % {error, {invalid_exporter, InvalidExporter}} + end + end). diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl new file mode 100644 index 000000000..b9f07d5c0 --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl @@ -0,0 +1,251 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_storage_exporter_s3). + +-include_lib("emqx/include/logger.hrl"). + +%% Exporter API +-export([start_export/3]). +-export([write/2]). +-export([complete/2]). +-export([discard/1]). +-export([list/2]). + +-export([ + start/1, + stop/1, + update/2 +]). + +-type options() :: emqx_s3:profile_config(). +-type transfer() :: emqx_ft:transfer(). +-type filemeta() :: emqx_ft:filemeta(). +-type exportinfo() :: #{ + transfer := transfer(), + name := file:name(), + uri := uri_string:uri_string(), + timestamp := emqx_datetime:epoch_second(), + size := _Bytes :: non_neg_integer(), + filemeta => filemeta() +}. + +-type query() :: emqx_ft_storage:query(cursor()). +-type page(T) :: emqx_ft_storage:page(T, cursor()). +-type cursor() :: iodata(). + +-type export_st() :: #{ + pid := pid(), + filemeta := filemeta(), + transfer := transfer() +}. + +-define(S3_PROFILE_ID, ?MODULE). +-define(FILEMETA_VSN, <<"1">>). +-define(S3_LIST_LIMIT, 500). + +%%-------------------------------------------------------------------- +%% Exporter behaviour +%%-------------------------------------------------------------------- + +-spec start_export(options(), transfer(), filemeta()) -> + {ok, export_st()} | {error, term()}. +start_export(_Options, Transfer, Filemeta) -> + Options = #{ + key => s3_key(Transfer, Filemeta), + headers => s3_headers(Transfer, Filemeta) + }, + case emqx_s3:start_uploader(?S3_PROFILE_ID, Options) of + {ok, Pid} -> + true = erlang:link(Pid), + {ok, #{filemeta => Filemeta, pid => Pid}}; + {error, _Reason} = Error -> + Error + end. + +-spec write(export_st(), iodata()) -> + {ok, export_st()} | {error, term()}. +write(#{pid := Pid} = ExportSt, IoData) -> + case emqx_s3_uploader:write(Pid, IoData) of + ok -> + {ok, ExportSt}; + {error, _Reason} = Error -> + Error + end. + +-spec complete(export_st(), emqx_ft:checksum()) -> + ok | {error, term()}. +complete(#{pid := Pid} = _ExportSt, _Checksum) -> + emqx_s3_uploader:complete(Pid). + +-spec discard(export_st()) -> + ok. +discard(#{pid := Pid} = _ExportSt) -> + emqx_s3_uploader:abort(Pid). + +-spec list(options(), query()) -> + {ok, page(exportinfo())} | {error, term()}. +list(Options, Query) -> + emqx_s3:with_client(?S3_PROFILE_ID, fun(Client) -> list(Client, Options, Query) end). + +%%-------------------------------------------------------------------- +%% Exporter behaviour (lifecycle) +%%-------------------------------------------------------------------- + +-spec start(options()) -> ok | {error, term()}. +start(Options) -> + emqx_s3:start_profile(?S3_PROFILE_ID, Options). + +-spec stop(options()) -> ok. +stop(_Options) -> + ok = emqx_s3:stop_profile(?S3_PROFILE_ID). + +-spec update(options(), options()) -> ok. +update(_OldOptions, NewOptions) -> + emqx_s3:update_profile(?S3_PROFILE_ID, NewOptions). + +%%-------------------------------------------------------------------- +%% Internal functions +%% ------------------------------------------------------------------- + +s3_key(Transfer, #{name := Filename}) -> + s3_prefix(Transfer) ++ "/" ++ Filename. + +s3_prefix({ClientId, FileId} = _Transfer) -> + emqx_ft_fs_util:escape_filename(ClientId) ++ "/" ++ emqx_ft_fs_util:escape_filename(FileId). + +s3_headers({ClientId, FileId}, Filemeta) -> + #{ + %% The ClientID MUST be a UTF-8 Encoded String + <<"x-amz-meta-clientid">> => ClientId, + %% It [Topic Name] MUST be a UTF-8 Encoded String + <<"x-amz-meta-fileid">> => FileId, + <<"x-amz-meta-filemeta">> => s3_header_filemeta(Filemeta), + <<"x-amz-meta-filemeta-vsn">> => ?FILEMETA_VSN + }. + +s3_header_filemeta(Filemeta) -> + emqx_utils_json:encode(emqx_ft:encode_filemeta(Filemeta), [force_utf8, uescape]). + +list(Client, _Options, #{transfer := Transfer}) -> + case list_key_info(Client, [{prefix, s3_prefix(Transfer)}, {max_keys, ?S3_LIST_LIMIT}]) of + {ok, {Exports, _Marker}} -> + {ok, #{items => Exports}}; + {error, _Reason} = Error -> + Error + end; +list(Client, _Options, Query) -> + Limit = maps:get(limit, Query, undefined), + Marker = emqx_maybe:apply(fun decode_cursor/1, maps:get(following, Query, undefined)), + case list_pages(Client, Marker, Limit, []) of + {ok, {Exports, undefined}} -> + {ok, #{items => Exports}}; + {ok, {Exports, NextMarker}} -> + {ok, #{items => Exports, cursor => encode_cursor(NextMarker)}}; + {error, _Reason} = Error -> + Error + end. + +list_pages(Client, Marker, Limit, Acc) -> + MaxKeys = min(?S3_LIST_LIMIT, Limit), + ListOptions = [{marker, Marker} || Marker =/= undefined], + case list_key_info(Client, [{max_keys, MaxKeys} | ListOptions]) of + {ok, {Exports, NextMarker}} -> + list_accumulate(Client, Limit, NextMarker, [Exports | Acc]); + {error, _Reason} = Error -> + Error + end. + +list_accumulate(_Client, _Limit, undefined, Acc) -> + {ok, {flatten_pages(Acc), undefined}}; +list_accumulate(Client, undefined, Marker, Acc) -> + list_pages(Client, Marker, undefined, Acc); +list_accumulate(Client, Limit, Marker, Acc = [Exports | _]) -> + case Limit - length(Exports) of + 0 -> + {ok, {flatten_pages(Acc), Marker}}; + Left -> + list_pages(Client, Marker, Left, Acc) + end. + +flatten_pages(Pages) -> + lists:append(lists:reverse(Pages)). + +list_key_info(Client, ListOptions) -> + case emqx_s3_client:list(Client, ListOptions) of + {ok, Result} -> + ?SLOG(debug, #{msg => "list_key_info", result => Result}), + KeyInfos = proplists:get_value(contents, Result, []), + Exports = lists:filtermap( + fun(KeyInfo) -> key_info_to_exportinfo(Client, KeyInfo) end, KeyInfos + ), + Marker = + case proplists:get_value(is_truncated, Result, false) of + true -> + next_marker(KeyInfos); + false -> + undefined + end, + {ok, {Exports, Marker}}; + {error, _Reason} = Error -> + Error + end. + +encode_cursor(Key) -> + unicode:characters_to_binary(Key). + +decode_cursor(Cursor) -> + case unicode:characters_to_list(Cursor) of + Key when is_list(Key) -> + Key; + _ -> + error({badarg, cursor}) + end. + +next_marker(KeyInfos) -> + proplists:get_value(key, lists:last(KeyInfos)). + +key_info_to_exportinfo(Client, KeyInfo) -> + Key = proplists:get_value(key, KeyInfo), + case parse_transfer_and_name(Key) of + {ok, {Transfer, Name}} -> + {true, #{ + transfer => Transfer, + name => unicode:characters_to_binary(Name), + uri => emqx_s3_client:uri(Client, Key), + timestamp => datetime_to_epoch_second(proplists:get_value(last_modified, KeyInfo)), + size => proplists:get_value(size, KeyInfo) + }}; + {error, _Reason} -> + false + end. + +-define(EPOCH_START, 62167219200). + +datetime_to_epoch_second(DateTime) -> + calendar:datetime_to_gregorian_seconds(DateTime) - ?EPOCH_START. + +parse_transfer_and_name(Key) -> + case string:split(Key, "/", all) of + [ClientId, FileId, Name] -> + Transfer = { + emqx_ft_fs_util:unescape_filename(ClientId), + emqx_ft_fs_util:unescape_filename(FileId) + }, + {ok, {Transfer, Name}}; + _ -> + {error, invalid_key} + end. diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl new file mode 100644 index 000000000..010d004a1 --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -0,0 +1,506 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% Filesystem storage backend +%% +%% NOTE +%% If you plan to change storage layout please consult `emqx_ft_storage_fs_gc` +%% to see how much it would break or impair GC. + +-module(emqx_ft_storage_fs). + +-behaviour(emqx_ft_storage). + +-include_lib("emqx/include/logger.hrl"). +-include_lib("snabbkaffe/include/trace.hrl"). + +-export([child_spec/1]). + +% Segments-related API +-export([store_filemeta/3]). +-export([store_segment/3]). +-export([read_filemeta/2]). +-export([list/3]). +-export([pread/5]). +-export([lookup_local_assembler/1]). +-export([assemble/3]). + +-export([transfers/1]). + +% GC API +% TODO: This is quickly becomes hairy. +-export([get_root/1]). +-export([get_subdir/2]). +-export([get_subdir/3]). + +-export([files/2]). + +-export([on_config_update/2]). +-export([start/1]). +-export([stop/1]). + +-export_type([storage/0]). +-export_type([filefrag/1]). +-export_type([filefrag/0]). +-export_type([transferinfo/0]). + +-export_type([file_error/0]). + +-type transfer() :: emqx_ft:transfer(). +-type offset() :: emqx_ft:offset(). +-type filemeta() :: emqx_ft:filemeta(). +-type segment() :: emqx_ft:segment(). + +-type segmentinfo() :: #{ + offset := offset(), + size := _Bytes :: non_neg_integer() +}. + +-type transferinfo() :: #{ + filemeta => filemeta() +}. + +% TODO naming +-type filefrag(T) :: #{ + path := file:name(), + timestamp := emqx_datetime:epoch_second(), + size := _Bytes :: non_neg_integer(), + fragment := T +}. + +-type filefrag() :: filefrag( + {filemeta, filemeta()} + | {segment, segmentinfo()} +). + +-define(FRAGDIR, frags). +-define(TEMPDIR, tmp). +-define(MANIFEST, "MANIFEST.json"). +-define(SEGMENT, "SEG"). + +-type segments() :: #{ + root := file:name(), + gc := #{ + interval := non_neg_integer(), + maximum_segments_ttl := non_neg_integer(), + minimum_segments_ttl := non_neg_integer() + } +}. + +-type storage() :: #{ + type := 'local', + segments := segments(), + exporter := emqx_ft_storage_exporter:exporter() +}. + +-type file_error() :: + file:posix() + %% Filename is incompatible with the backing filesystem. + | badarg + %% System limit (e.g. number of ports) reached. + | system_limit. + +%% Related resources childspecs +-spec child_spec(storage()) -> + [supervisor:child_spec()]. +child_spec(Storage) -> + [ + #{ + id => emqx_ft_storage_fs_gc, + start => {emqx_ft_storage_fs_gc, start_link, [Storage]}, + restart => permanent + } + ]. + +%% Store manifest in the backing filesystem. +%% Atomic operation. +-spec store_filemeta(storage(), transfer(), filemeta()) -> + % Quota? Some lower level errors? + ok | {error, conflict} | {error, file_error()}. +store_filemeta(Storage, Transfer, Meta) -> + Filepath = mk_filepath(Storage, Transfer, get_subdirs_for(fragment), ?MANIFEST), + case read_file(Filepath, fun decode_filemeta/1) of + {ok, Meta} -> + _ = touch_file(Filepath), + ok; + {ok, Conflict} -> + ?SLOG(warning, #{ + msg => "filemeta_conflict", transfer => Transfer, new => Meta, old => Conflict + }), + % TODO + % We won't see conflicts in case of concurrent `store_filemeta` + % requests. It's rather odd scenario so it's fine not to worry + % about it too much now. + {error, conflict}; + {error, Reason} when Reason =:= notfound; Reason =:= corrupted; Reason =:= enoent -> + write_file_atomic(Storage, Transfer, Filepath, encode_filemeta(Meta)); + {error, _} = Error -> + Error + end. + +%% Store a segment in the backing filesystem. +%% Atomic operation. +-spec store_segment(storage(), transfer(), segment()) -> + % Where is the checksum gets verified? Upper level probably. + % Quota? Some lower level errors? + ok | {error, file_error()}. +store_segment(Storage, Transfer, Segment = {_Offset, Content}) -> + Filename = mk_segment_filename(Segment), + Filepath = mk_filepath(Storage, Transfer, get_subdirs_for(fragment), Filename), + write_file_atomic(Storage, Transfer, Filepath, Content). + +-spec read_filemeta(storage(), transfer()) -> + {ok, filemeta()} | {error, corrupted} | {error, file_error()}. +read_filemeta(Storage, Transfer) -> + Filepath = mk_filepath(Storage, Transfer, get_subdirs_for(fragment), ?MANIFEST), + read_file(Filepath, fun decode_filemeta/1). + +-spec list(storage(), transfer(), _What :: fragment) -> + % Some lower level errors? {error, notfound}? + % Result will contain zero or only one filemeta. + {ok, [filefrag({filemeta, filemeta()} | {segment, segmentinfo()})]} + | {error, file_error()}. +list(Storage, Transfer, What = fragment) -> + Dirname = mk_filedir(Storage, Transfer, get_subdirs_for(What)), + case file:list_dir(Dirname) of + {ok, Filenames} -> + % TODO + % In case of `What = result` there might be more than one file (though + % extremely bad luck is needed for that, e.g. concurrent assemblers with + % different filemetas from different nodes). This might be unexpected for a + % client given the current protocol, yet might be helpful in the future. + {ok, filtermap_files(fun mk_filefrag/2, Dirname, Filenames)}; + {error, enoent} -> + {ok, []}; + {error, _} = Error -> + Error + end. + +-spec pread(storage(), transfer(), filefrag(), offset(), _Size :: non_neg_integer()) -> + {ok, _Content :: iodata()} | {error, eof} | {error, file_error()}. +pread(_Storage, _Transfer, Frag, Offset, Size) -> + Filepath = maps:get(path, Frag), + case file:open(Filepath, [read, raw, binary]) of + {ok, IoDevice} -> + % NOTE + % Reading empty file is always `eof`. + Read = file:pread(IoDevice, Offset, Size), + ok = file:close(IoDevice), + case Read of + {ok, Content} -> + {ok, Content}; + eof -> + {error, eof}; + {error, Reason} -> + {error, Reason} + end; + {error, Reason} -> + {error, Reason} + end. + +-spec assemble(storage(), transfer(), emqx_ft:bytes()) -> + {async, _Assembler :: pid()} | ok | {error, _TODO}. +assemble(Storage, Transfer, Size) -> + LookupSources = [ + fun() -> lookup_local_assembler(Transfer) end, + fun() -> lookup_remote_assembler(Transfer) end, + fun() -> check_if_already_exported(Storage, Transfer) end, + fun() -> ensure_local_assembler(Storage, Transfer, Size) end + ], + lookup_assembler(LookupSources). + +%% + +files(Storage, Query) -> + emqx_ft_storage_exporter:list(Storage, Query). + +%% + +on_config_update(StorageOld, StorageNew) -> + % NOTE: this will reset GC timer, frequent changes would postpone GC indefinitely + ok = emqx_ft_storage_fs_gc:reset(StorageNew), + emqx_ft_storage_exporter:on_config_update(StorageOld, StorageNew). + +start(Storage) -> + ok = lists:foreach( + fun(ChildSpec) -> + {ok, _Child} = supervisor:start_child(emqx_ft_sup, ChildSpec) + end, + child_spec(Storage) + ), + ok = emqx_ft_storage_exporter:on_config_update(undefined, Storage), + ok. + +stop(Storage) -> + ok = emqx_ft_storage_exporter:on_config_update(Storage, undefined), + ok = lists:foreach( + fun(#{id := ChildId}) -> + _ = supervisor:terminate_child(emqx_ft_sup, ChildId), + ok = supervisor:delete_child(emqx_ft_sup, ChildId) + end, + child_spec(Storage) + ), + ok. + +%% + +lookup_assembler([LastSource]) -> + LastSource(); +lookup_assembler([Source | Sources]) -> + case Source() of + {error, not_found} -> lookup_assembler(Sources); + Result -> Result + end. + +check_if_already_exported(Storage, Transfer) -> + case files(Storage, #{transfer => Transfer}) of + {ok, #{items := [_ | _]}} -> ok; + _ -> {error, not_found} + end. + +lookup_local_assembler(Transfer) -> + case emqx_ft_assembler:where(Transfer) of + Pid when is_pid(Pid) -> {async, Pid}; + _ -> {error, not_found} + end. + +lookup_remote_assembler(Transfer) -> + Nodes = emqx:running_nodes() -- [node()], + Assemblers = lists:flatmap( + fun + ({ok, {async, Pid}}) -> [Pid]; + (_) -> [] + end, + emqx_ft_storage_fs_proto_v1:list_assemblers(Nodes, Transfer) + ), + case Assemblers of + [Pid | _] -> {async, Pid}; + _ -> {error, not_found} + end. + +ensure_local_assembler(Storage, Transfer, Size) -> + {ok, Pid} = emqx_ft_assembler_sup:ensure_child(Storage, Transfer, Size), + {async, Pid}. + +-spec transfers(storage()) -> + {ok, #{transfer() => transferinfo()}}. +transfers(Storage) -> + % TODO `Continuation` + % There might be millions of transfers on the node, we need a protocol and + % storage schema to iterate through them effectively. + ClientIds = try_list_dir(get_root(Storage)), + {ok, + lists:foldl( + fun(ClientId, Acc) -> transfers(Storage, ClientId, Acc) end, + #{}, + ClientIds + )}. + +transfers(Storage, ClientId, AccIn) -> + Dirname = filename:join(get_root(Storage), ClientId), + case file:list_dir(Dirname) of + {ok, FileIds} -> + lists:foldl( + fun(FileId, Acc) -> + Transfer = dirnames_to_transfer(ClientId, FileId), + read_transferinfo(Storage, Transfer, Acc) + end, + AccIn, + FileIds + ); + {error, _Reason} -> + ?tp(warning, "list_dir_failed", #{ + storage => Storage, + directory => Dirname + }), + AccIn + end. + +read_transferinfo(Storage, Transfer, Acc) -> + case read_filemeta(Storage, Transfer) of + {ok, Filemeta} -> + Acc#{Transfer => #{filemeta => Filemeta}}; + {error, enoent} -> + Acc#{Transfer => #{}}; + {error, Reason} -> + ?tp(warning, "read_transferinfo_failed", #{ + storage => Storage, + transfer => Transfer, + reason => Reason + }), + Acc + end. + +-spec get_root(storage()) -> + file:name(). +get_root(Storage) -> + case emqx_utils_maps:deep_find([segments, root], Storage) of + {ok, Root} -> + Root; + {not_found, _, _} -> + filename:join([emqx:data_dir(), file_transfer, segments]) + end. + +-spec get_subdir(storage(), transfer()) -> + file:name(). +get_subdir(Storage, Transfer) -> + mk_filedir(Storage, Transfer, []). + +-spec get_subdir(storage(), transfer(), fragment | temporary) -> + file:name(). +get_subdir(Storage, Transfer, What) -> + mk_filedir(Storage, Transfer, get_subdirs_for(What)). + +get_subdirs_for(fragment) -> + [?FRAGDIR]; +get_subdirs_for(temporary) -> + [?TEMPDIR]. + +-define(PRELUDE(Vsn, Meta), [<<"filemeta">>, Vsn, Meta]). + +encode_filemeta(Meta) -> + emqx_utils_json:encode(?PRELUDE(_Vsn = 1, emqx_ft:encode_filemeta(Meta))). + +decode_filemeta(Binary) when is_binary(Binary) -> + ?PRELUDE(_Vsn = 1, Map) = emqx_utils_json:decode(Binary, [return_maps]), + case emqx_ft:decode_filemeta(Map) of + {ok, Meta} -> + Meta; + {error, Reason} -> + error(Reason) + end. + +mk_segment_filename({Offset, Content}) -> + lists:concat([?SEGMENT, ".", Offset, ".", byte_size(Content)]). + +break_segment_filename(Filename) -> + Regex = "^" ?SEGMENT "[.]([0-9]+)[.]([0-9]+)$", + Result = re:run(Filename, Regex, [{capture, all_but_first, list}]), + case Result of + {match, [Offset, Size]} -> + {ok, #{offset => list_to_integer(Offset), size => list_to_integer(Size)}}; + nomatch -> + {error, invalid} + end. + +mk_filedir(Storage, {ClientId, FileId}, SubDirs) -> + filename:join([ + get_root(Storage), + emqx_ft_fs_util:escape_filename(ClientId), + emqx_ft_fs_util:escape_filename(FileId) + | SubDirs + ]). + +dirnames_to_transfer(ClientId, FileId) -> + {emqx_ft_fs_util:unescape_filename(ClientId), emqx_ft_fs_util:unescape_filename(FileId)}. + +mk_filepath(Storage, Transfer, SubDirs, Filename) -> + filename:join(mk_filedir(Storage, Transfer, SubDirs), Filename). + +try_list_dir(Dirname) -> + case file:list_dir(Dirname) of + {ok, List} -> List; + {error, _} -> [] + end. + +-include_lib("kernel/include/file.hrl"). + +read_file(Filepath, DecodeFun) -> + emqx_ft_fs_util:read_decode_file(Filepath, DecodeFun). + +write_file_atomic(Storage, Transfer, Filepath, Content) when is_binary(Content) -> + TempFilepath = mk_temp_filepath(Storage, Transfer, filename:basename(Filepath)), + Result = emqx_utils:pipeline( + [ + fun filelib:ensure_dir/1, + fun write_contents/2, + fun(_) -> mv_temp_file(TempFilepath, Filepath) end + ], + TempFilepath, + Content + ), + case Result of + {ok, _, _} -> + _ = file:delete(TempFilepath), + ok; + {error, Reason, _} -> + {error, Reason} + end. + +mk_temp_filepath(Storage, Transfer, Filename) -> + Unique = erlang:unique_integer([positive]), + filename:join(get_subdir(Storage, Transfer, temporary), mk_filename([Unique, ".", Filename])). + +mk_filename(Comps) -> + lists:append(lists:map(fun mk_filename_component/1, Comps)). + +mk_filename_component(I) when is_integer(I) -> integer_to_list(I); +mk_filename_component(A) when is_atom(A) -> atom_to_list(A); +mk_filename_component(B) when is_binary(B) -> unicode:characters_to_list(B); +mk_filename_component(S) when is_list(S) -> S. + +write_contents(Filepath, Content) -> + file:write_file(Filepath, Content). + +mv_temp_file(TempFilepath, Filepath) -> + _ = filelib:ensure_dir(Filepath), + file:rename(TempFilepath, Filepath). + +touch_file(Filepath) -> + Now = erlang:localtime(), + file:change_time(Filepath, _Mtime = Now, _Atime = Now). + +filtermap_files(Fun, Dirname, Filenames) -> + lists:filtermap(fun(Filename) -> Fun(Dirname, Filename) end, Filenames). + +mk_filefrag(Dirname, Filename = ?MANIFEST) -> + mk_filefrag(Dirname, Filename, filemeta, fun read_frag_filemeta/2); +mk_filefrag(Dirname, Filename = ?SEGMENT ++ _) -> + mk_filefrag(Dirname, Filename, segment, fun read_frag_segmentinfo/2); +mk_filefrag(_Dirname, _Filename) -> + ?tp(warning, "rogue_file_found", #{ + directory => _Dirname, + filename => _Filename + }), + false. + +mk_filefrag(Dirname, Filename, Tag, Fun) -> + Filepath = filename:join(Dirname, Filename), + % TODO error handling? + {ok, Fileinfo} = file:read_file_info(Filepath), + case Fun(Filename, Filepath) of + {ok, Frag} -> + {true, #{ + path => Filepath, + timestamp => Fileinfo#file_info.mtime, + size => Fileinfo#file_info.size, + fragment => {Tag, Frag} + }}; + {error, _Reason} -> + ?tp(warning, "mk_filefrag_failed", #{ + directory => Dirname, + filename => Filename, + type => Tag, + reason => _Reason + }), + false + end. + +read_frag_filemeta(_Filename, Filepath) -> + read_file(Filepath, fun decode_filemeta/1). + +read_frag_segmentinfo(Filename, _Filepath) -> + break_segment_filename(Filename). diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl new file mode 100644 index 000000000..4e9a6d56c --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_gc.erl @@ -0,0 +1,393 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% Filesystem storage GC +%% +%% This is conceptually a part of the Filesystem storage backend, even +%% though it's tied to the backend module with somewhat narrow interface. + +-module(emqx_ft_storage_fs_gc). + +-include_lib("emqx_ft/include/emqx_ft_storage_fs.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/types.hrl"). +-include_lib("kernel/include/file.hrl"). +-include_lib("snabbkaffe/include/trace.hrl"). + +-export([start_link/1]). + +-export([collect/0]). +-export([collect/3]). +-export([reset/0]). +-export([reset/1]). + +-behaviour(gen_server). +-export([init/1]). +-export([handle_call/3]). +-export([handle_cast/2]). +-export([handle_info/2]). + +-record(st, { + next_gc_timer :: maybe(reference()), + last_gc :: maybe(gcstats()) +}). + +-type gcstats() :: #gcstats{}. + +-define(IS_ENABLED(INTERVAL), (is_integer(INTERVAL) andalso INTERVAL > 0)). + +%% + +start_link(Storage) -> + gen_server:start_link(mk_server_ref(global), ?MODULE, Storage, []). + +-spec collect() -> gcstats(). +collect() -> + gen_server:call(mk_server_ref(global), {collect, erlang:system_time()}, infinity). + +-spec reset() -> ok | {error, _}. +reset() -> + emqx_ft_storage:with_storage_type(local, fun reset/1). + +-spec reset(emqx_ft_storage_fs:storage()) -> ok. +reset(Storage) -> + gen_server:cast(mk_server_ref(global), {reset, gc_interval(Storage)}). + +collect(Storage, Transfer, Nodes) -> + gc_enabled(Storage) andalso cast_collect(mk_server_ref(global), Storage, Transfer, Nodes). + +mk_server_ref(Name) -> + % TODO + {via, gproc, {n, l, {?MODULE, Name}}}. + +%% + +init(Storage) -> + St = #st{}, + {ok, start_timer(gc_interval(Storage), St)}. + +handle_call({collect, CalledAt}, _From, St) -> + StNext = maybe_collect_garbage(CalledAt, St), + {reply, StNext#st.last_gc, StNext}; +handle_call(Call, From, St) -> + ?SLOG(error, #{msg => "unexpected_call", call => Call, from => From}), + {noreply, St}. + +handle_cast({collect, Storage, Transfer, [Node | Rest]}, St) -> + ok = do_collect_transfer(Storage, Transfer, Node, St), + case Rest of + [_ | _] -> + cast_collect(self(), Storage, Transfer, Rest); + [] -> + ok + end, + {noreply, St}; +handle_cast({reset, Interval}, St) -> + {noreply, start_timer(Interval, cancel_timer(St))}; +handle_cast(Cast, St) -> + ?SLOG(error, #{msg => "unexpected_cast", cast => Cast}), + {noreply, St}. + +handle_info({timeout, TRef, collect}, St = #st{next_gc_timer = TRef}) -> + StNext = do_collect_garbage(St), + {noreply, start_timer(StNext#st{next_gc_timer = undefined})}. + +do_collect_transfer(Storage, Transfer, Node, St = #st{}) when Node == node() -> + Stats = try_collect_transfer(Storage, Transfer, complete, init_gcstats()), + ok = maybe_report(Stats, St), + ok; +do_collect_transfer(_Storage, _Transfer, _Node, _St = #st{}) -> + % TODO + ok. + +cast_collect(Ref, Storage, Transfer, Nodes) -> + gen_server:cast(Ref, {collect, Storage, Transfer, Nodes}). + +maybe_collect_garbage(_CalledAt, St = #st{last_gc = undefined}) -> + do_collect_garbage(St); +maybe_collect_garbage(CalledAt, St = #st{last_gc = #gcstats{finished_at = FinishedAt}}) -> + case FinishedAt > CalledAt of + true -> + St; + false -> + start_timer(do_collect_garbage(cancel_timer(St))) + end. + +do_collect_garbage(St = #st{}) -> + emqx_ft_storage:with_storage_type(local, fun(Storage) -> + Stats = collect_garbage(Storage), + ok = maybe_report(Stats, Storage), + St#st{last_gc = Stats} + end). + +maybe_report(#gcstats{errors = Errors}, Storage) when map_size(Errors) > 0 -> + ?tp(warning, "garbage_collection_errors", #{errors => Errors, storage => Storage}); +maybe_report(#gcstats{} = _Stats, _Storage) -> + ?tp(garbage_collection, #{stats => _Stats, storage => _Storage}). + +start_timer(St) -> + Interval = emqx_ft_storage:with_storage_type(local, fun gc_interval/1), + start_timer(Interval, St). + +start_timer(Interval, St = #st{next_gc_timer = undefined}) when ?IS_ENABLED(Interval) -> + St#st{next_gc_timer = emqx_utils:start_timer(Interval, collect)}; +start_timer(Interval, St) -> + ?SLOG(warning, #{msg => "periodic_gc_disabled", interval => Interval}), + St. + +cancel_timer(St = #st{next_gc_timer = undefined}) -> + St; +cancel_timer(St = #st{next_gc_timer = TRef}) -> + ok = emqx_utils:cancel_timer(TRef), + St#st{next_gc_timer = undefined}. + +gc_enabled(Storage) -> + ?IS_ENABLED(gc_interval(Storage)). + +gc_interval(Storage) -> + emqx_ft_conf:gc_interval(Storage). + +%% + +collect_garbage(Storage) -> + Stats = init_gcstats(), + {ok, Transfers} = emqx_ft_storage_fs:transfers(Storage), + collect_garbage(Storage, Transfers, Stats). + +collect_garbage(Storage, Transfers, Stats) -> + finish_gcstats( + maps:fold( + fun(Transfer, TransferInfo, StatsAcc) -> + % TODO: throttling? + try_collect_transfer(Storage, Transfer, TransferInfo, StatsAcc) + end, + Stats, + Transfers + ) + ). + +try_collect_transfer(Storage, Transfer, TransferInfo = #{}, Stats) -> + % File transfer might still be incomplete. + % Any outdated fragments and temporary files should be collectable. As a kind of + % heuristic we only delete transfer directory itself only if it is also outdated + % _and was empty at the start of GC_, as a precaution against races between + % writers and GCs. + Cutoff = + case get_segments_ttl(Storage, TransferInfo) of + TTL when is_integer(TTL) -> + erlang:system_time(second) - TTL; + undefined -> + 0 + end, + {FragCleaned, Stats1} = collect_outdated_fragments(Storage, Transfer, Cutoff, Stats), + {TempCleaned, Stats2} = collect_outdated_tempfiles(Storage, Transfer, Cutoff, Stats1), + % TODO: collect empty directories separately + case FragCleaned and TempCleaned of + true -> + collect_transfer_directory(Storage, Transfer, Cutoff, Stats2); + false -> + Stats2 + end; +try_collect_transfer(Storage, Transfer, complete, Stats) -> + % File transfer is complete. + % We should be good to delete fragments and temporary files with their respective + % directories altogether. + {_, Stats1} = collect_fragments(Storage, Transfer, Stats), + {_, Stats2} = collect_tempfiles(Storage, Transfer, Stats1), + Stats2. + +collect_fragments(Storage, Transfer, Stats) -> + Dirname = emqx_ft_storage_fs:get_subdir(Storage, Transfer, fragment), + maybe_collect_directory(Dirname, true, Stats). + +collect_tempfiles(Storage, Transfer, Stats) -> + Dirname = emqx_ft_storage_fs:get_subdir(Storage, Transfer, temporary), + maybe_collect_directory(Dirname, true, Stats). + +collect_outdated_fragments(Storage, Transfer, Cutoff, Stats) -> + Dirname = emqx_ft_storage_fs:get_subdir(Storage, Transfer, fragment), + maybe_collect_directory(Dirname, filter_older_than(Cutoff), Stats). + +collect_outdated_tempfiles(Storage, Transfer, Cutoff, Stats) -> + Dirname = emqx_ft_storage_fs:get_subdir(Storage, Transfer, temporary), + maybe_collect_directory(Dirname, filter_older_than(Cutoff), Stats). + +collect_transfer_directory(Storage, Transfer, Cutoff, Stats) -> + Dirname = emqx_ft_storage_fs:get_subdir(Storage, Transfer), + Filter = + case Stats of + #gcstats{directories = 0} -> + % Nothing were collected, this is a leftover from a past complete transfer GC. + filter_older_than(Cutoff); + #gcstats{} -> + % Usual incomplete transfer GC, collect directories unconditionally. + true + end, + case collect_empty_directory(Dirname, Filter, Stats) of + {true, StatsNext} -> + collect_parents(Dirname, get_segments_root(Storage), StatsNext); + {false, StatsNext} -> + StatsNext + end. + +filter_older_than(Cutoff) -> + fun(_Filepath, #file_info{mtime = ModifiedAt}) -> ModifiedAt =< Cutoff end. + +collect_parents(Dirname, Until, Stats) -> + Parent = filename:dirname(Dirname), + case is_same_filepath(Parent, Until) orelse file:del_dir(Parent) of + true -> + Stats; + ok -> + ?tp(garbage_collected_directory, #{path => Dirname}), + collect_parents(Parent, Until, account_gcstat_directory(Stats)); + {error, eexist} -> + Stats; + {error, Reason} -> + register_gcstat_error({directory, Parent}, Reason, Stats) + end. + +maybe_collect_directory(Dirpath, Filter, Stats) -> + case filelib:is_dir(Dirpath) of + true -> + collect_filepath(Dirpath, Filter, Stats); + false -> + {true, Stats} + end. + +-spec collect_filepath(file:name(), Filter, gcstats()) -> {boolean(), gcstats()} when + Filter :: boolean() | fun((file:name(), file:file_info()) -> boolean()). +collect_filepath(Filepath, Filter, Stats) -> + case file:read_link_info(Filepath, [{time, posix}, raw]) of + {ok, Fileinfo} -> + collect_filepath(Filepath, Fileinfo, Filter, Stats); + {error, Reason} -> + {Reason == enoent, register_gcstat_error({path, Filepath}, Reason, Stats)} + end. + +collect_filepath(Filepath, #file_info{type = directory} = Fileinfo, Filter, Stats) -> + collect_directory(Filepath, Fileinfo, Filter, Stats); +collect_filepath(Filepath, #file_info{type = regular} = Fileinfo, Filter, Stats) -> + case filter_filepath(Filter, Filepath, Fileinfo) andalso file:delete(Filepath, [raw]) of + false -> + {false, Stats}; + ok -> + ?tp(garbage_collected_file, #{path => Filepath}), + {true, account_gcstat(Fileinfo, Stats)}; + {error, Reason} -> + {Reason == enoent, register_gcstat_error({file, Filepath}, Reason, Stats)} + end; +collect_filepath(Filepath, Fileinfo, _Filter, Stats) -> + {false, register_gcstat_error({file, Filepath}, {unexpected, Fileinfo}, Stats)}. + +collect_directory(Dirpath, Fileinfo, Filter, Stats) -> + case file:list_dir(Dirpath) of + {ok, Filenames} -> + {Clean, StatsNext} = collect_files(Dirpath, Filenames, Filter, Stats), + case Clean of + true -> + collect_empty_directory(Dirpath, Fileinfo, Filter, StatsNext); + false -> + {false, StatsNext} + end; + {error, Reason} -> + {false, register_gcstat_error({directory, Dirpath}, Reason, Stats)} + end. + +collect_files(Dirname, Filenames, Filter, Stats) -> + lists:foldl( + fun(Filename, {Complete, StatsAcc}) -> + Filepath = filename:join(Dirname, Filename), + {Collected, StatsNext} = collect_filepath(Filepath, Filter, StatsAcc), + {Collected andalso Complete, StatsNext} + end, + {true, Stats}, + Filenames + ). + +collect_empty_directory(Dirpath, Filter, Stats) -> + case file:read_link_info(Dirpath, [{time, posix}, raw]) of + {ok, Dirinfo} -> + collect_empty_directory(Dirpath, Dirinfo, Filter, Stats); + {error, Reason} -> + {Reason == enoent, register_gcstat_error({directory, Dirpath}, Reason, Stats)} + end. + +collect_empty_directory(Dirpath, Dirinfo, Filter, Stats) -> + case filter_filepath(Filter, Dirpath, Dirinfo) andalso file:del_dir(Dirpath) of + false -> + {false, Stats}; + ok -> + ?tp(garbage_collected_directory, #{path => Dirpath}), + {true, account_gcstat_directory(Stats)}; + {error, Reason} -> + {false, register_gcstat_error({directory, Dirpath}, Reason, Stats)} + end. + +filter_filepath(Filter, _, _) when is_boolean(Filter) -> + Filter; +filter_filepath(Filter, Filepath, Fileinfo) when is_function(Filter) -> + Filter(Filepath, Fileinfo). + +is_same_filepath(P1, P2) when is_binary(P1) andalso is_binary(P2) -> + filename:absname(P1) == filename:absname(P2); +is_same_filepath(P1, P2) when is_list(P1) andalso is_list(P2) -> + filename:absname(P1) == filename:absname(P2); +is_same_filepath(P1, P2) when is_binary(P1) -> + is_same_filepath(P1, filepath_to_binary(P2)). + +filepath_to_binary(S) -> + unicode:characters_to_binary(S, unicode, file:native_name_encoding()). + +get_segments_ttl(Storage, TransferInfo) -> + clamp(emqx_ft_conf:segments_ttl(Storage), try_get_filemeta_ttl(TransferInfo)). + +try_get_filemeta_ttl(#{filemeta := Filemeta}) -> + maps:get(segments_ttl, Filemeta, undefined); +try_get_filemeta_ttl(#{}) -> + undefined. + +clamp({Min, Max}, V) -> + min(Max, max(Min, V)); +clamp(undefined, V) -> + V. + +%% + +init_gcstats() -> + #gcstats{started_at = erlang:system_time()}. + +finish_gcstats(Stats) -> + Stats#gcstats{finished_at = erlang:system_time()}. + +account_gcstat(Fileinfo, Stats = #gcstats{files = Files, space = Space}) -> + Stats#gcstats{ + files = Files + 1, + space = Space + Fileinfo#file_info.size + }. + +account_gcstat_directory(Stats = #gcstats{directories = Directories}) -> + Stats#gcstats{ + directories = Directories + 1 + }. + +register_gcstat_error(Subject, Error, Stats = #gcstats{errors = Errors}) -> + Stats#gcstats{errors = Errors#{Subject => Error}}. + +%% + +get_segments_root(Storage) -> + emqx_ft_storage_fs:get_root(Storage). diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl new file mode 100644 index 000000000..e6358cb14 --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_proxy.erl @@ -0,0 +1,36 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% This methods are called via rpc by `emqx_ft_storage_fs` +%% They populate the call with actual storage which may be configured differently +%% on a concrete node. + +-module(emqx_ft_storage_fs_proxy). + +-export([ + list_local/2, + pread_local/4, + lookup_local_assembler/1 +]). + +list_local(Transfer, What) -> + emqx_ft_storage:with_storage_type(local, list, [Transfer, What]). + +pread_local(Transfer, Frag, Offset, Size) -> + emqx_ft_storage:with_storage_type(local, pread, [Transfer, Frag, Offset, Size]). + +lookup_local_assembler(Transfer) -> + emqx_ft_storage:with_storage_type(local, lookup_local_assembler, [Transfer]). diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl new file mode 100644 index 000000000..513872edd --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_reader.erl @@ -0,0 +1,139 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_storage_fs_reader). + +-behaviour(gen_server). + +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/types.hrl"). + +%% API +-export([ + start_link/2, + start_supervised/2, + table/1, + table/2, + read/2 +]). + +%% gen_server callbacks +-export([ + init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3 +]). + +-define(DEFAULT_CHUNK_SIZE, 1024). +-define(IS_FILENAME(Filename), (is_list(Filename) or is_binary(Filename))). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- + +-spec table(pid()) -> qlc:query_handle(). +table(ReaderPid) when is_pid(ReaderPid) -> + table(ReaderPid, ?DEFAULT_CHUNK_SIZE). + +-spec table(pid(), pos_integer()) -> qlc:query_handle(). +table(ReaderPid, Bytes) when is_pid(ReaderPid) andalso is_integer(Bytes) andalso Bytes > 0 -> + NextFun = fun NextFun(Pid) -> + case emqx_ft_storage_fs_reader_proto_v1:read(node(Pid), Pid, Bytes) of + eof -> + []; + {ok, Data} -> + [Data] ++ fun() -> NextFun(Pid) end; + {ErrorKind, Reason} when ErrorKind =:= badrpc; ErrorKind =:= error -> + ?SLOG(warning, #{msg => "file_read_error", kind => ErrorKind, reason => Reason}), + [] + end + end, + qlc:table(fun() -> NextFun(ReaderPid) end, []). + +-spec start_link(pid(), filename:filename()) -> startlink_ret(). +start_link(CallerPid, Filename) when + is_pid(CallerPid) andalso + ?IS_FILENAME(Filename) +-> + gen_server:start_link(?MODULE, [CallerPid, Filename], []). + +-spec start_supervised(pid(), filename:filename()) -> startlink_ret(). +start_supervised(CallerPid, Filename) when + is_pid(CallerPid) andalso + ?IS_FILENAME(Filename) +-> + emqx_ft_storage_fs_reader_sup:start_child(CallerPid, Filename). + +-spec read(pid(), pos_integer()) -> {ok, binary()} | eof | {error, term()}. +read(Pid, Bytes) when + is_pid(Pid) andalso + is_integer(Bytes) andalso + Bytes > 0 +-> + gen_server:call(Pid, {read, Bytes}). + +%%-------------------------------------------------------------------- +%% gen_server callbacks +%%-------------------------------------------------------------------- + +init([CallerPid, Filename]) -> + MRef = erlang:monitor(process, CallerPid), + case file:open(Filename, [read, raw, binary]) of + {ok, File} -> + {ok, #{ + filename => Filename, + file => File, + caller_pid => CallerPid, + mref => MRef + }}; + {error, Reason} -> + {stop, Reason} + end. + +handle_call({read, Bytes}, _From, #{file := File} = State) -> + case file:read(File, Bytes) of + {ok, Data} -> + ?SLOG(debug, #{msg => "read", bytes => byte_size(Data)}), + {reply, {ok, Data}, State}; + eof -> + ?SLOG(debug, #{msg => "read", eof => true}), + {stop, normal, eof, State}; + {error, Reason} = Error -> + {stop, Reason, Error, State} + end; +handle_call(Msg, _From, State) -> + {reply, {error, {bad_call, Msg}}, State}. + +handle_info( + {'DOWN', MRef, process, CallerPid, _Reason}, #{mref := MRef, caller_pid := CallerPid} = State +) -> + {stop, {caller_down, CallerPid}, State}; +handle_info(Msg, State) -> + ?SLOG(warning, #{msg => "unexpected_message", info_msg => Msg}), + {noreply, State}. + +handle_cast(Msg, State) -> + ?SLOG(warning, #{msg => "unexpected_message", case_msg => Msg}), + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs_reader_sup.erl b/apps/emqx_ft/src/emqx_ft_storage_fs_reader_sup.erl new file mode 100644 index 000000000..8c8aea6b3 --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_storage_fs_reader_sup.erl @@ -0,0 +1,49 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_storage_fs_reader_sup). + +-behaviour(supervisor). + +-export([ + init/1, + start_link/0, + start_child/2 +]). + +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +start_child(CallerPid, Filename) -> + Childspec = #{ + id => {CallerPid, Filename}, + start => {emqx_ft_storage_fs_reader, start_link, [CallerPid, Filename]}, + restart => temporary + }, + case supervisor:start_child(?MODULE, Childspec) of + {ok, Pid} -> + {ok, Pid}; + {error, {Reason, _Child}} -> + {error, Reason} + end. + +init(_) -> + SupFlags = #{ + strategy => one_for_one, + intensity => 10, + period => 1000 + }, + {ok, {SupFlags, []}}. diff --git a/apps/emqx_ft/src/emqx_ft_sup.erl b/apps/emqx_ft/src/emqx_ft_sup.erl new file mode 100644 index 000000000..0308668ab --- /dev/null +++ b/apps/emqx_ft/src/emqx_ft_sup.erl @@ -0,0 +1,65 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_sup). + +-behaviour(supervisor). + +-export([start_link/0]). + +-export([init/1]). + +-define(SERVER, ?MODULE). + +start_link() -> + supervisor:start_link({local, ?SERVER}, ?MODULE, []). + +init([]) -> + SupFlags = #{ + strategy => one_for_one, + intensity => 100, + period => 10 + }, + + AssemblerSup = #{ + id => emqx_ft_assembler_sup, + start => {emqx_ft_assembler_sup, start_link, []}, + restart => permanent, + shutdown => infinity, + type => supervisor, + modules => [emqx_ft_assembler_sup] + }, + + FileReaderSup = #{ + id => emqx_ft_storage_fs_reader_sup, + start => {emqx_ft_storage_fs_reader_sup, start_link, []}, + restart => permanent, + shutdown => infinity, + type => supervisor, + modules => [emqx_ft_storage_fs_reader_sup] + }, + + Responder = #{ + id => emqx_ft_responder_sup, + start => {emqx_ft_responder_sup, start_link, []}, + restart => permanent, + shutdown => infinity, + type => worker, + modules => [emqx_ft_responder_sup] + }, + + ChildSpecs = [Responder, AssemblerSup, FileReaderSup], + {ok, {SupFlags, ChildSpecs}}. diff --git a/apps/emqx_ft/src/proto/emqx_ft_storage_exporter_fs_proto_v1.erl b/apps/emqx_ft/src/proto/emqx_ft_storage_exporter_fs_proto_v1.erl new file mode 100644 index 000000000..222891f54 --- /dev/null +++ b/apps/emqx_ft/src/proto/emqx_ft_storage_exporter_fs_proto_v1.erl @@ -0,0 +1,54 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_storage_exporter_fs_proto_v1). + +-behaviour(emqx_bpapi). + +-export([introduced_in/0]). + +-export([list_exports/2]). +-export([read_export_file/3]). + +-include_lib("emqx/include/bpapi.hrl"). + +introduced_in() -> + "5.0.17". + +-spec list_exports([node()], emqx_ft_storage:query(_LocalCursor)) -> + emqx_rpc:erpc_multicall( + {ok, [emqx_ft_storage:file_info()]} + | {error, file:posix() | disabled | {invalid_storage_type, _}} + ). +list_exports(Nodes, Query) -> + erpc:multicall( + Nodes, + emqx_ft_storage_exporter_fs_proxy, + list_exports_local, + [Query] + ). + +-spec read_export_file(node(), file:name(), pid()) -> + {ok, emqx_ft_storage:reader()} + | {error, term()} + | no_return(). +read_export_file(Node, Filepath, CallerPid) -> + erpc:call( + Node, + emqx_ft_storage_exporter_fs_proxy, + read_export_file_local, + [Filepath, CallerPid] + ). diff --git a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl new file mode 100644 index 000000000..989a48555 --- /dev/null +++ b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_proto_v1.erl @@ -0,0 +1,49 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_storage_fs_proto_v1). + +-behaviour(emqx_bpapi). + +-export([introduced_in/0]). + +-export([multilist/3]). +-export([pread/5]). +-export([list_assemblers/2]). + +-type offset() :: emqx_ft:offset(). +-type transfer() :: emqx_ft:transfer(). +-type filefrag() :: emqx_ft_storage_fs:filefrag(). + +-include_lib("emqx/include/bpapi.hrl"). + +introduced_in() -> + "5.0.17". + +-spec multilist([node()], transfer(), fragment | result) -> + emqx_rpc:erpc_multicall({ok, [filefrag()]} | {error, term()}). +multilist(Nodes, Transfer, What) -> + erpc:multicall(Nodes, emqx_ft_storage_fs_proxy, list_local, [Transfer, What]). + +-spec pread(node(), transfer(), filefrag(), offset(), _Size :: non_neg_integer()) -> + {ok, [filefrag()]} | {error, term()} | no_return(). +pread(Node, Transfer, Frag, Offset, Size) -> + erpc:call(Node, emqx_ft_storage_fs_proxy, pread_local, [Transfer, Frag, Offset, Size]). + +-spec list_assemblers([node()], transfer()) -> + emqx_rpc:erpc_multicall([pid()]). +list_assemblers(Nodes, Transfer) -> + erpc:multicall(Nodes, emqx_ft_storage_fs_proxy, lookup_local_assembler, [Transfer]). diff --git a/apps/emqx_ft/src/proto/emqx_ft_storage_fs_reader_proto_v1.erl b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_reader_proto_v1.erl new file mode 100644 index 000000000..ea089111d --- /dev/null +++ b/apps/emqx_ft/src/proto/emqx_ft_storage_fs_reader_proto_v1.erl @@ -0,0 +1,35 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_storage_fs_reader_proto_v1). + +-behaviour(emqx_bpapi). + +-export([introduced_in/0]). + +-export([read/3]). + +-include_lib("emqx/include/bpapi.hrl"). + +introduced_in() -> + "5.0.17". + +-spec read(node(), pid(), pos_integer()) -> + {ok, binary()} | eof | {error, term()} | no_return(). +read(Node, Pid, Bytes) when + is_atom(Node) andalso is_pid(Pid) andalso is_integer(Bytes) andalso Bytes > 0 +-> + emqx_rpc:call(Node, emqx_ft_storage_fs_reader, read, [Pid, Bytes]). diff --git a/apps/emqx_ft/test/emqx_ft_SUITE.erl b/apps/emqx_ft/test/emqx_ft_SUITE.erl new file mode 100644 index 000000000..7d64f9716 --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_SUITE.erl @@ -0,0 +1,782 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("stdlib/include/assert.hrl"). + +-define(assertRCName(RCName, PublishRes), + ?assertMatch( + {ok, #{reason_code_name := RCName}}, + PublishRes + ) +). + +all() -> + [ + {group, single_node}, + {group, cluster} + ]. + +groups() -> + [ + {single_node, [parallel], [ + t_assemble_crash, + t_corrupted_segment_retry, + t_invalid_checksum, + t_invalid_fileid, + t_invalid_filename, + t_invalid_meta, + t_invalid_topic_format, + t_meta_conflict, + t_nasty_clientids_fileids, + t_no_meta, + t_no_segment, + t_simple_transfer + ]}, + {cluster, [], [ + t_switch_node, + t_unreliable_migrating_client, + {g_concurrent_fins, [{repeat_until_any_fail, 8}], [ + t_concurrent_fins + ]} + ]} + ]. + +init_per_suite(Config) -> + ok = emqx_common_test_helpers:start_apps([emqx_ft], set_special_configs(Config)), + Config. + +end_per_suite(_Config) -> + ok = emqx_common_test_helpers:stop_apps([emqx_ft]), + ok. + +set_special_configs(Config) -> + fun + (emqx_ft) -> + % NOTE + % Inhibit local fs GC to simulate it isn't fast enough to collect + % complete transfers. + Storage = emqx_utils_maps:deep_merge( + emqx_ft_test_helpers:local_storage(Config), + #{<<"local">> => #{<<"segments">> => #{<<"gc">> => #{<<"interval">> => 0}}}} + ), + emqx_ft_test_helpers:load_config(#{ + <<"enable">> => true, + <<"storage">> => Storage + }); + (_) -> + ok + end. + +init_per_testcase(Case, Config) -> + ClientId = atom_to_binary(Case), + case ?config(group, Config) of + cluster -> + [{clientid, ClientId} | Config]; + _ -> + {ok, C} = emqtt:start_link([{proto_ver, v5}, {clientid, ClientId}]), + {ok, _} = emqtt:connect(C), + [{client, C}, {clientid, ClientId} | Config] + end. +end_per_testcase(_Case, Config) -> + _ = [ok = emqtt:stop(C) || {client, C} <- Config], + ok. + +init_per_group(Group = cluster, Config) -> + Cluster = mk_cluster_specs(Config), + ct:pal("Starting ~p", [Cluster]), + Nodes = [ + emqx_common_test_helpers:start_slave(Name, Opts#{join_to => node()}) + || {Name, Opts} <- Cluster + ], + [{group, Group}, {cluster_nodes, Nodes} | Config]; +init_per_group(Group, Config) -> + [{group, Group} | Config]. + +end_per_group(cluster, Config) -> + ok = lists:foreach( + fun emqx_ft_test_helpers:stop_additional_node/1, + ?config(cluster_nodes, Config) + ); +end_per_group(_Group, _Config) -> + ok. + +mk_cluster_specs(Config) -> + Specs = [ + {core, emqx_ft_SUITE1, #{listener_ports => [{tcp, 2883}]}}, + {core, emqx_ft_SUITE2, #{listener_ports => [{tcp, 3883}]}} + ], + CommOpts = [ + {env, [{emqx, boot_modules, [broker, listeners]}]}, + {apps, [emqx_ft]}, + {conf, [{[listeners, Proto, default, enabled], false} || Proto <- [ssl, ws, wss]]}, + {env_handler, set_special_configs(Config)} + ], + emqx_common_test_helpers:emqx_cluster( + Specs, + CommOpts + ). + +%%-------------------------------------------------------------------- +%% Tests +%%-------------------------------------------------------------------- + +t_invalid_topic_format(Config) -> + C = ?config(client, Config), + + ?assertRCName( + unspecified_error, + emqtt:publish(C, <<"$file/fileid">>, <<>>, 1) + ), + ?assertRCName( + unspecified_error, + emqtt:publish(C, <<"$file/fileid/">>, <<>>, 1) + ), + ?assertRCName( + unspecified_error, + emqtt:publish(C, <<"$file/fileid/offset">>, <<>>, 1) + ), + ?assertRCName( + unspecified_error, + emqtt:publish(C, <<"$file/fileid/fin/offset">>, <<>>, 1) + ), + ?assertRCName( + unspecified_error, + emqtt:publish(C, <<"$file/">>, <<>>, 1) + ), + ?assertRCName( + unspecified_error, + emqtt:publish(C, <<"$file/X/Y/Z">>, <<>>, 1) + ), + %% should not be handled by `emqx_ft` + ?assertRCName( + no_matching_subscribers, + emqtt:publish(C, <<"$file">>, <<>>, 1) + ). + +t_invalid_fileid(Config) -> + C = ?config(client, Config), + ?assertRCName( + unspecified_error, + emqtt:publish(C, <<"$file//init">>, <<>>, 1) + ). + +t_invalid_filename(Config) -> + C = ?config(client, Config), + ?assertRCName( + unspecified_error, + emqtt:publish(C, mk_init_topic(<<"f1">>), encode_meta(meta(".", <<>>)), 1) + ), + ?assertRCName( + unspecified_error, + emqtt:publish(C, mk_init_topic(<<"f2">>), encode_meta(meta("..", <<>>)), 1) + ), + ?assertRCName( + unspecified_error, + emqtt:publish(C, mk_init_topic(<<"f2">>), encode_meta(meta("../nice", <<>>)), 1) + ), + ?assertRCName( + unspecified_error, + emqtt:publish(C, mk_init_topic(<<"f3">>), encode_meta(meta("/etc/passwd", <<>>)), 1) + ), + ?assertRCName( + unspecified_error, + emqtt:publish( + C, + mk_init_topic(<<"f4">>), + encode_meta(meta(lists:duplicate(1000, $A), <<>>)), + 1 + ) + ), + ?assertRCName( + success, + emqtt:publish(C, mk_init_topic(<<"f5">>), encode_meta(meta("146%", <<>>)), 1) + ). + +t_simple_transfer(Config) -> + C = ?config(client, Config), + + Filename = "topsecret.pdf", + FileId = <<"f1">>, + + Data = [<<"first">>, <<"second">>, <<"third">>], + + Meta = #{size := Filesize} = meta(Filename, Data), + + ?assertRCName( + success, + emqtt:publish(C, mk_init_topic(FileId), encode_meta(Meta), 1) + ), + + lists:foreach( + fun({Chunk, Offset}) -> + ?assertRCName( + success, + emqtt:publish(C, mk_segment_topic(FileId, Offset), Chunk, 1) + ) + end, + with_offsets(Data) + ), + + ?assertRCName( + success, + emqtt:publish(C, mk_fin_topic(FileId, Filesize), <<>>, 1) + ), + + [Export] = list_files(?config(clientid, Config)), + ?assertEqual( + {ok, iolist_to_binary(Data)}, + read_export(Export) + ). + +t_nasty_clientids_fileids(_Config) -> + Transfers = [ + {<<".">>, <<".">>}, + {<<"🌚"/utf8>>, <<"🌝"/utf8>>}, + {<<"../..">>, <<"😤"/utf8>>}, + {<<"/etc/passwd">>, <<"whitehat">>}, + {<<"; rm -rf / ;">>, <<"whitehat">>} + ], + + ok = lists:foreach( + fun({ClientId, FileId}) -> + ok = emqx_ft_test_helpers:upload_file(ClientId, FileId, "justfile", ClientId), + [Export] = list_files(ClientId), + ?assertEqual({ok, ClientId}, read_export(Export)) + end, + Transfers + ). + +t_meta_conflict(Config) -> + C = ?config(client, Config), + + Filename = "topsecret.pdf", + FileId = <<"f1">>, + + Meta = meta(Filename, [<<"x">>]), + + ?assertRCName( + success, + emqtt:publish(C, mk_init_topic(FileId), encode_meta(Meta), 1) + ), + + ConflictMeta = Meta#{name => "conflict.pdf"}, + + ?assertRCName( + unspecified_error, + emqtt:publish(C, mk_init_topic(FileId), encode_meta(ConflictMeta), 1) + ). + +t_no_meta(Config) -> + C = ?config(client, Config), + + FileId = <<"f1">>, + Data = <<"first">>, + + ?assertRCName( + success, + emqtt:publish(C, mk_segment_topic(FileId, 0), Data, 1) + ), + + ?assertRCName( + unspecified_error, + emqtt:publish(C, mk_fin_topic(FileId, 42), <<>>, 1) + ). + +t_no_segment(Config) -> + C = ?config(client, Config), + + Filename = "topsecret.pdf", + FileId = <<"f1">>, + + Data = [<<"first">>, <<"second">>, <<"third">>], + + Meta = #{size := Filesize} = meta(Filename, Data), + + ?assertRCName( + success, + emqtt:publish(C, mk_init_topic(FileId), encode_meta(Meta), 1) + ), + + lists:foreach( + fun({Chunk, Offset}) -> + ?assertRCName( + success, + emqtt:publish(C, mk_segment_topic(FileId, Offset), Chunk, 1) + ) + end, + %% Skip the first segment + tl(with_offsets(Data)) + ), + + ?assertRCName( + unspecified_error, + emqtt:publish(C, mk_fin_topic(FileId, Filesize), <<>>, 1) + ). + +t_invalid_meta(Config) -> + C = ?config(client, Config), + + FileId = <<"f1">>, + + %% Invalid schema + Meta = #{foo => <<"bar">>}, + MetaPayload = emqx_utils_json:encode(Meta), + ?assertRCName( + unspecified_error, + emqtt:publish(C, mk_init_topic(FileId), MetaPayload, 1) + ), + + %% Invalid JSON + ?assertRCName( + unspecified_error, + emqtt:publish(C, mk_init_topic(FileId), <<"{oops;">>, 1) + ). + +t_invalid_checksum(Config) -> + C = ?config(client, Config), + + Filename = "topsecret.pdf", + FileId = <<"f1">>, + + Data = [<<"first">>, <<"second">>, <<"third">>], + + Meta = #{size := Filesize} = meta(Filename, Data), + MetaPayload = encode_meta(Meta#{checksum => {sha256, sha256(<<"invalid">>)}}), + + ?assertRCName( + success, + emqtt:publish(C, mk_init_topic(FileId), MetaPayload, 1) + ), + + lists:foreach( + fun({Chunk, Offset}) -> + ?assertRCName( + success, + emqtt:publish(C, mk_segment_topic(FileId, Offset), Chunk, 1) + ) + end, + with_offsets(Data) + ), + + ?assertRCName( + unspecified_error, + emqtt:publish(C, mk_fin_topic(FileId, Filesize), <<>>, 1) + ). + +t_corrupted_segment_retry(Config) -> + C = ?config(client, Config), + + Filename = "corruption.pdf", + FileId = <<"4242-4242">>, + + Data = [<<"first">>, <<"second">>, <<"third">>], + [ + {Seg1, Offset1}, + {Seg2, Offset2}, + {Seg3, Offset3} + ] = with_offsets(Data), + [ + Checksum1, + Checksum2, + Checksum3 + ] = [binary:encode_hex(sha256(S)) || S <- Data], + + Meta = #{size := Filesize} = meta(Filename, Data), + + ?assertRCName(success, emqtt:publish(C, mk_init_topic(FileId), encode_meta(Meta), 1)), + + ?assertRCName( + success, + emqtt:publish(C, mk_segment_topic(FileId, Offset1, Checksum1), Seg1, 1) + ), + + % segment is corrupted + ?assertRCName( + unspecified_error, + emqtt:publish(C, mk_segment_topic(FileId, Offset2, Checksum2), <>, 1) + ), + + % retry + ?assertRCName( + success, + emqtt:publish(C, mk_segment_topic(FileId, Offset2, Checksum2), Seg2, 1) + ), + + ?assertRCName( + success, + emqtt:publish(C, mk_segment_topic(FileId, Offset3, Checksum3), Seg3, 1) + ), + + ?assertRCName( + success, + emqtt:publish(C, mk_fin_topic(FileId, Filesize), <<>>, 1) + ). + +t_switch_node(Config) -> + [Node | _] = ?config(cluster_nodes, Config), + AdditionalNodePort = emqx_ft_test_helpers:tcp_port(Node), + + ClientId = <<"t_switch_node-migrating_client">>, + + {ok, C1} = emqtt:start_link([{proto_ver, v5}, {clientid, ClientId}, {port, AdditionalNodePort}]), + {ok, _} = emqtt:connect(C1), + + Filename = "multinode_upload.txt", + FileId = <<"f1">>, + + Data = [<<"first">>, <<"second">>, <<"third">>], + [{Data0, Offset0}, {Data1, Offset1}, {Data2, Offset2}] = with_offsets(Data), + + %% First, publist metadata and the first segment to the additional node + + Meta = #{size := Filesize} = meta(Filename, Data), + + ?assertRCName( + success, + emqtt:publish(C1, mk_init_topic(FileId), encode_meta(Meta), 1) + ), + ?assertRCName( + success, + emqtt:publish(C1, mk_segment_topic(FileId, Offset0), Data0, 1) + ), + + %% Then, switch the client to the main node + %% and publish the rest of the segments + + ok = emqtt:stop(C1), + {ok, C2} = emqtt:start_link([{proto_ver, v5}, {clientid, ClientId}]), + {ok, _} = emqtt:connect(C2), + + ?assertRCName( + success, + emqtt:publish(C2, mk_segment_topic(FileId, Offset1), Data1, 1) + ), + ?assertRCName( + success, + emqtt:publish(C2, mk_segment_topic(FileId, Offset2), Data2, 1) + ), + + ?assertRCName( + success, + emqtt:publish(C2, mk_fin_topic(FileId, Filesize), <<>>, 1) + ), + + ok = emqtt:stop(C2), + + %% Now check consistency of the file + + [Export] = list_files(ClientId), + ?assertEqual( + {ok, iolist_to_binary(Data)}, + read_export(Export) + ). + +t_assemble_crash(Config) -> + C = ?config(client, Config), + + meck:new(emqx_ft_storage_fs), + meck:expect(emqx_ft_storage_fs, assemble, fun(_, _, _) -> meck:exception(error, oops) end), + + ?assertRCName( + unspecified_error, + emqtt:publish(C, <<"$file/someid/fin">>, <<>>, 1) + ). + +t_unreliable_migrating_client(Config) -> + NodeSelf = node(), + [Node1, Node2] = ?config(cluster_nodes, Config), + + ClientId = ?config(clientid, Config), + FileId = emqx_guid:to_hexstr(emqx_guid:gen()), + Filename = "migratory-birds-in-southern-hemisphere-2013.pdf", + Filesize = 1000, + Gen = emqx_ft_content_gen:new({{ClientId, FileId}, Filesize}, 16), + Payload = iolist_to_binary(emqx_ft_content_gen:consume(Gen, fun({Chunk, _, _}) -> Chunk end)), + Meta = meta(Filename, Payload), + + Context = #{ + clientid => ClientId, + fileid => FileId, + filesize => Filesize, + payload => Payload + }, + Commands = [ + % Connect to the broker on the current node + {fun connect_mqtt_client/2, [NodeSelf]}, + % Send filemeta and 3 initial segments + % (assuming client chose 100 bytes as a desired segment size) + {fun send_filemeta/2, [Meta]}, + {fun send_segment/3, [0, 100]}, + {fun send_segment/3, [100, 100]}, + {fun send_segment/3, [200, 100]}, + % Disconnect the client cleanly + {fun stop_mqtt_client/1, []}, + % Connect to the broker on `Node1` + {fun connect_mqtt_client/2, [Node1]}, + % Connect to the broker on `Node2` without first disconnecting from `Node1` + % Client forgot the state for some reason and started the transfer again. + % (assuming this is usual for a client on a device that was rebooted) + {fun connect_mqtt_client/2, [Node2]}, + {fun send_filemeta/2, [Meta]}, + % This time it chose 200 bytes as a segment size + {fun send_segment/3, [0, 200]}, + {fun send_segment/3, [200, 200]}, + % But now it downscaled back to 100 bytes segments + {fun send_segment/3, [400, 100]}, + % Client lost connectivity and reconnected + % (also had last few segments unacked and decided to resend them) + {fun connect_mqtt_client/2, [Node2]}, + {fun send_segment/3, [200, 200]}, + {fun send_segment/3, [400, 200]}, + % Client lost connectivity and reconnected, this time to another node + % (also had last segment unacked and decided to resend it) + {fun connect_mqtt_client/2, [Node1]}, + {fun send_segment/3, [400, 200]}, + {fun send_segment/3, [600, eof]}, + {fun send_finish/1, []}, + % Client lost connectivity and reconnected, this time to the current node + % (client had `fin` unacked and decided to resend it) + {fun connect_mqtt_client/2, [NodeSelf]}, + {fun send_finish/1, []} + ], + _Context = run_commands(Commands, Context), + + Exports = list_files(?config(clientid, Config)), + + Node1Str = atom_to_list(Node1), + % TODO: this testcase is specific to local fs storage backend + ?assertMatch( + [#{"node" := Node1Str}], + fs_exported_file_attributes(Exports) + ), + + [ + ?assertEqual({ok, Payload}, read_export(Export)) + || Export <- Exports + ]. + +t_concurrent_fins(Config) -> + ct:timetrap({seconds, 10}), + + NodeSelf = node(), + [Node1, Node2] = ?config(cluster_nodes, Config), + + ClientId = iolist_to_binary([ + ?config(clientid, Config), + integer_to_list(erlang:unique_integer()) + ]), + FileId = emqx_guid:to_hexstr(emqx_guid:gen()), + Filename = "migratory-birds-in-southern-hemisphere-2013.pdf", + Filesize = 100, + Gen = emqx_ft_content_gen:new({{ClientId, FileId}, Filesize}, 16), + Payload = iolist_to_binary(emqx_ft_content_gen:consume(Gen, fun({Chunk, _, _}) -> Chunk end)), + Meta = meta(Filename, Payload), + + %% Send filemeta and segments to Node1 + Context0 = #{ + clientid => ClientId, + fileid => FileId, + filesize => Filesize, + payload => Payload + }, + + Context1 = run_commands( + [ + {fun connect_mqtt_client/2, [Node1]}, + {fun send_filemeta/2, [Meta]}, + {fun send_segment/3, [0, 100]}, + {fun stop_mqtt_client/1, []} + ], + Context0 + ), + + %% Now send fins concurrently to the 3 nodes + Nodes = [Node1, Node2, NodeSelf], + SendFin = fun(Node) -> + run_commands( + [ + {fun connect_mqtt_client/2, [Node]}, + {fun send_finish/1, []} + ], + Context1 + ) + end, + + PidMons = lists:map( + fun(Node) -> + erlang:spawn_monitor(fun F() -> + _ = erlang:process_flag(trap_exit, true), + try + SendFin(Node) + catch + C:E -> + % NOTE: random delay to avoid livelock conditions + ct:pal("Node ~p did not send finish successfully: ~p:~p", [Node, C, E]), + ok = timer:sleep(rand:uniform(10)), + F() + end + end) + end, + Nodes + ), + ok = lists:foreach( + fun({Pid, MRef}) -> + receive + {'DOWN', MRef, process, Pid, normal} -> ok + end + end, + PidMons + ), + + %% Only one node should have the file + Exports = list_files(ClientId), + case fs_exported_file_attributes(Exports) of + [#{"node" := _Node}] -> + ok; + [#{"node" := _Node} | _] = Files -> + % ...But we can't really guarantee that + ct:comment({multiple_files_on_different_nodes, Files}) + end. + +%%------------------------------------------------------------------------------ +%% Command helpers +%%------------------------------------------------------------------------------ + +%% Command runners + +run_commands(Commands, Context) -> + lists:foldl(fun run_command/2, Context, Commands). + +run_command({Command, Args}, Context) -> + ct:pal("COMMAND ~p ~p", [erlang:fun_info(Command, name), Args]), + erlang:apply(Command, Args ++ [Context]). + +%% Commands + +connect_mqtt_client(Node, ContextIn) -> + Context = #{clientid := ClientId} = disown_mqtt_client(ContextIn), + NodePort = emqx_ft_test_helpers:tcp_port(Node), + {ok, Client} = emqtt:start_link([{proto_ver, v5}, {clientid, ClientId}, {port, NodePort}]), + {ok, _} = emqtt:connect(Client), + Context#{client => Client}. + +stop_mqtt_client(Context = #{client := Client}) -> + _ = emqtt:stop(Client), + maps:remove(client, Context). + +disown_mqtt_client(Context = #{client := Client}) -> + _ = erlang:unlink(Client), + maps:remove(client, Context); +disown_mqtt_client(Context = #{}) -> + Context. + +send_filemeta(Meta, Context = #{client := Client, fileid := FileId}) -> + ?assertRCName( + success, + emqtt:publish(Client, mk_init_topic(FileId), encode_meta(Meta), 1) + ), + Context. + +send_segment(Offset, Size, Context = #{client := Client, fileid := FileId, payload := Payload}) -> + Data = + case Size of + eof -> + binary:part(Payload, Offset, byte_size(Payload) - Offset); + N -> + binary:part(Payload, Offset, N) + end, + ?assertRCName( + success, + emqtt:publish(Client, mk_segment_topic(FileId, Offset), Data, 1) + ), + Context. + +send_finish(Context = #{client := Client, fileid := FileId, filesize := Filesize}) -> + ?assertRCName( + success, + emqtt:publish(Client, mk_fin_topic(FileId, Filesize), <<>>, 1) + ), + Context. + +%%------------------------------------------------------------------------------ +%% Helpers +%%------------------------------------------------------------------------------ + +fs_exported_file_attributes(FSExports) -> + lists:map( + fun(#{uri := URIString}) -> + #{query := QS} = uri_string:parse(URIString), + maps:from_list(uri_string:dissect_query(QS)) + end, + lists:sort(FSExports) + ). + +mk_init_topic(FileId) -> + <<"$file/", FileId/binary, "/init">>. + +mk_segment_topic(FileId, Offset) when is_integer(Offset) -> + mk_segment_topic(FileId, integer_to_binary(Offset)); +mk_segment_topic(FileId, Offset) when is_binary(Offset) -> + <<"$file/", FileId/binary, "/", Offset/binary>>. + +mk_segment_topic(FileId, Offset, Checksum) when is_integer(Offset) -> + mk_segment_topic(FileId, integer_to_binary(Offset), Checksum); +mk_segment_topic(FileId, Offset, Checksum) when is_binary(Offset) -> + <<"$file/", FileId/binary, "/", Offset/binary, "/", Checksum/binary>>. + +mk_fin_topic(FileId, Size) when is_integer(Size) -> + mk_fin_topic(FileId, integer_to_binary(Size)); +mk_fin_topic(FileId, Size) when is_binary(Size) -> + <<"$file/", FileId/binary, "/fin/", Size/binary>>. + +with_offsets(Items) -> + {List, _} = lists:mapfoldl( + fun(Item, Offset) -> + {{Item, integer_to_binary(Offset)}, Offset + byte_size(Item)} + end, + 0, + Items + ), + List. + +sha256(Data) -> + crypto:hash(sha256, Data). + +meta(FileName, Data) -> + FullData = iolist_to_binary(Data), + #{ + name => FileName, + checksum => {sha256, sha256(FullData)}, + expire_at => erlang:system_time(_Unit = second) + 3600, + size => byte_size(FullData) + }. + +encode_meta(Meta) -> + emqx_utils_json:encode(emqx_ft:encode_filemeta(Meta)). + +list_files(ClientId) -> + {ok, #{items := Files}} = emqx_ft_storage:files(), + [File || File = #{transfer := {CId, _}} <- Files, CId == ClientId]. + +read_export(#{path := AbsFilepath}) -> + % TODO: only works for the local filesystem exporter right now + file:read_file(AbsFilepath). diff --git a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl new file mode 100644 index 000000000..f69e13a6d --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl @@ -0,0 +1,304 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_api_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("stdlib/include/assert.hrl"). + +-import(emqx_dashboard_api_test_helpers, [host/0, uri/1]). + +all() -> + [ + {group, single}, + {group, cluster} + ]. + +groups() -> + [ + {single, [], emqx_common_test_helpers:all(?MODULE)}, + {cluster, [], emqx_common_test_helpers:all(?MODULE)} + ]. + +init_per_suite(Config) -> + ok = emqx_mgmt_api_test_util:init_suite( + [emqx_conf, emqx_ft], emqx_ft_test_helpers:env_handler(Config) + ), + {ok, _} = emqx:update_config([rpc, port_discovery], manual), + Config. +end_per_suite(_Config) -> + ok = emqx_mgmt_api_test_util:end_suite([emqx_ft, emqx_conf]), + ok. + +init_per_group(Group = cluster, Config) -> + Cluster = mk_cluster_specs(Config), + ct:pal("Starting ~p", [Cluster]), + Nodes = [ + emqx_common_test_helpers:start_slave(Name, Opts#{join_to => node()}) + || {Name, Opts} <- Cluster + ], + [{group, Group}, {cluster_nodes, Nodes} | Config]; +init_per_group(Group, Config) -> + [{group, Group} | Config]. + +end_per_group(cluster, Config) -> + ok = lists:foreach( + fun emqx_ft_test_helpers:stop_additional_node/1, + ?config(cluster_nodes, Config) + ); +end_per_group(_Group, _Config) -> + ok. + +mk_cluster_specs(Config) -> + Specs = [ + {core, emqx_ft_api_SUITE1, #{listener_ports => [{tcp, 2883}]}}, + {core, emqx_ft_api_SUITE2, #{listener_ports => [{tcp, 3883}]}} + ], + CommOpts = [ + {env, [{emqx, boot_modules, [broker, listeners]}]}, + {apps, [emqx_ft]}, + {conf, [{[listeners, Proto, default, enabled], false} || Proto <- [ssl, ws, wss]]}, + {env_handler, emqx_ft_test_helpers:env_handler(Config)} + ], + emqx_common_test_helpers:emqx_cluster( + Specs, + CommOpts + ). + +init_per_testcase(Case, Config) -> + [{tc, Case} | Config]. +end_per_testcase(t_ft_disabled, _Config) -> + emqx_config:put([file_transfer, enable], true); +end_per_testcase(_Case, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Tests +%%-------------------------------------------------------------------- + +t_list_files(Config) -> + ClientId = client_id(Config), + FileId = <<"f1">>, + + Node = lists:last(cluster(Config)), + ok = emqx_ft_test_helpers:upload_file(ClientId, FileId, "f1", <<"data">>, Node), + + {ok, 200, #{<<"files">> := Files}} = + request_json(get, uri(["file_transfer", "files"])), + + ?assertMatch( + [#{<<"clientid">> := ClientId, <<"fileid">> := <<"f1">>}], + [File || File = #{<<"clientid">> := CId} <- Files, CId == ClientId] + ), + + {ok, 200, #{<<"files">> := FilesTransfer}} = + request_json(get, uri(["file_transfer", "files", ClientId, FileId])), + + ?assertMatch( + [#{<<"clientid">> := ClientId, <<"fileid">> := <<"f1">>}], + FilesTransfer + ), + + ?assertMatch( + {ok, 404, #{<<"code">> := <<"FILES_NOT_FOUND">>}}, + request_json(get, uri(["file_transfer", "files", ClientId, <<"no-such-file">>])) + ). + +t_download_transfer(Config) -> + ClientId = client_id(Config), + FileId = <<"f1">>, + + Node = lists:last(cluster(Config)), + ok = emqx_ft_test_helpers:upload_file(ClientId, FileId, "f1", <<"data">>, Node), + + ?assertMatch( + {ok, 400, #{<<"code">> := <<"BAD_REQUEST">>}}, + request_json( + get, + uri(["file_transfer", "file"]) ++ query(#{fileref => FileId}) + ) + ), + + ?assertMatch( + {ok, 503, _}, + request( + get, + uri(["file_transfer", "file"]) ++ + query(#{ + fileref => FileId, + node => <<"nonode@nohost">> + }) + ) + ), + + ?assertMatch( + {ok, 404, _}, + request( + get, + uri(["file_transfer", "file"]) ++ + query(#{ + fileref => <<"unknown_file">>, + node => node() + }) + ) + ), + + {ok, 200, #{<<"files">> := [File]}} = + request_json(get, uri(["file_transfer", "files", ClientId, FileId])), + + {ok, 200, Response} = request(get, host() ++ maps:get(<<"uri">>, File)), + + ?assertEqual( + <<"data">>, + Response + ). + +t_list_files_paging(Config) -> + ClientId = client_id(Config), + NFiles = 20, + Nodes = cluster(Config), + Uploads = [ + {mk_file_id("file:", N), mk_file_name(N), pick(N, Nodes)} + || N <- lists:seq(1, NFiles) + ], + ok = lists:foreach( + fun({FileId, Name, Node}) -> + ok = emqx_ft_test_helpers:upload_file(ClientId, FileId, Name, <<"data">>, Node) + end, + Uploads + ), + + ?assertMatch( + {ok, 200, #{<<"files">> := [_, _, _], <<"cursor">> := _}}, + request_json(get, uri(["file_transfer", "files"]) ++ query(#{limit => 3})) + ), + + {ok, 200, #{<<"files">> := Files}} = + request_json(get, uri(["file_transfer", "files"]) ++ query(#{limit => 100})), + + ?assert(length(Files) >= NFiles), + + ?assertNotMatch( + {ok, 200, #{<<"cursor">> := _}}, + request_json(get, uri(["file_transfer", "files"]) ++ query(#{limit => 100})) + ), + + ?assertMatch( + {ok, 400, #{<<"code">> := <<"BAD_REQUEST">>}}, + request_json(get, uri(["file_transfer", "files"]) ++ query(#{limit => 0})) + ), + + ?assertMatch( + {ok, 400, #{<<"code">> := <<"BAD_REQUEST">>}}, + request_json( + get, + uri(["file_transfer", "files"]) ++ query(#{following => <<"whatsthat!?">>}) + ) + ), + + PageThrough = fun PageThrough(Query, Acc) -> + case request_json(get, uri(["file_transfer", "files"]) ++ query(Query)) of + {ok, 200, #{<<"files">> := FilesPage, <<"cursor">> := Cursor}} -> + PageThrough(Query#{following => Cursor}, Acc ++ FilesPage); + {ok, 200, #{<<"files">> := FilesPage}} -> + Acc ++ FilesPage + end + end, + + ?assertEqual(Files, PageThrough(#{limit => 1}, [])), + ?assertEqual(Files, PageThrough(#{limit => 8}, [])), + ?assertEqual(Files, PageThrough(#{limit => NFiles}, [])). + +t_ft_disabled(_Config) -> + ?assertMatch( + {ok, 200, _}, + request_json(get, uri(["file_transfer", "files"])) + ), + + ?assertMatch( + {ok, 400, _}, + request_json( + get, + uri(["file_transfer", "file"]) ++ query(#{fileref => <<"f1">>}) + ) + ), + + ok = emqx_config:put([file_transfer, enable], false), + + ?assertMatch( + {ok, 503, _}, + request_json(get, uri(["file_transfer", "files"])) + ), + + ?assertMatch( + {ok, 503, _}, + request_json( + get, + uri(["file_transfer", "file"]) ++ query(#{fileref => <<"f1">>, node => node()}) + ) + ). + +%%-------------------------------------------------------------------- +%% Helpers +%%-------------------------------------------------------------------- + +cluster(Config) -> + [node() | proplists:get_value(cluster_nodes, Config, [])]. + +client_id(Config) -> + iolist_to_binary(io_lib:format("~s.~s", [?config(group, Config), ?config(tc, Config)])). + +mk_file_id(Prefix, N) -> + iolist_to_binary([Prefix, integer_to_list(N)]). + +mk_file_name(N) -> + "file." ++ integer_to_list(N). + +request(Method, Url) -> + emqx_mgmt_api_test_util:request(Method, Url, []). + +request_json(Method, Url) -> + case emqx_mgmt_api_test_util:request(Method, Url, []) of + {ok, Code, Body} -> + {ok, Code, json(Body)}; + Otherwise -> + Otherwise + end. + +json(Body) when is_binary(Body) -> + emqx_utils_json:decode(Body, [return_maps]). + +query(Params) -> + KVs = lists:map(fun({K, V}) -> uri_encode(K) ++ "=" ++ uri_encode(V) end, maps:to_list(Params)), + "?" ++ string:join(KVs, "&"). + +uri_encode(T) -> + emqx_http_lib:uri_encode(to_list(T)). + +to_list(A) when is_atom(A) -> + atom_to_list(A); +to_list(A) when is_integer(A) -> + integer_to_list(A); +to_list(B) when is_binary(B) -> + binary_to_list(B); +to_list(L) when is_list(L) -> + L. + +pick(N, List) -> + lists:nth(1 + (N rem length(List)), List). diff --git a/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl new file mode 100644 index 000000000..c1deeb3bc --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_assembler_SUITE.erl @@ -0,0 +1,265 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_assembler_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("stdlib/include/assert.hrl"). +-include_lib("kernel/include/file.hrl"). + +all() -> + [ + t_assemble_empty_transfer, + t_assemble_complete_local_transfer, + t_assemble_incomplete_transfer, + t_assemble_no_meta, + + % NOTE + % It depends on the side effects of all previous testcases. + t_list_transfers + ]. + +init_per_suite(Config) -> + Apps = application:ensure_all_started(gproc), + [{suite_apps, Apps} | Config]. + +end_per_suite(_Config) -> + ok. + +init_per_testcase(TC, Config) -> + ok = snabbkaffe:start_trace(), + {ok, Pid} = emqx_ft_assembler_sup:start_link(), + [ + {storage_root, <<"file_transfer_root">>}, + {exports_root, <<"file_transfer_exports">>}, + {file_id, atom_to_binary(TC)}, + {assembler_sup, Pid} + | Config + ]. + +end_per_testcase(_TC, Config) -> + ok = inspect_storage_root(Config), + ok = gen:stop(?config(assembler_sup, Config)), + ok = snabbkaffe:stop(), + ok. + +%% + +-define(CLIENTID1, <<"thatsme">>). +-define(CLIENTID2, <<"thatsnotme">>). + +t_assemble_empty_transfer(Config) -> + Storage = storage(Config), + Transfer = {?CLIENTID1, ?config(file_id, Config)}, + Filename = "important.pdf", + Meta = #{ + name => Filename, + size => 0, + expire_at => 42 + }, + ok = emqx_ft_storage_fs:store_filemeta(Storage, Transfer, Meta), + ?assertMatch( + {ok, [ + #{ + path := _, + timestamp := {{_, _, _}, {_, _, _}}, + fragment := {filemeta, Meta} + } + ]}, + emqx_ft_storage_fs:list(Storage, Transfer, fragment) + ), + Status = complete_assemble(Storage, Transfer, 0), + ?assertEqual({shutdown, ok}, Status), + {ok, [_Result = #{size := _Size = 0}]} = list_exports(Config, Transfer), + % ?assertEqual( + % {error, eof}, + % emqx_ft_storage_fs:pread(Storage, Transfer, Result, 0, Size) + % ), + ok. + +t_assemble_complete_local_transfer(Config) -> + Storage = storage(Config), + Transfer = {?CLIENTID2, ?config(file_id, Config)}, + Filename = "topsecret.pdf", + TransferSize = 10000 + rand:uniform(50000), + SegmentSize = 4096, + Gen = emqx_ft_content_gen:new({Transfer, TransferSize}, SegmentSize), + Hash = emqx_ft_content_gen:hash(Gen, crypto:hash_init(sha256)), + Meta = #{ + name => Filename, + checksum => {sha256, Hash}, + expire_at => 42 + }, + + ok = emqx_ft_storage_fs:store_filemeta(Storage, Transfer, Meta), + _ = emqx_ft_content_gen:consume( + Gen, + fun({Content, SegmentNum, _Meta}) -> + Offset = (SegmentNum - 1) * SegmentSize, + ?assertEqual( + ok, + emqx_ft_storage_fs:store_segment(Storage, Transfer, {Offset, Content}) + ) + end + ), + + {ok, Fragments} = emqx_ft_storage_fs:list(Storage, Transfer, fragment), + ?assertEqual((TransferSize div SegmentSize) + 1 + 1, length(Fragments)), + ?assertEqual( + [Meta], + [FM || #{fragment := {filemeta, FM}} <- Fragments], + Fragments + ), + + Status = complete_assemble(Storage, Transfer, TransferSize), + ?assertEqual({shutdown, ok}, Status), + + ?assertMatch( + {ok, [ + #{ + size := TransferSize, + meta := #{} + } + ]}, + list_exports(Config, Transfer) + ), + {ok, [#{path := AssemblyFilename}]} = list_exports(Config, Transfer), + ?assertMatch( + {ok, #file_info{type = regular, size = TransferSize}}, + file:read_file_info(AssemblyFilename) + ), + ok = emqx_ft_content_gen:check_file_consistency( + {Transfer, TransferSize}, + 100, + AssemblyFilename + ). + +t_assemble_incomplete_transfer(Config) -> + Storage = storage(Config), + Transfer = {?CLIENTID2, ?config(file_id, Config)}, + Filename = "incomplete.pdf", + TransferSize = 10000 + rand:uniform(50000), + SegmentSize = 4096, + Gen = emqx_ft_content_gen:new({Transfer, TransferSize}, SegmentSize), + Hash = emqx_ft_content_gen:hash(Gen, crypto:hash_init(sha256)), + Meta = #{ + name => Filename, + checksum => {sha256, Hash}, + size => TransferSize, + expire_at => 42 + }, + ok = emqx_ft_storage_fs:store_filemeta(Storage, Transfer, Meta), + Status = complete_assemble(Storage, Transfer, TransferSize), + ?assertMatch({shutdown, {error, _}}, Status). + +t_assemble_no_meta(Config) -> + Storage = storage(Config), + Transfer = {?CLIENTID2, ?config(file_id, Config)}, + Status = complete_assemble(Storage, Transfer, 42), + ?assertMatch({shutdown, {error, {incomplete, _}}}, Status). + +complete_assemble(Storage, Transfer, Size) -> + complete_assemble(Storage, Transfer, Size, 1000). + +complete_assemble(Storage, Transfer, Size, Timeout) -> + {async, Pid} = emqx_ft_storage_fs:assemble(Storage, Transfer, Size), + MRef = erlang:monitor(process, Pid), + Pid ! kickoff, + receive + {'DOWN', MRef, process, Pid, Result} -> + Result + after Timeout -> + ct:fail("Assembler did not finish in time") + end. + +%% + +t_list_transfers(Config) -> + {ok, Exports} = list_exports(Config), + ?assertMatch( + [ + #{ + transfer := {?CLIENTID2, <<"t_assemble_complete_local_transfer">>}, + path := _, + size := Size, + meta := #{name := "topsecret.pdf"} + }, + #{ + transfer := {?CLIENTID1, <<"t_assemble_empty_transfer">>}, + path := _, + size := 0, + meta := #{name := "important.pdf"} + } + ] when Size > 0, + lists:sort(Exports) + ). + +%% + +-include_lib("kernel/include/file.hrl"). + +inspect_storage_root(Config) -> + inspect_dir(?config(storage_root, Config)). + +inspect_dir(Dir) -> + FileInfos = filelib:fold_files( + Dir, + ".*", + true, + fun(Filename, Acc) -> orddict:store(Filename, inspect_file(Filename), Acc) end, + orddict:new() + ), + ct:pal("inspect '~s': ~p", [Dir, FileInfos]). + +inspect_file(Filename) -> + {ok, Info} = file:read_file_info(Filename), + {Info#file_info.type, Info#file_info.size, Info#file_info.mtime}. + +mk_fileid() -> + integer_to_binary(erlang:system_time(millisecond)). + +list_exports(Config) -> + {emqx_ft_storage_exporter_fs, Options} = exporter(Config), + emqx_ft_storage_exporter_fs:list_local(Options). + +list_exports(Config, Transfer) -> + {emqx_ft_storage_exporter_fs, Options} = exporter(Config), + emqx_ft_storage_exporter_fs:list_local_transfer(Options, Transfer). + +exporter(Config) -> + emqx_ft_storage_exporter:exporter(storage(Config)). + +storage(Config) -> + emqx_utils_maps:deep_get( + [storage, local], + emqx_ft_schema:translate(#{ + <<"storage">> => #{ + <<"local">> => #{ + <<"segments">> => #{ + <<"root">> => ?config(storage_root, Config) + }, + <<"exporter">> => #{ + <<"local">> => #{ + <<"root">> => ?config(exports_root, Config) + } + } + } + } + }) + ). diff --git a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl new file mode 100644 index 000000000..1f53f88af --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl @@ -0,0 +1,249 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_conf_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("stdlib/include/assert.hrl"). +-include_lib("snabbkaffe/include/test_macros.hrl"). + +all() -> emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_testcase(_Case, Config) -> + _ = emqx_config:save_schema_mod_and_names(emqx_ft_schema), + ok = emqx_common_test_helpers:start_apps( + [emqx_conf, emqx_ft], fun + (emqx_ft) -> + emqx_ft_test_helpers:load_config(#{}); + (_) -> + ok + end + ), + {ok, _} = emqx:update_config([rpc, port_discovery], manual), + Config. + +end_per_testcase(_Case, _Config) -> + ok = emqx_common_test_helpers:stop_apps([emqx_ft, emqx_conf]), + ok = emqx_config:erase(file_transfer). + +%%-------------------------------------------------------------------- +%% Tests +%%-------------------------------------------------------------------- + +t_update_config(_Config) -> + ?assertMatch( + {error, #{kind := validation_error}}, + emqx_conf:update( + [file_transfer], + #{<<"storage">> => #{<<"unknown">> => #{<<"foo">> => 42}}}, + #{} + ) + ), + ?assertMatch( + {ok, _}, + emqx_conf:update( + [file_transfer], + #{ + <<"enable">> => true, + <<"storage">> => #{ + <<"local">> => #{ + <<"segments">> => #{ + <<"root">> => <<"/tmp/path">>, + <<"gc">> => #{ + <<"interval">> => <<"5m">> + } + }, + <<"exporter">> => #{ + <<"local">> => #{ + <<"root">> => <<"/tmp/exports">> + } + } + } + } + }, + #{} + ) + ), + ?assertEqual( + <<"/tmp/path">>, + emqx_config:get([file_transfer, storage, local, segments, root]) + ), + ?assertEqual( + 5 * 60 * 1000, + emqx_ft_storage:with_storage_type(local, fun emqx_ft_conf:gc_interval/1) + ), + ?assertEqual( + {5 * 60, 24 * 60 * 60}, + emqx_ft_storage:with_storage_type(local, fun emqx_ft_conf:segments_ttl/1) + ). + +t_disable_restore_config(Config) -> + ?assertMatch( + {ok, _}, + emqx_conf:update( + [file_transfer], + #{<<"enable">> => true, <<"storage">> => #{<<"local">> => #{}}}, + #{} + ) + ), + ?assertEqual( + 60 * 60 * 1000, + emqx_ft_storage:with_storage_type(local, fun emqx_ft_conf:gc_interval/1) + ), + % Verify that transfers work + ok = emqx_ft_test_helpers:upload_file(gen_clientid(), <<"f1">>, "f1", <>), + % Verify that clearing storage settings reverts config to defaults + ?assertMatch( + {ok, _}, + emqx_conf:update( + [file_transfer], + #{<<"enable">> => false, <<"storage">> => undefined}, + #{} + ) + ), + ?assertEqual( + false, + emqx_ft_conf:enabled() + ), + ?assertMatch( + #{local := #{exporter := #{local := _}}}, + emqx_ft_conf:storage() + ), + ClientId = gen_clientid(), + Client = emqx_ft_test_helpers:start_client(ClientId), + % Verify that transfers fail cleanly when storage is disabled + ?check_trace( + ?assertMatch( + {ok, #{reason_code_name := no_matching_subscribers}}, + emqtt:publish( + Client, + <<"$file/f2/init">>, + emqx_utils_json:encode(emqx_ft:encode_filemeta(#{name => "f2", size => 42})), + 1 + ) + ), + fun(Trace) -> + ?assertMatch([], ?of_kind("file_transfer_init", Trace)) + end + ), + ok = emqtt:stop(Client), + % Restore local storage backend + Root = iolist_to_binary(emqx_ft_test_helpers:root(Config, node(), [segments])), + ?assertMatch( + {ok, _}, + emqx_conf:update( + [file_transfer], + #{ + <<"enable">> => true, + <<"storage">> => #{ + <<"local">> => #{ + <<"segments">> => #{ + <<"root">> => Root, + <<"gc">> => #{<<"interval">> => <<"1s">>} + } + } + } + }, + #{} + ) + ), + % Verify that GC is getting triggered eventually + ?check_trace( + ?block_until(#{?snk_kind := garbage_collection}, 5000, 0), + fun(Trace) -> + ?assertMatch( + [ + #{ + ?snk_kind := garbage_collection, + storage := #{segments := #{root := Root}} + } + ], + ?of_kind(garbage_collection, Trace) + ) + end + ), + % Verify that transfers work again + ok = emqx_ft_test_helpers:upload_file(gen_clientid(), <<"f1">>, "f1", <>). + +t_switch_exporter(_Config) -> + ?assertMatch( + {ok, _}, + emqx_conf:update( + [file_transfer], + #{<<"enable">> => true}, + #{} + ) + ), + ?assertMatch( + #{local := #{exporter := #{local := _}}}, + emqx_ft_conf:storage() + ), + % Verify that switching to a different exporter works + ?assertMatch( + {ok, _}, + emqx_conf:update( + [file_transfer, storage, local, exporter], + #{ + <<"s3">> => #{ + <<"bucket">> => <<"emqx">>, + <<"host">> => <<"https://localhost">>, + <<"port">> => 9000, + <<"transport_options">> => #{ + <<"ipv6_probe">> => false + } + } + }, + #{} + ) + ), + ?assertMatch( + #{local := #{exporter := #{s3 := _}}}, + emqx_ft_conf:storage() + ), + % Verify that switching back to local exporter works + ?assertMatch( + {ok, _}, + emqx_conf:remove( + [file_transfer, storage, local, exporter], + #{} + ) + ), + ?assertMatch( + {ok, _}, + emqx_conf:update( + [file_transfer, storage, local, exporter], + #{<<"local">> => #{}}, + #{} + ) + ), + ?assertMatch( + #{local := #{exporter := #{local := #{}}}}, + emqx_ft_conf:storage() + ), + % Verify that transfers work + ok = emqx_ft_test_helpers:upload_file(gen_clientid(), <<"f1">>, "f1", <>). + +gen_clientid() -> + emqx_base62:encode(emqx_guid:gen()). diff --git a/apps/emqx_ft/test/emqx_ft_content_gen.erl b/apps/emqx_ft/test/emqx_ft_content_gen.erl new file mode 100644 index 000000000..286c1a588 --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_content_gen.erl @@ -0,0 +1,232 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% Inspired by +%% https://github.com/kafka4beam/kflow/blob/master/src/testbed/payload_gen.erl + +-module(emqx_ft_content_gen). + +-include_lib("eunit/include/eunit.hrl"). + +-dialyzer(no_improper_lists). + +-export([new/2]). +-export([generate/3]). +-export([next/1]). +-export([consume/1]). +-export([consume/2]). +-export([fold/3]). + +-export([hash/2]). +-export([check_file_consistency/3]). + +-export_type([cont/1]). +-export_type([stream/1]). +-export_type([binary_payload/0]). + +-define(hash_size, 16). + +-type payload() :: {Seed :: term(), Size :: integer()}. + +-type binary_payload() :: { + binary(), _ChunkNum :: non_neg_integer(), _Meta :: #{} +}. + +-type cont(Data) :: + fun(() -> stream(Data)) + | stream(Data). + +-type stream(Data) :: + maybe_improper_list(Data, cont(Data)) + | eos. + +-record(chunk_state, { + seed :: term(), + payload_size :: non_neg_integer(), + offset :: non_neg_integer(), + chunk_size :: non_neg_integer() +}). + +-type chunk_state() :: #chunk_state{}. + +%% ----------------------------------------------------------------------------- +%% Generic streams +%% ----------------------------------------------------------------------------- + +%% @doc Consume one element from the stream. +-spec next(cont(A)) -> stream(A). +next(Fun) when is_function(Fun, 0) -> + Fun(); +next(L) -> + L. + +%% @doc Consume all elements of the stream and feed them into a +%% callback (e.g. brod:produce) +-spec consume(cont(A), fun((A) -> Ret)) -> [Ret]. +consume([Data | Cont], Callback) -> + [Callback(Data) | consume(next(Cont), Callback)]; +consume(Cont, Callback) when is_function(Cont, 0) -> + consume(next(Cont), Callback); +consume(eos, _Callback) -> + []. + +%% @equiv consume(Stream, fun(A) -> A end) +-spec consume(cont(A)) -> [A]. +consume(Stream) -> + consume(Stream, fun(A) -> A end). + +-spec fold(fun((A, Acc) -> Acc), Acc, cont(A)) -> Acc. +fold(Fun, Acc, [Data | Cont]) -> + fold(Fun, Fun(Data, Acc), next(Cont)); +fold(Fun, Acc, Cont) when is_function(Cont, 0) -> + fold(Fun, Acc, next(Cont)); +fold(_Fun, Acc, eos) -> + Acc. + +%% ----------------------------------------------------------------------------- +%% Binary streams +%% ----------------------------------------------------------------------------- + +%% @doc Stream of binary chunks. +%% Limitation: `ChunkSize' should be dividable by `?hash_size' +-spec new(payload(), integer()) -> cont(binary_payload()). +new({Seed, Size}, ChunkSize) when ChunkSize rem ?hash_size =:= 0 -> + fun() -> + generate_next_chunk(#chunk_state{ + seed = Seed, + payload_size = Size, + chunk_size = ChunkSize, + offset = 0 + }) + end. + +%% @doc Generate chunks of data and feed them into +%% `Callback' +-spec generate(payload(), integer(), fun((binary_payload()) -> A)) -> [A]. +generate(Payload, ChunkSize, Callback) -> + consume(new(Payload, ChunkSize), Callback). + +-spec hash(cont(binary_payload()), crypto:hash_state()) -> binary(). +hash(Stream, HashCtxIn) -> + crypto:hash_final( + fold( + fun({Chunk, _, _}, HashCtx) -> + crypto:hash_update(HashCtx, Chunk) + end, + HashCtxIn, + Stream + ) + ). + +-spec check_consistency( + payload(), + integer(), + fun((integer()) -> {ok, binary()} | undefined) +) -> ok. +check_consistency({Seed, Size}, SampleSize, Callback) -> + SeedHash = seed_hash(Seed), + Random = [rand:uniform(Size) - 1 || _ <- lists:seq(1, SampleSize)], + %% Always check first and last bytes, and one that should not exist: + Samples = [0, Size - 1, Size | Random], + lists:foreach( + fun + (N) when N < Size -> + Expected = do_get_byte(N, SeedHash), + ?assertEqual( + {N, {ok, Expected}}, + {N, Callback(N)} + ); + (N) -> + ?assertMatch(undefined, Callback(N)) + end, + Samples + ). + +-spec check_file_consistency( + payload(), + integer(), + file:filename() +) -> ok. +check_file_consistency(Payload, SampleSize, FileName) -> + {ok, FD} = file:open(FileName, [read, raw]), + try + Fun = fun(N) -> + case file:pread(FD, [{N, 1}]) of + {ok, [[X]]} -> {ok, X}; + {ok, [eof]} -> undefined + end + end, + check_consistency(Payload, SampleSize, Fun) + after + file:close(FD) + end. + +%% ============================================================================= +%% Internal functions +%% ============================================================================= + +%% @doc Continue generating chunks +-spec generate_next_chunk(chunk_state()) -> stream(binary()). +generate_next_chunk(#chunk_state{offset = Offset, payload_size = Size}) when Offset >= Size -> + eos; +generate_next_chunk(State0 = #chunk_state{offset = Offset, chunk_size = ChunkSize}) -> + State = State0#chunk_state{offset = Offset + ChunkSize}, + Payload = generate_chunk( + State#chunk_state.seed, + Offset, + ChunkSize, + State#chunk_state.payload_size + ), + [Payload | fun() -> generate_next_chunk(State) end]. + +generate_chunk(Seed, Offset, ChunkSize, Size) -> + SeedHash = seed_hash(Seed), + To = min(Offset + ChunkSize, Size) - 1, + Payload = iolist_to_binary([ + generator_fun(I, SeedHash) + || I <- lists:seq(Offset div 16, To div 16) + ]), + ChunkNum = Offset div ChunkSize + 1, + Meta = #{ + chunk_size => ChunkSize, + chunk_count => ceil(Size / ChunkSize) + }, + Chunk = + case Offset + ChunkSize of + NextOffset when NextOffset > Size -> + binary:part(Payload, 0, Size rem ChunkSize); + _ -> + Payload + end, + {Chunk, ChunkNum, Meta}. + +%% @doc First argument is a chunk number, the second one is a seed. +%% This implementation is hardly efficient, but it was chosen for +%% clarity reasons +-spec generator_fun(integer(), binary()) -> binary(). +generator_fun(N, Seed) -> + crypto:hash(md5, <>). + +%% @doc Hash any term +-spec seed_hash(term()) -> binary(). +seed_hash(Seed) -> + crypto:hash(md5, term_to_binary(Seed)). + +%% @private Get byte at offset `N' +-spec do_get_byte(integer(), binary()) -> byte(). +do_get_byte(N, Seed) -> + Chunk = generator_fun(N div ?hash_size, Seed), + binary:at(Chunk, N rem ?hash_size). diff --git a/apps/emqx_ft/test/emqx_ft_fs_util_SUITE.erl b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE.erl new file mode 100644 index 000000000..e4aa70f81 --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE.erl @@ -0,0 +1,250 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_fs_util_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("stdlib/include/assert.hrl"). +-include_lib("kernel/include/file.hrl"). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +t_fold_single_level(Config) -> + Root = ?config(data_dir, Config), + ?assertMatch( + [ + {"a", #file_info{type = directory}, ["a"]}, + {"c", #file_info{type = directory}, ["c"]}, + {"d", #file_info{type = directory}, ["d"]} + ], + sort(fold(fun cons/4, [], Root, ['*'])) + ). + +t_fold_multi_level(Config) -> + Root = ?config(data_dir, Config), + ?assertMatch( + [ + {"a/b/foo/42", #file_info{type = regular}, ["42", "foo", "b", "a"]}, + {"a/b/foo/Я", #file_info{type = regular}, ["Я", "foo", "b", "a"]}, + {"d/e/baz/needle", #file_info{type = regular}, ["needle", "baz", "e", "d"]} + ], + sort(fold(fun cons/4, [], Root, ['*', '*', '*', '*'])) + ), + ?assertMatch( + [ + {"a/b/foo", #file_info{type = directory}, ["foo", "b", "a"]}, + {"c/bar/中文", #file_info{type = regular}, ["中文", "bar", "c"]}, + {"d/e/baz", #file_info{type = directory}, ["baz", "e", "d"]} + ], + sort(fold(fun cons/4, [], Root, ['*', '*', '*'])) + ). + +t_fold_no_glob(Config) -> + Root = ?config(data_dir, Config), + ?assertMatch( + [{"", #file_info{type = directory}, []}], + sort(fold(fun cons/4, [], Root, [])) + ). + +t_fold_glob_too_deep(Config) -> + Root = ?config(data_dir, Config), + ?assertMatch( + [], + sort(fold(fun cons/4, [], Root, ['*', '*', '*', '*', '*'])) + ). + +t_fold_invalid_root(Config) -> + Root = ?config(data_dir, Config), + ?assertMatch( + [], + sort(fold(fun cons/4, [], filename:join([Root, "a", "link"]), ['*'])) + ), + ?assertMatch( + [], + sort(fold(fun cons/4, [], filename:join([Root, "d", "haystack"]), ['*'])) + ). + +t_fold_filter_unicode(Config) -> + Root = ?config(data_dir, Config), + ?assertMatch( + [ + {"a/b/foo/42", #file_info{type = regular}, ["42", "foo", "b", "a"]}, + {"d/e/baz/needle", #file_info{type = regular}, ["needle", "baz", "e", "d"]} + ], + sort(fold(fun cons/4, [], Root, ['*', '*', '*', fun is_latin1/1])) + ), + ?assertMatch( + [ + {"a/b/foo/Я", #file_info{type = regular}, ["Я", "foo", "b", "a"]} + ], + sort(fold(fun cons/4, [], Root, ['*', '*', '*', is_not(fun is_latin1/1)])) + ). + +t_fold_filter_levels(Config) -> + Root = ?config(data_dir, Config), + ?assertMatch( + [ + {"a/b/foo", #file_info{type = directory}, ["foo", "b", "a"]}, + {"d/e/baz", #file_info{type = directory}, ["baz", "e", "d"]} + ], + sort(fold(fun cons/4, [], Root, [fun is_letter/1, fun is_letter/1, '*'])) + ). + +t_fold_errors(Config) -> + Root = ?config(data_dir, Config), + ok = meck:new(emqx_ft_fs_util, [passthrough]), + ok = meck:expect(emqx_ft_fs_util, read_info, fun(AbsFilepath) -> + ct:pal("read_info(~p)", [AbsFilepath]), + Filename = filename:basename(AbsFilepath), + case Filename of + "b" -> {error, eacces}; + "link" -> {error, enotsup}; + "bar" -> {error, enotdir}; + "needle" -> {error, ebusy}; + _ -> meck:passthrough([AbsFilepath]) + end + end), + ?assertMatch( + [ + {"a/b", {error, eacces}, ["b", "a"]}, + {"a/link", {error, enotsup}, ["link", "a"]}, + {"c/link", {error, enotsup}, ["link", "c"]}, + {"d/e/baz/needle", {error, ebusy}, ["needle", "baz", "e", "d"]} + ], + sort(fold(fun cons/4, [], Root, ['*', '*', '*', '*'])) + ). + +t_seek_fold(Config) -> + Root = ?config(data_dir, Config), + ?assertMatch( + [ + {leaf, "a/b/foo/42", #file_info{type = regular}, ["42", "foo", "b", "a"]}, + {leaf, "a/b/foo/Я", #file_info{type = regular}, ["Я", "foo", "b", "a"]}, + {leaf, "d/e/baz/needle", #file_info{type = regular}, ["needle", "baz", "e", "d"]} + | _Nodes + ], + sort( + emqx_ft_fs_iterator:fold( + fun cons/2, + [], + emqx_ft_fs_iterator:seek(["a", "a"], Root, ['*', '*', '*', '*']) + ) + ) + ), + ?assertMatch( + [ + {leaf, "a/b/foo/Я", #file_info{type = regular}, ["Я", "foo", "b", "a"]}, + {leaf, "d/e/baz/needle", #file_info{type = regular}, ["needle", "baz", "e", "d"]} + | _Nodes + ], + sort( + emqx_ft_fs_iterator:fold( + fun cons/2, + [], + emqx_ft_fs_iterator:seek(["a", "b", "foo", "42"], Root, ['*', '*', '*', '*']) + ) + ) + ), + ?assertMatch( + [ + {leaf, "d/e/baz/needle", #file_info{type = regular}, ["needle", "baz", "e", "d"]} + | _Nodes + ], + sort( + emqx_ft_fs_iterator:fold( + fun cons/2, + [], + emqx_ft_fs_iterator:seek(["c", "d", "e", "f"], Root, ['*', '*', '*', '*']) + ) + ) + ). + +t_seek_empty(Config) -> + Root = ?config(data_dir, Config), + ?assertEqual( + emqx_ft_fs_iterator:fold( + fun cons/2, + [], + emqx_ft_fs_iterator:new(Root, ['*', '*', '*', '*']) + ), + emqx_ft_fs_iterator:fold( + fun cons/2, + [], + emqx_ft_fs_iterator:seek([], Root, ['*', '*', '*', '*']) + ) + ). + +t_seek_past_end(Config) -> + Root = ?config(data_dir, Config), + ?assertEqual( + none, + emqx_ft_fs_iterator:next( + emqx_ft_fs_iterator:seek(["g", "h"], Root, ['*', '*', '*', '*']) + ) + ). + +t_seek_with_filter(Config) -> + Root = ?config(data_dir, Config), + ?assertMatch( + [ + {leaf, "d/e/baz", #file_info{type = directory}, ["baz", "e", "d"]} + | _Nodes + ], + sort( + emqx_ft_fs_iterator:fold( + fun cons/2, + [], + emqx_ft_fs_iterator:seek(["a", "link"], Root, ['*', fun is_letter/1, '*']) + ) + ) + ). + +%% + +fold(FoldFun, Acc, Root, Glob) -> + emqx_ft_fs_util:fold(FoldFun, Acc, Root, Glob). + +is_not(F) -> + fun(X) -> not F(X) end. + +is_latin1(Filename) -> + case unicode:characters_to_binary(Filename, unicode, latin1) of + {error, _, _} -> + false; + _ -> + true + end. + +is_letter(Filename) -> + case Filename of + [_] -> + true; + _ -> + false + end. + +cons(Path, Info, Stack, Acc) -> + [{Path, Info, Stack} | Acc]. + +cons(Entry, Acc) -> + [Entry | Acc]. + +sort(L) when is_list(L) -> + lists:sort(L). diff --git a/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/a/b/foo/42 b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/a/b/foo/42 new file mode 100644 index 000000000..e69de29bb diff --git a/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/a/b/foo/Я b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/a/b/foo/Я new file mode 100644 index 000000000..ac31ffd53 --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/a/b/foo/Я @@ -0,0 +1 @@ +Ты diff --git a/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/a/link b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/a/link new file mode 120000 index 000000000..1b271d838 --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/a/link @@ -0,0 +1 @@ +../c \ No newline at end of file diff --git a/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/c/bar/中文 b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/c/bar/中文 new file mode 100644 index 000000000..2e11eb72f --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/c/bar/中文 @@ -0,0 +1 @@ +Zhōngwén diff --git a/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/c/link b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/c/link new file mode 120000 index 000000000..82f488f26 --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/c/link @@ -0,0 +1 @@ +../a \ No newline at end of file diff --git a/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/d/e/baz/needle b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/d/e/baz/needle new file mode 100644 index 000000000..d755762d1 --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/d/e/baz/needle @@ -0,0 +1 @@ +haystack diff --git a/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/d/haystack b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/d/haystack new file mode 100644 index 000000000..a6b681bf4 --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/d/haystack @@ -0,0 +1 @@ +needle diff --git a/apps/emqx_ft/test/emqx_ft_fs_util_tests.erl b/apps/emqx_ft/test/emqx_ft_fs_util_tests.erl new file mode 100644 index 000000000..1939e74c6 --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_fs_util_tests.erl @@ -0,0 +1,65 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_fs_util_tests). + +-include_lib("eunit/include/eunit.hrl"). + +filename_safe_test_() -> + [ + ?_assertEqual(ok, emqx_ft_fs_util:is_filename_safe("im.safe")), + ?_assertEqual(ok, emqx_ft_fs_util:is_filename_safe(<<"im.safe">>)), + ?_assertEqual(ok, emqx_ft_fs_util:is_filename_safe(<<".safe.100%">>)), + ?_assertEqual(ok, emqx_ft_fs_util:is_filename_safe(<<"safe.as.🦺"/utf8>>)) + ]. + +filename_unsafe_test_() -> + [ + ?_assertEqual({error, empty}, emqx_ft_fs_util:is_filename_safe("")), + ?_assertEqual({error, special}, emqx_ft_fs_util:is_filename_safe(".")), + ?_assertEqual({error, special}, emqx_ft_fs_util:is_filename_safe("..")), + ?_assertEqual({error, special}, emqx_ft_fs_util:is_filename_safe(<<"..">>)), + ?_assertEqual({error, unsafe}, emqx_ft_fs_util:is_filename_safe(<<".././..">>)), + ?_assertEqual({error, unsafe}, emqx_ft_fs_util:is_filename_safe("/etc/passwd")), + ?_assertEqual({error, unsafe}, emqx_ft_fs_util:is_filename_safe("../cookie")), + ?_assertEqual({error, unsafe}, emqx_ft_fs_util:is_filename_safe("C:$cookie")), + ?_assertEqual({error, nonprintable}, emqx_ft_fs_util:is_filename_safe([1, 2, 3])), + ?_assertEqual({error, nonprintable}, emqx_ft_fs_util:is_filename_safe(<<4, 5, 6>>)), + ?_assertEqual({error, nonprintable}, emqx_ft_fs_util:is_filename_safe([$a, 16#7F, $z])) + ]. + +-define(NAMES, [ + {"just.file", <<"just.file">>}, + {".hidden", <<".hidden">>}, + {".~what", <<".~what">>}, + {"100%25.file", <<"100%.file">>}, + {"%2E%2E", <<"..">>}, + {"...", <<"...">>}, + {"%2Fetc%2Fpasswd", <<"/etc/passwd">>}, + {"%01%02%0A ", <<1, 2, 10, 32>>} +]). + +escape_filename_test_() -> + [ + ?_assertEqual(Filename, emqx_ft_fs_util:escape_filename(Input)) + || {Filename, Input} <- ?NAMES + ]. + +unescape_filename_test_() -> + [ + ?_assertEqual(Input, emqx_ft_fs_util:unescape_filename(Filename)) + || {Filename, Input} <- ?NAMES + ]. diff --git a/apps/emqx_ft/test/emqx_ft_responder_SUITE.erl b/apps/emqx_ft/test/emqx_ft_responder_SUITE.erl new file mode 100644 index 000000000..751861206 --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_responder_SUITE.erl @@ -0,0 +1,84 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_responder_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("stdlib/include/assert.hrl"). + +all() -> emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + ok = emqx_common_test_helpers:start_apps([emqx_ft], emqx_ft_test_helpers:env_handler(Config)), + Config. + +end_per_suite(_Config) -> + ok = emqx_common_test_helpers:stop_apps([emqx_ft]), + ok. + +init_per_testcase(_Case, Config) -> + Config. + +end_per_testcase(_Case, _Config) -> + ok. + +t_start_ack(_Config) -> + Key = <<"test">>, + DefaultAction = fun({ack, Ref}) -> Ref end, + ?assertMatch( + {ok, _Pid}, + emqx_ft_responder:start(Key, DefaultAction, 1000) + ), + ?assertMatch( + {error, {already_started, _Pid}}, + emqx_ft_responder:start(Key, DefaultAction, 1000) + ), + Ref = make_ref(), + ?assertEqual( + Ref, + emqx_ft_responder:ack(Key, Ref) + ), + ?assertExit( + {noproc, _}, + emqx_ft_responder:ack(Key, Ref) + ). + +t_timeout(_Config) -> + Key = <<"test">>, + Self = self(), + DefaultAction = fun(timeout) -> Self ! {timeout, Key} end, + {ok, _Pid} = emqx_ft_responder:start(Key, DefaultAction, 20), + receive + {timeout, Key} -> + ok + after 100 -> + ct:fail("emqx_ft_responder not called") + end, + ?assertExit( + {noproc, _}, + emqx_ft_responder:ack(Key, oops) + ). + +t_unknown_msgs(_Config) -> + {ok, Pid} = emqx_ft_responder:start(make_ref(), fun(_) -> ok end, 100), + Pid ! {unknown_msg, <<"test">>}, + ok = gen_server:cast(Pid, {unknown_msg, <<"test">>}), + ?assertEqual( + {error, unknown_call}, + gen_server:call(Pid, {unknown_call, <<"test">>}) + ). diff --git a/apps/emqx_ft/test/emqx_ft_storage_exporter_s3_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_exporter_s3_SUITE.erl new file mode 100644 index 000000000..e717fe262 --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_storage_exporter_s3_SUITE.erl @@ -0,0 +1,199 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_storage_exporter_s3_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("stdlib/include/assert.hrl"). + +-define(assertS3Data(Data, Url), + case httpc:request(Url) of + {ok, {{_StatusLine, 200, "OK"}, _Headers, Body}} -> + ?assertEqual(Data, list_to_binary(Body), "S3 data mismatch"); + OtherResponse -> + ct:fail("Unexpected response: ~p", [OtherResponse]) + end +). + +all() -> emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + Config. +end_per_suite(_Config) -> + ok. + +set_special_configs(Config) -> + fun + (emqx_ft) -> + Storage = emqx_ft_test_helpers:local_storage(Config, #{ + exporter => s3, bucket_name => ?config(bucket_name, Config) + }), + emqx_ft_test_helpers:load_config(#{<<"enable">> => true, <<"storage">> => Storage}); + (_) -> + ok + end. + +init_per_testcase(Case, Config0) -> + ClientId = atom_to_binary(Case), + BucketName = create_bucket(), + Config1 = [{bucket_name, BucketName}, {clientid, ClientId} | Config0], + ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_ft], set_special_configs(Config1)), + Config1. +end_per_testcase(_Case, _Config) -> + ok = emqx_common_test_helpers:stop_apps([emqx_ft, emqx_conf]), + ok. + +%%-------------------------------------------------------------------- +%% Test Cases +%%------------------------------------------------------------------- + +t_happy_path(Config) -> + ClientId = ?config(clientid, Config), + + FileId = <<"🌚"/utf8>>, + Name = "cool_name", + Data = <<"data"/utf8>>, + + ?assertEqual( + ok, + emqx_ft_test_helpers:upload_file(ClientId, FileId, Name, Data) + ), + + {ok, #{items := [#{uri := Uri}]}} = emqx_ft_storage:files(), + + ?assertS3Data(Data, Uri), + + Key = binary_to_list(ClientId) ++ "/" ++ binary_to_list(FileId) ++ "/" ++ Name, + Meta = erlcloud_s3:get_object_metadata( + ?config(bucket_name, Config), Key, emqx_ft_test_helpers:aws_config() + ), + + ?assertEqual( + ClientId, + metadata_field("clientid", Meta) + ), + + ?assertEqual( + FileId, + metadata_field("fileid", Meta) + ), + + NameBin = list_to_binary(Name), + ?assertMatch( + #{ + <<"name">> := NameBin, + <<"size">> := 4 + }, + emqx_utils_json:decode(metadata_field("filemeta", Meta), [return_maps]) + ). + +t_upload_error(Config) -> + ClientId = ?config(clientid, Config), + + FileId = <<"🌚"/utf8>>, + Name = "cool_name", + Data = <<"data"/utf8>>, + + {ok, _} = emqx_conf:update( + [file_transfer, storage, local, exporter, s3, bucket], <<"invalid-bucket">>, #{} + ), + + ?assertEqual( + {error, unspecified_error}, + emqx_ft_test_helpers:upload_file(ClientId, FileId, Name, Data) + ). + +t_paging(Config) -> + ClientId = ?config(clientid, Config), + N = 1050, + + FileId = fun integer_to_binary/1, + Name = "cool_name", + Data = fun integer_to_binary/1, + + ok = lists:foreach( + fun(I) -> + ok = emqx_ft_test_helpers:upload_file(ClientId, FileId(I), Name, Data(I)) + end, + lists:seq(1, N) + ), + + {ok, #{items := [#{uri := Uri}]}} = emqx_ft_storage:files(#{transfer => {ClientId, FileId(123)}}), + + ?assertS3Data(Data(123), Uri), + + lists:foreach( + fun(PageSize) -> + Pages = file_pages(#{limit => PageSize}), + ?assertEqual( + expected_page_count(PageSize, N), + length(Pages) + ), + FileIds = [ + FId + || #{transfer := {_, FId}} <- lists:concat(Pages) + ], + ?assertEqual( + lists:sort([FileId(I) || I <- lists:seq(1, N)]), + lists:sort(FileIds) + ) + end, + %% less than S3 limit, greater than S3 limit + [20, 550] + ). + +t_invalid_cursor(_Config) -> + InvalidUtf8 = <<16#80>>, + ?assertError( + {badarg, cursor}, + emqx_ft_storage:files(#{following => InvalidUtf8}) + ). + +%%-------------------------------------------------------------------- +%% Helper Functions +%%-------------------------------------------------------------------- + +expected_page_count(PageSize, Total) -> + case Total rem PageSize of + 0 -> Total div PageSize; + _ -> Total div PageSize + 1 + end. + +file_pages(Query) -> + case emqx_ft_storage:files(Query) of + {ok, #{items := Items, cursor := NewCursor}} -> + [Items] ++ file_pages(Query#{following => NewCursor}); + {ok, #{items := Items}} -> + [Items]; + {error, Error} -> + ct:fail("Failed to download files: ~p", [Error]) + end. + +metadata_field(Field, Meta) -> + Key = "x-amz-meta-" ++ Field, + case lists:keyfind(Key, 1, Meta) of + {Key, Value} -> list_to_binary(Value); + false -> false + end. + +create_bucket() -> + BucketName = emqx_s3_test_helpers:unique_bucket(), + _ = application:ensure_all_started(lhttpc), + ok = erlcloud_s3:create_bucket(BucketName, emqx_ft_test_helpers:aws_config()), + BucketName. diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl new file mode 100644 index 000000000..50925cfb9 --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl @@ -0,0 +1,93 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_storage_fs_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("stdlib/include/assert.hrl"). + +all() -> + [ + {group, cluster} + ]. + +-define(CLUSTER_CASES, [t_multinode_exports]). + +groups() -> + [ + {cluster, [sequence], ?CLUSTER_CASES} + ]. + +init_per_suite(Config) -> + ok = emqx_common_test_helpers:start_apps([emqx_ft], emqx_ft_test_helpers:env_handler(Config)), + Config. +end_per_suite(_Config) -> + ok = emqx_common_test_helpers:stop_apps([emqx_ft]), + ok. + +init_per_testcase(Case, Config) -> + [{tc, Case} | Config]. +end_per_testcase(_Case, _Config) -> + ok. + +init_per_group(cluster, Config) -> + Node = emqx_ft_test_helpers:start_additional_node(Config, emqx_ft_storage_fs1), + [{additional_node, Node} | Config]; +init_per_group(_Group, Config) -> + Config. + +end_per_group(cluster, Config) -> + ok = emqx_ft_test_helpers:stop_additional_node(?config(additional_node, Config)); +end_per_group(_Group, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Tests +%%-------------------------------------------------------------------- + +t_multinode_exports(Config) -> + Node1 = ?config(additional_node, Config), + ok = emqx_ft_test_helpers:upload_file(<<"c/1">>, <<"f:1">>, "fn1", <<"data">>, Node1), + + Node2 = node(), + ok = emqx_ft_test_helpers:upload_file(<<"c/2">>, <<"f:2">>, "fn2", <<"data">>, Node2), + + ?assertMatch( + [ + #{transfer := {<<"c/1">>, <<"f:1">>}, name := "fn1"}, + #{transfer := {<<"c/2">>, <<"f:2">>}, name := "fn2"} + ], + lists:sort(list_files(Config)) + ). + +%%-------------------------------------------------------------------- +%% Helpers +%%-------------------------------------------------------------------- + +client_id(Config) -> + atom_to_binary(?config(tc, Config), utf8). + +storage(Config) -> + RawConfig = #{<<"storage">> => emqx_ft_test_helpers:local_storage(Config)}, + #{storage := #{local := Storage}} = emqx_ft_schema:translate(RawConfig), + Storage. + +list_files(Config) -> + {ok, #{items := Files}} = emqx_ft_storage_fs:files(storage(Config), #{}), + Files. diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl new file mode 100644 index 000000000..a7ffd5675 --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_gc_SUITE.erl @@ -0,0 +1,363 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_storage_fs_gc_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("emqx_ft/include/emqx_ft_storage_fs.hrl"). +-include_lib("stdlib/include/assert.hrl"). +-include_lib("snabbkaffe/include/test_macros.hrl"). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + _ = application:load(emqx_ft), + ok = emqx_common_test_helpers:start_apps([]), + Config. + +end_per_suite(_Config) -> + ok = emqx_common_test_helpers:stop_apps([]), + ok. + +init_per_testcase(TC, Config) -> + SegmentsRoot = emqx_ft_test_helpers:root(Config, node(), [TC, segments]), + ExportsRoot = emqx_ft_test_helpers:root(Config, node(), [TC, exports]), + ok = emqx_common_test_helpers:start_app( + emqx_ft, + fun(emqx_ft) -> + emqx_ft_test_helpers:load_config(#{ + <<"enable">> => true, + <<"storage">> => #{ + <<"local">> => #{ + <<"segments">> => #{<<"root">> => SegmentsRoot}, + <<"exporter">> => #{ + <<"local">> => #{<<"root">> => ExportsRoot} + } + } + } + }) + end + ), + ok = snabbkaffe:start_trace(), + Config. + +end_per_testcase(_TC, _Config) -> + ok = snabbkaffe:stop(), + ok = application:stop(emqx_ft), + ok. + +%% + +-define(NSEGS(Filesize, SegmentSize), (ceil(Filesize / SegmentSize) + 1)). + +t_gc_triggers_periodically(_Config) -> + Interval = 500, + ok = set_gc_config(interval, Interval), + ok = emqx_ft_storage_fs_gc:reset(), + ?check_trace( + timer:sleep(Interval * 3), + fun(Trace) -> + [Event, _ | _] = ?of_kind(garbage_collection, Trace), + ?assertMatch( + #{ + stats := #gcstats{ + files = 0, + directories = 0, + space = 0, + errors = #{} = Errors + } + } when map_size(Errors) == 0, + Event + ) + end + ). + +t_gc_triggers_manually(_Config) -> + ?check_trace( + ?assertMatch( + #gcstats{files = 0, directories = 0, space = 0, errors = #{} = Errors} when + map_size(Errors) == 0, + emqx_ft_storage_fs_gc:collect() + ), + fun(Trace) -> + [Event] = ?of_kind(garbage_collection, Trace), + ?assertMatch( + #{stats := #gcstats{}}, + Event + ) + end + ). + +t_gc_complete_transfers(_Config) -> + {local, Storage} = emqx_ft_storage:backend(), + ok = set_gc_config(minimum_segments_ttl, 0), + ok = set_gc_config(maximum_segments_ttl, 3), + ok = set_gc_config(interval, 500), + ok = emqx_ft_storage_fs_gc:reset(), + Transfers = [ + { + T1 = {<<"client1">>, mk_file_id()}, + #{name => "cat.cur", segments_ttl => 10}, + emqx_ft_content_gen:new({?LINE, S1 = 42}, SS1 = 16) + }, + { + T2 = {<<"client2">>, mk_file_id()}, + #{name => "cat.ico", segments_ttl => 10}, + emqx_ft_content_gen:new({?LINE, S2 = 420}, SS2 = 64) + }, + { + T3 = {<<"client42">>, mk_file_id()}, + #{name => "cat.jpg", segments_ttl => 10}, + emqx_ft_content_gen:new({?LINE, S3 = 42000}, SS3 = 1024) + } + ], + % 1. Start all transfers + TransferSizes = emqx_utils:pmap( + fun(Transfer) -> start_transfer(Storage, Transfer) end, + Transfers + ), + ?assertEqual([S1, S2, S3], TransferSizes), + ?assertMatch( + #gcstats{files = 0, directories = 0, errors = #{} = Es} when map_size(Es) == 0, + emqx_ft_storage_fs_gc:collect() + ), + % 2. Complete just the first transfer + {ok, {ok, Event}} = ?wait_async_action( + ?assertEqual(ok, complete_transfer(Storage, T1, S1)), + #{?snk_kind := garbage_collection}, + 1000 + ), + ?assertMatch( + #{ + stats := #gcstats{ + files = Files, + directories = 2, + space = Space, + errors = #{} = Es + } + } when Files == ?NSEGS(S1, SS1) andalso Space > S1 andalso map_size(Es) == 0, + Event + ), + % 3. Complete rest of transfers + {ok, Sub} = snabbkaffe_collector:subscribe( + ?match_event(#{?snk_kind := garbage_collection}), + 2, + 1000, + 0 + ), + ?assertEqual( + [ok, ok], + emqx_utils:pmap( + fun({Transfer, Size}) -> complete_transfer(Storage, Transfer, Size) end, + [{T2, S2}, {T3, S3}] + ) + ), + {ok, Events} = snabbkaffe_collector:receive_events(Sub), + CFiles = lists:sum([Stats#gcstats.files || #{stats := Stats} <- Events]), + CDirectories = lists:sum([Stats#gcstats.directories || #{stats := Stats} <- Events]), + CSpace = lists:sum([Stats#gcstats.space || #{stats := Stats} <- Events]), + CErrors = lists:foldl( + fun maps:merge/2, + #{}, + [Stats#gcstats.errors || #{stats := Stats} <- Events] + ), + ?assertEqual(?NSEGS(S2, SS2) + ?NSEGS(S3, SS3), CFiles), + ?assertEqual(2 + 2, CDirectories), + ?assertMatch(Space when Space > S2 + S3, CSpace), + ?assertMatch(Errors when map_size(Errors) == 0, CErrors), + % 4. Ensure that empty transfer directories will be eventually collected + {ok, _} = ?block_until( + #{ + ?snk_kind := garbage_collection, + stats := #gcstats{ + files = 0, + directories = 6, + space = 0 + } + }, + 5000, + 0 + ). + +t_gc_incomplete_transfers(_Config) -> + ok = set_gc_config(minimum_segments_ttl, 0), + ok = set_gc_config(maximum_segments_ttl, 4), + {local, Storage} = emqx_ft_storage:backend(), + Transfers = [ + { + {<<"client43"/utf8>>, <<"file-🦕"/utf8>>}, + #{name => "dog.cur", segments_ttl => 1}, + emqx_ft_content_gen:new({?LINE, S1 = 123}, SS1 = 32) + }, + { + {<<"client44">>, <<"file-🦖"/utf8>>}, + #{name => "dog.ico", segments_ttl => 2}, + emqx_ft_content_gen:new({?LINE, S2 = 456}, SS2 = 64) + }, + { + {<<"client1337">>, <<"file-🦀"/utf8>>}, + #{name => "dog.jpg", segments_ttl => 3000}, + emqx_ft_content_gen:new({?LINE, S3 = 7890}, SS3 = 128) + }, + { + {<<"client31337">>, <<"file-⏳"/utf8>>}, + #{name => "dog.jpg"}, + emqx_ft_content_gen:new({?LINE, S4 = 1230}, SS4 = 256) + } + ], + % 1. Start transfers, send all the segments but don't trigger completion. + _ = emqx_utils:pmap(fun(Transfer) -> start_transfer(Storage, Transfer) end, Transfers), + % 2. Enable periodic GC every 0.5 seconds. + ok = set_gc_config(interval, 500), + ok = emqx_ft_storage_fs_gc:reset(), + % 3. First we need the first transfer to be collected. + {ok, _} = ?block_until( + #{ + ?snk_kind := garbage_collection, + stats := #gcstats{ + files = Files, + directories = 4, + space = Space + } + } when Files == (?NSEGS(S1, SS1)) andalso Space > S1, + 5000, + 0 + ), + % 4. Then the second one. + {ok, _} = ?block_until( + #{ + ?snk_kind := garbage_collection, + stats := #gcstats{ + files = Files, + directories = 4, + space = Space + } + } when Files == (?NSEGS(S2, SS2)) andalso Space > S2, + 5000, + 0 + ), + % 5. Then transfers 3 and 4 because 3rd has too big TTL and 4th has no specific TTL. + {ok, _} = ?block_until( + #{ + ?snk_kind := garbage_collection, + stats := #gcstats{ + files = Files, + directories = 4 * 2, + space = Space + } + } when Files == (?NSEGS(S3, SS3) + ?NSEGS(S4, SS4)) andalso Space > S3 + S4, + 5000, + 0 + ). + +t_gc_handling_errors(_Config) -> + ok = set_gc_config(minimum_segments_ttl, 0), + ok = set_gc_config(maximum_segments_ttl, 0), + {local, Storage} = emqx_ft_storage:backend(), + Transfer1 = {<<"client1">>, mk_file_id()}, + Transfer2 = {<<"client2">>, mk_file_id()}, + Filemeta = #{name => "oops.pdf"}, + Size = 420, + SegSize = 16, + _ = start_transfer( + Storage, + {Transfer1, Filemeta, emqx_ft_content_gen:new({?LINE, Size}, SegSize)} + ), + _ = start_transfer( + Storage, + {Transfer2, Filemeta, emqx_ft_content_gen:new({?LINE, Size}, SegSize)} + ), + % 1. Throw some chaos in the transfer directory. + DirFragment1 = emqx_ft_storage_fs:get_subdir(Storage, Transfer1, fragment), + DirTemporary1 = emqx_ft_storage_fs:get_subdir(Storage, Transfer1, temporary), + PathShadyLink = filename:join(DirTemporary1, "linked-here"), + ok = file:make_symlink(DirFragment1, PathShadyLink), + DirTransfer2 = emqx_ft_storage_fs:get_subdir(Storage, Transfer2), + PathTripUp = filename:join(DirTransfer2, "trip-up-here"), + ok = file:write_file(PathTripUp, <<"HAHA">>), + ok = timer:sleep(timer:seconds(1)), + % 2. Observe the errors are reported consistently. + ?check_trace( + ?assertMatch( + #gcstats{ + files = Files, + directories = 3, + space = Space, + errors = #{ + % NOTE: dangling symlink looks like `enoent` for some reason + {file, PathShadyLink} := {unexpected, _}, + {directory, DirTransfer2} := eexist + } + } when Files == ?NSEGS(Size, SegSize) * 2 andalso Space > Size * 2, + emqx_ft_storage_fs_gc:collect() + ), + fun(Trace) -> + ?assertMatch( + [ + #{ + errors := #{ + {file, PathShadyLink} := {unexpected, _}, + {directory, DirTransfer2} := eexist + } + } + ], + ?of_kind("garbage_collection_errors", Trace) + ) + end + ). + +%% + +set_gc_config(Name, Value) -> + emqx_config:put([file_transfer, storage, local, segments, gc, Name], Value). + +start_transfer(Storage, {Transfer, Meta, Gen}) -> + ?assertEqual( + ok, + emqx_ft_storage_fs:store_filemeta(Storage, Transfer, Meta) + ), + emqx_ft_content_gen:fold( + fun({Content, SegmentNum, #{chunk_size := SegmentSize}}, _Transferred) -> + Offset = (SegmentNum - 1) * SegmentSize, + ?assertEqual( + ok, + emqx_ft_storage_fs:store_segment(Storage, Transfer, {Offset, Content}) + ), + Offset + byte_size(Content) + end, + 0, + Gen + ). + +complete_transfer(Storage, Transfer, Size) -> + complete_transfer(Storage, Transfer, Size, 100). + +complete_transfer(Storage, Transfer, Size, Timeout) -> + {async, Pid} = emqx_ft_storage_fs:assemble(Storage, Transfer, Size), + MRef = erlang:monitor(process, Pid), + Pid ! kickoff, + receive + {'DOWN', MRef, process, Pid, {shutdown, Result}} -> + Result + after Timeout -> + ct:fail("Assembler did not finish in time") + end. + +mk_file_id() -> + emqx_guid:to_hexstr(emqx_guid:gen()). diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_reader_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_reader_SUITE.erl new file mode 100644 index 000000000..217205f6f --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_reader_SUITE.erl @@ -0,0 +1,153 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_storage_fs_reader_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("stdlib/include/assert.hrl"). + +all() -> emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + ok = emqx_common_test_helpers:start_apps([emqx_ft], emqx_ft_test_helpers:env_handler(Config)), + Config. + +end_per_suite(_Config) -> + ok = emqx_common_test_helpers:stop_apps([emqx_ft]), + ok. + +init_per_testcase(_Case, Config) -> + file:make_dir(?config(data_dir, Config)), + Data = <<"hello world">>, + Path = expand_path(Config, "test_file"), + ok = mk_test_file(Path, Data), + [{path, Path} | Config]. + +end_per_testcase(_Case, _Config) -> + ok. + +t_successful_read(Config) -> + Path = ?config(path, Config), + + {ok, ReaderPid} = emqx_ft_storage_fs_reader:start_link(self(), Path), + ?assertEqual( + {ok, <<"hello ">>}, + emqx_ft_storage_fs_reader:read(ReaderPid, 6) + ), + ?assertEqual( + {ok, <<"world">>}, + emqx_ft_storage_fs_reader:read(ReaderPid, 6) + ), + ?assertEqual( + eof, + emqx_ft_storage_fs_reader:read(ReaderPid, 6) + ), + ?assertNot(is_process_alive(ReaderPid)). + +t_caller_dead(Config) -> + erlang:process_flag(trap_exit, true), + + Path = ?config(path, Config), + + CallerPid = spawn_link( + fun() -> + receive + stop -> ok + end + end + ), + {ok, ReaderPid} = emqx_ft_storage_fs_reader:start_link(CallerPid, Path), + _ = erlang:monitor(process, ReaderPid), + ?assertEqual( + {ok, <<"hello ">>}, + emqx_ft_storage_fs_reader:read(ReaderPid, 6) + ), + CallerPid ! stop, + receive + {'DOWN', _, process, ReaderPid, _} -> ok + after 1000 -> + ct:fail("Reader process did not die") + end. + +t_tables(Config) -> + Path = ?config(path, Config), + + {ok, ReaderPid0} = emqx_ft_storage_fs_reader:start_link(self(), Path), + + ReaderQH0 = emqx_ft_storage_fs_reader:table(ReaderPid0, 6), + ?assertEqual( + [<<"hello ">>, <<"world">>], + qlc:eval(ReaderQH0) + ), + + {ok, ReaderPid1} = emqx_ft_storage_fs_reader:start_link(self(), Path), + + ReaderQH1 = emqx_ft_storage_fs_reader:table(ReaderPid1), + ?assertEqual( + [<<"hello world">>], + qlc:eval(ReaderQH1) + ). + +t_bad_messages(Config) -> + Path = ?config(path, Config), + + {ok, ReaderPid} = emqx_ft_storage_fs_reader:start_link(self(), Path), + + ReaderPid ! {bad, message}, + gen_server:cast(ReaderPid, {bad, message}), + + ?assertEqual( + {error, {bad_call, {bad, message}}}, + gen_server:call(ReaderPid, {bad, message}) + ). + +t_nonexistent_file(_Config) -> + ?assertEqual( + {error, enoent}, + emqx_ft_storage_fs_reader:start_link(self(), "/a/b/c/bar") + ). + +t_start_supervised(Config) -> + Path = ?config(path, Config), + + {ok, ReaderPid} = emqx_ft_storage_fs_reader:start_supervised(self(), Path), + ?assertEqual( + {ok, <<"hello ">>}, + emqx_ft_storage_fs_reader:read(ReaderPid, 6) + ). + +t_rpc_error(_Config) -> + ReaderQH = emqx_ft_storage_fs_reader:table(fake_remote_pid('dummy@127.0.0.1'), 6), + ?assertEqual( + [], + qlc:eval(ReaderQH) + ). + +mk_test_file(Path, Data) -> + ok = file:write_file(Path, Data). + +expand_path(Config, Filename) -> + filename:join([?config(data_dir, Config), Filename]). + +%% This is a hack to create a pid that is not registered on the local node. +%% https://www.erlang.org/doc/apps/erts/erl_ext_dist.html#new_pid_ext +fake_remote_pid(Node) -> + <<131, NodeAtom/binary>> = term_to_binary(Node), + PidBin = <<131, 88, NodeAtom/binary, 1:32/big, 1:32/big, 1:32/big>>, + binary_to_term(PidBin). diff --git a/apps/emqx_ft/test/emqx_ft_test_helpers.erl b/apps/emqx_ft/test/emqx_ft_test_helpers.erl new file mode 100644 index 000000000..2eb6d84db --- /dev/null +++ b/apps/emqx_ft/test/emqx_ft_test_helpers.erl @@ -0,0 +1,128 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ft_test_helpers). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("common_test/include/ct.hrl"). + +-define(S3_HOST, <<"minio">>). +-define(S3_PORT, 9000). + +start_additional_node(Config, Name) -> + emqx_common_test_helpers:start_slave( + Name, + [ + {apps, [emqx_ft]}, + {join_to, node()}, + {configure_gen_rpc, true}, + {env_handler, env_handler(Config)} + ] + ). + +stop_additional_node(Node) -> + ok = rpc:call(Node, ekka, leave, []), + ok = rpc:call(Node, emqx_common_test_helpers, stop_apps, [[emqx_ft]]), + ok = emqx_common_test_helpers:stop_slave(Node), + ok. + +env_handler(Config) -> + fun + (emqx_ft) -> + load_config(#{<<"enable">> => true, <<"storage">> => local_storage(Config)}); + (_) -> + ok + end. + +local_storage(Config) -> + local_storage(Config, #{exporter => local}). + +local_storage(Config, Opts) -> + #{ + <<"local">> => #{ + <<"segments">> => #{<<"root">> => root(Config, node(), [segments])}, + <<"exporter">> => exporter(Config, Opts) + } + }. + +exporter(Config, #{exporter := local}) -> + #{<<"local">> => #{<<"root">> => root(Config, node(), [exports])}}; +exporter(_Config, #{exporter := s3, bucket_name := BucketName}) -> + BaseConfig = emqx_s3_test_helpers:base_raw_config(tcp), + #{ + <<"s3">> => BaseConfig#{ + <<"bucket">> => list_to_binary(BucketName), + <<"host">> => ?S3_HOST, + <<"port">> => ?S3_PORT + } + }. + +load_config(Config) -> + emqx_common_test_helpers:load_config(emqx_ft_schema, #{<<"file_transfer">> => Config}). + +tcp_port(Node) -> + {_, Port} = rpc:call(Node, emqx_config, get, [[listeners, tcp, default, bind]]), + Port. + +root(Config, Node, Tail) -> + iolist_to_binary(filename:join([?config(priv_dir, Config), "file_transfer", Node | Tail])). + +start_client(ClientId) -> + start_client(ClientId, node()). + +start_client(ClientId, Node) -> + Port = tcp_port(Node), + {ok, Client} = emqtt:start_link([{proto_ver, v5}, {clientid, ClientId}, {port, Port}]), + {ok, _} = emqtt:connect(Client), + Client. + +upload_file(ClientId, FileId, Name, Data) -> + upload_file(ClientId, FileId, Name, Data, node()). + +upload_file(ClientId, FileId, Name, Data, Node) -> + C1 = start_client(ClientId, Node), + + Size = byte_size(Data), + Meta = #{ + name => Name, + expire_at => erlang:system_time(_Unit = second) + 3600, + size => Size + }, + MetaPayload = emqx_utils_json:encode(emqx_ft:encode_filemeta(Meta)), + + ct:pal("MetaPayload = ~ts", [MetaPayload]), + + MetaTopic = <<"$file/", FileId/binary, "/init">>, + {ok, #{reason_code_name := success}} = emqtt:publish(C1, MetaTopic, MetaPayload, 1), + {ok, #{reason_code_name := success}} = emqtt:publish( + C1, <<"$file/", FileId/binary, "/0">>, Data, 1 + ), + + FinTopic = <<"$file/", FileId/binary, "/fin/", (integer_to_binary(Size))/binary>>, + FinResult = + case emqtt:publish(C1, FinTopic, <<>>, 1) of + {ok, #{reason_code_name := success}} -> + ok; + {ok, #{reason_code_name := Error}} -> + {error, Error} + end, + ok = emqtt:stop(C1), + FinResult. + +aws_config() -> + emqx_s3_test_helpers:aws_config(tcp, binary_to_list(?S3_HOST), ?S3_PORT). diff --git a/apps/emqx_ft/test/props/prop_emqx_ft_assembly.erl b/apps/emqx_ft/test/props/prop_emqx_ft_assembly.erl new file mode 100644 index 000000000..9a2b7df0b --- /dev/null +++ b/apps/emqx_ft/test/props/prop_emqx_ft_assembly.erl @@ -0,0 +1,221 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(prop_emqx_ft_assembly). + +-include_lib("proper/include/proper.hrl"). + +-import(emqx_proper_types, [scaled/2]). + +-define(COVERAGE_TIMEOUT, 5000). + +prop_coverage() -> + ?FORALL( + {Filesize, Segsizes}, + {filesize_t(), segsizes_t()}, + ?FORALL( + Fragments, + noshrink(segments_t(Filesize, Segsizes)), + ?TIMEOUT( + ?COVERAGE_TIMEOUT, + begin + ASM1 = append_segments(mk_assembly(Filesize), Fragments), + {Time, ASM2} = timer:tc(emqx_ft_assembly, update, [ASM1]), + measure( + #{"Fragments" => length(Fragments), "Time" => Time}, + case emqx_ft_assembly:status(ASM2) of + complete -> + Coverage = emqx_ft_assembly:coverage(ASM2), + measure( + #{"CoverageLength" => length(Coverage)}, + is_coverage_complete(Coverage) + ); + {incomplete, {missing, {segment, _, _}}} -> + measure("CoverageLength", 0, true) + end + ) + end + ) + ) + ). + +prop_coverage_likely_incomplete() -> + ?FORALL( + {Filesize, Segsizes, Hole}, + {filesize_t(), segsizes_t(), filesize_t()}, + ?FORALL( + Fragments, + noshrink(segments_t(Filesize, Segsizes, Hole)), + ?TIMEOUT( + ?COVERAGE_TIMEOUT, + begin + ASM1 = append_segments(mk_assembly(Filesize), Fragments), + {Time, ASM2} = timer:tc(emqx_ft_assembly, update, [ASM1]), + measure( + #{"Fragments" => length(Fragments), "Time" => Time}, + case emqx_ft_assembly:status(ASM2) of + complete -> + % NOTE: this is still possible due to the nature of `SUCHTHATMAYBE` + IsComplete = emqx_ft_assembly:coverage(ASM2), + collect(complete, is_coverage_complete(IsComplete)); + {incomplete, {missing, {segment, _, _}}} -> + collect(incomplete, true) + end + ) + end + ) + ) + ). + +prop_coverage_complete() -> + ?FORALL( + {Filesize, Segsizes}, + {filesize_t(), ?SUCHTHAT([BaseSegsize | _], segsizes_t(), BaseSegsize > 0)}, + ?FORALL( + {Fragments, RemoteNode}, + noshrink({segments_t(Filesize, Segsizes), remote_node_t()}), + begin + % Ensure that we have complete coverage + ASM1 = mk_assembly(Filesize), + ASM2 = append_coverage(ASM1, RemoteNode, Filesize, Segsizes), + ASM3 = append_segments(ASM2, Fragments), + {Time, ASM4} = timer:tc(emqx_ft_assembly, update, [ASM3]), + measure( + #{"CoverageMax" => nsegs(Filesize, Segsizes), "Time" => Time}, + case emqx_ft_assembly:status(ASM4) of + complete -> + Coverage = emqx_ft_assembly:coverage(ASM4), + measure( + #{"Coverage" => length(Coverage)}, + is_coverage_complete(Coverage) + ); + {incomplete, _} -> + false + end + ) + end + ) + ). + +measure(NamedSamples, Test) -> + maps:fold(fun(Name, Sample, Acc) -> measure(Name, Sample, Acc) end, Test, NamedSamples). + +is_coverage_complete([]) -> + true; +is_coverage_complete(Coverage = [_ | Tail]) -> + is_coverage_complete(Coverage, Tail). + +is_coverage_complete([_], []) -> + true; +is_coverage_complete( + [{_Node1, #{fragment := {segment, #{offset := O1, size := S1}}}} | Rest], + [{_Node2, #{fragment := {segment, #{offset := O2}}}} | Tail] +) -> + (O1 + S1 == O2) andalso is_coverage_complete(Rest, Tail). + +mk_assembly(Filesize) -> + emqx_ft_assembly:append(emqx_ft_assembly:new(Filesize), node(), mk_filemeta(Filesize)). + +append_segments(ASMIn, Fragments) -> + lists:foldl( + fun({Node, {Offset, Size}}, ASM) -> + emqx_ft_assembly:append(ASM, Node, mk_segment(Offset, Size)) + end, + ASMIn, + Fragments + ). + +append_coverage(ASM, Node, Filesize, Segsizes = [BaseSegsize | _]) -> + append_coverage(ASM, Node, Filesize, BaseSegsize, 0, nsegs(Filesize, Segsizes)). + +append_coverage(ASM, Node, Filesize, Segsize, I, NSegs) when I < NSegs -> + Offset = I * Segsize, + Size = min(Segsize, Filesize - Offset), + ASMNext = emqx_ft_assembly:append(ASM, Node, mk_segment(Offset, Size)), + append_coverage(ASMNext, Node, Filesize, Segsize, I + 1, NSegs); +append_coverage(ASM, _Node, _Filesize, _Segsize, _, _NSegs) -> + ASM. + +mk_filemeta(Filesize) -> + #{ + path => "MANIFEST.json", + fragment => {filemeta, #{name => ?MODULE_STRING, size => Filesize}} + }. + +mk_segment(Offset, Size) -> + #{ + path => "SEG" ++ integer_to_list(Offset) ++ integer_to_list(Size), + fragment => {segment, #{offset => Offset, size => Size}} + }. + +nsegs(Filesize, [BaseSegsize | _]) -> + Filesize div max(1, BaseSegsize) + 1. + +segments_t(Filesize, Segsizes) -> + scaled(nsegs(Filesize, Segsizes), list({node_t(), segment_t(Filesize, Segsizes)})). + +segments_t(Filesize, Segsizes, Hole) -> + scaled(nsegs(Filesize, Segsizes), list({node_t(), segment_t(Filesize, Segsizes, Hole)})). + +segment_t(Filesize, Segsizes, Hole) -> + ?SUCHTHATMAYBE( + {Offset, Size}, + segment_t(Filesize, Segsizes), + (Hole rem Filesize) =< Offset orelse (Hole rem Filesize) > (Offset + Size) + ). + +segment_t(Filesize, Segsizes) -> + ?LET( + Segsize, + oneof(Segsizes), + ?LET( + Index, + range(0, Filesize div max(1, Segsize)), + {Index * Segsize, min(Segsize, Filesize - (Index * Segsize))} + ) + ). + +filesize_t() -> + scaled(2500, non_neg_integer()). + +segsizes_t() -> + ?LET( + BaseSize, + segsize_t(), + oneof([ + [BaseSize, BaseSize * 2], + [BaseSize, BaseSize * 2, BaseSize * 3], + [BaseSize, BaseSize * 2, BaseSize * 5] + ]) + ). + +segsize_t() -> + scaled(50, non_neg_integer()). + +remote_node_t() -> + oneof([ + 'emqx42@emqx.local', + 'emqx43@emqx.local', + 'emqx44@emqx.local' + ]). + +node_t() -> + oneof([ + node(), + 'emqx42@emqx.local', + 'emqx43@emqx.local', + 'emqx44@emqx.local' + ]). diff --git a/apps/emqx_machine/src/emqx_machine.app.src b/apps/emqx_machine/src/emqx_machine.app.src index a44d2b36e..7cf0e4b53 100644 --- a/apps/emqx_machine/src/emqx_machine.app.src +++ b/apps/emqx_machine/src/emqx_machine.app.src @@ -3,7 +3,7 @@ {id, "emqx_machine"}, {description, "The EMQX Machine"}, % strict semver, bump manually! - {vsn, "0.2.3"}, + {vsn, "0.2.4"}, {modules, []}, {registered, []}, {applications, [kernel, stdlib, emqx_ctl]}, diff --git a/apps/emqx_machine/src/emqx_machine_boot.erl b/apps/emqx_machine/src/emqx_machine_boot.erl index e3f84079b..266855877 100644 --- a/apps/emqx_machine/src/emqx_machine_boot.erl +++ b/apps/emqx_machine/src/emqx_machine_boot.erl @@ -26,7 +26,7 @@ -dialyzer({no_match, [basic_reboot_apps/0]}). -ifdef(TEST). --export([sorted_reboot_apps/1]). +-export([sorted_reboot_apps/1, reboot_apps/0]). -endif. %% these apps are always (re)started by emqx_machine @@ -120,7 +120,7 @@ restart_type(App) -> %% the list of (re)started apps depends on release type/edition reboot_apps() -> - {ok, ConfigApps0} = application:get_env(emqx_machine, applications), + ConfigApps0 = application:get_env(emqx_machine, applications, []), BaseRebootApps = basic_reboot_apps(), ConfigApps = lists:filter(fun(App) -> not lists:member(App, BaseRebootApps) end, ConfigApps0), BaseRebootApps ++ ConfigApps. @@ -154,6 +154,8 @@ basic_reboot_apps() -> ee -> CE ++ [ + emqx_s3, + emqx_ft, emqx_eviction_agent, emqx_node_rebalance ] diff --git a/apps/emqx_management/src/emqx_mgmt_api_api_keys.erl b/apps/emqx_management/src/emqx_mgmt_api_api_keys.erl index c39b11273..ba21adaa5 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_api_keys.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_api_keys.erl @@ -18,6 +18,7 @@ -behaviour(minirest_api). -include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). -export([api_spec/0, fields/1, paths/0, schema/1, namespace/0]). -export([api_key/2, api_key_by_name/2]). @@ -36,14 +37,14 @@ schema("/api_key") -> #{ 'operationId' => api_key, get => #{ - description => "Return api_key list", + description => ?DESC(api_key_list), tags => ?TAGS, responses => #{ 200 => delete([api_secret], fields(app)) } }, post => #{ - description => "Create new api_key", + description => ?DESC(create_new_api_key), tags => ?TAGS, 'requestBody' => delete([created_at, api_key, api_secret], fields(app)), responses => #{ @@ -56,7 +57,7 @@ schema("/api_key/:name") -> #{ 'operationId' => api_key_by_name, get => #{ - description => "Return the specific api_key", + description => ?DESC(get_api_key), tags => ?TAGS, parameters => [hoconsc:ref(name)], responses => #{ @@ -65,7 +66,7 @@ schema("/api_key/:name") -> } }, put => #{ - description => "Update the specific api_key", + description => ?DESC(update_api_key), tags => ?TAGS, parameters => [hoconsc:ref(name)], 'requestBody' => delete([created_at, api_key, api_secret, name], fields(app)), @@ -75,7 +76,7 @@ schema("/api_key/:name") -> } }, delete => #{ - description => "Delete the specific api_key", + description => ?DESC(delete_api_key), tags => ?TAGS, parameters => [hoconsc:ref(name)], responses => #{ diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl index 846a1d466..7720ab66f 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl @@ -20,6 +20,7 @@ -include_lib("typerefl/include/types.hrl"). -include_lib("emqx/include/emqx.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). -include_lib("emqx/include/logger.hrl"). @@ -101,7 +102,7 @@ schema("/clients") -> #{ 'operationId' => clients, get => #{ - description => <<"List clients">>, + description => ?DESC(list_clients), tags => ?TAGS, parameters => [ hoconsc:ref(emqx_dashboard_swagger, page), @@ -214,7 +215,7 @@ schema("/clients/:clientid") -> #{ 'operationId' => client, get => #{ - description => <<"Get clients info by client ID">>, + description => ?DESC(clients_info_from_id), tags => ?TAGS, parameters => [{clientid, hoconsc:mk(binary(), #{in => path})}], responses => #{ @@ -225,7 +226,7 @@ schema("/clients/:clientid") -> } }, delete => #{ - description => <<"Kick out client by client ID">>, + description => ?DESC(kick_client_id), tags => ?TAGS, parameters => [ {clientid, hoconsc:mk(binary(), #{in => path})} @@ -242,7 +243,7 @@ schema("/clients/:clientid/authorization/cache") -> #{ 'operationId' => authz_cache, get => #{ - description => <<"Get client authz cache in the cluster.">>, + description => ?DESC(get_authz_cache), tags => ?TAGS, parameters => [{clientid, hoconsc:mk(binary(), #{in => path})}], responses => #{ @@ -253,7 +254,7 @@ schema("/clients/:clientid/authorization/cache") -> } }, delete => #{ - description => <<"Clean client authz cache in the cluster.">>, + description => ?DESC(clean_authz_cache), tags => ?TAGS, parameters => [{clientid, hoconsc:mk(binary(), #{in => path})}], responses => #{ @@ -268,7 +269,7 @@ schema("/clients/:clientid/subscriptions") -> #{ 'operationId' => subscriptions, get => #{ - description => <<"Get client subscriptions">>, + description => ?DESC(get_client_subs), tags => ?TAGS, parameters => [{clientid, hoconsc:mk(binary(), #{in => path})}], responses => #{ @@ -285,7 +286,7 @@ schema("/clients/:clientid/subscribe") -> #{ 'operationId' => subscribe, post => #{ - description => <<"Subscribe">>, + description => ?DESC(subscribe), tags => ?TAGS, parameters => [{clientid, hoconsc:mk(binary(), #{in => path})}], 'requestBody' => hoconsc:mk(hoconsc:ref(?MODULE, subscribe)), @@ -301,7 +302,7 @@ schema("/clients/:clientid/subscribe/bulk") -> #{ 'operationId' => subscribe_batch, post => #{ - description => <<"Subscribe">>, + description => ?DESC(subscribe_g), tags => ?TAGS, parameters => [{clientid, hoconsc:mk(binary(), #{in => path})}], 'requestBody' => hoconsc:mk(hoconsc:array(hoconsc:ref(?MODULE, subscribe))), @@ -317,7 +318,7 @@ schema("/clients/:clientid/unsubscribe") -> #{ 'operationId' => unsubscribe, post => #{ - description => <<"Unsubscribe">>, + description => ?DESC(unsubscribe), tags => ?TAGS, parameters => [{clientid, hoconsc:mk(binary(), #{in => path})}], 'requestBody' => hoconsc:mk(hoconsc:ref(?MODULE, unsubscribe)), @@ -333,7 +334,7 @@ schema("/clients/:clientid/unsubscribe/bulk") -> #{ 'operationId' => unsubscribe_batch, post => #{ - description => <<"Unsubscribe">>, + description => ?DESC(unsubscribe_g), tags => ?TAGS, parameters => [{clientid, hoconsc:mk(binary(), #{in => path})}], 'requestBody' => hoconsc:mk(hoconsc:array(hoconsc:ref(?MODULE, unsubscribe))), @@ -349,7 +350,7 @@ schema("/clients/:clientid/keepalive") -> #{ 'operationId' => set_keepalive, put => #{ - description => <<"Set the online client keepalive by seconds">>, + description => ?DESC(set_keepalive_seconds), tags => ?TAGS, parameters => [{clientid, hoconsc:mk(binary(), #{in => path})}], 'requestBody' => hoconsc:mk(hoconsc:ref(?MODULE, keepalive)), diff --git a/apps/emqx_management/src/emqx_mgmt_api_cluster.erl b/apps/emqx_management/src/emqx_mgmt_api_cluster.erl index e74b6c362..ba185e369 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_cluster.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_cluster.erl @@ -39,7 +39,7 @@ schema("/cluster") -> #{ 'operationId' => cluster_info, get => #{ - description => "Get cluster info", + desc => ?DESC(get_cluster_info), tags => [<<"Cluster">>], responses => #{ 200 => [ @@ -54,7 +54,7 @@ schema("/cluster/:node/invite") -> #{ 'operationId' => invite_node, put => #{ - description => "Invite node to cluster", + desc => ?DESC(invite_node), tags => [<<"Cluster">>], parameters => [hoconsc:ref(node)], responses => #{ @@ -67,7 +67,7 @@ schema("/cluster/:node/force_leave") -> #{ 'operationId' => force_leave, delete => #{ - description => "Force leave node from cluster", + desc => ?DESC(force_remove_node), tags => [<<"Cluster">>], parameters => [hoconsc:ref(node)], responses => #{ diff --git a/apps/emqx_management/src/emqx_mgmt_api_configs.erl b/apps/emqx_management/src/emqx_mgmt_api_configs.erl index c8b21449e..faff589eb 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_configs.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_configs.erl @@ -67,8 +67,7 @@ schema("/configs") -> 'operationId' => configs, get => #{ tags => ?TAGS, - description => - <<"Get all the configurations of the specified node, including hot and non-hot updatable items.">>, + description => ?DESC(get_conf_node), parameters => [ {node, hoconsc:mk( @@ -77,8 +76,7 @@ schema("/configs") -> in => query, required => false, example => <<"emqx@127.0.0.1">>, - desc => - <<"Node's name: If you do not fill in the fields, this node will be used by default.">> + description => ?DESC(node_name) } )} ], @@ -95,12 +93,7 @@ schema("/configs_reset/:rootname") -> 'operationId' => config_reset, post => #{ tags => ?TAGS, - description => - << - "Reset the config entry specified by the query string parameter `conf_path`.
" - "- For a config entry that has default value, this resets it to the default value;\n" - "- For a config entry that has no default value, an error 400 will be returned" - >>, + description => ?DESC(rest_conf_query), %% We only return "200" rather than the new configs that has been changed, as %% the schema of the changed configs is depends on the request parameter %% `conf_path`, it cannot be defined here. @@ -134,12 +127,12 @@ schema("/configs/global_zone") -> 'operationId' => global_zone_configs, get => #{ tags => ?TAGS, - description => <<"Get the global zone configs">>, + description => ?DESC(get_global_zone_configs), responses => #{200 => Schema} }, put => #{ tags => ?TAGS, - description => <<"Update globbal zone configs">>, + description => ?DESC(update_globar_zone_configs), 'requestBody' => Schema, responses => #{ 200 => Schema, @@ -153,7 +146,7 @@ schema("/configs/limiter") -> 'operationId' => limiter, get => #{ tags => ?TAGS, - description => <<"Get the node-level limiter configs">>, + description => ?DESC(get_node_level_limiter_congigs), responses => #{ 200 => hoconsc:mk(hoconsc:ref(emqx_limiter_schema, limiter)), 404 => emqx_dashboard_swagger:error_codes(['NOT_FOUND'], <<"config not found">>) @@ -161,7 +154,7 @@ schema("/configs/limiter") -> }, put => #{ tags => ?TAGS, - description => <<"Update the node-level limiter configs">>, + description => ?DESC(update_node_level_limiter_congigs), 'requestBody' => hoconsc:mk(hoconsc:ref(emqx_limiter_schema, limiter)), responses => #{ 200 => hoconsc:mk(hoconsc:ref(emqx_limiter_schema, limiter)), @@ -172,15 +165,22 @@ schema("/configs/limiter") -> }; schema(Path) -> {RootKey, {_Root, Schema}} = find_schema(Path), + GetDesc = iolist_to_binary([ + <<"Get the sub-configurations under *">>, + RootKey, + <<"*">> + ]), + PutDesc = iolist_to_binary([ + <<"Update the sub-configurations under *">>, + RootKey, + <<"*">> + ]), #{ 'operationId' => config, get => #{ tags => ?TAGS, - description => iolist_to_binary([ - <<"Get the sub-configurations under *">>, - RootKey, - <<"*">> - ]), + desc => GetDesc, + summary => GetDesc, responses => #{ 200 => Schema, 404 => emqx_dashboard_swagger:error_codes(['NOT_FOUND'], <<"config not found">>) @@ -188,11 +188,8 @@ schema(Path) -> }, put => #{ tags => ?TAGS, - description => iolist_to_binary([ - <<"Update the sub-configurations under *">>, - RootKey, - <<"*">> - ]), + desc => PutDesc, + summary => PutDesc, 'requestBody' => Schema, responses => #{ 200 => Schema, diff --git a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl index 152ccc599..1f1dda5f2 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl @@ -66,7 +66,7 @@ schema("/listeners_status") -> 'operationId' => listener_type_status, get => #{ tags => [<<"listeners">>], - desc => <<"List all running node's listeners live status. group by listener type">>, + description => ?DESC(list_node_live_statuses), responses => #{ 200 => emqx_dashboard_swagger:schema_with_example( @@ -81,7 +81,7 @@ schema("/listeners") -> 'operationId' => list_listeners, get => #{ tags => [<<"listeners">>], - desc => <<"List all running node's listeners for the specified type.">>, + description => ?DESC(list_listeners), parameters => [ {type, ?HOCON( @@ -99,7 +99,7 @@ schema("/listeners") -> }, post => #{ tags => [<<"listeners">>], - desc => <<"Create the specified listener on all nodes.">>, + description => ?DESC(create_on_all_nodes), parameters => [], 'requestBody' => create_listener_schema(#{bind => true}), responses => #{ @@ -113,7 +113,7 @@ schema("/listeners/:id") -> 'operationId' => crud_listeners_by_id, get => #{ tags => [<<"listeners">>], - desc => <<"List all running node's listeners for the specified id.">>, + description => ?DESC(list_by_id), parameters => [?R_REF(listener_id)], responses => #{ 200 => listener_schema(#{bind => true}), @@ -122,7 +122,7 @@ schema("/listeners/:id") -> }, put => #{ tags => [<<"listeners">>], - desc => <<"Update the specified listener on all nodes.">>, + description => ?DESC(update_lisener), parameters => [?R_REF(listener_id)], 'requestBody' => listener_schema(#{bind => false}), responses => #{ @@ -133,7 +133,7 @@ schema("/listeners/:id") -> }, post => #{ tags => [<<"listeners">>], - desc => <<"Create the specified listener on all nodes.">>, + description => ?DESC(create_on_all_nodes), parameters => [?R_REF(listener_id)], 'requestBody' => listener_schema(#{bind => true}), responses => #{ @@ -144,7 +144,7 @@ schema("/listeners/:id") -> }, delete => #{ tags => [<<"listeners">>], - desc => <<"Delete the specified listener on all nodes.">>, + description => ?DESC(delete_on_all_nodes), parameters => [?R_REF(listener_id)], responses => #{ 204 => <<"Listener deleted">>, @@ -157,7 +157,7 @@ schema("/listeners/:id/start") -> 'operationId' => start_listeners_by_id, post => #{ tags => [<<"listeners">>], - desc => <<"Start the listener on all nodes.">>, + description => ?DESC(start_on_all_nodes), parameters => [ ?R_REF(listener_id) ], @@ -172,7 +172,7 @@ schema("/listeners/:id/stop") -> 'operationId' => stop_listeners_by_id, post => #{ tags => [<<"listeners">>], - desc => <<"Stop the listener on all nodes.">>, + description => ?DESC(stop_on_all_nodes), parameters => [ ?R_REF(listener_id) ], @@ -187,7 +187,7 @@ schema("/listeners/:id/restart") -> 'operationId' => restart_listeners_by_id, post => #{ tags => [<<"listeners">>], - desc => <<"Restart listeners on all nodes.">>, + description => ?DESC(restart_on_all_nodes), parameters => [ ?R_REF(listener_id) ], diff --git a/apps/emqx_management/src/emqx_mgmt_api_metrics.erl b/apps/emqx_management/src/emqx_mgmt_api_metrics.erl index 0fcc45d8e..7ad0777c7 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_metrics.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_metrics.erl @@ -19,6 +19,7 @@ -behaviour(minirest_api). -include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hocon_types.hrl"). -import(hoconsc, [mk/2, ref/2]). @@ -73,7 +74,7 @@ schema("/metrics") -> 'operationId' => metrics, get => #{ - description => <<"EMQX metrics">>, + description => ?DESC(emqx_metrics), tags => [<<"Metrics">>], parameters => [ diff --git a/apps/emqx_management/src/emqx_mgmt_api_nodes.erl b/apps/emqx_management/src/emqx_mgmt_api_nodes.erl index ecf465f43..827fea44a 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_nodes.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_nodes.erl @@ -18,6 +18,7 @@ -behaviour(minirest_api). -include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). -import(hoconsc, [mk/2, ref/1, ref/2, enum/1, array/1]). @@ -60,7 +61,7 @@ schema("/nodes") -> 'operationId' => nodes, get => #{ - description => <<"List EMQX nodes">>, + description => ?DESC(list_nodes), tags => [<<"Nodes">>], responses => #{ @@ -76,7 +77,7 @@ schema("/nodes/:node") -> 'operationId' => node, get => #{ - description => <<"Get node info">>, + description => ?DESC(get_node_info), tags => [<<"Nodes">>], parameters => [ref(node_name)], responses => @@ -94,7 +95,7 @@ schema("/nodes/:node/metrics") -> 'operationId' => node_metrics, get => #{ - description => <<"Get node metrics">>, + description => ?DESC(get_node_metrics), tags => [<<"Nodes">>], parameters => [ref(node_name)], responses => @@ -112,7 +113,7 @@ schema("/nodes/:node/stats") -> 'operationId' => node_stats, get => #{ - description => <<"Get node stats">>, + description => ?DESC(get_node_stats), tags => [<<"Nodes">>], parameters => [ref(node_name)], responses => diff --git a/apps/emqx_management/src/emqx_mgmt_api_stats.erl b/apps/emqx_management/src/emqx_mgmt_api_stats.erl index 5f4bbce65..b57565671 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_stats.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_stats.erl @@ -18,6 +18,7 @@ -behaviour(minirest_api). -include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). -import( hoconsc, @@ -49,7 +50,7 @@ schema("/stats") -> 'operationId' => list, get => #{ - description => <<"EMQX stats">>, + description => ?DESC(emqx_stats), tags => [<<"Metrics">>], parameters => [ref(aggregate)], responses => diff --git a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl index 1b69835f9..08a90e623 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl @@ -21,6 +21,7 @@ -include_lib("typerefl/include/types.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). -export([ api_spec/0, @@ -58,7 +59,7 @@ schema("/subscriptions") -> #{ 'operationId' => subscriptions, get => #{ - description => <<"List subscriptions">>, + description => ?DESC(list_subs), tags => [<<"Subscriptions">>], parameters => parameters(), responses => #{ diff --git a/apps/emqx_management/src/emqx_mgmt_api_topics.erl b/apps/emqx_management/src/emqx_mgmt_api_topics.erl index 4100269e5..6b0e1f622 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_topics.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_topics.erl @@ -18,6 +18,7 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). %% API -behaviour(minirest_api). @@ -51,7 +52,7 @@ schema("/topics") -> #{ 'operationId' => topics, get => #{ - description => <<"Topics list">>, + description => ?DESC(topic_list), tags => ?TAGS, parameters => [ topic_param(query), @@ -71,7 +72,7 @@ schema("/topics/:topic") -> #{ 'operationId' => topic, get => #{ - description => <<"Lookup topic info by name">>, + description => ?DESC(topic_info_by_name), tags => ?TAGS, parameters => [topic_param(path)], responses => #{ diff --git a/apps/emqx_management/src/emqx_mgmt_api_trace.erl b/apps/emqx_management/src/emqx_mgmt_api_trace.erl index 25cc2734f..331c62681 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_trace.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_trace.erl @@ -21,6 +21,7 @@ -include_lib("typerefl/include/types.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). -export([ api_spec/0, @@ -73,14 +74,14 @@ schema("/trace") -> #{ 'operationId' => trace, get => #{ - description => "List all trace", + description => ?DESC(list_all), tags => ?TAGS, responses => #{ 200 => hoconsc:array(hoconsc:ref(trace)) } }, post => #{ - description => "Create new trace", + description => ?DESC(create_new), tags => ?TAGS, 'requestBody' => delete([status, log_size], fields(trace)), responses => #{ @@ -102,7 +103,7 @@ schema("/trace") -> } }, delete => #{ - description => "Clear all traces", + description => ?DESC(clear_all), tags => ?TAGS, responses => #{ 204 => <<"No Content">> @@ -113,7 +114,7 @@ schema("/trace/:name") -> #{ 'operationId' => delete_trace, delete => #{ - description => "Delete specified trace", + description => ?DESC(delete_trace), tags => ?TAGS, parameters => [hoconsc:ref(name)], responses => #{ @@ -126,7 +127,7 @@ schema("/trace/:name/stop") -> #{ 'operationId' => update_trace, put => #{ - description => "Stop trace by name", + description => ?DESC(stop_trace), tags => ?TAGS, parameters => [hoconsc:ref(name)], responses => #{ @@ -139,7 +140,7 @@ schema("/trace/:name/download") -> #{ 'operationId' => download_trace_log, get => #{ - description => "Download trace log by name", + description => ?DESC(download_log_by_name), tags => ?TAGS, parameters => [hoconsc:ref(name), hoconsc:ref(node)], responses => #{ @@ -161,7 +162,7 @@ schema("/trace/:name/log_detail") -> #{ 'operationId' => log_file_detail, get => #{ - description => "get trace log file's metadata, such as size, last update time", + description => ?DESC(get_trace_file_metadata), tags => ?TAGS, parameters => [hoconsc:ref(name)], responses => #{ @@ -174,7 +175,7 @@ schema("/trace/:name/log") -> #{ 'operationId' => stream_log_file, get => #{ - description => "view trace log", + description => ?DESC(view_trace_log), tags => ?TAGS, parameters => [ hoconsc:ref(name), @@ -204,9 +205,8 @@ schema("/trace/:name/log") -> fields(log_file_detail) -> fields(node) ++ [ - {size, hoconsc:mk(integer(), #{desc => "file size"})}, - {mtime, - hoconsc:mk(integer(), #{desc => "the modification and last access times of a file"})} + {size, hoconsc:mk(integer(), #{description => ?DESC(file_size)})}, + {mtime, hoconsc:mk(integer(), #{description => ?DESC(modification_date)})} ]; fields(trace) -> [ @@ -214,7 +214,7 @@ fields(trace) -> hoconsc:mk( binary(), #{ - desc => "Unique and format by [a-zA-Z0-9-_]", + description => ?DESC(format), validator => fun ?MODULE:validate_name/1, required => true, example => <<"EMQX-TRACE-1">> @@ -224,7 +224,7 @@ fields(trace) -> hoconsc:mk( hoconsc:enum([clientid, topic, ip_address]), #{ - desc => "" "Filter type" "", + description => ?DESC(filter_type), required => true, example => <<"clientid">> } @@ -233,7 +233,7 @@ fields(trace) -> hoconsc:mk( binary(), #{ - desc => "" "support mqtt wildcard topic." "", + description => ?DESC(support_wildcard), required => false, example => <<"/dev/#">> } @@ -242,7 +242,7 @@ fields(trace) -> hoconsc:mk( binary(), #{ - desc => "" "mqtt clientid." "", + description => ?DESC(mqtt_clientid), required => false, example => <<"dev-001">> } @@ -252,7 +252,7 @@ fields(trace) -> hoconsc:mk( binary(), #{ - desc => "client ip address", + description => ?DESC(client_ip_addess), required => false, example => <<"127.0.0.1">> } @@ -261,7 +261,7 @@ fields(trace) -> hoconsc:mk( hoconsc:enum([running, stopped, waiting]), #{ - desc => "trace status", + description => ?DESC(trace_status), required => false, example => running } @@ -283,7 +283,7 @@ fields(trace) -> hoconsc:mk( emqx_datetime:epoch_second(), #{ - desc => "rfc3339 timestamp or epoch second", + description => ?DESC(time_format), required => false, example => <<"2021-11-04T18:17:38+08:00">> } @@ -292,7 +292,7 @@ fields(trace) -> hoconsc:mk( emqx_datetime:epoch_second(), #{ - desc => "rfc3339 timestamp or epoch second", + description => ?DESC(time_format), required => false, example => <<"2021-11-05T18:17:38+08:00">> } @@ -301,7 +301,7 @@ fields(trace) -> hoconsc:mk( hoconsc:array(map()), #{ - desc => "trace log size", + description => ?DESC(trace_log_size), example => [#{<<"node">> => <<"emqx@127.0.0.1">>, <<"size">> => 1024}], required => false } @@ -326,7 +326,7 @@ fields(node) -> hoconsc:mk( binary(), #{ - desc => "Node name", + description => ?DESC(node_name), in => query, required => false, example => "emqx@127.0.0.1" @@ -341,7 +341,7 @@ fields(bytes) -> %% across different OS range(0, ?MAX_SINT32), #{ - desc => "Maximum number of bytes to send in response", + description => ?DESC(max_response_bytes), in => query, required => false, default => 1000, @@ -356,7 +356,7 @@ fields(position) -> hoconsc:mk( integer(), #{ - desc => "Offset from the current trace position.", + description => ?DESC(current_trace_offset), in => query, required => false, default => 0 diff --git a/apps/emqx_node_rebalance/etc/emqx_node_rebalance.conf b/apps/emqx_node_rebalance/etc/emqx_node_rebalance.conf index 8ace22435..e69de29bb 100644 --- a/apps/emqx_node_rebalance/etc/emqx_node_rebalance.conf +++ b/apps/emqx_node_rebalance/etc/emqx_node_rebalance.conf @@ -1,3 +0,0 @@ -##-------------------------------------------------------------------- -## EMQX Node Rebalance Plugin -##-------------------------------------------------------------------- diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src index bfd7e68fa..82a95c377 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_plugin_libs, [ {description, "EMQX Plugin utility libs"}, - {vsn, "4.3.10"}, + {vsn, "4.3.11"}, {modules, []}, {applications, [kernel, stdlib]}, {env, []} diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl b/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl index 9a4c01a2b..3bfac1ec4 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl @@ -32,7 +32,8 @@ proc_cql_param_str/2, split_insert_sql/1, detect_sql_type/1, - proc_batch_sql/3 + proc_batch_sql/3, + formalize_sql/1 ]). %% type converting @@ -126,7 +127,8 @@ proc_cql_param_str(Tokens, Data) -> -spec split_insert_sql(binary()) -> {ok, {InsertSQL, Params}} | {error, atom()} when InsertSQL :: binary(), Params :: binary(). -split_insert_sql(SQL) -> +split_insert_sql(SQL0) -> + SQL = formalize_sql(SQL0), case re:split(SQL, "((?i)values)", [{return, binary}]) of [Part1, _, Part3] -> case string:trim(Part1, leading) of @@ -173,6 +175,12 @@ proc_batch_sql(BatchReqs, InsertPart, Tokens) -> ), <>. +formalize_sql(Input) -> + %% 1. replace all whitespaces like '\r' '\n' or spaces to a single space char. + SQL = re:replace(Input, "\\s+", " ", [global, {return, binary}]), + %% 2. trims the result + string:trim(SQL). + unsafe_atom_key(Key) when is_atom(Key) -> Key; unsafe_atom_key(Key) when is_binary(Key) -> diff --git a/apps/emqx_prometheus/src/emqx_prometheus.app.src b/apps/emqx_prometheus/src/emqx_prometheus.app.src index f94b22d81..a0d4dee04 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.app.src +++ b/apps/emqx_prometheus/src/emqx_prometheus.app.src @@ -2,7 +2,7 @@ {application, emqx_prometheus, [ {description, "Prometheus for EMQX"}, % strict semver, bump manually! - {vsn, "5.0.10"}, + {vsn, "5.0.11"}, {modules, []}, {registered, [emqx_prometheus_sup]}, {applications, [kernel, stdlib, prometheus, emqx, emqx_management]}, diff --git a/apps/emqx_prometheus/src/emqx_prometheus_api.erl b/apps/emqx_prometheus/src/emqx_prometheus_api.erl index d3bfc0224..6fb57483d 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_api.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_api.erl @@ -19,6 +19,7 @@ -behaviour(minirest_api). -include("emqx_prometheus.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). -import(hoconsc, [ref/2]). @@ -50,14 +51,14 @@ schema("/prometheus") -> 'operationId' => prometheus, get => #{ - description => <<"Get Prometheus config info">>, + description => ?DESC(get_prom_conf_info), tags => ?TAGS, responses => #{200 => prometheus_config_schema()} }, put => #{ - description => <<"Update Prometheus config">>, + description => ?DESC(update_prom_conf_info), tags => ?TAGS, 'requestBody' => prometheus_config_schema(), responses => @@ -69,7 +70,7 @@ schema("/prometheus/stats") -> 'operationId' => stats, get => #{ - description => <<"Get Prometheus Data">>, + description => ?DESC(get_prom_data), tags => ?TAGS, security => [], responses => diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker_sup.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker_sup.erl index 104ad7ade..4b85fe92f 100644 --- a/apps/emqx_resource/src/emqx_resource_buffer_worker_sup.erl +++ b/apps/emqx_resource/src/emqx_resource_buffer_worker_sup.erl @@ -115,7 +115,8 @@ ensure_worker_started(ResId, Idx, Opts) -> id => ?CHILD_ID(Mod, ResId, Idx), start => {Mod, start_link, [ResId, Idx, Opts]}, restart => transient, - shutdown => 5000, + %% if we delay shutdown, when the pool is big, it will take a long time + shutdown => brutal_kill, type => worker, modules => [Mod] }, @@ -130,13 +131,12 @@ ensure_worker_removed(ResId, Idx) -> ChildId = ?CHILD_ID(emqx_resource_buffer_worker, ResId, Idx), case supervisor:terminate_child(?SERVER, ChildId) of ok -> - Res = supervisor:delete_child(?SERVER, ChildId), - _ = gproc_pool:remove_worker(ResId, {ResId, Idx}), - Res; - {error, not_found} -> + _ = supervisor:delete_child(?SERVER, ChildId), + %% no need to remove worker from the pool, + %% because the entire pool will be force deleted later ok; - {error, Reason} -> - {error, Reason} + {error, not_found} -> + ok end. ensure_disk_queue_dir_absent(ResourceId, Index) -> diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index f42d3c1b5..97ac355f4 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -53,7 +53,18 @@ % State record -record(data, { - id, group, mod, callback_mode, query_mode, config, opts, status, state, error, pid + id, + group, + mod, + callback_mode, + query_mode, + config, + opts, + status, + state, + error, + pid, + extra }). -type data() :: #data{}. @@ -181,7 +192,15 @@ remove(ResId) when is_binary(ResId) -> %% @doc Stops a running resource_manager and optionally clears the metrics for the resource -spec remove(resource_id(), boolean()) -> ok | {error, Reason :: term()}. remove(ResId, ClearMetrics) when is_binary(ResId) -> - safe_call(ResId, {remove, ClearMetrics}, ?T_OPERATION). + ResourceManagerPid = gproc:whereis_name(?NAME(ResId)), + try + safe_call(ResId, {remove, ClearMetrics}, ?T_OPERATION) + after + %% Ensure the supervisor has it removed, otherwise the immediate re-add will see a stale process + %% If the 'remove' call babove had succeeded, this is mostly a no-op but still needed to avoid race condition. + %% Otherwise this is a 'infinity' shutdown, so it may take arbitrary long. + emqx_resource_manager_sup:delete_child(ResourceManagerPid) + end. %% @doc Stops and then starts an instance that was already running -spec restart(resource_id(), creation_opts()) -> ok | {error, Reason :: term()}. @@ -387,7 +406,7 @@ handle_event(EventType, EventData, State, Data) -> event_type => EventType, event_data => EventData, state => State, - data => Data + data => emqx_utils:redact(Data) } ), keep_state_and_data. @@ -397,15 +416,15 @@ log_state_consistency(State, #data{status = State} = Data) -> log_state_consistency(State, Data) -> ?tp(warning, "inconsistent_state", #{ state => State, - data => Data + data => emqx_utils:redact(Data) }). log_cache_consistency(Data, Data) -> ok; log_cache_consistency(DataCached, Data) -> ?tp(warning, "inconsistent_cache", #{ - cache => DataCached, - data => Data + cache => emqx_utils:redact(DataCached), + data => emqx_utils:redact(Data) }). %%------------------------------------------------------------------------------ @@ -439,8 +458,10 @@ health_check_actions(Data) -> [{state_timeout, health_check_interval(Data#data.opts), health_check}]. handle_remove_event(From, ClearMetrics, Data) -> - _ = stop_resource(Data), + %% stop the buffer workers first, brutal_kill, so it should be fast ok = emqx_resource_buffer_worker_sup:stop_workers(Data#data.id, Data#data.opts), + %% now stop the resource, this can be slow + _ = stop_resource(Data), case ClearMetrics of true -> ok = emqx_metrics_worker:clear_metrics(?RES_METRICS, Data#data.id); false -> ok diff --git a/apps/emqx_resource/src/emqx_resource_manager_sup.erl b/apps/emqx_resource/src/emqx_resource_manager_sup.erl index 2f442cd56..73f1988c6 100644 --- a/apps/emqx_resource/src/emqx_resource_manager_sup.erl +++ b/apps/emqx_resource/src/emqx_resource_manager_sup.erl @@ -17,7 +17,7 @@ -behaviour(supervisor). --export([ensure_child/5]). +-export([ensure_child/5, delete_child/1]). -export([start_link/0]). @@ -27,6 +27,11 @@ ensure_child(ResId, Group, ResourceType, Config, Opts) -> _ = supervisor:start_child(?MODULE, [ResId, Group, ResourceType, Config, Opts]), ok. +delete_child(Pid) -> + _ = supervisor:terminate_child(?MODULE, Pid), + _ = supervisor:delete_child(?MODULE, Pid), + ok. + start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). @@ -36,7 +41,10 @@ init([]) -> id => emqx_resource_manager, start => {emqx_resource_manager, start_link, []}, restart => transient, - shutdown => brutal_kill, + %% never force kill a resource manager. + %% becasue otherwise it may lead to release leak, + %% resource_manager's terminate callback calls resource on_stop + shutdown => infinity, type => worker, modules => [emqx_resource_manager] } diff --git a/apps/emqx_resource/src/schema/emqx_resource_schema.erl b/apps/emqx_resource/src/schema/emqx_resource_schema.erl index 1de1a6545..5ad61d20b 100644 --- a/apps/emqx_resource/src/schema/emqx_resource_schema.erl +++ b/apps/emqx_resource/src/schema/emqx_resource_schema.erl @@ -23,6 +23,12 @@ -export([namespace/0, roots/0, fields/1, desc/1]). +%% range interval in ms +-define(HEALTH_CHECK_INTERVAL_RANGE_MIN, 1). +-define(HEALTH_CHECK_INTERVAL_RANGE_MAX, 3_600_000). +-define(AUTO_RESTART_INTERVAL_RANGE_MIN, 1). +-define(AUTO_RESTART_INTERVAL_RANGE_MAX, 3_600_000). + %% ------------------------------------------------------------------------------------------------- %% Hocon Schema Definitions @@ -87,8 +93,22 @@ health_check_interval(type) -> emqx_schema:duration_ms(); health_check_interval(desc) -> ?DESC("health_check_interval"); health_check_interval(default) -> ?HEALTHCHECK_INTERVAL_RAW; health_check_interval(required) -> false; +health_check_interval(validator) -> fun health_check_interval_range/1; health_check_interval(_) -> undefined. +health_check_interval_range(HealthCheckInterval) when + is_integer(HealthCheckInterval) andalso + HealthCheckInterval >= ?HEALTH_CHECK_INTERVAL_RANGE_MIN andalso + HealthCheckInterval =< ?HEALTH_CHECK_INTERVAL_RANGE_MAX +-> + ok; +health_check_interval_range(_HealthCheckInterval) -> + {error, #{ + msg => <<"Health Check Interval out of range">>, + min => ?HEALTH_CHECK_INTERVAL_RANGE_MIN, + max => ?HEALTH_CHECK_INTERVAL_RANGE_MAX + }}. + start_after_created(type) -> boolean(); start_after_created(desc) -> ?DESC("start_after_created"); start_after_created(default) -> ?START_AFTER_CREATED_RAW; @@ -105,8 +125,22 @@ auto_restart_interval(type) -> hoconsc:union([infinity, emqx_schema:duration_ms( auto_restart_interval(desc) -> ?DESC("auto_restart_interval"); auto_restart_interval(default) -> ?AUTO_RESTART_INTERVAL_RAW; auto_restart_interval(required) -> false; +auto_restart_interval(validator) -> fun auto_restart_interval_range/1; auto_restart_interval(_) -> undefined. +auto_restart_interval_range(AutoRestartInterval) when + is_integer(AutoRestartInterval) andalso + AutoRestartInterval >= ?AUTO_RESTART_INTERVAL_RANGE_MIN andalso + AutoRestartInterval =< ?AUTO_RESTART_INTERVAL_RANGE_MAX +-> + ok; +auto_restart_interval_range(_AutoRestartInterval) -> + {error, #{ + msg => <<"Auto Restart Interval out of range">>, + min => ?AUTO_RESTART_INTERVAL_RANGE_MIN, + max => ?AUTO_RESTART_INTERVAL_RANGE_MAX + }}. + query_mode(type) -> enum([sync, async]); query_mode(desc) -> ?DESC("query_mode"); query_mode(default) -> async; diff --git a/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl index c9926f56f..0424bfb60 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl @@ -112,7 +112,8 @@ fields("rule_test") -> ref("ctx_disconnected"), ref("ctx_connack"), ref("ctx_check_authz_complete"), - ref("ctx_bridge_mqtt") + ref("ctx_bridge_mqtt"), + ref("ctx_delivery_dropped") ]), #{ desc => ?DESC("test_context"), @@ -276,6 +277,15 @@ fields("ctx_bridge_mqtt") -> {"retain", sc(binary(), #{desc => ?DESC("event_retain")})}, {"message_received_at", publish_received_at_sc()}, qos() + ]; +fields("ctx_delivery_dropped") -> + [ + {"event_type", event_type_sc(delivery_dropped)}, + {"id", sc(binary(), #{desc => ?DESC("event_id")})}, + {"reason", sc(binary(), #{desc => ?DESC("event_ctx_dropped")})}, + {"from_clientid", sc(binary(), #{desc => ?DESC("event_from_clientid")})}, + {"from_username", sc(binary(), #{desc => ?DESC("event_from_username")})} + | msg_event_common_fields() ]. qos() -> diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index 02163f95b..47017f718 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -1075,37 +1075,34 @@ now_timestamp(Unit) -> time_unit(<<"second">>) -> second; time_unit(<<"millisecond">>) -> millisecond; time_unit(<<"microsecond">>) -> microsecond; -time_unit(<<"nanosecond">>) -> nanosecond. +time_unit(<<"nanosecond">>) -> nanosecond; +time_unit(second) -> second; +time_unit(millisecond) -> millisecond; +time_unit(microsecond) -> microsecond; +time_unit(nanosecond) -> nanosecond. format_date(TimeUnit, Offset, FormatString) -> - emqx_plugin_libs_rule:bin( - emqx_rule_date:date( - time_unit(TimeUnit), - emqx_plugin_libs_rule:str(Offset), - emqx_plugin_libs_rule:str(FormatString) - ) - ). + Unit = time_unit(TimeUnit), + TimeEpoch = erlang:system_time(Unit), + format_date(Unit, Offset, FormatString, TimeEpoch). format_date(TimeUnit, Offset, FormatString, TimeEpoch) -> + Unit = time_unit(TimeUnit), emqx_plugin_libs_rule:bin( - emqx_rule_date:date( - time_unit(TimeUnit), - emqx_plugin_libs_rule:str(Offset), - emqx_plugin_libs_rule:str(FormatString), - TimeEpoch + lists:concat( + emqx_calendar:format(TimeEpoch, Unit, Offset, FormatString) ) ). date_to_unix_ts(TimeUnit, FormatString, InputString) -> - date_to_unix_ts(TimeUnit, "Z", FormatString, InputString). + Unit = time_unit(TimeUnit), + emqx_calendar:parse(InputString, Unit, FormatString). date_to_unix_ts(TimeUnit, Offset, FormatString, InputString) -> - emqx_rule_date:parse_date( - time_unit(TimeUnit), - emqx_plugin_libs_rule:str(Offset), - emqx_plugin_libs_rule:str(FormatString), - emqx_plugin_libs_rule:str(InputString) - ). + Unit = time_unit(TimeUnit), + OffsetSecond = emqx_calendar:offset_second(Offset), + OffsetDelta = erlang:convert_time_unit(OffsetSecond, second, Unit), + date_to_unix_ts(Unit, FormatString, InputString) - OffsetDelta. %% @doc This is for sql funcs that should be handled in the specific modules. %% Here the emqx_rule_funcs module acts as a proxy, forwarding diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_api_rule_test_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_api_rule_test_SUITE.erl new file mode 100644 index 000000000..575d35238 --- /dev/null +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_api_rule_test_SUITE.erl @@ -0,0 +1,277 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_rule_engine_api_rule_test_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-define(CONF_DEFAULT, <<"rule_engine {rules {}}">>). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + application:load(emqx_conf), + ok = emqx_common_test_helpers:load_config(emqx_rule_engine_schema, ?CONF_DEFAULT), + ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_rule_engine]), + Config. + +end_per_suite(_Config) -> + emqx_common_test_helpers:stop_apps([emqx_conf, emqx_rule_engine]), + ok. + +t_ctx_pub(_) -> + SQL = <<"SELECT payload.msg as msg, clientid, username, payload, topic, qos FROM \"t/#\"">>, + Context = #{ + clientid => <<"c_emqx">>, + event_type => message_publish, + payload => <<"{\"msg\": \"hello\"}">>, + qos => 1, + topic => <<"t/a">>, + username => <<"u_emqx">> + }, + Expected = Context#{msg => <<"hello">>}, + do_test(SQL, Context, Expected). + +t_ctx_sub(_) -> + SQL = <<"SELECT clientid, username, topic, qos FROM \"$events/session_subscribed\"">>, + Context = #{ + clientid => <<"c_emqx">>, + event_type => session_subscribed, + qos => 1, + topic => <<"t/a">>, + username => <<"u_emqx">> + }, + + do_test(SQL, Context, Context). + +t_ctx_unsub(_) -> + SQL = <<"SELECT clientid, username, topic, qos FROM \"$events/session_unsubscribed\"">>, + Context = #{ + clientid => <<"c_emqx">>, + event_type => session_unsubscribed, + qos => 1, + topic => <<"t/a">>, + username => <<"u_emqx">> + }, + do_test(SQL, Context, Context). + +t_ctx_delivered(_) -> + SQL = + <<"SELECT from_clientid, from_username, topic, qos, node, timestamp FROM \"$events/message_delivered\"">>, + Context = #{ + clientid => <<"c_emqx_2">>, + event_type => message_delivered, + from_clientid => <<"c_emqx_1">>, + from_username => <<"u_emqx_1">>, + payload => <<"{\"msg\": \"hello\"}">>, + qos => 1, + topic => <<"t/a">>, + username => <<"u_emqx_2">> + }, + Expected = check_result([from_clientid, from_username, topic, qos], [node, timestamp], Context), + do_test(SQL, Context, Expected). + +t_ctx_acked(_) -> + SQL = + <<"SELECT from_clientid, from_username, topic, qos, node, timestamp FROM \"$events/message_acked\"">>, + + Context = #{ + clientid => <<"c_emqx_2">>, + event_type => message_acked, + from_clientid => <<"c_emqx_1">>, + from_username => <<"u_emqx_1">>, + payload => <<"{\"msg\": \"hello\"}">>, + qos => 1, + topic => <<"t/a">>, + username => <<"u_emqx_2">> + }, + + Expected = with_node_timestampe([from_clientid, from_username, topic, qos], Context), + + do_test(SQL, Context, Expected). + +t_ctx_droped(_) -> + SQL = <<"SELECT reason, topic, qos, node, timestamp FROM \"$events/message_dropped\"">>, + Topic = <<"t/a">>, + QoS = 1, + Reason = <<"no_subscribers">>, + Context = #{ + clientid => <<"c_emqx">>, + event_type => message_dropped, + payload => <<"{\"msg\": \"hello\"}">>, + qos => QoS, + reason => Reason, + topic => Topic, + username => <<"u_emqx">> + }, + + Expected = with_node_timestampe([reason, topic, qos], Context), + do_test(SQL, Context, Expected). + +t_ctx_connected(_) -> + SQL = + <<"SELECT clientid, username, keepalive, is_bridge FROM \"$events/client_connected\"">>, + + Context = + #{ + clean_start => true, + clientid => <<"c_emqx">>, + event_type => client_connected, + is_bridge => false, + peername => <<"127.0.0.1:52918">>, + username => <<"u_emqx">> + }, + Expected = check_result([clientid, username, keepalive, is_bridge], [], Context), + do_test(SQL, Context, Expected). + +t_ctx_disconnected(_) -> + SQL = + <<"SELECT clientid, username, reason, disconnected_at, node FROM \"$events/client_disconnected\"">>, + + Context = + #{ + clientid => <<"c_emqx">>, + event_type => client_disconnected, + reason => <<"normal">>, + username => <<"u_emqx">> + }, + Expected = check_result([clientid, username, reason], [disconnected_at, node], Context), + do_test(SQL, Context, Expected). + +t_ctx_connack(_) -> + SQL = + <<"SELECT clientid, username, reason_code, node FROM \"$events/client_connack\"">>, + + Context = + #{ + clean_start => true, + clientid => <<"c_emqx">>, + event_type => client_connack, + reason_code => <<"sucess">>, + username => <<"u_emqx">> + }, + Expected = check_result([clientid, username, reason_code], [node], Context), + do_test(SQL, Context, Expected). + +t_ctx_check_authz_complete(_) -> + SQL = + << + "SELECT clientid, username, topic, action, result,\n" + "authz_source, node FROM \"$events/client_check_authz_complete\"" + >>, + + Context = + #{ + action => <<"publish">>, + clientid => <<"c_emqx">>, + event_type => client_check_authz_complete, + result => <<"allow">>, + topic => <<"t/1">>, + username => <<"u_emqx">> + }, + Expected = check_result( + [clientid, username, topic, action], + [authz_source, node, result], + Context + ), + + do_test(SQL, Context, Expected). + +t_ctx_delivery_dropped(_) -> + SQL = + <<"SELECT from_clientid, from_username, reason, topic, qos FROM \"$events/delivery_dropped\"">>, + + Context = + #{ + clientid => <<"c_emqx_2">>, + event_type => delivery_dropped, + from_clientid => <<"c_emqx_1">>, + from_username => <<"u_emqx_1">>, + payload => <<"{\"msg\": \"hello\"}">>, + qos => 1, + reason => <<"queue_full">>, + topic => <<"t/a">>, + username => <<"u_emqx_2">> + }, + Expected = check_result([from_clientid, from_username, reason, qos, topic], [], Context), + do_test(SQL, Context, Expected). + +do_test(SQL, Context, Expected0) -> + Res = emqx_rule_engine_api:'/rule_test'( + post, + test_rule_params(SQL, Context) + ), + ?assertMatch({200, _}, Res), + {200, Result0} = Res, + Result = emqx_utils_maps:unsafe_atom_key_map(Result0), + case is_function(Expected0) of + false -> + Expected = maps:without([event_type], Expected0), + ?assertMatch(Expected, Result, Expected); + _ -> + Expected0(Result) + end, + ok. + +test_rule_params(Sql, Context) -> + #{ + body => #{ + <<"context">> => Context, + <<"sql">> => Sql + } + }. + +with_node_timestampe(Keys, Context) -> + check_result(Keys, [node, timestamp], Context). + +check_result(Keys, Exists, Context) -> + Log = fun(Format, Args) -> + lists:flatten(io_lib:format(Format, Args)) + end, + + Base = maps:with(Keys, Context), + + fun(Result) -> + maps:foreach( + fun(Key, Value) -> + ?assertEqual( + Value, + maps:get(Key, Result, undefined), + Log("Key:~p value error~nResult:~p~n", [Key, Result]) + ) + end, + Base + ), + + NotExists = fun(Key) -> Log("Key:~p not exists in result:~p~n", [Key, Result]) end, + lists:foreach( + fun(Key) -> + Find = maps:find(Key, Result), + Formatter = NotExists(Key), + ?assertMatch({ok, _}, Find, Formatter), + ?assertNotMatch({ok, undefined}, Find, Formatter), + ?assertNotMatch({ok, <<"undefined">>}, Find, Formatter) + end, + Exists + ), + + ?assertEqual(erlang:length(Keys) + erlang:length(Exists), maps:size(Result), Result) + end. diff --git a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl index d88637312..9bb6e2e6f 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl @@ -955,10 +955,10 @@ t_format_date_funcs(_) -> ?PROPTEST(prop_format_date_fun). prop_format_date_fun() -> - Args1 = [<<"second">>, <<"+07:00">>, <<"%m--%d--%y---%H:%M:%S%Z">>], + Args1 = [<<"second">>, <<"+07:00">>, <<"%m--%d--%Y---%H:%M:%S%z">>], ?FORALL( S, - range(0, 4000000000), + erlang:system_time(second), S == apply_func( date_to_unix_ts, @@ -971,14 +971,15 @@ prop_format_date_fun() -> ] ) ), - Args2 = [<<"millisecond">>, <<"+04:00">>, <<"--%m--%d--%y---%H:%M:%S%Z">>], + Args2 = [<<"millisecond">>, <<"+04:00">>, <<"--%m--%d--%Y---%H:%M:%S:%3N%z">>], + Args2DTUS = [<<"millisecond">>, <<"--%m--%d--%Y---%H:%M:%S:%3N%z">>], ?FORALL( S, - range(0, 4000000000), + erlang:system_time(millisecond), S == apply_func( date_to_unix_ts, - Args2 ++ + Args2DTUS ++ [ apply_func( format_date, @@ -987,14 +988,15 @@ prop_format_date_fun() -> ] ) ), - Args = [<<"second">>, <<"+08:00">>, <<"%y-%m-%d-%H:%M:%S%Z">>], + Args = [<<"second">>, <<"+08:00">>, <<"%Y-%m-%d-%H:%M:%S%z">>], + ArgsDTUS = [<<"second">>, <<"%Y-%m-%d-%H:%M:%S%z">>], ?FORALL( S, - range(0, 4000000000), + erlang:system_time(second), S == apply_func( date_to_unix_ts, - Args ++ + ArgsDTUS ++ [ apply_func( format_date, @@ -1003,24 +1005,12 @@ prop_format_date_fun() -> ] ) ), - %% When no offset is specified, the offset should be taken from the formatted time string - ArgsNoOffset = [<<"second">>, <<"%y-%m-%d-%H:%M:%S%Z">>], - ArgsOffset = [<<"second">>, <<"+08:00">>, <<"%y-%m-%d-%H:%M:%S%Z">>], - ?FORALL( - S, - range(0, 4000000000), - S == - apply_func( - date_to_unix_ts, - ArgsNoOffset ++ - [ - apply_func( - format_date, - ArgsOffset ++ [S] - ) - ] - ) - ). + % no offset in format string. force add offset + Second = erlang:system_time(second), + Args3 = [<<"second">>, <<"+04:00">>, <<"--%m--%d--%Y---%H:%M:%S">>, Second], + Formatters3 = apply_func(format_date, Args3), + Args3DTUS = [<<"second">>, <<"+04:00">>, <<"--%m--%d--%Y---%H:%M:%S">>, Formatters3], + Second == apply_func(date_to_unix_ts, Args3DTUS). %%------------------------------------------------------------------------------ %% Utility functions diff --git a/apps/emqx_s3/BSL.txt b/apps/emqx_s3/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_s3/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_s3/README.md b/apps/emqx_s3/README.md new file mode 100644 index 000000000..4ce1b0c0a --- /dev/null +++ b/apps/emqx_s3/README.md @@ -0,0 +1,135 @@ +# emqx_s3 + +EMQX S3 Application + +## Description + +This application provides functionality for uploading files to S3. + +## Usage + +The steps to integrate this application are: +* Integrate S3 configuration schema where needed. +* On _client_ application start: + * Call `emqx_s3:start_profile(ProfileName, ProfileConfig)` with configuration. + * Add `emqx_config_handler` hook to call `emqx_s3:start_profile(ProfileName, ProfileConfig)` when configuration is updated. +* On _client_ application stop, call `emqx_s3:stop_profile(ProfileName)`. + +`ProfileName` is a unique name used to distinguish different sets of S3 settings. Each profile has its own connection pool and configuration. + +To use S3 from a _client_ application: +* Create an uploader process with `{ok, Pid} = emqx_s3:start_uploader(ProfileName, #{key => MyKey})`. +* Write data with `emqx_s3_uploader:write(Pid, <<"data">>)`. +* Finish the uploader with `emqx_s3_uploader:complete(Pid)` or `emqx_s3_uploader:abort(Pid)`. + +### Configuration + +Example of integrating S3 configuration schema into a _client_ application `emqx_someapp`. + +```erlang +-module(emqx_someapp_schema). + +... + +roots() -> [someapp] +... + +fields(someapp) -> + [ + {other_setting, ...}, + {s3_settings, + mk( + hoconsc:ref(emqx_s3_schema, s3), + #{ + desc => ?DESC("s3_settings"), + required => true + } + )} + ]; +... + +``` + +### Application start and config hooks + +```erlang +-module(emqx_someapp_app). + +-behaviour(application). + +-export([start/2, stop/1]). + +-export([ + pre_config_update/3, + post_config_update/5 +]). + +start(_StartType, _StartArgs) -> + ProfileConfig = emqx_config:get([someapp, s3_settings]), + ProfileName = someapp, + ok = emqx_s3:start_profile(ProfileName, ProfileConfig), + ok = emqx_config_handler:add_handler([someapp], ?MODULE). + +stop(_State) -> + ok = emqx_conf:remove_handler([someapp]), + ProfileName = someapp, + ok = emqx_s3:stop_profile(ProfileName). + +pre_config_update(_Path, NewConfig, _OldConfig) -> + {ok, NewConfig}. + +post_config_update(Path, _Req, NewConfig, _OldConfig, _AppEnvs) -> + NewProfileConfig = maps:get(s3_settings, NewConfig), + ProfileName = someapp, + %% more graceful error handling may be needed + ok = emqx_s3:update_profile(ProfileName, NewProfileConfig). + +``` + +### Uploader usage + +```erlang +-module(emqx_someapp_logic). +... + +-spec do_upload_data(Key :: string(), Data :: binary()) -> ok. +do_upload_data(Key, Data) -> + ProfileName = someapp, + {ok, Pid} = emqx_s3:start_uploader(ProfileName, #{key => Key}), + ok = emqx_s3_uploader:write(Pid, Data), + ok = emqx_s3_uploader:complete(Pid). + +``` + +## Design + +![Design](./docs/s3_app.png) + +* Each profile has its own supervisor `emqx_s3_profile_sup`. +* Under each profile supervisor, there is a + * `emqx_s3_profile_uploader_sup` supervisor for uploader processes. + * `emqx_s3_profile_conf` server for managing profile configuration. + +When an uploader process is started, it checkouts the actual S3 configuration for the profile from the `emqx_s3_profile_conf` server. It uses the obtained configuration and connection pool to upload data to S3 till the termination, even if the configuration is updated. + +Other processes (`emqx_XXX`) can also checkout the actual S3 configuration for the profile from the `emqx_s3_profile_conf` server. + +`emqx_s3_profile_conf`: +* Keeps actual S3 configuration for the profile and creates a connection pool for the actual configuration. +* Creates a new connection pool when the configuration is updated. +* Keeps track of uploaders using connection pools. +* Drops connection pools when no uploaders are using it or after a timeout. + +The code is designed to allow a painless transition from `ehttpc` pool to any other HTTP pool/client. + +## Possible performance improvements + +One of the downsides of the current implementation is that there is a lot of message passing between the uploader client and the actual sockets. + +A possible improvement could be: +* Use a process-less HTTP client, like [Mint](https://github.com/elixir-mint/mint). +* Use a resource pool, like [NimblePool](https://github.com/dashbitco/nimble_pool) to manage the HTTP connections. It temporarily grants sockets to its clients. +* Do the buffering logic locally in the uploader client. +* Use `emqx_s3_client` directly from the uploader client. + +In this case, the data will be directly sent to the socket, without being sent to any intermediate processes. diff --git a/apps/emqx_s3/docker-ct b/apps/emqx_s3/docker-ct new file mode 100644 index 000000000..a5a001815 --- /dev/null +++ b/apps/emqx_s3/docker-ct @@ -0,0 +1,2 @@ +minio +toxiproxy diff --git a/apps/emqx_s3/docs/s3_app.png b/apps/emqx_s3/docs/s3_app.png new file mode 100644 index 000000000..cb7758844 Binary files /dev/null and b/apps/emqx_s3/docs/s3_app.png differ diff --git a/apps/emqx_s3/rebar.config b/apps/emqx_s3/rebar.config new file mode 100644 index 000000000..b1483e028 --- /dev/null +++ b/apps/emqx_s3/rebar.config @@ -0,0 +1,6 @@ +{deps, [ + {emqx, {path, "../../apps/emqx"}}, + {erlcloud, {git, "https://github.com/emqx/erlcloud", {tag, "3.6.8-emqx-1"}}} +]}. + +{project_plugins, [erlfmt]}. diff --git a/apps/emqx_s3/src/emqx_s3.app.src b/apps/emqx_s3/src/emqx_s3.app.src new file mode 100644 index 000000000..7864ffb29 --- /dev/null +++ b/apps/emqx_s3/src/emqx_s3.app.src @@ -0,0 +1,14 @@ +{application, emqx_s3, [ + {description, "EMQX S3"}, + {vsn, "5.0.6"}, + {modules, []}, + {registered, [emqx_s3_sup]}, + {applications, [ + kernel, + stdlib, + gproc, + erlcloud, + ehttpc + ]}, + {mod, {emqx_s3_app, []}} +]}. diff --git a/apps/emqx_s3/src/emqx_s3.erl b/apps/emqx_s3/src/emqx_s3.erl new file mode 100644 index 000000000..0c2592736 --- /dev/null +++ b/apps/emqx_s3/src/emqx_s3.erl @@ -0,0 +1,96 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3). + +-include_lib("emqx/include/types.hrl"). + +-export([ + start_profile/2, + stop_profile/1, + update_profile/2, + start_uploader/2, + with_client/2 +]). + +-export_type([ + profile_id/0, + profile_config/0, + acl/0 +]). + +-type profile_id() :: atom() | binary(). + +-type acl() :: + private + | public_read + | public_read_write + | authenticated_read + | bucket_owner_read + | bucket_owner_full_control. + +-type transport_options() :: #{ + headers => map(), + connect_timeout => pos_integer(), + enable_pipelining => pos_integer(), + max_retries => pos_integer(), + pool_size => pos_integer(), + pool_type => atom(), + ipv6_probe => boolean(), + ssl => map() +}. + +-type profile_config() :: #{ + bucket := string(), + access_key_id => string(), + secret_access_key => string(), + host := string(), + port := pos_integer(), + url_expire_time := pos_integer(), + acl => acl(), + min_part_size => pos_integer(), + transport_options => transport_options() +}. + +-define(IS_PROFILE_ID(ProfileId), (is_atom(ProfileId) orelse is_binary(ProfileId))). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- + +-spec start_profile(profile_id(), profile_config()) -> ok_or_error(term()). +start_profile(ProfileId, ProfileConfig) when ?IS_PROFILE_ID(ProfileId) -> + case emqx_s3_sup:start_profile(ProfileId, ProfileConfig) of + {ok, _} -> + ok; + {error, _} = Error -> + Error + end. + +-spec stop_profile(profile_id()) -> ok_or_error(term()). +stop_profile(ProfileId) when ?IS_PROFILE_ID(ProfileId) -> + emqx_s3_sup:stop_profile(ProfileId). + +-spec update_profile(profile_id(), profile_config()) -> ok_or_error(term()). +update_profile(ProfileId, ProfileConfig) when ?IS_PROFILE_ID(ProfileId) -> + emqx_s3_profile_conf:update_config(ProfileId, ProfileConfig). + +-spec start_uploader(profile_id(), emqx_s3_uploader:opts()) -> + supervisor:start_ret() | {error, profile_not_found}. +start_uploader(ProfileId, Opts) when ?IS_PROFILE_ID(ProfileId) -> + emqx_s3_profile_uploader_sup:start_uploader(ProfileId, Opts). + +-spec with_client(profile_id(), fun((emqx_s3_client:client()) -> Result)) -> + {error, profile_not_found} | Result. +with_client(ProfileId, Fun) when is_function(Fun, 1) andalso ?IS_PROFILE_ID(ProfileId) -> + case emqx_s3_profile_conf:checkout_config(ProfileId) of + {ok, ClientConfig, _UploadConfig} -> + try + Fun(emqx_s3_client:create(ClientConfig)) + after + emqx_s3_profile_conf:checkin_config(ProfileId) + end; + {error, _} = Error -> + Error + end. diff --git a/apps/emqx_s3/src/emqx_s3.hrl b/apps/emqx_s3/src/emqx_s3.hrl new file mode 100644 index 000000000..9a2cf8d2f --- /dev/null +++ b/apps/emqx_s3/src/emqx_s3.hrl @@ -0,0 +1,13 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-define(VIA_GPROC(Id), {via, gproc, {n, l, Id}}). + +-define(SAFE_CALL_VIA_GPROC(Id, Message, Timeout, NoProcError), + try gen_server:call(?VIA_GPROC(Id), Message, Timeout) of + Result -> Result + catch + exit:{noproc, _} -> {error, NoProcError} + end +). diff --git a/apps/emqx_s3/src/emqx_s3_app.erl b/apps/emqx_s3/src/emqx_s3_app.erl new file mode 100644 index 000000000..8d8b0f7b9 --- /dev/null +++ b/apps/emqx_s3/src/emqx_s3_app.erl @@ -0,0 +1,16 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3_app). + +-behaviour(application). + +-export([start/2, stop/1]). + +start(_Type, _Args) -> + {ok, Sup} = emqx_s3_sup:start_link(), + {ok, Sup}. + +stop(_State) -> + ok. diff --git a/apps/emqx_s3/src/emqx_s3_client.erl b/apps/emqx_s3/src/emqx_s3_client.erl new file mode 100644 index 000000000..3bc5861c6 --- /dev/null +++ b/apps/emqx_s3/src/emqx_s3_client.erl @@ -0,0 +1,428 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3_client). + +-include_lib("emqx/include/types.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("erlcloud/include/erlcloud_aws.hrl"). + +-export([ + create/1, + + put_object/3, + put_object/4, + + start_multipart/2, + start_multipart/3, + upload_part/5, + complete_multipart/4, + abort_multipart/3, + list/2, + uri/2, + + format/1, + format_request/1 +]). + +-export_type([ + client/0, + headers/0 +]). + +-type headers() :: #{binary() | string() => iodata()}. +-type erlcloud_headers() :: list({string(), iodata()}). + +-type key() :: string(). +-type part_number() :: non_neg_integer(). +-type upload_id() :: string(). +-type etag() :: string(). +-type http_pool() :: ehttpc:pool_name(). +-type pool_type() :: random | hash. +-type upload_options() :: list({acl, emqx_s3:acl()}). + +-opaque client() :: #{ + aws_config := aws_config(), + upload_options := upload_options(), + bucket := string(), + headers := erlcloud_headers(), + url_expire_time := non_neg_integer(), + pool_type := pool_type() +}. + +-type config() :: #{ + scheme := string(), + host := string(), + port := part_number(), + bucket := string(), + headers := headers(), + acl := emqx_s3:acl() | undefined, + url_expire_time := pos_integer(), + access_key_id := string() | undefined, + secret_access_key := string() | undefined, + http_pool := http_pool(), + pool_type := pool_type(), + request_timeout := timeout() | undefined, + max_retries := non_neg_integer() | undefined +}. + +-type s3_options() :: proplists:proplist(). + +-define(DEFAULT_REQUEST_TIMEOUT, 30000). +-define(DEFAULT_MAX_RETRIES, 2). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- + +-spec create(config()) -> client(). +create(Config) -> + #{ + aws_config => aws_config(Config), + upload_options => upload_options(Config), + bucket => maps:get(bucket, Config), + url_expire_time => maps:get(url_expire_time, Config), + headers => headers(Config), + pool_type => maps:get(pool_type, Config) + }. + +-spec put_object(client(), key(), iodata()) -> ok_or_error(term()). +put_object(Client, Key, Value) -> + put_object(Client, #{}, Key, Value). + +-spec put_object(client(), headers(), key(), iodata()) -> ok_or_error(term()). +put_object( + #{bucket := Bucket, upload_options := Options, headers := Headers, aws_config := AwsConfig}, + SpecialHeaders, + Key, + Value +) -> + AllHeaders = join_headers(Headers, SpecialHeaders), + try erlcloud_s3:put_object(Bucket, erlcloud_key(Key), Value, Options, AllHeaders, AwsConfig) of + Props when is_list(Props) -> + ok + catch + error:{aws_error, Reason} -> + ?SLOG(debug, #{msg => "put_object_fail", key => Key, reason => Reason}), + {error, Reason} + end. + +-spec start_multipart(client(), key()) -> ok_or_error(upload_id(), term()). +start_multipart(Client, Key) -> + start_multipart(Client, #{}, Key). + +-spec start_multipart(client(), headers(), key()) -> ok_or_error(upload_id(), term()). +start_multipart( + #{bucket := Bucket, upload_options := Options, headers := Headers, aws_config := AwsConfig}, + SpecialHeaders, + Key +) -> + AllHeaders = join_headers(Headers, SpecialHeaders), + case erlcloud_s3:start_multipart(Bucket, erlcloud_key(Key), Options, AllHeaders, AwsConfig) of + {ok, Props} -> + {ok, response_property('uploadId', Props)}; + {error, Reason} -> + ?SLOG(debug, #{msg => "start_multipart_fail", key => Key, reason => Reason}), + {error, Reason} + end. + +-spec upload_part(client(), key(), upload_id(), part_number(), iodata()) -> + ok_or_error(etag(), term()). +upload_part( + #{bucket := Bucket, headers := Headers, aws_config := AwsConfig}, + Key, + UploadId, + PartNumber, + Value +) -> + case + erlcloud_s3:upload_part( + Bucket, erlcloud_key(Key), UploadId, PartNumber, Value, Headers, AwsConfig + ) + of + {ok, Props} -> + {ok, response_property(etag, Props)}; + {error, Reason} -> + ?SLOG(debug, #{msg => "upload_part_fail", key => Key, reason => Reason}), + {error, Reason} + end. + +-spec complete_multipart(client(), key(), upload_id(), [etag()]) -> ok_or_error(term()). +complete_multipart( + #{bucket := Bucket, headers := Headers, aws_config := AwsConfig}, + Key, + UploadId, + ETags +) -> + case + erlcloud_s3:complete_multipart( + Bucket, erlcloud_key(Key), UploadId, ETags, Headers, AwsConfig + ) + of + ok -> + ok; + {error, Reason} -> + ?SLOG(debug, #{msg => "complete_multipart_fail", key => Key, reason => Reason}), + {error, Reason} + end. + +-spec abort_multipart(client(), key(), upload_id()) -> ok_or_error(term()). +abort_multipart(#{bucket := Bucket, headers := Headers, aws_config := AwsConfig}, Key, UploadId) -> + case erlcloud_s3:abort_multipart(Bucket, erlcloud_key(Key), UploadId, [], Headers, AwsConfig) of + ok -> + ok; + {error, Reason} -> + ?SLOG(debug, #{msg => "abort_multipart_fail", key => Key, reason => Reason}), + {error, Reason} + end. + +-spec list(client(), s3_options()) -> ok_or_error(proplists:proplist(), term()). +list(#{bucket := Bucket, aws_config := AwsConfig}, Options) -> + try erlcloud_s3:list_objects(Bucket, Options, AwsConfig) of + Result -> {ok, Result} + catch + error:{aws_error, Reason} -> + ?SLOG(debug, #{msg => "list_objects_fail", bucket => Bucket, reason => Reason}), + {error, Reason} + end. + +-spec uri(client(), key()) -> iodata(). +uri(#{bucket := Bucket, aws_config := AwsConfig, url_expire_time := ExpireTime}, Key) -> + erlcloud_s3:make_presigned_v4_url(ExpireTime, Bucket, get, erlcloud_key(Key), [], AwsConfig). + +-spec format(client()) -> term(). +format(#{aws_config := AwsConfig} = Client) -> + Client#{aws_config => AwsConfig#aws_config{secret_access_key = "***"}}. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +upload_options(#{acl := Acl}) when Acl =/= undefined -> + [ + {acl, Acl} + ]; +upload_options(#{}) -> + []. + +headers(#{headers := Headers}) -> + headers_user_to_erlcloud_request(Headers); +headers(#{}) -> + []. + +aws_config(#{ + scheme := Scheme, + host := Host, + port := Port, + access_key_id := AccessKeyId, + secret_access_key := SecretAccessKey, + http_pool := HttpPool, + pool_type := PoolType, + request_timeout := Timeout, + max_retries := MaxRetries +}) -> + #aws_config{ + s3_scheme = Scheme, + s3_host = Host, + s3_port = Port, + s3_bucket_access_method = path, + s3_bucket_after_host = true, + + access_key_id = AccessKeyId, + secret_access_key = SecretAccessKey, + + http_client = request_fun( + HttpPool, PoolType, with_default(MaxRetries, ?DEFAULT_MAX_RETRIES) + ), + + %% This value will be transparently passed to ehttpc + timeout = with_default(Timeout, ?DEFAULT_REQUEST_TIMEOUT), + %% We rely on retry mechanism of ehttpc + retry_num = 1 + }. + +-spec request_fun(http_pool(), pool_type(), non_neg_integer()) -> erlcloud_httpc:request_fun(). +request_fun(HttpPool, PoolType, MaxRetries) -> + fun(Url, Method, Headers, Body, Timeout, _Config) -> + with_path_and_query_only(Url, fun(PathQuery) -> + Request = make_request( + Method, PathQuery, headers_erlcloud_request_to_ehttpc(Headers), Body + ), + case pick_worker_safe(HttpPool, PoolType) of + {ok, Worker} -> + ehttpc_request(Worker, Method, Request, Timeout, MaxRetries); + {error, Reason} -> + ?SLOG(error, #{ + msg => "s3_request_fun_fail", + reason => Reason, + http_pool => HttpPool, + pool_type => PoolType, + method => Method, + request => Request, + timeout => Timeout, + max_retries => MaxRetries + }), + {error, Reason} + end + end) + end. + +ehttpc_request(HttpPool, Method, Request, Timeout, MaxRetries) -> + try timer:tc(fun() -> ehttpc:request(HttpPool, Method, Request, Timeout, MaxRetries) end) of + {Time, {ok, StatusCode, RespHeaders}} -> + ?SLOG(info, #{ + msg => "s3_ehttpc_request_ok", + status_code => StatusCode, + headers => RespHeaders, + time => Time + }), + {ok, { + {StatusCode, undefined}, headers_ehttpc_to_erlcloud_response(RespHeaders), undefined + }}; + {Time, {ok, StatusCode, RespHeaders, RespBody}} -> + ?SLOG(info, #{ + msg => "s3_ehttpc_request_ok", + status_code => StatusCode, + headers => RespHeaders, + body => RespBody, + time => Time + }), + {ok, { + {StatusCode, undefined}, headers_ehttpc_to_erlcloud_response(RespHeaders), RespBody + }}; + {Time, {error, Reason}} -> + ?SLOG(error, #{ + msg => "s3_ehttpc_request_fail", + reason => Reason, + timeout => Timeout, + pool => HttpPool, + method => Method, + time => Time + }), + {error, Reason} + catch + error:badarg -> + ?SLOG(error, #{ + msg => "s3_ehttpc_request_fail", + reason => badarg, + timeout => Timeout, + pool => HttpPool, + method => Method + }), + {error, no_ehttpc_pool}; + error:Reason -> + ?SLOG(error, #{ + msg => "s3_ehttpc_request_fail", + reason => Reason, + timeout => Timeout, + pool => HttpPool, + method => Method + }), + {error, Reason} + end. + +pick_worker_safe(HttpPool, PoolType) -> + try + {ok, pick_worker(HttpPool, PoolType)} + catch + error:badarg -> + {error, no_ehttpc_pool} + end. + +pick_worker(HttpPool, random) -> + ehttpc_pool:pick_worker(HttpPool); +pick_worker(HttpPool, hash) -> + ehttpc_pool:pick_worker(HttpPool, self()). + +-define(IS_BODY_EMPTY(Body), (Body =:= undefined orelse Body =:= <<>>)). +-define(NEEDS_NO_BODY(Method), (Method =:= get orelse Method =:= head orelse Method =:= delete)). + +make_request(Method, PathQuery, Headers, Body) when + ?IS_BODY_EMPTY(Body) andalso ?NEEDS_NO_BODY(Method) +-> + {PathQuery, Headers}; +make_request(_Method, PathQuery, Headers, Body) when ?IS_BODY_EMPTY(Body) -> + {PathQuery, [{<<"content-length">>, <<"0">>} | Headers], <<>>}; +make_request(_Method, PathQuery, Headers, Body) -> + {PathQuery, Headers, Body}. + +format_request({PathQuery, Headers, _Body}) -> {PathQuery, Headers, <<"...">>}; +format_request({PathQuery, Headers}) -> {PathQuery, Headers}. + +with_path_and_query_only(Url, Fun) -> + case string:split(Url, "//", leading) of + [_Scheme, UrlRem] -> + case string:split(UrlRem, "/", leading) of + [_HostPort, PathQuery] -> + Fun([$/ | PathQuery]); + _ -> + {error, {invalid_url, Url}} + end; + _ -> + {error, {invalid_url, Url}} + end. + +%% We need some header conversions to tie the emqx_s3, erlcloud and ehttpc APIs together. + +%% The request header flow is: + +%% UserHeaders -> [emqx_s3_client API] -> ErlcloudRequestHeaders0 -> +%% -> [erlcloud API] -> ErlcloudRequestHeaders1 -> [emqx_s3_client injected request_fun] -> +%% -> EhttpcRequestHeaders -> [ehttpc API] + +%% The response header flow is: + +%% [ehttpc API] -> EhttpcResponseHeaders -> [emqx_s3_client injected request_fun] -> +%% -> ErlcloudResponseHeaders0 -> [erlcloud API] -> [emqx_s3_client API] + +%% UserHeders (emqx_s3 API headers) are maps with string/binary keys. +%% ErlcloudRequestHeaders are lists of tuples with string keys and iodata values +%% ErlcloudResponseHeders are lists of tuples with lower case string keys and iodata values. +%% EhttpcHeaders are lists of tuples with binary keys and iodata values. + +%% Users provide headers as a map, but erlcloud expects a list of tuples with string keys and values. +headers_user_to_erlcloud_request(UserHeaders) -> + [{to_list_string(K), V} || {K, V} <- maps:to_list(UserHeaders)]. + +%% Ehttpc returns operates on headers as a list of tuples with binary keys. +%% Erlcloud expects a list of tuples with string values and lowcase string keys +%% from the underlying http library. +headers_ehttpc_to_erlcloud_response(EhttpcHeaders) -> + [{string:to_lower(to_list_string(K)), to_list_string(V)} || {K, V} <- EhttpcHeaders]. + +%% Ehttpc expects a list of tuples with binary keys. +%% Erlcloud provides a list of tuples with string keys. +headers_erlcloud_request_to_ehttpc(ErlcloudHeaders) -> + [{to_binary(K), V} || {K, V} <- ErlcloudHeaders]. + +join_headers(ErlcloudHeaders, UserSpecialHeaders) -> + ErlcloudHeaders ++ headers_user_to_erlcloud_request(UserSpecialHeaders). + +to_binary(Val) when is_list(Val) -> list_to_binary(Val); +to_binary(Val) when is_binary(Val) -> Val. + +to_list_string(Val) when is_binary(Val) -> + binary_to_list(Val); +to_list_string(Val) when is_list(Val) -> + Val. + +erlcloud_key(Characters) -> + binary_to_list(unicode:characters_to_binary(Characters)). + +response_property(Name, Props) -> + case proplists:get_value(Name, Props) of + undefined -> + %% This schould not happen for valid S3 implementations + ?SLOG(error, #{ + msg => "missing_s3_response_property", + name => Name, + props => Props + }), + error({missing_s3_response_property, Name}); + Value -> + Value + end. + +with_default(undefined, Default) -> Default; +with_default(Value, _Default) -> Value. diff --git a/apps/emqx_s3/src/emqx_s3_profile_conf.erl b/apps/emqx_s3/src/emqx_s3_profile_conf.erl new file mode 100644 index 000000000..87f006bcb --- /dev/null +++ b/apps/emqx_s3/src/emqx_s3_profile_conf.erl @@ -0,0 +1,388 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3_profile_conf). + +-behaviour(gen_server). + +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/types.hrl"). + +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-include("src/emqx_s3.hrl"). + +-export([ + start_link/2, + child_spec/2 +]). + +-export([ + checkout_config/1, + checkout_config/2, + checkin_config/1, + checkin_config/2, + + update_config/2, + update_config/3 +]). + +-export([ + init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3 +]). + +%% For test purposes +-export([ + client_config/2, + start_http_pool/2, + id/1 +]). + +-define(DEFAULT_CALL_TIMEOUT, 5000). + +-define(DEFAULT_HTTP_POOL_TIMEOUT, 60000). +-define(DEAFULT_HTTP_POOL_CLEANUP_INTERVAL, 60000). + +-define(SAFE_CALL_VIA_GPROC(ProfileId, Message, Timeout), + ?SAFE_CALL_VIA_GPROC(id(ProfileId), Message, Timeout, profile_not_found) +). + +-spec child_spec(emqx_s3:profile_id(), emqx_s3:profile_config()) -> supervisor:child_spec(). +child_spec(ProfileId, ProfileConfig) -> + #{ + id => ProfileId, + start => {?MODULE, start_link, [ProfileId, ProfileConfig]}, + restart => permanent, + shutdown => 5000, + type => worker, + modules => [?MODULE] + }. + +-spec start_link(emqx_s3:profile_id(), emqx_s3:profile_config()) -> gen_server:start_ret(). +start_link(ProfileId, ProfileConfig) -> + gen_server:start_link(?VIA_GPROC(id(ProfileId)), ?MODULE, [ProfileId, ProfileConfig], []). + +-spec update_config(emqx_s3:profile_id(), emqx_s3:profile_config()) -> ok_or_error(term()). +update_config(ProfileId, ProfileConfig) -> + update_config(ProfileId, ProfileConfig, ?DEFAULT_CALL_TIMEOUT). + +-spec update_config(emqx_s3:profile_id(), emqx_s3:profile_config(), timeout()) -> + ok_or_error(term()). +update_config(ProfileId, ProfileConfig, Timeout) -> + ?SAFE_CALL_VIA_GPROC(ProfileId, {update_config, ProfileConfig}, Timeout). + +-spec checkout_config(emqx_s3:profile_id()) -> + {ok, emqx_s3_client:config(), emqx_s3_uploader:config()} | {error, profile_not_found}. +checkout_config(ProfileId) -> + checkout_config(ProfileId, ?DEFAULT_CALL_TIMEOUT). + +-spec checkout_config(emqx_s3:profile_id(), timeout()) -> + {ok, emqx_s3_client:config(), emqx_s3_uploader:config()} | {error, profile_not_found}. +checkout_config(ProfileId, Timeout) -> + ?SAFE_CALL_VIA_GPROC(ProfileId, {checkout_config, self()}, Timeout). + +-spec checkin_config(emqx_s3:profile_id()) -> ok | {error, profile_not_found}. +checkin_config(ProfileId) -> + checkin_config(ProfileId, ?DEFAULT_CALL_TIMEOUT). + +-spec checkin_config(emqx_s3:profile_id(), timeout()) -> ok | {error, profile_not_found}. +checkin_config(ProfileId, Timeout) -> + ?SAFE_CALL_VIA_GPROC(ProfileId, {checkin_config, self()}, Timeout). + +%%-------------------------------------------------------------------- +%% gen_server callbacks +%%-------------------------------------------------------------------- + +init([ProfileId, ProfileConfig]) -> + _ = process_flag(trap_exit, true), + ok = cleanup_profile_pools(ProfileId), + case start_http_pool(ProfileId, ProfileConfig) of + {ok, PoolName} -> + HttpPoolCleanupInterval = http_pool_cleanup_interval(ProfileConfig), + {ok, #{ + profile_id => ProfileId, + profile_config => ProfileConfig, + client_config => client_config(ProfileConfig, PoolName), + uploader_config => uploader_config(ProfileConfig), + pool_name => PoolName, + pool_clients => emqx_s3_profile_http_pool_clients:create_table(), + %% We don't expose these options to users currently, but use in tests + http_pool_timeout => http_pool_timeout(ProfileConfig), + http_pool_cleanup_interval => HttpPoolCleanupInterval, + + outdated_pool_cleanup_tref => erlang:send_after( + HttpPoolCleanupInterval, self(), cleanup_outdated + ) + }}; + {error, Reason} -> + {stop, Reason} + end. + +handle_call( + {checkout_config, Pid}, + _From, + #{ + client_config := ClientConfig, + uploader_config := UploaderConfig + } = State +) -> + ok = register_client(Pid, State), + {reply, {ok, ClientConfig, UploaderConfig}, State}; +handle_call({checkin_config, Pid}, _From, State) -> + ok = unregister_client(Pid, State), + {reply, ok, State}; +handle_call( + {update_config, NewProfileConfig}, + _From, + #{profile_id := ProfileId} = State +) -> + case update_http_pool(ProfileId, NewProfileConfig, State) of + {ok, PoolName} -> + NewState = State#{ + profile_config => NewProfileConfig, + client_config => client_config(NewProfileConfig, PoolName), + uploader_config => uploader_config(NewProfileConfig), + http_pool_timeout => http_pool_timeout(NewProfileConfig), + http_pool_cleanup_interval => http_pool_cleanup_interval(NewProfileConfig), + pool_name => PoolName + }, + {reply, ok, NewState}; + {error, Reason} -> + {reply, {error, Reason}, State} + end; +handle_call(_Request, _From, State) -> + {reply, {error, not_implemented}, State}. + +handle_cast(_Request, State) -> + {noreply, State}. + +handle_info({'DOWN', _Ref, process, Pid, _Reason}, State) -> + ok = unregister_client(Pid, State), + {noreply, State}; +handle_info(cleanup_outdated, #{http_pool_cleanup_interval := HttpPoolCleanupInterval} = State0) -> + %% Maybe cleanup asynchoronously + ok = cleanup_outdated_pools(State0), + State1 = State0#{ + outdated_pool_cleanup_tref => erlang:send_after( + HttpPoolCleanupInterval, self(), cleanup_outdated + ) + }, + {noreply, State1}; +handle_info(_Info, State) -> + {noreply, State}. + +terminate(_Reason, #{profile_id := ProfileId}) -> + cleanup_profile_pools(ProfileId). + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +id(ProfileId) -> + {?MODULE, ProfileId}. + +client_config(ProfileConfig, PoolName) -> + HTTPOpts = maps:get(transport_options, ProfileConfig, #{}), + #{ + scheme => scheme(HTTPOpts), + host => maps:get(host, ProfileConfig), + port => maps:get(port, ProfileConfig), + url_expire_time => maps:get(url_expire_time, ProfileConfig), + headers => maps:get(headers, HTTPOpts, #{}), + acl => maps:get(acl, ProfileConfig, undefined), + bucket => maps:get(bucket, ProfileConfig), + access_key_id => maps:get(access_key_id, ProfileConfig, undefined), + secret_access_key => maps:get(secret_access_key, ProfileConfig, undefined), + request_timeout => maps:get(request_timeout, HTTPOpts, undefined), + max_retries => maps:get(max_retries, HTTPOpts, undefined), + pool_type => maps:get(pool_type, HTTPOpts, random), + http_pool => PoolName + }. + +uploader_config(#{max_part_size := MaxPartSize, min_part_size := MinPartSize} = _ProfileConfig) -> + #{ + min_part_size => MinPartSize, + max_part_size => MaxPartSize + }. + +scheme(#{ssl := #{enable := true}}) -> "https://"; +scheme(_TransportOpts) -> "http://". + +start_http_pool(ProfileId, ProfileConfig) -> + HttpConfig = http_config(ProfileConfig), + PoolName = pool_name(ProfileId), + case do_start_http_pool(PoolName, HttpConfig) of + ok -> + ok = emqx_s3_profile_http_pools:register(ProfileId, PoolName), + ok = ?tp(debug, "s3_start_http_pool", #{pool_name => PoolName, profile_id => ProfileId}), + {ok, PoolName}; + {error, _} = Error -> + Error + end. + +update_http_pool(ProfileId, ProfileConfig, #{pool_name := OldPoolName} = State) -> + HttpConfig = http_config(ProfileConfig), + OldHttpConfig = old_http_config(State), + case OldHttpConfig =:= HttpConfig of + true -> + {ok, OldPoolName}; + false -> + PoolName = pool_name(ProfileId), + case do_start_http_pool(PoolName, HttpConfig) of + ok -> + ok = set_old_pool_outdated(State), + ok = emqx_s3_profile_http_pools:register(ProfileId, PoolName), + {ok, PoolName}; + {error, _} = Error -> + Error + end + end. + +pool_name(ProfileId) -> + iolist_to_binary([ + <<"s3-http-">>, + profile_id_to_bin(ProfileId), + <<"-">>, + integer_to_binary(erlang:system_time(millisecond)), + <<"-">>, + integer_to_binary(erlang:unique_integer([positive])) + ]). +profile_id_to_bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8); +profile_id_to_bin(Bin) when is_binary(Bin) -> Bin. + +old_http_config(#{profile_config := ProfileConfig}) -> http_config(ProfileConfig). + +set_old_pool_outdated(#{ + profile_id := ProfileId, pool_name := PoolName, http_pool_timeout := HttpPoolTimeout +}) -> + _ = emqx_s3_profile_http_pools:set_outdated(ProfileId, PoolName, HttpPoolTimeout), + ok. + +cleanup_profile_pools(ProfileId) -> + lists:foreach( + fun(PoolName) -> + ok = stop_http_pool(ProfileId, PoolName) + end, + emqx_s3_profile_http_pools:all(ProfileId) + ). + +register_client(Pid, #{profile_id := ProfileId, pool_clients := PoolClients, pool_name := PoolName}) -> + MRef = monitor(process, Pid), + ok = emqx_s3_profile_http_pool_clients:register(PoolClients, Pid, MRef, PoolName), + _ = emqx_s3_profile_http_pools:register_client(ProfileId, PoolName), + ok. + +unregister_client( + Pid, + #{ + profile_id := ProfileId, pool_clients := PoolClients, pool_name := PoolName + } +) -> + case emqx_s3_profile_http_pool_clients:unregister(PoolClients, Pid) of + undefined -> + ok; + {MRef, PoolName} -> + true = erlang:demonitor(MRef, [flush]), + _ = emqx_s3_profile_http_pools:unregister_client(ProfileId, PoolName), + ok; + {MRef, OutdatedPoolName} -> + true = erlang:demonitor(MRef, [flush]), + ClientNum = emqx_s3_profile_http_pools:unregister_client(ProfileId, OutdatedPoolName), + maybe_stop_outdated_pool(ProfileId, OutdatedPoolName, ClientNum) + end. + +maybe_stop_outdated_pool(ProfileId, OutdatedPoolName, 0) -> + ok = stop_http_pool(ProfileId, OutdatedPoolName); +maybe_stop_outdated_pool(_ProfileId, _OutdatedPoolName, _ClientNum) -> + ok. + +cleanup_outdated_pools(#{profile_id := ProfileId}) -> + lists:foreach( + fun(PoolName) -> + ok = stop_http_pool(ProfileId, PoolName) + end, + emqx_s3_profile_http_pools:outdated(ProfileId) + ). + +%%-------------------------------------------------------------------- +%% HTTP Pool implementation dependent functions +%%-------------------------------------------------------------------- + +http_config( + #{ + host := Host, + port := Port, + transport_options := #{ + pool_type := PoolType, + pool_size := PoolSize, + enable_pipelining := EnablePipelining, + connect_timeout := ConnectTimeout + } = HTTPOpts + } +) -> + {Transport, TransportOpts} = + case scheme(HTTPOpts) of + "http://" -> + {tcp, []}; + "https://" -> + SSLOpts = emqx_tls_lib:to_client_opts(maps:get(ssl, HTTPOpts)), + {tls, SSLOpts} + end, + NTransportOpts = maybe_ipv6_probe(TransportOpts, maps:get(ipv6_probe, HTTPOpts, true)), + [ + {host, Host}, + {port, Port}, + {connect_timeout, ConnectTimeout}, + {keepalive, 30000}, + {pool_type, PoolType}, + {pool_size, PoolSize}, + {transport, Transport}, + {transport_opts, NTransportOpts}, + {enable_pipelining, EnablePipelining} + ]. + +maybe_ipv6_probe(TransportOpts, true) -> + emqx_utils:ipv6_probe(TransportOpts); +maybe_ipv6_probe(TransportOpts, false) -> + TransportOpts. + +http_pool_cleanup_interval(ProfileConfig) -> + maps:get( + http_pool_cleanup_interval, ProfileConfig, ?DEAFULT_HTTP_POOL_CLEANUP_INTERVAL + ). + +http_pool_timeout(ProfileConfig) -> + maps:get( + http_pool_timeout, ProfileConfig, ?DEFAULT_HTTP_POOL_TIMEOUT + ). + +stop_http_pool(ProfileId, PoolName) -> + case ehttpc_sup:stop_pool(PoolName) of + ok -> + ok; + {error, Reason} -> + ?SLOG(error, #{msg => "ehttpc_pool_stop_fail", pool_name => PoolName, reason => Reason}), + ok + end, + ok = emqx_s3_profile_http_pools:unregister(ProfileId, PoolName), + ok = ?tp(debug, "s3_stop_http_pool", #{pool_name => PoolName}). + +do_start_http_pool(PoolName, HttpConfig) -> + ?SLOG(warning, #{msg => "s3_start_http_pool", pool_name => PoolName, config => HttpConfig}), + case ehttpc_sup:start_pool(PoolName, HttpConfig) of + {ok, _} -> + ?SLOG(warning, #{msg => "s3_start_http_pool_success", pool_name => PoolName}), + ok; + {error, _} = Error -> + ?SLOG(error, #{msg => "s3_start_http_pool_fail", pool_name => PoolName, error => Error}), + Error + end. diff --git a/apps/emqx_s3/src/emqx_s3_profile_http_pool_clients.erl b/apps/emqx_s3/src/emqx_s3_profile_http_pool_clients.erl new file mode 100644 index 000000000..19e4d2ddb --- /dev/null +++ b/apps/emqx_s3/src/emqx_s3_profile_http_pool_clients.erl @@ -0,0 +1,36 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3_profile_http_pool_clients). + +-export([ + create_table/0, + + register/4, + unregister/2 +]). + +-define(TAB, ?MODULE). + +-spec create_table() -> ok. +create_table() -> + ets:new(?TAB, [ + private, + set + ]). + +-spec register(ets:tid(), pid(), reference(), emqx_s3_profile_http_pools:pool_name()) -> ok. +register(Tab, Pid, MRef, PoolName) -> + true = ets:insert(Tab, {Pid, {MRef, PoolName}}), + ok. + +-spec unregister(ets:tid(), pid()) -> + {reference(), emqx_s3_profile_http_pools:pool_name()} | undefined. +unregister(Tab, Pid) -> + case ets:take(Tab, Pid) of + [{Pid, {MRef, PoolName}}] -> + {MRef, PoolName}; + [] -> + undefined + end. diff --git a/apps/emqx_s3/src/emqx_s3_profile_http_pools.erl b/apps/emqx_s3/src/emqx_s3_profile_http_pools.erl new file mode 100644 index 000000000..944f2037d --- /dev/null +++ b/apps/emqx_s3/src/emqx_s3_profile_http_pools.erl @@ -0,0 +1,124 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3_profile_http_pools). + +-include_lib("stdlib/include/ms_transform.hrl"). + +-export([ + create_table/0, + + register/2, + unregister/2, + + register_client/2, + unregister_client/2, + + set_outdated/3, + + outdated/1, + all/1 +]). + +-export_type([pool_name/0]). + +-define(TAB, ?MODULE). + +-type pool_name() :: ecpool:pool_name(). + +-type pool_key() :: {emqx_s3:profile_id(), pool_name()}. + +-record(pool, { + key :: pool_key(), + client_count = 0 :: integer(), + deadline = undefined :: undefined | integer(), + extra = #{} :: map() +}). + +-spec create_table() -> ok. +create_table() -> + _ = ets:new(?TAB, [ + named_table, + public, + ordered_set, + {keypos, #pool.key}, + {read_concurrency, true}, + {write_concurrency, true} + ]), + ok. + +-spec register(emqx_s3:profile_id(), pool_name()) -> + ok. +register(ProfileId, PoolName) -> + Key = key(ProfileId, PoolName), + true = ets:insert(?TAB, #pool{ + key = Key, + client_count = 0, + deadline = undefined, + extra = #{} + }), + ok. + +-spec unregister(emqx_s3:profile_id(), pool_name()) -> + ok. +unregister(ProfileId, PoolName) -> + Key = key(ProfileId, PoolName), + true = ets:delete(?TAB, Key), + ok. + +-spec register_client(emqx_s3:profile_id(), pool_name()) -> + integer(). +register_client(ProfileId, PoolName) -> + Key = key(ProfileId, PoolName), + ets:update_counter(?TAB, Key, {#pool.client_count, 1}). + +-spec unregister_client(emqx_s3:profile_id(), pool_name()) -> + integer(). +unregister_client(ProfileId, PoolName) -> + Key = key(ProfileId, PoolName), + try + ets:update_counter(?TAB, Key, {#pool.client_count, -1}) + catch + error:badarg -> + undefined + end. + +-spec set_outdated(emqx_s3:profile_id(), pool_name(), integer()) -> + ok. +set_outdated(ProfileId, PoolName, Timeout) -> + Key = key(ProfileId, PoolName), + Now = erlang:monotonic_time(millisecond), + _ = ets:update_element(?TAB, Key, {#pool.deadline, Now + Timeout}), + ok. + +-spec outdated(emqx_s3:profile_id()) -> + [pool_name()]. +outdated(ProfileId) -> + Now = erlang:monotonic_time(millisecond), + MS = ets:fun2ms( + fun(#pool{key = {CurProfileId, CurPoolName}, deadline = CurDeadline}) when + CurProfileId =:= ProfileId andalso + CurDeadline =/= undefined andalso CurDeadline < Now + -> + CurPoolName + end + ), + ets:select(?TAB, MS). + +-spec all(emqx_s3:profile_id()) -> + [pool_name()]. +all(ProfileId) -> + MS = ets:fun2ms( + fun(#pool{key = {CurProfileId, CurPoolName}}) when CurProfileId =:= ProfileId -> + CurPoolName + end + ), + ets:select(?TAB, MS). + +%%-------------------------------------------------------------------- +%% Helpers +%%-------------------------------------------------------------------- + +key(ProfileId, PoolName) -> + {ProfileId, PoolName}. diff --git a/apps/emqx_s3/src/emqx_s3_profile_sup.erl b/apps/emqx_s3/src/emqx_s3_profile_sup.erl new file mode 100644 index 000000000..c39fc9f4b --- /dev/null +++ b/apps/emqx_s3/src/emqx_s3_profile_sup.erl @@ -0,0 +1,48 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3_profile_sup). + +-behaviour(supervisor). + +-include_lib("emqx/include/types.hrl"). + +-export([ + start_link/2, + child_spec/2 +]). + +-export([init/1]). + +-spec start_link(emqx_s3:profile_id(), emqx_s3:profile_config()) -> supervisor:start_ret(). +start_link(ProfileId, ProfileConfig) -> + supervisor:start_link(?MODULE, [ProfileId, ProfileConfig]). + +-spec child_spec(emqx_s3:profile_id(), emqx_s3:profile_config()) -> supervisor:child_spec(). +child_spec(ProfileId, ProfileConfig) -> + #{ + id => ProfileId, + start => {?MODULE, start_link, [ProfileId, ProfileConfig]}, + restart => permanent, + shutdown => 5000, + type => supervisor, + modules => [?MODULE] + }. + +%%-------------------------------------------------------------------- +%% supervisor callbacks +%%------------------------------------------------------------------- + +init([ProfileId, ProfileConfig]) -> + SupFlags = #{ + strategy => one_for_one, + intensity => 10, + period => 5 + }, + ChildSpecs = [ + %% Order matters + emqx_s3_profile_conf:child_spec(ProfileId, ProfileConfig), + emqx_s3_profile_uploader_sup:child_spec(ProfileId) + ], + {ok, {SupFlags, ChildSpecs}}. diff --git a/apps/emqx_s3/src/emqx_s3_profile_uploader_sup.erl b/apps/emqx_s3/src/emqx_s3_profile_uploader_sup.erl new file mode 100644 index 000000000..fb7b93a15 --- /dev/null +++ b/apps/emqx_s3/src/emqx_s3_profile_uploader_sup.erl @@ -0,0 +1,75 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3_profile_uploader_sup). + +-behaviour(supervisor). + +-include_lib("emqx/include/types.hrl"). +-include_lib("emqx/include/logger.hrl"). + +-include("src/emqx_s3.hrl"). + +-export([ + start_link/1, + child_spec/1, + id/1, + start_uploader/2 +]). + +-export([init/1]). + +-export_type([id/0]). + +-type id() :: {?MODULE, emqx_s3:profile_id()}. + +-spec start_link(emqx_s3:profile_id()) -> supervisor:start_ret(). +start_link(ProfileId) -> + supervisor:start_link(?VIA_GPROC(id(ProfileId)), ?MODULE, [ProfileId]). + +-spec child_spec(emqx_s3:profile_id()) -> supervisor:child_spec(). +child_spec(ProfileId) -> + #{ + id => id(ProfileId), + start => {?MODULE, start_link, [ProfileId]}, + restart => permanent, + shutdown => 5000, + type => supervisor, + modules => [?MODULE] + }. + +-spec id(emqx_s3:profile_id()) -> id(). +id(ProfileId) -> + {?MODULE, ProfileId}. + +-spec start_uploader(emqx_s3:profile_id(), emqx_s3_uploader:opts()) -> + supervisor:start_ret() | {error, profile_not_found}. +start_uploader(ProfileId, Opts) -> + try supervisor:start_child(?VIA_GPROC(id(ProfileId)), [Opts]) of + Result -> Result + catch + exit:{noproc, _} -> {error, profile_not_found} + end. + +%%-------------------------------------------------------------------- +%% supervisor callbacks +%%------------------------------------------------------------------- + +init([ProfileId]) -> + SupFlags = #{ + strategy => simple_one_for_one, + intensity => 10, + period => 5 + }, + ChildSpecs = [ + #{ + id => emqx_s3_uploader, + start => {emqx_s3_uploader, start_link, [ProfileId]}, + restart => temporary, + shutdown => 5000, + type => worker, + modules => [emqx_s3_uploader] + } + ], + {ok, {SupFlags, ChildSpecs}}. diff --git a/apps/emqx_s3/src/emqx_s3_schema.erl b/apps/emqx_s3/src/emqx_s3_schema.erl new file mode 100644 index 000000000..5866f8c2b --- /dev/null +++ b/apps/emqx_s3/src/emqx_s3_schema.erl @@ -0,0 +1,172 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3_schema). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). + +-import(hoconsc, [mk/2, ref/2]). + +-export([roots/0, fields/1, namespace/0, tags/0, desc/1]). + +-export([translate/1]). +-export([translate/2]). + +roots() -> + [s3]. + +namespace() -> "s3". + +tags() -> + [<<"S3">>]. + +fields(s3) -> + [ + {access_key_id, + mk( + string(), + #{ + desc => ?DESC("access_key_id"), + required => false + } + )}, + {secret_access_key, + mk( + string(), + #{ + desc => ?DESC("secret_access_key"), + required => false, + sensitive => true + } + )}, + {bucket, + mk( + string(), + #{ + desc => ?DESC("bucket"), + required => true + } + )}, + {host, + mk( + string(), + #{ + desc => ?DESC("host"), + required => true + } + )}, + {port, + mk( + pos_integer(), + #{ + desc => ?DESC("port"), + required => true + } + )}, + {url_expire_time, + mk( + emqx_schema:duration_s(), + #{ + default => "1h", + desc => ?DESC("url_expire_time"), + required => false + } + )}, + {min_part_size, + mk( + emqx_schema:bytesize(), + #{ + default => "5mb", + desc => ?DESC("min_part_size"), + required => true, + validator => fun part_size_validator/1 + } + )}, + {max_part_size, + mk( + emqx_schema:bytesize(), + #{ + default => "5gb", + desc => ?DESC("max_part_size"), + required => true, + validator => fun part_size_validator/1 + } + )}, + {acl, + mk( + hoconsc:enum([ + private, + public_read, + public_read_write, + authenticated_read, + bucket_owner_read, + bucket_owner_full_control + ]), + #{ + desc => ?DESC("acl"), + required => false + } + )}, + {transport_options, + mk( + ref(?MODULE, transport_options), + #{ + desc => ?DESC("transport_options"), + required => false + } + )} + ]; +fields(transport_options) -> + [ + {ipv6_probe, + mk( + boolean(), + #{ + default => true, + desc => ?DESC("ipv6_probe"), + required => false + } + )} + ] ++ + props_without( + [base_url, max_retries, retry_interval, request], emqx_connector_http:fields(config) + ) ++ + props_with( + [headers, max_retries, request_timeout], emqx_connector_http:fields("request") + ). + +desc(s3) -> + "S3 connection options"; +desc(transport_options) -> + "Options for the HTTP transport layer used by the S3 client". + +translate(Conf) -> + translate(Conf, #{}). + +translate(Conf, OptionsIn) -> + Options = maps:merge(#{atom_key => true}, OptionsIn), + #{s3 := TranslatedConf} = hocon_tconf:check_plain( + emqx_s3_schema, #{<<"s3">> => Conf}, Options, [s3] + ), + TranslatedConf. + +%%-------------------------------------------------------------------- +%% Helpers +%%-------------------------------------------------------------------- + +props_with(Keys, Proplist) -> + lists:filter(fun({K, _}) -> lists:member(K, Keys) end, Proplist). + +props_without(Keys, Proplist) -> + lists:filter(fun({K, _}) -> not lists:member(K, Keys) end, Proplist). + +part_size_validator(PartSizeLimit) -> + case + PartSizeLimit >= 5 * 1024 * 1024 andalso + PartSizeLimit =< 5 * 1024 * 1024 * 1024 + of + true -> ok; + false -> {error, "must be at least 5mb and less than 5gb"} + end. diff --git a/apps/emqx_s3/src/emqx_s3_sup.erl b/apps/emqx_s3/src/emqx_s3_sup.erl new file mode 100644 index 000000000..0f6b0160b --- /dev/null +++ b/apps/emqx_s3/src/emqx_s3_sup.erl @@ -0,0 +1,47 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3_sup). + +-behaviour(supervisor). + +-include_lib("emqx/include/types.hrl"). + +-export([ + start_link/0, + start_profile/2, + stop_profile/1 +]). + +-export([init/1]). + +-spec start_link() -> supervisor:start_ret(). +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +-spec start_profile(emqx_s3:profile_id(), emqx_s3:profile_config()) -> supervisor:startchild_ret(). +start_profile(ProfileId, ProfileConfig) -> + supervisor:start_child(?MODULE, emqx_s3_profile_sup:child_spec(ProfileId, ProfileConfig)). + +-spec stop_profile(emqx_s3:profile_id()) -> ok_or_error(term()). +stop_profile(ProfileId) -> + case supervisor:terminate_child(?MODULE, ProfileId) of + ok -> + supervisor:delete_child(?MODULE, ProfileId); + {error, Reason} -> + {error, Reason} + end. + +%%-------------------------------------------------------------------- +%% supervisor callbacks +%%------------------------------------------------------------------- + +init([]) -> + ok = emqx_s3_profile_http_pools:create_table(), + SupFlags = #{ + strategy => one_for_one, + intensity => 10, + period => 5 + }, + {ok, {SupFlags, []}}. diff --git a/apps/emqx_s3/src/emqx_s3_uploader.erl b/apps/emqx_s3/src/emqx_s3_uploader.erl new file mode 100644 index 000000000..595612f62 --- /dev/null +++ b/apps/emqx_s3/src/emqx_s3_uploader.erl @@ -0,0 +1,329 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3_uploader). + +-include_lib("emqx/include/types.hrl"). + +-behaviour(gen_statem). + +-export([ + start_link/2, + + write/2, + write/3, + + complete/1, + complete/2, + + abort/1, + abort/2 +]). + +-export([ + init/1, + callback_mode/0, + handle_event/4, + terminate/3, + code_change/4, + format_status/1, + format_status/2 +]). + +-export_type([opts/0]). + +-type opts() :: #{ + key := string(), + headers => emqx_s3_client:headers() +}. + +-type data() :: #{ + profile_id := emqx_s3:profile_id(), + client := emqx_s3_client:client(), + key := emqx_s3_client:key(), + buffer := iodata(), + buffer_size := non_neg_integer(), + min_part_size := pos_integer(), + max_part_size := pos_integer(), + upload_id := undefined | emqx_s3_client:upload_id(), + etags := [emqx_s3_client:etag()], + part_number := emqx_s3_client:part_number(), + headers := emqx_s3_client:headers() +}. + +%% 5MB +-define(DEFAULT_MIN_PART_SIZE, 5242880). +%% 5GB +-define(DEFAULT_MAX_PART_SIZE, 5368709120). + +-define(DEFAULT_TIMEOUT, 30000). + +-spec start_link(emqx_s3:profile_id(), opts()) -> gen_statem:start_ret(). +start_link(ProfileId, #{key := Key} = Opts) when is_list(Key) -> + gen_statem:start_link(?MODULE, [ProfileId, Opts], []). + +-spec write(pid(), iodata()) -> ok_or_error(term()). +write(Pid, WriteData) -> + write(Pid, WriteData, ?DEFAULT_TIMEOUT). + +-spec write(pid(), iodata(), timeout()) -> ok_or_error(term()). +write(Pid, WriteData, Timeout) -> + gen_statem:call(Pid, {write, wrap(WriteData)}, Timeout). + +-spec complete(pid()) -> ok_or_error(term()). +complete(Pid) -> + complete(Pid, ?DEFAULT_TIMEOUT). + +-spec complete(pid(), timeout()) -> ok_or_error(term()). +complete(Pid, Timeout) -> + gen_statem:call(Pid, complete, Timeout). + +-spec abort(pid()) -> ok_or_error(term()). +abort(Pid) -> + abort(Pid, ?DEFAULT_TIMEOUT). + +-spec abort(pid(), timeout()) -> ok_or_error(term()). +abort(Pid, Timeout) -> + gen_statem:call(Pid, abort, Timeout). + +%%-------------------------------------------------------------------- +%% gen_statem callbacks +%%-------------------------------------------------------------------- + +callback_mode() -> handle_event_function. + +init([ProfileId, #{key := Key} = Opts]) -> + process_flag(trap_exit, true), + {ok, ClientConfig, UploaderConfig} = emqx_s3_profile_conf:checkout_config(ProfileId), + Client = client(ClientConfig), + {ok, upload_not_started, #{ + profile_id => ProfileId, + client => Client, + headers => maps:get(headers, Opts, #{}), + key => Key, + buffer => [], + buffer_size => 0, + min_part_size => maps:get(min_part_size, UploaderConfig, ?DEFAULT_MIN_PART_SIZE), + max_part_size => maps:get(max_part_size, UploaderConfig, ?DEFAULT_MAX_PART_SIZE), + upload_id => undefined, + etags => [], + part_number => 1 + }}. + +handle_event({call, From}, {write, WriteDataWrapped}, State, Data0) -> + WriteData = unwrap(WriteDataWrapped), + case is_valid_part(WriteData, Data0) of + true -> + handle_write(State, From, WriteData, Data0); + false -> + {keep_state_and_data, {reply, From, {error, {too_large, iolist_size(WriteData)}}}} + end; +handle_event({call, From}, complete, upload_not_started, Data0) -> + case put_object(Data0) of + ok -> + {stop_and_reply, normal, {reply, From, ok}}; + {error, _} = Error -> + {stop_and_reply, Error, {reply, From, Error}, Data0} + end; +handle_event({call, From}, complete, upload_started, Data0) -> + case complete_upload(Data0) of + {ok, Data1} -> + {stop_and_reply, normal, {reply, From, ok}, Data1}; + {error, _} = Error -> + {stop_and_reply, Error, {reply, From, Error}, Data0} + end; +handle_event({call, From}, abort, upload_not_started, _Data) -> + {stop_and_reply, normal, {reply, From, ok}}; +handle_event({call, From}, abort, upload_started, Data0) -> + case abort_upload(Data0) of + ok -> + {stop_and_reply, normal, {reply, From, ok}}; + {error, _} = Error -> + {stop_and_reply, Error, {reply, From, Error}, Data0} + end. + +handle_write(upload_not_started, From, WriteData, Data0) -> + Data1 = append_buffer(Data0, WriteData), + case maybe_start_upload(Data1) of + not_started -> + {keep_state, Data1, {reply, From, ok}}; + {started, Data2} -> + case upload_part(Data2) of + {ok, Data3} -> + {next_state, upload_started, Data3, {reply, From, ok}}; + {error, _} = Error -> + {stop_and_reply, Error, {reply, From, Error}, Data2} + end; + {error, _} = Error -> + {stop_and_reply, Error, {reply, From, Error}, Data1} + end; +handle_write(upload_started, From, WriteData, Data0) -> + Data1 = append_buffer(Data0, WriteData), + case maybe_upload_part(Data1) of + {ok, Data2} -> + {keep_state, Data2, {reply, From, ok}}; + {error, _} = Error -> + {stop_and_reply, Error, {reply, From, Error}, Data1} + end. + +terminate(Reason, _State, #{client := Client, upload_id := UploadId, key := Key}) when + (UploadId =/= undefined) andalso (Reason =/= normal) +-> + emqx_s3_client:abort_multipart(Client, Key, UploadId); +terminate(_Reason, _State, _Data) -> + ok. + +code_change(_OldVsn, StateName, State, _Extra) -> + {ok, StateName, State}. + +format_status(#{data := #{client := Client} = Data} = Status) -> + Status#{ + data => Data#{ + client => emqx_s3_client:format(Client), + buffer => [<<"...">>] + } + }. + +format_status(_Opt, [PDict, State, #{client := Client} = Data]) -> + #{ + data => Data#{ + client => emqx_s3_client:format(Client), + buffer => [<<"...">>] + }, + state => State, + pdict => PDict + }. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +-spec maybe_start_upload(data()) -> not_started | {started, data()} | {error, term()}. +maybe_start_upload(#{buffer_size := BufferSize, min_part_size := MinPartSize} = Data) -> + case BufferSize >= MinPartSize of + true -> + start_upload(Data); + false -> + not_started + end. + +-spec start_upload(data()) -> {started, data()} | {error, term()}. +start_upload(#{client := Client, key := Key, headers := Headers} = Data) -> + case emqx_s3_client:start_multipart(Client, Headers, Key) of + {ok, UploadId} -> + NewData = Data#{upload_id => UploadId}, + {started, NewData}; + {error, _} = Error -> + Error + end. + +-spec maybe_upload_part(data()) -> ok_or_error(data(), term()). +maybe_upload_part(#{buffer_size := BufferSize, min_part_size := MinPartSize} = Data) -> + case BufferSize >= MinPartSize of + true -> + upload_part(Data); + false -> + {ok, Data} + end. + +-spec upload_part(data()) -> ok_or_error(data(), term()). +upload_part(#{buffer_size := 0} = Data) -> + {ok, Data}; +upload_part( + #{ + client := Client, + key := Key, + upload_id := UploadId, + buffer := Buffer, + part_number := PartNumber, + etags := ETags + } = Data +) -> + case emqx_s3_client:upload_part(Client, Key, UploadId, PartNumber, Buffer) of + {ok, ETag} -> + NewData = Data#{ + buffer => [], + buffer_size => 0, + part_number => PartNumber + 1, + etags => [{PartNumber, ETag} | ETags] + }, + {ok, NewData}; + {error, _} = Error -> + Error + end. + +-spec complete_upload(data()) -> ok_or_error(data(), term()). +complete_upload( + #{ + client := Client, + key := Key, + upload_id := UploadId + } = Data0 +) -> + case upload_part(Data0) of + {ok, #{etags := ETags} = Data1} -> + case + emqx_s3_client:complete_multipart( + Client, Key, UploadId, lists:reverse(ETags) + ) + of + ok -> + {ok, Data1}; + {error, _} = Error -> + Error + end; + {error, _} = Error -> + Error + end. + +-spec abort_upload(data()) -> ok_or_error(term()). +abort_upload( + #{ + client := Client, + key := Key, + upload_id := UploadId + } +) -> + case emqx_s3_client:abort_multipart(Client, Key, UploadId) of + ok -> + ok; + {error, _} = Error -> + Error + end. + +-spec put_object(data()) -> ok_or_error(term()). +put_object( + #{ + client := Client, + key := Key, + buffer := Buffer, + headers := Headers + } +) -> + case emqx_s3_client:put_object(Client, Headers, Key, Buffer) of + ok -> + ok; + {error, _} = Error -> + Error + end. + +-spec append_buffer(data(), iodata()) -> data(). +append_buffer(#{buffer := Buffer, buffer_size := BufferSize} = Data, WriteData) -> + Data#{ + buffer => [Buffer, WriteData], + buffer_size => BufferSize + iolist_size(WriteData) + }. + +-compile({inline, [wrap/1, unwrap/1]}). +wrap(Data) -> + fun() -> Data end. + +unwrap(WrappedData) -> + WrappedData(). + +is_valid_part(WriteData, #{max_part_size := MaxPartSize, buffer_size := BufferSize}) -> + BufferSize + iolist_size(WriteData) =< MaxPartSize. + +client(Config) -> + emqx_s3_client:create(Config). diff --git a/apps/emqx_s3/test/certs/ca.crt b/apps/emqx_s3/test/certs/ca.crt new file mode 100644 index 000000000..8a9dafccd --- /dev/null +++ b/apps/emqx_s3/test/certs/ca.crt @@ -0,0 +1,29 @@ +-----BEGIN CERTIFICATE----- +MIIE5DCCAswCCQCF3o0gIdaNDjANBgkqhkiG9w0BAQsFADA0MRIwEAYDVQQKDAlF +TVFYIFRlc3QxHjAcBgNVBAMMFUNlcnRpZmljYXRlIEF1dGhvcml0eTAeFw0yMTEy +MzAwODQxMTFaFw00OTA1MTcwODQxMTFaMDQxEjAQBgNVBAoMCUVNUVggVGVzdDEe +MBwGA1UEAwwVQ2VydGlmaWNhdGUgQXV0aG9yaXR5MIICIjANBgkqhkiG9w0BAQEF +AAOCAg8AMIICCgKCAgEAqmqSrxyH16j63QhqGLT1UO8I+m6BM3HfnJQM8laQdtJ0 +WgHqCh0/OphH3S7v4SfF4fNJDEJWMWuuzJzU9cTqHPLzhvo3+ZHcMIENgtY2p2Cf +7AQjEqFViEDyv2ZWNEe76BJeShntdY5NZr4gIPar99YGG/Ln8YekspleV+DU38rE +EX9WzhgBr02NN9z4NzIxeB+jdvPnxcXs3WpUxzfnUjOQf/T1tManvSdRbFmKMbxl +A8NLYK3oAYm8EbljWUINUNN6loqYhbigKv8bvo5S4xvRqmX86XB7sc0SApngtNcg +O0EKn8z/KVPDskE+8lMfGMiU2e2Tzw6Rph57mQPOPtIp5hPiKRik7ST9n0p6piXW +zRLplJEzSjf40I1u+VHmpXlWI/Fs8b1UkDSMiMVJf0LyWb4ziBSZOY2LtZzWHbWj +LbNgxQcwSS29tKgUwfEFmFcm+iOM59cPfkl2IgqVLh5h4zmKJJbfQKSaYb5fcKRf +50b1qsN40VbR3Pk/0lJ0/WqgF6kZCExmT1qzD5HJES/5grjjKA4zIxmHOVU86xOF +ouWvtilVR4PGkzmkFvwK5yRhBUoGH/A9BurhqOc0QCGay1kqHQFA6se4JJS+9KOS +x8Rn1Nm6Pi7sd6Le3cKmHTlyl5a/ofKqTCX2Qh+v/7y62V1V1wnoh3ipRjdPTnMC +AwEAATANBgkqhkiG9w0BAQsFAAOCAgEARCqaocvlMFUQjtFtepO2vyG1krn11xJ0 +e7md26i+g8SxCCYqQ9IqGmQBg0Im8fyNDKRN/LZoj5+A4U4XkG1yya91ZIrPpWyF +KUiRAItchNj3g1kHmI2ckl1N//6Kpx3DPaS7qXZaN3LTExf6Ph+StE1FnS0wVF+s +tsNIf6EaQ+ZewW3pjdlLeAws3jvWKUkROc408Ngvx74zbbKo/zAC4tz8oH9ZcpsT +WD8enVVEeUQKI6ItcpZ9HgTI9TFWgfZ1vYwvkoRwNIeabYI62JKmLEo2vGfGwWKr +c+GjnJ/tlVI2DpPljfWOnQ037/7yyJI/zo65+HPRmGRD6MuW/BdPDYOvOZUTcQKh +kANi5THSbJJgZcG3jb1NLebaUQ1H0zgVjn0g3KhUV+NJQYk8RQ7rHtB+MySqTKlM +kRkRjfTfR0Ykxpks7Mjvsb6NcZENf08ZFPd45+e/ptsxpiKu4e4W4bV7NZDvNKf9 +0/aD3oGYNMiP7s+KJ1lRSAjnBuG21Yk8FpzG+yr8wvJhV8aFgNQ5wIH86SuUTmN0 +5bVzFEIcUejIwvGoQEctNHBlOwHrb7zmB6OwyZeMapdXBQ+9UDhYg8ehDqdDOdfn +wsBcnjD2MwNhlE1hjL+tZWLNwSHiD6xx3LvNoXZu2HK8Cp3SOrkE69cFghYMIZZb +T+fp6tNL6LE= +-----END CERTIFICATE----- diff --git a/apps/emqx_s3/test/emqx_s3_SUITE.erl b/apps/emqx_s3/test/emqx_s3_SUITE.erl new file mode 100644 index 000000000..287dcb597 --- /dev/null +++ b/apps/emqx_s3/test/emqx_s3_SUITE.erl @@ -0,0 +1,66 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + {ok, _} = application:ensure_all_started(emqx_s3), + Config. + +end_per_suite(_Config) -> + ok = application:stop(emqx_s3). + +%%-------------------------------------------------------------------- +%% Test cases +%%-------------------------------------------------------------------- + +t_start_stop_update(_Config) -> + ProfileId = <<"test">>, + ProfileConfig = profile_config(), + + ?assertMatch( + ok, + emqx_s3:start_profile(ProfileId, ProfileConfig) + ), + + ?assertMatch( + {error, _}, + emqx_s3:start_profile(ProfileId, ProfileConfig) + ), + + ?assertEqual( + ok, + emqx_s3:update_profile(ProfileId, ProfileConfig) + ), + + ?assertMatch( + {error, _}, + emqx_s3:update_profile(<<"unknown">>, ProfileConfig) + ), + + ?assertEqual( + ok, + emqx_s3:stop_profile(ProfileId) + ), + + ?assertMatch( + {error, _}, + emqx_s3:stop_profile(ProfileId) + ). + +%%-------------------------------------------------------------------- +%% Helpers +%%-------------------------------------------------------------------- + +profile_config() -> + emqx_s3_test_helpers:base_config(tcp). diff --git a/apps/emqx_s3/test/emqx_s3_client_SUITE.erl b/apps/emqx_s3/test/emqx_s3_client_SUITE.erl new file mode 100644 index 000000000..434510867 --- /dev/null +++ b/apps/emqx_s3/test/emqx_s3_client_SUITE.erl @@ -0,0 +1,170 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3_client_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-define(PROFILE_ID, atom_to_binary(?MODULE)). + +all() -> + [ + {group, tcp}, + {group, tls} + ]. + +groups() -> + AllCases = emqx_common_test_helpers:all(?MODULE), + PoolGroups = [ + {group, pool_random}, + {group, pool_hash} + ], + [ + {tcp, [], PoolGroups}, + {tls, [], PoolGroups}, + {pool_random, [], AllCases}, + {pool_hash, [], AllCases} + ]. + +init_per_suite(Config) -> + {ok, _} = application:ensure_all_started(emqx_s3), + Config. + +end_per_suite(_Config) -> + ok = application:stop(emqx_s3). + +init_per_group(ConnTypeGroup, Config) when ConnTypeGroup =:= tcp; ConnTypeGroup =:= tls -> + [{conn_type, ConnTypeGroup} | Config]; +init_per_group(PoolTypeGroup, Config) when + PoolTypeGroup =:= pool_random; PoolTypeGroup =:= pool_hash +-> + PoolType = + case PoolTypeGroup of + pool_random -> random; + pool_hash -> hash + end, + [{pool_type, PoolType} | Config]. +end_per_group(_ConnType, _Config) -> + ok. + +init_per_testcase(_TestCase, Config0) -> + ConnType = ?config(conn_type, Config0), + + Bucket = emqx_s3_test_helpers:unique_bucket(), + TestAwsConfig = emqx_s3_test_helpers:aws_config(ConnType), + ok = erlcloud_s3:create_bucket(Bucket, TestAwsConfig), + Config1 = [ + {key, emqx_s3_test_helpers:unique_key()}, + {bucket, Bucket}, + {aws_config, TestAwsConfig} + | Config0 + ], + {ok, PoolName} = emqx_s3_profile_conf:start_http_pool(?PROFILE_ID, profile_config(Config1)), + [{ehttpc_pool_name, PoolName} | Config1]. + +end_per_testcase(_TestCase, Config) -> + ok = ehttpc_sup:stop_pool(?config(ehttpc_pool_name, Config)). + +%%-------------------------------------------------------------------- +%% Test cases +%%-------------------------------------------------------------------- + +t_multipart_upload(Config) -> + Key = ?config(key, Config), + + Client = client(Config), + + {ok, UploadId} = emqx_s3_client:start_multipart(Client, Key), + + Data = data(6_000_000), + + {ok, Etag1} = emqx_s3_client:upload_part(Client, Key, UploadId, 1, Data), + {ok, Etag2} = emqx_s3_client:upload_part(Client, Key, UploadId, 2, Data), + + ok = emqx_s3_client:complete_multipart( + Client, Key, UploadId, [{1, Etag1}, {2, Etag2}] + ). + +t_simple_put(Config) -> + Key = ?config(key, Config), + + Client = client(Config), + + Data = data(6_000_000), + + ok = emqx_s3_client:put_object(Client, Key, Data). + +t_list(Config) -> + Key = ?config(key, Config), + + Client = client(Config), + + ok = emqx_s3_client:put_object(Client, Key, <<"data">>), + + {ok, List} = emqx_s3_client:list(Client, Key), + + [KeyInfo] = proplists:get_value(contents, List), + ?assertMatch( + #{ + key := Key, + size := 4, + etag := _, + last_modified := _ + }, + maps:from_list(KeyInfo) + ). + +t_url(Config) -> + Key = ?config(key, Config), + + Client = client(Config), + ok = emqx_s3_client:put_object(Client, Key, <<"data">>), + + Url = emqx_s3_client:uri(Client, Key), + + ?assertMatch( + {ok, {{_StatusLine, 200, "OK"}, _Headers, "data"}}, + httpc:request(Url) + ). + +t_no_acl(Config) -> + Key = ?config(key, Config), + + ClientConfig = emqx_s3_profile_conf:client_config( + profile_config(Config), ?config(ehttpc_pool_name, Config) + ), + Client = emqx_s3_client:create(maps:without([acl], ClientConfig)), + + ok = emqx_s3_client:put_object(Client, Key, <<"data">>). + +%%-------------------------------------------------------------------- +%% Helpers +%%-------------------------------------------------------------------- + +client(Config) -> + ClientConfig = emqx_s3_profile_conf:client_config( + profile_config(Config), ?config(ehttpc_pool_name, Config) + ), + emqx_s3_client:create(ClientConfig). + +profile_config(Config) -> + ProfileConfig0 = emqx_s3_test_helpers:base_config(?config(conn_type, Config)), + ProfileConfig1 = maps:put( + bucket, + ?config(bucket, Config), + ProfileConfig0 + ), + ProfileConfig2 = emqx_utils_maps:deep_put( + [transport_options, pool_type], + ProfileConfig1, + ?config(pool_type, Config) + ), + ProfileConfig2. + +data(Size) -> + iolist_to_binary([$a || _ <- lists:seq(1, Size)]). diff --git a/apps/emqx_s3/test/emqx_s3_profile_conf_SUITE.erl b/apps/emqx_s3/test/emqx_s3_profile_conf_SUITE.erl new file mode 100644 index 000000000..433cfe07b --- /dev/null +++ b/apps/emqx_s3/test/emqx_s3_profile_conf_SUITE.erl @@ -0,0 +1,282 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3_profile_conf_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("emqx/include/asserts.hrl"). + +all() -> emqx_common_test_helpers:all(?MODULE). + +suite() -> [{timetrap, {minutes, 1}}]. + +init_per_suite(Config) -> + {ok, _} = application:ensure_all_started(emqx_s3), + Config. + +end_per_suite(_Config) -> + ok = application:stop(emqx_s3). + +init_per_testcase(_TestCase, Config) -> + ok = snabbkaffe:start_trace(), + TestAwsConfig = emqx_s3_test_helpers:aws_config(tcp), + + Bucket = emqx_s3_test_helpers:unique_bucket(), + ok = erlcloud_s3:create_bucket(Bucket, TestAwsConfig), + + ProfileBaseConfig = emqx_s3_test_helpers:base_config(tcp), + ProfileConfig = ProfileBaseConfig#{bucket => Bucket}, + ok = emqx_s3:start_profile(profile_id(), ProfileConfig), + + [{profile_config, ProfileConfig} | Config]. + +end_per_testcase(_TestCase, _Config) -> + ok = snabbkaffe:stop(), + _ = emqx_s3:stop_profile(profile_id()). + +%%-------------------------------------------------------------------- +%% Test cases +%%-------------------------------------------------------------------- + +t_regular_outdated_pool_cleanup(Config) -> + _ = process_flag(trap_exit, true), + Key = emqx_s3_test_helpers:unique_key(), + {ok, Pid} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + + [OldPool] = emqx_s3_profile_http_pools:all(profile_id()), + + ProfileBaseConfig = ?config(profile_config, Config), + ProfileConfig = emqx_utils_maps:deep_put( + [transport_options, pool_size], ProfileBaseConfig, 16 + ), + ok = emqx_s3:update_profile(profile_id(), ProfileConfig), + + ?assertEqual( + 2, + length(emqx_s3_profile_http_pools:all(profile_id())) + ), + + ?assertWaitEvent( + ok = emqx_s3_uploader:abort(Pid), + #{?snk_kind := "s3_stop_http_pool", pool_name := OldPool}, + 1000 + ), + + [NewPool] = emqx_s3_profile_http_pools:all(profile_id()), + + ?assertWaitEvent( + ok = emqx_s3:stop_profile(profile_id()), + #{?snk_kind := "s3_stop_http_pool", pool_name := NewPool}, + 1000 + ), + + ?assertEqual( + 0, + length(emqx_s3_profile_http_pools:all(profile_id())) + ). + +t_timeout_pool_cleanup(Config) -> + _ = process_flag(trap_exit, true), + + %% We restart the profile to set `http_pool_timeout` value suitable for test + ok = emqx_s3:stop_profile(profile_id()), + ProfileBaseConfig = ?config(profile_config, Config), + ProfileConfig = ProfileBaseConfig#{ + http_pool_timeout => 500, + http_pool_cleanup_interval => 100 + }, + ok = emqx_s3:start_profile(profile_id(), ProfileConfig), + + %% Start uploader + Key = emqx_s3_test_helpers:unique_key(), + {ok, Pid} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + ok = emqx_s3_uploader:write(Pid, <<"data">>), + + [OldPool] = emqx_s3_profile_http_pools:all(profile_id()), + + NewProfileConfig = emqx_utils_maps:deep_put( + [transport_options, pool_size], ProfileConfig, 16 + ), + + %% We update profile to create new pool and wait for the old one to be stopped by timeout + ?assertWaitEvent( + ok = emqx_s3:update_profile(profile_id(), NewProfileConfig), + #{?snk_kind := "s3_stop_http_pool", pool_name := OldPool}, + 1000 + ), + + %% The uploader now has no valid pool and should fail + ?assertMatch( + {error, _}, + emqx_s3_uploader:complete(Pid) + ). + +t_checkout_no_profile(_Config) -> + ?assertEqual( + {error, profile_not_found}, + emqx_s3_profile_conf:checkout_config(<<"no_such_profile">>) + ). + +t_httpc_pool_start_error(Config) -> + %% `ehhtpc_pool`s are lazy so it is difficult to trigger an error + %% passing some bad connection options. + %% So we emulate some unknown crash with `meck`. + meck:new(ehttpc_pool, [passthrough]), + meck:expect(ehttpc_pool, init, fun(_) -> meck:raise(error, badarg) end), + + ?assertMatch( + {error, _}, + emqx_s3:start_profile(<<"profile">>, ?config(profile_config, Config)) + ). + +t_httpc_pool_update_error(Config) -> + %% `ehhtpc_pool`s are lazy so it is difficult to trigger an error + %% passing some bad connection options. + %% So we emulate some unknown crash with `meck`. + meck:new(ehttpc_pool, [passthrough]), + meck:expect(ehttpc_pool, init, fun(_) -> meck:raise(error, badarg) end), + + ProfileBaseConfig = ?config(profile_config, Config), + NewProfileConfig = emqx_utils_maps:deep_put( + [transport_options, pool_size], ProfileBaseConfig, 16 + ), + + ?assertMatch( + {error, _}, + emqx_s3:start_profile(<<"profile">>, NewProfileConfig) + ). + +t_orphaned_pools_cleanup(_Config) -> + ProfileId = profile_id(), + Pid = gproc:where({n, l, emqx_s3_profile_conf:id(ProfileId)}), + + %% We kill conf and wait for it to restart + %% and create a new pool + ?assertWaitEvent( + exit(Pid, kill), + #{?snk_kind := "s3_start_http_pool", profile_id := ProfileId}, + 1000 + ), + + %% We should still have only one pool + ?assertEqual( + 1, + length(emqx_s3_profile_http_pools:all(ProfileId)) + ). + +t_orphaned_pools_cleanup_non_graceful(_Config) -> + ProfileId = profile_id(), + Pid = gproc:where({n, l, emqx_s3_profile_conf:id(ProfileId)}), + + %% We stop pool, conf server should not fail when attempting to stop it once more + [PoolName] = emqx_s3_profile_http_pools:all(ProfileId), + ok = ehttpc_pool:stop_pool(PoolName), + + %% We kill conf and wait for it to restart + %% and create a new pool + ?assertWaitEvent( + exit(Pid, kill), + #{?snk_kind := "s3_start_http_pool", profile_id := ProfileId}, + 1000 + ), + + %% We should still have only one pool + ?assertEqual( + 1, + length(emqx_s3_profile_http_pools:all(ProfileId)) + ). + +t_checkout_client(Config) -> + ProfileId = profile_id(), + Key = emqx_s3_test_helpers:unique_key(), + Caller = self(), + Pid = spawn_link(fun() -> + emqx_s3:with_client( + ProfileId, + fun(Client) -> + receive + put_object -> + Caller ! {put_object, emqx_s3_client:put_object(Client, Key, <<"data">>)} + end, + receive + list_objects -> + Caller ! {list_objects, emqx_s3_client:list(Client, [])} + end + end + ), + Caller ! client_released, + receive + stop -> ok + end + end), + + %% Ask spawned process to put object + Pid ! put_object, + receive + {put_object, ok} -> ok + after 1000 -> + ct:fail("put_object fail") + end, + + %% Now change config for the profile + ProfileBaseConfig = ?config(profile_config, Config), + NewProfileConfig0 = ProfileBaseConfig#{bucket => <<"new_bucket">>}, + NewProfileConfig1 = emqx_utils_maps:deep_put( + [transport_options, pool_size], NewProfileConfig0, 16 + ), + ok = emqx_s3:update_profile(profile_id(), NewProfileConfig1), + + %% We should have two pools now, because the old one is still in use + %% by the spawned process + ?assertEqual( + 2, + length(emqx_s3_profile_http_pools:all(ProfileId)) + ), + + %% Ask spawned process to list objects + Pid ! list_objects, + receive + {list_objects, Result} -> + {ok, OkResult} = Result, + Contents = proplists:get_value(contents, OkResult), + ?assertEqual(1, length(Contents)), + ?assertEqual(Key, proplists:get_value(key, hd(Contents))) + after 1000 -> + ct:fail("list_objects fail") + end, + + %% Wait till spawned process releases client + receive + client_released -> ok + after 1000 -> + ct:fail("client not released") + end, + + %% We should have only one pool now, because the old one is released + ?assertEqual( + 1, + length(emqx_s3_profile_http_pools:all(ProfileId)) + ). + +t_unknown_messages(_Config) -> + Pid = gproc:where({n, l, emqx_s3_profile_conf:id(profile_id())}), + + Pid ! unknown, + ok = gen_server:cast(Pid, unknown), + + ?assertEqual( + {error, not_implemented}, + gen_server:call(Pid, unknown) + ). + +%%-------------------------------------------------------------------- +%% Test helpers +%%-------------------------------------------------------------------- + +profile_id() -> + <<"test">>. diff --git a/apps/emqx_s3/test/emqx_s3_schema_SUITE.erl b/apps/emqx_s3/test/emqx_s3_schema_SUITE.erl new file mode 100644 index 000000000..63f659da0 --- /dev/null +++ b/apps/emqx_s3/test/emqx_s3_schema_SUITE.erl @@ -0,0 +1,172 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3_schema_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +%%-------------------------------------------------------------------- +%% Test cases +%%-------------------------------------------------------------------- + +t_minimal_config(_Config) -> + ?assertMatch( + #{ + bucket := "bucket", + host := "s3.us-east-1.endpoint.com", + port := 443, + min_part_size := 5242880, + transport_options := + #{ + connect_timeout := 15000, + enable_pipelining := 100, + pool_size := 8, + pool_type := random, + ssl := #{enable := false} + } + }, + emqx_s3_schema:translate(#{ + <<"bucket">> => <<"bucket">>, + <<"host">> => <<"s3.us-east-1.endpoint.com">>, + <<"port">> => 443 + }) + ). + +t_full_config(_Config) -> + ?assertMatch( + #{ + access_key_id := "access_key_id", + acl := public_read, + bucket := "bucket", + host := "s3.us-east-1.endpoint.com", + min_part_size := 10485760, + port := 443, + secret_access_key := "secret_access_key", + transport_options := + #{ + connect_timeout := 30000, + enable_pipelining := 200, + headers := #{<<"x-amz-acl">> := <<"public-read">>}, + max_retries := 3, + pool_size := 10, + pool_type := random, + request_timeout := 10000, + ssl := + #{ + cacertfile := <<"cacertfile.crt">>, + certfile := <<"server.crt">>, + ciphers := ["ECDHE-RSA-AES256-GCM-SHA384"], + depth := 10, + enable := true, + keyfile := <<"server.key">>, + reuse_sessions := true, + secure_renegotiate := true, + server_name_indication := "some-host", + verify := verify_peer, + versions := ['tlsv1.2'] + } + } + }, + emqx_s3_schema:translate(#{ + <<"access_key_id">> => <<"access_key_id">>, + <<"secret_access_key">> => <<"secret_access_key">>, + <<"bucket">> => <<"bucket">>, + <<"host">> => <<"s3.us-east-1.endpoint.com">>, + <<"port">> => 443, + <<"min_part_size">> => <<"10mb">>, + <<"acl">> => <<"public_read">>, + <<"transport_options">> => #{ + <<"connect_timeout">> => 30000, + <<"enable_pipelining">> => 200, + <<"pool_size">> => 10, + <<"pool_type">> => <<"random">>, + <<"ssl">> => #{ + <<"enable">> => true, + <<"keyfile">> => <<"server.key">>, + <<"certfile">> => <<"server.crt">>, + <<"cacertfile">> => <<"cacertfile.crt">>, + <<"server_name_indication">> => <<"some-host">>, + <<"verify">> => <<"verify_peer">>, + <<"versions">> => [<<"tlsv1.2">>], + <<"ciphers">> => [<<"ECDHE-RSA-AES256-GCM-SHA384">>] + }, + <<"request_timeout">> => <<"10s">>, + <<"max_retries">> => 3, + <<"headers">> => #{ + <<"x-amz-acl">> => <<"public-read">> + } + } + }) + ). + +t_sensitive_config_hidden(_Config) -> + ?assertMatch( + #{ + access_key_id := "access_key_id", + secret_access_key := <<"******">> + }, + emqx_s3_schema:translate( + #{ + <<"bucket">> => <<"bucket">>, + <<"host">> => <<"s3.us-east-1.endpoint.com">>, + <<"port">> => 443, + <<"access_key_id">> => <<"access_key_id">>, + <<"secret_access_key">> => <<"secret_access_key">> + }, + % NOTE: this is what Config API handler is doing + #{obfuscate_sensitive_values => true} + ) + ). + +t_invalid_limits(_Config) -> + ?assertException( + throw, + {emqx_s3_schema, [#{kind := validation_error, path := "s3.min_part_size"}]}, + emqx_s3_schema:translate(#{ + <<"bucket">> => <<"bucket">>, + <<"host">> => <<"s3.us-east-1.endpoint.com">>, + <<"port">> => 443, + <<"min_part_size">> => <<"1mb">> + }) + ), + + ?assertException( + throw, + {emqx_s3_schema, [#{kind := validation_error, path := "s3.min_part_size"}]}, + emqx_s3_schema:translate(#{ + <<"bucket">> => <<"bucket">>, + <<"host">> => <<"s3.us-east-1.endpoint.com">>, + <<"port">> => 443, + <<"min_part_size">> => <<"100000gb">> + }) + ), + + ?assertException( + throw, + {emqx_s3_schema, [#{kind := validation_error, path := "s3.max_part_size"}]}, + emqx_s3_schema:translate(#{ + <<"bucket">> => <<"bucket">>, + <<"host">> => <<"s3.us-east-1.endpoint.com">>, + <<"port">> => 443, + <<"max_part_size">> => <<"1mb">> + }) + ), + + ?assertException( + throw, + {emqx_s3_schema, [#{kind := validation_error, path := "s3.max_part_size"}]}, + emqx_s3_schema:translate(#{ + <<"bucket">> => <<"bucket">>, + <<"host">> => <<"s3.us-east-1.endpoint.com">>, + <<"port">> => 443, + <<"max_part_size">> => <<"100000gb">> + }) + ). diff --git a/apps/emqx_s3/test/emqx_s3_test_helpers.erl b/apps/emqx_s3/test/emqx_s3_test_helpers.erl new file mode 100644 index 000000000..a73f618af --- /dev/null +++ b/apps/emqx_s3/test/emqx_s3_test_helpers.erl @@ -0,0 +1,140 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3_test_helpers). + +-compile(nowarn_export_all). +-compile(export_all). + +-define(ACCESS_KEY_ID, "minioadmin"). +-define(SECRET_ACCESS_KEY, "minioadmin"). + +-define(TOXIPROXY_HOST, "toxiproxy"). +-define(TOXIPROXY_PORT, 8474). + +-define(TCP_HOST, ?TOXIPROXY_HOST). +-define(TCP_PORT, 19000). +-define(TLS_HOST, ?TOXIPROXY_HOST). +-define(TLS_PORT, 19100). + +-include_lib("erlcloud/include/erlcloud_aws.hrl"). + +-export([ + aws_config/1, + base_raw_config/1, + base_config/1, + + unique_key/0, + unique_bucket/0, + + with_failure/3 +]). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- + +aws_config(tcp) -> + aws_config(tcp, ?TCP_HOST, ?TCP_PORT); +aws_config(tls) -> + aws_config(tls, ?TLS_HOST, ?TLS_PORT). + +aws_config(tcp, Host, Port) -> + erlcloud_s3_new( + ?ACCESS_KEY_ID, + ?SECRET_ACCESS_KEY, + Host, + Port, + "http://" + ); +aws_config(tls, Host, Port) -> + erlcloud_s3_new( + ?ACCESS_KEY_ID, + ?SECRET_ACCESS_KEY, + Host, + Port, + "https://" + ). + +base_raw_config(tcp) -> + #{ + <<"bucket">> => <<"bucket">>, + <<"access_key_id">> => bin(?ACCESS_KEY_ID), + <<"secret_access_key">> => bin(?SECRET_ACCESS_KEY), + <<"host">> => ?TCP_HOST, + <<"port">> => ?TCP_PORT, + <<"max_part_size">> => 10 * 1024 * 1024, + <<"transport_options">> => + #{ + <<"request_timeout">> => 2000 + } + }; +base_raw_config(tls) -> + #{ + <<"bucket">> => <<"bucket">>, + <<"access_key_id">> => bin(?ACCESS_KEY_ID), + <<"secret_access_key">> => bin(?SECRET_ACCESS_KEY), + <<"host">> => ?TLS_HOST, + <<"port">> => ?TLS_PORT, + <<"max_part_size">> => 10 * 1024 * 1024, + <<"transport_options">> => + #{ + <<"request_timeout">> => 2000, + <<"ssl">> => #{ + <<"enable">> => true, + <<"cacertfile">> => bin(cert_path("ca.crt")), + <<"server_name_indication">> => <<"authn-server">>, + <<"verify">> => <<"verify_peer">> + } + } + }. + +base_config(ConnType) -> + emqx_s3_schema:translate(base_raw_config(ConnType)). + +unique_key() -> + "key-" ++ integer_to_list(erlang:system_time(millisecond)) ++ "-" ++ + integer_to_list(erlang:unique_integer([positive])). + +unique_bucket() -> + "bucket-" ++ integer_to_list(erlang:system_time(millisecond)) ++ "-" ++ + integer_to_list(erlang:unique_integer([positive])). + +with_failure(_ConnType, ehttpc_500, Fun) -> + try + meck:new(ehttpc, [passthrough, no_history]), + meck:expect(ehttpc, request, fun(_, _, _, _, _) -> {ok, 500, []} end), + Fun() + after + meck:unload(ehttpc) + end; +with_failure(ConnType, FailureType, Fun) -> + emqx_common_test_helpers:with_failure( + FailureType, + toxproxy_name(ConnType), + ?TOXIPROXY_HOST, + ?TOXIPROXY_PORT, + Fun + ). + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +toxproxy_name(tcp) -> "minio_tcp"; +toxproxy_name(tls) -> "minio_tls". + +cert_path(FileName) -> + Dir = code:lib_dir(emqx_s3, test), + filename:join([Dir, <<"certs">>, FileName]). + +bin(String) when is_list(String) -> list_to_binary(String); +bin(Binary) when is_binary(Binary) -> Binary. + +erlcloud_s3_new(AccessKeyId, SecretAccessKey, Host, Port, Scheme) -> + AwsConfig = erlcloud_s3:new(AccessKeyId, SecretAccessKey, Host, Port), + AwsConfig#aws_config{ + s3_scheme = Scheme, + s3_bucket_access_method = path + }. diff --git a/apps/emqx_s3/test/emqx_s3_uploader_SUITE.erl b/apps/emqx_s3/test/emqx_s3_uploader_SUITE.erl new file mode 100644 index 000000000..6ba0e3ed9 --- /dev/null +++ b/apps/emqx_s3/test/emqx_s3_uploader_SUITE.erl @@ -0,0 +1,591 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3_uploader_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-define(assertProcessExited(Reason, Pid), + receive + {'DOWN', _, _, Pid, Reason} -> + ok + after 3000 -> + ct:fail("uploader process did not exit") + end +). + +-define(assertObjectEqual(Value, AwsConfig, Bucket, Key), + ?assertEqual( + Value, + proplists:get_value( + content, + erlcloud_s3:get_object( + Bucket, + Key, + AwsConfig + ) + ) + ) +). + +all() -> + [ + {group, tcp}, + {group, tls} + ]. + +groups() -> + [ + {tcp, [ + {group, common_cases}, + {group, tcp_cases} + ]}, + {tls, [ + {group, common_cases}, + {group, tls_cases} + ]}, + {common_cases, [], [ + t_happy_path_simple_put, + t_happy_path_multi, + t_abort_multi, + t_abort_simple_put, + t_signed_url_download, + t_signed_nonascii_url_download, + + {group, noconn_errors}, + {group, timeout_errors}, + {group, http_errors} + ]}, + + {tcp_cases, [ + t_config_switch, + t_config_switch_http_settings, + t_too_large, + t_no_profile + ]}, + + {tls_cases, [ + t_tls_error + ]}, + + {noconn_errors, [{group, transport_errors}]}, + {timeout_errors, [{group, transport_errors}]}, + {http_errors, [{group, transport_errors}]}, + + {transport_errors, [ + t_start_multipart_error, + t_upload_part_error, + t_complete_multipart_error, + t_abort_multipart_error, + t_put_object_error + ]} + ]. + +suite() -> [{timetrap, {minutes, 1}}]. + +init_per_suite(Config) -> + {ok, _} = application:ensure_all_started(emqx_s3), + Config. + +end_per_suite(_Config) -> + ok = application:stop(emqx_s3). + +init_per_group(Group, Config) when Group =:= tcp orelse Group =:= tls -> + [{conn_type, Group} | Config]; +init_per_group(noconn_errors, Config) -> + [{failure, down} | Config]; +init_per_group(timeout_errors, Config) -> + [{failure, timeout} | Config]; +init_per_group(http_errors, Config) -> + [{failure, ehttpc_500} | Config]; +init_per_group(_ConnType, Config) -> + Config. + +end_per_group(_ConnType, _Config) -> + ok. + +init_per_testcase(_TestCase, Config) -> + ok = snabbkaffe:start_trace(), + ConnType = ?config(conn_type, Config), + TestAwsConfig = emqx_s3_test_helpers:aws_config(ConnType), + + Bucket = emqx_s3_test_helpers:unique_bucket(), + ok = erlcloud_s3:create_bucket(Bucket, TestAwsConfig), + + ProfileBaseConfig = emqx_s3_test_helpers:base_config(ConnType), + ProfileConfig = ProfileBaseConfig#{bucket => Bucket}, + ok = emqx_s3:start_profile(profile_id(), ProfileConfig), + + [{bucket, Bucket}, {test_aws_config, TestAwsConfig}, {profile_config, ProfileConfig} | Config]. + +end_per_testcase(_TestCase, _Config) -> + ok = snabbkaffe:stop(), + _ = emqx_s3:stop_profile(profile_id()). + +%%-------------------------------------------------------------------- +%% Test cases +%%-------------------------------------------------------------------- + +t_happy_path_simple_put(Config) -> + Key = emqx_s3_test_helpers:unique_key(), + {ok, Pid} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + + _ = erlang:monitor(process, Pid), + + Data = data($a, 1024, 10), + + lists:foreach( + fun(Chunk) -> + ?assertEqual( + ok, + emqx_s3_uploader:write(Pid, Chunk) + ) + end, + Data + ), + + ok = emqx_s3_uploader:complete(Pid), + + ?assertProcessExited( + normal, + Pid + ), + + ?assertObjectEqual( + iolist_to_binary(Data), + ?config(test_aws_config, Config), + ?config(bucket, Config), + Key + ). + +t_happy_path_multi(Config) -> + Key = emqx_s3_test_helpers:unique_key(), + {ok, Pid} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + + _ = erlang:monitor(process, Pid), + + Data = data($a, 1024 * 1024, 10), + + lists:foreach( + fun(Chunk) -> + ?assertEqual( + ok, + emqx_s3_uploader:write(Pid, Chunk) + ) + end, + Data + ), + + ok = emqx_s3_uploader:complete(Pid), + + ?assertProcessExited( + normal, + Pid + ), + + ?assertObjectEqual( + iolist_to_binary(Data), + ?config(test_aws_config, Config), + ?config(bucket, Config), + Key + ). + +t_signed_url_download(_Config) -> + Prefix = emqx_s3_test_helpers:unique_key(), + Key = Prefix ++ "/ascii.txt", + + {ok, Data} = upload(Key, 1024, 5), + + SignedUrl = emqx_s3:with_client(profile_id(), fun(Client) -> + emqx_s3_client:uri(Client, Key) + end), + + {ok, {_, _, Body}} = httpc:request(get, {SignedUrl, []}, [], []), + + ?assertEqual( + iolist_to_binary(Data), + iolist_to_binary(Body) + ). + +t_signed_nonascii_url_download(_Config) -> + Prefix = emqx_s3_test_helpers:unique_key(), + Key = Prefix ++ "/unicode-🫠.txt", + + {ok, Data} = upload(Key, 1024 * 1024, 8), + + SignedUrl = emqx_s3:with_client(profile_id(), fun(Client) -> + emqx_s3_client:uri(Client, Key) + end), + + {ok, {_, _, Body}} = httpc:request(get, {SignedUrl, []}, [], []), + + ?assertEqual( + iolist_to_binary(Data), + iolist_to_binary(Body) + ). + +t_abort_multi(Config) -> + Key = emqx_s3_test_helpers:unique_key(), + {ok, Pid} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + + _ = erlang:monitor(process, Pid), + + [Data] = data($a, 6 * 1024 * 1024, 1), + + ok = emqx_s3_uploader:write(Pid, Data), + + ?assertMatch( + [], + list_objects(Config) + ), + + ok = emqx_s3_uploader:abort(Pid), + + ?assertMatch( + [], + list_objects(Config) + ), + + ?assertProcessExited( + normal, + Pid + ). + +t_abort_simple_put(_Config) -> + Key = emqx_s3_test_helpers:unique_key(), + {ok, Pid} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + + _ = erlang:monitor(process, Pid), + + [Data] = data($a, 10 * 1024, 1), + + ok = emqx_s3_uploader:write(Pid, Data), + + ok = emqx_s3_uploader:abort(Pid), + + ?assertProcessExited( + normal, + Pid + ). + +t_config_switch(Config) -> + Key = emqx_s3_test_helpers:unique_key(), + OldBucket = ?config(bucket, Config), + {ok, Pid0} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + + [Data0, Data1] = data($a, 6 * 1024 * 1024, 2), + + ok = emqx_s3_uploader:write(Pid0, Data0), + + %% Switch to the new config, but without changing HTTP settings + ProfileConfig = ?config(profile_config, Config), + NewBucket = emqx_s3_test_helpers:unique_bucket(), + ok = erlcloud_s3:create_bucket(NewBucket, ?config(test_aws_config, Config)), + NewProfileConfig = ProfileConfig#{bucket => NewBucket}, + + ok = emqx_s3:update_profile(profile_id(), NewProfileConfig), + + %% Already started uploader should be OK and use previous config + ok = emqx_s3_uploader:write(Pid0, Data1), + ok = emqx_s3_uploader:complete(Pid0), + + ?assertObjectEqual( + iolist_to_binary([Data0, Data1]), + ?config(test_aws_config, Config), + OldBucket, + Key + ), + + %% Now check that new uploader uses new config + {ok, Pid1} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + ok = emqx_s3_uploader:write(Pid1, Data0), + ok = emqx_s3_uploader:complete(Pid1), + + ?assertObjectEqual( + iolist_to_binary(Data0), + ?config(test_aws_config, Config), + NewBucket, + Key + ). + +t_config_switch_http_settings(Config) -> + Key = emqx_s3_test_helpers:unique_key(), + OldBucket = ?config(bucket, Config), + {ok, Pid0} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + + [Data0, Data1] = data($a, 6 * 1024 * 1024, 2), + + ok = emqx_s3_uploader:write(Pid0, Data0), + + %% Switch to the new config, completely changing HTTP settings (tcp -> tls) + NewBucket = emqx_s3_test_helpers:unique_bucket(), + NewTestAwsConfig = emqx_s3_test_helpers:aws_config(tls), + ok = erlcloud_s3:create_bucket(NewBucket, NewTestAwsConfig), + NewProfileConfig0 = emqx_s3_test_helpers:base_config(tls), + NewProfileConfig1 = NewProfileConfig0#{bucket => NewBucket}, + + ok = emqx_s3:update_profile(profile_id(), NewProfileConfig1), + + %% Already started uploader should be OK and use previous config + ok = emqx_s3_uploader:write(Pid0, Data1), + ok = emqx_s3_uploader:complete(Pid0), + + ?assertObjectEqual( + iolist_to_binary([Data0, Data1]), + ?config(test_aws_config, Config), + OldBucket, + Key + ), + + %% Now check that new uploader uses new config + {ok, Pid1} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + ok = emqx_s3_uploader:write(Pid1, Data0), + ok = emqx_s3_uploader:complete(Pid1), + + ?assertObjectEqual( + iolist_to_binary(Data0), + NewTestAwsConfig, + NewBucket, + Key + ). + +t_start_multipart_error(Config) -> + _ = process_flag(trap_exit, true), + + Key = emqx_s3_test_helpers:unique_key(), + {ok, Pid} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + + _ = erlang:monitor(process, Pid), + + [Data] = data($a, 6 * 1024 * 1024, 1), + + emqx_s3_test_helpers:with_failure( + ?config(conn_type, Config), + ?config(failure, Config), + fun() -> + ?assertMatch( + {error, _}, + emqx_s3_uploader:write(Pid, Data) + ) + end + ), + + ?assertProcessExited( + {error, _}, + Pid + ). + +t_upload_part_error(Config) -> + _ = process_flag(trap_exit, true), + + Key = emqx_s3_test_helpers:unique_key(), + {ok, Pid} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + + _ = erlang:monitor(process, Pid), + + [Data0, Data1] = data($a, 6 * 1024 * 1024, 2), + + ok = emqx_s3_uploader:write(Pid, Data0), + + emqx_s3_test_helpers:with_failure( + ?config(conn_type, Config), + ?config(failure, Config), + fun() -> + ?assertMatch( + {error, _}, + emqx_s3_uploader:write(Pid, Data1) + ) + end + ), + + ?assertProcessExited( + {error, _}, + Pid + ). + +t_abort_multipart_error(Config) -> + _ = process_flag(trap_exit, true), + + Key = emqx_s3_test_helpers:unique_key(), + {ok, Pid} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + + _ = erlang:monitor(process, Pid), + + [Data] = data($a, 6 * 1024 * 1024, 1), + + ok = emqx_s3_uploader:write(Pid, Data), + + emqx_s3_test_helpers:with_failure( + ?config(conn_type, Config), + ?config(failure, Config), + fun() -> + ?assertMatch( + {error, _}, + emqx_s3_uploader:abort(Pid) + ) + end + ), + + ?assertProcessExited( + {error, _}, + Pid + ). + +t_complete_multipart_error(Config) -> + _ = process_flag(trap_exit, true), + + Key = emqx_s3_test_helpers:unique_key(), + {ok, Pid} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + + _ = erlang:monitor(process, Pid), + + [Data] = data($a, 6 * 1024 * 1024, 1), + + ok = emqx_s3_uploader:write(Pid, Data), + + emqx_s3_test_helpers:with_failure( + ?config(conn_type, Config), + ?config(failure, Config), + fun() -> + ?assertMatch( + {error, _}, + emqx_s3_uploader:complete(Pid) + ) + end + ), + + ?assertProcessExited( + {error, _}, + Pid + ). + +t_put_object_error(Config) -> + _ = process_flag(trap_exit, true), + + Key = emqx_s3_test_helpers:unique_key(), + {ok, Pid} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + + _ = erlang:monitor(process, Pid), + + %% Little data to avoid multipart upload + [Data] = data($a, 1024, 1), + + emqx_s3_test_helpers:with_failure( + ?config(conn_type, Config), + ?config(failure, Config), + fun() -> + ok = emqx_s3_uploader:write(Pid, Data), + ?assertMatch( + {error, _}, + emqx_s3_uploader:complete(Pid) + ) + end + ), + + ?assertProcessExited( + {error, _}, + Pid + ). + +t_too_large(Config) -> + Key = emqx_s3_test_helpers:unique_key(), + {ok, Pid} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + + _ = erlang:monitor(process, Pid), + + [Data] = data($a, 1024, 1), + + [DataLarge] = data($a, 20 * 1024 * 1024, 1), + + ?assertMatch( + {error, {too_large, _}}, + emqx_s3_uploader:write(Pid, DataLarge) + ), + + ok = emqx_s3_uploader:write(Pid, Data), + ok = emqx_s3_uploader:complete(Pid), + + ?assertProcessExited( + normal, + Pid + ), + + ?assertObjectEqual( + iolist_to_binary(Data), + ?config(test_aws_config, Config), + ?config(bucket, Config), + Key + ). + +t_tls_error(Config) -> + _ = process_flag(trap_exit, true), + + ProfileBaseConfig = ?config(profile_config, Config), + ProfileConfig = emqx_utils_maps:deep_put( + [transport_options, ssl, server_name_indication], ProfileBaseConfig, "invalid-hostname" + ), + ok = emqx_s3:update_profile(profile_id(), ProfileConfig), + Key = emqx_s3_test_helpers:unique_key(), + {ok, Pid} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + + _ = erlang:monitor(process, Pid), + + [Data] = data($a, 6 * 1024 * 1024, 1), + + ?assertMatch( + {error, _}, + emqx_s3_uploader:write(Pid, Data) + ), + + ?assertProcessExited( + {error, _}, + Pid + ). + +t_no_profile(_Config) -> + Key = emqx_s3_test_helpers:unique_key(), + ?assertMatch( + {error, profile_not_found}, + emqx_s3:start_uploader(<<"no-profile">>, #{key => Key}) + ). + +%%-------------------------------------------------------------------- +%% Test helpers +%%-------------------------------------------------------------------- + +profile_id() -> + <<"test">>. + +data(Byte, ChunkSize, ChunkCount) -> + Chunk = iolist_to_binary([Byte || _ <- lists:seq(1, ChunkSize)]), + [Chunk || _ <- lists:seq(1, ChunkCount)]. + +list_objects(Config) -> + Props = erlcloud_s3:list_objects(?config(bucket, Config), [], ?config(test_aws_config, Config)), + proplists:get_value(contents, Props). + +upload(Key, ChunkSize, ChunkCount) -> + {ok, Pid} = emqx_s3:start_uploader(profile_id(), #{key => Key}), + + _ = erlang:monitor(process, Pid), + + Data = data($a, ChunkSize, ChunkCount), + + ok = lists:foreach( + fun(Chunk) -> ?assertEqual(ok, emqx_s3_uploader:write(Pid, Chunk)) end, + Data + ), + + ok = emqx_s3_uploader:complete(Pid), + + ok = ?assertProcessExited( + normal, + Pid + ), + + {ok, Data}. diff --git a/changes/ce/fix-10340-en.md b/changes/ce/fix-10340.en.md similarity index 97% rename from changes/ce/fix-10340-en.md rename to changes/ce/fix-10340.en.md index c9ae7b81b..0f0436cec 100644 --- a/changes/ce/fix-10340-en.md +++ b/changes/ce/fix-10340.en.md @@ -1,4 +1,4 @@ -Fixed the issue that could lead to crash logs being printed when stopping EMQ X via systemd. +Fixed the issue that could lead to crash logs being printed when stopping EMQX via systemd. ``` 2023-03-29T16:43:25.915761+08:00 [error] Generic server memsup terminating. Reason: {port_died,normal}. Last message: {'EXIT',<0.2117.0>,{port_died,normal}}. State: [{data,[{"Timeout",60000}]},{items,{"Memory Usage",[{"Allocated",929959936},{"Total",3832242176}]}},{items,{"Worst Memory User",[{"Pid",<0.2031.0>},{"Memory",4720472}]}}]. 2023-03-29T16:43:25.924764+08:00 [error] crasher: initial call: memsup:init/1, pid: <0.2116.0>, registered_name: memsup, exit: {{port_died,normal},[{gen_server,handle_common_reply,8,[{file,"gen_server.erl"},{line,811}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,226}]}]}, ancestors: [os_mon_sup,<0.2114.0>], message_queue_len: 0, messages: [], links: [<0.2115.0>], dictionary: [], trap_exit: true, status: running, heap_size: 4185, stack_size: 29, reductions: 187637; neighbours: diff --git a/changes/ce/fix-10724.en.md b/changes/ce/fix-10724.en.md new file mode 100644 index 000000000..a38a9e842 --- /dev/null +++ b/changes/ce/fix-10724.en.md @@ -0,0 +1 @@ +A summary has been added for all endpoints in the HTTP API documentation (accessible at "http://emqx_host_name:18083/api-docs"). diff --git a/changes/ce/fix-10726.en.md b/changes/ce/fix-10726.en.md new file mode 100644 index 000000000..d7a1f5bab --- /dev/null +++ b/changes/ce/fix-10726.en.md @@ -0,0 +1 @@ +Validate Health Check Interval and Auto Restart Interval against the range from 1ms to 1 hour. diff --git a/changes/ce/fix-10743.en.md b/changes/ce/fix-10743.en.md new file mode 100644 index 000000000..95e6b3652 --- /dev/null +++ b/changes/ce/fix-10743.en.md @@ -0,0 +1 @@ +Fixes an issue where trying to get a bridge info or metrics could result in a crash when a node is joining a cluster. diff --git a/changes/ce/fix-10746.en.md b/changes/ce/fix-10746.en.md new file mode 100644 index 000000000..c8cf2d084 --- /dev/null +++ b/changes/ce/fix-10746.en.md @@ -0,0 +1 @@ +Add missing support of the event `$events/delivery_dropped` into the rule engine test API `rule_test`. diff --git a/changes/ce/fix-10747.en.md b/changes/ce/fix-10747.en.md new file mode 100644 index 000000000..a07c24074 --- /dev/null +++ b/changes/ce/fix-10747.en.md @@ -0,0 +1 @@ +Refactor date and time functions, `format_date` and `date_to_unix_ts`, in the rule engine to fix the implementation problem. diff --git a/changes/ce/fix-10755.en.md b/changes/ce/fix-10755.en.md new file mode 100644 index 000000000..6c887166b --- /dev/null +++ b/changes/ce/fix-10755.en.md @@ -0,0 +1,10 @@ +Fixed data bridge resource update race condition. + +In the 'delete + create' process for EMQX resource updates, +long bridge creation times could cause dashboard request timeouts. +If a bridge resource update was initiated before completion of its creation, +it led to an erroneous deletion from the runtime, despite being present in the config file. + +This fix addresses the race condition in bridge resource updates, +ensuring the accurate identification and addition of new resources, +maintaining consistency between runtime and configuration file statuses. diff --git a/changes/ce/fix-10761.en.md b/changes/ce/fix-10761.en.md new file mode 100644 index 000000000..3df84bb86 --- /dev/null +++ b/changes/ce/fix-10761.en.md @@ -0,0 +1 @@ +Fixing the issue where the default value of SSL certificate for Dashboard Listener was not correctly interpolated, which caused HTTPS to be inaccessible when verify_peer and cacertfile were using the default configuration. diff --git a/changes/ce/perf-10511.en.md b/changes/ce/perf-10511.en.md new file mode 100644 index 000000000..953ecf693 --- /dev/null +++ b/changes/ce/perf-10511.en.md @@ -0,0 +1 @@ +Improve the security and privacy of some resource logs by masking sensitive information in the data. diff --git a/changes/ee/feat-9927.en.md b/changes/ee/feat-9927.en.md new file mode 100644 index 000000000..c20a0c51e --- /dev/null +++ b/changes/ee/feat-9927.en.md @@ -0,0 +1 @@ +Introduce support for the File Transfer over MQTT feature as described in [EIP-0021](https://github.com/emqx/eip), with support to publish transferred files either to the node-local file system or to the S3 API compatible remote object storage. diff --git a/changes/ee/fix-10738.en.md b/changes/ee/fix-10738.en.md new file mode 100644 index 000000000..203fb5823 --- /dev/null +++ b/changes/ee/fix-10738.en.md @@ -0,0 +1,3 @@ +Add support for the `Supertable` and `Create Tables Automatically` features of TDEngine to its data bridge. +Before this fix, an insert with a supertable in the template will fail, like this: + `insert into ${clientid} using msg TAGS (${clientid}) values (${ts},${msg})`. diff --git a/dev b/dev index 425da993a..8cf07cfaf 100755 --- a/dev +++ b/dev @@ -323,10 +323,12 @@ boot() { ## Make erl command aware where to load all the beams ## this should be done before every erl command prepare_erl_libs "$PROFILE" + ## make sure copy acl.conf and certs to etc before render_hocon_conf + ## hocon will check rules inside acl.conf. + copy_other_conf_files render_hocon_conf generate_app_conf append_args_file - copy_other_conf_files APPS="$(apps_to_load)" BOOT_SEQUENCE=" diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl index 853247c1a..3636e3eb2 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl @@ -14,33 +14,37 @@ api_schemas(Method) -> [ - ref(emqx_bridge_gcp_pubsub, Method), - ref(emqx_bridge_kafka, Method ++ "_consumer"), - ref(emqx_bridge_kafka, Method ++ "_producer"), - ref(emqx_bridge_cassandra, Method), - ref(emqx_ee_bridge_mysql, Method), - ref(emqx_bridge_pgsql, Method), - ref(emqx_ee_bridge_mongodb, Method ++ "_rs"), - ref(emqx_ee_bridge_mongodb, Method ++ "_sharded"), - ref(emqx_ee_bridge_mongodb, Method ++ "_single"), - ref(emqx_ee_bridge_hstreamdb, Method), - ref(emqx_bridge_influxdb, Method ++ "_api_v1"), - ref(emqx_bridge_influxdb, Method ++ "_api_v2"), - ref(emqx_ee_bridge_redis, Method ++ "_single"), - ref(emqx_ee_bridge_redis, Method ++ "_sentinel"), - ref(emqx_ee_bridge_redis, Method ++ "_cluster"), - ref(emqx_bridge_timescale, Method), - ref(emqx_bridge_matrix, Method), - ref(emqx_bridge_tdengine, Method), - ref(emqx_bridge_dynamo, Method), - ref(emqx_bridge_rocketmq, Method), - ref(emqx_bridge_clickhouse, Method), - ref(emqx_bridge_sqlserver, Method), - ref(emqx_bridge_opents, Method), - ref(emqx_bridge_pulsar, Method ++ "_producer"), - ref(emqx_bridge_oracle, Method), - ref(emqx_bridge_iotdb, Method), - ref(emqx_bridge_rabbitmq, Method) + %% We need to map the `type' field of a request (binary) to a + %% bridge schema module. + api_ref(emqx_bridge_gcp_pubsub, <<"gcp_pubsub">>, Method), + api_ref(emqx_bridge_kafka, <<"kafka_consumer">>, Method ++ "_consumer"), + %% TODO: rename this to `kafka_producer' after alias support is added + %% to hocon; keeping this as just `kafka' for backwards compatibility. + api_ref(emqx_bridge_kafka, <<"kafka">>, Method ++ "_producer"), + api_ref(emqx_bridge_cassandra, <<"cassandra">>, Method), + api_ref(emqx_ee_bridge_mysql, <<"mysql">>, Method), + api_ref(emqx_bridge_pgsql, <<"pgsql">>, Method), + api_ref(emqx_ee_bridge_mongodb, <<"mongodb_rs">>, Method ++ "_rs"), + api_ref(emqx_ee_bridge_mongodb, <<"mongodb_sharded">>, Method ++ "_sharded"), + api_ref(emqx_ee_bridge_mongodb, <<"mongodb_single">>, Method ++ "_single"), + api_ref(emqx_ee_bridge_hstreamdb, <<"hstreamdb">>, Method), + api_ref(emqx_bridge_influxdb, <<"influxdb_api_v1">>, Method ++ "_api_v1"), + api_ref(emqx_bridge_influxdb, <<"influxdb_api_v2">>, Method ++ "_api_v2"), + api_ref(emqx_ee_bridge_redis, <<"redis_single">>, Method ++ "_single"), + api_ref(emqx_ee_bridge_redis, <<"redis_sentinel">>, Method ++ "_sentinel"), + api_ref(emqx_ee_bridge_redis, <<"redis_cluster">>, Method ++ "_cluster"), + api_ref(emqx_bridge_timescale, <<"timescale">>, Method), + api_ref(emqx_bridge_matrix, <<"matrix">>, Method), + api_ref(emqx_bridge_tdengine, <<"tdengine">>, Method), + api_ref(emqx_bridge_clickhouse, <<"clickhouse">>, Method), + api_ref(emqx_bridge_dynamo, <<"dynamo">>, Method), + api_ref(emqx_bridge_rocketmq, <<"rocketmq">>, Method), + api_ref(emqx_bridge_sqlserver, <<"sqlserver">>, Method), + api_ref(emqx_bridge_opents, <<"opents">>, Method), + api_ref(emqx_bridge_pulsar, <<"pulsar_producer">>, Method ++ "_producer"), + api_ref(emqx_bridge_oracle, <<"oracle">>, Method), + api_ref(emqx_bridge_iotdb, <<"iotdb">>, Method), + api_ref(emqx_bridge_rabbitmq, <<"rabbitmq">>, Method) ]. schema_modules() -> @@ -338,3 +342,6 @@ rabbitmq_structs() -> } )} ]. + +api_ref(Module, Type, Method) -> + {Type, ref(Module, Method)}. diff --git a/mix.exs b/mix.exs index c7fc1e3c0..565f4b259 100644 --- a/mix.exs +++ b/mix.exs @@ -58,7 +58,7 @@ defmodule EMQXUmbrella.MixProject do {:ekka, github: "emqx/ekka", tag: "0.15.1", override: true}, {:gen_rpc, github: "emqx/gen_rpc", tag: "2.8.1", override: true}, {:grpc, github: "emqx/grpc-erl", tag: "0.6.7", override: true}, - {:minirest, github: "emqx/minirest", tag: "1.3.8", override: true}, + {:minirest, github: "emqx/minirest", tag: "1.3.9", override: true}, {:ecpool, github: "emqx/ecpool", tag: "0.5.3", override: true}, {:replayq, github: "emqx/replayq", tag: "0.3.7", override: true}, {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true}, @@ -175,7 +175,9 @@ defmodule EMQXUmbrella.MixProject do :emqx_oracle, :emqx_bridge_oracle, :emqx_bridge_rabbitmq, - :emqx_bridge_clickhouse + :emqx_bridge_clickhouse, + :emqx_ft, + :emqx_s3 ]) end @@ -190,13 +192,6 @@ defmodule EMQXUmbrella.MixProject do {:snappyer, "1.2.8", override: true}, {:crc32cer, "0.1.8", override: true}, {:supervisor3, "1.1.12", override: true}, - {:erlcloud, github: "emqx/erlcloud", tag: "3.5.16-emqx-1", override: true}, - # erlcloud's rebar.config requires rebar3 and does not support Mix, - # so it tries to fetch deps from git. We need to override this. - {:lhttpc, "1.6.2", override: true}, - {:eini, "1.2.9", override: true}, - {:base16, "1.0.0", override: true}, - # end of erlcloud's deps {:opentsdb, github: "emqx/opentsdb-client-erl", tag: "v0.5.1", override: true}, # The following two are dependencies of rabbit_common. They are needed here to # make mix not complain about conflicting versions @@ -212,7 +207,14 @@ defmodule EMQXUmbrella.MixProject do github: "emqx/rabbitmq-server", tag: "v3.11.13-emqx", sparse: "deps/amqp_client", - override: true} + override: true}, + {:erlcloud, github: "emqx/erlcloud", tag: "3.6.8-emqx-1", override: true}, + # erlcloud's rebar.config requires rebar3 and does not support Mix, + # so it tries to fetch deps from git. We need to override this. + {:lhttpc, github: "erlcloud/lhttpc", tag: "1.6.2", override: true}, + {:eini, "1.2.9", override: true}, + {:base16, "1.0.0", override: true} + # end of erlcloud's deps ] end @@ -411,7 +413,8 @@ defmodule EMQXUmbrella.MixProject do emqx_bridge_rabbitmq: :permanent, emqx_ee_schema_registry: :permanent, emqx_eviction_agent: :permanent, - emqx_node_rebalance: :permanent + emqx_node_rebalance: :permanent, + emqx_ft: :permanent ], else: [] ) diff --git a/rebar.config b/rebar.config index 0c9a79e24..4b4838f61 100644 --- a/rebar.config +++ b/rebar.config @@ -65,7 +65,7 @@ , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.1"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}} , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.7"}}} - , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.8"}}} + , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.9"}}} , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.3"}}} , {replayq, {git, "https://github.com/emqx/replayq.git", {tag, "0.3.7"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} diff --git a/rebar.config.erl b/rebar.config.erl index a740ab183..5046009d9 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -99,6 +99,8 @@ is_community_umbrella_app("apps/emqx_bridge_oracle") -> false; is_community_umbrella_app("apps/emqx_bridge_sqlserver") -> false; is_community_umbrella_app("apps/emqx_oracle") -> false; is_community_umbrella_app("apps/emqx_bridge_rabbitmq") -> false; +is_community_umbrella_app("apps/emqx_ft") -> false; +is_community_umbrella_app("apps/emqx_s3") -> false; is_community_umbrella_app(_) -> true. is_jq_supported() -> @@ -480,7 +482,8 @@ relx_apps_per_edition(ee) -> emqx_bridge_rabbitmq, emqx_ee_schema_registry, emqx_eviction_agent, - emqx_node_rebalance + emqx_node_rebalance, + emqx_ft ]; relx_apps_per_edition(ce) -> []. diff --git a/rel/i18n/emqx_authn_api.hocon b/rel/i18n/emqx_authn_api.hocon index 07f9c6c3e..90240c4f4 100644 --- a/rel/i18n/emqx_authn_api.hocon +++ b/rel/i18n/emqx_authn_api.hocon @@ -2,42 +2,68 @@ emqx_authn_api { authentication_get.desc: """List authenticators for global authentication.""" +authentication_get.label: +"""List authenticators""" authentication_id_delete.desc: """Delete authenticator from global authentication chain.""" +authentication_id_delete.label: +"""Delete authenticator""" authentication_id_get.desc: """Get authenticator from global authentication chain.""" +authentication_id_get.label: +"""Get authenticator""" authentication_id_position_put.desc: """Move authenticator in global authentication chain.""" +authentication_id_position_put.label: +"""Move authenticator""" authentication_id_put.desc: """Update authenticator from global authentication chain.""" +authentication_id_put.label: +"""Update authenticator""" authentication_id_status_get.desc: """Get authenticator status from global authentication chain.""" +authentication_id_status_get.label: +"""Get authenticator status""" authentication_id_users_get.desc: """List users in authenticator in global authentication chain.""" +authentication_id_users_get.label: +"""List users in authenticator""" authentication_id_users_post.desc: """Create users for authenticator in global authentication chain.""" +authentication_id_users_post.label: +"""Create users for authenticator""" authentication_id_users_user_id_delete.desc: """Delete user in authenticator in global authentication chain.""" +authentication_id_users_user_id_delete.label: +"""Delete user in authenticator""" authentication_id_users_user_id_get.desc: """Get user from authenticator in global authentication chain.""" +authentication_id_users_user_id_get.label: +"""Get user from authenticator""" authentication_id_users_user_id_put.desc: """Update user in authenticator in global authentication chain.""" +authentication_id_users_user_id_put.label: +"""Update user in authenticator""" authentication_post.desc: """Create authenticator for global authentication.""" +authentication_post.label: +"""Create authenticator""" is_superuser.desc: """Is superuser""" +is_superuser.label: +"""Is superuser""" like_user_id.desc: """Fuzzy search user_id (username or clientid).""" @@ -47,50 +73,82 @@ like_user_id.label: listeners_listener_id_authentication_get.desc: """List authenticators for listener authentication.""" +listeners_listener_id_authentication_get.label: +"""List authenticators for listener""" listeners_listener_id_authentication_id_delete.desc: """Delete authenticator from listener authentication chain.""" +listeners_listener_id_authentication_id_delete.label: +"""Delete authenticator from listener""" listeners_listener_id_authentication_id_get.desc: """Get authenticator from listener authentication chain.""" +listeners_listener_id_authentication_id_get.label: +"""Get authenticator from listener""" listeners_listener_id_authentication_id_position_put.desc: """Move authenticator in listener authentication chain.""" +listeners_listener_id_authentication_id_position_put.label: +"""Move authenticator in listener""" listeners_listener_id_authentication_id_put.desc: """Update authenticator from listener authentication chain.""" +listeners_listener_id_authentication_id_put.label: +"""Update authenticator from listener""" listeners_listener_id_authentication_id_status_get.desc: """Get authenticator status from listener authentication chain.""" +listeners_listener_id_authentication_id_status_get.label: +"""Get authenticator status from listener""" listeners_listener_id_authentication_id_users_get.desc: """List users in authenticator in listener authentication chain.""" +listeners_listener_id_authentication_id_users_get.label: +"""List users in authenticator in listener""" listeners_listener_id_authentication_id_users_post.desc: """Create users for authenticator in listener authentication chain.""" +listeners_listener_id_authentication_id_users_post.label: +"""Create users for authenticator in listener""" listeners_listener_id_authentication_id_users_user_id_delete.desc: """Delete user in authenticator in listener authentication chain.""" +listeners_listener_id_authentication_id_users_user_id_delete.label: +"""Delete user in authenticator in listener""" listeners_listener_id_authentication_id_users_user_id_get.desc: """Get user from authenticator in listener authentication chain.""" +listeners_listener_id_authentication_id_users_user_id_get.label: +"""Get user from authenticator in listener""" listeners_listener_id_authentication_id_users_user_id_put.desc: """Update user in authenticator in listener authentication chain.""" +listeners_listener_id_authentication_id_users_user_id_put.label: +"""Update user in authenticator in listener""" listeners_listener_id_authentication_post.desc: """Create authenticator for listener authentication.""" +listeners_listener_id_authentication_post.label: +"""Create authenticator for listener""" param_auth_id.desc: """Authenticator ID.""" +param_auth_id.label: +"""Authenticator ID""" param_listener_id.desc: """Listener ID.""" +param_listener_id.label: +"""Listener ID""" param_position.desc: """Position of authenticator in chain. Possible values are 'front', 'rear', 'before:{other_authenticator}', 'after:{other_authenticator}'.""" +param_position.label: +"""Position of authenticator""" param_user_id.desc: """User ID.""" +param_user_id.label: +"""User ID""" } diff --git a/rel/i18n/emqx_authn_user_import_api.hocon b/rel/i18n/emqx_authn_user_import_api.hocon index f8fb1757c..3a97823df 100644 --- a/rel/i18n/emqx_authn_user_import_api.hocon +++ b/rel/i18n/emqx_authn_user_import_api.hocon @@ -2,8 +2,12 @@ emqx_authn_user_import_api { authentication_id_import_users_post.desc: """Import users into authenticator in global authentication chain.""" +authentication_id_import_users_post.label: +"""Global import users into authenticator""" listeners_listener_id_authentication_id_import_users_post.desc: """Import users into authenticator in listener authentication chain.""" +listeners_listener_id_authentication_id_import_users_post.label: +"""Import users into authenticator in listener""" } diff --git a/rel/i18n/emqx_authz_api_cache.hocon b/rel/i18n/emqx_authz_api_cache.hocon index 0789c1e71..f3e933d4b 100644 --- a/rel/i18n/emqx_authz_api_cache.hocon +++ b/rel/i18n/emqx_authz_api_cache.hocon @@ -2,5 +2,7 @@ emqx_authz_api_cache { authorization_cache_delete.desc: """Clean all authorization cache in the cluster.""" +authorization_cache_delete.label: +"""Clean authorization cache in cluster""" } diff --git a/rel/i18n/emqx_authz_api_mnesia.hocon b/rel/i18n/emqx_authz_api_mnesia.hocon index 4cfed2970..d0021c6a5 100644 --- a/rel/i18n/emqx_authz_api_mnesia.hocon +++ b/rel/i18n/emqx_authz_api_mnesia.hocon @@ -2,13 +2,11 @@ emqx_authz_api_mnesia { action.desc: """Authorized action (pub/sub/all)""" - action.label: """action""" clientid.desc: """ClientID""" - clientid.label: """clientid""" @@ -26,62 +24,87 @@ fuzzy_username.label: permission.desc: """Permission""" - permission.label: """permission""" rules_all_delete.desc: """Delete rules for 'all'""" +rules_all_delete.label: +"""Delete rules for 'all'""" rules_all_get.desc: """Show the list of rules for 'all'""" +rules_all_get.label: +"""Show rules for 'all'""" rules_all_post.desc: """Create/Update the list of rules for 'all'.""" +rules_all_post.label: +"""Update rules for 'all'""" rules_delete.desc: """Delete all rules for all 'users', 'clients' and 'all'""" +rules_delete.label: +"""Delete all rules""" topic.desc: """Rule on specific topic""" - topic.label: """topic""" user_clientid_delete.desc: """Delete rule for 'clientid'""" +user_clientid_delete.label: +"""Delete rule for 'clientid'""" user_clientid_get.desc: """Get rule for 'clientid'""" +user_clientid_get.label: +"""Get rule for 'clientid'""" user_clientid_put.desc: """Set rule for 'clientid'""" +user_clientid_put.label: +"""Set rule for 'clientid'""" user_username_delete.desc: """Delete rule for 'username'""" +user_username_delete.label: +"""Delete rule for 'username'""" user_username_get.desc: """Get rule for 'username'""" +user_username_get.label: +"""Get rule for 'username'""" user_username_put.desc: """Set rule for 'username'""" +user_username_put.label: +"""Set rule for 'username'""" username.desc: """Username""" - username.label: """username""" users_clientid_get.desc: """Show the list of rules for clients""" +users_clientid_get.label: +"""Show rules for clients""" users_clientid_post.desc: """Add new rule for 'clientid'""" +users_clientid_post.label: +"""Add rule for 'clientid'""" users_username_get.desc: """Show the list of rules for users""" +users_username_get.label: +"""Show rules for users""" users_username_post.desc: """Add new rule for 'username'""" +users_username_post.label: +"""Add rule for 'username'""" } diff --git a/rel/i18n/emqx_authz_api_sources.hocon b/rel/i18n/emqx_authz_api_sources.hocon index 5d2dda69e..1257d9eb8 100644 --- a/rel/i18n/emqx_authz_api_sources.hocon +++ b/rel/i18n/emqx_authz_api_sources.hocon @@ -2,46 +2,56 @@ emqx_authz_api_sources { authorization_sources_get.desc: """List all authorization sources""" +authorization_sources_get.label: +"""List all authorization sources""" authorization_sources_post.desc: """Add a new source""" +authorization_sources_post.label: +"""Add a new source""" authorization_sources_type_delete.desc: """Delete source""" +authorization_sources_type_delete.label: +"""Delete source""" authorization_sources_type_get.desc: """Get a authorization source""" +authorization_sources_type_get.label: +"""Get a authorization source""" authorization_sources_type_move_post.desc: """Change the exection order of sources""" +authorization_sources_type_move_post.label: +"""Change order of sources""" authorization_sources_type_put.desc: """Update source""" +authorization_sources_type_put.label: +"""Update source""" authorization_sources_type_status_get.desc: """Get a authorization source""" +authorization_sources_type_status_get.label: +"""Get a authorization source""" source.desc: """Authorization source""" - source.label: """source""" source_config.desc: """Source config""" - source_config.label: """source_config""" source_type.desc: """Authorization type""" - source_type.label: """source_type""" sources.desc: """Authorization sources""" - sources.label: """sources""" diff --git a/rel/i18n/emqx_auto_subscribe_api.hocon b/rel/i18n/emqx_auto_subscribe_api.hocon index df8e87e1a..518878a50 100644 --- a/rel/i18n/emqx_auto_subscribe_api.hocon +++ b/rel/i18n/emqx_auto_subscribe_api.hocon @@ -2,11 +2,17 @@ emqx_auto_subscribe_api { list_auto_subscribe_api.desc: """Get auto subscribe topic list""" +list_auto_subscribe_api.label: +"""Get auto subscribe topics""" update_auto_subscribe_api.desc: """Update auto subscribe topic list""" +update_auto_subscribe_api.label: +"""Update auto subscribe topics""" update_auto_subscribe_api_response409.desc: """Auto Subscribe topics max limit""" +update_auto_subscribe_api_response409.label: +"""Auto Subscribe topics max limit""" } diff --git a/rel/i18n/emqx_dashboard_api.hocon b/rel/i18n/emqx_dashboard_api.hocon index 3e5bb6239..01fc6eb16 100644 --- a/rel/i18n/emqx_dashboard_api.hocon +++ b/rel/i18n/emqx_dashboard_api.hocon @@ -2,24 +2,36 @@ emqx_dashboard_api { change_pwd_api.desc: """Change dashboard user password""" +change_pwd_api.label: +"""Change dashboard user password""" create_user_api.desc: """Create dashboard user""" +create_user_api.label: +"""Create dashboard user""" create_user_api_success.desc: """Create dashboard user success""" +create_user_api_success.label: +"""Create dashboard user success""" delete_user_api.desc: """Delete dashboard user""" +delete_user_api.label: +"""Delete dashboard user""" license.desc: """EMQX License. opensource or enterprise""" list_users_api.desc: """Dashboard list users""" +list_users_api.label: +"""Dashboard list users""" login_api.desc: """Get Dashboard Auth Token.""" +login_api.label: +"""Get Dashboard Auth Token.""" login_failed401.desc: """Login failed. Bad username or password""" @@ -32,6 +44,8 @@ login_success.desc: logout_api.desc: """Dashboard user logout""" +logout_api.label: +"""Dashboard user logout""" new_pwd.desc: """New password""" @@ -47,6 +61,8 @@ token.desc: update_user_api.desc: """Update dashboard user description""" +update_user_api.label: +"""Update dashboard user description""" update_user_api200.desc: """Update dashboard user success""" diff --git a/rel/i18n/emqx_dashboard_error_code_api.hocon b/rel/i18n/emqx_dashboard_error_code_api.hocon new file mode 100644 index 000000000..835f200ab --- /dev/null +++ b/rel/i18n/emqx_dashboard_error_code_api.hocon @@ -0,0 +1,13 @@ +emqx_dashboard_error_code_api { + +error_codes.desc: +"""API Error Codes""" +error_codes.label: +"""API Error Codes""" + +error_codes_u.desc: +"""API Error Codes""" +error_codes_u.label: +"""API Error Codes""" + +} diff --git a/rel/i18n/emqx_dashboard_monitor_api.hocon b/rel/i18n/emqx_dashboard_monitor_api.hocon new file mode 100644 index 000000000..1d45c45ae --- /dev/null +++ b/rel/i18n/emqx_dashboard_monitor_api.hocon @@ -0,0 +1,23 @@ +emqx_dashboard_monitor_api { + +list_monitor.desc: +"""List monitor data.""" +list_monitor.label: +"""List monitor data.""" + +list_monitor_node.desc: +"""List the monitor data on the node.""" +list_monitor_node.label: +"""List the monitor data on the node.""" + +current_status.desc: +"""Current status. Gauge and rate.""" +current_status.label: +"""Current status. Gauge and rate.""" + +current_status_node.desc: +"""Node current status. Gauge and rate.""" +current_status_node.label: +"""Node current status. Gauge and rate.""" + +} diff --git a/rel/i18n/emqx_delayed_api.hocon b/rel/i18n/emqx_delayed_api.hocon index 62e0fd775..b891c618c 100644 --- a/rel/i18n/emqx_delayed_api.hocon +++ b/rel/i18n/emqx_delayed_api.hocon @@ -2,71 +2,117 @@ emqx_delayed_api { bad_msgid_format.desc: """Bad Message ID format""" +bad_msgid_format.label: +"""Bad Message ID format""" count.desc: """Count of delayed messages""" +count.label: +"""Count of delayed messages""" delayed_interval.desc: """Delayed interval(second)""" +delayed_interval.label: +"""Delayed interval""" delayed_remaining.desc: """Delayed remaining(second)""" +delayed_remaining.label: +"""Delayed remaining""" delete_api.desc: """Delete delayed message""" +delete_api.label: +"""Delete delayed message""" expected_at.desc: """Expect publish time, in RFC 3339 format""" +expected_at.label: +"""Expect publish time""" from_clientid.desc: """From ClientID""" +from_clientid.label: +"""From ClientID""" from_username.desc: """From Username""" +from_username.label: +"""From Username""" get_message_api.desc: """View delayed message""" +get_message_api.label: +"""View delayed message""" illegality_limit.desc: """Max limit illegality""" +illegality_limit.label: +"""Max limit illegality""" list_api.desc: """List delayed messages""" +list_api.label: +"""List delayed messages""" msgid.desc: """Delayed Message ID""" +msgid.label: +"""Delayed Message ID""" msgid_not_found.desc: """Message ID not found""" +msgid_not_found.label: +"""Message ID not found""" node.desc: """The node where message from""" +node.label: +"""Node where message from""" payload.desc: """Payload, base64 encoded. Payload will be set to 'PAYLOAD_TO_LARGE' if its length is larger than 2048 bytes""" +payload.label: +"""Payload""" publish_at.desc: """Clinet publish message time, in RFC 3339 format""" +publish_at.label: +"""Client publish message time""" qos.desc: """QoS""" +qos.label: +"""QoS""" topic.desc: """Topic""" +topic.label: +"""Topic""" update_api.desc: """Enable or disable delayed, set max delayed messages""" +update_api.label: +"""Enable or disable delayed""" update_success.desc: """Enable or disable delayed successfully""" +update_success.label: +"""Enable or disable delayed successfully""" view_limit.desc: """Page limit""" +view_limit.label: +"""Page limit""" view_page.desc: """View page""" +view_page.label: +"""View page""" view_status_api.desc: """Get delayed status""" +view_status_api.label: +"""Get delayed status""" } diff --git a/rel/i18n/emqx_exhook_api.hocon b/rel/i18n/emqx_exhook_api.hocon index 9cb7177c1..dabf89364 100644 --- a/rel/i18n/emqx_exhook_api.hocon +++ b/rel/i18n/emqx_exhook_api.hocon @@ -2,70 +2,109 @@ emqx_exhook_api { add_server.desc: """Add a server""" +add_server.label: +"""Add a server""" delete_server.desc: """Delete the server""" +delete_server.label: +"""Delete the server""" get_detail.desc: """Get the detail information of Exhook server""" +get_detail.label: +"""Get server details""" get_hooks.desc: """Get the hooks information of server""" +get_hooks.label: +"""Get server hooks information""" hook_metrics.desc: """Metrics information of this hook in the current node""" +hook_metrics.label: +"""Hook metrics""" hook_name.desc: """The hook's name""" +hook_name.label: +"""Hook name""" hook_params.desc: """The parameters used when the hook is registered""" +hook_params.label: +"""Hook parameters""" list_all_servers.desc: """List all servers""" +list_all_servers.label: +"""List servers""" metric_failed.desc: """The number of times the hook execution failed""" +metric_failed.label: +"""Failed executions count""" metric_max_rate.desc: """Maximum call rate of hooks""" +metric_max_rate.label: +"""Max hook call rate""" metric_rate.desc: """The call rate of hooks""" +metric_rate.label: +"""Hook call rate""" metric_succeed.desc: """The number of times the hooks execution successful""" +metric_succeed.label: +"""Successful executions count""" metrics.desc: """Metrics information""" +metrics.label: +"""Metrics information""" move_api.desc: """Move the server. NOTE: The position should be "front | rear | before:{name} | after:{name}""" - move_api.label: """Change order of execution for registered Exhook server""" move_position.desc: """The target position to be moved""" +move_position.label: +"""Target position""" node.desc: """Node name""" +node.label: +"""Node name""" node_hook_metrics.desc: """Metrics information of this hook in all nodes""" +node_hook_metrics.label: +"""Node-wise hook metrics""" node_metrics.desc: """Metrics information of this server in all nodes""" +node_metrics.label: +"""Node-wise server metrics""" node_status.desc: """status of this server in all nodes""" +node_status.label: +"""Node-wise server status""" server_metrics.desc: """Metrics information of this server in the current node""" +server_metrics.label: +"""Server metrics""" server_name.desc: """The Exhook server name""" +server_name.label: +"""Server name""" status.desc: """The status of Exhook server.
@@ -74,8 +113,12 @@ connecting: connection failed, reconnecting
disconnected: failed to connect and didn't reconnect
disabled: this server is disabled
error: failed to view the status of this server""" +status.label: +"""Server status""" update_server.desc: """Update the server""" +update_server.label: +"""Update the server""" } diff --git a/rel/i18n/emqx_ft_api.hocon b/rel/i18n/emqx_ft_api.hocon new file mode 100644 index 000000000..9d88fcddd --- /dev/null +++ b/rel/i18n/emqx_ft_api.hocon @@ -0,0 +1,16 @@ +emqx_ft_api { + +file_list.desc: +"""List all uploaded files.""" + +file_list_transfer.desc: +"""List a file uploaded during specified transfer, identified by client id and file id.""" + +} + +emqx_ft_storage_exporter_fs_api { + +file_get.desc: +"""Get a file by its id.""" + +} diff --git a/rel/i18n/emqx_ft_schema.hocon b/rel/i18n/emqx_ft_schema.hocon new file mode 100644 index 000000000..bafda331a --- /dev/null +++ b/rel/i18n/emqx_ft_schema.hocon @@ -0,0 +1,62 @@ +emqx_ft_schema { + +enable.desc: +"""Enable the File Transfer feature.
+Enabling File Transfer implies reserving special MQTT topics in order to serve the protocol.
+This toggle also affects the availability of the File Transfer REST API and +storage-dependent background activities (e.g. garbage collection).""" + +init_timeout.desc: +"""Timeout for initializing the file transfer.
+After reaching the timeout, `init` message will be acked with an error""" + +assemble_timeout.desc: +"""Timeout for assembling and exporting file segments into a final file.
+After reaching the timeout, `fin` message will be acked with an error""" + +store_segment_timeout.desc: +"""Timeout for storing a file segment.
+After reaching the timeout, message with the segment will be acked with an error""" + +storage_backend.desc: +"""Storage settings for file transfer.""" + +local_storage.desc: +"""Local file system backend to store uploaded fragments and temporary data.""" + +local_storage_segments.desc: +"""Settings for local segments storage, which include uploaded transfer fragments and temporary data.""" + +local_storage_segments_root.desc: +"""File system path to keep uploaded fragments and temporary data.""" + +local_storage_exporter_backend.desc: +"""Exporter for the local file system storage backend.
+Exporter defines where and how fully transferred and assembled files are stored.""" + +local_storage_exporter.desc: +"""Exporter to the local file system.""" + +s3_exporter.desc: +"""Exporter to the S3 API compatible object storage.""" + +local_storage_exporter_root.desc: +"""Directory where the uploaded files are kept.""" + +local_storage_segments_gc.desc: +"""Garbage collection settings for the intermediate and temporary files in the local file system.""" + +storage_gc_interval.desc: +"""Interval of periodic garbage collection.""" + +storage_gc_max_segments_ttl.desc: +"""Maximum TTL of a segment kept in the local file system.
+This is a hard limit: no segment will outlive this TTL, even if some file transfer specifies a +TTL more than that.""" + +storage_gc_min_segments_ttl.desc: +"""Minimum TTL of a segment kept in the local file system.
+This is a hard limit: no segment will be garbage collected before reaching this TTL, +even if some file transfer specifies a TTL less than that.""" + +} diff --git a/rel/i18n/emqx_mgmt_api_alarms.hocon b/rel/i18n/emqx_mgmt_api_alarms.hocon index 0327fffcd..b9c289114 100644 --- a/rel/i18n/emqx_mgmt_api_alarms.hocon +++ b/rel/i18n/emqx_mgmt_api_alarms.hocon @@ -8,6 +8,8 @@ deactivate_at.desc: delete_alarms_api.desc: """Remove all historical alarms.""" +delete_alarms_api.label: +"""Remove all historical alarms.""" delete_alarms_api_response204.desc: """Historical alarms have been cleared successfully.""" @@ -26,6 +28,8 @@ The default is false.""" list_alarms_api.desc: """List currently activated alarms or historical alarms, determined by query parameters.""" +list_alarms_api.label: +"""List alarms""" message.desc: """Alarm message, which describes the alarm content in a human-readable format.""" diff --git a/rel/i18n/emqx_mgmt_api_api_keys.hocon b/rel/i18n/emqx_mgmt_api_api_keys.hocon new file mode 100644 index 000000000..85d5c4ec7 --- /dev/null +++ b/rel/i18n/emqx_mgmt_api_api_keys.hocon @@ -0,0 +1,33 @@ +emqx_mgmt_api_api_keys { + +api_key_list.desc: +"""Return api_key list""" +api_key_list.label: +"""Return api_key list""" + +create_new_api_key.desc: +"""Create new api_key""" +create_new_api_key.label: +"""Create new api_key""" + +get_api_key.desc: +"""Return the specific api_key""" +get_api_key.label: +"""Return the specific api_key""" + +update_api_key.desc: +"""Update the specific api_key""" +update_api_key.label: +"""Update the specific api_key""" + +delete_api_key.desc: +"""Delete the specific api_key""" +delete_api_key.label: +"""Delete the specific api_key""" + +format.desc: +"""Unique and format by [a-zA-Z0-9-_]""" +format.label: +"""Unique and format by [a-zA-Z0-9-_]""" + +} diff --git a/rel/i18n/emqx_mgmt_api_banned.hocon b/rel/i18n/emqx_mgmt_api_banned.hocon index 1a9700641..0a5439402 100644 --- a/rel/i18n/emqx_mgmt_api_banned.hocon +++ b/rel/i18n/emqx_mgmt_api_banned.hocon @@ -20,18 +20,24 @@ by.label: create_banned_api.desc: """Add a client ID, username or IP address to the blacklist.""" +create_banned_api.label: +"""Ban client ID, username or IP address""" create_banned_api_response400.desc: """Bad request, possibly due to wrong parameters or the existence of a banned object.""" delete_banned_api.desc: """Remove a client ID, username or IP address from the blacklist.""" +delete_banned_api.label: +"""Unban a client ID, username or IP address""" delete_banned_api_response404.desc: """The banned object was not found in the blacklist.""" list_banned_api.desc: """List all currently banned client IDs, usernames and IP addresses.""" +list_banned_api.label: +"""List all banned client IDs""" reason.desc: """Ban reason, record the reason why the current object was banned.""" diff --git a/rel/i18n/emqx_mgmt_api_clients.hocon b/rel/i18n/emqx_mgmt_api_clients.hocon new file mode 100644 index 000000000..d82642b1e --- /dev/null +++ b/rel/i18n/emqx_mgmt_api_clients.hocon @@ -0,0 +1,58 @@ +emqx_mgmt_api_clients { + +list_clients.desc: +"""List clients""" +list_clients.label: +"""List clients""" + +clients_info_from_id.desc: +"""Get clients info by client ID""" +clients_info_from_id.label: +"""Get clients info by client ID""" + +kick_client_id.desc: +"""Kick out client by client ID""" +kick_client_id.label: +"""Kick out client by client ID""" + +get_authz_cache.desc: +"""Get client authz cache in the cluster.""" +get_authz_cache.label: +"""Get client authz cache in the cluster.""" + +clean_authz_cache.desc: +"""Clean client authz cache in the cluster.""" +clean_authz_cache.label: +"""Clean client authz cache in the cluster.""" + +get_client_subs.desc: +"""Get client subscriptions""" +get_client_subs.label: +"""Get client subscriptions""" + +subscribe.desc: +"""Subscribe""" +subscribe.label: +"""Subscribe""" + +subscribe_g.desc: +"""Subscribe bulk""" +subscribe_g.label: +"""Subscribe bulk""" + +unsubscribe.desc: +"""Unsubscribe""" +unsubscribe.label: +"""Unsubscribe""" + +unsubscribe_g.desc: +"""Unsubscribe bulk""" +unsubscribe_g.label: +"""Unsubscribe bulk""" + +set_keepalive_seconds.desc: +"""Set the online client keepalive by seconds""" +set_keepalive_seconds.label: +"""Set the online client keepalive by seconds""" + +} diff --git a/rel/i18n/emqx_mgmt_api_cluster.hocon b/rel/i18n/emqx_mgmt_api_cluster.hocon new file mode 100644 index 000000000..f8b6de1a4 --- /dev/null +++ b/rel/i18n/emqx_mgmt_api_cluster.hocon @@ -0,0 +1,18 @@ +emqx_mgmt_api_cluster { + +get_cluster_info.desc: +"""Get cluster info""" +get_cluster_info.label: +"""Get cluster info""" + +invite_node.desc: +"""Invite node to cluster""" +invite_node.label: +"""Invite node to cluster""" + +force_remove_node.desc: +"""Force leave node from cluster""" +force_remove_node.label: +"""Force leave node from cluster""" + +} diff --git a/rel/i18n/emqx_mgmt_api_configs.hocon b/rel/i18n/emqx_mgmt_api_configs.hocon new file mode 100644 index 000000000..42bda7899 --- /dev/null +++ b/rel/i18n/emqx_mgmt_api_configs.hocon @@ -0,0 +1,40 @@ +emqx_mgmt_api_configs { + +get_conf_node.desc: +"""Get all the configurations of the specified node, including hot and non-hot updatable items.""" +get_conf_node.label: +"""Get all the configurations for node.""" + +node_name.desc: +"""Node's name: If you do not fill in the fields, this node will be used by default.""" +node_name.label: +"""Node's name.""" + +rest_conf_query.desc: +"""Reset the config entry specified by the query string parameter `conf_path`.
+- For a config entry that has default value, this resets it to the default value; +- For a config entry that has no default value, an error 400 will be returned""" +rest_conf_query.label: +"""Reset the config entry with query""" + +get_global_zone_configs.desc: +"""Get the global zone configs""" +get_global_zone_configs.label: +"""Get the global zone configs""" + +update_globar_zone_configs.desc: +"""Update globbal zone configs""" +update_globar_zone_configs.label: +"""Update globbal zone configs""" + +get_node_level_limiter_congigs.desc: +"""Get the node-level limiter configs""" +get_node_level_limiter_congigs.label: +"""Get the node-level limiter configs""" + +update_node_level_limiter_congigs.desc: +"""Update the node-level limiter configs""" +update_node_level_limiter_congigs.label: +"""Update the node-level limiter configs""" + +} diff --git a/rel/i18n/emqx_mgmt_api_listeners.hocon b/rel/i18n/emqx_mgmt_api_listeners.hocon new file mode 100644 index 000000000..b45f55977 --- /dev/null +++ b/rel/i18n/emqx_mgmt_api_listeners.hocon @@ -0,0 +1,59 @@ +emqx_mgmt_api_listeners { + +list_node_live_statuses.desc: +"""List all running node's listeners live status. group by listener type""" +list_node_live_statuses.label: +"""List all running node's listeners live status. group by listener type""" + +list_listeners.desc: +"""List all running node's listeners for the specified type.""" +list_listeners.label: +"""List all running node's listeners for the specified type.""" + +listener_type.desc: +"""Listener type""" +listener_type.label: +"""Listener type""" + +create_on_all_nodes.desc: +"""Create the specified listener on all nodes.""" +create_on_all_nodes.label: +"""Create the specified listener on all nodes.""" + +list_by_id.desc: +"""List all running node's listeners for the specified id.""" +list_by_id.label: +"""List all running node's listeners for the specified id.""" + +update_lisener.desc: +"""Update the specified listener on all nodes.""" +update_lisener.label: +"""Update the specified listener on all nodes.""" + +create_on_all_nodes_2.desc: +"""Create the specified listener on all nodes.""" +create_on_all_nodes_2.label: +"""Create the specified listener on all nodes.""" + +delete_on_all_nodes.desc: +"""Delete the specified listener on all nodes.""" +delete_on_all_nodes.label: +"""Delete the specified listener on all nodes.""" + +start_on_all_nodes.desc: +"""Start the listener on all nodes.""" +start_on_all_nodes.label: +"""Start the listener on all nodes.""" + +stop_on_all_nodes.desc: +"""Stop the listener on all nodes.""" +stop_on_all_nodes.label: +"""Stop the listener on all nodes.""" + +restart_on_all_nodes.desc: +"""Restart listeners on all nodes.""" +restart_on_all_nodes.label: +"""Restart listeners on all nodes.""" + + +} diff --git a/rel/i18n/emqx_mgmt_api_metrics.hocon b/rel/i18n/emqx_mgmt_api_metrics.hocon new file mode 100644 index 000000000..de1e7f20c --- /dev/null +++ b/rel/i18n/emqx_mgmt_api_metrics.hocon @@ -0,0 +1,8 @@ +emqx_mgmt_api_metrics { + +emqx_metrics.desc: +"""EMQX metrics""" +emqx_metrics.label: +"""EMQX metrics""" + +} diff --git a/rel/i18n/emqx_mgmt_api_nodes.hocon b/rel/i18n/emqx_mgmt_api_nodes.hocon new file mode 100644 index 000000000..e875b0dcc --- /dev/null +++ b/rel/i18n/emqx_mgmt_api_nodes.hocon @@ -0,0 +1,23 @@ +emqx_mgmt_api_nodes { + +list_nodes.desc: +"""List EMQX nodes""" +list_nodes.label: +"""List EMQX nodes""" + +get_node_info.desc: +"""Get node info""" +get_node_info.label: +"""Get node info""" + +get_node_metrics.desc: +"""Get node metrics""" +get_node_metrics.label: +"""Get node metrics""" + +get_node_stats.desc: +"""Get node stats""" +get_node_stats.label: +"""Get node stats""" + +} diff --git a/rel/i18n/emqx_mgmt_api_stats.hocon b/rel/i18n/emqx_mgmt_api_stats.hocon new file mode 100644 index 000000000..d3c57d6e8 --- /dev/null +++ b/rel/i18n/emqx_mgmt_api_stats.hocon @@ -0,0 +1,8 @@ +emqx_mgmt_api_stats { + +emqx_stats.desc: +"""EMQX stats""" +emqx_stats.label: +"""EMQX stats""" + +} diff --git a/rel/i18n/emqx_mgmt_api_subscriptions.hocon b/rel/i18n/emqx_mgmt_api_subscriptions.hocon new file mode 100644 index 000000000..a0c8a7a01 --- /dev/null +++ b/rel/i18n/emqx_mgmt_api_subscriptions.hocon @@ -0,0 +1,8 @@ +emqx_mgmt_api_subscriptions { + +list_subs.desc: +"""List subscriptions""" +list_subs.label: +"""List subscriptions""" + +} diff --git a/rel/i18n/emqx_mgmt_api_topics.hocon b/rel/i18n/emqx_mgmt_api_topics.hocon new file mode 100644 index 000000000..5b65af95a --- /dev/null +++ b/rel/i18n/emqx_mgmt_api_topics.hocon @@ -0,0 +1,13 @@ +emqx_mgmt_api_topics { + +topic_list.desc: +"""Topics list""" +topic_list.label: +"""Topics list""" + +topic_info_by_name.desc: +"""Lookup topic info by name""" +topic_info_by_name.label: +"""Lookup topic info by name""" + +} diff --git a/rel/i18n/emqx_mgmt_api_trace.hocon b/rel/i18n/emqx_mgmt_api_trace.hocon new file mode 100644 index 000000000..4af68a460 --- /dev/null +++ b/rel/i18n/emqx_mgmt_api_trace.hocon @@ -0,0 +1,118 @@ +emqx_mgmt_api_trace { + +list_all.desc: +"""List all trace""" +list_all.label: +"""List all trace""" + +create_new.desc: +"""Create new trace""" +create_new.label: +"""Create new trace""" + +clear_all.desc: +"""Clear all traces""" +clear_all.label: +"""Clear all traces""" + +delete_trace.desc: +"""Delete specified trace""" +delete_trace.label: +"""Delete specified trace""" + +stop_trace.desc: +"""Stop trace by name""" +stop_trace.label: +"""Stop trace by name""" + +download_log_by_name.desc: +"""Download trace log by name""" +download_log_by_name.label: +"""Download trace log by name""" + +trace_zip_file.desc: +"""A trace zip file""" +trace_zip_file.label: +"""A trace zip file""" + +get_trace_file_metadata.desc: +"""get trace log file's metadata, such as size, last update time""" +get_trace_file_metadata.label: +"""get trace log file's metadata""" + +view_trace_log.desc: +"""view trace log""" +view_trace_log.label: +"""view trace log""" + +file_size.desc: +"""file size""" +file_size.label: +"""file size""" + +modification_date.desc: +"""the modification and last access times of a file""" +modification_date.label: +"""last access time""" + +format.desc: +"""Unique and format by [a-zA-Z0-9-_]""" +format.label: +"""Unique and format by [a-zA-Z0-9-_]""" + +filter_type.desc: +"""Filter type""" +filter_type.label: +"""Filter type""" + +support_wildcard.desc: +"""support mqtt wildcard topic.""" +support_wildcard.label: +"""support mqtt wildcard topic""" + +mqtt_clientid.desc: +"""mqtt clientid.""" +mqtt_clientid.label: +"""mqtt clientid""" + +client_ip_addess.desc: +"""client ip address""" +client_ip_addess.label: +"""client ip address""" + +trace_status.desc: +"""trace status""" +trace_status.label: +"""trace status""" + +time_format.desc: +"""rfc3339 timestamp or epoch second""" +time_format.label: +"""rfc3339 timestamp or epoch second""" + +time_format_g.desc: +"""rfc3339 timestamp or epoch second""" +time_format_g.label: +"""rfc3339 timestamp or epoch second""" + +trace_log_size.desc: +"""trace log size""" +trace_log_size.label: +"""trace log size""" + +node_name.desc: +"""Node name""" +node_name.label: +"""Node name""" + +max_response_bytes.desc: +"""Maximum number of bytes to send in response""" +max_response_bytes.label: +"""Maximum response bytes""" + +current_trace_offset.desc: +"""Offset from the current trace position.""" +current_trace_offset.label: +"""Offset from the current trace position.""" + +} diff --git a/rel/i18n/emqx_prometheus_api.hocon b/rel/i18n/emqx_prometheus_api.hocon new file mode 100644 index 000000000..c468ccc2f --- /dev/null +++ b/rel/i18n/emqx_prometheus_api.hocon @@ -0,0 +1,18 @@ +emqx_prometheus_api { + +get_prom_conf_info.desc: +"""Get Prometheus config info""" +get_prom_conf_info.label: +"""Get Prometheus config info""" + +update_prom_conf_info.desc: +"""Update Prometheus config""" +update_prom_conf_info.label: +"""Update Prometheus config""" + +get_prom_data.desc: +"""Get Prometheus Data""" +get_prom_data.label: +"""Get Prometheus Data""" + +} diff --git a/rel/i18n/emqx_retainer_api.hocon b/rel/i18n/emqx_retainer_api.hocon index 5c7084778..cdb49f8d0 100644 --- a/rel/i18n/emqx_retainer_api.hocon +++ b/rel/i18n/emqx_retainer_api.hocon @@ -8,6 +8,8 @@ config_not_found.desc: delete_matching_api.desc: """Delete matching messages.""" +delete_matching_api.label: +"""Delete matching messages""" from_clientid.desc: """The clientid of publisher.""" @@ -17,12 +19,18 @@ from_username.desc: get_config_api.desc: """View config""" +get_config_api.label: +"""View config""" list_retained_api.desc: """List retained messages.""" +list_retained_api.label: +"""List retained messages.""" lookup_api.desc: """Lookup a message by a topic without wildcards.""" +lookup_api.label: +"""Lookup a message""" message_detail.desc: """Details of the message.""" @@ -59,5 +67,7 @@ update_config_success.desc: update_retainer_api.desc: """Update retainer config.""" +update_retainer_api.label: +"""Update retainer config""" } diff --git a/rel/i18n/emqx_rewrite_api.hocon b/rel/i18n/emqx_rewrite_api.hocon index 7e090e590..1b56cf24d 100644 --- a/rel/i18n/emqx_rewrite_api.hocon +++ b/rel/i18n/emqx_rewrite_api.hocon @@ -2,11 +2,17 @@ emqx_rewrite_api { list_topic_rewrite_api.desc: """List all rewrite rules""" +list_topic_rewrite_api.label: +"""List all rewrite rules""" update_topic_rewrite_api.desc: """Update all rewrite rules""" +update_topic_rewrite_api.label: +"""Update all rewrite rules""" update_topic_rewrite_api_response413.desc: """Rules count exceed max limit""" +update_topic_rewrite_api_response413.label: +"""Rules count exceed limit""" } diff --git a/rel/i18n/emqx_s3_schema.hocon b/rel/i18n/emqx_s3_schema.hocon new file mode 100644 index 000000000..df4b973fa --- /dev/null +++ b/rel/i18n/emqx_s3_schema.hocon @@ -0,0 +1,38 @@ +emqx_s3_schema { + +access_key_id.desc: +"""The access key ID of the S3 bucket.""" + +secret_access_key.desc: +"""The secret access key of the S3 bucket.""" + +bucket.desc: +"""The name of the S3 bucket.""" + +host.desc: +"""The host of the S3 endpoint.""" + +port.desc: +"""The port of the S3 endpoint.""" + +url_expire_time.desc: +"""The time in seconds for which the signed URLs to the S3 objects are valid.""" + +min_part_size.desc: +"""The minimum part size for multipart uploads.
+Uploaded data will be accumulated in memory until this size is reached.""" + +max_part_size.desc: +"""The maximum part size for multipart uploads.
+S3 uploader won't try to upload parts larger than this size.""" + +acl.desc: +"""The ACL to use for the uploaded objects.""" + +transport_options.desc: +"""Options for the HTTP transport layer used by the S3 client.""" + +ipv6_probe.desc: +"""Whether to probe for IPv6 support.""" + +} diff --git a/rel/i18n/emqx_slow_subs_api.hocon b/rel/i18n/emqx_slow_subs_api.hocon index edf473487..f1a31784b 100644 --- a/rel/i18n/emqx_slow_subs_api.hocon +++ b/rel/i18n/emqx_slow_subs_api.hocon @@ -2,15 +2,21 @@ emqx_slow_subs_api { clear_records_api.desc: """Clear current data and re count slow topic""" +clear_records_api.label: +"""Clear current data and re count slow topic""" clientid.desc: """Message clientid""" get_records_api.desc: """View slow topics statistics record data""" +get_records_api.label: +"""View slow topics statistics record data""" get_setting_api.desc: """View slow subs settings""" +get_setting_api.label: +"""View slow subs settings""" last_update_time.desc: """The timestamp of last update""" @@ -26,5 +32,7 @@ topic.desc: update_setting_api.desc: """Update slow subs settings""" +update_setting_api.label: +"""Update slow subs settings""" } diff --git a/rel/i18n/emqx_telemetry_api.hocon b/rel/i18n/emqx_telemetry_api.hocon index 5c61b8d3c..4651cc8a0 100644 --- a/rel/i18n/emqx_telemetry_api.hocon +++ b/rel/i18n/emqx_telemetry_api.hocon @@ -14,9 +14,13 @@ enable.desc: get_telemetry_data_api.desc: """Get telemetry data""" +get_telemetry_data_api.label: +"""Get telemetry data""" get_telemetry_status_api.desc: """Get telemetry status""" +get_telemetry_status_api.label: +"""Get telemetry status""" license.desc: """Get license information""" @@ -47,6 +51,8 @@ up_time.desc: update_telemetry_status_api.desc: """Enable or disable telemetry""" +update_telemetry_status_api.label: +"""Enable or disable telemetry""" uuid.desc: """Get UUID""" diff --git a/rel/i18n/emqx_topic_metrics_api.hocon b/rel/i18n/emqx_topic_metrics_api.hocon index 94c58f0cd..bffd831c3 100644 --- a/rel/i18n/emqx_topic_metrics_api.hocon +++ b/rel/i18n/emqx_topic_metrics_api.hocon @@ -2,104 +2,172 @@ emqx_topic_metrics_api { message_qos1_in_rate.desc: """QoS1 in messages rate""" +message_qos1_in_rate.label: +"""QoS1 in messages rate""" message_out_count.desc: """Out messages count""" +message_out_count.label: +"""Out messages count""" message_qos2_out_rate.desc: """QoS2 out messages rate""" +message_qos2_out_rate.label: +"""QoS2 out messages rate""" message_qos0_in_rate.desc: """QoS0 in messages rate""" +message_qos0_in_rate.label: +"""QoS0 in messages rate""" get_topic_metrics_api.desc: """List topic metrics""" +get_topic_metrics_api.label: +"""List topic metrics""" reset_time.desc: """Reset time. In rfc3339. Nullable if never reset""" +reset_time.label: +"""Reset time""" topic_metrics_api_response400.desc: """Bad request. Already exists or bad topic name""" +topic_metrics_api_response400.label: +"""Bad request""" reset_topic_desc.desc: """Topic Name. If this parameter is not present,all created topic metrics will be reset.""" +reset_topic_desc.label: +"""Topic Name""" topic_metrics_api_response409.desc: """Conflict. Topic metrics exceeded max limit 512""" +topic_metrics_api_response409.label: +"""Conflict""" post_topic_metrics_api.desc: """Create topic metrics""" +post_topic_metrics_api.label: +"""Create topic metrics""" message_dropped_rate.desc: """Dropped messages rate""" +message_dropped_rate.label: +"""Dropped messages rate""" message_qos2_in_rate.desc: """QoS2 in messages rate""" +message_qos2_in_rate.label: +"""QoS2 in messages rate""" message_in_rate.desc: """In messages rate""" +message_in_rate.label: +"""In messages rate""" message_qos0_out_rate.desc: """QoS0 out messages rate""" +message_qos0_out_rate.label: +"""QoS0 out messages rate""" message_qos2_in_count.desc: """QoS2 in messages count""" +message_qos2_in_count.label: +"""QoS2 in messages count""" message_dropped_count.desc: """Dropped messages count""" +message_dropped_count.label: +"""Dropped messages count""" topic_metrics_api_response404.desc: """Not Found. Topic metrics not found""" +topic_metrics_api_response404.label: +"""Not Found""" topic_in_path.desc: """Topic string. Notice: Topic string in url path must be encoded""" +topic_in_path.label: +"""Topic string""" action.desc: """Action. Only support reset""" +action.label: +"""Action""" message_qos0_in_count.desc: """QoS0 in messages count""" +message_qos0_in_count.label: +"""QoS0 in messages count""" message_qos1_out_rate.desc: """QoS1 out messages rate""" +message_qos1_out_rate.label: +"""QoS1 out messages rate""" topic.desc: """Topic""" +topic.label: +"""Topic""" reset_topic_metrics_api.desc: """Reset telemetry status""" +reset_topic_metrics_api.label: +"""Reset telemetry status""" create_time.desc: """Create time""" +create_time.label: +"""Create time""" metrics.desc: """Metrics""" +metrics.label: +"""Metrics""" message_qos1_out_count.desc: """QoS1 out messages count""" +message_qos1_out_count.label: +"""QoS1 out messages count""" gat_topic_metrics_data_api.desc: """Get topic metrics""" +gat_topic_metrics_data_api.label: +"""Get topic metrics""" message_qos1_in_count.desc: """QoS1 in messages count""" +message_qos1_in_count.label: +"""QoS1 in messages count""" delete_topic_metrics_data_api.desc: """Delete topic metrics""" +delete_topic_metrics_data_api.label: +"""Delete topic metrics""" message_qos0_out_count.desc: """QoS0 out messages count""" +message_qos0_out_count.label: +"""QoS0 out messages count""" topic_in_body.desc: """Raw topic string""" +topic_in_body.label: +"""Raw topic string""" message_in_count.desc: """In messages count""" +message_in_count.label: +"""In messages count""" message_qos2_out_count.desc: """QoS2 out messages count""" +message_qos2_out_count.label: +"""QoS2 out messages count""" message_out_rate.desc: """Out messages rate""" +message_out_rate.label: +"""Out messages rate""" } diff --git a/scripts/conf-test/old-confs/e5.0.0.conf b/scripts/conf-test/old-confs/e5.0.0.conf new file mode 100644 index 000000000..023c0a6ca --- /dev/null +++ b/scripts/conf-test/old-confs/e5.0.0.conf @@ -0,0 +1,374 @@ +node { + name = "emqx@127.0.0.1" + cookie = "emqxsecretcookie" + data_dir = "data" +} + +log { + file_handlers.default { + level = warning + file = "log/emqx.log" + } +} + +cluster { + name = emqxcl + discovery_strategy = manual +} + + +listeners.tcp.default { + bind = "0.0.0.0:1883" + max_connections = 1024000 +} + +listeners.ssl.default { + bind = "0.0.0.0:8883" + max_connections = 512000 + ssl_options { + keyfile = "etc/certs/key.pem" + certfile = "etc/certs/cert.pem" + cacertfile = "etc/certs/cacert.pem" + } +} + +listeners.ws.default { + bind = "0.0.0.0:8083" + max_connections = 1024000 + websocket.mqtt_path = "/mqtt" +} + +listeners.wss.default { + bind = "0.0.0.0:8084" + max_connections = 512000 + websocket.mqtt_path = "/mqtt" + ssl_options { + keyfile = "etc/certs/key.pem" + certfile = "etc/certs/cert.pem" + cacertfile = "etc/certs/cacert.pem" + } +} + +dashboard { + listeners.http { + bind = 18083 + } + default_username = "admin" + default_password = "public" +} + +authentication = [ + { + backend = "built_in_database" + mechanism = "password_based" + password_hash_algorithm {name = "sha256", salt_position = "suffix"} + user_id_type = "username" + } +] +authorization { + cache {enable = true} + deny_action = "ignore" + no_match = "allow" + sources = [ + { + body {username = "${username}"} + connect_timeout = "5s" + enable_pipelining = 100 + headers {"content-type" = "application/json"} + method = "post" + pool_size = 8 + request_timeout = "5s" + ssl {enable = false, verify = "verify_peer"} + type = "http" + url = "http://127.0.0.1:8080" + }, + { + enable = true + path = "etc/acl.conf" + type = "file" + } + ] +} +bridges { + webhook { + default { + connect_timeout = "5s" + enable_pipelining = 100 + headers {"content-type" = "application/json"} + method = "post" + pool_size = 4 + pool_type = "random" + request_timeout = "5s" + resource_opts { + async_inflight_window = 100 + auto_restart_interval = "60s" + health_check_interval = "15s" + max_queue_bytes = "1GB" + query_mode = "async" + request_timeout = "15s" + worker_pool_size = 4 + } + ssl {enable = false, verify = "verify_peer"} + url = "http://127.0.0.1:8080/webhook" + } + } +} +conn_congestion {enable_alarm = true, min_alarm_sustain_duration = "1m"} +exhook { + servers = [ + { + auto_reconnect = "60s" + enable = false + failed_action = "deny" + name = "exhook" + pool_size = 16 + request_timeout = "5s" + ssl {enable = false, verify = "verify_peer"} + url = "http://127.0.0.1:8000" + } + ] +} +flapping_detect { + ban_time = "5m" + enable = false + max_count = 15 + window_time = "1m" +} +force_gc { + bytes = "16MB" + count = 16000 + enable = true +} +force_shutdown { + enable = true + max_heap_size = "32MB" + max_message_queue_len = 1000 +} +gateway { + stomp { + enable_stats = true + frame { + max_body_length = 8192 + max_headers = 10 + max_headers_length = 1024 + } + idle_timeout = "30s" + listeners { + tcp { + default { + bind = "61613" + max_conn_rate = 1000 + max_connections = 1024000 + } + } + } + mountpoint = "" + } +} +limiter { + bytes_in {burst = "1000", rate = "infinity"} + client { + bytes_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "1000KB/s" + } + connection { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "1000" + } + internal { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + message_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "1000" + } + message_routing { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + } + connection {burst = 0, rate = "infinity"} + internal {burst = 0, rate = "infinity"} + message_in {burst = 0, rate = "infinity"} + message_routing {burst = 0, rate = "1000"} +} +log { + console_handler { + burst_limit { + enable = true + max_count = 10000 + window_time = "1s" + } + chars_limit = "unlimited" + drop_mode_qlen = 3000 + enable = true + flush_qlen = 8000 + formatter = "text" + level = "info" + max_depth = 100 + overload_kill { + enable = true + mem_size = "30MB" + qlen = 20000 + restart_after = "5s" + } + single_line = true + supervisor_reports = "error" + sync_mode_qlen = 100 + time_offset = "system" + } + file_handlers { + default { + burst_limit { + enable = true + max_count = 10000 + window_time = "1s" + } + chars_limit = "unlimited" + drop_mode_qlen = 3000 + enable = false + file = "log/emqx.log" + flush_qlen = 8000 + formatter = "text" + level = "warning" + max_depth = 100 + max_size = "50MB" + overload_kill { + enable = true + mem_size = "30MB" + qlen = 20000 + restart_after = "5s" + } + rotation {count = 10, enable = true} + single_line = true + supervisor_reports = "error" + sync_mode_qlen = 100 + time_offset = "system" + } + } +} +mqtt { + await_rel_timeout = "300s" + exclusive_subscription = false + idle_timeout = "15s" + ignore_loop_deliver = false + keepalive_backoff = 0.5 + max_awaiting_rel = 100 + max_clientid_len = 65535 + max_inflight = 32 + max_mqueue_len = 1000 + max_packet_size = "1MB" + max_qos_allowed = 2 + max_subscriptions = "infinity" + max_topic_alias = 65535 + max_topic_levels = 128 + mqueue_default_priority = "lowest" + mqueue_priorities = "disabled" + "mqueue_store_qos0" = true + peer_cert_as_clientid = "disabled" + peer_cert_as_username = "disabled" + response_information = "" + retain_available = true + retry_interval = "30s" + server_keepalive = "disabled" + session_expiry_interval = "2h" + shared_subscription = true + strict_mode = false + upgrade_qos = false + use_username_as_clientid = false + wildcard_subscription = true +} +overload_protection { + backoff_delay = 1 + backoff_gc = false + backoff_hibernation = true + backoff_new_conn = true + enable = false +} +retainer { + backend { + index_specs = [ + [1, 2, 3], + [1, 3], + [2, 3], + [3] + ] + max_retained_messages = 0 + storage_type = "ram" + type = "built_in_database" + } + enable = true + flow_control { + batch_deliver_limiter { + capacity = "infinity" + client { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + initial = "0" + rate = "infinity" + } + batch_deliver_number = 0 + batch_read_number = 0 + } + max_payload_size = "10MB" + msg_clear_interval = "0s" + msg_expiry_interval = "0s" + stop_publish_clear_msg = false +} +rule_engine { + ignore_sys_message = true + jq_function_default_timeout = "10s" + jq_implementation_module = "jq_nif" + rules { + "rule_k9y8" { + actions = ["webhook:default"] + description = "" + metadata {created_at = 1683875156258} + sql = "SELECT\n *\nFROM\n \"t/#\"" + } + } +} +stats {enable = true} +sys_topics { + sys_event_messages { + client_connected = true + client_disconnected = false + client_subscribed = false + client_unsubscribed = false + } + sys_heartbeat_interval = "30s" + sys_msg_interval = "1m" +} diff --git a/scripts/conf-test/old-confs/e5.0.1.conf b/scripts/conf-test/old-confs/e5.0.1.conf new file mode 100644 index 000000000..e60662b95 --- /dev/null +++ b/scripts/conf-test/old-confs/e5.0.1.conf @@ -0,0 +1,375 @@ +node { + name = "emqx@127.0.0.1" + cookie = "emqxsecretcookie" + data_dir = "data" +} + +log { + file_handlers.default { + level = warning + file = "log/emqx.log" + } +} + +cluster { + name = emqxcl + discovery_strategy = manual +} + +dashboard { + listeners.http { + bind = 18083 + } + default_username = "admin" + default_password = "public" +} + + +authentication = [ + { + backend = "built_in_database" + mechanism = "password_based" + password_hash_algorithm {name = "sha256", salt_position = "suffix"} + user_id_type = "username" + } +] +authorization { + cache {enable = true} + deny_action = "ignore" + no_match = "allow" + sources = [ + { + body {username = "${username}"} + connect_timeout = "5s" + enable_pipelining = 100 + headers {"content-type" = "application/json"} + method = "post" + pool_size = 8 + request_timeout = "5s" + ssl {enable = false, verify = "verify_peer"} + type = "http" + url = "http://127.0.0.1:8080" + }, + { + enable = true + path = "etc/acl.conf" + type = "file" + } + ] +} +bridges { + webhook { + test { + connect_timeout = "5s" + enable_pipelining = 100 + headers {"content-type" = "application/json"} + method = "post" + pool_size = 4 + pool_type = "random" + request_timeout = "15s" + resource_opts { + async_inflight_window = 100 + auto_restart_interval = "60s" + health_check_interval = "15s" + max_queue_bytes = "1GB" + query_mode = "async" + request_timeout = "15s" + worker_pool_size = 4 + } + ssl {enable = false, verify = "verify_peer"} + url = "http://127.0.0.1:8080" + } + } +} +gateway { + mqttsn { + broadcast = true + "enable_qos3" = true + enable_stats = true + gateway_id = 1 + idle_timeout = "30s" + listeners { + udp { + default { + bind = "1884" + max_conn_rate = 1000 + max_connections = 1024000 + } + } + } + mountpoint = "" + predefined = [] + } +} +limiter { + bytes_in {burst = "0", rate = "10MB"} + client { + bytes_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "100KB" + } + connection { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + internal { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + message_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + message_routing { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + } + connection {burst = "0", rate = "1000"} + internal {burst = "0", rate = "infinity"} + message_in {burst = "0", rate = "1000"} + message_routing {burst = "0", rate = "infinity"} +} +listeners { + ssl { + default { + bind = "0.0.0.0:8883" + max_connections = 512000 + ssl_options { + cacertfile = "etc/certs/cacert.pem" + certfile = "etc/certs/cert.pem" + keyfile = "etc/certs/key.pem" + } + } + } + tcp { + default { + acceptors = 16 + bind = "0.0.0.0:1883" + enable_authn = true + enabled = true + limiter { + bytes_in { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + client { + bytes_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + connection { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + message_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + message_routing { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + } + connection { + capacity = 1000 + initial = "0" + rate = "1000/s" + } + message_in { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + message_routing { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + } + max_connections = 2000 + proxy_protocol = false + proxy_protocol_timeout = "3s" + tcp_options { + active_n = 100 + backlog = 1024 + buffer = "4KB" + high_watermark = "1MB" + nodelay = true + reuseaddr = true + send_timeout = "15s" + send_timeout_close = true + } + zone = "default" + } + } + ws { + default { + bind = "0.0.0.0:8083" + max_connections = 1024000 + websocket {mqtt_path = "/mqtt"} + } + } + wss { + default { + bind = "0.0.0.0:8084" + max_connections = 512000 + ssl_options { + cacertfile = "etc/certs/cacert.pem" + certfile = "etc/certs/cert.pem" + keyfile = "etc/certs/key.pem" + } + websocket {mqtt_path = "/mqtt"} + } + } +} +log { + console_handler { + burst_limit { + enable = true + max_count = 10000 + window_time = "1s" + } + chars_limit = "unlimited" + drop_mode_qlen = 3000 + enable = true + flush_qlen = 8000 + formatter = "text" + level = "info" + max_depth = 100 + overload_kill { + enable = true + mem_size = "30MB" + qlen = 20000 + restart_after = "5s" + } + single_line = true + supervisor_reports = "error" + sync_mode_qlen = 100 + time_offset = "system" + } + file_handlers { + default { + burst_limit { + enable = true + max_count = 10000 + window_time = "1s" + } + chars_limit = "unlimited" + drop_mode_qlen = 3000 + enable = false + file = "log/emqx.log" + flush_qlen = 8000 + formatter = "text" + level = "warning" + max_depth = 100 + max_size = "50MB" + overload_kill { + enable = true + mem_size = "30MB" + qlen = 20000 + restart_after = "5s" + } + rotation {count = 10, enable = true} + single_line = true + supervisor_reports = "error" + sync_mode_qlen = 100 + time_offset = "system" + } + } +} +retainer { + backend { + index_specs = [ + [1, 2, 3], + [1, 3], + [2, 3], + [3] + ] + max_retained_messages = 0 + storage_type = "ram" + type = "built_in_database" + } + enable = true + flow_control { + batch_deliver_limiter { + capacity = "infinity" + client { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + initial = "0" + rate = "infinity" + } + batch_deliver_number = 0 + batch_read_number = 0 + } + max_payload_size = "5MB" + msg_clear_interval = "0s" + msg_expiry_interval = "0s" + stop_publish_clear_msg = false +} +rule_engine { + ignore_sys_message = true + jq_function_default_timeout = "10s" + jq_implementation_module = jq_nif + rules { + "rule_bw1q" { + actions = ["webhook:test"] + description = "" + metadata {created_at = 1683878250142} + sql = "SELECT\n *\nFROM\n \"t/#\"" + } + } +} diff --git a/scripts/conf-test/old-confs/e5.0.2.conf b/scripts/conf-test/old-confs/e5.0.2.conf new file mode 100644 index 000000000..f5f3b1394 --- /dev/null +++ b/scripts/conf-test/old-confs/e5.0.2.conf @@ -0,0 +1,454 @@ +node { + name = "emqx@127.0.0.1" + cookie = "emqxsecretcookie" + data_dir = "data" +} + +log { + file_handlers.default { + level = warning + file = "log/emqx.log" + } +} + +cluster { + name = emqxcl + discovery_strategy = manual +} + +dashboard { + listeners.http { + bind = 18083 + } + default_username = "admin" + default_password = "public" +} + +authentication = [ + { + backend = "built_in_database" + mechanism = "password_based" + password_hash_algorithm {name = "sha256", salt_position = "suffix"} + user_id_type = "username" + } +] +authorization { + cache {enable = true} + deny_action = "ignore" + no_match = "allow" + sources = [ + { + body {username = "${username}"} + connect_timeout = "15s" + enable_pipelining = 100 + headers {"content-type" = "application/json"} + method = "post" + pool_size = 8 + request_timeout = "30s" + ssl {enable = false, verify = "verify_peer"} + type = "http" + url = "http://127.0.0.1:8080" + }, + { + enable = true + path = "etc/acl.conf" + type = "file" + } + ] +} +bridges { + webhook { + tests { + connect_timeout = "15s" + enable_pipelining = 100 + headers {"content-type" = "application/json"} + method = "post" + pool_size = 8 + pool_type = "random" + request_timeout = "15s" + resource_opts { + async_inflight_window = 100 + auto_restart_interval = "60s" + health_check_interval = "15s" + max_queue_bytes = "1GB" + query_mode = "async" + request_timeout = "15s" + worker_pool_size = 4 + } + ssl {enable = false, verify = "verify_peer"} + url = "http://127.0.0.1:8080" + } + } +} +conn_congestion {enable_alarm = true, min_alarm_sustain_duration = "1m"} +exhook { + servers = [ + { + auto_reconnect = "60s" + enable = false + failed_action = "deny" + name = "test" + pool_size = 16 + request_timeout = "5s" + ssl {enable = false, verify = "verify_peer"} + url = "http://127.0.0.1:8080" + } + ] +} +flapping_detect { + ban_time = "5m" + enable = false + max_count = 15 + window_time = "1m" +} +force_gc { + bytes = "16MB" + count = 16000 + enable = true +} +force_shutdown { + enable = true + max_heap_size = "32MB" + max_message_queue_len = 1000 +} +gateway { + coap { + connection_required = false + enable_stats = true + listeners { + udp { + default { + bind = "5683" + max_conn_rate = 1000 + max_connections = 1024000 + } + } + } + mountpoint = "" + notify_type = "qos" + publish_qos = "coap" + subscribe_qos = "coap" + } +} +limiter { + bytes_in {burst = "0", rate = "infinity"} + client { + bytes_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + connection { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + internal { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "500" + } + message_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "200" + low_watermark = "60" + max_retry_time = "10s" + rate = "100" + } + message_routing { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + } + connection {burst = "0", rate = "infinity"} + internal {burst = "0", rate = "1000"} + message_in {burst = "0", rate = "10000"} + message_routing {burst = "0", rate = "infinity"} +} +listeners { + ssl { + default { + bind = "0.0.0.0:8883" + max_connections = 512000 + ssl_options { + cacertfile = "etc/certs/cacert.pem" + certfile = "etc/certs/cert.pem" + keyfile = "etc/certs/key.pem" + } + } + } + tcp { + default { + acceptors = 16 + bind = "0.0.0.0:1883" + enable_authn = true + enabled = true + limiter { + bytes_in { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + client { + bytes_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + connection { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + message_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + message_routing { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + } + connection { + capacity = "1000" + initial = "0" + rate = "1000/s" + } + message_in { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + message_routing { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + } + max_connections = 1024000 + mountpoint = "topic_prefix1/x" + proxy_protocol = false + proxy_protocol_timeout = "3s" + tcp_options { + active_n = 100 + backlog = 1024 + buffer = "4KB" + high_watermark = "1MB" + nodelay = true + reuseaddr = true + send_timeout = "15s" + send_timeout_close = true + } + zone = "default" + } + } + ws { + default { + bind = "0.0.0.0:8083" + max_connections = 1024000 + websocket {mqtt_path = "/mqtt"} + } + } + wss { + default { + bind = "0.0.0.0:8084" + max_connections = 512000 + ssl_options { + cacertfile = "etc/certs/cacert.pem" + certfile = "etc/certs/cert.pem" + keyfile = "etc/certs/key.pem" + } + websocket {mqtt_path = "/mqtt"} + } + } +} +log { + console_handler { + burst_limit { + enable = true + max_count = 10000 + window_time = "1s" + } + chars_limit = "unlimited" + drop_mode_qlen = 3000 + enable = true + flush_qlen = 8000 + formatter = "text" + level = "info" + max_depth = 100 + overload_kill { + enable = true + mem_size = "30MB" + qlen = "10000" + restart_after = "5s" + } + single_line = true + supervisor_reports = "error" + sync_mode_qlen = 100 + time_offset = "system" + } + file_handlers { + default { + burst_limit { + enable = true + max_count = 10000 + window_time = "1s" + } + chars_limit = "unlimited" + drop_mode_qlen = 3000 + enable = true + file = "log/emqx.log" + flush_qlen = 8000 + formatter = "text" + level = "warning" + max_depth = 100 + max_size = "1000MB" + overload_kill { + enable = true + mem_size = "30MB" + qlen = 20000 + restart_after = "5s" + } + rotation {count = 10, enable = true} + single_line = true + supervisor_reports = "error" + sync_mode_qlen = 100 + time_offset = "system" + } + } +} +mqtt { + await_rel_timeout = "300s" + exclusive_subscription = false + idle_timeout = "15s" + ignore_loop_deliver = false + keepalive_backoff = 0.75 + max_awaiting_rel = 100 + max_clientid_len = 65535 + max_inflight = 8 + max_mqueue_len = 1000 + max_packet_size = "1MB" + max_qos_allowed = 2 + max_subscriptions = "infinity" + max_topic_alias = 1024 + max_topic_levels = 128 + mqueue_default_priority = "lowest" + mqueue_priorities = "disabled" + "mqueue_store_qos0" = true + peer_cert_as_clientid = "disabled" + peer_cert_as_username = "disabled" + response_information = "" + retain_available = true + retry_interval = "30s" + server_keepalive = "disabled" + session_expiry_interval = "2h" + shared_subscription = true + strict_mode = false + upgrade_qos = false + use_username_as_clientid = false + wildcard_subscription = true +} +overload_protection { + backoff_delay = 1 + backoff_gc = false + backoff_hibernation = true + backoff_new_conn = true + enable = false +} +retainer { + backend { + index_specs = [ + [1, 2, 3], + [1, 3], + [2, 3], + [3] + ] + max_retained_messages = 0 + storage_type = "disc" + type = "built_in_database" + } + enable = true + flow_control { + batch_deliver_limiter { + capacity = "infinity" + client { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + initial = "0" + rate = "infinity" + } + batch_deliver_number = 0 + batch_read_number = 0 + } + max_payload_size = "1MB" + msg_clear_interval = "0s" + msg_expiry_interval = "120s" + stop_publish_clear_msg = false +} +rule_engine { + ignore_sys_message = true + jq_function_default_timeout = "10s" + jq_implementation_module = "jq_nif" + rules { + "rule_bx87" { + actions = ["webhook:tests"] + description = "" + metadata {created_at = 1683880400358} + sql = "SELECT\n *\nFROM\n \"t/#\"" + } + } +} +stats {enable = true} +sys_topics { + sys_event_messages { + client_connected = true + client_disconnected = true + client_subscribed = true + client_unsubscribed = true + } + sys_heartbeat_interval = "30s" + sys_msg_interval = "1m" +} diff --git a/scripts/conf-test/old-confs/e5.0.3.conf b/scripts/conf-test/old-confs/e5.0.3.conf new file mode 100644 index 000000000..1981b8d09 --- /dev/null +++ b/scripts/conf-test/old-confs/e5.0.3.conf @@ -0,0 +1,715 @@ +node { + name = "emqx@127.0.0.1" + cookie = "emqxsecretcookie" + data_dir = "data" +} + +cluster { + name = emqxcl + discovery_strategy = manual +} + + +dashboard { + listeners.http { + bind = 18083 + } +} + +authentication = [ + { + backend = "built_in_database" + mechanism = "password_based" + password_hash_algorithm {name = "sha256", salt_position = "suffix"} + user_id_type = "username" + } +] +authorization { + cache { + enable = true + max_size = 32 + ttl = "1m" + } + deny_action = "ignore" + no_match = "allow" + sources = [ + { + body {username = "${username}"} + connect_timeout = "15s" + enable_pipelining = 100 + headers {"content-type" = "application/json"} + method = "post" + pool_size = 8 + request_timeout = "30s" + ssl {enable = false, verify = "verify_peer"} + type = "http" + url = "http://127.0.0.1:8080" + }, + { + enable = true + path = "${EMQX_ETC_DIR}/acl.conf" + type = "file" + } + ] +} +delayed {enable = true, max_delayed_messages = 200} +exhook { + servers = [ + { + auto_reconnect = "60s" + enable = false + failed_action = "deny" + name = "test" + pool_size = 16 + request_timeout = "5s" + ssl {enable = false, verify = "verify_peer"} + url = "http://127.0.0.1:8080" + } + ] +} +gateway { + coap { + connection_required = false + enable_stats = true + listeners { + udp { + default { + bind = "5683" + max_conn_rate = 1000 + max_connections = 1024000 + } + } + } + mountpoint = "" + notify_type = "qos" + publish_qos = "coap" + subscribe_qos = "coap" + } +} +limiter { + bytes_in {burst = "10MB", rate = "10MB/s"} + client { + bytes_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "10KB" + low_watermark = "0" + max_retry_time = "10s" + rate = "100KB/s" + } + connection { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "100" + low_watermark = "0" + max_retry_time = "10s" + rate = "100" + } + internal { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "100" + low_watermark = "0" + max_retry_time = "10s" + rate = "100" + } + message_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "100" + low_watermark = "0" + max_retry_time = "10s" + rate = "100" + } + message_routing { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "100" + low_watermark = "0" + max_retry_time = "10s" + rate = "100" + } + } + connection {burst = "1000", rate = "1000"} + internal {burst = "1000", rate = "1000"} + message_in {burst = "1000", rate = "1000"} + message_routing {burst = "1000", rate = "1000"} +} +listeners { + ssl { + default { + acceptors = 16 + access_rules = ["allow all"] + bind = "0.0.0.0:8883" + enable_authn = true + enabled = true + limiter { + bytes_in { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + client { + bytes_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + connection { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + message_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + message_routing { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + } + connection { + capacity = "1000" + initial = "0" + rate = "1000/s" + } + message_in { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + message_routing { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + } + max_connections = 5000000 + mountpoint = "" + proxy_protocol = false + proxy_protocol_timeout = "3s" + ssl_options { + cacertfile = "${EMQX_ETC_DIR}/certs/cacert.pem" + certfile = "${EMQX_ETC_DIR}/certs/cert.pem" + ciphers = [] + client_renegotiation = true + depth = 10 + enable_crl_check = false + fail_if_no_peer_cert = false + gc_after_handshake = false + handshake_timeout = "15s" + hibernate_after = "5s" + honor_cipher_order = true + keyfile = "${EMQX_ETC_DIR}/certs/key.pem" + ocsp { + enable_ocsp_stapling = false + refresh_http_timeout = "15s" + refresh_interval = "5m" + } + reuse_sessions = true + secure_renegotiate = true + verify = "verify_none" + versions = ["tlsv1.3", "tlsv1.2", "tlsv1.1", "tlsv1"] + } + tcp_options { + active_n = 100 + backlog = 1024 + buffer = "4KB" + high_watermark = "1MB" + nodelay = true + reuseaddr = true + send_timeout = "15s" + send_timeout_close = true + } + zone = "default" + } + } + tcp { + default { + acceptors = 16 + access_rules = ["allow all"] + bind = "0.0.0.0:1883" + enable_authn = true + enabled = true + limiter { + bytes_in { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + client { + bytes_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + connection { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + message_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + message_routing { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + } + connection { + capacity = "1000" + initial = "0" + rate = "1000/s" + } + message_in { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + message_routing { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + } + max_connections = 12222 + mountpoint = "" + proxy_protocol = false + proxy_protocol_timeout = "3s" + tcp_options { + active_n = 100 + backlog = 1024 + buffer = "4KB" + high_watermark = "1MB" + nodelay = true + reuseaddr = true + send_timeout = "15s" + send_timeout_close = true + } + zone = "default" + } + } + ws { + default { + acceptors = 16 + access_rules = ["allow all"] + bind = "0.0.0.0:8083" + enable_authn = true + enabled = true + limiter { + bytes_in { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + client { + bytes_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + connection { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + message_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + message_routing { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + } + connection { + capacity = "1000" + initial = "0" + rate = "1000/s" + } + message_in { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + message_routing { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + } + max_connections = 5000000 + mountpoint = "" + proxy_protocol = false + proxy_protocol_timeout = "3s" + tcp_options { + active_n = 100 + backlog = 1024 + buffer = "4KB" + high_watermark = "1MB" + nodelay = true + reuseaddr = true + send_timeout = "15s" + send_timeout_close = true + } + websocket { + allow_origin_absence = true + check_origin_enable = false + check_origins = "http://localhost:18083, http://127.0.0.1:18083" + compress = false + deflate_opts { + client_context_takeover = "takeover" + client_max_window_bits = 15 + mem_level = 8 + server_context_takeover = "takeover" + server_max_window_bits = 15 + strategy = "default" + } + fail_if_no_subprotocol = true + idle_timeout = "7200s" + max_frame_size = "infinity" + mqtt_path = "/mqtt" + mqtt_piggyback = "multiple" + proxy_address_header = "x-forwarded-for" + proxy_port_header = "x-forwarded-port" + supported_subprotocols = "mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5" + } + zone = "default" + } + } + wss { + default { + acceptors = 16 + access_rules = ["allow all"] + bind = "0.0.0.0:8084" + enable_authn = true + enabled = true + limiter { + bytes_in { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + client { + bytes_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + connection { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + message_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + message_routing { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + } + connection { + capacity = "1000" + initial = "0" + rate = "1000/s" + } + message_in { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + message_routing { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + } + max_connections = 5000000 + mountpoint = "" + proxy_protocol = false + proxy_protocol_timeout = "3s" + ssl_options { + cacertfile = "${EMQX_ETC_DIR}/certs/cacert.pem" + certfile = "${EMQX_ETC_DIR}/certs/cert.pem" + ciphers = [] + client_renegotiation = true + depth = 10 + fail_if_no_peer_cert = false + handshake_timeout = "15s" + hibernate_after = "5s" + honor_cipher_order = true + keyfile = "${EMQX_ETC_DIR}/certs/key.pem" + reuse_sessions = true + secure_renegotiate = true + verify = "verify_none" + versions = ["tlsv1.3", "tlsv1.2", "tlsv1.1", "tlsv1"] + } + tcp_options { + active_n = 100 + backlog = 1024 + buffer = "4KB" + high_watermark = "1MB" + nodelay = true + reuseaddr = true + send_timeout = "15s" + send_timeout_close = true + } + websocket { + allow_origin_absence = true + check_origin_enable = false + check_origins = "http://localhost:18083, http://127.0.0.1:18083" + compress = false + deflate_opts { + client_context_takeover = "takeover" + client_max_window_bits = 15 + mem_level = 8 + server_context_takeover = "takeover" + server_max_window_bits = 15 + strategy = "default" + } + fail_if_no_subprotocol = true + idle_timeout = "7200s" + max_frame_size = "infinity" + mqtt_path = "/mqtt" + mqtt_piggyback = "multiple" + proxy_address_header = "x-forwarded-for" + proxy_port_header = "x-forwarded-port" + supported_subprotocols = "mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5" + } + zone = "default" + } + } +} +log { + console_handler { + burst_limit { + enable = true + max_count = 10000 + window_time = "1s" + } + chars_limit = "unlimited" + drop_mode_qlen = 3000 + enable = true + flush_qlen = 8000 + formatter = "text" + level = "warning" + max_depth = 100 + overload_kill { + enable = true + mem_size = "30MB" + qlen = "10000" + restart_after = "5s" + } + single_line = true + supervisor_reports = "error" + sync_mode_qlen = 100 + time_offset = "system" + } + file_handlers { + default { + burst_limit { + enable = true + max_count = 10000 + window_time = "1s" + } + chars_limit = "unlimited" + drop_mode_qlen = 3000 + enable = false + file = "${EMQX_LOG_DIR}/emqx.log" + flush_qlen = 8000 + formatter = "text" + level = "warning" + max_depth = 100 + max_size = "500MB" + overload_kill { + enable = true + mem_size = "30MB" + qlen = 20000 + restart_after = "5s" + } + rotation {count = 10, enable = true} + single_line = true + supervisor_reports = "error" + sync_mode_qlen = 100 + time_offset = "system" + } + } +} +mqtt { + await_rel_timeout = "300s" + exclusive_subscription = false + idle_timeout = "15s" + ignore_loop_deliver = false + keepalive_backoff = 0.75 + max_awaiting_rel = 100 + max_clientid_len = 65535 + max_inflight = 32 + max_mqueue_len = 1000 + max_packet_size = "20MB" + max_qos_allowed = 2 + max_subscriptions = "infinity" + max_topic_alias = 65535 + max_topic_levels = 128 + mqueue_default_priority = "lowest" + mqueue_priorities = "disabled" + "mqueue_store_qos0" = true + peer_cert_as_clientid = "disabled" + peer_cert_as_username = "disabled" + response_information = "" + retain_available = true + retry_interval = "30s" + server_keepalive = "disabled" + session_expiry_interval = "12h" + shared_subscription = true + strict_mode = false + upgrade_qos = false + use_username_as_clientid = false + wildcard_subscription = true +} +retainer { + backend { + index_specs = [ + [1, 2, 3], + [1, 3], + [2, 3], + [3] + ] + max_retained_messages = 0 + storage_type = "disc" + type = "built_in_database" + } + enable = true + flow_control { + batch_deliver_limiter { + capacity = "infinity" + client { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + initial = "0" + rate = "infinity" + } + batch_deliver_number = 0 + batch_read_number = 0 + } + max_payload_size = "20MB" + msg_clear_interval = "0s" + msg_expiry_interval = "0s" + stop_publish_clear_msg = false +} +sys_topics { + sys_event_messages { + client_connected = true + client_disconnected = true + client_subscribed = true + client_unsubscribed = false + } + sys_heartbeat_interval = "30s" + sys_msg_interval = "1m" +} +sysmon { + os { + cpu_check_interval = "60s" + cpu_high_watermark = "80%" + cpu_low_watermark = "60%" + mem_check_interval = "60s" + procmem_high_watermark = "5%" + sysmem_high_watermark = "70%" + } + vm { + busy_dist_port = true + busy_port = true + large_heap = "disabled" + long_gc = "disabled" + long_schedule = "240ms" + process_check_interval = "30s" + process_high_watermark = "80%" + process_low_watermark = "60%" + } +} diff --git a/scripts/conf-test/old-confs/v5.0.20.conf b/scripts/conf-test/old-confs/v5.0.20.conf new file mode 100644 index 000000000..b8361a7be --- /dev/null +++ b/scripts/conf-test/old-confs/v5.0.20.conf @@ -0,0 +1,458 @@ +node { + name = "emqx@127.0.0.1" + cookie = "emqxsecretcookie" + data_dir = "data" +} + +log { + file_handlers.default { + level = warning + file = "log/emqx.log" + } +} + +cluster { + name = emqxcl + discovery_strategy = manual +} + +dashboard { + listeners.http { + bind = 18083 + } + default_username = "admin" + default_password = "public" +} + +authentication = [ + { + backend = "built_in_database" + mechanism = "password_based" + password_hash_algorithm {name = "sha256", salt_position = "suffix"} + user_id_type = "username" + }, + { + backend = "mysql" + database = "mqtt_user" + mechanism = "password_based" + password_hash_algorithm {name = "sha256", salt_position = "suffix"} + pool_size = 8 + query = "SELECT password_hash, salt FROM mqtt_user where username = ${username} LIMIT 1" + query_timeout = "5s" + server = "127.0.0.1:3306" + ssl {enable = false, verify = "verify_peer"} + username = "root" + }, + { + backend = "mongodb" + collection = "users" + database = "mqtt" + filter {username = "${username}"} + mechanism = "password_based" + mongo_type = "single" + password_hash_algorithm {name = "sha256", salt_position = "suffix"} + password_hash_field = "password_hash" + pool_size = 8 + salt_field = "salt" + server = "127.0.0.1:27017" + ssl {enable = false, verify = "verify_peer"} + topology {connect_timeout_ms = "20s"} + }, + { + backend = "postgresql" + database = "mqtt_user" + mechanism = "password_based" + password_hash_algorithm {name = "sha256", salt_position = "suffix"} + pool_size = 8 + query = "SELECT password_hash, salt FROM mqtt_user where username = ${username} LIMIT 1" + server = "127.0.0.1:5432" + ssl {enable = false, verify = "verify_peer"} + username = "root" + }, + { + backend = "redis" + cmd = "HMGET mqtt_user:${username} password_hash salt" + database = 0 + mechanism = "password_based" + password_hash_algorithm {name = "sha256", salt_position = "suffix"} + pool_size = 8 + redis_type = "single" + server = "127.0.0.1:6379" + ssl {enable = false, verify = "verify_peer"} + }, + { + backend = "http" + body {password = "${password}", username = "${username}"} + connect_timeout = "15s" + enable_pipelining = 100 + headers {"content-type" = "application/json"} + mechanism = "password_based" + method = "post" + pool_size = 8 + request_timeout = "5s" + ssl {enable = false, verify = "verify_peer"} + url = "http://127.0.0.1:8080" + } +] +authorization { + cache {enable = true} + deny_action = "ignore" + no_match = "allow" + sources = [ + {type = "built_in_database"}, + { + enable = true + path = "etc/acl.conf" + type = "file" + } + ] +} +bridges { + webhook { + test { + connect_timeout = "15s" + enable_pipelining = 100 + headers {"content-type" = "application/json"} + method = "post" + pool_size = 8 + pool_type = "random" + request_timeout = "15s" + resource_opts { + async_inflight_window = 100 + auto_restart_interval = "60s" + health_check_interval = "15s" + max_queue_bytes = "1GB" + query_mode = "async" + request_timeout = "15s" + worker_pool_size = 4 + } + ssl {enable = false, verify = "verify_peer"} + url = "http://127.0.0.1:8080/webhook" + } + } +} +conn_congestion {enable_alarm = true, min_alarm_sustain_duration = "1m"} +exhook { + servers = [ + { + auto_reconnect = "60s" + enable = false + failed_action = "deny" + name = "test" + pool_size = 16 + request_timeout = "5s" + ssl {enable = false, verify = "verify_peer"} + url = "http://127.0.0.1:8080" + } + ] +} +flapping_detect { + ban_time = "5m" + enable = false + max_count = 15 + window_time = "1m" +} +force_gc { + bytes = "16MB" + count = 16000 + enable = true +} +force_shutdown { + enable = true + max_heap_size = "32MB" + max_message_queue_len = 1000 +} +gateway { + stomp { + enable_stats = true + frame { + max_body_length = 8192 + max_headers = 10 + max_headers_length = 1024 + } + idle_timeout = "30s" + listeners { + tcp { + default { + bind = "61613" + max_conn_rate = 1000 + max_connections = 1024000 + } + } + } + mountpoint = "" + } +} +limiter { + bytes_in {burst = "0", rate = "infinity"} + client { + bytes_in { + capacity = "infinity" + divisible = true + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + connection { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + internal { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + message_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "10" + low_watermark = "0" + max_retry_time = "10s" + rate = "100" + } + message_routing { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + } + connection {burst = "0", rate = "infinity"} + internal {burst = "0", rate = "infinity"} + message_in {burst = "100", rate = "1000/s"} + message_routing {burst = "0", rate = "infinity"} +} +listeners { + ssl { + default { + bind = "0.0.0.0:8883" + max_connections = 512000 + ssl_options { + cacertfile = "etc/certs/cacert.pem" + certfile = "etc/certs/cert.pem" + keyfile = "etc/certs/key.pem" + } + } + } + tcp { + default { + acceptors = 32 + bind = "0.0.0.0:1883" + enable_authn = true + enabled = true + limiter { + bytes_in { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + client { + bytes_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + connection { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + message_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + message_routing { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + } + connection { + capacity = 1000 + initial = "0" + rate = "1000/s" + } + message_in { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + message_routing { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + } + max_connections = 1024000 + proxy_protocol = false + proxy_protocol_timeout = "3s" + tcp_options { + active_n = 100 + backlog = 1024 + buffer = "4KB" + high_watermark = "1MB" + nodelay = true + reuseaddr = true + send_timeout = "15s" + send_timeout_close = true + } + zone = "default" + } + } + ws { + default { + bind = "0.0.0.0:8083" + max_connections = 1024000 + websocket {mqtt_path = "/mqtt"} + } + } + wss { + default { + bind = "0.0.0.0:8084" + max_connections = 512000 + ssl_options { + cacertfile = "etc/certs/cacert.pem" + certfile = "etc/certs/cert.pem" + keyfile = "etc/certs/key.pem" + } + websocket {mqtt_path = "/mqtt"} + } + } +} +mqtt { + await_rel_timeout = "300s" + exclusive_subscription = false + idle_timeout = "15s" + ignore_loop_deliver = false + keepalive_backoff = 0.75 + max_awaiting_rel = 100 + max_clientid_len = 65535 + max_inflight = 320 + max_mqueue_len = 1000 + max_packet_size = "10MB" + max_qos_allowed = 2 + max_subscriptions = "infinity" + max_topic_alias = 65535 + max_topic_levels = 128 + mqueue_default_priority = "lowest" + mqueue_priorities = "disabled" + "mqueue_store_qos0" = true + peer_cert_as_clientid = "disabled" + peer_cert_as_username = "disabled" + response_information = "" + retain_available = true + retry_interval = "30s" + server_keepalive = "disabled" + session_expiry_interval = "2h" + shared_subscription = true + strict_mode = false + upgrade_qos = false + use_username_as_clientid = false + wildcard_subscription = true +} +overload_protection { + backoff_delay = 1 + backoff_gc = false + backoff_hibernation = true + backoff_new_conn = true + enable = false +} +retainer { + backend { + index_specs = [ + [1, 2, 3], + [1, 3], + [2, 3], + [3] + ] + max_retained_messages = 0 + storage_type = "ram" + type = "built_in_database" + } + enable = true + flow_control { + batch_deliver_limiter { + capacity = "infinity" + client { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + initial = "0" + rate = "infinity" + } + batch_deliver_number = 0 + batch_read_number = 0 + } + max_payload_size = "10MB" + msg_clear_interval = "0s" + msg_expiry_interval = "0s" + stop_publish_clear_msg = false +} +rule_engine { + ignore_sys_message = true + jq_function_default_timeout = "10s" + jq_implementation_module = "jq_nif" + rules { + "rule_kcz2" { + actions = ["webhook:test"] + description = "" + metadata {created_at = 1684227520811} + sql = "SELECT\n *\nFROM\n \"t/#\"" + } + } +} +slow_subs { + enable = true + expire_interval = "300s" + stats_type = "whole" + threshold = "500ms" + top_k_num = 10 +} +stats {enable = true} +sys_topics { + sys_event_messages { + client_connected = true + client_disconnected = true + client_subscribed = true + client_unsubscribed = true + } + sys_heartbeat_interval = "30s" + sys_msg_interval = "1m" +} diff --git a/scripts/conf-test/old-confs/v5.0.21.conf b/scripts/conf-test/old-confs/v5.0.21.conf new file mode 100644 index 000000000..c205a8ac2 --- /dev/null +++ b/scripts/conf-test/old-confs/v5.0.21.conf @@ -0,0 +1,498 @@ +node { + name = "emqx@127.0.0.1" + cookie = "emqxsecretcookie" + data_dir = "data" +} + +log { + file_handlers.default { + level = warning + file = "log/emqx.log" + } +} + +cluster { + name = emqxcl + discovery_strategy = manual +} + +dashboard { + listeners.http { + bind = 18083 + } + default_username = "admin" + default_password = "public" +} + +authentication = [ + { + backend = "built_in_database" + mechanism = "password_based" + password_hash_algorithm {name = "sha256", salt_position = "suffix"} + user_id_type = "username" + }, + { + algorithm = "hmac-based" + from = "password" + mechanism = "jwt" + secret = "emqxsecret" + "secret_base64_encoded" = false + use_jwks = false + }, + { + backend = "http" + body {password = "${password}", username = "${username}"} + connect_timeout = "15s" + enable_pipelining = 100 + headers {"content-type" = "application/json"} + mechanism = "password_based" + method = "post" + pool_size = 8 + request_timeout = "5s" + ssl {enable = false, verify = "verify_peer"} + url = "http://127.0.0.1:8080" + } +] +authorization { + cache {enable = true} + deny_action = "ignore" + no_match = "allow" + sources = [ + { + cmd = "HGETALL mqtt_acl:${username}" + database = 0 + pool_size = 8 + redis_type = "single" + server = "127.0.0.1:6379" + ssl {enable = false, verify = "verify_peer"} + type = "redis" + }, + { + database = "mqtt_acl" + pool_size = 8 + query = "SELECT action, permission, topic FROM mqtt_acl where username = ${username}" + server = "127.0.0.1:5432" + ssl {enable = false, verify = "verify_peer"} + type = "postgresql" + username = "root" + }, + { + collection = "users" + database = "mqtt" + filter {username = "${username}"} + mongo_type = "single" + pool_size = 8 + server = "127.0.0.1:27017" + ssl {enable = false, verify = "verify_peer"} + topology {connect_timeout_ms = "20s"} + type = "mongodb" + }, + { + database = "mqtt_acl" + pool_size = 8 + query = "SELECT action, permission, topic FROM mqtt_acl where username = ${username}" + server = "127.0.0.1:3306" + ssl {enable = false, verify = "verify_peer"} + type = "mysql" + username = "root" + }, + { + body {username = "${username}"} + connect_timeout = "15s" + enable_pipelining = 100 + headers {"content-type" = "application/json"} + method = "post" + pool_size = 8 + request_timeout = "30s" + ssl {enable = false, verify = "verify_peer"} + type = "http" + url = "http://127.0.0.1:8080" + }, + { + enable = true + path = "etc/acl.conf" + type = "file" + } + ] +} +bridges { + webhook { + test { + connect_timeout = "15s" + enable_pipelining = 100 + headers {"content-type" = "application/json"} + method = "post" + pool_size = 8 + pool_type = "random" + request_timeout = "15s" + resource_opts { + async_inflight_window = 100 + auto_restart_interval = "60s" + health_check_interval = "15s" + max_queue_bytes = "1GB" + query_mode = "async" + request_timeout = "15s" + worker_pool_size = 4 + } + ssl {enable = false, verify = "verify_peer"} + url = "http://127.0.0.1:8080/test" + } + } +} +conn_congestion {enable_alarm = true, min_alarm_sustain_duration = "1m"} +delayed {enable = true, max_delayed_messages = 10} +exhook { + servers = [ + { + auto_reconnect = "60s" + enable = false + failed_action = "deny" + name = "testx" + pool_size = 16 + request_timeout = "5s" + ssl {enable = false, verify = "verify_peer"} + url = "http://127.0.0.1:8080/exhook" + } + ] +} +flapping_detect { + ban_time = "5m" + enable = false + max_count = 15 + window_time = "1m" +} +force_gc { + bytes = "16MB" + count = 16000 + enable = true +} +force_shutdown { + enable = true + max_heap_size = "32MB" + max_message_queue_len = 1000 +} +gateway { + coap { + connection_required = false + enable_stats = true + listeners { + udp { + default { + bind = "5683" + max_conn_rate = 1000 + max_connections = 1024000 + } + } + } + mountpoint = "" + notify_type = "qos" + publish_qos = "coap" + subscribe_qos = "coap" + } + exproto { + enable_stats = true + handler { + address = "http://127.0.0.1:9001" + ssl_options {enable = false} + } + idle_timeout = "30s" + listeners { + tcp { + default { + acceptors = 16 + bind = "7993" + max_conn_rate = 1000 + max_connections = 1024000 + } + } + } + mountpoint = "" + server { + bind = "127.0.0.1:9100" + ssl_options {verify = "verify_none"} + } + } + "lwm2m" { + auto_observe = true + enable_stats = true + idle_timeout = "30s" + lifetime_max = "86400s" + lifetime_min = "1s" + listeners { + udp { + default { + bind = "5783" + max_conn_rate = 1000 + max_connections = 1024000 + } + } + } + mountpoint = "" + qmode_time_window = "22s" + translators { + command {qos = 0, topic = "dn/#"} + notify {qos = 0, topic = "up/notify"} + register {qos = 0, topic = "up/resp"} + response {qos = 0, topic = "up/resp"} + update {qos = 0, topic = "up/update"} + } + update_msg_publish_condition = "contains_object_list" + xml_dir = "etc/lwm2m_xml/" + } + mqttsn { + broadcast = true + "enable_qos3" = true + enable_stats = true + gateway_id = 1 + idle_timeout = "30s" + listeners { + udp { + default { + bind = "1884" + max_conn_rate = 1000 + max_connections = 1024000 + } + } + } + mountpoint = "" + predefined = [] + } + stomp { + enable_stats = true + frame { + max_body_length = 8192 + max_headers = 10 + max_headers_length = 1024 + } + idle_timeout = "30s" + listeners { + tcp { + default { + bind = "61613" + max_conn_rate = 1000 + max_connections = 1024000 + } + } + } + mountpoint = "" + } +} +listeners { + ssl { + default { + bind = "0.0.0.0:8883" + max_connections = 512000 + ssl_options { + cacertfile = "etc/certs/cacert.pem" + certfile = "etc/certs/cert.pem" + keyfile = "etc/certs/key.pem" + } + } + } + tcp { + default { + acceptors = 32 + bind = "0.0.0.0:1883" + enable_authn = true + enabled = true + limiter { + bytes_in { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + client { + bytes_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + connection { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + message_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + message_routing { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + } + connection { + capacity = "1000" + initial = "0" + rate = "1000/s" + } + message_in { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + message_routing { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + } + max_connections = 1024000 + proxy_protocol = false + proxy_protocol_timeout = "3s" + tcp_options { + active_n = 100 + backlog = 1024 + buffer = "4KB" + high_watermark = "1MB" + nodelay = true + reuseaddr = true + send_timeout = "15s" + send_timeout_close = true + } + zone = "default" + } + } + ws { + default { + bind = "0.0.0.0:8083" + max_connections = 1024000 + websocket {mqtt_path = "/mqtt"} + } + } + wss { + default { + bind = "0.0.0.0:8084" + max_connections = 512000 + ssl_options { + cacertfile = "etc/certs/cacert.pem" + certfile = "etc/certs/cert.pem" + keyfile = "etc/certs/key.pem" + } + websocket {mqtt_path = "/mqtt"} + } + } +} +mqtt { + await_rel_timeout = "300s" + exclusive_subscription = false + idle_timeout = "15s" + ignore_loop_deliver = false + keepalive_backoff = 0.75 + max_awaiting_rel = 100 + max_clientid_len = 65535 + max_inflight = 320 + max_mqueue_len = 1000 + max_packet_size = "10MB" + max_qos_allowed = 2 + max_subscriptions = "infinity" + max_topic_alias = 65535 + max_topic_levels = 128 + mqueue_default_priority = "lowest" + mqueue_priorities = "disabled" + "mqueue_store_qos0" = true + peer_cert_as_clientid = "disabled" + peer_cert_as_username = "disabled" + response_information = "" + retain_available = true + retry_interval = "30s" + server_keepalive = "disabled" + session_expiry_interval = "2h" + shared_subscription = true + strict_mode = false + upgrade_qos = false + use_username_as_clientid = false + wildcard_subscription = true +} +overload_protection { + backoff_delay = 1 + backoff_gc = false + backoff_hibernation = true + backoff_new_conn = true + enable = false +} +retainer { + backend { + index_specs = [ + [1, 2, 3], + [1, 3], + [2, 3], + [3] + ] + max_retained_messages = 0 + storage_type = "ram" + type = "built_in_database" + } + enable = true + flow_control { + batch_deliver_limiter { + capacity = "infinity" + client { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + initial = "0" + rate = "infinity" + } + batch_deliver_number = 0 + batch_read_number = 0 + } + max_payload_size = "10MB" + msg_clear_interval = "0s" + msg_expiry_interval = "0s" + stop_publish_clear_msg = false +} +rule_engine { + ignore_sys_message = true + jq_function_default_timeout = "10s" + jq_implementation_module = "jq_nif" + rules { + rule_plna { + actions = ["webhook:test"] + description = "" + metadata {created_at = 1684226199152} + sql = "SELECT\n *\nFROM\n \"t/#\"" + } + } +} +slow_subs { + enable = true + expire_interval = "300s" + stats_type = "whole" + threshold = "500ms" + top_k_num = 10 +} +stats {enable = true} +sys_topics { + sys_event_messages { + client_connected = true + client_disconnected = true + client_subscribed = true + client_unsubscribed = true + } + sys_heartbeat_interval = "30s" + sys_msg_interval = "1m" +} diff --git a/scripts/conf-test/old-confs/v5.0.22.conf b/scripts/conf-test/old-confs/v5.0.22.conf new file mode 100644 index 000000000..c5cd6e1c9 --- /dev/null +++ b/scripts/conf-test/old-confs/v5.0.22.conf @@ -0,0 +1,502 @@ +node { + name = "emqx@127.0.0.1" + cookie = "emqxsecretcookie" + data_dir = "data" +} + +log { + file_handlers.default { + level = warning + file = "log/emqx.log" + } +} + +cluster { + name = emqxcl + discovery_strategy = manual +} + +dashboard { + listeners.http { + bind = 18083 + } + default_username = "admin" + default_password = "public" +} + +authentication = [ + { + backend = "built_in_database" + mechanism = "password_based" + password_hash_algorithm {name = "sha256", salt_position = "suffix"} + user_id_type = "username" + } +] +authorization { + cache {enable = true} + deny_action = "ignore" + no_match = "allow" + sources = [ + { + body {username = "${username}"} + connect_timeout = "15s" + enable_pipelining = 100 + headers {"content-type" = "application/json"} + method = "post" + pool_size = 8 + request_timeout = "30s" + ssl {enable = false, verify = "verify_peer"} + type = "http" + url = "http://127.0.0.1:8080" + }, + { + enable = true + path = "etc/acl.conf" + type = "file" + } + ] +} +bridges { + webhook { + test { + connect_timeout = "15s" + enable_pipelining = 100 + headers {"content-type" = "application/json"} + method = "post" + pool_size = 8 + pool_type = "random" + request_timeout = "15s" + resource_opts { + async_inflight_window = 100 + auto_restart_interval = "60s" + health_check_interval = "15s" + max_queue_bytes = "1GB" + query_mode = "async" + request_timeout = "15s" + worker_pool_size = 4 + } + ssl {enable = false, verify = "verify_peer"} + url = "http://127.0.0.1:8080/webhook" + } + } +} +conn_congestion {enable_alarm = true, min_alarm_sustain_duration = "1m"} +delayed {enable = true, max_delayed_messages = 1000} +exhook { + servers = [ + { + auto_reconnect = "60s" + enable = false + failed_action = "deny" + name = "test" + pool_size = 16 + request_timeout = "5s" + ssl {enable = false, verify = "verify_peer"} + url = "http://127.0.0.1:8080" + } + ] +} +flapping_detect { + ban_time = "5m" + enable = false + max_count = 15 + window_time = "1m" +} +force_gc { + bytes = "16MB" + count = 16000 + enable = true +} +force_shutdown { + enable = true + max_heap_size = "32MB" + max_message_queue_len = 1000 +} +gateway { + mqttsn { + broadcast = true + "enable_qos3" = true + enable_stats = true + gateway_id = 1 + idle_timeout = "30s" + listeners { + udp { + default { + bind = "1884" + max_conn_rate = 1000 + max_connections = 1024000 + } + } + } + mountpoint = "" + predefined = [] + } +} +limiter { + bytes_in {burst = "0", rate = "10MB/s"} + client { + bytes_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "100KB" + low_watermark = "0" + max_retry_time = "10s" + rate = "2000" + } + connection { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + internal { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + message_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + message_routing { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + } + connection {burst = "0", rate = "infinity"} + internal {burst = "1000", rate = "1000/s"} + message_in {burst = "0", rate = "infinity"} + message_routing {burst = "0", rate = "2000/m"} +} +listeners { + ssl { + default { + bind = "0.0.0.0:8883" + max_connections = 512000 + ssl_options { + cacertfile = "etc/certs/cacert.pem" + certfile = "etc/certs/cert.pem" + keyfile = "etc/certs/key.pem" + } + } + } + tcp { + default {bind = "0.0.0.0:1883", max_connections = 1024000} + } + ws { + default { + acceptors = 32 + bind = "0.0.0.0:8083" + enable_authn = true + enabled = true + limiter { + bytes_in { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + client { + bytes_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + connection { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + message_in { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + message_routing { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + } + connection { + capacity = "1000" + initial = "0" + rate = "1000/s" + } + message_in { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + message_routing { + capacity = "infinity" + initial = "0" + rate = "infinity" + } + } + max_connections = 1024000 + proxy_protocol = false + proxy_protocol_timeout = "3s" + tcp_options { + active_n = 100 + backlog = 1024 + buffer = "4KB" + high_watermark = "1MB" + nodelay = true + reuseaddr = true + send_timeout = "15s" + send_timeout_close = true + } + websocket { + allow_origin_absence = true + check_origin_enable = false + check_origins = "http://localhost:18083, http://127.0.0.1:18083" + compress = false + deflate_opts { + client_context_takeover = "takeover" + client_max_window_bits = 15 + mem_level = 8 + server_context_takeover = "takeover" + server_max_window_bits = 15 + strategy = "default" + } + fail_if_no_subprotocol = true + idle_timeout = "7200s" + max_frame_size = "infinity" + mqtt_path = "/mqtt" + mqtt_piggyback = "multiple" + proxy_address_header = "x-forwarded-for" + proxy_port_header = "x-forwarded-port" + supported_subprotocols = "mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5" + } + zone = "default" + } + } + wss { + default { + bind = "0.0.0.0:8084" + max_connections = 512000 + ssl_options { + cacertfile = "etc/certs/cacert.pem" + certfile = "etc/certs/cert.pem" + keyfile = "etc/certs/key.pem" + } + websocket {mqtt_path = "/mqtt"} + } + } +} +log { + console_handler { + burst_limit { + enable = true + max_count = 10000 + window_time = "1s" + } + chars_limit = 1000 + drop_mode_qlen = 3000 + enable = true + flush_qlen = 8000 + formatter = "text" + level = "warning" + max_depth = "unlimited" + overload_kill { + enable = true + mem_size = "30MB" + qlen = 20000 + restart_after = "5s" + } + single_line = true + supervisor_reports = "error" + sync_mode_qlen = 100 + time_offset = "system" + } + file_handlers { + default { + burst_limit { + enable = true + max_count = 10000 + window_time = "1s" + } + chars_limit = "unlimited" + drop_mode_qlen = 3000 + enable = false + file = "log/emqx.log" + flush_qlen = 8000 + formatter = "text" + level = "warning" + max_depth = 100 + max_size = "50MB" + overload_kill { + enable = true + mem_size = "30MB" + qlen = 20000 + restart_after = "5s" + } + rotation {count = 10, enable = true} + single_line = true + supervisor_reports = "error" + sync_mode_qlen = 100 + time_offset = "system" + } + } +} +mqtt { + await_rel_timeout = "300s" + exclusive_subscription = true + idle_timeout = "15s" + ignore_loop_deliver = true + keepalive_backoff = 1 + max_awaiting_rel = 100 + max_clientid_len = 65535 + max_inflight = 32 + max_mqueue_len = 1000 + max_packet_size = "10MB" + max_qos_allowed = 2 + max_subscriptions = 2000 + max_topic_alias = 65535 + max_topic_levels = 128 + mqueue_default_priority = "lowest" + mqueue_priorities = "disabled" + "mqueue_store_qos0" = true + peer_cert_as_clientid = "disabled" + peer_cert_as_username = "disabled" + response_information = "" + retain_available = true + retry_interval = "30s" + server_keepalive = 20 + session_expiry_interval = "2h" + shared_subscription = true + strict_mode = false + upgrade_qos = true + use_username_as_clientid = true + wildcard_subscription = true +} +overload_protection { + backoff_delay = 1 + backoff_gc = false + backoff_hibernation = true + backoff_new_conn = true + enable = false +} +retainer { + backend { + index_specs = [ + [1, 2, 3], + [1, 3], + [2, 3], + [3] + ] + max_retained_messages = 2000 + storage_type = "ram" + type = "built_in_database" + } + enable = true + flow_control { + batch_deliver_limiter { + capacity = "infinity" + client { + capacity = "infinity" + divisible = false + failure_strategy = "force" + initial = "0" + low_watermark = "0" + max_retry_time = "10s" + rate = "infinity" + } + initial = "0" + rate = "infinity" + } + batch_deliver_number = 0 + batch_read_number = 0 + } + max_payload_size = "1MB" + msg_clear_interval = "2000ms" + msg_expiry_interval = "1000ms" + stop_publish_clear_msg = false +} +rule_engine { + ignore_sys_message = true + jq_function_default_timeout = "10s" + rules { + "rule_fhd9" { + actions = ["webhook:test"] + description = "" + metadata {created_at = 1684225481677} + sql = "SELECT\n *\nFROM\n \"t/#\"" + } + } +} +stats {enable = true} +sys_topics { + sys_event_messages { + client_connected = true + client_disconnected = true + client_subscribed = true + client_unsubscribed = true + } + sys_heartbeat_interval = "30s" + sys_msg_interval = "1m" +} +sysmon { + os { + cpu_check_interval = "60s" + cpu_high_watermark = "80%" + cpu_low_watermark = "60%" + mem_check_interval = "60s" + procmem_high_watermark = "5%" + sysmem_high_watermark = "70%" + } + top { + db_hostname = "" + db_name = "postgres" + db_password = "******" + db_port = 5432 + db_username = "system_monitor" + max_procs = 1000000 + num_items = 10 + sample_interval = "2s" + } + vm { + busy_dist_port = false + busy_port = false + large_heap = "disabled" + long_gc = "disabled" + long_schedule = "disabled" + process_check_interval = "30s" + process_high_watermark = "80%" + process_low_watermark = "60%" + } +} diff --git a/scripts/conf-test/old-confs/v5.0.23.conf b/scripts/conf-test/old-confs/v5.0.23.conf new file mode 100644 index 000000000..2bf5056d2 --- /dev/null +++ b/scripts/conf-test/old-confs/v5.0.23.conf @@ -0,0 +1,280 @@ +node { + name = "emqx@127.0.0.1" + cookie = "emqxsecretcookie" + data_dir = "data" +} + +log { + file_handlers.default { + level = warning + file = "log/emqx.log" + } +} + +cluster { + name = emqxcl + discovery_strategy = manual +} + +dashboard { + listeners.http { + bind = 18083 + } + default_username = "admin" + default_password = "public" +} + +authentication = [ + { + algorithm = "hmac-based" + from = "password" + mechanism = "jwt" + secret = "emqxsecret" + "secret_base64_encoded" = false + use_jwks = false + verify_claims {} + } +] +authorization { + cache {enable = true} + deny_action = "ignore" + no_match = "allow" + sources = [ + {type = "built_in_database"}, + { + enable = true + path = "etc/acl.conf" + type = "file" + } + ] +} +bridges { + mqtt { + "to-public-broker" { + bridge_mode = false + clean_start = true + egress { + local {topic = "t/#"} + remote { + payload = "${payload}" + qos = 1 + retain = false + topic = "b/t" + } + } + enable = true + keepalive = "300s" + mode = "cluster_shareload" + proto_ver = "v4" + resource_opts { + auto_restart_interval = "60s" + health_check_interval = "15s" + inflight_window = 100 + max_buffer_bytes = "1GB" + query_mode = "async" + request_timeout = "15s" + worker_pool_size = 4 + } + retry_interval = "15s" + server = "broker.emqx.io:1883" + ssl {enable = false, verify = "verify_peer"} + } + } +} +exhook { + servers = [ + { + auto_reconnect = "60s" + enable = false + failed_action = "deny" + name = "test" + pool_size = 16 + request_timeout = "5s" + ssl {enable = false, verify = "verify_peer"} + url = "http://127.0.0.1:8080/exhook" + } + ] +} +gateway { + mqttsn { + broadcast = true + "enable_qos3" = true + enable_stats = true + gateway_id = 1 + idle_timeout = "30s" + listeners { + udp { + default { + bind = "1884" + max_conn_rate = 1000 + max_connections = 1024000 + } + } + } + mountpoint = "" + predefined = [] + } +} +listeners { + ssl { + default { + bind = "0.0.0.0:8883" + max_connections = 512000 + ssl_options { + cacertfile = "etc/certs/cacert.pem" + certfile = "etc/certs/cert.pem" + keyfile = "etc/certs/key.pem" + } + } + } + tcp { + default { + acceptors = 32 + bind = "0.0.0.0:1883" + enable_authn = true + enabled = true + limiter { + bytes {rate = "infinity"} + client { + bytes {rate = "infinity"} + messages {rate = "infinity"} + } + messages {rate = "infinity"} + } + max_connections = 1024000 + proxy_protocol = false + proxy_protocol_timeout = "3s" + tcp_options { + active_n = 100 + backlog = 1024 + buffer = "4KB" + high_watermark = "1MB" + nodelay = true + reuseaddr = true + send_timeout = "15s" + send_timeout_close = true + } + zone = "default" + } + } + ws { + default { + bind = "0.0.0.0:8083" + max_connections = 1024000 + websocket {mqtt_path = "/mqtt"} + } + } + wss { + default { + bind = "0.0.0.0:8084" + max_connections = 512000 + ssl_options { + cacertfile = "etc/certs/cacert.pem" + certfile = "etc/certs/cert.pem" + keyfile = "etc/certs/key.pem" + } + websocket {mqtt_path = "/mqtt"} + } + } +} +mqtt { + await_rel_timeout = "300s" + exclusive_subscription = true + idle_timeout = "15s" + ignore_loop_deliver = true + keepalive_backoff = 1 + max_awaiting_rel = 100 + max_clientid_len = 65535 + max_inflight = 32 + max_mqueue_len = 1000 + max_packet_size = "1MB" + max_qos_allowed = 2 + max_subscriptions = 100 + max_topic_alias = 65535 + max_topic_levels = 128 + mqueue_default_priority = "lowest" + mqueue_priorities = "disabled" + "mqueue_store_qos0" = true + peer_cert_as_clientid = "cn" + peer_cert_as_username = "cn" + response_information = "" + retain_available = true + retry_interval = "30s" + server_keepalive = 200 + session_expiry_interval = "2h" + shared_subscription = true + strict_mode = true + upgrade_qos = true + use_username_as_clientid = true + wildcard_subscription = true +} +retainer { + backend { + index_specs = [ + [1, 2, 3], + [1, 3], + [2, 3], + [3] + ] + max_retained_messages = 1000 + storage_type = "ram" + type = "built_in_database" + } + enable = true + max_payload_size = "1MB" + msg_clear_interval = "20000ms" + msg_expiry_interval = "10000ms" + stop_publish_clear_msg = false +} +rule_engine { + ignore_sys_message = true + jq_function_default_timeout = "10s" + rules { + rule_tcbl { + actions = ["mqtt:to-public-broker"] + description = "" + metadata {created_at = 1684223521242} + sql = "SELECT\n *\nFROM\n \"t/#\"" + } + } +} +sys_topics { + sys_event_messages { + client_connected = true + client_disconnected = true + client_subscribed = true + client_unsubscribed = true + } + sys_heartbeat_interval = "30s" + sys_msg_interval = "1m" +} +sysmon { + os { + cpu_check_interval = "60s" + cpu_high_watermark = "80%" + cpu_low_watermark = "60%" + mem_check_interval = "60s" + procmem_high_watermark = "5%" + sysmem_high_watermark = "70%" + } + top { + db_hostname = "" + db_name = "postgres" + db_password = "******" + db_port = 5432 + db_username = "system_monitor" + max_procs = 1000000 + num_items = 10 + sample_interval = "2s" + } + vm { + busy_dist_port = true + busy_port = true + large_heap = "32MB" + long_gc = "100ms" + long_schedule = "240ms" + process_check_interval = "30s" + process_high_watermark = "80%" + process_low_watermark = "60%" + } +} diff --git a/scripts/conf-test/old-confs/v5.0.24.conf b/scripts/conf-test/old-confs/v5.0.24.conf new file mode 100644 index 000000000..80ce00245 --- /dev/null +++ b/scripts/conf-test/old-confs/v5.0.24.conf @@ -0,0 +1,295 @@ +node { + name = "emqx@127.0.0.1" + cookie = "emqxsecretcookie" + data_dir = "data" +} + +log { + file_handlers.default { + level = warning + file = "log/emqx.log" + } +} + +cluster { + name = emqxcl + discovery_strategy = manual +} + +authentication = [ + { + backend = "built_in_database" + mechanism = "password_based" + password_hash_algorithm {name = "sha256", salt_position = "suffix"} + user_id_type = "username" + } +] +authorization { + cache {enable = true} + deny_action = "ignore" + no_match = "allow" + sources = [ + { + body {username = "${username}"} + connect_timeout = "15s" + enable_pipelining = 100 + headers {"content-type" = "application/json"} + method = "post" + pool_size = 8 + request_timeout = "30s" + ssl {enable = false, verify = "verify_peer"} + type = "http" + url = "http://127.0.0.1:8080" + }, + { + enable = true + path = "etc/acl.conf" + type = "file" + } + ] +} +bridges { + webhook { + test { + connect_timeout = "15s" + enable_pipelining = 100 + headers {"content-type" = "application/json"} + method = "post" + pool_size = 8 + pool_type = "random" + request_timeout = "15s" + resource_opts { + async_inflight_window = 100 + auto_restart_interval = "60s" + health_check_interval = "15s" + max_queue_bytes = "1GB" + query_mode = "async" + request_timeout = "15s" + worker_pool_size = 4 + } + ssl {enable = false, verify = "verify_peer"} + url = "http://127.0.0.1:8080/webhook" + } + } +} +exhook { + servers = [ + { + auto_reconnect = "60s" + enable = false + failed_action = "deny" + name = "test" + pool_size = 16 + request_timeout = "5s" + ssl {enable = false, verify = "verify_peer"} + url = "http://127.0.0.1:8080" + } + ] +} +gateway { + coap { + connection_required = false + enable_stats = true + listeners { + udp { + default { + bind = "5683" + max_conn_rate = 1000 + max_connections = 1024000 + } + } + } + mountpoint = "" + notify_type = "qos" + publish_qos = "coap" + subscribe_qos = "coap" + } +} +listeners { + ssl { + default { + bind = "0.0.0.0:8883" + max_connections = 512000 + ssl_options { + cacertfile = "etc/certs/cacert.pem" + certfile = "etc/certs/cert.pem" + keyfile = "etc/certs/key.pem" + } + } + } + tcp { + default { + acceptors = 16 + bind = "0.0.0.0:1883" + enable_authn = true + enabled = true + limiter { + bytes {rate = "infinity"} + client { + bytes {rate = "infinity"} + messages {rate = "infinity"} + } + messages {rate = "infinity"} + } + max_connections = 5000 + proxy_protocol = false + proxy_protocol_timeout = "3s" + tcp_options { + active_n = 100 + backlog = 1024 + buffer = "4KB" + high_watermark = "1MB" + nodelay = true + reuseaddr = true + send_timeout = "15s" + send_timeout_close = true + } + zone = "default" + } + } + ws { + default { + bind = "0.0.0.0:8083" + max_connections = 1024000 + websocket {mqtt_path = "/mqtt"} + } + } + wss { + default { + bind = "0.0.0.0:8084" + max_connections = 512000 + ssl_options { + cacertfile = "etc/certs/cacert.pem" + certfile = "etc/certs/cert.pem" + keyfile = "etc/certs/key.pem" + } + websocket {mqtt_path = "/mqtt"} + } + } +} +log { + console_handler { + burst_limit { + enable = true + max_count = 10000 + window_time = "1s" + } + chars_limit = "unlimited" + drop_mode_qlen = 3000 + enable = true + flush_qlen = 8000 + formatter = "text" + level = "warning" + max_depth = 100 + overload_kill { + enable = true + mem_size = "30MB" + qlen = "10000" + restart_after = "5s" + } + single_line = true + supervisor_reports = "error" + sync_mode_qlen = 100 + time_offset = "system" + } + file_handlers { + default { + burst_limit { + enable = true + max_count = 10000 + window_time = "1s" + } + chars_limit = "unlimited" + drop_mode_qlen = 3000 + enable = false + file = "log/emqx.log" + flush_qlen = 8000 + formatter = "text" + level = "info" + max_depth = 100 + max_size = "50MB" + overload_kill { + enable = true + mem_size = "30MB" + qlen = 20000 + restart_after = "5s" + } + rotation {count = 10, enable = true} + single_line = true + supervisor_reports = "error" + sync_mode_qlen = 100 + time_offset = "system" + } + } +} +mqtt { + await_rel_timeout = "300s" + exclusive_subscription = false + idle_timeout = "15s" + ignore_loop_deliver = false + keepalive_backoff = 0.75 + max_awaiting_rel = 100 + max_clientid_len = 65535 + max_inflight = 32 + max_mqueue_len = 1000 + max_packet_size = "10MB" + max_qos_allowed = 2 + max_subscriptions = "infinity" + max_topic_alias = 65535 + max_topic_levels = 128 + mqueue_default_priority = "lowest" + mqueue_priorities = "disabled" + "mqueue_store_qos0" = true + peer_cert_as_clientid = "disabled" + peer_cert_as_username = "disabled" + response_information = "" + retain_available = true + retry_interval = "30s" + server_keepalive = "disabled" + session_expiry_interval = "12h" + shared_subscription = true + strict_mode = false + upgrade_qos = false + use_username_as_clientid = false + wildcard_subscription = true +} +retainer { + backend { + index_specs = [ + [1, 2, 3], + [1, 3], + [2, 3], + [3] + ] + max_retained_messages = 0 + storage_type = "ram" + type = "built_in_database" + } + enable = true + max_payload_size = "10MB" + msg_clear_interval = "0s" + msg_expiry_interval = "0s" + stop_publish_clear_msg = false +} +rule_engine { + ignore_sys_message = true + jq_function_default_timeout = "10s" + rules { + "rule_s56b" { + actions = ["webhook:test"] + description = "" + metadata {created_at = 1683882654432} + sql = "SELECT\n *\nFROM\n \"t/#\"" + } + } +} +sys_topics { + sys_event_messages { + client_connected = true + client_disconnected = true + client_subscribed = true + client_unsubscribed = true + } + sys_heartbeat_interval = "30s" + sys_msg_interval = "1m" +} diff --git a/scripts/conf-test/old-confs/v5.0.25.conf b/scripts/conf-test/old-confs/v5.0.25.conf new file mode 100644 index 000000000..11c6e4dc0 --- /dev/null +++ b/scripts/conf-test/old-confs/v5.0.25.conf @@ -0,0 +1,437 @@ +node { + name = "emqx@127.0.0.1" + cookie = "emqxsecretcookie" + data_dir = "data" +} + +cluster { + name = emqxcl + discovery_strategy = manual +} + + +dashboard { + listeners.http { + bind = 18083 + } +} + +authentication = [ + { + backend = "built_in_database" + mechanism = "password_based" + password_hash_algorithm {name = "sha256", salt_position = "suffix"} + user_id_type = "username" + } +] +authorization { + cache { + enable = true + max_size = 32 + ttl = "1m" + } + deny_action = "ignore" + no_match = "allow" + sources = [ + { + body {username = "${username}"} + connect_timeout = "15s" + enable_pipelining = 100 + headers {"content-type" = "application/json"} + method = "post" + pool_size = 8 + request_timeout = "30s" + ssl {enable = false, verify = "verify_peer"} + type = "http" + url = "http://127.0.0.1:8080" + }, + { + enable = true + path = "${EMQX_ETC_DIR}/acl.conf" + type = "file" + } + ] +} +bridges { + webhook { + test { + connect_timeout = "15s" + enable_pipelining = 100 + headers {"content-type" = "application/json"} + method = "post" + pool_size = 8 + pool_type = "random" + request_timeout = "15s" + resource_opts { + auto_restart_interval = "60s" + health_check_interval = "15s" + inflight_window = 100 + max_buffer_bytes = "1GB" + query_mode = "async" + request_timeout = "15s" + worker_pool_size = 4 + } + ssl {enable = false, verify = "verify_peer"} + url = "http://127.0.0.1:8080/webhook" + } + } +} +delayed {enable = false, max_delayed_messages = 2000} +exhook { + servers = [ + { + auto_reconnect = "60s" + enable = false + failed_action = "deny" + name = "test" + pool_size = 16 + request_timeout = "5s" + ssl {enable = false, verify = "verify_peer"} + url = "http://127.0.0.1:8080" + } + ] +} +gateway { + "lwm2m" { + auto_observe = true + enable_stats = true + idle_timeout = "30s" + lifetime_max = "86400s" + lifetime_min = "1s" + listeners { + udp { + default { + bind = "5783" + max_conn_rate = 1000 + max_connections = 1024000 + } + } + } + mountpoint = "" + qmode_time_window = "22s" + translators { + command {qos = 0, topic = "dn/#"} + notify {qos = 0, topic = "up/notify"} + register {qos = 0, topic = "up/resp"} + response {qos = 0, topic = "up/resp"} + update {qos = 0, topic = "up/update"} + } + update_msg_publish_condition = "contains_object_list" + xml_dir = "etc/lwm2m_xml/" + } +} +limiter {max_conn_rate = "2000/s"} +listeners { + ssl { + default { + acceptors = 16 + access_rules = ["allow all"] + bind = "0.0.0.0:8883" + enable_authn = true + enabled = true + max_connections = 5000000 + mountpoint = "" + proxy_protocol = false + proxy_protocol_timeout = "3s" + ssl_options { + cacertfile = "${EMQX_ETC_DIR}/certs/cacert.pem" + certfile = "${EMQX_ETC_DIR}/certs/cert.pem" + ciphers = [] + client_renegotiation = true + depth = 10 + enable_crl_check = false + fail_if_no_peer_cert = false + gc_after_handshake = false + handshake_timeout = "15s" + hibernate_after = "5s" + honor_cipher_order = true + keyfile = "${EMQX_ETC_DIR}/certs/key.pem" + ocsp { + enable_ocsp_stapling = false + refresh_http_timeout = "15s" + refresh_interval = "5m" + } + reuse_sessions = true + secure_renegotiate = true + verify = "verify_none" + versions = ["tlsv1.3", "tlsv1.2"] + } + tcp_options { + active_n = 100 + backlog = 1024 + buffer = "4KB" + high_watermark = "1MB" + nodelay = true + reuseaddr = true + send_timeout = "15s" + send_timeout_close = true + } + zone = "default" + } + } + tcp { + default { + acceptors = 16 + access_rules = ["allow all"] + bind = "0.0.0.0:1883" + enable_authn = true + enabled = true + max_connections = 5000000 + mountpoint = "" + proxy_protocol = false + proxy_protocol_timeout = "3s" + tcp_options { + active_n = 100 + backlog = 1024 + buffer = "4KB" + high_watermark = "1MB" + nodelay = true + reuseaddr = true + send_timeout = "15s" + send_timeout_close = true + } + zone = "default" + } + } + ws { + default { + acceptors = 32 + access_rules = ["allow all"] + bind = "0.0.0.0:8083" + enable_authn = true + enabled = true + max_connections = 5000000 + mountpoint = "" + proxy_protocol = false + proxy_protocol_timeout = "3s" + tcp_options { + active_n = 100 + backlog = 1024 + buffer = "40KB" + high_watermark = "1MB" + nodelay = true + reuseaddr = true + send_timeout = "15s" + send_timeout_close = true + } + websocket { + allow_origin_absence = true + check_origin_enable = false + check_origins = "http://localhost:18083, http://127.0.0.1:18083" + compress = false + deflate_opts { + client_context_takeover = "takeover" + client_max_window_bits = 15 + mem_level = 8 + server_context_takeover = "takeover" + server_max_window_bits = 15 + strategy = "default" + } + fail_if_no_subprotocol = true + idle_timeout = "7200s" + max_frame_size = "infinity" + mqtt_path = "/mqtt" + mqtt_piggyback = "multiple" + proxy_address_header = "x-forwarded-for" + proxy_port_header = "x-forwarded-port" + supported_subprotocols = "mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5" + } + zone = "default" + } + } + wss { + default { + acceptors = 16 + access_rules = ["allow all"] + bind = "0.0.0.0:8084" + enable_authn = true + enabled = true + max_connections = 5000000 + mountpoint = "" + proxy_protocol = false + proxy_protocol_timeout = "3s" + ssl_options { + cacertfile = "${EMQX_ETC_DIR}/certs/cacert.pem" + certfile = "${EMQX_ETC_DIR}/certs/cert.pem" + ciphers = [] + client_renegotiation = true + depth = 10 + fail_if_no_peer_cert = false + handshake_timeout = "15s" + hibernate_after = "5s" + honor_cipher_order = true + keyfile = "${EMQX_ETC_DIR}/certs/key.pem" + reuse_sessions = true + secure_renegotiate = true + verify = "verify_none" + versions = ["tlsv1.3", "tlsv1.2"] + } + tcp_options { + active_n = 100 + backlog = 1024 + buffer = "4KB" + high_watermark = "1MB" + nodelay = true + reuseaddr = true + send_timeout = "15s" + send_timeout_close = true + } + websocket { + allow_origin_absence = true + check_origin_enable = false + check_origins = "http://localhost:18083, http://127.0.0.1:18083" + compress = false + deflate_opts { + client_context_takeover = "takeover" + client_max_window_bits = 15 + mem_level = 8 + server_context_takeover = "takeover" + server_max_window_bits = 15 + strategy = "default" + } + fail_if_no_subprotocol = true + idle_timeout = "7200s" + max_frame_size = "infinity" + mqtt_path = "/mqtt" + mqtt_piggyback = "multiple" + proxy_address_header = "x-forwarded-for" + proxy_port_header = "x-forwarded-port" + supported_subprotocols = "mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5" + } + zone = "default" + } + } +} +log { + console_handler { + burst_limit { + enable = true + max_count = 10000 + window_time = "1s" + } + chars_limit = 10000 + drop_mode_qlen = 3000 + enable = true + flush_qlen = 8000 + formatter = "text" + level = "info" + max_depth = 100 + overload_kill { + enable = true + mem_size = "30MB" + qlen = 20000 + restart_after = "5s" + } + single_line = true + supervisor_reports = "error" + sync_mode_qlen = 100 + time_offset = "system" + } + file_handlers { + default { + burst_limit { + enable = true + max_count = 10000 + window_time = "1s" + } + chars_limit = "unlimited" + drop_mode_qlen = 3000 + enable = false + file = "${EMQX_LOG_DIR}/emqx.log" + flush_qlen = 8000 + formatter = "text" + level = "warning" + max_depth = 100 + max_size = "50MB" + overload_kill { + enable = true + mem_size = "30MB" + qlen = 20000 + restart_after = "5s" + } + rotation {count = 10, enable = true} + single_line = true + supervisor_reports = "error" + sync_mode_qlen = 100 + time_offset = "system" + } + } +} +mqtt { + await_rel_timeout = "300s" + exclusive_subscription = true + idle_timeout = "15s" + ignore_loop_deliver = true + keepalive_backoff = 1 + max_awaiting_rel = 100 + max_clientid_len = 65535 + max_inflight = 32 + max_mqueue_len = 1000 + max_packet_size = "10MB" + max_qos_allowed = 2 + max_subscriptions = 15 + max_topic_alias = 65535 + max_topic_levels = 128 + mqueue_default_priority = "lowest" + mqueue_priorities = "disabled" + "mqueue_store_qos0" = true + peer_cert_as_clientid = "cn" + peer_cert_as_username = "cn" + response_information = "" + retain_available = true + retry_interval = "30s" + server_keepalive = 15 + session_expiry_interval = "2h" + shared_subscription = true + strict_mode = true + upgrade_qos = true + use_username_as_clientid = true + wildcard_subscription = true +} +retainer { + backend { + index_specs = [ + [1, 2, 3], + [1, 3], + [2, 3], + [3] + ] + max_retained_messages = 120 + storage_type = "ram" + type = "built_in_database" + } + enable = true + max_payload_size = "1MB" + msg_clear_interval = "2000ms" + msg_expiry_interval = "10000ms" + stop_publish_clear_msg = false +} +rule_engine { + ignore_sys_message = true + jq_function_default_timeout = "10s" + rules { + "rule_t7cl" { + actions = ["webhook:test"] + description = "" + metadata {created_at = 1684222459298} + sql = "SELECT\n *\nFROM\n \"t/#\"" + } + } +} +slow_subs { + enable = true + expire_interval = "3000s" + stats_type = "internal" + threshold = "5000ms" + top_k_num = 1000 +} +sys_topics { + sys_event_messages { + client_connected = true + client_disconnected = true + client_subscribed = true + client_unsubscribed = true + } + sys_heartbeat_interval = "30s" + sys_msg_interval = "1m" +} diff --git a/scripts/conf-test/run.sh b/scripts/conf-test/run.sh new file mode 100755 index 000000000..4c2f18615 --- /dev/null +++ b/scripts/conf-test/run.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +set -euo pipefail + +PROFILE="${PROFILE:-emqx}" +EMQX_ROOT="${EMQX_ROOT:-_build/$PROFILE/rel/emqx}" +EMQX_WAIT_FOR_START="${EMQX_WAIT_FOR_START:-30}" +export EMQX_WAIT_FOR_START + +start_emqx_with_conf() { + echo "Starting $PROFILE with $1" + "$EMQX_ROOT"/bin/emqx start + "$EMQX_ROOT"/bin/emqx stop +} + +MINOR_VSN=$(./pkg-vsn.sh "$PROFILE" | cut -d. -f1,2) + +if [ "$PROFILE" = "emqx" ]; then + PREFIX="v" +else + PREFIX="e" +fi + +FILES=$(ls ./scripts/conf-test/old-confs/$PREFIX"$MINOR_VSN"*) + +cp "$EMQX_ROOT"/etc/emqx.conf "$EMQX_ROOT"/etc/emqx.conf.bak +cleanup() { + cp "$EMQX_ROOT"/etc/emqx.conf.bak "$EMQX_ROOT"/etc/emqx.conf +} +trap cleanup EXIT + +for file in $FILES; do + cp "$file" "$EMQX_ROOT"/etc/emqx.conf + start_emqx_with_conf "$file" +done diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index 4824fbdf3..b24c760f0 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -92,6 +92,12 @@ if [ "${WHICH_APP}" = 'novalue' ]; then exit 1 fi +if [ ! -d "${WHICH_APP}" ]; then + echo "must provide an existing path for --app arg" + help + exit 1 +fi + if [[ "${WHICH_APP}" == lib-ee* && (-z "${PROFILE+x}" || "${PROFILE}" != emqx-enterprise) ]]; then echo 'You are trying to run an enterprise test case without the emqx-enterprise profile.' echo 'This will most likely not work.' @@ -203,6 +209,10 @@ for dep in ${CT_DEPS}; do rabbitmq) FILES+=( '.ci/docker-compose-file/docker-compose-rabbitmq.yaml' ) ;; + minio) + FILES+=( '.ci/docker-compose-file/docker-compose-minio-tcp.yaml' + '.ci/docker-compose-file/docker-compose-minio-tls.yaml' ) + ;; *) echo "unknown_ct_dependency $dep" exit 1