diff --git a/.ci/docker-compose-file/docker-compose-kafka.yaml b/.ci/docker-compose-file/docker-compose-kafka.yaml index bfae12182..158db93f1 100644 --- a/.ci/docker-compose-file/docker-compose-kafka.yaml +++ b/.ci/docker-compose-file/docker-compose-kafka.yaml @@ -18,7 +18,7 @@ services: - /tmp/emqx-ci/emqx-shared-secret:/var/lib/secret kdc: hostname: kdc.emqx.net - image: ghcr.io/emqx/emqx-builder/5.3-2:1.15.7-26.2.1-2-ubuntu22.04 + image: ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-ubuntu22.04 container_name: kdc.emqx.net expose: - 88 # kdc diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml index c21c0dc82..d354a9281 100644 --- a/.ci/docker-compose-file/docker-compose.yaml +++ b/.ci/docker-compose-file/docker-compose.yaml @@ -3,7 +3,7 @@ version: '3.9' services: erlang: container_name: erlang - image: ${DOCKER_CT_RUNNER_IMAGE:-ghcr.io/emqx/emqx-builder/5.3-2:1.15.7-26.2.1-2-ubuntu22.04} + image: ${DOCKER_CT_RUNNER_IMAGE:-ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-ubuntu22.04} env_file: - credentials.env - conf.env diff --git a/.github/workflows/_pr_entrypoint.yaml b/.github/workflows/_pr_entrypoint.yaml index fcec256b0..779b73d16 100644 --- a/.github/workflows/_pr_entrypoint.yaml +++ b/.github/workflows/_pr_entrypoint.yaml @@ -20,15 +20,15 @@ permissions: jobs: sanity-checks: runs-on: ubuntu-22.04 - container: "ghcr.io/emqx/emqx-builder/5.3-2:1.15.7-26.2.1-2-ubuntu22.04" + container: "ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-ubuntu22.04" outputs: ct-matrix: ${{ steps.matrix.outputs.ct-matrix }} ct-host: ${{ steps.matrix.outputs.ct-host }} ct-docker: ${{ steps.matrix.outputs.ct-docker }} version-emqx: ${{ steps.matrix.outputs.version-emqx }} version-emqx-enterprise: ${{ steps.matrix.outputs.version-emqx-enterprise }} - builder: "ghcr.io/emqx/emqx-builder/5.3-2:1.15.7-26.2.1-2-ubuntu22.04" - builder_vsn: "5.3-2" + builder: "ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-ubuntu22.04" + builder_vsn: "5.3-4" otp_vsn: "26.2.1-2" elixir_vsn: "1.15.7" @@ -95,12 +95,12 @@ jobs: MATRIX="$(echo "${APPS}" | jq -c ' [ (.[] | select(.profile == "emqx") | . + { - builder: "5.3-2", + builder: "5.3-4", otp: "26.2.1-2", elixir: "1.15.7" }), (.[] | select(.profile == "emqx-enterprise") | . + { - builder: "5.3-2", + builder: "5.3-4", otp: ["26.2.1-2"][], elixir: "1.15.7" }) diff --git a/.github/workflows/_push-entrypoint.yaml b/.github/workflows/_push-entrypoint.yaml index e9dbb41a0..b141989d5 100644 --- a/.github/workflows/_push-entrypoint.yaml +++ b/.github/workflows/_push-entrypoint.yaml @@ -23,7 +23,7 @@ env: jobs: prepare: runs-on: ubuntu-22.04 - container: 'ghcr.io/emqx/emqx-builder/5.3-2:1.15.7-26.2.1-2-ubuntu22.04' + container: 'ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-ubuntu22.04' outputs: profile: ${{ steps.parse-git-ref.outputs.profile }} release: ${{ steps.parse-git-ref.outputs.release }} @@ -31,8 +31,8 @@ jobs: ct-matrix: ${{ steps.matrix.outputs.ct-matrix }} ct-host: ${{ steps.matrix.outputs.ct-host }} ct-docker: ${{ steps.matrix.outputs.ct-docker }} - builder: 'ghcr.io/emqx/emqx-builder/5.3-2:1.15.7-26.2.1-2-ubuntu22.04' - builder_vsn: '5.3-2' + builder: 'ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-ubuntu22.04' + builder_vsn: '5.3-4' otp_vsn: '26.2.1-2' elixir_vsn: '1.15.7' @@ -62,12 +62,12 @@ jobs: MATRIX="$(echo "${APPS}" | jq -c ' [ (.[] | select(.profile == "emqx") | . + { - builder: "5.3-2", + builder: "5.3-4", otp: "26.2.1-2", elixir: "1.15.7" }), (.[] | select(.profile == "emqx-enterprise") | . + { - builder: "5.3-2", + builder: "5.3-4", otp: ["26.2.1-2"][], elixir: "1.15.7" }) diff --git a/.github/workflows/build_and_push_docker_images.yaml b/.github/workflows/build_and_push_docker_images.yaml index f1b4a3d90..583b0b42a 100644 --- a/.github/workflows/build_and_push_docker_images.yaml +++ b/.github/workflows/build_and_push_docker_images.yaml @@ -61,7 +61,7 @@ on: builder_vsn: required: false type: string - default: '5.3-2' + default: '5.3-4' permissions: contents: read diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 3b19e7094..fecb55f3f 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -63,7 +63,7 @@ on: builder_vsn: required: false type: string - default: '5.3-2' + default: '5.3-4' permissions: contents: read diff --git a/.github/workflows/build_packages_cron.yaml b/.github/workflows/build_packages_cron.yaml index 16c50ceb8..c3e76d1b2 100644 --- a/.github/workflows/build_packages_cron.yaml +++ b/.github/workflows/build_packages_cron.yaml @@ -23,8 +23,8 @@ jobs: fail-fast: false matrix: profile: - - ['emqx', 'master', '5.3-2:1.15.7-26.2.1-2'] - - ['emqx-enterprise', 'release-56', '5.3-2:1.15.7-26.2.1-2'] + - ['emqx', 'master', '5.3-4:1.15.7-26.2.1-2'] + - ['emqx-enterprise', 'release-56', '5.3-4:1.15.7-26.2.1-2'] os: - debian10 - ubuntu22.04 diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 44e94c35a..c36a6ebd3 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -27,11 +27,11 @@ on: builder: required: false type: string - default: 'ghcr.io/emqx/emqx-builder/5.3-2:1.15.7-26.2.1-2-ubuntu22.04' + default: 'ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-ubuntu22.04' builder_vsn: required: false type: string - default: '5.3-2' + default: '5.3-4' otp_vsn: required: false type: string diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml index 7b9c14d5f..5c2a9a4d3 100644 --- a/.github/workflows/codeql.yaml +++ b/.github/workflows/codeql.yaml @@ -17,7 +17,7 @@ jobs: actions: read security-events: write container: - image: ghcr.io/emqx/emqx-builder/5.3-2:1.15.7-26.2.1-2-ubuntu22.04 + image: ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-ubuntu22.04 strategy: fail-fast: false diff --git a/.github/workflows/performance_test.yaml b/.github/workflows/performance_test.yaml index 16874163c..fc524a5f9 100644 --- a/.github/workflows/performance_test.yaml +++ b/.github/workflows/performance_test.yaml @@ -26,7 +26,7 @@ jobs: prepare: runs-on: ubuntu-latest if: github.repository_owner == 'emqx' - container: ghcr.io/emqx/emqx-builder/5.3-2:1.15.7-26.2.1-2-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-ubuntu20.04 outputs: BENCH_ID: ${{ steps.prepare.outputs.BENCH_ID }} PACKAGE_FILE: ${{ steps.package_file.outputs.PACKAGE_FILE }} diff --git a/Makefile b/Makefile index 2f01ad16e..83d041260 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ REBAR = $(CURDIR)/rebar3 BUILD = $(CURDIR)/build SCRIPTS = $(CURDIR)/scripts export EMQX_RELUP ?= true -export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.3-2:1.15.7-26.2.1-2-debian12 +export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-debian12 export EMQX_DEFAULT_RUNNER = public.ecr.aws/debian/debian:12-slim export EMQX_REL_FORM ?= tgz export QUICER_DOWNLOAD_FROM_RELEASE = 1 @@ -20,8 +20,8 @@ endif # Dashboard version # from https://github.com/emqx/emqx-dashboard5 -export EMQX_DASHBOARD_VERSION ?= v1.8.1 -export EMQX_EE_DASHBOARD_VERSION ?= e1.6.1 +export EMQX_DASHBOARD_VERSION ?= v1.9.0-beta.1 +export EMQX_EE_DASHBOARD_VERSION ?= e1.7.0-beta.1 PROFILE ?= emqx REL_PROFILES := emqx emqx-enterprise diff --git a/README-CN.md b/README-CN.md index c2c5e80f4..c6efab682 100644 --- a/README-CN.md +++ b/README-CN.md @@ -86,7 +86,7 @@ EMQX Cloud 文档：[docs.emqx.com/zh/cloud/latest/](https://docs.emqx.com/zh/cl `master` 分支是最新的 5 版本，`main-v4.4` 是 4.4 版本。 -EMQX 4.4 版本需要 OTP 24；5 版本则可以使用 OTP 24 和 25 构建。 +EMQX 4.4 版本需要 OTP 24；5 版本则可以使用 OTP 25 和 26 构建。 ```bash git clone https://github.com/emqx/emqx.git diff --git a/README.md b/README.md index ad710b5e6..98f78c297 100644 --- a/README.md +++ b/README.md @@ -98,7 +98,7 @@ The `master` branch tracks the latest version 5. For version 4.4 checkout the `m EMQX 4.4 requires OTP 24. EMQX 5.0 ~ 5.3 can be built with OTP 24 or 25. -EMQX 5.4 and newer can be built with OTP 24 or 25. +EMQX 5.4 and newer can be built with OTP 25 or 26. ```bash git clone https://github.com/emqx/emqx.git diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index 270acf07a..18e059422 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -32,7 +32,7 @@ %% `apps/emqx/src/bpapi/README.md' %% Opensource edition --define(EMQX_RELEASE_CE, "5.6.1"). +-define(EMQX_RELEASE_CE, "5.7.0-alpha.1"). %% Enterprise edition --define(EMQX_RELEASE_EE, "5.6.1"). +-define(EMQX_RELEASE_EE, "5.7.0-alpha.1"). diff --git a/apps/emqx/include/emqx_trace.hrl b/apps/emqx/include/emqx_trace.hrl index d1e70b184..27dd8b6c8 100644 --- a/apps/emqx/include/emqx_trace.hrl +++ b/apps/emqx/include/emqx_trace.hrl @@ -38,4 +38,10 @@ -define(SHARD, ?COMMON_SHARD). -define(MAX_SIZE, 30). +-define(EMQX_TRACE_STOP_ACTION(REASON), + {unrecoverable_error, {action_stopped_after_template_rendering, REASON}} +). + +-define(EMQX_TRACE_STOP_ACTION_MATCH, ?EMQX_TRACE_STOP_ACTION(_)). + -endif. diff --git a/apps/emqx/priv/bpapi.versions b/apps/emqx/priv/bpapi.versions index a2913c37a..10d27fc63 100644 --- a/apps/emqx/priv/bpapi.versions +++ b/apps/emqx/priv/bpapi.versions @@ -46,6 +46,7 @@ {emqx_metrics,2}. {emqx_mgmt_api_plugins,1}. {emqx_mgmt_api_plugins,2}. +{emqx_mgmt_api_plugins,3}. {emqx_mgmt_cluster,1}. {emqx_mgmt_cluster,2}. {emqx_mgmt_cluster,3}. diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index b58734cf8..b58cd0cb7 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -30,7 +30,7 @@ {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.11.2"}}}, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.19.3"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.3.1"}}}, - {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.42.1"}}}, + {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.42.2"}}}, {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.3"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}, diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index efb5133bc..05358f889 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -1636,8 +1636,17 @@ maybe_set_client_initial_attrs(ConnPkt, #{zone := Zone} = ClientInfo) -> initialize_client_attrs(Inits, ClientInfo) -> lists:foldl( fun(#{expression := Variform, set_as_attr := Name}, Acc) -> - Attrs = maps:get(client_attrs, ClientInfo, #{}), + Attrs = maps:get(client_attrs, Acc, #{}), case emqx_variform:render(Variform, ClientInfo) of + {ok, <<>>} -> + ?SLOG( + debug, + #{ + msg => "client_attr_rednered_to_empty_string", + set_as_attr => Name + } + ), + Acc; {ok, Value} -> ?SLOG( debug, diff --git a/apps/emqx/src/emqx_logger_jsonfmt.erl b/apps/emqx/src/emqx_logger_jsonfmt.erl index 2efb0e032..92c0bb561 100644 --- a/apps/emqx/src/emqx_logger_jsonfmt.erl +++ b/apps/emqx/src/emqx_logger_jsonfmt.erl @@ -32,7 +32,7 @@ -export([format/2]). %% For CLI HTTP API outputs --export([best_effort_json/1, best_effort_json/2]). +-export([best_effort_json/1, best_effort_json/2, best_effort_json_obj/1]). -ifdef(TEST). -include_lib("proper/include/proper.hrl"). @@ -65,10 +65,13 @@ best_effort_json(Input) -> best_effort_json(Input, [pretty, force_utf8]). best_effort_json(Input, Opts) -> - Config = #{depth => unlimited, single_line => true, chars_limit => unlimited}, - JsonReady = best_effort_json_obj(Input, Config), + JsonReady = best_effort_json_obj(Input), emqx_utils_json:encode(JsonReady, Opts). +best_effort_json_obj(Input) -> + Config = #{depth => unlimited, single_line => true, chars_limit => unlimited}, + best_effort_json_obj(Input, Config). + -spec format(logger:log_event(), config()) -> iodata(). format(#{level := Level, msg := Msg, meta := Meta}, Config0) when is_map(Config0) -> Config = add_default_config(Config0), diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 02e31387e..b4ba08bbc 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -217,14 +217,7 @@ roots(high) -> importance => ?IMPORTANCE_MEDIUM } )}, - {zones, - sc( - map(name, ref("zone")), - #{ - desc => ?DESC(zones), - importance => ?IMPORTANCE_HIDDEN - } - )} + {zones, zones_field_schema()} ] ++ emqx_schema_hooks:injection_point( 'roots.high', @@ -1859,7 +1852,7 @@ base_listener(Bind) -> #{ desc => ?DESC(base_listener_zone), default => 'default', - importance => ?IMPORTANCE_HIDDEN + importance => ?IMPORTANCE_LOW } )}, {"limiter", @@ -1883,6 +1876,22 @@ base_listener(Bind) -> )} ] ++ emqx_limiter_schema:short_paths_fields(). +%% @hidden Starting from 5.7, listeners.{TYPE}.{NAME}.zone is no longer hidden +%% However, the root key 'zones' is still hidden because the fields' schema +%% just repeat other root field's schema, which makes the dumped schema doc +%% unnecessarily bloated. +%% +%% zone schema is documented here since 5.7: +%% https://docs.emqx.com/en/enterprise/latest/configuration/configuration.html +zones_field_schema() -> + sc( + map(name, ref("zone")), + #{ + desc => ?DESC(zones), + importance => ?IMPORTANCE_HIDDEN + } + ). + desc("persistent_session_store") -> "Settings for message persistence."; desc("persistent_session_builtin") -> @@ -3712,7 +3721,7 @@ default_mem_check_interval() -> description_schema() -> sc( - string(), + binary(), #{ default => <<"">>, desc => ?DESC(description), diff --git a/apps/emqx/src/emqx_trace/emqx_trace.erl b/apps/emqx/src/emqx_trace/emqx_trace.erl index 7bbe59b2b..91de65b39 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace.erl @@ -29,7 +29,9 @@ unsubscribe/2, log/3, log/4, - rendered_action_template/2 + rendered_action_template/2, + make_rendered_action_template_trace_context/1, + rendered_action_template_with_ctx/2 ]). -export([ @@ -70,6 +72,12 @@ -export_type([ruleid/0]). -type ruleid() :: binary(). +-export_type([rendered_action_template_ctx/0]). +-opaque rendered_action_template_ctx() :: #{ + trace_ctx := map(), + action_id := any() +}. + publish(#message{topic = <<"$SYS/", _/binary>>}) -> ignore; publish(#message{from = From, topic = Topic, payload = Payload}) when @@ -87,7 +95,7 @@ unsubscribe(<<"$SYS/", _/binary>>, _SubOpts) -> unsubscribe(Topic, SubOpts) -> ?TRACE("UNSUBSCRIBE", "unsubscribe", #{topic => Topic, sub_opts => SubOpts}). -rendered_action_template(ActionID, RenderResult) -> +rendered_action_template(<<"action:", _/binary>> = ActionID, RenderResult) -> TraceResult = ?TRACE( "QUERY_RENDER", "action_template_rendered", @@ -107,11 +115,55 @@ rendered_action_template(ActionID, RenderResult) -> ) ), MsgBin = unicode:characters_to_binary(StopMsg), - error({unrecoverable_error, {action_stopped_after_template_rendering, MsgBin}}); + error(?EMQX_TRACE_STOP_ACTION(MsgBin)); _ -> ok end, - TraceResult. + TraceResult; +rendered_action_template(_ActionID, _RenderResult) -> + %% We do nothing if we don't get a valid Action ID. This can happen when + %% called from connectors that are used for actions as well as authz and + %% authn. + ok. + +%% The following two functions are used for connectors that don't do the +%% rendering in the main process (the one that called on_*query). In this case +%% we need to pass the trace context to the sub process that do the rendering +%% so that the result of the rendering can be traced correctly. It is also +%% important to ensure that the error that can be thrown from +%% rendered_action_template_with_ctx is handled in the appropriate way in the +%% sub process. +-spec make_rendered_action_template_trace_context(any()) -> rendered_action_template_ctx(). +make_rendered_action_template_trace_context(ActionID) -> + MetaData = + case logger:get_process_metadata() of + undefined -> #{}; + M -> M + end, + #{trace_ctx => MetaData, action_id => ActionID}. + +-spec rendered_action_template_with_ctx(rendered_action_template_ctx(), Result :: term()) -> term(). +rendered_action_template_with_ctx( + #{ + trace_ctx := LogMetaData, + action_id := ActionID + }, + RenderResult +) -> + OldMetaData = + case logger:get_process_metadata() of + undefined -> #{}; + M -> M + end, + try + logger:set_process_metadata(LogMetaData), + emqx_trace:rendered_action_template( + ActionID, + RenderResult + ) + after + logger:set_process_metadata(OldMetaData) + end. log(List, Msg, Meta) -> log(debug, List, Msg, Meta). diff --git a/apps/emqx/src/emqx_trace/emqx_trace_json_formatter.erl b/apps/emqx/src/emqx_trace/emqx_trace_json_formatter.erl index 35b09b9b0..8f748ed9f 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace_json_formatter.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace_json_formatter.erl @@ -48,6 +48,21 @@ prepare_log_map(LogMap, PEncode) -> NewKeyValuePairs = [prepare_key_value(K, V, PEncode) || {K, V} <- maps:to_list(LogMap)], maps:from_list(NewKeyValuePairs). +prepare_key_value(K, {Formatter, V}, PEncode) when is_function(Formatter, 1) -> + %% A cusom formatter is provided with the value + try + NewV = Formatter(V), + prepare_key_value(K, NewV, PEncode) + catch + _:_ -> + {K, V} + end; +prepare_key_value(K, {ok, Status, Headers, Body}, PEncode) when + is_integer(Status), is_list(Headers), is_binary(Body) +-> + %% This is unlikely anything else then info about a HTTP request so we make + %% it more structured + prepare_key_value(K, #{status => Status, headers => Headers, body => Body}, PEncode); prepare_key_value(payload = K, V, PEncode) -> NewV = try diff --git a/apps/emqx/test/emqx_client_SUITE.erl b/apps/emqx/test/emqx_client_SUITE.erl index f0afe6195..d47162ed7 100644 --- a/apps/emqx/test/emqx_client_SUITE.erl +++ b/apps/emqx/test/emqx_client_SUITE.erl @@ -422,6 +422,10 @@ t_client_attr_from_user_property(_Config) -> #{ expression => Compiled, set_as_attr => <<"group">> + }, + #{ + expression => Compiled, + set_as_attr => <<"group2">> } ]), SslConf = emqx_common_test_helpers:client_mtls('tlsv1.3'), @@ -436,7 +440,7 @@ t_client_attr_from_user_property(_Config) -> {ok, _} = emqtt:connect(Client), %% assert only two chars are extracted ?assertMatch( - #{clientinfo := #{client_attrs := #{<<"group">> := <<"g1">>}}}, + #{clientinfo := #{client_attrs := #{<<"group">> := <<"g1">>, <<"group2">> := <<"g1">>}}}, emqx_cm:get_chan_info(ClientId) ), emqtt:disconnect(Client). diff --git a/apps/emqx_auth/etc/acl.conf b/apps/emqx_auth/etc/acl.conf index dbeec6852..3cc0ed5b8 100644 --- a/apps/emqx_auth/etc/acl.conf +++ b/apps/emqx_auth/etc/acl.conf @@ -1,27 +1,4 @@ -%%-------------------------------------------------------------------- -%% -type(ipaddr() :: {ipaddr, string()}). -%% -%% -type(ipaddrs() :: {ipaddrs, [string()]}). -%% -%% -type(username() :: {user | username, string()} | {user | username, {re, regex()}}). -%% -%% -type(clientid() :: {client | clientid, string()} | {client | clientid, {re, regex()}}). -%% -%% -type(who() :: ipaddr() | ipaddrs() | username() | clientid() | -%% {'and', [ipaddr() | ipaddrs() | username() | clientid()]} | -%% {'or', [ipaddr() | ipaddrs() | username() | clientid()]} | -%% all). -%% -%% -type(action() :: subscribe | publish | all). -%% -%% -type(topic_filters() :: string()). -%% -%% -type(topics() :: [topic_filters() | {eq, topic_filters()}]). -%% -%% -type(permission() :: allow | deny). -%% -%% -type(rule() :: {permission(), who(), action(), topics()} | {permission(), all}). -%%-------------------------------------------------------------------- +%%-------------- Default ACL rules ------------------------------------------------------- {allow, {username, {re, "^dashboard$"}}, subscribe, ["$SYS/#"]}. @@ -30,3 +7,117 @@ {deny, all, subscribe, ["$SYS/#", {eq, "#"}]}. {allow, all}. +%% NOTE! when deploy in production: +%% - Change the last rule to `{deny, all}.` +%% - Set config `authorization.no_match = deny` + +%% See docs below +%% +%% ------------ The formal spec ---------------------------------------------------------- +%% +%% -type ipaddr() :: {ipaddr, string()}. +%% -type ipaddrs() :: {ipaddrs, [string()]}. +%% -type username() :: {user | username, string()} | {user | username, {re, regex()}}. +%% -type clientid() :: {client | clientid, string()} | {client | clientid, {re, regex()}}. +%% -type who() :: ipaddr() | ipaddrs() | username() | clientid() | +%% {'and', [ipaddr() | ipaddrs() | username() | clientid()]} | +%% {'or', [ipaddr() | ipaddrs() | username() | clientid()]} | +%% all. +%% -type simple_action() :: subscribe | publish | all. +%% -type complex_action() :: {simple_action(), [{qos, 0..2}, {retain, true|false|all}]}. +%% -type action() :: simple_action() | complex_action(). +%% -type topic() :: string(). +%% -type topic_filter() :: string(). +%% -type topic_match() :: topic() | topic_filter() | {eq, topic() | topic_filter()}. +%% -type perm() :: allow | deny. +%% -type rule() :: {perm(), who(), action(), [topic_match()]} | {perm(), all}. + +%%-------------- Viusal aid for the spec ------------------------------------------------- +%% +%% rule() +%% ├── {perm(), who(), action(), [topic_match()]} +%% │ │ │ │ ├── topic() :: string() +%% │ │ │ │ ├── topic_filter() :: string() +%% │ │ │ │ └── {eq, topic() | topic_filter()} +%% │ │ │ │ +%% │ │ │ ├── simple_action() +%% │ │ │ │ ├── publish +%% │ │ │ │ ├── subscribe +%% │ │ │ │ └── all +%% │ │ │ └── {simple_action(), [{qos,0..2},{retain,true|false|all}]} +%% │ │ │ +%% │ │ ├── ipaddr() +%% │ │ │ └── {ipaddr, string()} +%% │ │ ├── ipaddrs() +%% │ │ │ └── {ipaddrs, [string()]} +%% │ │ ├── username() +%% │ │ │ ├── {user | username, string()} +%% │ │ │ └── {user | username, {re, regex()}} +%% │ │ ├── clientid() +%% │ │ │ ├── {client | clientid, string()} +%% │ │ │ └── {client | clientid, {re, regex()}} +%% │ │ ├── {'and', [ipaddr() | ipaddrs() | username() | clientid()]} +%% │ │ ├── {'or', [ipaddr() | ipaddrs() | username() | clientid()]} +%% │ │ └── all +%% │ │ +%% │ ├── allow +%% │ └── deny +%% │ +%% └── {perm(), all} +%% + +%% This file defines a set of ACL rules for MQTT client pub/sub authorization. +%% The content is of Erlang-term format. +%% Each Erlang-term is a tuple `{...}` terminated by dot `.` +%% +%% NOTE: When deploy to production, the last rule should be changed to {deny, all}. +%% +%% NOTE: It's a good practice to keep the nubmer of rules small, because in worst case +%% scenarios, all rules have to be traversed for each message publish. +%% +%% A rule is a 4-element tuple. +%% For example, `{allow, {username, "Jon"}, subscribe, ["#"]}` allows Jon to subscribe to +%% any topic they want. +%% +%% Below is an explanation: +%% +%% - `perm()`: The permission. +%% Defines whether this is an `allow` or `deny` rule. +%% +%% - `who()`: The MQTT client matching condition. +%% - `all`: A rule which applies to all clients. +%% - `{ipaddr, IpAddress}`: Matches a client by source IP address. CIDR notation is allowed. +%% - `{ipaddrs, [IpAddress]}`: Matches clients by a set of IP addresses. CIDR notation is allowed. +%% - `{clientid, ClientID}`: Matches a client by ID. +%% - `{username, Username}`: Matches a client by username. +%% - `{..., {re, ..}}`: Regular expression to match either clientid or username. +%% - `{'and', [...]}`: Combines a list of matching conditions. +%% - `{'or', [...]}`: Combines a list of matching conditions. +%% +%% - `action()`: Matches publish or subscribe actions (or both). +%% Applies the rule to `publish` or `subscribe` actions. +%% The special value `all` denotes allowing or denying both `publish` and `subscribe`. +%% It can also be associated with `qos` and `retain` flags to match the action with +%% more specifics. For example, `{publish, [{qos,0},{retain,false}]}` should only +%% match the `publish` action when the message has QoS 0, and without retained flag set. +%% +%% - `[topic_match()]`: +%% A list of topics, topic-filters, or template rendering to match the topic being +%% subscribed to or published. +%% For example, `{allow, {username, "Jan"}, publish, ["jan/#"]}` permits Jan to publish +%% to any topic matching the wildcard pattern "jan/#". +%% A special tuple `{eq, topic_match()}` is useful to allow or deny the specific wildcard +%% subscription instead of performing a topic match. +%% A `topic_match()` can also contain a placeholder rendered with actual value at runtime, +%% for example, `{allow, all, publish, "${clientid}/#"}` allows all clients to publish to +%% topics prefixed by their own client ID. +%% +%% Supported placeholders are: +%% - `${cn}`: TLS certificate common name. +%% - `${clientid}`: The client ID. +%% - `${username}`: The username. +%% - `${client_attrs.NAME}`: A client attribute named `NAME`, which can be initialized by +%% `mqtt.client_attrs_init` config or extended by certain authentication backends. +%% NOTE: Placeholder is not rendered as empty string if the referencing value is not +%% foud. For example, `${client_attrs.group}/#` is not rendered as `/#` if the +%% client does not have a `group` attribute. diff --git a/apps/emqx_auth/src/emqx_authz/emqx_authz_rule.erl b/apps/emqx_auth/src/emqx_authz/emqx_authz_rule.erl index 2fb6fb8f5..aa86a7c1e 100644 --- a/apps/emqx_auth/src/emqx_authz/emqx_authz_rule.erl +++ b/apps/emqx_auth/src/emqx_authz/emqx_authz_rule.erl @@ -386,8 +386,9 @@ match_who(_, _) -> match_topics(_ClientInfo, _Topic, []) -> false; match_topics(ClientInfo, Topic, [{pattern, PatternFilter} | Filters]) -> - TopicFilter = bin(emqx_template:render_strict(PatternFilter, ClientInfo)), - match_topic(emqx_topic:words(Topic), emqx_topic:words(TopicFilter)) orelse + TopicFilter = render_topic(PatternFilter, ClientInfo), + (is_binary(TopicFilter) andalso + match_topic(emqx_topic:words(Topic), emqx_topic:words(TopicFilter))) orelse match_topics(ClientInfo, Topic, Filters); match_topics(ClientInfo, Topic, [TopicFilter | Filters]) -> match_topic(emqx_topic:words(Topic), TopicFilter) orelse @@ -397,3 +398,16 @@ match_topic(Topic, {'eq', TopicFilter}) -> Topic =:= TopicFilter; match_topic(Topic, TopicFilter) -> emqx_topic:match(Topic, TopicFilter). + +render_topic(Topic, ClientInfo) -> + try + bin(emqx_template:render_strict(Topic, ClientInfo)) + catch + error:Reason -> + ?SLOG(debug, #{ + msg => "failed_to_render_topic_template", + template => Topic, + reason => Reason + }), + error + end. diff --git a/apps/emqx_auth/test/emqx_authz/emqx_authz_SUITE.erl b/apps/emqx_auth/test/emqx_authz/emqx_authz_SUITE.erl index 70dd0bbb6..575eb4109 100644 --- a/apps/emqx_auth/test/emqx_authz/emqx_authz_SUITE.erl +++ b/apps/emqx_auth/test/emqx_authz/emqx_authz_SUITE.erl @@ -173,7 +173,16 @@ end_per_testcase(_TestCase, _Config) -> -define(SOURCE_FILE_CLIENT_ATTR, ?SOURCE_FILE( << - "{allow,all,all,[\"${client_attrs.alias}/#\"]}.\n" + "{allow,all,all,[\"${client_attrs.alias}/#\",\"client_attrs_backup\"]}.\n" + "{deny, all}." + >> + ) +). + +-define(SOURCE_FILE_CLIENT_NO_SUCH_ATTR, + ?SOURCE_FILE( + << + "{allow,all,all,[\"${client_attrs.nonexist}/#\",\"client_attrs_backup\"]}.\n" "{deny, all}." >> ) @@ -572,11 +581,41 @@ t_alias_prefix(_Config) -> ?assertMatch({ok, _}, emqtt:connect(C)), ?assertMatch({ok, _, [?RC_SUCCESS]}, emqtt:subscribe(C, SubTopic)), ?assertMatch({ok, _, [?RC_NOT_AUTHORIZED]}, emqtt:subscribe(C, SubTopicNotAllowed)), + ?assertMatch({ok, _, [?RC_NOT_AUTHORIZED]}, emqtt:subscribe(C, <<"/#">>)), unlink(C), emqtt:stop(C), + NonMatching = <<"clientid_which_has_no_dash">>, + {ok, C2} = emqtt:start_link([{clientid, NonMatching}, {proto_ver, v5}]), + ?assertMatch({ok, _}, emqtt:connect(C2)), + ?assertMatch({ok, _, [?RC_SUCCESS]}, emqtt:subscribe(C2, <<"client_attrs_backup">>)), + %% assert '${client_attrs.alias}/#' is not rendered as '/#' + ?assertMatch({ok, _, [?RC_NOT_AUTHORIZED]}, emqtt:subscribe(C2, <<"/#">>)), + unlink(C2), + emqtt:stop(C2), emqx_config:put_zone_conf(default, [mqtt, client_attrs_init], []), ok. +t_non_existing_attr(_Config) -> + {ok, _} = emqx_authz:update(?CMD_REPLACE, [?SOURCE_FILE_CLIENT_NO_SUCH_ATTR]), + %% '^.*-(.*)$': extract the suffix after the last '-' + {ok, Compiled} = emqx_variform:compile("concat(regex_extract(clientid,'^.*-(.*)$'))"), + emqx_config:put_zone_conf(default, [mqtt, client_attrs_init], [ + #{ + expression => Compiled, + %% this is intended to be different from 'nonexist' + set_as_attr => <<"existing">> + } + ]), + ClientId = <<"org1-name3">>, + {ok, C} = emqtt:start_link([{clientid, ClientId}, {proto_ver, v5}]), + ?assertMatch({ok, _}, emqtt:connect(C)), + ?assertMatch({ok, _, [?RC_SUCCESS]}, emqtt:subscribe(C, <<"client_attrs_backup">>)), + %% assert '${client_attrs.nonexist}/#' is not rendered as '/#' + ?assertMatch({ok, _, [?RC_NOT_AUTHORIZED]}, emqtt:subscribe(C, <<"/#">>)), + unlink(C), + emqtt:stop(C), + ok. + %% client is allowed by ACL to publish to its LWT topic, is connected, %% and then gets banned and kicked out while connected. Should not %% publish LWT. diff --git a/apps/emqx_bridge/src/emqx_action_info.erl b/apps/emqx_bridge/src/emqx_action_info.erl index a8aaf9fdd..dc71d5806 100644 --- a/apps/emqx_bridge/src/emqx_action_info.erl +++ b/apps/emqx_bridge/src/emqx_action_info.erl @@ -118,7 +118,8 @@ hard_coded_action_info_modules_ee() -> emqx_bridge_pulsar_action_info, emqx_bridge_greptimedb_action_info, emqx_bridge_tdengine_action_info, - emqx_bridge_s3_action_info + emqx_bridge_s3_upload_action_info, + emqx_bridge_s3_aggreg_upload_action_info ]. -else. hard_coded_action_info_modules_ee() -> diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl b/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl index a3316e39d..b057a648e 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl @@ -173,6 +173,11 @@ source_hookpoint(Config) -> BridgeId = emqx_bridge_resource:bridge_id(Type, Name), emqx_bridge_v2:source_hookpoint(BridgeId). +action_hookpoint(Config) -> + #{kind := action, type := Type, name := Name} = get_common_values(Config), + BridgeId = emqx_bridge_resource:bridge_id(Type, Name), + emqx_bridge_resource:bridge_hookpoint(BridgeId). + add_source_hookpoint(Config) -> Hookpoint = source_hookpoint(Config), ok = emqx_hooks:add(Hookpoint, {?MODULE, source_hookpoint_callback, [self()]}, 1000), @@ -378,6 +383,14 @@ start_connector_api(ConnectorName, ConnectorType) -> ct:pal("connector update (http) result:\n ~p", [Res]), Res. +get_connector_api(ConnectorType, ConnectorName) -> + ConnectorId = emqx_connector_resource:connector_id(ConnectorType, ConnectorName), + Path = emqx_mgmt_api_test_util:api_path(["connectors", ConnectorId]), + ct:pal("get connector ~s (http)", [ConnectorId]), + Res = request(get, Path, _Params = []), + ct:pal("get connector (http) result:\n ~p", [Res]), + Res. + create_action_api(Config) -> create_action_api(Config, _Overrides = #{}). diff --git a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl index eb12cbaae..ef79f78fe 100644 --- a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl +++ b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl @@ -223,6 +223,11 @@ do_single_query(InstId, Request, Async, #{pool_name := PoolName} = State) -> } ), {PreparedKeyOrCQL1, Data} = proc_cql_params(Type, PreparedKeyOrCQL, Params, State), + emqx_trace:rendered_action_template(PreparedKeyOrCQL, #{ + type => Type, + key_or_cql => PreparedKeyOrCQL1, + data => Data + }), Res = exec_cql_query(InstId, PoolName, Type, Async, PreparedKeyOrCQL1, Data), handle_result(Res). @@ -261,6 +266,14 @@ do_batch_query(InstId, Requests, Async, #{pool_name := PoolName} = State) -> state => State } ), + ChannelID = + case Requests of + [{CID, _} | _] -> CID; + _ -> none + end, + emqx_trace:rendered_action_template(ChannelID, #{ + cqls => CQLs + }), Res = exec_cql_batch_query(InstId, PoolName, Async, CQLs), handle_result(Res). diff --git a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl index 942f7590b..2c824aa95 100644 --- a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl +++ b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl @@ -386,7 +386,7 @@ on_query( SimplifiedRequestType = query_type(RequestType), Templates = get_templates(RequestType, State), SQL = get_sql(SimplifiedRequestType, Templates, DataOrSQL), - ClickhouseResult = execute_sql_in_clickhouse_server(PoolName, SQL), + ClickhouseResult = execute_sql_in_clickhouse_server(RequestType, PoolName, SQL), transform_and_log_clickhouse_result(ClickhouseResult, ResourceID, SQL). get_templates(ChannId, State) -> @@ -398,7 +398,7 @@ get_templates(ChannId, State) -> end. get_sql(channel_message, #{send_message_template := PreparedSQL}, Data) -> - emqx_placeholder:proc_tmpl(PreparedSQL, Data); + emqx_placeholder:proc_tmpl(PreparedSQL, Data, #{return => full_binary}); get_sql(_, _, SQL) -> SQL. @@ -425,7 +425,7 @@ on_batch_query(ResourceID, BatchReq, #{pool_name := PoolName} = State) -> %% Create batch insert SQL statement SQL = objects_to_sql(ObjectsToInsert, Templates), %% Do the actual query in the database - ResultFromClickhouse = execute_sql_in_clickhouse_server(PoolName, SQL), + ResultFromClickhouse = execute_sql_in_clickhouse_server(ChannId, PoolName, SQL), %% Transform the result to a better format transform_and_log_clickhouse_result(ResultFromClickhouse, ResourceID, SQL). @@ -464,7 +464,8 @@ objects_to_sql(_, _) -> %% This function is used by on_query/3 and on_batch_query/3 to send a query to %% the database server and receive a result -execute_sql_in_clickhouse_server(PoolName, SQL) -> +execute_sql_in_clickhouse_server(Id, PoolName, SQL) -> + emqx_trace:rendered_action_template(Id, #{rendered_sql => SQL}), ecpool:pick_and_do( PoolName, {?MODULE, execute_sql_in_clickhouse_server_using_connection, [SQL]}, diff --git a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl index 372472dda..f89786929 100644 --- a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl +++ b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl @@ -9,6 +9,7 @@ -include_lib("emqx_resource/include/emqx_resource.hrl"). -include_lib("typerefl/include/types.hrl"). -include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/emqx_trace.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -include_lib("hocon/include/hoconsc.hrl"). @@ -133,20 +134,10 @@ on_add_channel( create_channel_state( #{parameters := Conf} = _ChannelConfig ) -> - Keys = maps:with([hash_key, range_key], Conf), - Keys1 = maps:fold( - fun(K, V, Acc) -> - Acc#{K := erlang:binary_to_existing_atom(V)} - end, - Keys, - Keys - ), - - Base = maps:without([template, hash_key, range_key], Conf), - Base1 = maps:merge(Base, Keys1), + Base = maps:without([template], Conf), Templates = parse_template_from_conf(Conf), - State = Base1#{ + State = Base#{ templates => Templates }, {ok, State}. @@ -246,12 +237,16 @@ do_query( table := Table, templates := Templates } = ChannelState, + TraceRenderedCTX = + emqx_trace:make_rendered_action_template_trace_context(ChannelId), Result = case ensuare_dynamo_keys(Query, ChannelState) of true -> ecpool:pick_and_do( PoolName, - {emqx_bridge_dynamo_connector_client, query, [Table, QueryTuple, Templates]}, + {emqx_bridge_dynamo_connector_client, query, [ + Table, QueryTuple, Templates, TraceRenderedCTX + ]}, no_handover ); _ -> @@ -259,6 +254,8 @@ do_query( end, case Result of + {error, ?EMQX_TRACE_STOP_ACTION(_)} = Error -> + Error; {error, Reason} -> ?tp( dynamo_connector_query_return, @@ -311,12 +308,12 @@ get_query_tuple([InsertQuery | _]) -> ensuare_dynamo_keys({_, Data} = Query, State) when is_map(Data) -> ensuare_dynamo_keys([Query], State); ensuare_dynamo_keys([{_, Data} | _] = Queries, State) when is_map(Data) -> - Keys = maps:to_list(maps:with([hash_key, range_key], State)), + Keys = maps:values(maps:with([hash_key, range_key], State)), lists:all( fun({_, Query}) -> lists:all( - fun({_, Key}) -> - maps:is_key(Key, Query) + fun(Key) -> + is_dynamo_key_existing(Key, Query) end, Keys ) @@ -364,3 +361,17 @@ get_host_info(Server) -> redact(Data) -> emqx_utils:redact(Data, fun(Any) -> Any =:= aws_secret_access_key end). + +is_dynamo_key_existing(Bin, Query) when is_binary(Bin) -> + case maps:is_key(Bin, Query) of + true -> + true; + _ -> + try + Key = erlang:binary_to_existing_atom(Bin), + maps:is_key(Key, Query) + catch + _:_ -> + false + end + end. diff --git a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector_client.erl b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector_client.erl index 4f924ef67..f257ae389 100644 --- a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector_client.erl +++ b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector_client.erl @@ -10,7 +10,7 @@ -export([ start_link/1, is_connected/2, - query/4 + query/5 ]). %% gen_server callbacks @@ -40,8 +40,8 @@ is_connected(Pid, Timeout) -> {false, Error} end. -query(Pid, Table, Query, Templates) -> - gen_server:call(Pid, {query, Table, Query, Templates}, infinity). +query(Pid, Table, Query, Templates, TraceRenderedCTX) -> + gen_server:call(Pid, {query, Table, Query, Templates, TraceRenderedCTX}, infinity). %%-------------------------------------------------------------------- %% @doc @@ -77,14 +77,14 @@ handle_call(is_connected, _From, State) -> {false, Error} end, {reply, IsConnected, State}; -handle_call({query, Table, Query, Templates}, _From, State) -> - Result = do_query(Table, Query, Templates), +handle_call({query, Table, Query, Templates, TraceRenderedCTX}, _From, State) -> + Result = do_query(Table, Query, Templates, TraceRenderedCTX), {reply, Result, State}; handle_call(_Request, _From, State) -> {reply, ok, State}. handle_cast({query, Table, Query, Templates, {ReplyFun, [Context]}}, State) -> - Result = do_query(Table, Query, Templates), + Result = do_query(Table, Query, Templates, {fun(_, _) -> ok end, none}), ReplyFun(Context, Result), {noreply, State}; handle_cast(_Request, State) -> @@ -102,15 +102,29 @@ code_change(_OldVsn, State, _Extra) -> %%%=================================================================== %%% Internal functions %%%=================================================================== -do_query(Table, Query0, Templates) -> +do_query(Table, Query0, Templates, TraceRenderedCTX) -> try Query = apply_template(Query0, Templates), + emqx_trace:rendered_action_template_with_ctx(TraceRenderedCTX, #{ + table => Table, + query => {fun trace_format_query/1, Query} + }), execute(Query, Table) catch + error:{unrecoverable_error, Reason} -> + {error, {unrecoverable_error, Reason}}; _Type:Reason -> {error, {unrecoverable_error, {invalid_request, Reason}}} end. +trace_format_query({Type, Data}) -> + #{type => Type, data => Data}; +trace_format_query([_ | _] = Batch) -> + BatchData = [trace_format_query(Q) || Q <- Batch], + #{type => batch, data => BatchData}; +trace_format_query(Query) -> + Query. + %% some simple query commands for authn/authz or test execute({insert_item, Msg}, Table) -> Item = convert_to_item(Msg), diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_producer.erl b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_producer.erl index 13040dccf..12d5d1f2f 100644 --- a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_producer.erl +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_producer.erl @@ -284,6 +284,13 @@ do_send_requests_sync(ConnectorState, Requests, InstanceId) -> Method = post, ReqOpts = #{request_ttl => RequestTTL}, Request = {prepared_request, {Method, Path, Body}, ReqOpts}, + emqx_trace:rendered_action_template(MessageTag, #{ + method => Method, + path => Path, + body => Body, + options => ReqOpts, + is_async => false + }), Result = emqx_bridge_gcp_pubsub_client:query_sync(Request, Client), QueryMode = sync, handle_result(Result, Request, QueryMode, InstanceId). @@ -312,6 +319,13 @@ do_send_requests_async(ConnectorState, Requests, ReplyFunAndArgs0) -> ReqOpts = #{request_ttl => RequestTTL}, Request = {prepared_request, {Method, Path, Body}, ReqOpts}, ReplyFunAndArgs = {fun ?MODULE:reply_delegator/2, [ReplyFunAndArgs0]}, + emqx_trace:rendered_action_template(MessageTag, #{ + method => Method, + path => Path, + body => Body, + options => ReqOpts, + is_async => true + }), emqx_bridge_gcp_pubsub_client:query_async( Request, ReplyFunAndArgs, Client ). diff --git a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl index 6bdd8a4cd..97eedf3f6 100644 --- a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl +++ b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl @@ -128,7 +128,7 @@ on_query(InstId, {Channel, Message}, State) -> greptimedb_connector_send_query, #{points => Points, batch => false, mode => sync} ), - do_query(InstId, Client, Points); + do_query(InstId, Channel, Client, Points); {error, ErrorPoints} -> ?tp( greptimedb_connector_send_query_error, @@ -152,7 +152,7 @@ on_batch_query(InstId, [{Channel, _} | _] = BatchData, State) -> greptimedb_connector_send_query, #{points => Points, batch => true, mode => sync} ), - do_query(InstId, Client, Points); + do_query(InstId, Channel, Client, Points); {error, Reason} -> ?tp( greptimedb_connector_send_query_error, @@ -173,7 +173,7 @@ on_query_async(InstId, {Channel, Message}, {ReplyFun, Args}, State) -> greptimedb_connector_send_query, #{points => Points, batch => false, mode => async} ), - do_async_query(InstId, Client, Points, {ReplyFun, Args}); + do_async_query(InstId, Channel, Client, Points, {ReplyFun, Args}); {error, ErrorPoints} = Err -> ?tp( greptimedb_connector_send_query_error, @@ -195,7 +195,7 @@ on_batch_query_async(InstId, [{Channel, _} | _] = BatchData, {ReplyFun, Args}, S greptimedb_connector_send_query, #{points => Points, batch => true, mode => async} ), - do_async_query(InstId, Client, Points, {ReplyFun, Args}); + do_async_query(InstId, Channel, Client, Points, {ReplyFun, Args}); {error, Reason} -> ?tp( greptimedb_connector_send_query_error, @@ -420,7 +420,8 @@ is_auth_key(_) -> %% ------------------------------------------------------------------------------------------------- %% Query -do_query(InstId, Client, Points) -> +do_query(InstId, Channel, Client, Points) -> + emqx_trace:rendered_action_template(Channel, #{points => Points, is_async => false}), case greptimedb:write_batch(Client, Points) of {ok, #{response := {affected_rows, #{value := Rows}}}} -> ?SLOG(debug, #{ @@ -452,12 +453,13 @@ do_query(InstId, Client, Points) -> end end. -do_async_query(InstId, Client, Points, ReplyFunAndArgs) -> +do_async_query(InstId, Channel, Client, Points, ReplyFunAndArgs) -> ?SLOG(info, #{ msg => "greptimedb_write_point_async", connector => InstId, points => Points }), + emqx_trace:rendered_action_template(Channel, #{points => Points, is_async => true}), WrappedReplyFunAndArgs = {fun ?MODULE:reply_callback/2, [ReplyFunAndArgs]}, ok = greptimedb:async_write_batch(Client, Points, WrappedReplyFunAndArgs). diff --git a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl index cf195fc9f..cf53291b2 100644 --- a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl +++ b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl @@ -134,8 +134,11 @@ on_query( #{ producer := Producer, partition_key := PartitionKey, record_template := HRecordTemplate } = maps:get(ChannelID, Channels), - try to_record(PartitionKey, HRecordTemplate, Data) of - Record -> append_record(InstId, Producer, Record, false) + try + KeyAndRawRecord = to_key_and_raw_record(PartitionKey, HRecordTemplate, Data), + emqx_trace:rendered_action_template(ChannelID, #{record => KeyAndRawRecord}), + Record = to_record(KeyAndRawRecord), + append_record(InstId, Producer, Record, false) catch _:_ -> ?FAILED_TO_APPLY_HRECORD_TEMPLATE end. @@ -148,8 +151,13 @@ on_batch_query( #{ producer := Producer, partition_key := PartitionKey, record_template := HRecordTemplate } = maps:get(ChannelID, Channels), - try to_multi_part_records(PartitionKey, HRecordTemplate, BatchList) of - Records -> append_record(InstId, Producer, Records, true) + try + KeyAndRawRecordList = to_multi_part_key_and_partition_key( + PartitionKey, HRecordTemplate, BatchList + ), + emqx_trace:rendered_action_template(ChannelID, #{records => KeyAndRawRecordList}), + Records = [to_record(Item) || Item <- KeyAndRawRecordList], + append_record(InstId, Producer, Records, true) catch _:_ -> ?FAILED_TO_APPLY_HRECORD_TEMPLATE end. @@ -348,20 +356,20 @@ ensure_start_producer(ProducerName, ProducerOptions) -> produce_name(ActionId) -> list_to_binary("backend_hstream_producer:" ++ to_string(ActionId)). -to_record(PartitionKeyTmpl, HRecordTmpl, Data) -> +to_key_and_raw_record(PartitionKeyTmpl, HRecordTmpl, Data) -> PartitionKey = emqx_placeholder:proc_tmpl(PartitionKeyTmpl, Data), RawRecord = emqx_placeholder:proc_tmpl(HRecordTmpl, Data), - to_record(PartitionKey, RawRecord). + #{partition_key => PartitionKey, raw_record => RawRecord}. -to_record(PartitionKey, RawRecord) when is_binary(PartitionKey) -> - to_record(binary_to_list(PartitionKey), RawRecord); -to_record(PartitionKey, RawRecord) -> +to_record(#{partition_key := PartitionKey, raw_record := RawRecord}) when is_binary(PartitionKey) -> + to_record(#{partition_key => binary_to_list(PartitionKey), raw_record => RawRecord}); +to_record(#{partition_key := PartitionKey, raw_record := RawRecord}) -> hstreamdb:to_record(PartitionKey, raw, RawRecord). -to_multi_part_records(PartitionKeyTmpl, HRecordTmpl, BatchList) -> +to_multi_part_key_and_partition_key(PartitionKeyTmpl, HRecordTmpl, BatchList) -> lists:map( fun({_, Data}) -> - to_record(PartitionKeyTmpl, HRecordTmpl, Data) + to_key_and_raw_record(PartitionKeyTmpl, HRecordTmpl, Data) end, BatchList ). diff --git a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl index 9be7457e1..ef9c6c70d 100644 --- a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl +++ b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl @@ -359,7 +359,7 @@ on_query(InstId, {Method, Request, Timeout}, State) -> on_query( InstId, {ActionId, KeyOrNum, Method, Request, Timeout, Retry}, - #{base_path := BasePath} = State + #{base_path := BasePath, host := Host} = State ) -> ?TRACE( "QUERY", @@ -373,7 +373,7 @@ on_query( } ), NRequest = formalize_request(Method, BasePath, Request), - trace_rendered_action_template(ActionId, Method, NRequest, Timeout), + trace_rendered_action_template(ActionId, Host, Method, NRequest, Timeout), Worker = resolve_pool_worker(State, KeyOrNum), Result0 = ehttpc:request( Worker, @@ -469,7 +469,7 @@ on_query_async( InstId, {ActionId, KeyOrNum, Method, Request, Timeout}, ReplyFunAndArgs, - #{base_path := BasePath} = State + #{base_path := BasePath, host := Host} = State ) -> Worker = resolve_pool_worker(State, KeyOrNum), ?TRACE( @@ -483,7 +483,7 @@ on_query_async( } ), NRequest = formalize_request(Method, BasePath, Request), - trace_rendered_action_template(ActionId, Method, NRequest, Timeout), + trace_rendered_action_template(ActionId, Host, Method, NRequest, Timeout), MaxAttempts = maps:get(max_attempts, State, 3), Context = #{ attempt => 1, @@ -503,15 +503,16 @@ on_query_async( ), {ok, Worker}. -trace_rendered_action_template(ActionId, Method, NRequest, Timeout) -> +trace_rendered_action_template(ActionId, Host, Method, NRequest, Timeout) -> case NRequest of {Path, Headers} -> emqx_trace:rendered_action_template( ActionId, #{ + host => Host, path => Path, method => Method, - headers => emqx_utils_redact:redact_headers(Headers), + headers => {fun emqx_utils_redact:redact_headers/1, Headers}, timeout => Timeout } ); @@ -519,15 +520,19 @@ trace_rendered_action_template(ActionId, Method, NRequest, Timeout) -> emqx_trace:rendered_action_template( ActionId, #{ + host => Host, path => Path, method => Method, - headers => emqx_utils_redact:redact_headers(Headers), + headers => {fun emqx_utils_redact:redact_headers/1, Headers}, timeout => Timeout, - body => Body + body => {fun log_format_body/1, Body} } ) end. +log_format_body(Body) -> + unicode:characters_to_binary(Body). + resolve_pool_worker(State, undefined) -> resolve_pool_worker(State, self()); resolve_pool_worker(#{pool_name := PoolName} = State, Key) -> @@ -831,6 +836,8 @@ transform_result(Result) -> Result; {ok, _TooManyRequests = StatusCode = 429, Headers} -> {error, {recoverable_error, #{status_code => StatusCode, headers => Headers}}}; + {ok, _ServiceUnavailable = StatusCode = 503, Headers} -> + {error, {recoverable_error, #{status_code => StatusCode, headers => Headers}}}; {ok, StatusCode, Headers} -> {error, {unrecoverable_error, #{status_code => StatusCode, headers => Headers}}}; {ok, _TooManyRequests = StatusCode = 429, Headers, Body} -> @@ -838,6 +845,11 @@ transform_result(Result) -> {recoverable_error, #{ status_code => StatusCode, headers => Headers, body => Body }}}; + {ok, _ServiceUnavailable = StatusCode = 503, Headers, Body} -> + {error, + {recoverable_error, #{ + status_code => StatusCode, headers => Headers, body => Body + }}}; {ok, StatusCode, Headers, Body} -> {error, {unrecoverable_error, #{ diff --git a/apps/emqx_bridge_http/test/emqx_bridge_http_SUITE.erl b/apps/emqx_bridge_http/test/emqx_bridge_http_SUITE.erl index 9d215d815..7f9418bd3 100644 --- a/apps/emqx_bridge_http/test/emqx_bridge_http_SUITE.erl +++ b/apps/emqx_bridge_http/test/emqx_bridge_http_SUITE.erl @@ -93,6 +93,14 @@ init_per_testcase(t_too_many_requests, Config) -> ), ok = emqx_bridge_http_connector_test_server:set_handler(too_many_requests_http_handler()), [{http_server, #{port => HTTPPort, path => HTTPPath}} | Config]; +init_per_testcase(t_service_unavailable, Config) -> + HTTPPath = <<"/path">>, + ServerSSLOpts = false, + {ok, {HTTPPort, _Pid}} = emqx_bridge_http_connector_test_server:start_link( + _Port = random, HTTPPath, ServerSSLOpts + ), + ok = emqx_bridge_http_connector_test_server:set_handler(service_unavailable_http_handler()), + [{http_server, #{port => HTTPPort, path => HTTPPath}} | Config]; init_per_testcase(t_rule_action_expired, Config) -> [ {bridge_name, ?BRIDGE_NAME} @@ -115,6 +123,7 @@ init_per_testcase(_TestCase, Config) -> end_per_testcase(TestCase, _Config) when TestCase =:= t_path_not_found; TestCase =:= t_too_many_requests; + TestCase =:= t_service_unavailable; TestCase =:= t_rule_action_expired; TestCase =:= t_bridge_probes_header_atoms; TestCase =:= t_send_async_connection_timeout; @@ -260,6 +269,12 @@ not_found_http_handler() -> end. too_many_requests_http_handler() -> + fail_then_success_http_handler(429). + +service_unavailable_http_handler() -> + fail_then_success_http_handler(503). + +fail_then_success_http_handler(FailStatusCode) -> GetAndBump = fun() -> NCalled = persistent_term:get({?MODULE, times_called}, 0), @@ -272,7 +287,7 @@ too_many_requests_http_handler() -> {ok, Body, Req} = cowboy_req:read_body(Req0), TestPid ! {http, cowboy_req:headers(Req), Body}, Rep = - case N >= 2 of + case N >= 3 of true -> cowboy_req:reply( 200, @@ -282,9 +297,13 @@ too_many_requests_http_handler() -> ); false -> cowboy_req:reply( - 429, + FailStatusCode, #{<<"content-type">> => <<"text/plain">>}, - <<"slow down, buddy">>, + %% Body and no body to trigger different code paths + case N of + 1 -> <<"slow down, buddy">>; + _ -> <<>> + end, Req ) end, @@ -570,6 +589,12 @@ t_path_not_found(Config) -> ok. t_too_many_requests(Config) -> + check_send_is_retried(Config). + +t_service_unavailable(Config) -> + check_send_is_retried(Config). + +check_send_is_retried(Config) -> ?check_trace( begin #{port := Port, path := Path} = ?config(http_server, Config), diff --git a/apps/emqx_bridge_http/test/emqx_bridge_http_v2_SUITE.erl b/apps/emqx_bridge_http/test/emqx_bridge_http_v2_SUITE.erl index f9ae0c7e0..3b73abfd9 100644 --- a/apps/emqx_bridge_http/test/emqx_bridge_http_v2_SUITE.erl +++ b/apps/emqx_bridge_http/test/emqx_bridge_http_v2_SUITE.erl @@ -69,10 +69,21 @@ end_per_suite(Config) -> suite() -> [{timetrap, {seconds, 60}}]. +init_per_testcase(t_update_with_sensitive_data, Config) -> + HTTPPath = <<"/foo/bar">>, + ServerSSLOpts = false, + {ok, {HTTPPort, _Pid}} = emqx_bridge_http_connector_test_server:start_link( + _Port = random, HTTPPath, ServerSSLOpts + ), + ok = emqx_bridge_http_connector_test_server:set_handler(success_handler()), + [{path, HTTPPath}, {http_server, #{port => HTTPPort, path => HTTPPath}} | Config]; init_per_testcase(_TestCase, Config) -> Server = start_http_server(#{response_delay_ms => 0}), [{http_server, Server} | Config]. +end_per_testcase(t_update_with_sensitive_data, Config) -> + ok = emqx_bridge_http_connector_test_server:stop(), + end_per_testcase(common, proplists:delete(http_server, Config)); end_per_testcase(_TestCase, Config) -> case ?config(http_server, Config) of undefined -> ok; @@ -112,6 +123,69 @@ t_compose_connector_url_and_action_path(Config) -> ), ok. +%% Checks that we can successfully update a connector containing sensitive headers and +%% they won't be clobbered by the update. +t_update_with_sensitive_data(Config) -> + ?check_trace( + begin + ConnectorCfg0 = make_connector_config(Config), + AuthHeader = <<"Bearer some_token">>, + ConnectorCfg1 = emqx_utils_maps:deep_merge( + ConnectorCfg0, + #{<<"headers">> => #{<<"authorization">> => AuthHeader}} + ), + ActionCfg = make_action_config(Config), + CreateConfig = [ + {bridge_kind, action}, + {action_type, ?BRIDGE_TYPE}, + {action_name, ?BRIDGE_NAME}, + {action_config, ActionCfg}, + {connector_type, ?BRIDGE_TYPE}, + {connector_name, ?CONNECTOR_NAME}, + {connector_config, ConnectorCfg1} + ], + {ok, {{_, 201, _}, _, #{<<"headers">> := #{<<"authorization">> := Obfuscated}}}} = + emqx_bridge_v2_testlib:create_connector_api(CreateConfig), + {ok, _} = + emqx_bridge_v2_testlib:create_kind_api(CreateConfig), + BridgeId = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, ?BRIDGE_NAME), + {ok, _} = emqx_bridge_v2_testlib:create_rule_api( + #{ + sql => <<"select * from \"t/http\" ">>, + actions => [BridgeId] + } + ), + emqx:publish(emqx_message:make(<<"t/http">>, <<"1">>)), + ?assertReceive({http, #{<<"authorization">> := AuthHeader}, _}), + + %% Now update the connector and see if the header stays deobfuscated. We send the old + %% auth header as an obfuscated value to simulate the behavior of the frontend. + ConnectorCfg2 = emqx_utils_maps:deep_merge( + ConnectorCfg1, + #{ + <<"headers">> => #{ + <<"authorization">> => Obfuscated, + <<"other_header">> => <<"new">> + } + } + ), + {ok, _} = emqx_bridge_v2_testlib:update_connector_api( + ?CONNECTOR_NAME, + ?BRIDGE_TYPE, + ConnectorCfg2 + ), + + emqx:publish(emqx_message:make(<<"t/http">>, <<"2">>)), + %% Should not be obfuscated. + ?assertReceive({http, #{<<"authorization">> := AuthHeader}, _}, 2_000), + + ok + end, + [] + ), + + ok. + %%-------------------------------------------------------------------- %% helpers %%-------------------------------------------------------------------- @@ -123,7 +197,10 @@ make_connector_config(Config) -> <<"url">> => iolist_to_binary(io_lib:format("http://localhost:~p", [Port])), <<"headers">> => #{}, <<"pool_type">> => <<"hash">>, - <<"pool_size">> => 1 + <<"pool_size">> => 1, + <<"resource_opts">> => #{ + <<"health_check_interval">> => <<"100ms">> + } }. make_action_config(Config) -> @@ -136,5 +213,22 @@ make_action_config(Config) -> <<"method">> => <<"post">>, <<"headers">> => #{}, <<"body">> => <<"${.}">> + }, + <<"resource_opts">> => #{ + <<"health_check_interval">> => <<"100ms">> } }. + +success_handler() -> + TestPid = self(), + fun(Req0, State) -> + {ok, Body, Req} = cowboy_req:read_body(Req0), + TestPid ! {http, cowboy_req:headers(Req), Body}, + Rep = cowboy_req:reply( + 200, + #{<<"content-type">> => <<"application/json">>}, + <<"{}">>, + Req + ), + {ok, Rep, State} + end. diff --git a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl index 94419c7d9..f239d3735 100644 --- a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl +++ b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl @@ -130,7 +130,7 @@ on_query(InstId, {Channel, Message}, #{channels := ChannelConf}) -> influxdb_connector_send_query, #{points => Points, batch => false, mode => sync} ), - do_query(InstId, Client, Points); + do_query(InstId, Channel, Client, Points); {error, ErrorPoints} -> ?tp( influxdb_connector_send_query_error, @@ -152,7 +152,7 @@ on_batch_query(InstId, BatchData, #{channels := ChannelConf}) -> influxdb_connector_send_query, #{points => Points, batch => true, mode => sync} ), - do_query(InstId, Client, Points); + do_query(InstId, Channel, Client, Points); {error, Reason} -> ?tp( influxdb_connector_send_query_error, @@ -175,7 +175,7 @@ on_query_async( influxdb_connector_send_query, #{points => Points, batch => false, mode => async} ), - do_async_query(InstId, Client, Points, {ReplyFun, Args}); + do_async_query(InstId, Channel, Client, Points, {ReplyFun, Args}); {error, ErrorPoints} = Err -> ?tp( influxdb_connector_send_query_error, @@ -200,7 +200,7 @@ on_batch_query_async( influxdb_connector_send_query, #{points => Points, batch => true, mode => async} ), - do_async_query(InstId, Client, Points, {ReplyFun, Args}); + do_async_query(InstId, Channel, Client, Points, {ReplyFun, Args}); {error, Reason} -> ?tp( influxdb_connector_send_query_error, @@ -496,7 +496,8 @@ is_auth_key(_) -> %% ------------------------------------------------------------------------------------------------- %% Query -do_query(InstId, Client, Points) -> +do_query(InstId, Channel, Client, Points) -> + emqx_trace:rendered_action_template(Channel, #{points => Points, is_async => false}), case influxdb:write(Client, Points) of ok -> ?SLOG(debug, #{ @@ -527,12 +528,13 @@ do_query(InstId, Client, Points) -> end end. -do_async_query(InstId, Client, Points, ReplyFunAndArgs) -> +do_async_query(InstId, Channel, Client, Points, ReplyFunAndArgs) -> ?SLOG(info, #{ msg => "influxdb_write_point_async", connector => InstId, points => Points }), + emqx_trace:rendered_action_template(Channel, #{points => Points, is_async => true}), WrappedReplyFunAndArgs = {fun ?MODULE:reply_callback/2, [ReplyFunAndArgs]}, {ok, _WorkerPid} = influxdb:write_async(Client, Points, WrappedReplyFunAndArgs). diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl index 6bb1690ff..16bca153a 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl @@ -319,6 +319,9 @@ on_query( emqx_bridge_kafka_impl_producer_sync_query, #{headers_config => KafkaHeaders, instance_id => InstId} ), + emqx_trace:rendered_action_template(MessageTag, #{ + message => KafkaMessage, send_type => sync + }), do_send_msg(sync, KafkaMessage, Producers, SyncTimeout) catch throw:{bad_kafka_header, _} = Error -> @@ -376,6 +379,9 @@ on_query_async( emqx_bridge_kafka_impl_producer_async_query, #{headers_config => KafkaHeaders, instance_id => InstId} ), + emqx_trace:rendered_action_template(MessageTag, #{ + message => KafkaMessage, send_type => async + }), do_send_msg(async, KafkaMessage, Producers, AsyncReplyFn) catch error:{invalid_partition_count, _Count, _Partitioner} -> diff --git a/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_impl_producer.erl b/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_impl_producer.erl index c8a522e01..8744dfd71 100644 --- a/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_impl_producer.erl +++ b/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_impl_producer.erl @@ -261,6 +261,11 @@ do_send_requests_sync( stream_name := StreamName } = maps:get(ChannelId, InstalledChannels), Records = render_records(Requests, Templates), + StructuredRecords = [ + #{data => Data, partition_key => PartitionKey} + || {Data, PartitionKey} <- Records + ], + emqx_trace:rendered_action_template(ChannelId, StructuredRecords), Result = ecpool:pick_and_do( PoolName, {emqx_bridge_kinesis_connector_client, query, [Records, StreamName]}, diff --git a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_connector.erl b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_connector.erl index a0d53d454..69c2242e4 100644 --- a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_connector.erl +++ b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_connector.erl @@ -66,10 +66,15 @@ on_query(InstanceId, {Channel, Message0}, #{channels := Channels, connector_stat payload_template := PayloadTemplate, collection_template := CollectionTemplate } = ChannelState0 = maps:get(Channel, Channels), + Collection = emqx_placeholder:proc_tmpl(CollectionTemplate, Message0), ChannelState = ChannelState0#{ - collection => emqx_placeholder:proc_tmpl(CollectionTemplate, Message0) + collection => Collection }, Message = render_message(PayloadTemplate, Message0), + emqx_trace:rendered_action_template(Channel, #{ + collection => Collection, + data => Message + }), Res = emqx_mongodb:on_query( InstanceId, {Channel, Message}, diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl index 900f6143f..f133bf334 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl @@ -264,7 +264,7 @@ on_query( ), Channels = maps:get(installed_channels, State), ChannelConfig = maps:get(ChannelId, Channels), - handle_send_result(with_egress_client(PoolName, send, [Msg, ChannelConfig])); + handle_send_result(with_egress_client(ChannelId, PoolName, send, [Msg, ChannelConfig])); on_query(ResourceId, {_ChannelId, Msg}, #{}) -> ?SLOG(error, #{ msg => "forwarding_unavailable", @@ -283,7 +283,7 @@ on_query_async( Callback = {fun on_async_result/2, [CallbackIn]}, Channels = maps:get(installed_channels, State), ChannelConfig = maps:get(ChannelId, Channels), - Result = with_egress_client(PoolName, send_async, [Msg, Callback, ChannelConfig]), + Result = with_egress_client(ChannelId, PoolName, send_async, [Msg, Callback, ChannelConfig]), case Result of ok -> ok; @@ -300,8 +300,11 @@ on_query_async(ResourceId, {_ChannelId, Msg}, _Callback, #{}) -> reason => "Egress is not configured" }). -with_egress_client(ResourceId, Fun, Args) -> - ecpool:pick_and_do(ResourceId, {emqx_bridge_mqtt_egress, Fun, Args}, no_handover). +with_egress_client(ActionID, ResourceId, Fun, Args) -> + TraceRenderedCTX = emqx_trace:make_rendered_action_template_trace_context(ActionID), + ecpool:pick_and_do( + ResourceId, {emqx_bridge_mqtt_egress, Fun, [TraceRenderedCTX | Args]}, no_handover + ). on_async_result(Callback, Result) -> apply_callback_function(Callback, handle_send_result(Result)). @@ -337,6 +340,8 @@ classify_error({shutdown, _} = Reason) -> {recoverable_error, Reason}; classify_error(shutdown = Reason) -> {recoverable_error, Reason}; +classify_error({unrecoverable_error, _Reason} = Error) -> + Error; classify_error(Reason) -> {unrecoverable_error, Reason}. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_egress.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_egress.erl index d23899ef1..a4a0b0d37 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_egress.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_egress.erl @@ -22,8 +22,8 @@ -export([ config/1, - send/3, - send_async/4 + send/4, + send_async/5 ]). -type message() :: emqx_types:message() | map(). @@ -42,25 +42,40 @@ config(#{remote := RC = #{}} = Conf) -> Conf#{remote => emqx_bridge_mqtt_msg:parse(RC)}. --spec send(pid(), message(), egress()) -> ok. -send(Pid, MsgIn, Egress) -> - emqtt:publish(Pid, export_msg(MsgIn, Egress)). +-spec send(pid(), emqx_trace:rendered_action_template_ctx(), message(), egress()) -> + ok | {error, {unrecoverable_error, term()}}. +send(Pid, TraceRenderedCTX, MsgIn, Egress) -> + try + emqtt:publish(Pid, export_msg(MsgIn, Egress, TraceRenderedCTX)) + catch + error:{unrecoverable_error, Reason} -> + {error, {unrecoverable_error, Reason}} + end. --spec send_async(pid(), message(), callback(), egress()) -> - ok | {ok, pid()}. -send_async(Pid, MsgIn, Callback, Egress) -> - ok = emqtt:publish_async(Pid, export_msg(MsgIn, Egress), _Timeout = infinity, Callback), - {ok, Pid}. +-spec send_async(pid(), emqx_trace:rendered_action_template_ctx(), message(), callback(), egress()) -> + {ok, pid()} | {error, {unrecoverable_error, term()}}. +send_async(Pid, TraceRenderedCTX, MsgIn, Callback, Egress) -> + try + ok = emqtt:publish_async( + Pid, export_msg(MsgIn, Egress, TraceRenderedCTX), _Timeout = infinity, Callback + ), + {ok, Pid} + catch + error:{unrecoverable_error, Reason} -> + {error, {unrecoverable_error, Reason}} + end. -export_msg(Msg, #{remote := Remote}) -> - to_remote_msg(Msg, Remote). +export_msg(Msg, #{remote := Remote}, TraceRenderedCTX) -> + to_remote_msg(Msg, Remote, TraceRenderedCTX). --spec to_remote_msg(message(), emqx_bridge_mqtt_msg:msgvars()) -> +-spec to_remote_msg( + message(), emqx_bridge_mqtt_msg:msgvars(), emqx_trace:rendered_action_template_ctx() +) -> remote_message(). -to_remote_msg(#message{flags = Flags} = Msg, Vars) -> +to_remote_msg(#message{flags = Flags} = Msg, Vars, TraceRenderedCTX) -> {EventMsg, _} = emqx_rule_events:eventmsg_publish(Msg), - to_remote_msg(EventMsg#{retain => maps:get(retain, Flags, false)}, Vars); -to_remote_msg(Msg = #{}, Remote) -> + to_remote_msg(EventMsg#{retain => maps:get(retain, Flags, false)}, Vars, TraceRenderedCTX); +to_remote_msg(Msg = #{}, Remote, TraceRenderedCTX) -> #{ topic := Topic, payload := Payload, @@ -68,6 +83,13 @@ to_remote_msg(Msg = #{}, Remote) -> retain := Retain } = emqx_bridge_mqtt_msg:render(Msg, Remote), PubProps = maps:get(pub_props, Msg, #{}), + emqx_trace:rendered_action_template_with_ctx(TraceRenderedCTX, #{ + qos => QoS, + retain => Retain, + topic => Topic, + props => PubProps, + payload => Payload + }), #mqtt_msg{ qos = QoS, retain = Retain, diff --git a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql_connector.erl b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql_connector.erl index 6720e1fb7..da9377814 100644 --- a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql_connector.erl +++ b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql_connector.erl @@ -104,10 +104,12 @@ on_query( #{channels := Channels, connector_state := ConnectorState} ) when is_binary(Channel) -> ChannelConfig = maps:get(Channel, Channels), + MergedState0 = maps:merge(ConnectorState, ChannelConfig), + MergedState1 = MergedState0#{channel_id => Channel}, Result = emqx_mysql:on_query( InstanceId, Request, - maps:merge(ConnectorState, ChannelConfig) + MergedState1 ), ?tp(mysql_connector_on_query_return, #{instance_id => InstanceId, result => Result}), Result; @@ -121,10 +123,12 @@ on_batch_query( ) when is_binary(element(1, Req)) -> Channel = element(1, Req), ChannelConfig = maps:get(Channel, Channels), + MergedState0 = maps:merge(ConnectorState, ChannelConfig), + MergedState1 = MergedState0#{channel_id => Channel}, Result = emqx_mysql:on_batch_query( InstanceId, BatchRequest, - maps:merge(ConnectorState, ChannelConfig) + MergedState1 ), ?tp(mysql_connector_on_batch_query_return, #{instance_id => InstanceId, result => Result}), Result; diff --git a/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl b/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl index abef958ff..509d53284 100644 --- a/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl +++ b/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl @@ -167,9 +167,10 @@ on_batch_query( BatchReq, #{channels := Channels} = State ) -> + [{ChannelId, _} | _] = BatchReq, case try_render_messages(BatchReq, Channels) of {ok, Datas} -> - do_query(InstanceId, Datas, State); + do_query(InstanceId, ChannelId, Datas, State); Error -> Error end. @@ -222,12 +223,13 @@ on_get_channel_status(InstanceId, ChannelId, #{channels := Channels} = State) -> %% Helper fns %%======================================================================================== -do_query(InstanceId, Query, #{pool_name := PoolName} = State) -> +do_query(InstanceId, ChannelID, Query, #{pool_name := PoolName} = State) -> ?TRACE( "QUERY", "opents_connector_received", #{connector => InstanceId, query => Query, state => State} ), + emqx_trace:rendered_action_template(ChannelID, #{query => Query}), ?tp(opents_bridge_on_query, #{instance_id => InstanceId}), diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_connector.erl b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_connector.erl index 8c39c3671..0cddfab66 100644 --- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_connector.erl +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_connector.erl @@ -196,6 +196,11 @@ on_query(_InstanceId, {ChannelId, Message}, State) -> {error, channel_not_found}; {ok, #{message := MessageTmpl, sync_timeout := SyncTimeout, producers := Producers}} -> PulsarMessage = render_message(Message, MessageTmpl), + emqx_trace:rendered_action_template(ChannelId, #{ + message => PulsarMessage, + sync_timeout => SyncTimeout, + is_async => false + }), try pulsar:send_sync(Producers, [PulsarMessage], SyncTimeout) catch @@ -217,12 +222,16 @@ on_query_async(_InstanceId, {ChannelId, Message}, AsyncReplyFn, State) -> ?tp_span( pulsar_producer_on_query_async, #{instance_id => _InstanceId, message => Message}, - on_query_async2(Producers, Message, MessageTmpl, AsyncReplyFn) + on_query_async2(ChannelId, Producers, Message, MessageTmpl, AsyncReplyFn) ) end. -on_query_async2(Producers, Message, MessageTmpl, AsyncReplyFn) -> +on_query_async2(ChannelId, Producers, Message, MessageTmpl, AsyncReplyFn) -> PulsarMessage = render_message(Message, MessageTmpl), + emqx_trace:rendered_action_template(ChannelId, #{ + message => PulsarMessage, + is_async => true + }), pulsar:send(Producers, [PulsarMessage], #{callback_fn => AsyncReplyFn}). %%------------------------------------------------------------------------------------- diff --git a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl index 1ef1c6617..dacb47a57 100644 --- a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl +++ b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl @@ -9,6 +9,7 @@ -include_lib("emqx_resource/include/emqx_resource.hrl"). -include_lib("typerefl/include/types.hrl"). -include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/emqx_trace.hrl"). -include_lib("hocon/include/hoconsc.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). @@ -41,7 +42,7 @@ -export([connect/1]). %% Internal callbacks --export([publish_messages/4]). +-export([publish_messages/5]). namespace() -> "rabbitmq". @@ -214,9 +215,10 @@ on_query(ResourceID, {ChannelId, Data} = MsgReq, State) -> #{channels := Channels} = State, case maps:find(ChannelId, Channels) of {ok, #{param := ProcParam, rabbitmq := RabbitMQ}} -> + TraceRenderedCTX = emqx_trace:make_rendered_action_template_trace_context(ChannelId), Res = ecpool:pick_and_do( ResourceID, - {?MODULE, publish_messages, [RabbitMQ, ProcParam, [MsgReq]]}, + {?MODULE, publish_messages, [RabbitMQ, ProcParam, [MsgReq], TraceRenderedCTX]}, no_handover ), handle_result(Res); @@ -234,9 +236,10 @@ on_batch_query(ResourceID, [{ChannelId, _Data} | _] = Batch, State) -> #{channels := Channels} = State, case maps:find(ChannelId, Channels) of {ok, #{param := ProcParam, rabbitmq := RabbitMQ}} -> + TraceRenderedCTX = emqx_trace:make_rendered_action_template_trace_context(ChannelId), Res = ecpool:pick_and_do( ResourceID, - {?MODULE, publish_messages, [RabbitMQ, ProcParam, Batch]}, + {?MODULE, publish_messages, [RabbitMQ, ProcParam, Batch, TraceRenderedCTX]}, no_handover ), handle_result(Res); @@ -255,7 +258,8 @@ publish_messages( wait_for_publish_confirmations := WaitForPublishConfirmations, publish_confirmation_timeout := PublishConfirmationTimeout }, - Messages + Messages, + TraceRenderedCTX ) -> try publish_messages( @@ -267,15 +271,18 @@ publish_messages( PayloadTmpl, Messages, WaitForPublishConfirmations, - PublishConfirmationTimeout + PublishConfirmationTimeout, + TraceRenderedCTX ) catch + error:?EMQX_TRACE_STOP_ACTION_MATCH = Reason -> + {error, Reason}; %% if send a message to a non-existent exchange, RabbitMQ client will crash %% {shutdown,{server_initiated_close,404,<<"NOT_FOUND - no exchange 'xyz' in vhost '/'">>} %% so we catch and return {recoverable_error, Reason} to increase metrics _Type:Reason -> Msg = iolist_to_binary(io_lib:format("RabbitMQ: publish_failed: ~p", [Reason])), - erlang:error({recoverable_error, Msg}) + {error, {recoverable_error, Msg}} end. publish_messages( @@ -287,7 +294,8 @@ publish_messages( PayloadTmpl, Messages, WaitForPublishConfirmations, - PublishConfirmationTimeout + PublishConfirmationTimeout, + TraceRenderedCTX ) -> case maps:find(Conn, RabbitMQ) of {ok, Channel} -> @@ -299,18 +307,33 @@ publish_messages( exchange = Exchange, routing_key = RoutingKey }, + FormattedMsgs = [ + format_data(PayloadTmpl, M) + || {_, M} <- Messages + ], + emqx_trace:rendered_action_template_with_ctx(TraceRenderedCTX, #{ + messages => FormattedMsgs, + properties => #{ + headers => [], + delivery_mode => DeliveryMode + }, + method => #{ + exchange => Exchange, + routing_key => RoutingKey + } + }), lists:foreach( - fun({_, MsgRaw}) -> + fun(Msg) -> amqp_channel:cast( Channel, Method, #amqp_msg{ - payload = format_data(PayloadTmpl, MsgRaw), + payload = Msg, props = MessageProperties } ) end, - Messages + FormattedMsgs ), case WaitForPublishConfirmations of true -> diff --git a/apps/emqx_bridge_redis/src/emqx_bridge_redis_connector.erl b/apps/emqx_bridge_redis/src/emqx_bridge_redis_connector.erl index b7477b385..3f7c4897c 100644 --- a/apps/emqx_bridge_redis/src/emqx_bridge_redis_connector.erl +++ b/apps/emqx_bridge_redis/src/emqx_bridge_redis_connector.erl @@ -107,7 +107,7 @@ on_query(InstId, {cmd, Cmd}, #{conn_st := RedisConnSt}) -> Result; on_query( InstId, - {_MessageTag, _Data} = Msg, + {MessageTag, _Data} = Msg, #{channels := Channels, conn_st := RedisConnSt} ) -> case try_render_message([Msg], Channels) of @@ -116,6 +116,10 @@ on_query( redis_bridge_connector_cmd, #{cmd => Cmd, batch => false, mode => sync} ), + emqx_trace:rendered_action_template( + MessageTag, + #{command => Cmd, batch => false, mode => sync} + ), Result = query(InstId, {cmd, Cmd}, RedisConnSt), ?tp( redis_bridge_connector_send_done, @@ -135,6 +139,11 @@ on_batch_query( redis_bridge_connector_send, #{batch_data => BatchData, batch => true, mode => sync} ), + [{ChannelID, _} | _] = BatchData, + emqx_trace:rendered_action_template( + ChannelID, + #{commands => Cmds, batch => ture, mode => sync} + ), Result = query(InstId, {cmds, Cmds}, RedisConnSt), ?tp( redis_bridge_connector_send_done, diff --git a/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq_connector.erl b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq_connector.erl index f9b4ec5d4..314afb350 100644 --- a/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq_connector.erl +++ b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq_connector.erl @@ -264,7 +264,11 @@ do_query( TopicKey = get_topic_key(Query, TopicTks), Data = apply_template(Query, Templates, DispatchStrategy), - + emqx_trace:rendered_action_template(ChannelId, #{ + topic_key => TopicKey, + data => Data, + request_timeout => RequestTimeout + }), Result = safe_do_produce( ChannelId, InstanceId, QueryFunc, ClientId, TopicKey, Data, ProducerOpts, RequestTimeout ), diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3.app.src b/apps/emqx_bridge_s3/src/emqx_bridge_s3.app.src index a6000067a..5cdf3fb82 100644 --- a/apps/emqx_bridge_s3/src/emqx_bridge_s3.app.src +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3.app.src @@ -10,9 +10,15 @@ emqx_s3 ]}, {env, [ - {emqx_action_info_modules, [emqx_bridge_s3_action_info]}, - {emqx_connector_info_modules, [emqx_bridge_s3_connector_info]} + {emqx_action_info_modules, [ + emqx_bridge_s3_upload_action_info, + emqx_bridge_s3_aggreg_upload_action_info + ]}, + {emqx_connector_info_modules, [ + emqx_bridge_s3_connector_info + ]} ]}, + {mod, {emqx_bridge_s3_app, []}}, {modules, []}, {links, []} ]}. diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3.erl b/apps/emqx_bridge_s3/src/emqx_bridge_s3.erl index 79cc560d2..49f033554 100644 --- a/apps/emqx_bridge_s3/src/emqx_bridge_s3.erl +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3.erl @@ -19,7 +19,6 @@ ]). -export([ - bridge_v2_examples/1, connector_examples/1 ]). @@ -39,58 +38,11 @@ fields(Field) when Field == "post_connector" -> emqx_connector_schema:api_fields(Field, ?CONNECTOR, fields(s3_connector_config)); -fields(Field) when - Field == "get_bridge_v2"; - Field == "put_bridge_v2"; - Field == "post_bridge_v2" --> - emqx_bridge_v2_schema:api_fields(Field, ?ACTION, fields(?ACTION)); -fields(action) -> - {?ACTION, - hoconsc:mk( - hoconsc:map(name, hoconsc:ref(?MODULE, ?ACTION)), - #{ - desc => <<"S3 Action Config">>, - required => false - } - )}; fields("config_connector") -> emqx_connector_schema:common_fields() ++ fields(s3_connector_config); -fields(?ACTION) -> - emqx_bridge_v2_schema:make_producer_action_schema( - hoconsc:mk( - ?R_REF(s3_upload_parameters), - #{ - required => true, - desc => ?DESC(s3_upload) - } - ), - #{ - resource_opts_ref => ?R_REF(s3_action_resource_opts) - } - ); fields(s3_connector_config) -> emqx_s3_schema:fields(s3_client) ++ emqx_connector_schema:resource_opts_ref(?MODULE, s3_connector_resource_opts); -fields(s3_upload_parameters) -> - emqx_s3_schema:fields(s3_upload) ++ - [ - {content, - hoconsc:mk( - emqx_schema:template(), - #{ - required => false, - default => <<"${.}">>, - desc => ?DESC(s3_object_content) - } - )} - ]; -fields(s3_action_resource_opts) -> - UnsupportedOpts = [batch_size, batch_time], - lists:filter( - fun({N, _}) -> not lists:member(N, UnsupportedOpts) end, - emqx_bridge_v2_schema:action_resource_opts_fields() - ); fields(s3_connector_resource_opts) -> CommonOpts = emqx_connector_schema:common_resource_opts_subfields(), lists:filter( @@ -100,14 +52,6 @@ fields(s3_connector_resource_opts) -> desc("config_connector") -> ?DESC(config_connector); -desc(?ACTION) -> - ?DESC(s3_upload); -desc(s3_upload) -> - ?DESC(s3_upload); -desc(s3_upload_parameters) -> - ?DESC(s3_upload_parameters); -desc(s3_action_resource_opts) -> - ?DESC(emqx_resource_schema, resource_opts); desc(s3_connector_resource_opts) -> ?DESC(emqx_resource_schema, resource_opts); desc(_Name) -> @@ -115,54 +59,6 @@ desc(_Name) -> %% Examples -bridge_v2_examples(Method) -> - [ - #{ - <<"s3">> => #{ - summary => <<"S3 Simple Upload">>, - value => action_example(Method) - } - } - ]. - -action_example(post) -> - maps:merge( - action_example(put), - #{ - type => atom_to_binary(?ACTION), - name => <<"my_s3_action">> - } - ); -action_example(get) -> - maps:merge( - action_example(put), - #{ - status => <<"connected">>, - node_status => [ - #{ - node => <<"emqx@localhost">>, - status => <<"connected">> - } - ] - } - ); -action_example(put) -> - #{ - enable => true, - connector => <<"my_s3_connector">>, - description => <<"My action">>, - parameters => #{ - bucket => <<"${clientid}">>, - key => <<"${topic}">>, - content => <<"${payload}">>, - acl => <<"public_read">> - }, - resource_opts => #{ - query_mode => <<"sync">>, - inflight_window => 10 - } - }. - connector_examples(Method) -> [ #{ diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3.hrl b/apps/emqx_bridge_s3/src/emqx_bridge_s3.hrl index 6d500d056..62a80d260 100644 --- a/apps/emqx_bridge_s3/src/emqx_bridge_s3.hrl +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3.hrl @@ -5,7 +5,12 @@ -ifndef(__EMQX_BRIDGE_S3_HRL__). -define(__EMQX_BRIDGE_S3_HRL__, true). --define(ACTION, s3). +%% Actions +-define(ACTION_UPLOAD, s3). +-define(BRIDGE_TYPE_UPLOAD, <<"s3">>). +-define(ACTION_AGGREGATED_UPLOAD, s3_aggregated_upload). +-define(BRIDGE_TYPE_AGGREGATED_UPLOAD, <<"s3_aggregated_upload">>). + -define(CONNECTOR, s3). -endif. diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_buffer.erl b/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_buffer.erl new file mode 100644 index 000000000..503b864a7 --- /dev/null +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_buffer.erl @@ -0,0 +1,138 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +%% This module provides pretty stupid interface for writing and reading +%% Erlang terms to/from a file descriptor (i.e. IO device), with a goal +%% of being able to write and read terms in a streaming fashion, and to +%% survive partial corruption. +%% +%% Layout of the file is as follows: +%% ``` +%% ETF(Header { Metadata }) +%% ETF(Record1 ByteSize) +%% ETF(Record1) +%% ETF(Record2 ByteSize) +%% ETF(Record2) +%% ... +%% ``` +%% ^ ETF = Erlang External Term Format (i.e. `erlang:term_to_binary/1`). +-module(emqx_bridge_s3_aggreg_buffer). + +-export([ + new_writer/2, + write/2, + takeover/1 +]). + +-export([ + new_reader/1, + read/1 +]). + +-export_type([writer/0, reader/0]). + +-record(reader, { + fd :: file:io_device() | eof, + buffer :: binary(), + hasread = 0 :: non_neg_integer() +}). + +-type writer() :: file:io_device(). +-type reader() :: #reader{}. + +%% + +-define(VSN, 1). +-define(HEADER(MD), [?MODULE, ?VSN, MD]). + +-define(READAHEAD_BYTES, 64 * 4096). +-define(SANE_TERM_SIZE, 256 * 1024 * 1024). + +%% + +-spec new_writer(file:io_device(), _Meta) -> writer(). +new_writer(FD, Meta) -> + %% TODO: Validate header is not too big? + Header = term_to_iovec(?HEADER(Meta)), + case file:write(FD, Header) of + ok -> + FD; + {error, Reason} -> + error({buffer_write_failed, Reason}) + end. + +-spec write(_Term, writer()) -> ok | {error, file:posix()}. +write(Term, FD) -> + IOData = term_to_iovec(Term), + Marker = term_to_binary(iolist_size(IOData)), + file:write(FD, [Marker | IOData]). + +%% + +-spec new_reader(file:io_device()) -> {_Meta, reader()}. +new_reader(FD) -> + Reader0 = #reader{fd = FD, buffer = <<>>}, + Reader1 = read_buffered(?READAHEAD_BYTES, Reader0), + case read_next_term(Reader1) of + {?HEADER(MD), Reader} -> + {MD, Reader}; + {UnexpectedHeader, _Reader} -> + error({buffer_unexpected_header, UnexpectedHeader}); + eof -> + error({buffer_incomplete, header}) + end. + +-spec read(reader()) -> {_Term, reader()} | eof. +read(Reader0) -> + case read_next_term(read_buffered(_LargeEnough = 16, Reader0)) of + {Size, Reader1} when is_integer(Size) andalso Size > 0 andalso Size < ?SANE_TERM_SIZE -> + case read_next_term(read_buffered(Size, Reader1)) of + {Term, Reader} -> + {Term, Reader}; + eof -> + error({buffer_incomplete, Size}) + end; + {UnexpectedSize, _Reader} -> + error({buffer_unexpected_record_size, UnexpectedSize}); + eof -> + eof + end. + +-spec takeover(reader()) -> writer(). +takeover(#reader{fd = FD, hasread = HasRead}) -> + case file:position(FD, HasRead) of + {ok, HasRead} -> + case file:truncate(FD) of + ok -> + FD; + {error, Reason} -> + error({buffer_takeover_failed, Reason}) + end; + {error, Reason} -> + error({buffer_takeover_failed, Reason}) + end. + +read_next_term(#reader{fd = eof, buffer = <<>>}) -> + eof; +read_next_term(Reader = #reader{buffer = Buffer, hasread = HasRead}) -> + {Term, UsedBytes} = erlang:binary_to_term(Buffer, [safe, used]), + BufferSize = byte_size(Buffer), + BufferLeft = binary:part(Buffer, UsedBytes, BufferSize - UsedBytes), + {Term, Reader#reader{buffer = BufferLeft, hasread = HasRead + UsedBytes}}. + +read_buffered(_Size, Reader = #reader{fd = eof}) -> + Reader; +read_buffered(Size, Reader = #reader{fd = FD, buffer = Buffer0}) -> + BufferSize = byte_size(Buffer0), + ReadSize = erlang:max(Size, ?READAHEAD_BYTES), + case BufferSize < Size andalso file:read(FD, ReadSize) of + false -> + Reader; + {ok, Data} -> + Reader#reader{buffer = <>}; + eof -> + Reader#reader{fd = eof}; + {error, Reason} -> + error({buffer_read_failed, Reason}) + end. diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_csv.erl b/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_csv.erl new file mode 100644 index 000000000..33895d8c1 --- /dev/null +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_csv.erl @@ -0,0 +1,105 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +%% CSV container implementation for `emqx_bridge_s3_aggregator`. +-module(emqx_bridge_s3_aggreg_csv). + +%% Container API +-export([ + new/1, + fill/2, + close/1 +]). + +-export_type([container/0]). + +-record(csv, { + columns :: [binary()] | undefined, + column_order :: [binary()], + %% A string or character that separates each field in a record from the next. + %% Default: "," + field_separator :: char() | iodata(), + %% A string or character that delimits boundaries of a record. + %% Default: "\n" + record_delimiter :: char() | iodata(), + quoting_mp :: _ReMP +}). + +-type container() :: #csv{}. + +-type options() :: #{ + %% Which columns have to be ordered first in the resulting CSV? + column_order => [column()] +}. + +-type record() :: emqx_bridge_s3_aggregator:record(). +-type column() :: binary(). + +%% + +-spec new(options()) -> container(). +new(Opts) -> + {ok, MP} = re:compile("[\\[\\],\\r\\n\"]", [unicode]), + #csv{ + column_order = maps:get(column_order, Opts, []), + field_separator = $,, + record_delimiter = $\n, + quoting_mp = MP + }. + +-spec fill([record()], container()) -> {iodata(), container()}. +fill(Records = [Record | _], CSV0 = #csv{columns = undefined}) -> + Columns = mk_columns(Record, CSV0), + Header = emit_header(Columns, CSV0), + {Writes, CSV} = fill(Records, CSV0#csv{columns = Columns}), + {[Header | Writes], CSV}; +fill(Records, CSV) -> + Writes = [emit_row(R, CSV) || R <- Records], + {Writes, CSV}. + +-spec close(container()) -> iodata(). +close(#csv{}) -> + []. + +%% + +mk_columns(Record, #csv{column_order = ColumnOrder}) -> + Columns = [emqx_utils_conv:bin(C) || C <- lists:sort(maps:keys(Record))], + Unoredered = Columns -- ColumnOrder, + ColumnOrder ++ Unoredered. + +-spec emit_header([column()], container()) -> iodata(). +emit_header([C], #csv{record_delimiter = Delim}) -> + [C, Delim]; +emit_header([C | Rest], CSV = #csv{field_separator = Sep}) -> + [C, Sep | emit_header(Rest, CSV)]; +emit_header([], #csv{record_delimiter = Delim}) -> + [Delim]. + +-spec emit_row(record(), container()) -> iodata(). +emit_row(Record, CSV = #csv{columns = Columns}) -> + emit_row(Record, Columns, CSV). + +emit_row(Record, [C], CSV = #csv{record_delimiter = Delim}) -> + [emit_cell(C, Record, CSV), Delim]; +emit_row(Record, [C | Rest], CSV = #csv{field_separator = Sep}) -> + [emit_cell(C, Record, CSV), Sep | emit_row(Record, Rest, CSV)]; +emit_row(#{}, [], #csv{record_delimiter = Delim}) -> + [Delim]. + +emit_cell(Column, Record, CSV) -> + case emqx_template:lookup(Column, Record) of + {ok, Value} -> + encode_cell(emqx_template:to_string(Value), CSV); + {error, undefined} -> + _Empty = "" + end. + +encode_cell(V, #csv{quoting_mp = MP}) -> + case re:run(V, MP, []) of + nomatch -> + V; + _ -> + [$", re:replace(V, <<"\"">>, <<"\"\"">>, [global, unicode]), $"] + end. diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_delivery.erl b/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_delivery.erl new file mode 100644 index 000000000..02099dbec --- /dev/null +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_delivery.erl @@ -0,0 +1,212 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +%% This module takes aggregated records from a buffer and delivers them to S3, +%% wrapped in a configurable container (though currently there's only CSV). +-module(emqx_bridge_s3_aggreg_delivery). + +-include_lib("snabbkaffe/include/trace.hrl"). +-include("emqx_bridge_s3_aggregator.hrl"). + +-export([start_link/3]). + +%% Internal exports +-export([ + init/4, + loop/3 +]). + +-behaviour(emqx_template). +-export([lookup/2]). + +%% Sys +-export([ + system_continue/3, + system_terminate/4, + format_status/2 +]). + +-record(delivery, { + name :: _Name, + container :: emqx_bridge_s3_aggreg_csv:container(), + reader :: emqx_bridge_s3_aggreg_buffer:reader(), + upload :: emqx_s3_upload:t(), + empty :: boolean() +}). + +-type state() :: #delivery{}. + +%% + +start_link(Name, Buffer, Opts) -> + proc_lib:start_link(?MODULE, init, [self(), Name, Buffer, Opts]). + +%% + +-spec init(pid(), _Name, buffer(), _Opts :: map()) -> no_return(). +init(Parent, Name, Buffer, Opts) -> + ?tp(s3_aggreg_delivery_started, #{action => Name, buffer => Buffer}), + Reader = open_buffer(Buffer), + Delivery = init_delivery(Name, Reader, Buffer, Opts#{action => Name}), + _ = erlang:process_flag(trap_exit, true), + ok = proc_lib:init_ack({ok, self()}), + loop(Delivery, Parent, []). + +init_delivery(Name, Reader, Buffer, Opts = #{container := ContainerOpts}) -> + #delivery{ + name = Name, + container = mk_container(ContainerOpts), + reader = Reader, + upload = mk_upload(Buffer, Opts), + empty = true + }. + +open_buffer(#buffer{filename = Filename}) -> + case file:open(Filename, [read, binary, raw]) of + {ok, FD} -> + {_Meta, Reader} = emqx_bridge_s3_aggreg_buffer:new_reader(FD), + Reader; + {error, Reason} -> + error({buffer_open_failed, Reason}) + end. + +mk_container(#{type := csv, column_order := OrderOpt}) -> + %% TODO: Deduplicate? + ColumnOrder = lists:map(fun emqx_utils_conv:bin/1, OrderOpt), + emqx_bridge_s3_aggreg_csv:new(#{column_order => ColumnOrder}). + +mk_upload( + Buffer, + Opts = #{ + bucket := Bucket, + upload_options := UploadOpts, + client_config := Config, + uploader_config := UploaderConfig + } +) -> + Client = emqx_s3_client:create(Bucket, Config), + Key = mk_object_key(Buffer, Opts), + emqx_s3_upload:new(Client, Key, UploadOpts, UploaderConfig). + +mk_object_key(Buffer, #{action := Name, key := Template}) -> + emqx_template:render_strict(Template, {?MODULE, {Name, Buffer}}). + +%% + +-spec loop(state(), pid(), [sys:debug_option()]) -> no_return(). +loop(Delivery, Parent, Debug) -> + %% NOTE: This function is mocked in tests. + receive + Msg -> handle_msg(Msg, Delivery, Parent, Debug) + after 0 -> + process_delivery(Delivery, Parent, Debug) + end. + +process_delivery(Delivery0 = #delivery{reader = Reader0}, Parent, Debug) -> + case emqx_bridge_s3_aggreg_buffer:read(Reader0) of + {Records = [#{} | _], Reader} -> + Delivery1 = Delivery0#delivery{reader = Reader}, + Delivery2 = process_append_records(Records, Delivery1), + Delivery = process_write(Delivery2), + loop(Delivery, Parent, Debug); + {[], Reader} -> + Delivery = Delivery0#delivery{reader = Reader}, + loop(Delivery, Parent, Debug); + eof -> + process_complete(Delivery0); + {Unexpected, _Reader} -> + exit({buffer_unexpected_record, Unexpected}) + end. + +process_append_records(Records, Delivery = #delivery{container = Container0, upload = Upload0}) -> + {Writes, Container} = emqx_bridge_s3_aggreg_csv:fill(Records, Container0), + {ok, Upload} = emqx_s3_upload:append(Writes, Upload0), + Delivery#delivery{ + container = Container, + upload = Upload, + empty = false + }. + +process_write(Delivery = #delivery{upload = Upload0}) -> + case emqx_s3_upload:write(Upload0) of + {ok, Upload} -> + Delivery#delivery{upload = Upload}; + {cont, Upload} -> + process_write(Delivery#delivery{upload = Upload}); + {error, Reason} -> + _ = emqx_s3_upload:abort(Upload0), + exit({upload_failed, Reason}) + end. + +process_complete(#delivery{name = Name, empty = true}) -> + ?tp(s3_aggreg_delivery_completed, #{action => Name, upload => empty}), + exit({shutdown, {skipped, empty}}); +process_complete(#delivery{name = Name, container = Container, upload = Upload0}) -> + Trailer = emqx_bridge_s3_aggreg_csv:close(Container), + {ok, Upload} = emqx_s3_upload:append(Trailer, Upload0), + case emqx_s3_upload:complete(Upload) of + {ok, Completed} -> + ?tp(s3_aggreg_delivery_completed, #{action => Name, upload => Completed}), + ok; + {error, Reason} -> + _ = emqx_s3_upload:abort(Upload), + exit({upload_failed, Reason}) + end. + +%% + +handle_msg({system, From, Msg}, Delivery, Parent, Debug) -> + sys:handle_system_msg(Msg, From, Parent, ?MODULE, Debug, Delivery); +handle_msg({'EXIT', Parent, Reason}, Delivery, Parent, Debug) -> + system_terminate(Reason, Parent, Debug, Delivery); +handle_msg(_Msg, Delivery, Parent, Debug) -> + loop(Parent, Debug, Delivery). + +-spec system_continue(pid(), [sys:debug_option()], state()) -> no_return(). +system_continue(Parent, Debug, Delivery) -> + loop(Delivery, Parent, Debug). + +-spec system_terminate(_Reason, pid(), [sys:debug_option()], state()) -> _. +system_terminate(_Reason, _Parent, _Debug, #delivery{upload = Upload}) -> + emqx_s3_upload:abort(Upload). + +-spec format_status(normal, Args :: [term()]) -> _StateFormatted. +format_status(_Normal, [_PDict, _SysState, _Parent, _Debug, Delivery]) -> + Delivery#delivery{ + upload = emqx_s3_upload:format(Delivery#delivery.upload) + }. + +%% + +-spec lookup(emqx_template:accessor(), {_Name, buffer()}) -> + {ok, integer() | string()} | {error, undefined}. +lookup([<<"action">>], {Name, _Buffer}) -> + {ok, mk_fs_safe_string(Name)}; +lookup(Accessor, {_Name, Buffer = #buffer{}}) -> + lookup_buffer_var(Accessor, Buffer); +lookup(_Accessor, _Context) -> + {error, undefined}. + +lookup_buffer_var([<<"datetime">>, Format], #buffer{since = Since}) -> + {ok, format_timestamp(Since, Format)}; +lookup_buffer_var([<<"datetime_until">>, Format], #buffer{until = Until}) -> + {ok, format_timestamp(Until, Format)}; +lookup_buffer_var([<<"sequence">>], #buffer{seq = Seq}) -> + {ok, Seq}; +lookup_buffer_var([<<"node">>], #buffer{}) -> + {ok, mk_fs_safe_string(atom_to_binary(erlang:node()))}; +lookup_buffer_var(_Binding, _Context) -> + {error, undefined}. + +format_timestamp(Timestamp, <<"rfc3339utc">>) -> + String = calendar:system_time_to_rfc3339(Timestamp, [{unit, second}, {offset, "Z"}]), + mk_fs_safe_string(String); +format_timestamp(Timestamp, <<"rfc3339">>) -> + String = calendar:system_time_to_rfc3339(Timestamp, [{unit, second}]), + mk_fs_safe_string(String); +format_timestamp(Timestamp, <<"unix">>) -> + Timestamp. + +mk_fs_safe_string(String) -> + unicode:characters_to_binary(string:replace(String, ":", "_", all)). diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_upload.erl b/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_upload.erl new file mode 100644 index 000000000..02d43f4f0 --- /dev/null +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_upload.erl @@ -0,0 +1,275 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_s3_aggreg_upload). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). +-include("emqx_bridge_s3.hrl"). + +-define(ACTION, ?ACTION_AGGREGATED_UPLOAD). + +-define(DEFAULT_BATCH_SIZE, 100). +-define(DEFAULT_BATCH_TIME, <<"10ms">>). + +-behaviour(hocon_schema). +-export([ + namespace/0, + roots/0, + fields/1, + desc/1 +]). + +%% Interpreting options +-export([ + mk_key_template/1, + mk_upload_options/1 +]). + +%% emqx_bridge_v2_schema API +-export([bridge_v2_examples/1]). + +%%------------------------------------------------------------------------------------------------- +%% `hocon_schema' API +%%------------------------------------------------------------------------------------------------- + +namespace() -> + "bridge_s3". + +roots() -> + []. + +fields(Field) when + Field == "get_bridge_v2"; + Field == "put_bridge_v2"; + Field == "post_bridge_v2" +-> + emqx_bridge_v2_schema:api_fields(Field, ?ACTION, fields(?ACTION)); +fields(action) -> + {?ACTION, + hoconsc:mk( + hoconsc:map(name, hoconsc:ref(?MODULE, ?ACTION)), + #{ + desc => <<"S3 Aggregated Upload Action Config">>, + required => false + } + )}; +fields(?ACTION) -> + emqx_bridge_v2_schema:make_producer_action_schema( + hoconsc:mk( + ?R_REF(s3_aggregated_upload_parameters), + #{ + required => true, + desc => ?DESC(s3_aggregated_upload_parameters) + } + ), + #{ + resource_opts_ref => ?R_REF(s3_aggreg_upload_resource_opts) + } + ); +fields(s3_aggregated_upload_parameters) -> + lists:append([ + [ + {container, + hoconsc:mk( + %% TODO: Support selectors once there are more than one container. + hoconsc:union(fun + (all_union_members) -> [?REF(s3_aggregated_container_csv)]; + ({value, _Valur}) -> [?REF(s3_aggregated_container_csv)] + end), + #{ + required => true, + default => #{<<"type">> => <<"csv">>}, + desc => ?DESC(s3_aggregated_container) + } + )}, + {aggregation, + hoconsc:mk( + ?REF(s3_aggregation), + #{ + required => true, + desc => ?DESC(s3_aggregation) + } + )} + ], + emqx_resource_schema:override(emqx_s3_schema:fields(s3_upload), [ + {key, #{desc => ?DESC(s3_aggregated_upload_key)}} + ]), + emqx_s3_schema:fields(s3_uploader) + ]); +fields(s3_aggregated_container_csv) -> + [ + {type, + hoconsc:mk( + csv, + #{ + required => true, + desc => ?DESC(s3_aggregated_container_csv) + } + )}, + {column_order, + hoconsc:mk( + hoconsc:array(string()), + #{ + required => false, + default => [], + desc => ?DESC(s3_aggregated_container_csv_column_order) + } + )} + ]; +fields(s3_aggregation) -> + [ + %% TODO: Needs bucketing? (e.g. messages falling in this 1h interval) + {time_interval, + hoconsc:mk( + emqx_schema:duration_s(), + #{ + required => false, + default => <<"1h">>, + desc => ?DESC(s3_aggregation_interval) + } + )}, + {max_records, + hoconsc:mk( + pos_integer(), + #{ + required => false, + default => <<"1000000">>, + desc => ?DESC(s3_aggregation_max_records) + } + )} + ]; +fields(s3_aggreg_upload_resource_opts) -> + %% NOTE: This action should benefit from generous batching defaults. + emqx_bridge_v2_schema:action_resource_opts_fields([ + {batch_size, #{default => ?DEFAULT_BATCH_SIZE}}, + {batch_time, #{default => ?DEFAULT_BATCH_TIME}} + ]). + +desc(Name) when + Name == s3_aggregated_upload; + Name == s3_aggregated_upload_parameters; + Name == s3_aggregation; + Name == s3_aggregated_container_csv +-> + ?DESC(Name); +desc(s3_aggreg_upload_resource_opts) -> + ?DESC(emqx_resource_schema, resource_opts); +desc(_Name) -> + undefined. + +%% Interpreting options + +-spec mk_key_template(_Parameters :: map()) -> emqx_template:str(). +mk_key_template(#{key := Key}) -> + Template = emqx_template:parse(Key), + {_, BindingErrors} = emqx_template:render(Template, #{}), + {UsedBindings, _} = lists:unzip(BindingErrors), + SuffixTemplate = mk_suffix_template(UsedBindings), + case emqx_template:is_const(SuffixTemplate) of + true -> + Template; + false -> + Template ++ SuffixTemplate + end. + +mk_suffix_template(UsedBindings) -> + RequiredBindings = ["action", "node", "datetime.", "sequence"], + SuffixBindings = [ + mk_default_binding(RB) + || RB <- RequiredBindings, + lists:all(fun(UB) -> string:prefix(UB, RB) == nomatch end, UsedBindings) + ], + SuffixTemplate = [["/", B] || B <- SuffixBindings], + emqx_template:parse(SuffixTemplate). + +mk_default_binding("datetime.") -> + "${datetime.rfc3339utc}"; +mk_default_binding(Binding) -> + "${" ++ Binding ++ "}". + +-spec mk_upload_options(_Parameters :: map()) -> emqx_s3_client:upload_options(). +mk_upload_options(Parameters) -> + Headers = mk_upload_headers(Parameters), + #{ + headers => Headers, + acl => maps:get(acl, Parameters, undefined) + }. + +mk_upload_headers(Parameters = #{container := Container}) -> + Headers = normalize_headers(maps:get(headers, Parameters, #{})), + ContainerHeaders = mk_container_headers(Container), + maps:merge(ContainerHeaders, Headers). + +normalize_headers(Headers) -> + maps:fold( + fun(Header, Value, Acc) -> + maps:put(string:lowercase(emqx_utils_conv:str(Header)), Value, Acc) + end, + #{}, + Headers + ). + +mk_container_headers(#{type := csv}) -> + #{"content-type" => "text/csv"}; +mk_container_headers(#{}) -> + #{}. + +%% Examples + +bridge_v2_examples(Method) -> + [ + #{ + <<"s3_aggregated_upload">> => #{ + summary => <<"S3 Aggregated Upload">>, + value => s3_action_example(Method) + } + } + ]. + +s3_action_example(post) -> + maps:merge( + s3_action_example(put), + #{ + type => atom_to_binary(?ACTION_UPLOAD), + name => <<"my_s3_action">> + } + ); +s3_action_example(get) -> + maps:merge( + s3_action_example(put), + #{ + status => <<"connected">>, + node_status => [ + #{ + node => <<"emqx@localhost">>, + status => <<"connected">> + } + ] + } + ); +s3_action_example(put) -> + #{ + enable => true, + connector => <<"my_s3_connector">>, + description => <<"My action">>, + parameters => #{ + bucket => <<"mqtt-aggregated">>, + key => <<"${action}/${node}/${datetime.rfc3339utc}_N${sequence}.csv">>, + acl => <<"public_read">>, + aggregation => #{ + time_interval => <<"15m">>, + max_records => 100_000 + }, + <<"container">> => #{ + type => <<"csv">>, + column_order => [<<"clientid">>, <<"topic">>, <<"publish_received_at">>] + } + }, + resource_opts => #{ + health_check_interval => <<"10s">>, + query_mode => <<"async">>, + inflight_window => 100 + } + }. diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_upload_action_info.erl b/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_upload_action_info.erl new file mode 100644 index 000000000..b179073e5 --- /dev/null +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_upload_action_info.erl @@ -0,0 +1,21 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_s3_aggreg_upload_action_info). + +-behaviour(emqx_action_info). + +-include("emqx_bridge_s3.hrl"). + +-export([ + action_type_name/0, + connector_type_name/0, + schema_module/0 +]). + +action_type_name() -> ?ACTION_AGGREGATED_UPLOAD. + +connector_type_name() -> s3. + +schema_module() -> emqx_bridge_s3_aggreg_upload. diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_upload_sup.erl b/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_upload_sup.erl new file mode 100644 index 000000000..973187b7e --- /dev/null +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggreg_upload_sup.erl @@ -0,0 +1,72 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_s3_aggreg_upload_sup). + +-export([ + start_link/3, + start_link_delivery_sup/2 +]). + +-export([ + start_delivery/2, + start_delivery_proc/3 +]). + +-behaviour(supervisor). +-export([init/1]). + +-define(SUPREF(NAME), {via, gproc, {n, l, {?MODULE, NAME}}}). + +%% + +start_link(Name, AggregOpts, DeliveryOpts) -> + supervisor:start_link(?MODULE, {root, Name, AggregOpts, DeliveryOpts}). + +start_link_delivery_sup(Name, DeliveryOpts) -> + supervisor:start_link(?SUPREF(Name), ?MODULE, {delivery, Name, DeliveryOpts}). + +%% + +start_delivery(Name, Buffer) -> + supervisor:start_child(?SUPREF(Name), [Buffer]). + +start_delivery_proc(Name, DeliveryOpts, Buffer) -> + emqx_bridge_s3_aggreg_delivery:start_link(Name, Buffer, DeliveryOpts). + +%% + +init({root, Name, AggregOpts, DeliveryOpts}) -> + SupFlags = #{ + strategy => one_for_one, + intensity => 10, + period => 5 + }, + AggregatorChildSpec = #{ + id => aggregator, + start => {emqx_bridge_s3_aggregator, start_link, [Name, AggregOpts]}, + type => worker, + restart => permanent + }, + DeliverySupChildSpec = #{ + id => delivery_sup, + start => {?MODULE, start_link_delivery_sup, [Name, DeliveryOpts]}, + type => supervisor, + restart => permanent + }, + {ok, {SupFlags, [DeliverySupChildSpec, AggregatorChildSpec]}}; +init({delivery, Name, DeliveryOpts}) -> + SupFlags = #{ + strategy => simple_one_for_one, + intensity => 100, + period => 5 + }, + ChildSpec = #{ + id => delivery, + start => {?MODULE, start_delivery_proc, [Name, DeliveryOpts]}, + type => worker, + restart => temporary, + shutdown => 1000 + }, + {ok, {SupFlags, [ChildSpec]}}. diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggregator.erl b/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggregator.erl new file mode 100644 index 000000000..47ecdeb4a --- /dev/null +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggregator.erl @@ -0,0 +1,486 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +%% This module manages buffers for aggregating records and offloads them +%% to separate "delivery" processes when they are full or time interval +%% is over. +-module(emqx_bridge_s3_aggregator). + +-include_lib("emqx/include/logger.hrl"). +-include_lib("snabbkaffe/include/trace.hrl"). + +-include("emqx_bridge_s3_aggregator.hrl"). + +-export([ + start_link/2, + push_records/3, + tick/2, + take_error/1 +]). + +-behaviour(gen_server). +-export([ + init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2 +]). + +-export_type([ + record/0, + timestamp/0 +]). + +%% Record. +-type record() :: #{binary() => _}. + +%% Unix timestamp, seconds since epoch. +-type timestamp() :: _Seconds :: non_neg_integer(). + +%% + +-define(VSN, 1). +-define(SRVREF(NAME), {via, gproc, {n, l, {?MODULE, NAME}}}). + +%% + +start_link(Name, Opts) -> + gen_server:start_link(?SRVREF(Name), ?MODULE, mk_state(Name, Opts), []). + +push_records(Name, Timestamp, Records = [_ | _]) -> + %% FIXME: Error feedback. + case pick_buffer(Name, Timestamp) of + undefined -> + BufferNext = next_buffer(Name, Timestamp), + write_records_limited(Name, BufferNext, Records); + Buffer -> + write_records_limited(Name, Buffer, Records) + end; +push_records(_Name, _Timestamp, []) -> + ok. + +tick(Name, Timestamp) -> + case pick_buffer(Name, Timestamp) of + #buffer{} -> + ok; + _Outdated -> + send_close_buffer(Name, Timestamp) + end. + +take_error(Name) -> + gen_server:call(?SRVREF(Name), take_error). + +%% + +write_records_limited(Name, Buffer = #buffer{max_records = undefined}, Records) -> + write_records(Name, Buffer, Records); +write_records_limited(Name, Buffer = #buffer{max_records = MaxRecords}, Records) -> + NR = length(Records), + case inc_num_records(Buffer, NR) of + NR -> + %% NOTE: Allow unconditionally if it's the first write. + write_records(Name, Buffer, Records); + NWritten when NWritten > MaxRecords -> + NextBuffer = rotate_buffer(Name, Buffer), + write_records_limited(Name, NextBuffer, Records); + _ -> + write_records(Name, Buffer, Records) + end. + +write_records(Name, Buffer = #buffer{fd = Writer}, Records) -> + case emqx_bridge_s3_aggreg_buffer:write(Records, Writer) of + ok -> + ?tp(s3_aggreg_records_written, #{action => Name, records => Records}), + ok; + {error, terminated} -> + BufferNext = rotate_buffer(Name, Buffer), + write_records(Name, BufferNext, Records); + {error, _} = Error -> + Error + end. + +inc_num_records(#buffer{cnt_records = Counter}, Size) -> + inc_counter(Counter, Size). + +next_buffer(Name, Timestamp) -> + gen_server:call(?SRVREF(Name), {next_buffer, Timestamp}). + +rotate_buffer(Name, #buffer{fd = FD}) -> + gen_server:call(?SRVREF(Name), {rotate_buffer, FD}). + +send_close_buffer(Name, Timestamp) -> + gen_server:cast(?SRVREF(Name), {close_buffer, Timestamp}). + +%% + +-record(st, { + name :: _Name, + tab :: ets:tid() | undefined, + buffer :: buffer() | undefined, + queued :: buffer() | undefined, + deliveries = #{} :: #{reference() => buffer()}, + errors = queue:new() :: queue:queue(_Error), + interval :: emqx_schema:duration_s(), + max_records :: pos_integer(), + work_dir :: file:filename() +}). + +-type state() :: #st{}. + +mk_state(Name, Opts) -> + Interval = maps:get(time_interval, Opts), + MaxRecords = maps:get(max_records, Opts), + WorkDir = maps:get(work_dir, Opts), + ok = ensure_workdir(WorkDir), + #st{ + name = Name, + interval = Interval, + max_records = MaxRecords, + work_dir = WorkDir + }. + +ensure_workdir(WorkDir) -> + %% NOTE + %% Writing MANIFEST as a means to ensure the work directory is writable. It's not + %% (yet) read back because there's only one version of the implementation. + ok = filelib:ensure_path(WorkDir), + ok = write_manifest(WorkDir). + +write_manifest(WorkDir) -> + Manifest = #{<<"version">> => ?VSN}, + file:write_file(filename:join(WorkDir, "MANIFEST"), hocon_pp:do(Manifest, #{})). + +%% + +-spec init(state()) -> {ok, state()}. +init(St0 = #st{name = Name}) -> + _ = erlang:process_flag(trap_exit, true), + St1 = St0#st{tab = create_tab(Name)}, + St = recover(St1), + _ = announce_current_buffer(St), + {ok, St}. + +handle_call({next_buffer, Timestamp}, _From, St0) -> + St = #st{buffer = Buffer} = handle_next_buffer(Timestamp, St0), + {reply, Buffer, St, 0}; +handle_call({rotate_buffer, FD}, _From, St0) -> + St = #st{buffer = Buffer} = handle_rotate_buffer(FD, St0), + {reply, Buffer, St, 0}; +handle_call(take_error, _From, St0) -> + {MaybeError, St} = handle_take_error(St0), + {reply, MaybeError, St}. + +handle_cast({close_buffer, Timestamp}, St) -> + {noreply, handle_close_buffer(Timestamp, St)}; +handle_cast(_Cast, St) -> + {noreply, St}. + +handle_info(timeout, St) -> + {noreply, handle_queued_buffer(St)}; +handle_info({'DOWN', MRef, _, Pid, Reason}, St0 = #st{name = Name, deliveries = Ds0}) -> + case maps:take(MRef, Ds0) of + {Buffer, Ds} -> + St = St0#st{deliveries = Ds}, + {noreply, handle_delivery_exit(Buffer, Reason, St)}; + error -> + ?SLOG(notice, #{ + msg => "unexpected_down_signal", + action => Name, + pid => Pid, + reason => Reason + }), + {noreply, St0} + end; +handle_info(_Msg, St) -> + {noreply, St}. + +terminate(_Reason, #st{name = Name}) -> + cleanup_tab(Name). + +%% + +handle_next_buffer(Timestamp, St = #st{buffer = #buffer{until = Until}}) when Timestamp < Until -> + St; +handle_next_buffer(Timestamp, St0 = #st{buffer = Buffer = #buffer{since = PrevSince}}) -> + BufferClosed = close_buffer(Buffer), + St = enqueue_closed_buffer(BufferClosed, St0), + handle_next_buffer(Timestamp, PrevSince, St); +handle_next_buffer(Timestamp, St = #st{buffer = undefined}) -> + handle_next_buffer(Timestamp, Timestamp, St). + +handle_next_buffer(Timestamp, PrevSince, St0) -> + NextBuffer = allocate_next_buffer(Timestamp, PrevSince, St0), + St = St0#st{buffer = NextBuffer}, + _ = announce_current_buffer(St), + St. + +handle_rotate_buffer( + FD, + St0 = #st{buffer = Buffer = #buffer{since = Since, seq = Seq, fd = FD}} +) -> + BufferClosed = close_buffer(Buffer), + NextBuffer = allocate_buffer(Since, Seq + 1, St0), + St = enqueue_closed_buffer(BufferClosed, St0#st{buffer = NextBuffer}), + _ = announce_current_buffer(St), + St; +handle_rotate_buffer(_ClosedFD, St) -> + St. + +enqueue_closed_buffer(Buffer, St = #st{queued = undefined}) -> + St#st{queued = Buffer}; +enqueue_closed_buffer(Buffer, St0) -> + %% NOTE: Should never really happen unless interval / max records are too tight. + St = handle_queued_buffer(St0), + St#st{queued = Buffer}. + +handle_queued_buffer(St = #st{queued = undefined}) -> + St; +handle_queued_buffer(St = #st{queued = Buffer}) -> + enqueue_delivery(Buffer, St#st{queued = undefined}). + +allocate_next_buffer(Timestamp, PrevSince, St = #st{interval = Interval}) -> + Since = compute_since(Timestamp, PrevSince, Interval), + allocate_buffer(Since, 0, St). + +compute_since(Timestamp, PrevSince, Interval) -> + Timestamp - (Timestamp - PrevSince) rem Interval. + +allocate_buffer(Since, Seq, St = #st{name = Name}) -> + Buffer = #buffer{filename = Filename, cnt_records = Counter} = mk_buffer(Since, Seq, St), + {ok, FD} = file:open(Filename, [write, binary]), + Writer = emqx_bridge_s3_aggreg_buffer:new_writer(FD, _Meta = []), + _ = add_counter(Counter), + ?tp(s3_aggreg_buffer_allocated, #{action => Name, filename => Filename}), + Buffer#buffer{fd = Writer}. + +recover_buffer(Buffer = #buffer{filename = Filename, cnt_records = Counter}) -> + {ok, FD} = file:open(Filename, [read, write, binary]), + case recover_buffer_writer(FD, Filename) of + {ok, Writer, NWritten} -> + _ = add_counter(Counter, NWritten), + Buffer#buffer{fd = Writer}; + {error, Reason} -> + ?SLOG(warning, #{ + msg => "existing_buffer_recovery_failed", + filename => Filename, + reason => Reason, + details => "Buffer is corrupted beyond repair, will be discarded." + }), + _ = file:close(FD), + _ = file:delete(Filename), + undefined + end. + +recover_buffer_writer(FD, Filename) -> + try emqx_bridge_s3_aggreg_buffer:new_reader(FD) of + {_Meta, Reader} -> recover_buffer_writer(FD, Filename, Reader, 0) + catch + error:Reason -> + {error, Reason} + end. + +recover_buffer_writer(FD, Filename, Reader0, NWritten) -> + try emqx_bridge_s3_aggreg_buffer:read(Reader0) of + {Records, Reader} when is_list(Records) -> + recover_buffer_writer(FD, Filename, Reader, NWritten + length(Records)); + {Unexpected, _Reader} -> + %% Buffer is corrupted, should be discarded. + {error, {buffer_unexpected_record, Unexpected}}; + eof -> + %% Buffer is fine, continue writing at the end. + {ok, FD, NWritten} + catch + error:Reason -> + %% Buffer is truncated or corrupted somewhere in the middle. + %% Continue writing after the last valid record. + ?SLOG(warning, #{ + msg => "existing_buffer_recovered_partially", + filename => Filename, + reason => Reason, + details => + "Buffer is truncated or corrupted somewhere in the middle. " + "Corrupted records will be discarded." + }), + Writer = emqx_bridge_s3_aggreg_buffer:takeover(Reader0), + {ok, Writer, NWritten} + end. + +mk_buffer( + Since, + Seq, + #st{tab = Tab, interval = Interval, max_records = MaxRecords, work_dir = WorkDir} +) -> + Name = mk_filename(Since, Seq), + Counter = {Tab, {Since, Seq}}, + #buffer{ + since = Since, + until = Since + Interval, + seq = Seq, + filename = filename:join(WorkDir, Name), + max_records = MaxRecords, + cnt_records = Counter + }. + +handle_close_buffer( + Timestamp, + St0 = #st{buffer = Buffer = #buffer{until = Until}} +) when Timestamp >= Until -> + St = St0#st{buffer = undefined}, + _ = announce_current_buffer(St), + enqueue_delivery(close_buffer(Buffer), St); +handle_close_buffer(_Timestamp, St = #st{buffer = undefined}) -> + St. + +close_buffer(Buffer = #buffer{fd = FD}) -> + ok = file:close(FD), + Buffer#buffer{fd = undefined}. + +discard_buffer(#buffer{filename = Filename, cnt_records = Counter}) -> + %% NOTE: Hopefully, no process is touching this counter anymore. + _ = del_counter(Counter), + file:delete(Filename). + +pick_buffer(Name, Timestamp) -> + case lookup_current_buffer(Name) of + #buffer{until = Until} = Buffer when Timestamp < Until -> + Buffer; + #buffer{since = Since} when Timestamp < Since -> + %% TODO: Support timestamps going back. + error({invalid_timestamp, Timestamp}); + _Outdated -> + undefined + end. + +announce_current_buffer(#st{tab = Tab, buffer = Buffer}) -> + ets:insert(Tab, {buffer, Buffer}). + +lookup_current_buffer(Name) -> + ets:lookup_element(lookup_tab(Name), buffer, 2). + +%% + +enqueue_delivery(Buffer, St = #st{name = Name, deliveries = Ds}) -> + {ok, Pid} = emqx_bridge_s3_aggreg_upload_sup:start_delivery(Name, Buffer), + MRef = erlang:monitor(process, Pid), + St#st{deliveries = Ds#{MRef => Buffer}}. + +handle_delivery_exit(Buffer, Normal, St = #st{name = Name}) when + Normal == normal; Normal == noproc +-> + ?SLOG(debug, #{ + msg => "aggregated_buffer_delivery_completed", + action => Name, + buffer => Buffer#buffer.filename + }), + ok = discard_buffer(Buffer), + St; +handle_delivery_exit(Buffer, {shutdown, {skipped, Reason}}, St = #st{name = Name}) -> + ?SLOG(info, #{ + msg => "aggregated_buffer_delivery_skipped", + action => Name, + buffer => {Buffer#buffer.since, Buffer#buffer.seq}, + reason => Reason + }), + ok = discard_buffer(Buffer), + St; +handle_delivery_exit(Buffer, Error, St = #st{name = Name}) -> + ?SLOG(error, #{ + msg => "aggregated_buffer_delivery_failed", + action => Name, + buffer => {Buffer#buffer.since, Buffer#buffer.seq}, + filename => Buffer#buffer.filename, + reason => Error + }), + %% TODO: Retries? + enqueue_status_error(Error, St). + +enqueue_status_error({upload_failed, Error}, St = #st{errors = QErrors}) -> + %% TODO + %% This code feels too specific, errors probably need classification. + St#st{errors = queue:in(Error, QErrors)}; +enqueue_status_error(_AnotherError, St) -> + St. + +handle_take_error(St = #st{errors = QErrors0}) -> + case queue:out(QErrors0) of + {{value, Error}, QErrors} -> + {[Error], St#st{errors = QErrors}}; + {empty, QErrors} -> + {[], St#st{errors = QErrors}} + end. + +%% + +recover(St0 = #st{work_dir = WorkDir}) -> + {ok, Filenames} = file:list_dir(WorkDir), + ExistingBuffers = lists:flatmap(fun(FN) -> read_existing_file(FN, St0) end, Filenames), + case lists:reverse(lists:keysort(#buffer.since, ExistingBuffers)) of + [Buffer | ClosedBuffers] -> + St = lists:foldl(fun enqueue_delivery/2, St0, ClosedBuffers), + St#st{buffer = recover_buffer(Buffer)}; + [] -> + St0 + end. + +read_existing_file("MANIFEST", _St) -> + []; +read_existing_file(Name, St) -> + case parse_filename(Name) of + {Since, Seq} -> + [read_existing_buffer(Since, Seq, Name, St)]; + error -> + %% TODO: log? + [] + end. + +read_existing_buffer(Since, Seq, Name, St = #st{work_dir = WorkDir}) -> + Filename = filename:join(WorkDir, Name), + Buffer = mk_buffer(Since, Seq, St), + Buffer#buffer{filename = Filename}. + +%% + +mk_filename(Since, Seq) -> + "T" ++ integer_to_list(Since) ++ "_" ++ pad_number(Seq, 4). + +parse_filename(Filename) -> + case re:run(Filename, "^T(\\d+)_(\\d+)$", [{capture, all_but_first, list}]) of + {match, [Since, Seq]} -> + {list_to_integer(Since), list_to_integer(Seq)}; + nomatch -> + error + end. + +%% + +add_counter({Tab, Counter}) -> + add_counter({Tab, Counter}, 0). + +add_counter({Tab, Counter}, N) -> + ets:insert(Tab, {Counter, N}). + +inc_counter({Tab, Counter}, Size) -> + ets:update_counter(Tab, Counter, {2, Size}). + +del_counter({Tab, Counter}) -> + ets:delete(Tab, Counter). + +%% + +create_tab(Name) -> + Tab = ets:new(?MODULE, [public, set, {write_concurrency, auto}]), + ok = persistent_term:put({?MODULE, Name}, Tab), + Tab. + +lookup_tab(Name) -> + persistent_term:get({?MODULE, Name}). + +cleanup_tab(Name) -> + persistent_term:erase({?MODULE, Name}). + +%% + +pad_number(I, L) -> + string:pad(integer_to_list(I), L, leading, $0). diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggregator.hrl b/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggregator.hrl new file mode 100644 index 000000000..7ac62c6b5 --- /dev/null +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3_aggregator.hrl @@ -0,0 +1,15 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-record(buffer, { + since :: emqx_bridge_s3_aggregator:timestamp(), + until :: emqx_bridge_s3_aggregator:timestamp(), + seq :: non_neg_integer(), + filename :: file:filename(), + fd :: file:io_device() | undefined, + max_records :: pos_integer() | undefined, + cnt_records :: {ets:tab(), _Counter} | undefined +}). + +-type buffer() :: #buffer{}. diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_app.erl b/apps/emqx_bridge_s3/src/emqx_bridge_s3_app.erl new file mode 100644 index 000000000..e5b77f9d6 --- /dev/null +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3_app.erl @@ -0,0 +1,16 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_s3_app). + +-behaviour(application). +-export([start/2, stop/1]). + +%% + +start(_StartType, _StartArgs) -> + emqx_bridge_s3_sup:start_link(). + +stop(_State) -> + ok. diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_connector.erl b/apps/emqx_bridge_s3/src/emqx_bridge_s3_connector.erl index 4407222d5..d135a087a 100644 --- a/apps/emqx_bridge_s3/src/emqx_bridge_s3_connector.erl +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3_connector.erl @@ -7,6 +7,7 @@ -include_lib("emqx/include/logger.hrl"). -include_lib("snabbkaffe/include/trace.hrl"). -include_lib("emqx_resource/include/emqx_resource.hrl"). +-include("emqx_bridge_s3.hrl"). -behaviour(emqx_resource). -export([ @@ -17,7 +18,7 @@ on_remove_channel/3, on_get_channels/1, on_query/3, - % on_batch_query/3, + on_batch_query/3, on_get_status/2, on_get_channel_status/3 ]). @@ -31,12 +32,31 @@ }. -type channel_config() :: #{ - parameters := #{ - bucket := string(), - key := string(), - content := string(), - acl => emqx_s3:acl() - } + bridge_type := binary(), + parameters := s3_upload_parameters() | s3_aggregated_upload_parameters() +}. + +-type s3_upload_parameters() :: #{ + bucket := string(), + key := string(), + content := string(), + acl => emqx_s3:acl() +}. + +-type s3_aggregated_upload_parameters() :: #{ + bucket := string(), + key := string(), + acl => emqx_s3:acl(), + aggregation => #{ + time_interval := emqx_schema:duration_s(), + max_records := pos_integer() + }, + container := #{ + type := csv, + column_order => [string()] + }, + min_part_size := emqx_schema:bytesize(), + max_part_size := emqx_schema:bytesize() }. -type channel_state() :: #{ @@ -123,12 +143,13 @@ on_get_status(_InstId, State = #{client_config := Config}) -> -spec on_add_channel(_InstanceId :: resource_id(), state(), channel_id(), channel_config()) -> {ok, state()} | {error, _Reason}. on_add_channel(_InstId, State = #{channels := Channels}, ChannelId, Config) -> - ChannelState = init_channel_state(Config), + ChannelState = start_channel(State, Config), {ok, State#{channels => Channels#{ChannelId => ChannelState}}}. -spec on_remove_channel(_InstanceId :: resource_id(), state(), channel_id()) -> {ok, state()}. on_remove_channel(_InstId, State = #{channels := Channels}, ChannelId) -> + ok = stop_channel(maps:get(ChannelId, Channels, undefined)), {ok, State#{channels => maps:remove(ChannelId, Channels)}}. -spec on_get_channels(_InstanceId :: resource_id()) -> @@ -138,27 +159,121 @@ on_get_channels(InstId) -> -spec on_get_channel_status(_InstanceId :: resource_id(), channel_id(), state()) -> channel_status(). -on_get_channel_status(_InstId, ChannelId, #{channels := Channels}) -> +on_get_channel_status(_InstId, ChannelId, State = #{channels := Channels}) -> case maps:get(ChannelId, Channels, undefined) of - _ChannelState = #{} -> - %% TODO - %% Since bucket name may be templated, we can't really provide any - %% additional information regarding the channel health. - ?status_connected; + ChannelState = #{} -> + channel_status(ChannelState, State); undefined -> ?status_disconnected end. -init_channel_state(#{parameters := Parameters}) -> +start_channel(_State, #{ + bridge_type := ?BRIDGE_TYPE_UPLOAD, + parameters := Parameters = #{ + bucket := Bucket, + key := Key, + content := Content + } +}) -> #{ - bucket => emqx_template:parse(maps:get(bucket, Parameters)), - key => emqx_template:parse(maps:get(key, Parameters)), - content => emqx_template:parse(maps:get(content, Parameters)), - upload_options => #{ - acl => maps:get(acl, Parameters, undefined) - } + type => ?ACTION_UPLOAD, + bucket => emqx_template:parse(Bucket), + key => emqx_template:parse(Key), + content => emqx_template:parse(Content), + upload_options => upload_options(Parameters) + }; +start_channel(State, #{ + bridge_type := Type = ?BRIDGE_TYPE_AGGREGATED_UPLOAD, + bridge_name := Name, + parameters := Parameters = #{ + aggregation := #{ + time_interval := TimeInterval, + max_records := MaxRecords + }, + container := Container, + bucket := Bucket + } +}) -> + AggregOpts = #{ + time_interval => TimeInterval, + max_records => MaxRecords, + work_dir => work_dir(Type, Name) + }, + DeliveryOpts = #{ + bucket => Bucket, + key => emqx_bridge_s3_aggreg_upload:mk_key_template(Parameters), + container => Container, + upload_options => emqx_bridge_s3_aggreg_upload:mk_upload_options(Parameters), + client_config => maps:get(client_config, State), + uploader_config => maps:with([min_part_size, max_part_size], Parameters) + }, + _ = emqx_bridge_s3_sup:delete_child({Type, Name}), + {ok, SupPid} = emqx_bridge_s3_sup:start_child(#{ + id => {Type, Name}, + start => {emqx_bridge_s3_aggreg_upload_sup, start_link, [Name, AggregOpts, DeliveryOpts]}, + type => supervisor, + restart => permanent + }), + #{ + type => ?ACTION_AGGREGATED_UPLOAD, + name => Name, + bucket => Bucket, + supervisor => SupPid, + on_stop => fun() -> emqx_bridge_s3_sup:delete_child({Type, Name}) end }. +upload_options(Parameters) -> + #{acl => maps:get(acl, Parameters, undefined)}. + +work_dir(Type, Name) -> + filename:join([emqx:data_dir(), bridge, Type, Name]). + +stop_channel(#{on_stop := OnStop}) -> + OnStop(); +stop_channel(_ChannelState) -> + ok. + +channel_status(#{type := ?ACTION_UPLOAD}, _State) -> + %% TODO + %% Since bucket name may be templated, we can't really provide any additional + %% information regarding the channel health. + ?status_connected; +channel_status(#{type := ?ACTION_AGGREGATED_UPLOAD, name := Name, bucket := Bucket}, State) -> + %% NOTE: This will effectively trigger uploads of buffers yet to be uploaded. + Timestamp = erlang:system_time(second), + ok = emqx_bridge_s3_aggregator:tick(Name, Timestamp), + ok = check_bucket_accessible(Bucket, State), + ok = check_aggreg_upload_errors(Name), + ?status_connected. + +check_bucket_accessible(Bucket, #{client_config := Config}) -> + case emqx_s3_client:aws_config(Config) of + {error, Reason} -> + throw({unhealthy_target, Reason}); + AWSConfig -> + try erlcloud_s3:list_objects(Bucket, [{max_keys, 1}], AWSConfig) of + Props when is_list(Props) -> + ok + catch + error:{aws_error, {http_error, 404, _, _Reason}} -> + throw({unhealthy_target, "Bucket does not exist"}); + error:{aws_error, {socket_error, Reason}} -> + throw({unhealthy_target, emqx_utils:format(Reason)}) + end + end. + +check_aggreg_upload_errors(Name) -> + case emqx_bridge_s3_aggregator:take_error(Name) of + [Error] -> + %% TODO + %% This approach means that, for example, 3 upload failures will cause + %% the channel to be marked as unhealthy for 3 consecutive health checks. + ErrorMessage = emqx_utils:format(Error), + throw({unhealthy_target, ErrorMessage}); + [] -> + ok + end. + %% Queries -type query() :: {_Tag :: channel_id(), _Data :: emqx_jsonish:t()}. @@ -167,14 +282,28 @@ init_channel_state(#{parameters := Parameters}) -> {ok, _Result} | {error, _Reason}. on_query(InstId, {Tag, Data}, #{client_config := Config, channels := Channels}) -> case maps:get(Tag, Channels, undefined) of - ChannelState = #{} -> - run_simple_upload(InstId, Data, ChannelState, Config); + ChannelState = #{type := ?ACTION_UPLOAD} -> + run_simple_upload(InstId, Tag, Data, ChannelState, Config); + ChannelState = #{type := ?ACTION_AGGREGATED_UPLOAD} -> + run_aggregated_upload(InstId, [Data], ChannelState); + undefined -> + {error, {unrecoverable_error, {invalid_message_tag, Tag}}} + end. + +-spec on_batch_query(_InstanceId :: resource_id(), [query()], state()) -> + {ok, _Result} | {error, _Reason}. +on_batch_query(InstId, [{Tag, Data0} | Rest], #{channels := Channels}) -> + case maps:get(Tag, Channels, undefined) of + ChannelState = #{type := ?ACTION_AGGREGATED_UPLOAD} -> + Records = [Data0 | [Data || {_, Data} <- Rest]], + run_aggregated_upload(InstId, Records, ChannelState); undefined -> {error, {unrecoverable_error, {invalid_message_tag, Tag}}} end. run_simple_upload( InstId, + ChannelID, Data, #{ bucket := BucketTemplate, @@ -188,6 +317,11 @@ run_simple_upload( Client = emqx_s3_client:create(Bucket, Config), Key = render_key(KeyTemplate, Data), Content = render_content(ContentTemplate, Data), + emqx_trace:rendered_action_template(ChannelID, #{ + bucket => Bucket, + key => Key, + content => Content + }), case emqx_s3_client:put_object(Client, Key, UploadOpts, Content) of ok -> ?tp(s3_bridge_connector_upload_ok, #{ @@ -200,6 +334,16 @@ run_simple_upload( {error, map_error(Reason)} end. +run_aggregated_upload(InstId, Records, #{name := Name}) -> + Timestamp = erlang:system_time(second), + case emqx_bridge_s3_aggregator:push_records(Name, Timestamp, Records) of + ok -> + ?tp(s3_bridge_aggreg_push_ok, #{instance_id => InstId, name => Name}), + ok; + {error, Reason} -> + {error, {unrecoverable_error, Reason}} + end. + map_error({socket_error, _} = Reason) -> {recoverable_error, Reason}; map_error(Reason = {aws_error, Status, _, _Body}) when Status >= 500 -> diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_connector_info.erl b/apps/emqx_bridge_s3/src/emqx_bridge_s3_connector_info.erl index be3c29bae..560334610 100644 --- a/apps/emqx_bridge_s3/src/emqx_bridge_s3_connector_info.erl +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3_connector_info.erl @@ -20,7 +20,7 @@ type_name() -> s3. bridge_types() -> - [s3]. + [s3, s3_aggregated_upload]. resource_callback_module() -> emqx_bridge_s3_connector. diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_sup.erl b/apps/emqx_bridge_s3/src/emqx_bridge_s3_sup.erl new file mode 100644 index 000000000..230711a76 --- /dev/null +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3_sup.erl @@ -0,0 +1,42 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_s3_sup). + +-export([ + start_link/0, + start_child/1, + delete_child/1 +]). + +-behaviour(supervisor). +-export([init/1]). + +-define(SUPREF, ?MODULE). + +%% + +start_link() -> + supervisor:start_link({local, ?SUPREF}, ?MODULE, root). + +start_child(ChildSpec) -> + supervisor:start_child(?SUPREF, ChildSpec). + +delete_child(ChildId) -> + case supervisor:terminate_child(?SUPREF, ChildId) of + ok -> + supervisor:delete_child(?SUPREF, ChildId); + Error -> + Error + end. + +%% + +init(root) -> + SupFlags = #{ + strategy => one_for_one, + intensity => 1, + period => 1 + }, + {ok, {SupFlags, []}}. diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_upload.erl b/apps/emqx_bridge_s3/src/emqx_bridge_s3_upload.erl new file mode 100644 index 000000000..44e2360b8 --- /dev/null +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3_upload.erl @@ -0,0 +1,143 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_s3_upload). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). +-include("emqx_bridge_s3.hrl"). + +-define(ACTION, ?ACTION_UPLOAD). + +-behaviour(hocon_schema). +-export([ + namespace/0, + roots/0, + fields/1, + desc/1 +]). + +-export([ + bridge_v2_examples/1 +]). + +%%------------------------------------------------------------------------------------------------- +%% `hocon_schema' API +%%------------------------------------------------------------------------------------------------- + +namespace() -> + "bridge_s3". + +roots() -> + []. + +fields(Field) when + Field == "get_bridge_v2"; + Field == "put_bridge_v2"; + Field == "post_bridge_v2" +-> + emqx_bridge_v2_schema:api_fields(Field, ?ACTION, fields(?ACTION)); +fields(action) -> + {?ACTION, + hoconsc:mk( + hoconsc:map(name, hoconsc:ref(?MODULE, ?ACTION)), + #{ + desc => <<"S3 Upload Action Config">>, + required => false + } + )}; +fields(?ACTION) -> + emqx_bridge_v2_schema:make_producer_action_schema( + hoconsc:mk( + ?R_REF(s3_upload_parameters), + #{ + required => true, + desc => ?DESC(s3_upload) + } + ), + #{ + resource_opts_ref => ?R_REF(s3_action_resource_opts) + } + ); +fields(s3_upload_parameters) -> + emqx_s3_schema:fields(s3_upload) ++ + [ + {content, + hoconsc:mk( + emqx_schema:template(), + #{ + required => false, + default => <<"${.}">>, + desc => ?DESC(s3_object_content) + } + )} + ]; +fields(s3_action_resource_opts) -> + UnsupportedOpts = [batch_size, batch_time], + lists:filter( + fun({N, _}) -> not lists:member(N, UnsupportedOpts) end, + emqx_bridge_v2_schema:action_resource_opts_fields() + ). + +desc(s3) -> + ?DESC(s3_upload); +desc(Name) when + Name == s3_upload; + Name == s3_upload_parameters +-> + ?DESC(Name); +desc(s3_action_resource_opts) -> + ?DESC(emqx_resource_schema, resource_opts); +desc(_Name) -> + undefined. + +%% Examples + +bridge_v2_examples(Method) -> + [ + #{ + <<"s3">> => #{ + summary => <<"S3 Simple Upload">>, + value => s3_upload_action_example(Method) + } + } + ]. + +s3_upload_action_example(post) -> + maps:merge( + s3_upload_action_example(put), + #{ + type => atom_to_binary(?ACTION_UPLOAD), + name => <<"my_s3_action">> + } + ); +s3_upload_action_example(get) -> + maps:merge( + s3_upload_action_example(put), + #{ + status => <<"connected">>, + node_status => [ + #{ + node => <<"emqx@localhost">>, + status => <<"connected">> + } + ] + } + ); +s3_upload_action_example(put) -> + #{ + enable => true, + connector => <<"my_s3_connector">>, + description => <<"My action">>, + parameters => #{ + bucket => <<"${clientid}">>, + key => <<"${topic}">>, + content => <<"${payload}">>, + acl => <<"public_read">> + }, + resource_opts => #{ + query_mode => <<"sync">>, + inflight_window => 10 + } + }. diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_action_info.erl b/apps/emqx_bridge_s3/src/emqx_bridge_s3_upload_action_info.erl similarity index 69% rename from apps/emqx_bridge_s3/src/emqx_bridge_s3_action_info.erl rename to apps/emqx_bridge_s3/src/emqx_bridge_s3_upload_action_info.erl index 646173bf4..15a76ae7c 100644 --- a/apps/emqx_bridge_s3/src/emqx_bridge_s3_action_info.erl +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3_upload_action_info.erl @@ -2,18 +2,20 @@ %% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_bridge_s3_action_info). +-module(emqx_bridge_s3_upload_action_info). -behaviour(emqx_action_info). +-include("emqx_bridge_s3.hrl"). + -export([ action_type_name/0, connector_type_name/0, schema_module/0 ]). -action_type_name() -> s3. +action_type_name() -> ?ACTION_UPLOAD. connector_type_name() -> s3. -schema_module() -> emqx_bridge_s3. +schema_module() -> emqx_bridge_s3_upload. diff --git a/apps/emqx_bridge_s3/test/emqx_bridge_s3_SUITE.erl b/apps/emqx_bridge_s3/test/emqx_bridge_s3_SUITE.erl index 738f9186f..322666b1f 100644 --- a/apps/emqx_bridge_s3/test/emqx_bridge_s3_SUITE.erl +++ b/apps/emqx_bridge_s3/test/emqx_bridge_s3_SUITE.erl @@ -11,8 +11,6 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("snabbkaffe/include/test_macros.hrl"). --import(emqx_utils_conv, [bin/1]). - %% See `emqx_bridge_s3.hrl`. -define(BRIDGE_TYPE, <<"s3">>). -define(CONNECTOR_TYPE, <<"s3">>). @@ -79,67 +77,56 @@ end_per_testcase(_TestCase, _Config) -> connector_config(Name, _Config) -> BaseConf = emqx_s3_test_helpers:base_raw_config(tcp), - parse_and_check_config(<<"connectors">>, ?CONNECTOR_TYPE, Name, #{ - <<"enable">> => true, - <<"description">> => <<"S3 Connector">>, - <<"host">> => emqx_utils_conv:bin(maps:get(<<"host">>, BaseConf)), - <<"port">> => maps:get(<<"port">>, BaseConf), - <<"access_key_id">> => maps:get(<<"access_key_id">>, BaseConf), - <<"secret_access_key">> => maps:get(<<"secret_access_key">>, BaseConf), - <<"transport_options">> => #{ - <<"headers">> => #{ - <<"content-type">> => <> + emqx_bridge_s3_test_helpers:parse_and_check_config( + <<"connectors">>, ?CONNECTOR_TYPE, Name, #{ + <<"enable">> => true, + <<"description">> => <<"S3 Connector">>, + <<"host">> => emqx_utils_conv:bin(maps:get(<<"host">>, BaseConf)), + <<"port">> => maps:get(<<"port">>, BaseConf), + <<"access_key_id">> => maps:get(<<"access_key_id">>, BaseConf), + <<"secret_access_key">> => maps:get(<<"secret_access_key">>, BaseConf), + <<"transport_options">> => #{ + <<"headers">> => #{ + <<"content-type">> => <> + }, + <<"connect_timeout">> => <<"500ms">>, + <<"request_timeout">> => <<"1s">>, + <<"pool_size">> => 4, + <<"max_retries">> => 0, + <<"enable_pipelining">> => 1 }, - <<"connect_timeout">> => <<"500ms">>, - <<"request_timeout">> => <<"1s">>, - <<"pool_size">> => 4, - <<"max_retries">> => 0, - <<"enable_pipelining">> => 1 - }, - <<"resource_opts">> => #{ - <<"health_check_interval">> => <<"5s">>, - <<"start_timeout">> => <<"5s">> + <<"resource_opts">> => #{ + <<"health_check_interval">> => <<"5s">>, + <<"start_timeout">> => <<"5s">> + } } - }). + ). action_config(Name, ConnectorId) -> - parse_and_check_config(<<"actions">>, ?BRIDGE_TYPE, Name, #{ - <<"enable">> => true, - <<"connector">> => ConnectorId, - <<"parameters">> => #{ - <<"bucket">> => <<"${clientid}">>, - <<"key">> => <<"${topic}">>, - <<"content">> => <<"${payload}">>, - <<"acl">> => <<"public_read">> - }, - <<"resource_opts">> => #{ - <<"buffer_mode">> => <<"memory_only">>, - <<"buffer_seg_bytes">> => <<"10MB">>, - <<"health_check_interval">> => <<"3s">>, - <<"inflight_window">> => 40, - <<"max_buffer_bytes">> => <<"256MB">>, - <<"metrics_flush_interval">> => <<"1s">>, - <<"query_mode">> => <<"sync">>, - <<"request_ttl">> => <<"60s">>, - <<"resume_interval">> => <<"3s">>, - <<"worker_pool_size">> => <<"4">> + emqx_bridge_s3_test_helpers:parse_and_check_config( + <<"actions">>, ?BRIDGE_TYPE, Name, #{ + <<"enable">> => true, + <<"connector">> => ConnectorId, + <<"parameters">> => #{ + <<"bucket">> => <<"${clientid}">>, + <<"key">> => <<"${topic}">>, + <<"content">> => <<"${payload}">>, + <<"acl">> => <<"public_read">> + }, + <<"resource_opts">> => #{ + <<"buffer_mode">> => <<"memory_only">>, + <<"buffer_seg_bytes">> => <<"10MB">>, + <<"health_check_interval">> => <<"3s">>, + <<"inflight_window">> => 40, + <<"max_buffer_bytes">> => <<"256MB">>, + <<"metrics_flush_interval">> => <<"1s">>, + <<"query_mode">> => <<"sync">>, + <<"request_ttl">> => <<"60s">>, + <<"resume_interval">> => <<"3s">>, + <<"worker_pool_size">> => <<"4">> + } } - }). - -parse_and_check_config(Root, Type, Name, ConfigIn) -> - Schema = - case Root of - <<"connectors">> -> emqx_connector_schema; - <<"actions">> -> emqx_bridge_v2_schema - end, - #{Root := #{Type := #{Name := Config}}} = - hocon_tconf:check_plain( - Schema, - #{Root => #{Type => #{Name => ConfigIn}}}, - #{required => false, atom_key => false} - ), - ct:pal("parsed config: ~p", [Config]), - ConfigIn. + ). t_start_stop(Config) -> emqx_bridge_v2_testlib:t_start_stop(Config, s3_bridge_stopped). @@ -190,7 +177,7 @@ t_sync_query(Config) -> ok = erlcloud_s3:create_bucket(Bucket, AwsConfig), ok = emqx_bridge_v2_testlib:t_sync_query( Config, - fun() -> mk_message(Bucket, Topic, Payload) end, + fun() -> emqx_bridge_s3_test_helpers:mk_message_event(Bucket, Topic, Payload) end, fun(Res) -> ?assertMatch(ok, Res) end, s3_bridge_connector_upload_ok ), @@ -224,15 +211,10 @@ t_query_retry_recoverable(Config) -> heal_failure, [timeout, ?PROXY_NAME, ProxyHost, ProxyPort] ), - Message = mk_message(Bucket, Topic, Payload), + Message = emqx_bridge_s3_test_helpers:mk_message_event(Bucket, Topic, Payload), %% Verify that the message is sent eventually. ok = emqx_bridge_v2:send_message(?BRIDGE_TYPE, BridgeName, Message, #{}), ?assertMatch( #{content := Payload}, maps:from_list(erlcloud_s3:get_object(Bucket, Topic, AwsConfig)) ). - -mk_message(ClientId, Topic, Payload) -> - Message = emqx_message:make(bin(ClientId), bin(Topic), Payload), - {Event, _} = emqx_rule_events:eventmsg_publish(Message), - Event. diff --git a/apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_buffer_SUITE.erl b/apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_buffer_SUITE.erl new file mode 100644 index 000000000..199e070d3 --- /dev/null +++ b/apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_buffer_SUITE.erl @@ -0,0 +1,181 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_s3_aggreg_buffer_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +%% CT Setup + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + WorkDir = emqx_cth_suite:work_dir(Config), + ok = filelib:ensure_path(WorkDir), + [{work_dir, WorkDir} | Config]. + +end_per_suite(_Config) -> + ok. + +%% Testcases + +t_write_read_cycle(Config) -> + Filename = mk_filename(?FUNCTION_NAME, Config), + Metadata = {?MODULE, #{tc => ?FUNCTION_NAME}}, + {ok, WFD} = file:open(Filename, [write, binary]), + Writer = emqx_bridge_s3_aggreg_buffer:new_writer(WFD, Metadata), + Terms = [ + [], + [[[[[[[[]]]]]]]], + 123, + lists:seq(1, 100), + lists:seq(1, 1000), + lists:seq(1, 10000), + lists:seq(1, 100000), + #{<<"id">> => 123456789, <<"ts">> => <<"2028-02-29T12:34:56Z">>, <<"gauge">> => 42.42}, + {<<"text/plain">>, _Huge = rand:bytes(1048576)}, + {<<"application/json">>, emqx_utils_json:encode(#{j => <<"son">>, null => null})} + ], + ok = lists:foreach( + fun(T) -> ?assertEqual(ok, emqx_bridge_s3_aggreg_buffer:write(T, Writer)) end, + Terms + ), + ok = file:close(WFD), + {ok, RFD} = file:open(Filename, [read, binary, raw]), + {MetadataRead, Reader} = emqx_bridge_s3_aggreg_buffer:new_reader(RFD), + ?assertEqual(Metadata, MetadataRead), + TermsRead = read_until_eof(Reader), + ?assertEqual(Terms, TermsRead). + +t_read_empty(Config) -> + Filename = mk_filename(?FUNCTION_NAME, Config), + {ok, WFD} = file:open(Filename, [write, binary]), + ok = file:close(WFD), + {ok, RFD} = file:open(Filename, [read, binary]), + ?assertError( + {buffer_incomplete, header}, + emqx_bridge_s3_aggreg_buffer:new_reader(RFD) + ). + +t_read_garbage(Config) -> + Filename = mk_filename(?FUNCTION_NAME, Config), + {ok, WFD} = file:open(Filename, [write, binary]), + ok = file:write(WFD, rand:bytes(1048576)), + ok = file:close(WFD), + {ok, RFD} = file:open(Filename, [read, binary]), + ?assertError( + badarg, + emqx_bridge_s3_aggreg_buffer:new_reader(RFD) + ). + +t_read_truncated(Config) -> + Filename = mk_filename(?FUNCTION_NAME, Config), + {ok, WFD} = file:open(Filename, [write, binary]), + Metadata = {?MODULE, #{tc => ?FUNCTION_NAME}}, + Writer = emqx_bridge_s3_aggreg_buffer:new_writer(WFD, Metadata), + Terms = [ + [[[[[[[[[[[]]]]]]]]]]], + lists:seq(1, 100000), + #{<<"id">> => 123456789, <<"ts">> => <<"2029-02-30T12:34:56Z">>, <<"gauge">> => 42.42}, + {<<"text/plain">>, _Huge = rand:bytes(1048576)} + ], + LastTerm = + {<<"application/json">>, emqx_utils_json:encode(#{j => <<"son">>, null => null})}, + ok = lists:foreach( + fun(T) -> ?assertEqual(ok, emqx_bridge_s3_aggreg_buffer:write(T, Writer)) end, + Terms + ), + {ok, WPos} = file:position(WFD, cur), + ?assertEqual(ok, emqx_bridge_s3_aggreg_buffer:write(LastTerm, Writer)), + ok = file:close(WFD), + ok = emqx_bridge_s3_test_helpers:truncate_at(Filename, WPos + 1), + {ok, RFD1} = file:open(Filename, [read, binary]), + {Metadata, Reader0} = emqx_bridge_s3_aggreg_buffer:new_reader(RFD1), + {ReadTerms1, Reader1} = read_terms(length(Terms), Reader0), + ?assertEqual(Terms, ReadTerms1), + ?assertError( + badarg, + emqx_bridge_s3_aggreg_buffer:read(Reader1) + ), + ok = emqx_bridge_s3_test_helpers:truncate_at(Filename, WPos div 2), + {ok, RFD2} = file:open(Filename, [read, binary]), + {Metadata, Reader2} = emqx_bridge_s3_aggreg_buffer:new_reader(RFD2), + {ReadTerms2, Reader3} = read_terms(_FitsInto = 3, Reader2), + ?assertEqual(lists:sublist(Terms, 3), ReadTerms2), + ?assertError( + badarg, + emqx_bridge_s3_aggreg_buffer:read(Reader3) + ). + +t_read_truncated_takeover_write(Config) -> + Filename = mk_filename(?FUNCTION_NAME, Config), + {ok, WFD} = file:open(Filename, [write, binary]), + Metadata = {?MODULE, #{tc => ?FUNCTION_NAME}}, + Writer1 = emqx_bridge_s3_aggreg_buffer:new_writer(WFD, Metadata), + Terms1 = [ + [[[[[[[[[[[]]]]]]]]]]], + lists:seq(1, 10000), + lists:duplicate(1000, ?FUNCTION_NAME), + {<<"text/plain">>, _Huge = rand:bytes(1048576)} + ], + Terms2 = [ + {<<"application/json">>, emqx_utils_json:encode(#{j => <<"son">>, null => null})}, + {<<"application/x-octet-stream">>, rand:bytes(102400)} + ], + ok = lists:foreach( + fun(T) -> ?assertEqual(ok, emqx_bridge_s3_aggreg_buffer:write(T, Writer1)) end, + Terms1 + ), + {ok, WPos} = file:position(WFD, cur), + ok = file:close(WFD), + ok = emqx_bridge_s3_test_helpers:truncate_at(Filename, WPos div 2), + {ok, RWFD} = file:open(Filename, [read, write, binary]), + {Metadata, Reader0} = emqx_bridge_s3_aggreg_buffer:new_reader(RWFD), + {ReadTerms1, Reader1} = read_terms(_Survived = 3, Reader0), + ?assertEqual( + lists:sublist(Terms1, 3), + ReadTerms1 + ), + ?assertError( + badarg, + emqx_bridge_s3_aggreg_buffer:read(Reader1) + ), + Writer2 = emqx_bridge_s3_aggreg_buffer:takeover(Reader1), + ok = lists:foreach( + fun(T) -> ?assertEqual(ok, emqx_bridge_s3_aggreg_buffer:write(T, Writer2)) end, + Terms2 + ), + ok = file:close(RWFD), + {ok, RFD} = file:open(Filename, [read, binary]), + {Metadata, Reader2} = emqx_bridge_s3_aggreg_buffer:new_reader(RFD), + ReadTerms2 = read_until_eof(Reader2), + ?assertEqual( + lists:sublist(Terms1, 3) ++ Terms2, + ReadTerms2 + ). + +%% + +mk_filename(Name, Config) -> + filename:join(?config(work_dir, Config), Name). + +read_terms(0, Reader) -> + {[], Reader}; +read_terms(N, Reader0) -> + {Term, Reader1} = emqx_bridge_s3_aggreg_buffer:read(Reader0), + {Terms, Reader} = read_terms(N - 1, Reader1), + {[Term | Terms], Reader}. + +read_until_eof(Reader0) -> + case emqx_bridge_s3_aggreg_buffer:read(Reader0) of + {Term, Reader} -> + [Term | read_until_eof(Reader)]; + eof -> + [] + end. diff --git a/apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_csv_tests.erl b/apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_csv_tests.erl new file mode 100644 index 000000000..6da70c6fe --- /dev/null +++ b/apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_csv_tests.erl @@ -0,0 +1,72 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_s3_aggreg_csv_tests). + +-include_lib("eunit/include/eunit.hrl"). + +encoding_test() -> + CSV = emqx_bridge_s3_aggreg_csv:new(#{}), + ?assertEqual( + "A,B,Ç\n" + "1.2345,string,0.0\n" + "0.3333333333,\"[]\",-0.0\n" + "111111,🫠,0.0\n" + "111.111,\"\"\"quoted\"\"\",\"line\r\nbreak\"\n" + "222.222,,\n", + fill_close(CSV, [ + [ + #{<<"A">> => 1.2345, <<"B">> => "string", <<"Ç"/utf8>> => +0.0}, + #{<<"A">> => 1 / 3, <<"B">> => "[]", <<"Ç"/utf8>> => -0.0}, + #{<<"A">> => 111111, <<"B">> => "🫠", <<"Ç"/utf8>> => 0.0}, + #{<<"A">> => 111.111, <<"B">> => "\"quoted\"", <<"Ç"/utf8>> => "line\r\nbreak"}, + #{<<"A">> => 222.222, <<"B">> => "", <<"Ç"/utf8>> => undefined} + ] + ]) + ). + +column_order_test() -> + Order = [<<"ID">>, <<"TS">>], + CSV = emqx_bridge_s3_aggreg_csv:new(#{column_order => Order}), + ?assertEqual( + "ID,TS,A,B,D\n" + "1,2024-01-01,12.34,str,\"[]\"\n" + "2,2024-01-02,23.45,ing,\n" + "3,,45,,'\n" + "4,2024-01-04,,,\n", + fill_close(CSV, [ + [ + #{ + <<"A">> => 12.34, + <<"B">> => "str", + <<"ID">> => 1, + <<"TS">> => "2024-01-01", + <<"D">> => <<"[]">> + }, + #{ + <<"TS">> => "2024-01-02", + <<"C">> => <<"null">>, + <<"ID">> => 2, + <<"A">> => 23.45, + <<"B">> => "ing" + } + ], + [ + #{<<"A">> => 45, <<"D">> => <<"'">>, <<"ID">> => 3}, + #{<<"ID">> => 4, <<"TS">> => "2024-01-04"} + ] + ]) + ). + +fill_close(CSV, LRecords) -> + string(fill_close_(CSV, LRecords)). + +fill_close_(CSV0, [Records | LRest]) -> + {Writes, CSV} = emqx_bridge_s3_aggreg_csv:fill(Records, CSV0), + [Writes | fill_close_(CSV, LRest)]; +fill_close_(CSV, []) -> + [emqx_bridge_s3_aggreg_csv:close(CSV)]. + +string(Writes) -> + unicode:characters_to_list(Writes). diff --git a/apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_upload_SUITE.erl b/apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_upload_SUITE.erl new file mode 100644 index 000000000..6577b45ed --- /dev/null +++ b/apps/emqx_bridge_s3/test/emqx_bridge_s3_aggreg_upload_SUITE.erl @@ -0,0 +1,465 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_s3_aggreg_upload_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/test_macros.hrl"). + +-import(emqx_utils_conv, [bin/1]). + +%% See `emqx_bridge_s3.hrl`. +-define(BRIDGE_TYPE, <<"s3_aggregated_upload">>). +-define(CONNECTOR_TYPE, <<"s3">>). + +-define(PROXY_NAME, "minio_tcp"). + +-define(CONF_TIME_INTERVAL, 4000). +-define(CONF_MAX_RECORDS, 100). +-define(CONF_COLUMN_ORDER, ?CONF_COLUMN_ORDER([])). +-define(CONF_COLUMN_ORDER(T), [ + <<"publish_received_at">>, + <<"clientid">>, + <<"topic">>, + <<"payload">>, + <<"empty">> + | T +]). + +-define(LIMIT_TOLERANCE, 1.1). + +%% CT Setup + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + % Setup toxiproxy + ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"), + ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")), + _ = emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), + Apps = emqx_cth_suite:start( + [ + emqx, + emqx_conf, + emqx_connector, + emqx_bridge_s3, + emqx_bridge, + emqx_rule_engine, + emqx_management, + {emqx_dashboard, "dashboard.listeners.http { enable = true, bind = 18083 }"} + ], + #{work_dir => emqx_cth_suite:work_dir(Config)} + ), + {ok, _} = emqx_common_test_http:create_default_app(), + [ + {apps, Apps}, + {proxy_host, ProxyHost}, + {proxy_port, ProxyPort}, + {proxy_name, ?PROXY_NAME} + | Config + ]. + +end_per_suite(Config) -> + ok = emqx_cth_suite:stop(?config(apps, Config)). + +%% Testcases + +init_per_testcase(TestCase, Config) -> + ct:timetrap(timer:seconds(15)), + ok = snabbkaffe:start_trace(), + TS = erlang:system_time(), + Name = iolist_to_binary(io_lib:format("~s-~p", [TestCase, TS])), + Bucket = unicode:characters_to_list(string:replace(Name, "_", "-", all)), + ConnectorConfig = connector_config(Name, Config), + ActionConfig = action_config(Name, Name, Bucket), + ok = emqx_bridge_s3_test_helpers:create_bucket(Bucket), + [ + {connector_type, ?CONNECTOR_TYPE}, + {connector_name, Name}, + {connector_config, ConnectorConfig}, + {bridge_type, ?BRIDGE_TYPE}, + {bridge_name, Name}, + {bridge_config, ActionConfig}, + {s3_bucket, Bucket} + | Config + ]. + +end_per_testcase(_TestCase, _Config) -> + ok = snabbkaffe:stop(), + ok. + +connector_config(Name, _Config) -> + BaseConf = emqx_s3_test_helpers:base_raw_config(tcp), + emqx_bridge_s3_test_helpers:parse_and_check_config( + <<"connectors">>, ?CONNECTOR_TYPE, Name, #{ + <<"enable">> => true, + <<"description">> => <<"S3 Connector">>, + <<"host">> => emqx_utils_conv:bin(maps:get(<<"host">>, BaseConf)), + <<"port">> => maps:get(<<"port">>, BaseConf), + <<"access_key_id">> => maps:get(<<"access_key_id">>, BaseConf), + <<"secret_access_key">> => maps:get(<<"secret_access_key">>, BaseConf), + <<"transport_options">> => #{ + <<"connect_timeout">> => <<"500ms">>, + <<"request_timeout">> => <<"1s">>, + <<"pool_size">> => 4, + <<"max_retries">> => 0 + }, + <<"resource_opts">> => #{ + <<"health_check_interval">> => <<"1s">> + } + } + ). + +action_config(Name, ConnectorId, Bucket) -> + emqx_bridge_s3_test_helpers:parse_and_check_config( + <<"actions">>, ?BRIDGE_TYPE, Name, #{ + <<"enable">> => true, + <<"connector">> => ConnectorId, + <<"parameters">> => #{ + <<"bucket">> => unicode:characters_to_binary(Bucket), + <<"key">> => <<"${action}/${node}/${datetime.rfc3339}">>, + <<"acl">> => <<"public_read">>, + <<"headers">> => #{ + <<"X-AMZ-Meta-Version">> => <<"42">> + }, + <<"aggregation">> => #{ + <<"time_interval">> => <<"4s">>, + <<"max_records">> => ?CONF_MAX_RECORDS + }, + <<"container">> => #{ + <<"type">> => <<"csv">>, + <<"column_order">> => ?CONF_COLUMN_ORDER + } + }, + <<"resource_opts">> => #{ + <<"health_check_interval">> => <<"1s">>, + <<"max_buffer_bytes">> => <<"64MB">>, + <<"query_mode">> => <<"async">>, + <<"worker_pool_size">> => 4 + } + } + ). + +t_start_stop(Config) -> + emqx_bridge_v2_testlib:t_start_stop(Config, s3_bridge_stopped). + +t_create_via_http(Config) -> + emqx_bridge_v2_testlib:t_create_via_http(Config). + +t_on_get_status(Config) -> + emqx_bridge_v2_testlib:t_on_get_status(Config, #{}). + +t_aggreg_upload(Config) -> + Bucket = ?config(s3_bucket, Config), + BridgeName = ?config(bridge_name, Config), + BridgeNameString = unicode:characters_to_list(BridgeName), + NodeString = atom_to_list(node()), + %% Create a bridge with the sample configuration. + ?assertMatch({ok, _Bridge}, emqx_bridge_v2_testlib:create_bridge(Config)), + %% Prepare some sample messages that look like Rule SQL productions. + MessageEvents = lists:map(fun mk_message_event/1, [ + {<<"C1">>, T1 = <<"a/b/c">>, P1 = <<"{\"hello\":\"world\"}">>}, + {<<"C2">>, T2 = <<"foo/bar">>, P2 = <<"baz">>}, + {<<"C3">>, T3 = <<"t/42">>, P3 = <<"">>} + ]), + ok = send_messages(BridgeName, MessageEvents), + %% Wait until the delivery is completed. + ?block_until(#{?snk_kind := s3_aggreg_delivery_completed, action := BridgeName}), + %% Check the uploaded objects. + _Uploads = [#{key := Key}] = emqx_bridge_s3_test_helpers:list_objects(Bucket), + ?assertMatch( + [BridgeNameString, NodeString, _Datetime, _Seq = "0"], + string:split(Key, "/", all) + ), + Upload = #{content := Content} = emqx_bridge_s3_test_helpers:get_object(Bucket, Key), + ?assertMatch( + #{content_type := "text/csv", "x-amz-meta-version" := "42"}, + Upload + ), + %% Verify that column order is respected. + ?assertMatch( + {ok, [ + ?CONF_COLUMN_ORDER(_), + [TS, <<"C1">>, T1, P1, <<>> | _], + [TS, <<"C2">>, T2, P2, <<>> | _], + [TS, <<"C3">>, T3, P3, <<>> | _] + ]}, + erl_csv:decode(Content) + ). + +t_aggreg_upload_rule(Config) -> + Bucket = ?config(s3_bucket, Config), + BridgeName = ?config(bridge_name, Config), + ClientID = emqx_utils_conv:bin(?FUNCTION_NAME), + %% Create a bridge with the sample configuration and a simple SQL rule. + ?assertMatch({ok, _Bridge}, emqx_bridge_v2_testlib:create_bridge(Config)), + ?assertMatch( + {ok, _Rule}, + emqx_bridge_v2_testlib:create_rule_and_action_http(?BRIDGE_TYPE, <<>>, Config, #{ + sql => << + "SELECT" + " *," + " strlen(payload) as psize," + " unix_ts_to_rfc3339(publish_received_at, 'millisecond') as publish_received_at" + " FROM 's3/#'" + >> + }) + ), + ok = lists:foreach(fun emqx:publish/1, [ + emqx_message:make(?FUNCTION_NAME, T1 = <<"s3/m1">>, P1 = <<"[HELLO]">>), + emqx_message:make(?FUNCTION_NAME, T2 = <<"s3/m2">>, P2 = <<"[WORLD]">>), + emqx_message:make(?FUNCTION_NAME, T3 = <<"s3/empty">>, P3 = <<>>), + emqx_message:make(?FUNCTION_NAME, <<"not/s3">>, <<"should not be here">>) + ]), + ?block_until(#{?snk_kind := s3_aggreg_delivery_completed, action := BridgeName}), + %% Check the uploaded objects. + _Uploads = [#{key := Key}] = emqx_bridge_s3_test_helpers:list_objects(Bucket), + _CSV = [Header | Rows] = fetch_parse_csv(Bucket, Key), + %% Verify that column order is respected and event fields are preserved. + ?assertMatch(?CONF_COLUMN_ORDER(_), Header), + ?assertEqual( + [<<"event">>, <<"qos">>, <<"psize">>], + [C || C <- [<<"event">>, <<"qos">>, <<"psize">>], lists:member(C, Header)] + ), + %% Verify that all the matching messages are present. + ?assertMatch( + [ + [_TS1, ClientID, T1, P1 | _], + [_TS2, ClientID, T2, P2 | _], + [_TS3, ClientID, T3, P3 | _] + ], + Rows + ), + %% Verify that timestamp column now has RFC3339 format. + [_Row = [TS1 | _] | _Rest] = Rows, + ?assert( + is_integer(emqx_rule_funcs:rfc3339_to_unix_ts(TS1, millisecond)), + TS1 + ). + +t_aggreg_upload_restart(Config) -> + %% NOTE + %% This test verifies that the bridge will reuse existing aggregation buffer + %% after a restart. + Bucket = ?config(s3_bucket, Config), + BridgeName = ?config(bridge_name, Config), + %% Create a bridge with the sample configuration. + ?assertMatch({ok, _Bridge}, emqx_bridge_v2_testlib:create_bridge(Config)), + %% Send some sample messages that look like Rule SQL productions. + MessageEvents = lists:map(fun mk_message_event/1, [ + {<<"C1">>, T1 = <<"a/b/c">>, P1 = <<"{\"hello\":\"world\"}">>}, + {<<"C2">>, T2 = <<"foo/bar">>, P2 = <<"baz">>}, + {<<"C3">>, T3 = <<"t/42">>, P3 = <<"">>} + ]), + ok = send_messages(BridgeName, MessageEvents), + {ok, _} = ?block_until(#{?snk_kind := s3_aggreg_records_written, action := BridgeName}), + %% Restart the bridge. + {ok, _} = emqx_bridge_v2:disable_enable(disable, ?BRIDGE_TYPE, BridgeName), + {ok, _} = emqx_bridge_v2:disable_enable(enable, ?BRIDGE_TYPE, BridgeName), + %% Send some more messages. + ok = send_messages(BridgeName, MessageEvents), + {ok, _} = ?block_until(#{?snk_kind := s3_aggreg_records_written, action := BridgeName}), + %% Wait until the delivery is completed. + {ok, _} = ?block_until(#{?snk_kind := s3_aggreg_delivery_completed, action := BridgeName}), + %% Check there's still only one upload. + _Uploads = [#{key := Key}] = emqx_bridge_s3_test_helpers:list_objects(Bucket), + _Upload = #{content := Content} = emqx_bridge_s3_test_helpers:get_object(Bucket, Key), + ?assertMatch( + {ok, [ + _Header = [_ | _], + [TS1, <<"C1">>, T1, P1 | _], + [TS1, <<"C2">>, T2, P2 | _], + [TS1, <<"C3">>, T3, P3 | _], + [TS2, <<"C1">>, T1, P1 | _], + [TS2, <<"C2">>, T2, P2 | _], + [TS2, <<"C3">>, T3, P3 | _] + ]}, + erl_csv:decode(Content) + ). + +t_aggreg_upload_restart_corrupted(Config) -> + %% NOTE + %% This test verifies that the bridge can recover from a buffer file corruption, + %% and does so while preserving uncompromised data. + Bucket = ?config(s3_bucket, Config), + BridgeName = ?config(bridge_name, Config), + BatchSize = ?CONF_MAX_RECORDS div 2, + %% Create a bridge with the sample configuration. + ?assertMatch({ok, _Bridge}, emqx_bridge_v2_testlib:create_bridge(Config)), + %% Send some sample messages that look like Rule SQL productions. + Messages1 = [ + {integer_to_binary(N), <<"a/b/c">>, <<"{\"hello\":\"world\"}">>} + || N <- lists:seq(1, BatchSize) + ], + %% Ensure that they span multiple batch queries. + ok = send_messages_delayed(BridgeName, lists:map(fun mk_message_event/1, Messages1), 1), + {ok, _} = ?block_until( + #{?snk_kind := s3_aggreg_records_written, action := BridgeName}, + infinity, + 0 + ), + %% Find out the buffer file. + {ok, #{filename := Filename}} = ?block_until( + #{?snk_kind := s3_aggreg_buffer_allocated, action := BridgeName} + ), + %% Stop the bridge, corrupt the buffer file, and restart the bridge. + {ok, _} = emqx_bridge_v2:disable_enable(disable, ?BRIDGE_TYPE, BridgeName), + BufferFileSize = filelib:file_size(Filename), + ok = emqx_bridge_s3_test_helpers:truncate_at(Filename, BufferFileSize div 2), + {ok, _} = emqx_bridge_v2:disable_enable(enable, ?BRIDGE_TYPE, BridgeName), + %% Send some more messages. + Messages2 = [ + {integer_to_binary(N), <<"c/d/e">>, <<"{\"hello\":\"world\"}">>} + || N <- lists:seq(1, BatchSize) + ], + ok = send_messages_delayed(BridgeName, lists:map(fun mk_message_event/1, Messages2), 0), + %% Wait until the delivery is completed. + {ok, _} = ?block_until(#{?snk_kind := s3_aggreg_delivery_completed, action := BridgeName}), + %% Check that upload contains part of the first batch and all of the second batch. + _Uploads = [#{key := Key}] = emqx_bridge_s3_test_helpers:list_objects(Bucket), + CSV = [_Header | Rows] = fetch_parse_csv(Bucket, Key), + NRows = length(Rows), + ?assert( + NRows > BatchSize, + CSV + ), + ?assertEqual( + lists:sublist(Messages1, NRows - BatchSize) ++ Messages2, + [{ClientID, Topic, Payload} || [_TS, ClientID, Topic, Payload | _] <- Rows], + CSV + ). + +t_aggreg_pending_upload_restart(Config) -> + %% NOTE + %% This test verifies that the bridge will finish uploading a buffer file after + %% a restart. + Bucket = ?config(s3_bucket, Config), + BridgeName = ?config(bridge_name, Config), + %% Create a bridge with the sample configuration. + ?assertMatch({ok, _Bridge}, emqx_bridge_v2_testlib:create_bridge(Config)), + %% Send few large messages that will require multipart upload. + %% Ensure that they span multiple batch queries. + Payload = iolist_to_binary(lists:duplicate(128 * 1024, "PAYLOAD!")), + Messages = [{integer_to_binary(N), <<"a/b/c">>, Payload} || N <- lists:seq(1, 10)], + ok = send_messages_delayed(BridgeName, lists:map(fun mk_message_event/1, Messages), 10), + %% Wait until the multipart upload is started. + {ok, #{key := ObjectKey}} = + ?block_until(#{?snk_kind := s3_client_multipart_started, bucket := Bucket}), + %% Stop the bridge. + {ok, _} = emqx_bridge_v2:disable_enable(disable, ?BRIDGE_TYPE, BridgeName), + %% Verify that pending uploads have been gracefully aborted. + %% NOTE: Minio does not support multipart upload listing w/o prefix. + ?assertEqual( + [], + emqx_bridge_s3_test_helpers:list_pending_uploads(Bucket, ObjectKey) + ), + %% Restart the bridge. + {ok, _} = emqx_bridge_v2:disable_enable(enable, ?BRIDGE_TYPE, BridgeName), + %% Wait until the delivery is completed. + {ok, _} = ?block_until(#{?snk_kind := s3_aggreg_delivery_completed, action := BridgeName}), + %% Check that delivery contains all the messages. + _Uploads = [#{key := Key}] = emqx_bridge_s3_test_helpers:list_objects(Bucket), + [_Header | Rows] = fetch_parse_csv(Bucket, Key), + ?assertEqual( + Messages, + [{CID, Topic, PL} || [_TS, CID, Topic, PL | _] <- Rows] + ). + +t_aggreg_next_rotate(Config) -> + %% NOTE + %% This is essentially a stress test that tries to verify that buffer rotation + %% and windowing work correctly under high rate, high concurrency conditions. + Bucket = ?config(s3_bucket, Config), + BridgeName = ?config(bridge_name, Config), + NSenders = 4, + %% Create a bridge with the sample configuration. + ?assertMatch({ok, _Bridge}, emqx_bridge_v2_testlib:create_bridge(Config)), + %% Start separate processes to send messages. + Senders = [ + spawn_link(fun() -> run_message_sender(BridgeName, N) end) + || N <- lists:seq(1, NSenders) + ], + %% Give them some time to send messages so that rotation and windowing will happen. + ok = timer:sleep(round(?CONF_TIME_INTERVAL * 1.5)), + %% Stop the senders. + _ = [Sender ! {stop, self()} || Sender <- Senders], + NSent = receive_sender_reports(Senders), + %% Wait for the last delivery to complete. + ok = timer:sleep(round(?CONF_TIME_INTERVAL * 0.5)), + ?block_until(#{?snk_kind := s3_aggreg_delivery_completed, action := BridgeName}, infinity, 0), + %% There should be at least 2 time windows of aggregated records. + Uploads = [K || #{key := K} <- emqx_bridge_s3_test_helpers:list_objects(Bucket)], + DTs = [DT || K <- Uploads, [_Action, _Node, DT | _] <- [string:split(K, "/", all)]], + ?assert( + ordsets:size(ordsets:from_list(DTs)) > 1, + Uploads + ), + %% Uploads should not contain more than max allowed records. + CSVs = [{K, fetch_parse_csv(Bucket, K)} || K <- Uploads], + NRecords = [{K, length(CSV) - 1} || {K, CSV} <- CSVs], + ?assertEqual( + [], + [{K, NR} || {K, NR} <- NRecords, NR > ?CONF_MAX_RECORDS * ?LIMIT_TOLERANCE] + ), + %% No message should be lost. + ?assertEqual( + NSent, + lists:sum([NR || {_, NR} <- NRecords]) + ). + +run_message_sender(BridgeName, N) -> + ClientID = integer_to_binary(N), + Topic = <<"a/b/c/", ClientID/binary>>, + run_message_sender(BridgeName, N, ClientID, Topic, N, 0). + +run_message_sender(BridgeName, N, ClientID, Topic, Delay, NSent) -> + Payload = integer_to_binary(N * 1_000_000 + NSent), + Message = emqx_bridge_s3_test_helpers:mk_message_event(ClientID, Topic, Payload), + _ = send_message(BridgeName, Message), + receive + {stop, From} -> + From ! {sent, self(), NSent + 1} + after Delay -> + run_message_sender(BridgeName, N, ClientID, Topic, Delay, NSent + 1) + end. + +receive_sender_reports([Sender | Rest]) -> + receive + {sent, Sender, NSent} -> NSent + receive_sender_reports(Rest) + end; +receive_sender_reports([]) -> + 0. + +%% + +mk_message_event({ClientID, Topic, Payload}) -> + emqx_bridge_s3_test_helpers:mk_message_event(ClientID, Topic, Payload). + +send_messages(BridgeName, MessageEvents) -> + lists:foreach( + fun(M) -> send_message(BridgeName, M) end, + MessageEvents + ). + +send_messages_delayed(BridgeName, MessageEvents, Delay) -> + lists:foreach( + fun(M) -> + send_message(BridgeName, M), + timer:sleep(Delay) + end, + MessageEvents + ). + +send_message(BridgeName, Message) -> + ?assertEqual(ok, emqx_bridge_v2:send_message(?BRIDGE_TYPE, BridgeName, Message, #{})). + +fetch_parse_csv(Bucket, Key) -> + #{content := Content} = emqx_bridge_s3_test_helpers:get_object(Bucket, Key), + {ok, CSV} = erl_csv:decode(Content), + CSV. diff --git a/apps/emqx_bridge_s3/test/emqx_bridge_s3_test_helpers.erl b/apps/emqx_bridge_s3/test/emqx_bridge_s3_test_helpers.erl new file mode 100644 index 000000000..21729369b --- /dev/null +++ b/apps/emqx_bridge_s3/test/emqx_bridge_s3_test_helpers.erl @@ -0,0 +1,58 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_s3_test_helpers). + +-compile(nowarn_export_all). +-compile(export_all). + +-import(emqx_utils_conv, [bin/1]). + +parse_and_check_config(Root, Type, Name, Config) -> + Schema = + case Root of + <<"connectors">> -> emqx_connector_schema; + <<"actions">> -> emqx_bridge_v2_schema + end, + #{Root := #{Type := #{Name := _ConfigParsed}}} = + hocon_tconf:check_plain( + Schema, + #{Root => #{Type => #{Name => Config}}}, + #{required => false, atom_key => false} + ), + Config. + +mk_message_event(ClientId, Topic, Payload) -> + Message = emqx_message:make(bin(ClientId), bin(Topic), Payload), + {Event, _} = emqx_rule_events:eventmsg_publish(Message), + emqx_utils_maps:binary_key_map(Event). + +create_bucket(Bucket) -> + AwsConfig = emqx_s3_test_helpers:aws_config(tcp), + erlcloud_s3:create_bucket(Bucket, AwsConfig). + +list_objects(Bucket) -> + AwsConfig = emqx_s3_test_helpers:aws_config(tcp), + Response = erlcloud_s3:list_objects(Bucket, AwsConfig), + false = proplists:get_value(is_truncated, Response), + Contents = proplists:get_value(contents, Response), + lists:map(fun maps:from_list/1, Contents). + +get_object(Bucket, Key) -> + AwsConfig = emqx_s3_test_helpers:aws_config(tcp), + maps:from_list(erlcloud_s3:get_object(Bucket, Key, AwsConfig)). + +list_pending_uploads(Bucket, Key) -> + AwsConfig = emqx_s3_test_helpers:aws_config(tcp), + {ok, Props} = erlcloud_s3:list_multipart_uploads(Bucket, [{prefix, Key}], [], AwsConfig), + Uploads = proplists:get_value(uploads, Props), + lists:map(fun maps:from_list/1, Uploads). + +%% File utilities + +truncate_at(Filename, Pos) -> + {ok, FD} = file:open(Filename, [read, write, binary]), + {ok, Pos} = file:position(FD, Pos), + ok = file:truncate(FD), + ok = file:close(FD). diff --git a/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl b/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl index 1eb9746dc..683551316 100644 --- a/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl +++ b/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl @@ -413,6 +413,9 @@ do_query( %% only insert sql statement for single query and batch query case apply_template(QueryTuple, Templates) of {?ACTION_SEND_MESSAGE, SQL} -> + emqx_trace:rendered_action_template(ChannelId, #{ + sql => SQL + }), Result = ecpool:pick_and_do( PoolName, {?MODULE, worker_do_insert, [SQL, State]}, diff --git a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_connector.erl b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_connector.erl index 045af348f..a6d47229c 100644 --- a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_connector.erl +++ b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_connector.erl @@ -273,6 +273,8 @@ do_query( Result = case try_render_message(Query, Channels) of {ok, Msg} -> + [{ChannelID, _} | _] = Query, + emqx_trace:rendered_action_template(ChannelID, #{message => Msg}), ecpool:pick_and_do( PoolName, {emqx_bridge_syskeeper_client, forward, [Msg, AckTimeout + ?EXTRA_CALL_TIMEOUT]}, diff --git a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl index 7bb342ed1..67b0e77bc 100644 --- a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl +++ b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl @@ -10,6 +10,7 @@ -include_lib("hocon/include/hoconsc.hrl"). -include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/emqx_trace.hrl"). -include_lib("typerefl/include/types.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -include_lib("emqx_resource/include/emqx_resource.hrl"). @@ -32,7 +33,7 @@ -export([connector_examples/1]). --export([connect/1, do_get_status/1, execute/3, do_batch_insert/4]). +-export([connect/1, do_get_status/1, execute/3, do_batch_insert/5]). -import(hoconsc, [mk/2, enum/1, ref/2]). @@ -186,6 +187,7 @@ on_query(InstanceId, {ChannelId, Data}, #{channels := Channels} = State) -> case maps:find(ChannelId, Channels) of {ok, #{insert := Tokens, opts := Opts}} -> Query = emqx_placeholder:proc_tmpl(Tokens, Data), + emqx_trace:rendered_action_template(ChannelId, #{query => Query}), do_query_job(InstanceId, {?MODULE, execute, [Query, Opts]}, State); _ -> {error, {unrecoverable_error, {invalid_channel_id, InstanceId}}} @@ -199,9 +201,10 @@ on_batch_query( ) -> case maps:find(ChannelId, Channels) of {ok, #{batch := Tokens, opts := Opts}} -> + TraceRenderedCTX = emqx_trace:make_rendered_action_template_trace_context(ChannelId), do_query_job( InstanceId, - {?MODULE, do_batch_insert, [Tokens, BatchReq, Opts]}, + {?MODULE, do_batch_insert, [Tokens, BatchReq, Opts, TraceRenderedCTX]}, State ); _ -> @@ -338,9 +341,18 @@ do_query_job(InstanceId, Job, #{pool_name := PoolName} = State) -> execute(Conn, Query, Opts) -> tdengine:insert(Conn, Query, Opts). -do_batch_insert(Conn, Tokens, BatchReqs, Opts) -> +do_batch_insert(Conn, Tokens, BatchReqs, Opts, TraceRenderedCTX) -> SQL = aggregate_query(Tokens, BatchReqs, <<"INSERT INTO">>), - execute(Conn, SQL, Opts). + try + emqx_trace:rendered_action_template_with_ctx( + TraceRenderedCTX, + #{query => SQL} + ), + execute(Conn, SQL, Opts) + catch + error:?EMQX_TRACE_STOP_ACTION_MATCH = Reason -> + {error, Reason} + end. aggregate_query(BatchTks, BatchReqs, Acc) -> lists:foldl( diff --git a/apps/emqx_conf/src/emqx_conf_cli.erl b/apps/emqx_conf/src/emqx_conf_cli.erl index d6462a0b6..f1909e59b 100644 --- a/apps/emqx_conf/src/emqx_conf_cli.erl +++ b/apps/emqx_conf/src/emqx_conf_cli.erl @@ -242,7 +242,7 @@ load_config(Bin, Opts) when is_binary(Bin) -> load_config_from_raw(RawConf0, Opts) -> SchemaMod = emqx_conf:schema_module(), RawConf1 = emqx_config:upgrade_raw_conf(SchemaMod, RawConf0), - case check_config(RawConf1) of + case check_config(RawConf1, Opts) of {ok, RawConf} -> %% It has been ensured that the connector is always the first configuration to be updated. %% However, when deleting the connector, we need to clean up the dependent actions/sources first; @@ -395,24 +395,28 @@ suggest_msg(#{kind := validation_error, reason := unknown_fields}, Mode) -> suggest_msg(_, _) -> <<"">>. -check_config(Conf) -> - case check_keys_is_not_readonly(Conf) of - ok -> - Conf1 = emqx_config:fill_defaults(Conf), - case check_config_schema(Conf1) of - ok -> {ok, Conf1}; +check_config(Conf0, Opts) -> + case check_keys_is_not_readonly(Conf0, Opts) of + {ok, Conf1} -> + Conf = emqx_config:fill_defaults(Conf1), + case check_config_schema(Conf) of + ok -> {ok, Conf}; {error, Reason} -> {error, Reason} end; Error -> Error end. -check_keys_is_not_readonly(Conf) -> +check_keys_is_not_readonly(Conf, Opts) -> + IgnoreReadonly = maps:get(ignore_readonly, Opts, false), Keys = maps:keys(Conf), ReadOnlyKeys = [atom_to_binary(K) || K <- ?READONLY_KEYS], case lists:filter(fun(K) -> lists:member(K, Keys) end, ReadOnlyKeys) of [] -> - ok; + {ok, Conf}; + BadKeys when IgnoreReadonly -> + ?SLOG(info, #{msg => "readonly_root_keys_ignored", keys => BadKeys}), + {ok, maps:without(BadKeys, Conf)}; BadKeys -> BadKeysStr = lists:join(<<",">>, BadKeys), {error, ?UPDATE_READONLY_KEYS_PROHIBITED, BadKeysStr} diff --git a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_saml.erl b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_saml.erl index 33c4059df..eaa550d64 100644 --- a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_saml.erl +++ b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_saml.erl @@ -72,7 +72,7 @@ dashboard_addr(desc) -> ?DESC(dashboard_addr); dashboard_addr(default) -> <<"https://127.0.0.1:18083">>; dashboard_addr(_) -> undefined. -%% TOOD: support raw xml metadata in hocon (maybe?🤔) +%% TODO: support raw xml metadata in hocon (maybe?🤔) idp_metadata_url(type) -> binary(); idp_metadata_url(desc) -> ?DESC(idp_metadata_url); idp_metadata_url(default) -> <<"https://idp.example.com">>; diff --git a/apps/emqx_management/src/emqx_mgmt_api_configs.erl b/apps/emqx_management/src/emqx_mgmt_api_configs.erl index f013dfcd1..75341facc 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_configs.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_configs.erl @@ -147,7 +147,9 @@ schema("/configs") -> hoconsc:mk( hoconsc:enum([replace, merge]), #{in => query, default => merge, required => false} - )} + )}, + {ignore_readonly, + hoconsc:mk(boolean(), #{in => query, default => false, required => false})} ], 'requestBody' => #{ content => @@ -361,16 +363,18 @@ configs(get, #{query_string := QueryStr, headers := Headers}, _Req) -> {ok, <<"text/plain">>} -> get_configs_v2(QueryStr); {error, _} = Error -> {400, #{code => 'INVALID_ACCEPT', message => ?ERR_MSG(Error)}} end; -configs(put, #{body := Conf, query_string := #{<<"mode">> := Mode}}, _Req) -> - case emqx_conf_cli:load_config(Conf, #{mode => Mode, log => none}) of +configs(put, #{body := Conf, query_string := #{<<"mode">> := Mode} = QS}, _Req) -> + IngnoreReadonly = maps:get(<<"ignore_readonly">>, QS, false), + case + emqx_conf_cli:load_config(Conf, #{ + mode => Mode, log => none, ignore_readonly => IngnoreReadonly + }) + of ok -> {200}; %% bad hocon format - {error, MsgList = [{_, _} | _]} -> - JsonFun = fun(K, V) -> {K, emqx_utils_maps:binary_string(V)} end, - JsonMap = emqx_utils_maps:jsonable_map(maps:from_list(MsgList), JsonFun), - {400, #{<<"content-type">> => <<"text/plain">>}, JsonMap}; - {error, Msg} -> + {error, Errors} -> + Msg = emqx_logger_jsonfmt:best_effort_json_obj(#{errors => Errors}), {400, #{<<"content-type">> => <<"text/plain">>}, Msg} end. diff --git a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl index 9e05d39a7..94b16320a 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl @@ -19,7 +19,7 @@ -include_lib("typerefl/include/types.hrl"). -include_lib("emqx/include/logger.hrl"). -%%-include_lib("emqx_plugins/include/emqx_plugins.hrl"). +-include_lib("emqx_plugins/include/emqx_plugins.hrl"). -export([ api_spec/0, @@ -34,6 +34,8 @@ upload_install/2, plugin/2, update_plugin/2, + plugin_config/2, + plugin_schema/2, update_boot_order/2 ]). @@ -43,7 +45,8 @@ install_package/2, delete_package/1, describe_package/1, - ensure_action/2 + ensure_action/2, + do_update_plugin_config/3 ]). -define(NAME_RE, "^[A-Za-z]+[A-Za-z0-9-_.]*$"). @@ -52,7 +55,11 @@ %% app_name must be a snake_case (no '-' allowed). -define(VSN_WILDCARD, "-*.tar.gz"). -namespace() -> "plugins". +-define(CONTENT_PLUGIN, plugin). +-define(CONTENT_CONFIG, config). + +namespace() -> + "plugins". api_spec() -> emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). @@ -64,6 +71,8 @@ paths() -> "/plugins/:name", "/plugins/install", "/plugins/:name/:action", + "/plugins/:name/config", + "/plugins/:name/schema", "/plugins/:name/move" ]. @@ -97,15 +106,15 @@ schema("/plugins/install") -> schema => #{ type => object, properties => #{ - plugin => #{type => string, format => binary} + ?CONTENT_PLUGIN => #{type => string, format => binary} } }, - encoding => #{plugin => #{'contentType' => 'application/gzip'}} + encoding => #{?CONTENT_PLUGIN => #{'contentType' => 'application/gzip'}} } } }, responses => #{ - 200 => <<"OK">>, + 204 => <<"Install plugin successfully">>, 400 => emqx_dashboard_swagger:error_codes( ['UNEXPECTED_ERROR', 'ALREADY_INSTALLED', 'BAD_PLUGIN_INFO'] ) @@ -117,7 +126,7 @@ schema("/plugins/:name") -> 'operationId' => plugin, get => #{ summary => <<"Get a plugin description">>, - description => "Describs plugin according to its `release.json` and `README.md`.", + description => "Describe a plugin according to its `release.json` and `README.md`.", tags => ?TAGS, parameters => [hoconsc:ref(name)], responses => #{ @@ -152,22 +161,80 @@ schema("/plugins/:name/:action") -> {action, hoconsc:mk(hoconsc:enum([start, stop]), #{desc => "Action", in => path})} ], responses => #{ - 200 => <<"OK">>, + 204 => <<"Trigger action successfully">>, 404 => emqx_dashboard_swagger:error_codes(['NOT_FOUND'], <<"Plugin Not Found">>) } } }; +schema("/plugins/:name/config") -> + #{ + 'operationId' => plugin_config, + get => #{ + summary => <<"Get plugin config">>, + description => + "Get plugin config. Config schema is defined by user's schema.avsc file.
", + tags => ?TAGS, + parameters => [hoconsc:ref(name)], + responses => #{ + %% avro data, json encoded + 200 => hoconsc:mk(binary()), + 404 => emqx_dashboard_swagger:error_codes(['NOT_FOUND'], <<"Plugin Not Found">>) + } + }, + put => #{ + summary => + <<"Update plugin config">>, + description => + "Update plugin config. Config schema defined by user's schema.avsc file.
", + tags => ?TAGS, + parameters => [hoconsc:ref(name)], + 'requestBody' => #{ + content => #{ + 'application/json' => #{ + schema => #{ + type => object + } + } + } + }, + responses => #{ + 204 => <<"Config updated successfully">>, + 400 => emqx_dashboard_swagger:error_codes( + ['BAD_CONFIG', 'UNEXPECTED_ERROR'], <<"Update plugin config failed">> + ), + 404 => emqx_dashboard_swagger:error_codes(['NOT_FOUND'], <<"Plugin Not Found">>) + } + } + }; +schema("/plugins/:name/schema") -> + #{ + 'operationId' => plugin_schema, + get => #{ + summary => <<"Get installed plugin's AVRO schema">>, + description => "Get plugin's config AVRO schema.", + tags => ?TAGS, + parameters => [hoconsc:ref(name)], + responses => #{ + %% avro schema and i18n json object + 200 => hoconsc:mk(binary()), + 404 => emqx_dashboard_swagger:error_codes( + ['NOT_FOUND', 'FILE_NOT_EXISTED'], + <<"Plugin Not Found or Plugin not given a schema file">> + ) + } + } + }; schema("/plugins/:name/move") -> #{ 'operationId' => update_boot_order, post => #{ - summary => <<"Move plugin within plugin hiearchy">>, + summary => <<"Move plugin within plugin hierarchy">>, description => "Setting the boot order of plugins.", tags => ?TAGS, parameters => [hoconsc:ref(name)], 'requestBody' => move_request_body(), responses => #{ - 200 => <<"OK">>, + 204 => <<"Boot order changed successfully">>, 400 => emqx_dashboard_swagger:error_codes(['MOVE_FAILED'], <<"Move failed">>) } } @@ -338,7 +405,7 @@ upload_install(post, #{body := #{<<"plugin">> := Plugin}}) when is_map(Plugin) - %% File bin is too large, we use rpc:multicall instead of cluster_rpc:multicall NameVsn = string:trim(FileName, trailing, ".tar.gz"), case emqx_plugins:describe(NameVsn) of - {error, #{error := "bad_info_file", return := {enoent, _}}} -> + {error, #{error_msg := "bad_info_file", reason := {enoent, _}}} -> case emqx_plugins:parse_name_vsn(FileName) of {ok, AppName, _Vsn} -> AppDir = filename:join(emqx_plugins:install_dir(), AppName), @@ -382,7 +449,7 @@ do_install_package(FileName, Bin) -> {[_ | _] = Res, []} = emqx_mgmt_api_plugins_proto_v2:install_package(Nodes, FileName, Bin), case lists:filter(fun(R) -> R =/= ok end, Res) of [] -> - {200}; + {204}; Filtered -> %% crash if we have unexpected errors or results [] = lists:filter( @@ -394,7 +461,7 @@ do_install_package(FileName, Bin) -> ), Reason = case hd(Filtered) of - {error, #{error := Reason0}} -> Reason0; + {error, #{error_msg := Reason0}} -> Reason0; {error, #{reason := Reason0}} -> Reason0 end, {400, #{ @@ -418,6 +485,50 @@ update_plugin(put, #{bindings := #{name := Name, action := Action}}) -> Res = emqx_mgmt_api_plugins_proto_v2:ensure_action(Name, Action), return(204, Res). +plugin_config(get, #{bindings := #{name := NameVsn}}) -> + case emqx_plugins:describe(NameVsn) of + {ok, _} -> + case emqx_plugins:get_config(NameVsn) of + {ok, AvroJson} -> + {200, #{<<"content-type">> => <<"'application/json'">>}, AvroJson}; + {error, _} -> + {400, #{ + code => 'BAD_CONFIG', + message => <<"Failed to get plugin config">> + }} + end; + _ -> + {404, plugin_not_found_msg()} + end; +plugin_config(put, #{bindings := #{name := NameVsn}, body := AvroJsonMap}) -> + case emqx_plugins:describe(NameVsn) of + {ok, _} -> + case emqx_plugins:decode_plugin_avro_config(NameVsn, AvroJsonMap) of + {ok, AvroValueConfig} -> + Nodes = emqx:running_nodes(), + %% cluster call with config in map (binary key-value) + _Res = emqx_mgmt_api_plugins_proto_v3:update_plugin_config( + Nodes, NameVsn, AvroJsonMap, AvroValueConfig + ), + {204}; + {error, Reason} -> + {400, #{ + code => 'BAD_CONFIG', + message => readable_error_msg(Reason) + }} + end; + _ -> + {404, plugin_not_found_msg()} + end. + +plugin_schema(get, #{bindings := #{name := NameVsn}}) -> + case emqx_plugins:describe(NameVsn) of + {ok, _Plugin} -> + {200, format_plugin_avsc_and_i18n(NameVsn)}; + _ -> + {404, plugin_not_found_msg()} + end. + update_boot_order(post, #{bindings := #{name := Name}, body := Body}) -> case parse_position(Body, Name) of {error, Reason} -> @@ -425,11 +536,11 @@ update_boot_order(post, #{bindings := #{name := Name}, body := Body}) -> Position -> case emqx_plugins:ensure_enabled(Name, Position, _ConfLocation = global) of ok -> - {200}; + {204}; {error, Reason} -> {400, #{ code => 'MOVE_FAILED', - message => iolist_to_binary(io_lib:format("~p", [Reason])) + message => readable_error_msg(Reason) }} end end. @@ -443,7 +554,7 @@ install_package(FileName, Bin) -> ok = file:write_file(File, Bin), PackageName = string:trim(FileName, trailing, ".tar.gz"), case emqx_plugins:ensure_installed(PackageName) of - {error, #{return := not_found}} = NotFound -> + {error, #{reason := not_found}} = NotFound -> NotFound; {error, Reason} = Error -> ?SLOG(error, Reason#{msg => "failed_to_install_plugin"}), @@ -454,9 +565,9 @@ install_package(FileName, Bin) -> end. %% For RPC plugin get -describe_package(Name) -> +describe_package(NameVsn) -> Node = node(), - case emqx_plugins:describe(Name) of + case emqx_plugins:describe(NameVsn) of {ok, Plugin} -> {Node, [Plugin]}; _ -> {Node, []} end. @@ -487,12 +598,32 @@ ensure_action(Name, restart) -> _ = emqx_plugins:restart(Name), ok. +%% for RPC plugin avro encoded config update +do_update_plugin_config(Name, AvroJsonMap, PluginConfigMap) -> + %% TODO: maybe use `PluginConfigMap` to validate config + emqx_plugins:put_config(Name, AvroJsonMap, PluginConfigMap). + +%%-------------------------------------------------------------------- +%% Helper functions +%%-------------------------------------------------------------------- + return(Code, ok) -> {Code}; -return(_, {error, #{error := "bad_info_file", return := {enoent, _} = Reason}}) -> - {404, #{code => 'NOT_FOUND', message => iolist_to_binary(io_lib:format("~p", [Reason]))}}; +return(_, {error, #{error_msg := "bad_info_file", reason := {enoent, _} = Reason}}) -> + {404, #{code => 'NOT_FOUND', message => readable_error_msg(Reason)}}; +return(_, {error, #{error_msg := "bad_avro_config_file", reason := {enoent, _} = Reason}}) -> + {404, #{code => 'NOT_FOUND', message => readable_error_msg(Reason)}}; return(_, {error, Reason}) -> - {400, #{code => 'PARAM_ERROR', message => iolist_to_binary(io_lib:format("~p", [Reason]))}}. + {400, #{code => 'PARAM_ERROR', message => readable_error_msg(Reason)}}. + +plugin_not_found_msg() -> + #{ + code => 'NOT_FOUND', + message => <<"Plugin Not Found">> + }. + +readable_error_msg(Msg) -> + emqx_utils:readable_error_msg(Msg). parse_position(#{<<"position">> := <<"front">>}, _) -> front; @@ -563,6 +694,18 @@ aggregate_status([{Node, Plugins} | List], Acc) -> ), aggregate_status(List, NewAcc). +format_plugin_avsc_and_i18n(NameVsn) -> + #{ + avsc => try_read_file(fun() -> emqx_plugins:plugin_avsc(NameVsn) end), + i18n => try_read_file(fun() -> emqx_plugins:plugin_i18n(NameVsn) end) + }. + +try_read_file(Fun) -> + case Fun() of + {ok, Json} -> Json; + _ -> null + end. + % running_status: running loaded, stopped %% config_status: not_configured disable enable plugin_status(#{running_status := running}) -> running; diff --git a/apps/emqx_management/src/proto/emqx_mgmt_api_plugins_proto_v3.erl b/apps/emqx_management/src/proto/emqx_mgmt_api_plugins_proto_v3.erl new file mode 100644 index 000000000..641d35f70 --- /dev/null +++ b/apps/emqx_management/src/proto/emqx_mgmt_api_plugins_proto_v3.erl @@ -0,0 +1,69 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_mgmt_api_plugins_proto_v3). + +-behaviour(emqx_bpapi). + +-export([ + introduced_in/0, + get_plugins/1, + install_package/3, + describe_package/2, + delete_package/1, + ensure_action/2, + update_plugin_config/4 +]). + +-include_lib("emqx/include/bpapi.hrl"). + +introduced_in() -> + "5.7.0". + +-spec get_plugins([node()]) -> emqx_rpc:multicall_result(). +get_plugins(Nodes) -> + rpc:multicall(Nodes, emqx_mgmt_api_plugins, get_plugins, [], 15000). + +-spec install_package([node()], binary() | string(), binary()) -> emqx_rpc:multicall_result(). +install_package(Nodes, Filename, Bin) -> + rpc:multicall(Nodes, emqx_mgmt_api_plugins, install_package, [Filename, Bin], 25000). + +-spec describe_package([node()], binary() | string()) -> emqx_rpc:multicall_result(). +describe_package(Nodes, Name) -> + rpc:multicall(Nodes, emqx_mgmt_api_plugins, describe_package, [Name], 10000). + +-spec delete_package(binary() | string()) -> ok | {error, any()}. +delete_package(Name) -> + emqx_cluster_rpc:multicall(emqx_mgmt_api_plugins, delete_package, [Name], all, 10000). + +-spec ensure_action(binary() | string(), 'restart' | 'start' | 'stop') -> ok | {error, any()}. +ensure_action(Name, Action) -> + emqx_cluster_rpc:multicall(emqx_mgmt_api_plugins, ensure_action, [Name, Action], all, 10000). + +-spec update_plugin_config( + [node()], + binary() | string(), + binary(), + map() +) -> + emqx_rpc:multicall_result(). +update_plugin_config(Nodes, NameVsn, AvroJsonMap, PluginConfig) -> + rpc:multicall( + Nodes, + emqx_mgmt_api_plugins, + do_update_plugin_config, + [NameVsn, AvroJsonMap, PluginConfig], + 10000 + ). diff --git a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl index 2c90c9dac..6d4d94013 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl @@ -331,18 +331,20 @@ t_configs_key(_Config) -> Log ), Log1 = emqx_utils_maps:deep_put([<<"log">>, <<"console">>, <<"level">>], Log, <<"error">>), - ?assertEqual(<<>>, update_configs_with_binary(iolist_to_binary(hocon_pp:do(Log1, #{})))), + ?assertEqual({ok, <<>>}, update_configs_with_binary(iolist_to_binary(hocon_pp:do(Log1, #{})))), ?assertEqual(<<"error">>, read_conf([<<"log">>, <<"console">>, <<"level">>])), BadLog = emqx_utils_maps:deep_put([<<"log">>, <<"console">>, <<"level">>], Log, <<"erro1r">>), {error, Error} = update_configs_with_binary(iolist_to_binary(hocon_pp:do(BadLog, #{}))), ExpectError = #{ - <<"log">> => - #{ - <<"kind">> => <<"validation_error">>, - <<"path">> => <<"log.console.level">>, - <<"reason">> => <<"unable_to_convert_to_enum_symbol">>, - <<"value">> => <<"erro1r">> - } + <<"errors">> => #{ + <<"log">> => + #{ + <<"kind">> => <<"validation_error">>, + <<"path">> => <<"log.console.level">>, + <<"reason">> => <<"unable_to_convert_to_enum_symbol">>, + <<"value">> => <<"erro1r">> + } + } }, ?assertEqual(ExpectError, emqx_utils_json:decode(Error, [return_maps])), ReadOnlyConf = #{ @@ -355,7 +357,8 @@ t_configs_key(_Config) -> }, ReadOnlyBin = iolist_to_binary(hocon_pp:do(ReadOnlyConf, #{})), {error, ReadOnlyError} = update_configs_with_binary(ReadOnlyBin), - ?assertEqual(<<"Cannot update read-only key 'cluster'.">>, ReadOnlyError), + ?assertEqual(<<"{\"errors\":\"Cannot update read-only key 'cluster'.\"}">>, ReadOnlyError), + ?assertMatch({ok, <<>>}, update_configs_with_binary(ReadOnlyBin, _InogreReadonly = true)), ok. t_get_configs_in_different_accept(_Config) -> @@ -405,7 +408,7 @@ t_create_webhook_v1_bridges_api(Config) -> WebHookFile = filename:join(?config(data_dir, Config), "webhook_v1.conf"), ?assertMatch({ok, _}, hocon:files([WebHookFile])), {ok, WebHookBin} = file:read_file(WebHookFile), - ?assertEqual(<<>>, update_configs_with_binary(WebHookBin)), + ?assertEqual({ok, <<>>}, update_configs_with_binary(WebHookBin)), Actions = #{ <<"http">> => @@ -487,6 +490,22 @@ t_create_webhook_v1_bridges_api(Config) -> ?assertEqual(#{<<"webhook">> => #{}}, emqx_conf:get_raw([<<"bridges">>])), ok. +t_config_update_parse_error(_Config) -> + ?assertMatch( + {error, <<"{\"errors\":\"{parse_error,", _/binary>>}, + update_configs_with_binary(<<"not an object">>) + ), + ?assertMatch( + {error, <<"{\"errors\":\"{parse_error,", _/binary>>}, + update_configs_with_binary(<<"a = \"tlsv1\"\"\"3e-01">>) + ). + +t_config_update_unknown_root(_Config) -> + ?assertMatch( + {error, <<"{\"errors\":{\"a\":\"{root_key_not_found,", _/binary>>}, + update_configs_with_binary(<<"a = \"tlsv1.3\"">>) + ). + %% Helpers get_config(Name) -> @@ -539,18 +558,29 @@ get_configs_with_binary(Key, Node) -> end. update_configs_with_binary(Bin) -> - Path = emqx_mgmt_api_test_util:api_path(["configs"]), + update_configs_with_binary(Bin, _InogreReadonly = undefined). + +update_configs_with_binary(Bin, IgnoreReadonly) -> + Path = + case IgnoreReadonly of + undefined -> + emqx_mgmt_api_test_util:api_path(["configs"]); + Boolean -> + emqx_mgmt_api_test_util:api_path([ + "configs?ignore_readonly=" ++ atom_to_list(Boolean) + ]) + end, Auth = emqx_mgmt_api_test_util:auth_header_(), Headers = [{"accept", "text/plain"}, Auth], case httpc:request(put, {Path, Headers, "text/plain", Bin}, [], [{body_format, binary}]) of {ok, {{"HTTP/1.1", Code, _}, _Headers, Body}} when Code >= 200 andalso Code =< 299 -> - Body; - {ok, {{"HTTP/1.1", _Code, _}, _Headers, Body}} -> + {ok, Body}; + {ok, {{"HTTP/1.1", 400, _}, _Headers, Body}} -> {error, Body}; Error -> - Error + error({unexpected, Error}) end. update_config(Name, Change) -> diff --git a/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl index 8106afc4a..e563ba262 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl @@ -37,10 +37,10 @@ init_per_suite(Config) -> ok = filelib:ensure_dir(WorkDir), DemoShDir1 = string:replace(WorkDir, "emqx_mgmt_api_plugins", "emqx_plugins"), DemoShDir = lists:flatten(string:replace(DemoShDir1, "emqx_management", "emqx_plugins")), - OrigInstallDir = emqx_plugins:get_config(install_dir, undefined), + OrigInstallDir = emqx_plugins:get_config_interal(install_dir, undefined), ok = filelib:ensure_dir(DemoShDir), emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_plugins]), - emqx_plugins:put_config(install_dir, DemoShDir), + emqx_plugins:put_config_internal(install_dir, DemoShDir), [{demo_sh_dir, DemoShDir}, {orig_install_dir, OrigInstallDir} | Config]. end_per_suite(Config) -> @@ -48,7 +48,7 @@ end_per_suite(Config) -> %% restore config case proplists:get_value(orig_install_dir, Config) of undefined -> ok; - OrigInstallDir -> emqx_plugins:put_config(install_dir, OrigInstallDir) + OrigInstallDir -> emqx_plugins:put_config_internal(install_dir, OrigInstallDir) end, emqx_mgmt_api_test_util:end_suite([emqx_plugins, emqx_conf]), ok. @@ -271,7 +271,7 @@ install_plugin(FilePath) -> Token ) of - {ok, {{"HTTP/1.1", 200, "OK"}, _Headers, <<>>}} -> ok; + {ok, {{"HTTP/1.1", 204, "No Content"}, _Headers, <<>>}} -> ok; Error -> Error end. @@ -288,7 +288,7 @@ install_plugin(Config, FilePath) -> Auth ) of - {ok, {{"HTTP/1.1", 200, "OK"}, _Headers, <<>>}} -> ok; + {ok, {{"HTTP/1.1", 204, "No Content"}, _Headers, <<>>}} -> ok; Error -> Error end. diff --git a/apps/emqx_message_validation/src/emqx_message_validation_schema.erl b/apps/emqx_message_validation/src/emqx_message_validation_schema.erl index 9a27915d8..f151cdf62 100644 --- a/apps/emqx_message_validation/src/emqx_message_validation_schema.erl +++ b/apps/emqx_message_validation/src/emqx_message_validation_schema.erl @@ -65,6 +65,7 @@ fields(validation) -> #{ desc => ?DESC("topics"), converter => fun ensure_array/2, + validator => fun validate_unique_topics/1, required => true } )}, @@ -269,3 +270,23 @@ do_validate_unique_schema_checks( end; do_validate_unique_schema_checks([_Check | Rest], Seen, Duplicated) -> do_validate_unique_schema_checks(Rest, Seen, Duplicated). + +validate_unique_topics(Topics) -> + Grouped = maps:groups_from_list( + fun(T) -> T end, + Topics + ), + DuplicatedMap = maps:filter( + fun(_T, Ts) -> length(Ts) > 1 end, + Grouped + ), + case maps:keys(DuplicatedMap) of + [] -> + ok; + Duplicated -> + Msg = iolist_to_binary([ + <<"duplicated topics: ">>, + lists:join(", ", Duplicated) + ]), + {error, Msg} + end. diff --git a/apps/emqx_message_validation/test/emqx_message_validation_tests.erl b/apps/emqx_message_validation/test/emqx_message_validation_tests.erl index c344f6202..7c3dfc9d8 100644 --- a/apps/emqx_message_validation/test/emqx_message_validation_tests.erl +++ b/apps/emqx_message_validation/test/emqx_message_validation_tests.erl @@ -232,6 +232,65 @@ check_test_() -> duplicated_check_test_() -> [ + {"duplicated topics 1", + ?_assertThrow( + {_Schema, [ + #{ + reason := <<"duplicated topics: t/1">>, + kind := validation_error, + path := "message_validation.validations.1.topics" + } + ]}, + parse_and_check([ + validation( + <<"foo">>, + [schema_check(json, <<"a">>)], + #{<<"topics">> => [<<"t/1">>, <<"t/1">>]} + ) + ]) + )}, + {"duplicated topics 2", + ?_assertThrow( + {_Schema, [ + #{ + reason := <<"duplicated topics: t/1">>, + kind := validation_error, + path := "message_validation.validations.1.topics" + } + ]}, + parse_and_check([ + validation( + <<"foo">>, + [schema_check(json, <<"a">>)], + #{<<"topics">> => [<<"t/1">>, <<"t/#">>, <<"t/1">>]} + ) + ]) + )}, + {"duplicated topics 3", + ?_assertThrow( + {_Schema, [ + #{ + reason := <<"duplicated topics: t/1, t/2">>, + kind := validation_error, + path := "message_validation.validations.1.topics" + } + ]}, + parse_and_check([ + validation( + <<"foo">>, + [schema_check(json, <<"a">>)], + #{ + <<"topics">> => [ + <<"t/1">>, + <<"t/#">>, + <<"t/1">>, + <<"t/2">>, + <<"t/2">> + ] + } + ) + ]) + )}, {"duplicated sql checks are not checked", ?_assertMatch( [#{<<"checks">> := [_, _]}], diff --git a/apps/emqx_mysql/src/emqx_mysql.app.src b/apps/emqx_mysql/src/emqx_mysql.app.src index f23d7b092..9637cc473 100644 --- a/apps/emqx_mysql/src/emqx_mysql.app.src +++ b/apps/emqx_mysql/src/emqx_mysql.app.src @@ -1,6 +1,6 @@ {application, emqx_mysql, [ {description, "EMQX MySQL Database Connector"}, - {vsn, "0.1.8"}, + {vsn, "0.1.9"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_mysql/src/emqx_mysql.erl b/apps/emqx_mysql/src/emqx_mysql.erl index e77965d67..ff851558a 100644 --- a/apps/emqx_mysql/src/emqx_mysql.erl +++ b/apps/emqx_mysql/src/emqx_mysql.erl @@ -498,6 +498,8 @@ on_sql_query( ) -> LogMeta = #{connector => InstId, sql => SQLOrKey, state => State}, ?TRACE("QUERY", "mysql_connector_received", LogMeta), + ChannelID = maps:get(channel_id, State, no_channel), + emqx_trace:rendered_action_template(ChannelID, #{sql => SQLOrKey}), Worker = ecpool:get_client(PoolName), case ecpool_worker:client(Worker) of {ok, Conn} -> diff --git a/apps/emqx_oracle/src/emqx_oracle.erl b/apps/emqx_oracle/src/emqx_oracle.erl index 4e88a3d96..e90665cc4 100644 --- a/apps/emqx_oracle/src/emqx_oracle.erl +++ b/apps/emqx_oracle/src/emqx_oracle.erl @@ -210,7 +210,7 @@ on_query( }), Type = query, {NameOrSQL2, Data} = proc_sql_params(TypeOrKey, NameOrSQL, Params, State), - Res = on_sql_query(InstId, PoolName, Type, ?SYNC_QUERY_MODE, NameOrSQL2, Data), + Res = on_sql_query(InstId, TypeOrKey, PoolName, Type, ?SYNC_QUERY_MODE, NameOrSQL2, Data), handle_result(Res). on_batch_query( @@ -244,7 +244,9 @@ on_batch_query( Datas2 = [emqx_placeholder:proc_sql(TokenList, Data) || Data <- Datas], St = maps:get(BinKey, Sts), case - on_sql_query(InstId, PoolName, execute_batch, ?SYNC_QUERY_MODE, St, Datas2) + on_sql_query( + InstId, BinKey, PoolName, execute_batch, ?SYNC_QUERY_MODE, St, Datas2 + ) of {ok, Results} -> handle_batch_result(Results, 0); @@ -281,7 +283,13 @@ proc_sql_params(TypeOrKey, SQLOrData, Params, #{ end end. -on_sql_query(InstId, PoolName, Type, ApplyMode, NameOrSQL, Data) -> +on_sql_query(InstId, ChannelID, PoolName, Type, ApplyMode, NameOrSQL, Data) -> + emqx_trace:rendered_action_template(ChannelID, #{ + type => Type, + apply_mode => ApplyMode, + name_or_sql => NameOrSQL, + data => Data + }), case ecpool:pick_and_do(PoolName, {?MODULE, Type, [NameOrSQL, Data]}, ApplyMode) of {error, Reason} = Result -> ?tp( diff --git a/apps/emqx_plugins/include/emqx_plugins.hrl b/apps/emqx_plugins/include/emqx_plugins.hrl index 05959e46f..95dc50e4f 100644 --- a/apps/emqx_plugins/include/emqx_plugins.hrl +++ b/apps/emqx_plugins/include/emqx_plugins.hrl @@ -19,4 +19,25 @@ -define(CONF_ROOT, plugins). +-define(PLUGIN_SERDE_TAB, emqx_plugins_schema_serde_tab). + +-define(CONFIG_FORMAT_AVRO, config_format_avro). +-define(CONFIG_FORMAT_MAP, config_format_map). + +-type schema_name() :: binary(). +-type avsc() :: binary(). + +-type encoded_data() :: iodata(). +-type decoded_data() :: map(). + +-record(plugin_schema_serde, { + name :: schema_name(), + eval_context :: term(), + %% TODO: fields to mark schema import status + %% scheam_imported :: boolean(), + %% for future use + extra = [] +}). +-type plugin_schema_serde() :: #plugin_schema_serde{}. + -endif. diff --git a/apps/emqx_plugins/rebar.config b/apps/emqx_plugins/rebar.config index 9f17b7657..63d56ac72 100644 --- a/apps/emqx_plugins/rebar.config +++ b/apps/emqx_plugins/rebar.config @@ -1,5 +1,8 @@ %% -*- mode: erlang -*- -{deps, [{emqx, {path, "../emqx"}}]}. +{deps, [ + {emqx, {path, "../emqx"}}, + {erlavro, {git, "https://github.com/emqx/erlavro.git", {tag, "2.10.0"}}} +]}. {project_plugins, [erlfmt]}. diff --git a/apps/emqx_plugins/src/emqx_plugins.app.src b/apps/emqx_plugins/src/emqx_plugins.app.src index b26836475..6501d5654 100644 --- a/apps/emqx_plugins/src/emqx_plugins.app.src +++ b/apps/emqx_plugins/src/emqx_plugins.app.src @@ -1,9 +1,9 @@ %% -*- mode: erlang -*- {application, emqx_plugins, [ {description, "EMQX Plugin Management"}, - {vsn, "0.1.8"}, + {vsn, "0.2.0"}, {modules, []}, {mod, {emqx_plugins_app, []}}, - {applications, [kernel, stdlib, emqx]}, + {applications, [kernel, stdlib, emqx, erlavro]}, {env, []} ]}. diff --git a/apps/emqx_plugins/src/emqx_plugins.erl b/apps/emqx_plugins/src/emqx_plugins.erl index ebe5b7932..67d25bf7a 100644 --- a/apps/emqx_plugins/src/emqx_plugins.erl +++ b/apps/emqx_plugins/src/emqx_plugins.erl @@ -16,7 +16,6 @@ -module(emqx_plugins). --include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/logger.hrl"). -include("emqx_plugins.hrl"). @@ -24,6 +23,16 @@ -include_lib("eunit/include/eunit.hrl"). -endif. +-export([ + describe/1, + plugin_avsc/1, + plugin_i18n/1, + plugin_avro/1, + parse_name_vsn/1, + make_name_vsn_string/2 +]). + +%% Package operations -export([ ensure_installed/1, ensure_uninstalled/1, @@ -35,21 +44,30 @@ delete_package/1 ]). +%% Plugin runtime management -export([ ensure_started/0, ensure_started/1, ensure_stopped/0, ensure_stopped/1, restart/1, - list/0, - describe/1, - parse_name_vsn/1 + list/0 ]). +%% Plugin config APIs -export([ + get_config/1, get_config/2, - put_config/2, - get_tar/1 + get_config/3, + get_config/4, + put_config/3 +]). + +%% Package utils +-export([ + decode_plugin_avro_config/2, + install_dir/0, + avsc_file_path/1 ]). %% `emqx_config_handler' API @@ -57,21 +75,33 @@ post_config_update/5 ]). -%% internal +%% RPC call +-export([get_tar/1]). + +%% Internal export -export([do_ensure_started/1]). --export([ - install_dir/0 -]). +%% for test cases +-export([put_config_internal/2]). -ifdef(TEST). -compile(export_all). -compile(nowarn_export_all). -endif. +%% Defines +-define(PLUGIN_PERSIS_CONFIG_KEY(NameVsn), {?MODULE, NameVsn}). + +-define(RAW_BIN, binary). +-define(JSON_MAP, json_map). + %% "my_plugin-0.1.0" -type name_vsn() :: binary() | string(). %% the parse result of the JSON info file -type plugin() :: map(). +-type schema_json() :: map(). +-type i18n_json() :: map(). +-type avro_binary() :: binary(). +-type plugin_config() :: map(). -type position() :: no_move | front | rear | {before, name_vsn()} | {behind, name_vsn()}. %%-------------------------------------------------------------------- @@ -80,12 +110,39 @@ %% @doc Describe a plugin. -spec describe(name_vsn()) -> {ok, plugin()} | {error, any()}. -describe(NameVsn) -> read_plugin(NameVsn, #{fill_readme => true}). +describe(NameVsn) -> + read_plugin_info(NameVsn, #{fill_readme => true}). + +-spec plugin_avsc(name_vsn()) -> {ok, schema_json()} | {error, any()}. +plugin_avsc(NameVsn) -> + read_plugin_avsc(NameVsn). + +-spec plugin_i18n(name_vsn()) -> {ok, i18n_json()} | {error, any()}. +plugin_i18n(NameVsn) -> + read_plugin_i18n(NameVsn). + +-spec plugin_avro(name_vsn()) -> {ok, avro_binary()} | {error, any()}. +plugin_avro(NameVsn) -> + read_plugin_avro(NameVsn). + +parse_name_vsn(NameVsn) when is_binary(NameVsn) -> + parse_name_vsn(binary_to_list(NameVsn)); +parse_name_vsn(NameVsn) when is_list(NameVsn) -> + case lists:splitwith(fun(X) -> X =/= $- end, NameVsn) of + {AppName, [$- | Vsn]} -> {ok, list_to_atom(AppName), Vsn}; + _ -> {error, "bad_name_vsn"} + end. + +make_name_vsn_string(Name, Vsn) -> + binary_to_list(iolist_to_binary([Name, "-", Vsn])). + +%%-------------------------------------------------------------------- +%% Package operations %% @doc Install a .tar.gz package placed in install_dir. -spec ensure_installed(name_vsn()) -> ok | {error, map()}. ensure_installed(NameVsn) -> - case read_plugin(NameVsn, #{}) of + case read_plugin_info(NameVsn, #{}) of {ok, _} -> ok; {error, _} -> @@ -93,36 +150,197 @@ ensure_installed(NameVsn) -> do_ensure_installed(NameVsn) end. -do_ensure_installed(NameVsn) -> - TarGz = pkg_file(NameVsn), - case erl_tar:extract(TarGz, [compressed, memory]) of - {ok, TarContent} -> - ok = write_tar_file_content(install_dir(), TarContent), - case read_plugin(NameVsn, #{}) of - {ok, _} -> - ok; - {error, Reason} -> - ?SLOG(warning, Reason#{msg => "failed_to_read_after_install"}), - ok = delete_tar_file_content(install_dir(), TarContent), - {error, Reason} - end; - {error, {_, enoent}} -> +%% @doc Ensure files and directories for the given plugin are being deleted. +%% If a plugin is running, or enabled, an error is returned. +-spec ensure_uninstalled(name_vsn()) -> ok | {error, any()}. +ensure_uninstalled(NameVsn) -> + case read_plugin_info(NameVsn, #{}) of + {ok, #{running_status := RunningSt}} when RunningSt =/= stopped -> {error, #{ - reason => "failed_to_extract_plugin_package", - path => TarGz, - return => not_found + error_msg => "bad_plugin_running_status", + hint => "stop_the_plugin_first" }}; - {error, Reason} -> + {ok, #{config_status := enabled}} -> {error, #{ - reason => "bad_plugin_package", - path => TarGz, - return => Reason - }} + error_msg => "bad_plugin_config_status", + hint => "disable_the_plugin_first" + }}; + _ -> + purge(NameVsn), + ensure_delete(NameVsn) end. +%% @doc Ensure a plugin is enabled to the end of the plugins list. +-spec ensure_enabled(name_vsn()) -> ok | {error, any()}. +ensure_enabled(NameVsn) -> + ensure_enabled(NameVsn, no_move). + +%% @doc Ensure a plugin is enabled at the given position of the plugin list. +-spec ensure_enabled(name_vsn(), position()) -> ok | {error, any()}. +ensure_enabled(NameVsn, Position) -> + ensure_state(NameVsn, Position, _Enabled = true, _ConfLocation = local). + +-spec ensure_enabled(name_vsn(), position(), local | global) -> ok | {error, any()}. +ensure_enabled(NameVsn, Position, ConfLocation) when + ConfLocation =:= local; ConfLocation =:= global +-> + ensure_state(NameVsn, Position, _Enabled = true, ConfLocation). + +%% @doc Ensure a plugin is disabled. +-spec ensure_disabled(name_vsn()) -> ok | {error, any()}. +ensure_disabled(NameVsn) -> + ensure_state(NameVsn, no_move, false, _ConfLocation = local). + +%% @doc Delete extracted dir +%% In case one lib is shared by multiple plugins. +%% it might be the case that purging one plugin's install dir +%% will cause deletion of loaded beams. +%% It should not be a problem, because shared lib should +%% reside in all the plugin install dirs. +-spec purge(name_vsn()) -> ok. +purge(NameVsn) -> + _ = maybe_purge_plugin_config(NameVsn), + purge_plugin(NameVsn). + +%% @doc Delete the package file. +-spec delete_package(name_vsn()) -> ok. +delete_package(NameVsn) -> + File = pkg_file_path(NameVsn), + _ = emqx_plugins_serde:delete_schema(NameVsn), + case file:delete(File) of + ok -> + ?SLOG(info, #{msg => "purged_plugin_dir", path => File}), + ok; + {error, enoent} -> + ok; + {error, Reason} -> + ?SLOG(error, #{ + msg => "failed_to_delete_package_file", + path => File, + reason => Reason + }), + {error, Reason} + end. + +%%-------------------------------------------------------------------- +%% Plugin runtime management + +%% @doc Start all configured plugins are started. +-spec ensure_started() -> ok. +ensure_started() -> + ok = for_plugins(fun ?MODULE:do_ensure_started/1). + +%% @doc Start a plugin from Management API or CLI. +%% the input is a - string. +-spec ensure_started(name_vsn()) -> ok | {error, term()}. +ensure_started(NameVsn) -> + case do_ensure_started(NameVsn) of + ok -> + ok; + {error, Reason} -> + ?SLOG(alert, Reason#{msg => "failed_to_start_plugin"}), + {error, Reason} + end. + +%% @doc Stop all plugins before broker stops. +-spec ensure_stopped() -> ok. +ensure_stopped() -> + for_plugins(fun ?MODULE:ensure_stopped/1). + +%% @doc Stop a plugin from Management API or CLI. +-spec ensure_stopped(name_vsn()) -> ok | {error, term()}. +ensure_stopped(NameVsn) -> + tryit( + "stop_plugin", + fun() -> + Plugin = do_read_plugin(NameVsn), + ensure_apps_stopped(Plugin) + end + ). + +get_config(Name, Vsn, Options, Default) -> + get_config(make_name_vsn_string(Name, Vsn), Options, Default). + +-spec get_config(name_vsn()) -> + {ok, plugin_config()} + | {error, term()}. +get_config(NameVsn) -> + get_config(bin(NameVsn), #{format => ?CONFIG_FORMAT_MAP}). + +-spec get_config(name_vsn(), Options :: map()) -> + {ok, avro_binary() | plugin_config()} + | {error, term()}. +get_config(NameVsn, #{format := ?CONFIG_FORMAT_AVRO}) -> + %% no default value when get raw binary config + case read_plugin_avro(NameVsn) of + {ok, _AvroJson} = Res -> Res; + {error, _Reason} = Err -> Err + end; +get_config(NameVsn, Options = #{format := ?CONFIG_FORMAT_MAP}) -> + get_config(NameVsn, Options, #{}). + +get_config(NameVsn, #{format := ?CONFIG_FORMAT_MAP}, Default) -> + {ok, persistent_term:get(?PLUGIN_PERSIS_CONFIG_KEY(NameVsn), Default)}. + +%% @doc Update plugin's config. +%% RPC call from Management API or CLI. +%% the avro Json Map and plugin config ALWAYS be valid before calling this function. +put_config(NameVsn, AvroJsonMap, _DecodedPluginConfig) -> + AvroJsonBin = emqx_utils_json:encode(AvroJsonMap), + ok = write_avro_bin(NameVsn, AvroJsonBin), + ok = persistent_term:put(?PLUGIN_PERSIS_CONFIG_KEY(NameVsn), AvroJsonMap), + ok. + +%% @doc Stop and then start the plugin. +restart(NameVsn) -> + case ensure_stopped(NameVsn) of + ok -> ensure_started(NameVsn); + {error, Reason} -> {error, Reason} + end. + +%% @doc List all installed plugins. +%% Including the ones that are installed, but not enabled in config. +-spec list() -> [plugin()]. +list() -> + Pattern = filename:join([install_dir(), "*", "release.json"]), + All = lists:filtermap( + fun(JsonFilePath) -> + [_, NameVsn | _] = lists:reverse(filename:split(JsonFilePath)), + case read_plugin_info(NameVsn, #{}) of + {ok, Info} -> + {true, Info}; + {error, Reason} -> + ?SLOG(warning, Reason), + false + end + end, + filelib:wildcard(Pattern) + ), + do_list(configured(), All). + +%%-------------------------------------------------------------------- +%% Package utils + +-spec decode_plugin_avro_config(name_vsn(), map() | binary()) -> {ok, map()} | {error, any()}. +decode_plugin_avro_config(NameVsn, AvroJsonMap) when is_map(AvroJsonMap) -> + decode_plugin_avro_config(NameVsn, emqx_utils_json:encode(AvroJsonMap)); +decode_plugin_avro_config(NameVsn, AvroJsonBin) -> + case emqx_plugins_serde:decode(NameVsn, AvroJsonBin) of + {ok, Config} -> {ok, Config}; + {error, ReasonMap} -> {error, ReasonMap} + end. + +get_config_interal(Key, Default) when is_atom(Key) -> + get_config_interal([Key], Default); +get_config_interal(Path, Default) -> + emqx_conf:get([?CONF_ROOT | Path], Default). + +put_config_internal(Key, Value) -> + do_put_config_internal(Key, Value, _ConfLocation = local). + -spec get_tar(name_vsn()) -> {ok, binary()} | {error, any}. get_tar(NameVsn) -> - TarGz = pkg_file(NameVsn), + TarGz = pkg_file_path(NameVsn), case file:read_file(TarGz) of {ok, Content} -> {ok, Content}; @@ -135,10 +353,14 @@ get_tar(NameVsn) -> end end. +%%-------------------------------------------------------------------- +%% Internal +%%-------------------------------------------------------------------- + maybe_create_tar(NameVsn, TarGzName, InstallDir) when is_binary(InstallDir) -> maybe_create_tar(NameVsn, TarGzName, binary_to_list(InstallDir)); maybe_create_tar(NameVsn, TarGzName, InstallDir) -> - case filelib:wildcard(filename:join(dir(NameVsn), "**")) of + case filelib:wildcard(filename:join(plugin_dir(NameVsn), "**")) of [_ | _] = PluginFiles -> InstallDir1 = string:trim(InstallDir, trailing, "/") ++ "/", PluginFiles1 = [{string:prefix(F, InstallDir1), F} || F <- PluginFiles], @@ -207,24 +429,32 @@ top_dir_test_() -> ]. -endif. -%% @doc Ensure files and directories for the given plugin are being deleted. -%% If a plugin is running, or enabled, an error is returned. --spec ensure_uninstalled(name_vsn()) -> ok | {error, any()}. -ensure_uninstalled(NameVsn) -> - case read_plugin(NameVsn, #{}) of - {ok, #{running_status := RunningSt}} when RunningSt =/= stopped -> +do_ensure_installed(NameVsn) -> + TarGz = pkg_file_path(NameVsn), + case erl_tar:extract(TarGz, [compressed, memory]) of + {ok, TarContent} -> + ok = write_tar_file_content(install_dir(), TarContent), + case read_plugin_info(NameVsn, #{}) of + {ok, _} -> + ok = maybe_post_op_after_install(NameVsn), + ok; + {error, Reason} -> + ?SLOG(warning, Reason#{msg => "failed_to_read_after_install"}), + ok = delete_tar_file_content(install_dir(), TarContent), + {error, Reason} + end; + {error, {_, enoent}} -> {error, #{ - reason => "bad_plugin_running_status", - hint => "stop_the_plugin_first" + error_msg => "failed_to_extract_plugin_package", + path => TarGz, + reason => not_found }}; - {ok, #{config_status := enabled}} -> + {error, Reason} -> {error, #{ - reason => "bad_plugin_config_status", - hint => "disable_the_plugin_first" - }}; - _ -> - purge(NameVsn), - ensure_delete(NameVsn) + error_msg => "bad_plugin_package", + path => TarGz, + reason => Reason + }} end. ensure_delete(NameVsn0) -> @@ -233,37 +463,19 @@ ensure_delete(NameVsn0) -> put_configured(lists:filter(fun(#{name_vsn := N1}) -> bin(N1) =/= NameVsn end, List)), ok. -%% @doc Ensure a plugin is enabled to the end of the plugins list. --spec ensure_enabled(name_vsn()) -> ok | {error, any()}. -ensure_enabled(NameVsn) -> - ensure_enabled(NameVsn, no_move). - -%% @doc Ensure a plugin is enabled at the given position of the plugin list. --spec ensure_enabled(name_vsn(), position()) -> ok | {error, any()}. -ensure_enabled(NameVsn, Position) -> - ensure_state(NameVsn, Position, _Enabled = true, _ConfLocation = local). - --spec ensure_enabled(name_vsn(), position(), local | global) -> ok | {error, any()}. -ensure_enabled(NameVsn, Position, ConfLocation) when - ConfLocation =:= local; ConfLocation =:= global --> - ensure_state(NameVsn, Position, _Enabled = true, ConfLocation). - -%% @doc Ensure a plugin is disabled. --spec ensure_disabled(name_vsn()) -> ok | {error, any()}. -ensure_disabled(NameVsn) -> - ensure_state(NameVsn, no_move, false, _ConfLocation = local). - ensure_state(NameVsn, Position, State, ConfLocation) when is_binary(NameVsn) -> ensure_state(binary_to_list(NameVsn), Position, State, ConfLocation); ensure_state(NameVsn, Position, State, ConfLocation) -> - case read_plugin(NameVsn, #{}) of + case read_plugin_info(NameVsn, #{}) of {ok, _} -> Item = #{ name_vsn => NameVsn, enable => State }, - tryit("ensure_state", fun() -> ensure_configured(Item, Position, ConfLocation) end); + tryit( + "ensure_state", + fun() -> ensure_configured(Item, Position, ConfLocation) end + ); {error, Reason} -> {error, Reason} end. @@ -295,7 +507,7 @@ add_new_configured(Configured, {Action, NameVsn}, Item) -> {Front, Rear} = lists:splitwith(SplitFun, Configured), Rear =:= [] andalso throw(#{ - error => "position_anchor_plugin_not_configured", + error_msg => "position_anchor_plugin_not_configured", hint => "maybe_install_and_configure", name_vsn => NameVsn }), @@ -307,37 +519,21 @@ add_new_configured(Configured, {Action, NameVsn}, Item) -> Front ++ [Anchor, Item | Rear0] end. -%% @doc Delete the package file. --spec delete_package(name_vsn()) -> ok. -delete_package(NameVsn) -> - File = pkg_file(NameVsn), - case file:delete(File) of - ok -> - ?SLOG(info, #{msg => "purged_plugin_dir", path => File}), - ok; - {error, enoent} -> - ok; - {error, Reason} -> - ?SLOG(error, #{ - msg => "failed_to_delete_package_file", - path => File, - reason => Reason - }), - {error, Reason} - end. +maybe_purge_plugin_config(NameVsn) -> + _ = persistent_term:erase(?PLUGIN_PERSIS_CONFIG_KEY(NameVsn)), + ok. -%% @doc Delete extracted dir -%% In case one lib is shared by multiple plugins. -%% it might be the case that purging one plugin's install dir -%% will cause deletion of loaded beams. -%% It should not be a problem, because shared lib should -%% reside in all the plugin install dirs. --spec purge(name_vsn()) -> ok. -purge(NameVsn) -> - Dir = dir(NameVsn), +purge_plugin(NameVsn) -> + Dir = plugin_dir(NameVsn), + purge_plugin_dir(Dir). + +purge_plugin_dir(Dir) -> case file:del_dir_r(Dir) of ok -> - ?SLOG(info, #{msg => "purged_plugin_dir", dir => Dir}); + ?SLOG(info, #{ + msg => "purged_plugin_dir", + dir => Dir + }); {error, enoent} -> ok; {error, Reason} -> @@ -349,72 +545,10 @@ purge(NameVsn) -> {error, Reason} end. -%% @doc Start all configured plugins are started. --spec ensure_started() -> ok. -ensure_started() -> - ok = for_plugins(fun ?MODULE:do_ensure_started/1). - -%% @doc Start a plugin from Management API or CLI. -%% the input is a - string. --spec ensure_started(name_vsn()) -> ok | {error, term()}. -ensure_started(NameVsn) -> - case do_ensure_started(NameVsn) of - ok -> - ok; - {error, Reason} -> - ?SLOG(alert, #{ - msg => "failed_to_start_plugin", - reason => Reason - }), - {error, Reason} - end. - -%% @doc Stop all plugins before broker stops. --spec ensure_stopped() -> ok. -ensure_stopped() -> - for_plugins(fun ?MODULE:ensure_stopped/1). - -%% @doc Stop a plugin from Management API or CLI. --spec ensure_stopped(name_vsn()) -> ok | {error, term()}. -ensure_stopped(NameVsn) -> - tryit( - "stop_plugin", - fun() -> - Plugin = do_read_plugin(NameVsn), - ensure_apps_stopped(Plugin) - end - ). - -%% @doc Stop and then start the plugin. -restart(NameVsn) -> - case ensure_stopped(NameVsn) of - ok -> ensure_started(NameVsn); - {error, Reason} -> {error, Reason} - end. - -%% @doc List all installed plugins. -%% Including the ones that are installed, but not enabled in config. --spec list() -> [plugin()]. -list() -> - Pattern = filename:join([install_dir(), "*", "release.json"]), - All = lists:filtermap( - fun(JsonFile) -> - case read_plugin({file, JsonFile}, #{}) of - {ok, Info} -> - {true, Info}; - {error, Reason} -> - ?SLOG(warning, Reason), - false - end - end, - filelib:wildcard(Pattern) - ), - list(configured(), All). - %% Make sure configured ones are ordered in front. -list([], All) -> +do_list([], All) -> All; -list([#{name_vsn := NameVsn} | Rest], All) -> +do_list([#{name_vsn := NameVsn} | Rest], All) -> SplitF = fun(#{<<"name">> := Name, <<"rel_vsn">> := Vsn}) -> bin([Name, "-", Vsn]) =/= bin(NameVsn) end, @@ -424,9 +558,9 @@ list([#{name_vsn := NameVsn} | Rest], All) -> msg => "configured_plugin_not_installed", name_vsn => NameVsn }), - list(Rest, All); + do_list(Rest, All); {Front, [I | Rear]} -> - [I | list(Rest, Front ++ Rear)] + [I | do_list(Rest, Front ++ Rear)] end. do_ensure_started(NameVsn) -> @@ -439,23 +573,26 @@ do_ensure_started(NameVsn) -> ok = load_code_start_apps(NameVsn, Plugin); {error, plugin_not_found} -> ?SLOG(error, #{ - msg => "plugin_not_found", + error_msg => "plugin_not_found", name_vsn => NameVsn - }) + }), + ok end end ). +%%-------------------------------------------------------------------- + %% try the function, catch 'throw' exceptions as normal 'error' return %% other exceptions with stacktrace logged. tryit(WhichOp, F) -> try F() catch - throw:Reason -> + throw:ReasonMap -> %% thrown exceptions are known errors %% translate to a return value without stacktrace - {error, Reason}; + {error, ReasonMap}; error:Reason:Stacktrace -> %% unexpected errors, log stacktrace ?SLOG(warning, #{ @@ -469,40 +606,57 @@ tryit(WhichOp, F) -> %% read plugin info from the JSON file %% returns {ok, Info} or {error, Reason} -read_plugin(NameVsn, Options) -> +read_plugin_info(NameVsn, Options) -> tryit( - "read_plugin_info", - fun() -> {ok, do_read_plugin(NameVsn, Options)} end + atom_to_list(?FUNCTION_NAME), + fun() -> {ok, do_read_plugin2(NameVsn, Options)} end ). -do_read_plugin(Plugin) -> do_read_plugin(Plugin, #{}). +do_read_plugin(NameVsn) -> + do_read_plugin2(NameVsn, #{}). -do_read_plugin({file, InfoFile}, Options) -> - [_, NameVsn | _] = lists:reverse(filename:split(InfoFile)), - case hocon:load(InfoFile, #{format => richmap}) of - {ok, RichMap} -> - Info0 = check_plugin(hocon_maps:ensure_plain(RichMap), NameVsn, InfoFile), - Info1 = plugins_readme(NameVsn, Options, Info0), - plugin_status(NameVsn, Info1); - {error, Reason} -> - throw(#{ - error => "bad_info_file", - path => InfoFile, - return => Reason - }) - end; -do_read_plugin(NameVsn, Options) -> - do_read_plugin({file, info_file(NameVsn)}, Options). +do_read_plugin2(NameVsn, Option) -> + do_read_plugin3(NameVsn, info_file_path(NameVsn), Option). + +do_read_plugin3(NameVsn, InfoFilePath, Options) -> + {ok, PlainMap} = (read_file_fun(InfoFilePath, "bad_info_file", #{read_mode => ?JSON_MAP}))(), + Info0 = check_plugin(PlainMap, NameVsn, InfoFilePath), + Info1 = plugins_readme(NameVsn, Options, Info0), + plugin_status(NameVsn, Info1). + +read_plugin_avsc(NameVsn) -> + read_plugin_avsc(NameVsn, #{read_mode => ?JSON_MAP}). +read_plugin_avsc(NameVsn, Options) -> + tryit( + atom_to_list(?FUNCTION_NAME), + read_file_fun(avsc_file_path(NameVsn), "bad_avsc_file", Options) + ). + +read_plugin_i18n(NameVsn) -> + read_plugin_i18n(NameVsn, #{read_mode => ?JSON_MAP}). +read_plugin_i18n(NameVsn, Options) -> + tryit( + atom_to_list(?FUNCTION_NAME), + read_file_fun(i18n_file_path(NameVsn), "bad_i18n_file", Options) + ). + +read_plugin_avro(NameVsn) -> + read_plugin_avro(NameVsn, #{read_mode => ?RAW_BIN}). +read_plugin_avro(NameVsn, Options) -> + tryit( + atom_to_list(?FUNCTION_NAME), + read_file_fun(avro_config_file(NameVsn), "bad_avro_file", Options) + ). ensure_exists_and_installed(NameVsn) -> - case filelib:is_dir(dir(NameVsn)) of + case filelib:is_dir(plugin_dir(NameVsn)) of true -> ok; false -> %% Do we have the package, but it's not extracted yet? case get_tar(NameVsn) of {ok, TarContent} -> - ok = file:write_file(pkg_file(NameVsn), TarContent), + ok = file:write_file(pkg_file_path(NameVsn), TarContent), ok = do_ensure_installed(NameVsn); _ -> %% If not, try to get it from the cluster. @@ -514,7 +668,7 @@ do_get_from_cluster(NameVsn) -> Nodes = [N || N <- mria:running_nodes(), N /= node()], case get_from_any_node(Nodes, NameVsn, []) of {ok, TarContent} -> - ok = file:write_file(pkg_file(NameVsn), TarContent), + ok = file:write_file(pkg_file_path(NameVsn), TarContent), ok = do_ensure_installed(NameVsn); {error, NodeErrors} when Nodes =/= [] -> ?SLOG(error, #{ @@ -581,10 +735,6 @@ plugin_status(NameVsn, Info) -> config_status => ConfSt }. -bin(A) when is_atom(A) -> atom_to_binary(A, utf8); -bin(L) when is_list(L) -> unicode:characters_to_binary(L, utf8); -bin(B) when is_binary(B) -> B. - check_plugin( #{ <<"name">> := Name, @@ -593,7 +743,7 @@ check_plugin( <<"description">> := _ } = Info, NameVsn, - File + FilePath ) -> case bin(NameVsn) =:= bin([Name, "-", Vsn]) of true -> @@ -605,7 +755,7 @@ check_plugin( catch _:_ -> throw(#{ - error => "bad_rel_apps", + error_msg => "bad_rel_apps", rel_apps => Apps, hint => "A non-empty string list of app_name-app_vsn format" }) @@ -613,16 +763,16 @@ check_plugin( Info; false -> throw(#{ - error => "name_vsn_mismatch", + error_msg => "name_vsn_mismatch", name_vsn => NameVsn, - path => File, + path => FilePath, name => Name, rel_vsn => Vsn }) end; check_plugin(_What, NameVsn, File) -> throw(#{ - error => "bad_info_file_content", + error_msg => "bad_info_file_content", mandatory_fields => [rel_vsn, name, rel_apps, description], name_vsn => NameVsn, path => File @@ -678,7 +828,7 @@ do_load_plugin_app(AppName, Ebin) -> ok; {error, Reason} -> throw(#{ - error => "failed_to_load_plugin_beam", + error_msg => "failed_to_load_plugin_beam", path => BeamFile, reason => Reason }) @@ -693,7 +843,7 @@ do_load_plugin_app(AppName, Ebin) -> ok; {error, Reason} -> throw(#{ - error => "failed_to_load_plugin_app", + error_msg => "failed_to_load_plugin_app", name => AppName, reason => Reason }) @@ -710,7 +860,7 @@ start_app(App) -> ok; {error, {ErrApp, Reason}} -> throw(#{ - error => "failed_to_start_plugin_app", + error_msg => "failed_to_start_plugin_app", app => App, err_app => ErrApp, reason => Reason @@ -775,7 +925,7 @@ stop_app(App) -> ?SLOG(debug, #{msg => "plugin_not_started", app => App}), ok = unload_moudle_and_app(App); {error, Reason} -> - throw(#{error => "failed_to_stop_app", app => App, reason => Reason}) + throw(#{error_msg => "failed_to_stop_app", app => App, reason => Reason}) end. unload_moudle_and_app(App) -> @@ -802,94 +952,22 @@ is_needed_by(AppToStop, RunningApp) -> undefined -> false end. -put_config(Key, Value) -> - put_config(Key, Value, _ConfLocation = local). - -put_config(Key, Value, ConfLocation) when is_atom(Key) -> - put_config([Key], Value, ConfLocation); -put_config(Path, Values, _ConfLocation = local) when is_list(Path) -> +do_put_config_internal(Key, Value, ConfLocation) when is_atom(Key) -> + do_put_config_internal([Key], Value, ConfLocation); +do_put_config_internal(Path, Values, _ConfLocation = local) when is_list(Path) -> Opts = #{rawconf_with_defaults => true, override_to => cluster}, %% Already in cluster_rpc, don't use emqx_conf:update, dead calls case emqx:update_config([?CONF_ROOT | Path], bin_key(Values), Opts) of {ok, _} -> ok; Error -> Error end; -put_config(Path, Values, _ConfLocation = global) when is_list(Path) -> +do_put_config_internal(Path, Values, _ConfLocation = global) when is_list(Path) -> Opts = #{rawconf_with_defaults => true, override_to => cluster}, case emqx_conf:update([?CONF_ROOT | Path], bin_key(Values), Opts) of {ok, _} -> ok; Error -> Error end. -bin_key(Map) when is_map(Map) -> - maps:fold(fun(K, V, Acc) -> Acc#{bin(K) => V} end, #{}, Map); -bin_key(List = [#{} | _]) -> - lists:map(fun(M) -> bin_key(M) end, List); -bin_key(Term) -> - Term. - -get_config(Key, Default) when is_atom(Key) -> - get_config([Key], Default); -get_config(Path, Default) -> - emqx_conf:get([?CONF_ROOT | Path], Default). - -install_dir() -> get_config(install_dir, ""). - -put_configured(Configured) -> - put_configured(Configured, _ConfLocation = local). - -put_configured(Configured, ConfLocation) -> - ok = put_config(states, bin_key(Configured), ConfLocation). - -configured() -> - get_config(states, []). - -for_plugins(ActionFun) -> - case lists:flatmap(fun(I) -> for_plugin(I, ActionFun) end, configured()) of - [] -> ok; - Errors -> erlang:error(#{function => ActionFun, errors => Errors}) - end. - -for_plugin(#{name_vsn := NameVsn, enable := true}, Fun) -> - case Fun(NameVsn) of - ok -> []; - {error, Reason} -> [{NameVsn, Reason}] - end; -for_plugin(#{name_vsn := NameVsn, enable := false}, _Fun) -> - ?SLOG(debug, #{ - msg => "plugin_disabled", - name_vsn => NameVsn - }), - []. - -parse_name_vsn(NameVsn) when is_binary(NameVsn) -> - parse_name_vsn(binary_to_list(NameVsn)); -parse_name_vsn(NameVsn) when is_list(NameVsn) -> - case lists:splitwith(fun(X) -> X =/= $- end, NameVsn) of - {AppName, [$- | Vsn]} -> {ok, list_to_atom(AppName), Vsn}; - _ -> {error, "bad_name_vsn"} - end. - -pkg_file(NameVsn) -> - filename:join([install_dir(), bin([NameVsn, ".tar.gz"])]). - -dir(NameVsn) -> - filename:join([install_dir(), NameVsn]). - -info_file(NameVsn) -> - filename:join([dir(NameVsn), "release.json"]). - -readme_file(NameVsn) -> - filename:join([dir(NameVsn), "README.md"]). - -running_apps() -> - lists:map( - fun({N, _, V}) -> - {N, V} - end, - application:which_applications(infinity) - ). - %%-------------------------------------------------------------------- %% `emqx_config_handler' API %%-------------------------------------------------------------------- @@ -913,3 +991,137 @@ enable_disable_plugin(NameVsn, {#{enable := false}, #{enable := true}}) -> ok; enable_disable_plugin(_NameVsn, _Diff) -> ok. + +%%-------------------------------------------------------------------- +%% Helper functions +%%-------------------------------------------------------------------- + +install_dir() -> + get_config_interal(install_dir, ""). + +put_configured(Configured) -> + put_configured(Configured, _ConfLocation = local). + +put_configured(Configured, ConfLocation) -> + ok = do_put_config_internal(states, bin_key(Configured), ConfLocation). + +configured() -> + get_config_interal(states, []). + +for_plugins(ActionFun) -> + case lists:flatmap(fun(I) -> for_plugin(I, ActionFun) end, configured()) of + [] -> ok; + Errors -> erlang:error(#{function => ActionFun, errors => Errors}) + end. + +for_plugin(#{name_vsn := NameVsn, enable := true}, Fun) -> + case Fun(NameVsn) of + ok -> []; + {error, Reason} -> [{NameVsn, Reason}] + end; +for_plugin(#{name_vsn := NameVsn, enable := false}, _Fun) -> + ?SLOG(debug, #{ + msg => "plugin_disabled", + name_vsn => NameVsn + }), + []. + +maybe_post_op_after_install(NameVsn) -> + _ = maybe_load_config_schema(NameVsn), + _ = maybe_create_config_dir(NameVsn), + ok. + +maybe_load_config_schema(NameVsn) -> + AvscPath = avsc_file_path(NameVsn), + filelib:is_regular(AvscPath) andalso + do_load_config_schema(NameVsn, AvscPath). + +do_load_config_schema(NameVsn, AvscPath) -> + case emqx_plugins_serde:add_schema(NameVsn, AvscPath) of + ok -> ok; + {error, already_exists} -> ok; + {error, _Reason} -> ok + end. + +maybe_create_config_dir(NameVsn) -> + ConfigDir = plugin_config_dir(NameVsn), + case filelib:ensure_path(ConfigDir) of + ok -> + ok; + {error, Reason} -> + ?SLOG(warning, #{ + msg => "failed_to_create_plugin_config_dir", + dir => ConfigDir, + reason => Reason + }), + {error, {mkdir_failed, ConfigDir, Reason}} + end. + +write_avro_bin(NameVsn, AvroBin) -> + ok = file:write_file(avro_config_file(NameVsn), AvroBin). + +read_file_fun(Path, ErrMsg, #{read_mode := ?RAW_BIN}) -> + fun() -> + case file:read_file(Path) of + {ok, Bin} -> + {ok, Bin}; + {error, Reason} -> + ErrMeta = #{error_msg => ErrMsg, reason => Reason}, + throw(ErrMeta) + end + end; +read_file_fun(Path, ErrMsg, #{read_mode := ?JSON_MAP}) -> + fun() -> + case hocon:load(Path, #{format => richmap}) of + {ok, RichMap} -> + {ok, hocon_maps:ensure_plain(RichMap)}; + {error, Reason} -> + ErrMeta = #{error_msg => ErrMsg, reason => Reason}, + throw(ErrMeta) + end + end. + +%% Directorys +plugin_dir(NameVsn) -> + filename:join([install_dir(), NameVsn]). + +plugin_config_dir(NameVsn) -> + filename:join([plugin_dir(NameVsn), "data", "configs"]). + +%% Files +pkg_file_path(NameVsn) -> + filename:join([install_dir(), bin([NameVsn, ".tar.gz"])]). + +info_file_path(NameVsn) -> + filename:join([plugin_dir(NameVsn), "release.json"]). + +avsc_file_path(NameVsn) -> + filename:join([plugin_dir(NameVsn), "config_schema.avsc"]). + +avro_config_file(NameVsn) -> + filename:join([plugin_config_dir(NameVsn), "config.avro"]). + +i18n_file_path(NameVsn) -> + filename:join([plugin_dir(NameVsn), "config_i18n.json"]). + +readme_file(NameVsn) -> + filename:join([plugin_dir(NameVsn), "README.md"]). + +running_apps() -> + lists:map( + fun({N, _, V}) -> + {N, V} + end, + application:which_applications(infinity) + ). + +bin_key(Map) when is_map(Map) -> + maps:fold(fun(K, V, Acc) -> Acc#{bin(K) => V} end, #{}, Map); +bin_key(List = [#{} | _]) -> + lists:map(fun(M) -> bin_key(M) end, List); +bin_key(Term) -> + Term. + +bin(A) when is_atom(A) -> atom_to_binary(A, utf8); +bin(L) when is_list(L) -> unicode:characters_to_binary(L, utf8); +bin(B) when is_binary(B) -> B. diff --git a/apps/emqx_plugins/src/emqx_plugins_serde.erl b/apps/emqx_plugins/src/emqx_plugins_serde.erl new file mode 100644 index 000000000..fc1321ff1 --- /dev/null +++ b/apps/emqx_plugins/src/emqx_plugins_serde.erl @@ -0,0 +1,279 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2017-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_plugins_serde). + +-include("emqx_plugins.hrl"). +-include_lib("emqx/include/logger.hrl"). + +%% API +-export([ + start_link/0, + lookup_serde/1, + add_schema/2, + get_schema/1, + delete_schema/1 +]). + +%% `gen_server' API +-export([ + init/1, + handle_call/3, + handle_cast/2, + handle_continue/2, + terminate/2 +]). + +-export([ + decode/2, + encode/2 +]). + +%%------------------------------------------------------------------------------------------------- +%% API +%%------------------------------------------------------------------------------------------------- + +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +-spec lookup_serde(schema_name()) -> {ok, plugin_schema_serde()} | {error, not_found}. +lookup_serde(SchemaName) -> + case ets:lookup(?PLUGIN_SERDE_TAB, to_bin(SchemaName)) of + [] -> + {error, not_found}; + [Serde] -> + {ok, Serde} + end. + +-spec add_schema(schema_name(), avsc()) -> ok | {error, term()}. +add_schema(NameVsn, Path) -> + case lookup_serde(NameVsn) of + {ok, _Serde} -> + ?SLOG(warning, #{msg => "plugin_schema_already_exists", plugin => NameVsn}), + {error, already_exists}; + {error, not_found} -> + case gen_server:call(?MODULE, {build_serdes, to_bin(NameVsn), Path}, infinity) of + ok -> + ?SLOG(debug, #{msg => "plugin_schema_added", plugin => NameVsn}), + ok; + {error, Reason} = E -> + ?SLOG(error, #{ + msg => "plugin_schema_add_failed", + plugin => NameVsn, + reason => emqx_utils:readable_error_msg(Reason) + }), + E + end + end. + +get_schema(NameVsn) -> + Path = emqx_plugins:avsc_file_path(NameVsn), + case read_avsc_file(Path) of + {ok, Avsc} -> + {ok, Avsc}; + {error, Reason} -> + ?SLOG(warning, Reason), + {error, Reason} + end. + +-spec delete_schema(schema_name()) -> ok | {error, term()}. +delete_schema(NameVsn) -> + case lookup_serde(NameVsn) of + {ok, _Serde} -> + async_delete_serdes([NameVsn]), + ok; + {error, not_found} -> + {error, not_found} + end. + +-spec decode(schema_name(), encoded_data()) -> {ok, decoded_data()} | {error, any()}. +decode(SerdeName, RawData) -> + with_serde( + ?FUNCTION_NAME, + SerdeName, + [RawData] + ). + +-spec encode(schema_name(), decoded_data()) -> {ok, encoded_data()} | {error, any()}. +encode(SerdeName, Data) -> + with_serde( + ?FUNCTION_NAME, + SerdeName, + [Data] + ). + +%%------------------------------------------------------------------------------------------------- +%% `gen_server' API +%%------------------------------------------------------------------------------------------------- + +init(_) -> + process_flag(trap_exit, true), + ok = emqx_utils_ets:new(?PLUGIN_SERDE_TAB, [ + public, ordered_set, {keypos, #plugin_schema_serde.name} + ]), + State = #{}, + AvscPaths = get_plugin_avscs(), + {ok, State, {continue, {build_serdes, AvscPaths}}}. + +handle_continue({build_serdes, AvscPaths}, State) -> + _ = build_serdes(AvscPaths), + {noreply, State}. + +handle_call({build_serdes, NameVsn, AvscPath}, _From, State) -> + BuildRes = do_build_serde({NameVsn, AvscPath}), + {reply, BuildRes, State}; +handle_call(_Call, _From, State) -> + {reply, {error, unknown_call}, State}. + +handle_cast({delete_serdes, Names}, State) -> + lists:foreach(fun ensure_serde_absent/1, Names), + {noreply, State}; +handle_cast(_Cast, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +%%------------------------------------------------------------------------------------------------- +%% Internal fns +%%------------------------------------------------------------------------------------------------- + +-spec get_plugin_avscs() -> [{string(), string()}]. +get_plugin_avscs() -> + Pattern = filename:join([emqx_plugins:install_dir(), "*", "config_schema.avsc"]), + lists:foldl( + fun(AvscPath, AccIn) -> + [_, NameVsn | _] = lists:reverse(filename:split(AvscPath)), + [{NameVsn, AvscPath} | AccIn] + end, + _Acc0 = [], + filelib:wildcard(Pattern) + ). + +build_serdes(AvscPaths) -> + ok = lists:foreach(fun do_build_serde/1, AvscPaths). + +do_build_serde({NameVsn, AvscPath}) -> + try + Serde = make_serde(NameVsn, AvscPath), + true = ets:insert(?PLUGIN_SERDE_TAB, Serde), + ok + catch + Kind:Error:Stacktrace -> + ?SLOG( + error, + #{ + msg => "error_building_plugin_schema_serde", + name => NameVsn, + kind => Kind, + error => Error, + stacktrace => Stacktrace + } + ), + {error, Error} + end. + +make_serde(NameVsn, AvscPath) -> + {ok, AvscBin} = read_avsc_file(AvscPath), + Store0 = avro_schema_store:new([map]), + %% import the schema into the map store with an assigned name + %% if it's a named schema (e.g. struct), then Name is added as alias + Store = avro_schema_store:import_schema_json(NameVsn, AvscBin, Store0), + #plugin_schema_serde{ + name = NameVsn, + eval_context = Store + }. + +ensure_serde_absent(Name) when not is_binary(Name) -> + ensure_serde_absent(to_bin(Name)); +ensure_serde_absent(Name) -> + case lookup_serde(Name) of + {ok, _Serde} -> + _ = ets:delete(?PLUGIN_SERDE_TAB, Name), + ok; + {error, not_found} -> + ok + end. + +async_delete_serdes(Names) -> + gen_server:cast(?MODULE, {delete_serdes, Names}). + +with_serde(Op, SerdeName, Args) -> + WhichOp = which_op(Op), + ErrMsg = error_msg(Op), + try + eval_serde(Op, ErrMsg, SerdeName, Args) + catch + throw:Reason -> + ?SLOG(error, Reason#{ + which_op => WhichOp, + reason => emqx_utils:readable_error_msg(Reason) + }), + {error, Reason}; + error:Reason:Stacktrace -> + %% unexpected errors, log stacktrace + ?SLOG(warning, #{ + msg => "plugin_schema_op_failed", + which_op => WhichOp, + exception => Reason, + stacktrace => Stacktrace + }), + {error, #{ + which_op => WhichOp, + reason => Reason + }} + end. + +eval_serde(Op, ErrMsg, SerdeName, Args) -> + case lookup_serde(SerdeName) of + {ok, Serde} -> + eval_serde(Op, Serde, Args); + {error, not_found} -> + throw(#{ + error_msg => ErrMsg, + reason => plugin_serde_not_found, + serde_name => SerdeName + }) + end. + +eval_serde(decode, #plugin_schema_serde{name = Name, eval_context = Store}, [Data]) -> + Opts = avro:make_decoder_options([{map_type, map}, {record_type, map}, {encoding, avro_json}]), + {ok, avro_json_decoder:decode_value(Data, Name, Store, Opts)}; +eval_serde(encode, #plugin_schema_serde{name = Name, eval_context = Store}, [Data]) -> + {ok, avro_json_encoder:encode(Store, Name, Data)}; +eval_serde(_, _, _) -> + throw(#{error_msg => "unexpected_plugin_avro_op"}). + +which_op(Op) -> + atom_to_list(Op) ++ "_avro_json". + +error_msg(Op) -> + atom_to_list(Op) ++ "_avro_data". + +read_avsc_file(Path) -> + case file:read_file(Path) of + {ok, Bin} -> + {ok, Bin}; + {error, _} -> + {error, #{ + error => "failed_to_read_plugin_schema", + path => Path + }} + end. + +to_bin(A) when is_atom(A) -> atom_to_binary(A); +to_bin(L) when is_list(L) -> iolist_to_binary(L); +to_bin(B) when is_binary(B) -> B. diff --git a/apps/emqx_plugins/src/emqx_plugins_sup.erl b/apps/emqx_plugins/src/emqx_plugins_sup.erl index f2092fb28..9f396c14d 100644 --- a/apps/emqx_plugins/src/emqx_plugins_sup.erl +++ b/apps/emqx_plugins/src/emqx_plugins_sup.erl @@ -32,4 +32,14 @@ init([]) -> intensity => 100, period => 10 }, - {ok, {SupFlags, []}}. + ChildSpecs = [child_spec(emqx_plugins_serde)], + {ok, {SupFlags, ChildSpecs}}. + +child_spec(Mod) -> + #{ + id => Mod, + start => {Mod, start_link, []}, + restart => permanent, + shutdown => 5_000, + type => worker + }. diff --git a/apps/emqx_plugins/test/emqx_plugins_SUITE.erl b/apps/emqx_plugins/test/emqx_plugins_SUITE.erl index 0f66e20dc..80f3d7a48 100644 --- a/apps/emqx_plugins/test/emqx_plugins_SUITE.erl +++ b/apps/emqx_plugins/test/emqx_plugins_SUITE.erl @@ -346,7 +346,7 @@ t_enable_disable(Config) -> ?assertEqual([#{name_vsn => NameVsn, enable => true}], emqx_plugins:configured()), ?assertMatch( {error, #{ - reason := "bad_plugin_config_status", + error_msg := "bad_plugin_config_status", hint := "disable_the_plugin_first" }}, emqx_plugins:ensure_uninstalled(NameVsn) @@ -374,15 +374,15 @@ t_bad_tar_gz(Config) -> ok = file:write_file(FakeTarTz, "a\n"), ?assertMatch( {error, #{ - reason := "bad_plugin_package", - return := eof + error_msg := "bad_plugin_package", + reason := eof }}, emqx_plugins:ensure_installed("fake-vsn") ), ?assertMatch( {error, #{ - reason := "failed_to_extract_plugin_package", - return := not_found + error_msg := "failed_to_extract_plugin_package", + reason := not_found }}, emqx_plugins:ensure_installed("nonexisting") ), @@ -412,7 +412,7 @@ t_bad_tar_gz2(Config) -> ?assert(filelib:is_regular(TarGz)), %% failed to install, it also cleans up the bad content of .tar.gz file ?assertMatch({error, _}, emqx_plugins:ensure_installed(NameVsn)), - ?assertEqual({error, enoent}, file:read_file_info(emqx_plugins:dir(NameVsn))), + ?assertEqual({error, enoent}, file:read_file_info(emqx_plugins:plugin_dir(NameVsn))), %% but the tar.gz file is still around ?assert(filelib:is_regular(TarGz)), ok. @@ -440,8 +440,8 @@ t_tar_vsn_content_mismatch(Config) -> %% failed to install, it also cleans up content of the bad .tar.gz file even %% if in other directory ?assertMatch({error, _}, emqx_plugins:ensure_installed(NameVsn)), - ?assertEqual({error, enoent}, file:read_file_info(emqx_plugins:dir(NameVsn))), - ?assertEqual({error, enoent}, file:read_file_info(emqx_plugins:dir("foo-0.2"))), + ?assertEqual({error, enoent}, file:read_file_info(emqx_plugins:plugin_dir(NameVsn))), + ?assertEqual({error, enoent}, file:read_file_info(emqx_plugins:plugin_dir("foo-0.2"))), %% the tar.gz file is still around ?assert(filelib:is_regular(TarGz)), ok. @@ -455,15 +455,15 @@ t_bad_info_json(Config) -> ok = write_info_file(Config, NameVsn, "bad-syntax"), ?assertMatch( {error, #{ - error := "bad_info_file", - return := {parse_error, _} + error_msg := "bad_info_file", + reason := {parse_error, _} }}, emqx_plugins:describe(NameVsn) ), ok = write_info_file(Config, NameVsn, "{\"bad\": \"obj\"}"), ?assertMatch( {error, #{ - error := "bad_info_file_content", + error_msg := "bad_info_file_content", mandatory_fields := _ }}, emqx_plugins:describe(NameVsn) @@ -499,7 +499,7 @@ t_elixir_plugin(Config) -> ok = emqx_plugins:ensure_installed(NameVsn), %% idempotent ok = emqx_plugins:ensure_installed(NameVsn), - {ok, Info} = emqx_plugins:read_plugin(NameVsn, #{}), + {ok, Info} = emqx_plugins:read_plugin_info(NameVsn, #{}), ?assertEqual([Info], emqx_plugins:list()), %% start ok = emqx_plugins:ensure_started(NameVsn), @@ -626,9 +626,9 @@ group_t_copy_plugin_to_a_new_node({init, Config}) -> } ), [CopyFromNode] = emqx_cth_cluster:start([SpecCopyFrom#{join_to => undefined}]), - ok = rpc:call(CopyFromNode, emqx_plugins, put_config, [install_dir, FromInstallDir]), + ok = rpc:call(CopyFromNode, emqx_plugins, put_config_internal, [install_dir, FromInstallDir]), [CopyToNode] = emqx_cth_cluster:start([SpecCopyTo#{join_to => undefined}]), - ok = rpc:call(CopyToNode, emqx_plugins, put_config, [install_dir, ToInstallDir]), + ok = rpc:call(CopyToNode, emqx_plugins, put_config_internal, [install_dir, ToInstallDir]), NameVsn = filename:basename(Package, ?PACKAGE_SUFFIX), ok = rpc:call(CopyFromNode, emqx_plugins, ensure_installed, [NameVsn]), ok = rpc:call(CopyFromNode, emqx_plugins, ensure_started, [NameVsn]), @@ -658,7 +658,7 @@ group_t_copy_plugin_to_a_new_node(Config) -> CopyFromNode = proplists:get_value(copy_from_node, Config), CopyToNode = proplists:get_value(copy_to_node, Config), CopyToDir = proplists:get_value(to_install_dir, Config), - CopyFromPluginsState = rpc:call(CopyFromNode, emqx_plugins, get_config, [[states], []]), + CopyFromPluginsState = rpc:call(CopyFromNode, emqx_plugins, get_config_interal, [[states], []]), NameVsn = proplists:get_value(name_vsn, Config), PluginName = proplists:get_value(plugin_name, Config), PluginApp = list_to_atom(PluginName), @@ -681,7 +681,7 @@ group_t_copy_plugin_to_a_new_node(Config) -> ), ok = rpc:call(CopyToNode, ekka, join, [CopyFromNode]), %% Mimic cluster-override conf copying - ok = rpc:call(CopyToNode, emqx_plugins, put_config, [[states], CopyFromPluginsState]), + ok = rpc:call(CopyToNode, emqx_plugins, put_config_internal, [[states], CopyFromPluginsState]), %% Plugin copying is triggered upon app restart on a new node. %% This is similar to emqx_conf, which copies cluster-override conf upon start, %% see: emqx_conf_app:init_conf/0 @@ -734,7 +734,7 @@ group_t_copy_plugin_to_a_new_node_single_node(Config) -> %% successfully even if it's not extracted yet. Simply starting %% the node would crash if not working properly. ct:pal("~p config:\n ~p", [ - CopyToNode, erpc:call(CopyToNode, emqx_plugins, get_config, [[], #{}]) + CopyToNode, erpc:call(CopyToNode, emqx_plugins, get_config_interal, [[], #{}]) ]), ct:pal("~p install_dir:\n ~p", [ CopyToNode, erpc:call(CopyToNode, file, list_dir, [ToInstallDir]) diff --git a/apps/emqx_plugins/test/emqx_plugins_tests.erl b/apps/emqx_plugins/test/emqx_plugins_tests.erl index d064cba9d..1ae0bcef3 100644 --- a/apps/emqx_plugins/test/emqx_plugins_tests.erl +++ b/apps/emqx_plugins/test/emqx_plugins_tests.erl @@ -16,6 +16,7 @@ -module(emqx_plugins_tests). +-include("emqx_plugins.hrl"). -include_lib("eunit/include/eunit.hrl"). -compile(nowarn_export_all). @@ -28,20 +29,20 @@ ensure_configured_test_todo() -> after emqx_plugins:put_configured([]) end, - meck:unload(emqx). + unmeck_emqx(). test_ensure_configured() -> ok = emqx_plugins:put_configured([]), P1 = #{name_vsn => "p-1", enable => true}, P2 = #{name_vsn => "p-2", enable => true}, P3 = #{name_vsn => "p-3", enable => false}, - emqx_plugins:ensure_configured(P1, front), - emqx_plugins:ensure_configured(P2, {before, <<"p-1">>}), - emqx_plugins:ensure_configured(P3, {before, <<"p-1">>}), + emqx_plugins:ensure_configured(P1, front, local), + emqx_plugins:ensure_configured(P2, {before, <<"p-1">>}, local), + emqx_plugins:ensure_configured(P3, {before, <<"p-1">>}, local), ?assertEqual([P2, P3, P1], emqx_plugins:configured()), ?assertThrow( #{error := "position_anchor_plugin_not_configured"}, - emqx_plugins:ensure_configured(P3, {before, <<"unknown-x">>}) + emqx_plugins:ensure_configured(P3, {before, <<"unknown-x">>}, local) ). read_plugin_test() -> @@ -49,34 +50,34 @@ read_plugin_test() -> with_rand_install_dir( fun(_Dir) -> NameVsn = "bar-5", - InfoFile = emqx_plugins:info_file(NameVsn), + InfoFile = emqx_plugins:info_file_path(NameVsn), FakeInfo = "name=bar, rel_vsn=\"5\", rel_apps=[justname_no_vsn]," "description=\"desc bar\"", try ok = write_file(InfoFile, FakeInfo), ?assertMatch( - {error, #{error := "bad_rel_apps"}}, - emqx_plugins:read_plugin(NameVsn, #{}) + {error, #{error_msg := "bad_rel_apps"}}, + emqx_plugins:read_plugin_info(NameVsn, #{}) ) after emqx_plugins:purge(NameVsn) end end ), - meck:unload(emqx). + unmeck_emqx(). with_rand_install_dir(F) -> N = rand:uniform(10000000), TmpDir = integer_to_list(N), OriginalInstallDir = emqx_plugins:install_dir(), ok = filelib:ensure_dir(filename:join([TmpDir, "foo"])), - ok = emqx_plugins:put_config(install_dir, TmpDir), + ok = emqx_plugins:put_config_internal(install_dir, TmpDir), try F(TmpDir) after file:del_dir_r(TmpDir), - ok = emqx_plugins:put_config(install_dir, OriginalInstallDir) + ok = emqx_plugins:put_config_internal(install_dir, OriginalInstallDir) end. write_file(Path, Content) -> @@ -90,7 +91,7 @@ delete_package_test() -> meck_emqx(), with_rand_install_dir( fun(_Dir) -> - File = emqx_plugins:pkg_file("a-1"), + File = emqx_plugins:pkg_file_path("a-1"), ok = write_file(File, "a"), ok = emqx_plugins:delete_package("a-1"), %% delete again should be ok @@ -100,7 +101,7 @@ delete_package_test() -> ?assertMatch({error, _}, emqx_plugins:delete_package("a-1")) end ), - meck:unload(emqx). + unmeck_emqx(). %% purge plugin's install dir should mostly work and return ok %% but it may fail in case the dir is read-only @@ -108,8 +109,8 @@ purge_test() -> meck_emqx(), with_rand_install_dir( fun(_Dir) -> - File = emqx_plugins:info_file("a-1"), - Dir = emqx_plugins:dir("a-1"), + File = emqx_plugins:info_file_path("a-1"), + Dir = emqx_plugins:plugin_dir("a-1"), ok = filelib:ensure_dir(File), ?assertMatch({ok, _}, file:read_file_info(Dir)), ?assertEqual(ok, emqx_plugins:purge("a-1")), @@ -120,10 +121,11 @@ purge_test() -> ?assertEqual(ok, emqx_plugins:purge("a-1")) end ), - meck:unload(emqx). + unmeck_emqx(). meck_emqx() -> meck:new(emqx, [unstick, passthrough]), + meck:new(emqx_plugins_serde), meck:expect( emqx, update_config, @@ -131,4 +133,14 @@ meck_emqx() -> emqx_config:put(Path, Values) end ), + meck:expect( + emqx_plugins_serde, + delete_schema, + fun(_NameVsn) -> ok end + ), + ok. + +unmeck_emqx() -> + meck:unload(emqx), + meck:unload(emqx_plugins_serde), ok. diff --git a/apps/emqx_postgresql/src/emqx_postgresql.erl b/apps/emqx_postgresql/src/emqx_postgresql.erl index f27ec8615..761c9c0f6 100644 --- a/apps/emqx_postgresql/src/emqx_postgresql.erl +++ b/apps/emqx_postgresql/src/emqx_postgresql.erl @@ -304,7 +304,7 @@ on_query( }), Type = pgsql_query_type(TypeOrKey), {NameOrSQL2, Data} = proc_sql_params(TypeOrKey, NameOrSQL, Params, State), - Res = on_sql_query(InstId, PoolName, Type, NameOrSQL2, Data), + Res = on_sql_query(TypeOrKey, InstId, PoolName, Type, NameOrSQL2, Data), ?tp(postgres_bridge_connector_on_query_return, #{instance_id => InstId, result => Res}), handle_result(Res). @@ -337,7 +337,7 @@ on_batch_query( {_Statement, RowTemplate} -> PrepStatement = get_prepared_statement(BinKey, State), Rows = [render_prepare_sql_row(RowTemplate, Data) || {_Key, Data} <- BatchReq], - case on_sql_query(InstId, PoolName, execute_batch, PrepStatement, Rows) of + case on_sql_query(Key, InstId, PoolName, execute_batch, PrepStatement, Rows) of {error, _Error} = Result -> handle_result(Result); {_Column, Results} -> @@ -386,7 +386,15 @@ get_prepared_statement(Key, #{prepares := PrepStatements}) -> BinKey = to_bin(Key), maps:get(BinKey, PrepStatements). -on_sql_query(InstId, PoolName, Type, NameOrSQL, Data) -> +on_sql_query(Key, InstId, PoolName, Type, NameOrSQL, Data) -> + emqx_trace:rendered_action_template( + Key, + #{ + statement_type => Type, + statement_or_name => NameOrSQL, + data => Data + } + ), try ecpool:pick_and_do(PoolName, {?MODULE, Type, [NameOrSQL, Data]}, no_handover) of {error, Reason} -> ?tp( diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index 6d9ad50e4..1c0c3f4ca 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -81,15 +81,15 @@ hc_workers = #{ resource => #{}, channel => #{ - pending => [], - previous_status => #{} + ongoing => #{}, + pending => [] } } :: #{ resource := #{{pid(), reference()} => true}, channel := #{ {pid(), reference()} => channel_id(), - pending := [channel_id()], - previous_status := #{channel_id() => channel_status_map()} + ongoing := #{channel_id() => channel_status_map()}, + pending := [channel_id()] } }, %% Callers waiting on health check @@ -446,6 +446,7 @@ init({DataIn, Opts}) -> terminate({shutdown, removed}, _State, _Data) -> ok; terminate(_Reason, _State, Data) -> + ok = terminate_health_check_workers(Data), _ = maybe_stop_resource(Data), _ = erase_cache(Data), ok. @@ -555,22 +556,22 @@ handle_event( {keep_state_and_data, {reply, From, {ok, Channels}}}; handle_event( info, - {'DOWN', Ref, process, Pid, Res}, + {'EXIT', Pid, Res}, State0, Data0 = #data{hc_workers = #{resource := RHCWorkers}} ) when - is_map_key({Pid, Ref}, RHCWorkers) + is_map_key(Pid, RHCWorkers) -> - handle_resource_health_check_worker_down(State0, Data0, {Pid, Ref}, Res); + handle_resource_health_check_worker_down(State0, Data0, Pid, Res); handle_event( info, - {'DOWN', Ref, process, Pid, Res}, + {'EXIT', Pid, Res}, _State, Data0 = #data{hc_workers = #{channel := CHCWorkers}} ) when - is_map_key({Pid, Ref}, CHCWorkers) + is_map_key(Pid, CHCWorkers) -> - handle_channel_health_check_worker_down(Data0, {Pid, Ref}, Res); + handle_channel_health_check_worker_down(Data0, Pid, Res); % Ignore all other events handle_event(EventType, EventData, State, Data) -> ?SLOG( @@ -634,6 +635,7 @@ health_check_actions(Data) -> handle_remove_event(From, ClearMetrics, Data) -> %% stop the buffer workers first, brutal_kill, so it should be fast ok = emqx_resource_buffer_worker_sup:stop_workers(Data#data.id, Data#data.opts), + ok = terminate_health_check_workers(Data), %% now stop the resource, this can be slow _ = stop_resource(Data), case ClearMetrics of @@ -793,6 +795,35 @@ safe_call_remove_channel(_ResId, _Mod, undefined = State, _ChannelID) -> safe_call_remove_channel(ResId, Mod, State, ChannelID) -> emqx_resource:call_remove_channel(ResId, Mod, State, ChannelID). +%% For cases where we need to terminate and there are running health checks. +terminate_health_check_workers(Data) -> + #data{ + hc_workers = #{resource := RHCWorkers, channel := CHCWorkers}, + hc_pending_callers = #{resource := RPending, channel := CPending} + } = Data, + maps:foreach( + fun(Pid, _) -> + exit(Pid, kill) + end, + RHCWorkers + ), + maps:foreach( + fun + (Pid, _) when is_pid(Pid) -> + exit(Pid, kill); + (_, _) -> + ok + end, + CHCWorkers + ), + Pending = lists:flatten([RPending, maps:values(CPending)]), + lists:foreach( + fun(From) -> + gen_statem:reply(From, {error, resource_shutting_down}) + end, + Pending + ). + make_test_id() -> RandId = iolist_to_binary(emqx_utils:gen_id(16)), <>. @@ -916,14 +947,14 @@ start_resource_health_check(#data{hc_workers = #{resource := HCWorkers}}) when keep_state_and_data; start_resource_health_check(#data{} = Data0) -> #data{hc_workers = HCWorkers0 = #{resource := RHCWorkers0}} = Data0, - WorkerRef = {_Pid, _Ref} = spawn_resource_health_check_worker(Data0), - HCWorkers = HCWorkers0#{resource := RHCWorkers0#{WorkerRef => true}}, + WorkerPid = spawn_resource_health_check_worker(Data0), + HCWorkers = HCWorkers0#{resource := RHCWorkers0#{WorkerPid => true}}, Data = Data0#data{hc_workers = HCWorkers}, {keep_state, Data}. --spec spawn_resource_health_check_worker(data()) -> {pid(), reference()}. +-spec spawn_resource_health_check_worker(data()) -> pid(). spawn_resource_health_check_worker(#data{} = Data) -> - spawn_monitor(?MODULE, worker_resource_health_check, [Data]). + spawn_link(?MODULE, worker_resource_health_check, [Data]). %% separated so it can be spec'ed and placate dialyzer tantrums... -spec worker_resource_health_check(data()) -> no_return(). @@ -1008,12 +1039,12 @@ handle_manual_channel_health_check( #data{ added_channels = Channels, hc_pending_callers = #{channel := CPending0} = Pending0, - hc_workers = #{channel := #{previous_status := PreviousStatus}} + hc_workers = #{channel := #{ongoing := Ongoing}} } = Data0, ChannelId ) when is_map_key(ChannelId, Channels), - is_map_key(ChannelId, PreviousStatus) + is_map_key(ChannelId, Ongoing) -> %% Ongoing health check. CPending = maps:update_with( @@ -1158,65 +1189,66 @@ resource_not_connected_channel_error_msg(ResourceStatus, ChannelId, Data1) -> %% `?status_connected'. -spec trigger_health_check_for_added_channels(data()) -> data(). trigger_health_check_for_added_channels(Data0 = #data{hc_workers = HCWorkers0}) -> - #{channel := CHCWorkers0} = HCWorkers0, - PreviousStatus = maps:from_list([ - {ChannelId, OldStatus} - || {ChannelId, OldStatus} <- maps:to_list(Data0#data.added_channels), - channel_status_is_channel_added(OldStatus) - ]), - ChannelsToCheck = maps:keys(PreviousStatus), + #{ + channel := CHCWorkers0 = + #{ + pending := CPending0, + ongoing := Ongoing0 + } + } = HCWorkers0, + NewOngoing = maps:filter( + fun(ChannelId, OldStatus) -> + not is_map_key(ChannelId, Ongoing0) and + channel_status_is_channel_added(OldStatus) + end, + Data0#data.added_channels + ), + ChannelsToCheck = maps:keys(NewOngoing), case ChannelsToCheck of [] -> %% Nothing to do. Data0; [ChannelId | Rest] -> %% Shooting one check at a time. We could increase concurrency in the future. - CHCWorkers = CHCWorkers0#{pending := Rest, previous_status := PreviousStatus}, + CHCWorkers = CHCWorkers0#{ + pending := CPending0 ++ Rest, + ongoing := maps:merge(Ongoing0, NewOngoing) + }, Data1 = Data0#data{hc_workers = HCWorkers0#{channel := CHCWorkers}}, start_channel_health_check(Data1, ChannelId) end. --spec continue_channel_health_check_connected(data()) -> data(). -continue_channel_health_check_connected(Data0) -> +-spec continue_channel_health_check_connected(channel_id(), channel_status_map(), data()) -> data(). +continue_channel_health_check_connected(ChannelId, OldStatus, Data0) -> #data{hc_workers = HCWorkers0} = Data0, - #{channel := #{previous_status := PreviousStatus} = CHCWorkers0} = HCWorkers0, - CHCWorkers = CHCWorkers0#{previous_status := #{}}, + #{channel := CHCWorkers0} = HCWorkers0, + CHCWorkers = emqx_utils_maps:deep_remove([ongoing, ChannelId], CHCWorkers0), Data1 = Data0#data{hc_workers = HCWorkers0#{channel := CHCWorkers}}, %% Remove the added channels with a a status different from connected or connecting - CheckedChannels = [ - {ChannelId, NewStatus} - || {ChannelId, NewStatus} <- maps:to_list(Data0#data.added_channels), - is_map_key(ChannelId, PreviousStatus) - ], - ChannelsToRemove = [ - ChannelId - || {ChannelId, NewStatus} <- CheckedChannels, - not channel_status_is_channel_added(NewStatus) - ], + NewStatus = maps:get(ChannelId, Data0#data.added_channels), + ChannelsToRemove = [ChannelId || not channel_status_is_channel_added(NewStatus)], Data = remove_channels_in_list(ChannelsToRemove, Data1, true), %% Raise/clear alarms - lists:foreach( - fun - ({ID, #{status := ?status_connected}}) -> - _ = maybe_clear_alarm(ID); - ({ID, NewStatus}) -> - OldStatus = maps:get(ID, PreviousStatus), - _ = maybe_alarm(NewStatus, ID, NewStatus, OldStatus) - end, - CheckedChannels - ), + case NewStatus of + #{status := ?status_connected} -> + _ = maybe_clear_alarm(ChannelId), + ok; + _ -> + _ = maybe_alarm(NewStatus, ChannelId, NewStatus, OldStatus), + ok + end, Data. -spec start_channel_health_check(data(), channel_id()) -> data(). start_channel_health_check(#data{} = Data0, ChannelId) -> #data{hc_workers = HCWorkers0 = #{channel := CHCWorkers0}} = Data0, - WorkerRef = {_Pid, _Ref} = spawn_channel_health_check_worker(Data0, ChannelId), - HCWorkers = HCWorkers0#{channel := CHCWorkers0#{WorkerRef => ChannelId}}, + WorkerPid = spawn_channel_health_check_worker(Data0, ChannelId), + HCWorkers = HCWorkers0#{channel := CHCWorkers0#{WorkerPid => ChannelId}}, Data0#data{hc_workers = HCWorkers}. --spec spawn_channel_health_check_worker(data(), channel_id()) -> {pid(), reference()}. +-spec spawn_channel_health_check_worker(data(), channel_id()) -> pid(). spawn_channel_health_check_worker(#data{} = Data, ChannelId) -> - spawn_monitor(?MODULE, worker_channel_health_check, [Data, ChannelId]). + spawn_link(?MODULE, worker_channel_health_check, [Data, ChannelId]). %% separated so it can be spec'ed and placate dialyzer tantrums... -spec worker_channel_health_check(data(), channel_id()) -> no_return(). @@ -1240,19 +1272,24 @@ handle_channel_health_check_worker_down(Data0, WorkerRef, ExitResult) -> %% `emqx_resource:call_channel_health_check' catches all exceptions. AddedChannels = maps:put(ChannelId, NewStatus, AddedChannels0) end, + #{ongoing := Ongoing0} = CHCWorkers1, + {PreviousChanStatus, Ongoing1} = maps:take(ChannelId, Ongoing0), + CHCWorkers2 = CHCWorkers1#{ongoing := Ongoing1}, + CHCWorkers3 = emqx_utils_maps:deep_remove([ongoing, ChannelId], CHCWorkers2), Data1 = Data0#data{added_channels = AddedChannels}, {Replies, Data2} = reply_pending_channel_health_check_callers(ChannelId, NewStatus, Data1), case CHCWorkers1 of #{pending := [NextChannelId | Rest]} -> - CHCWorkers = CHCWorkers1#{pending := Rest}, + CHCWorkers = CHCWorkers3#{pending := Rest}, HCWorkers = HCWorkers0#{channel := CHCWorkers}, Data3 = Data2#data{hc_workers = HCWorkers}, - Data = start_channel_health_check(Data3, NextChannelId), + Data4 = continue_channel_health_check_connected(ChannelId, PreviousChanStatus, Data3), + Data = start_channel_health_check(Data4, NextChannelId), {keep_state, update_state(Data, Data0), Replies}; #{pending := []} -> - HCWorkers = HCWorkers0#{channel := CHCWorkers1}, + HCWorkers = HCWorkers0#{channel := CHCWorkers3}, Data3 = Data2#data{hc_workers = HCWorkers}, - Data = continue_channel_health_check_connected(Data3), + Data = continue_channel_health_check_connected(ChannelId, PreviousChanStatus, Data3), {keep_state, update_state(Data, Data0), Replies} end. @@ -1308,7 +1345,7 @@ remove_runtime_data(#data{} = Data0) -> Data0#data{ hc_workers = #{ resource => #{}, - channel => #{pending => [], previous_status => #{}} + channel => #{pending => [], ongoing => #{}} }, hc_pending_callers = #{ resource => [], diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index 4e28efb5f..604f43d82 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -132,8 +132,6 @@ %% String Funcs -export([ - coalesce/1, - coalesce/2, lower/1, ltrim/1, reverse/1, @@ -759,10 +757,6 @@ is_array(_) -> false. %% String Funcs %%------------------------------------------------------------------------------ -coalesce(List) -> emqx_variform_bif:coalesce(List). - -coalesce(A, B) -> emqx_variform_bif:coalesce(A, B). - lower(S) -> emqx_variform_bif:lower(S). ltrim(S) -> emqx_variform_bif:ltrim(S). diff --git a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl index f99341a9b..5ec4bdc6e 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl @@ -18,6 +18,7 @@ -include("rule_engine.hrl"). -include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/emqx_trace.hrl"). -include_lib("emqx_resource/include/emqx_resource_errors.hrl"). -export([ @@ -141,22 +142,24 @@ apply_rule(Rule = #{id := RuleID}, Columns, Envs) -> set_process_trace_metadata(RuleID, #{clientid := ClientID} = Columns) -> logger:update_process_metadata(#{ - clientid => ClientID - }), - set_process_trace_metadata(RuleID, maps:remove(clientid, Columns)); + clientid => ClientID, + rule_id => RuleID, + rule_trigger_time => rule_trigger_time(Columns) + }); set_process_trace_metadata(RuleID, Columns) -> - EventTimestamp = - case Columns of - #{timestamp := Timestamp} -> - Timestamp; - _ -> - erlang:system_time(millisecond) - end, logger:update_process_metadata(#{ rule_id => RuleID, - rule_trigger_time => EventTimestamp + rule_trigger_time => rule_trigger_time(Columns) }). +rule_trigger_time(Columns) -> + case Columns of + #{timestamp := Timestamp} -> + Timestamp; + _ -> + erlang:system_time(millisecond) + end. + reset_process_trace_metadata(#{clientid := _ClientID}) -> Meta = logger:get_process_metadata(), Meta1 = maps:remove(clientid, Meta), @@ -722,7 +725,7 @@ inc_action_metrics(TraceCtx, Result) -> do_inc_action_metrics( #{rule_id := RuleId, action_id := ActId} = TraceContext, - {error, {unrecoverable_error, {action_stopped_after_template_rendering, Explanation}} = _Reason} + {error, ?EMQX_TRACE_STOP_ACTION(Explanation) = _Reason} ) -> TraceContext1 = maps:remove(action_id, TraceContext), trace_action( diff --git a/apps/emqx_s3/src/emqx_s3_client.erl b/apps/emqx_s3/src/emqx_s3_client.erl index a415cf8d4..58029ae85 100644 --- a/apps/emqx_s3/src/emqx_s3_client.erl +++ b/apps/emqx_s3/src/emqx_s3_client.erl @@ -6,6 +6,7 @@ -include_lib("emqx/include/types.hrl"). -include_lib("emqx/include/logger.hrl"). +-include_lib("snabbkaffe/include/trace.hrl"). -include_lib("erlcloud/include/erlcloud_aws.hrl"). -export([ @@ -133,7 +134,13 @@ start_multipart( Headers = join_headers(BaseHeaders, maps:get(headers, UploadOpts, undefined)), case erlcloud_s3:start_multipart(Bucket, ECKey, ECOpts, Headers, AwsConfig) of {ok, Props} -> - {ok, response_property('uploadId', Props)}; + UploadId = response_property('uploadId', Props), + ?tp(s3_client_multipart_started, #{ + bucket => Bucket, + key => Key, + upload_id => UploadId + }), + {ok, UploadId}; {error, Reason} -> ?SLOG(debug, #{msg => "start_multipart_fail", key => Key, reason => Reason}), {error, Reason} @@ -177,6 +184,11 @@ complete_multipart( ) of ok -> + ?tp(s3_client_multipart_completed, #{ + bucket => Bucket, + key => Key, + upload_id => UploadId + }), ok; {error, Reason} -> ?SLOG(debug, #{msg => "complete_multipart_fail", key => Key, reason => Reason}), @@ -193,6 +205,11 @@ abort_multipart( ) -> case erlcloud_s3:abort_multipart(Bucket, erlcloud_key(Key), UploadId, [], Headers, AwsConfig) of ok -> + ?tp(s3_client_multipart_aborted, #{ + bucket => Bucket, + key => Key, + upload_id => UploadId + }), ok; {error, Reason} -> ?SLOG(debug, #{msg => "abort_multipart_fail", key => Key, reason => Reason}), diff --git a/apps/emqx_s3/src/emqx_s3_profile_conf.erl b/apps/emqx_s3/src/emqx_s3_profile_conf.erl index 67f9d33b3..7bb5d87f1 100644 --- a/apps/emqx_s3/src/emqx_s3_profile_conf.erl +++ b/apps/emqx_s3/src/emqx_s3_profile_conf.erl @@ -53,7 +53,7 @@ emqx_s3_client:bucket(), emqx_s3_client:config(), emqx_s3_client:upload_options(), - emqx_s3_uploader:config() + emqx_s3_upload:config() }. -define(DEFAULT_CALL_TIMEOUT, 5000). diff --git a/apps/emqx_s3/src/emqx_s3_schema.erl b/apps/emqx_s3/src/emqx_s3_schema.erl index 1199948d0..1fa6d31cd 100644 --- a/apps/emqx_s3/src/emqx_s3_schema.erl +++ b/apps/emqx_s3/src/emqx_s3_schema.erl @@ -102,6 +102,14 @@ fields(s3_upload) -> desc => ?DESC("acl"), required => false } + )}, + {headers, + hoconsc:mk( + map(), + #{ + required => false, + desc => ?DESC("upload_headers") + } )} ]; fields(s3_uploader) -> diff --git a/apps/emqx_s3/src/emqx_s3_upload.erl b/apps/emqx_s3/src/emqx_s3_upload.erl new file mode 100644 index 000000000..565c6b8bc --- /dev/null +++ b/apps/emqx_s3/src/emqx_s3_upload.erl @@ -0,0 +1,217 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_s3_upload). + +-include_lib("emqx/include/types.hrl"). + +-export([ + new/4, + append/2, + write/1, + complete/1, + abort/1 +]). + +-export([format/1]). + +-export_type([t/0, config/0]). + +-type config() :: #{ + min_part_size => pos_integer(), + max_part_size => pos_integer() +}. + +-type t() :: #{ + started := boolean(), + client := emqx_s3_client:client(), + key := emqx_s3_client:key(), + upload_opts := emqx_s3_client:upload_options(), + buffer := iodata(), + buffer_size := non_neg_integer(), + min_part_size := pos_integer(), + max_part_size := pos_integer(), + upload_id := undefined | emqx_s3_client:upload_id(), + etags := [emqx_s3_client:etag()], + part_number := emqx_s3_client:part_number() +}. + +%% 5MB +-define(DEFAULT_MIN_PART_SIZE, 5242880). +%% 5GB +-define(DEFAULT_MAX_PART_SIZE, 5368709120). + +%% + +-spec new( + emqx_s3_client:client(), + emqx_s3_client:key(), + emqx_s3_client:upload_options(), + config() +) -> + t(). +new(Client, Key, UploadOpts, Config) -> + #{ + started => false, + client => Client, + key => Key, + upload_opts => UploadOpts, + buffer => [], + buffer_size => 0, + min_part_size => maps:get(min_part_size, Config, ?DEFAULT_MIN_PART_SIZE), + max_part_size => maps:get(max_part_size, Config, ?DEFAULT_MAX_PART_SIZE), + upload_id => undefined, + etags => [], + part_number => 1 + }. + +-spec append(iodata(), t()) -> {ok, t()} | {error, term()}. +append(WriteData, #{buffer := Buffer, buffer_size := BufferSize} = Upload) -> + case is_valid_part(WriteData, Upload) of + true -> + {ok, Upload#{ + buffer => [Buffer, WriteData], + buffer_size => BufferSize + iolist_size(WriteData) + }}; + false -> + {error, {too_large, iolist_size(WriteData)}} + end. + +-spec write(t()) -> {ok, t()} | {cont, t()} | {error, term()}. +write(U0 = #{started := false}) -> + case maybe_start_upload(U0) of + not_started -> + {ok, U0}; + {started, U1} -> + {cont, U1#{started := true}}; + {error, _} = Error -> + Error + end; +write(U0 = #{started := true}) -> + maybe_upload_part(U0). + +-spec complete(t()) -> {ok, t()} | {error, term()}. +complete( + #{ + started := true, + client := Client, + key := Key, + upload_id := UploadId + } = U0 +) -> + case upload_part(U0) of + {ok, #{etags := ETagsRev} = U1} -> + ETags = lists:reverse(ETagsRev), + case emqx_s3_client:complete_multipart(Client, Key, UploadId, ETags) of + ok -> + {ok, U1}; + {error, _} = Error -> + Error + end; + {error, _} = Error -> + Error + end; +complete(#{started := false} = Upload) -> + put_object(Upload). + +-spec abort(t()) -> ok_or_error(term()). +abort(#{ + started := true, + client := Client, + key := Key, + upload_id := UploadId +}) -> + case emqx_s3_client:abort_multipart(Client, Key, UploadId) of + ok -> + ok; + {error, _} = Error -> + Error + end; +abort(#{started := false}) -> + ok. + +%%-------------------------------------------------------------------- + +-spec format(t()) -> map(). +format(Upload = #{client := Client}) -> + Upload#{ + client => emqx_s3_client:format(Client), + buffer => [<<"...">>] + }. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +-spec maybe_start_upload(t()) -> not_started | {started, t()} | {error, term()}. +maybe_start_upload(#{buffer_size := BufferSize, min_part_size := MinPartSize} = Data) -> + case BufferSize >= MinPartSize of + true -> + start_upload(Data); + false -> + not_started + end. + +-spec start_upload(t()) -> {started, t()} | {error, term()}. +start_upload(#{client := Client, key := Key, upload_opts := UploadOpts} = Data) -> + case emqx_s3_client:start_multipart(Client, Key, UploadOpts) of + {ok, UploadId} -> + NewData = Data#{upload_id => UploadId}, + {started, NewData}; + {error, _} = Error -> + Error + end. + +-spec maybe_upload_part(t()) -> ok_or_error(t(), term()). +maybe_upload_part(#{buffer_size := BufferSize, min_part_size := MinPartSize} = Data) -> + case BufferSize >= MinPartSize of + true -> + upload_part(Data); + false -> + {ok, Data} + end. + +-spec upload_part(t()) -> ok_or_error(t(), term()). +upload_part(#{buffer_size := 0} = Upload) -> + {ok, Upload}; +upload_part( + #{ + client := Client, + key := Key, + upload_id := UploadId, + buffer := Buffer, + part_number := PartNumber, + etags := ETags + } = Upload +) -> + case emqx_s3_client:upload_part(Client, Key, UploadId, PartNumber, Buffer) of + {ok, ETag} -> + {ok, Upload#{ + buffer => [], + buffer_size => 0, + part_number => PartNumber + 1, + etags => [{PartNumber, ETag} | ETags] + }}; + {error, _} = Error -> + Error + end. + +-spec put_object(t()) -> ok_or_error(t(), term()). +put_object( + #{ + client := Client, + key := Key, + upload_opts := UploadOpts, + buffer := Buffer + } = Upload +) -> + case emqx_s3_client:put_object(Client, Key, UploadOpts, Buffer) of + ok -> + {ok, Upload}; + {error, _} = Error -> + Error + end. + +is_valid_part(WriteData, #{max_part_size := MaxPartSize, buffer_size := BufferSize}) -> + BufferSize + iolist_size(WriteData) =< MaxPartSize. diff --git a/apps/emqx_s3/src/emqx_s3_uploader.erl b/apps/emqx_s3/src/emqx_s3_uploader.erl index 160ecbfef..99a89eb92 100644 --- a/apps/emqx_s3/src/emqx_s3_uploader.erl +++ b/apps/emqx_s3/src/emqx_s3_uploader.erl @@ -6,7 +6,7 @@ -include_lib("emqx/include/types.hrl"). --behaviour(gen_statem). +-behaviour(gen_server). -export([ start_link/3, @@ -25,46 +25,23 @@ -export([ init/1, - callback_mode/0, - handle_event/4, - terminate/3, - code_change/4, - format_status/1, - format_status/2 + handle_call/3, + handle_cast/2, + terminate/2, + format_status/1 ]). --export_type([config/0]). - --type config() :: #{ - min_part_size => pos_integer(), - max_part_size => pos_integer() -}. - -type data() :: #{ profile_id => emqx_s3:profile_id(), - client := emqx_s3_client:client(), - key := emqx_s3_client:key(), - upload_opts := emqx_s3_client:upload_options(), - buffer := iodata(), - buffer_size := non_neg_integer(), - min_part_size := pos_integer(), - max_part_size := pos_integer(), - upload_id := undefined | emqx_s3_client:upload_id(), - etags := [emqx_s3_client:etag()], - part_number := emqx_s3_client:part_number() + upload := emqx_s3_upload:t() | aborted }. -%% 5MB --define(DEFAULT_MIN_PART_SIZE, 5242880). -%% 5GB --define(DEFAULT_MAX_PART_SIZE, 5368709120). - -define(DEFAULT_TIMEOUT, 30000). -spec start_link(emqx_s3:profile_id(), emqx_s3_client:key(), emqx_s3_client:upload_options()) -> - gen_statem:start_ret(). + gen_server:start_ret(). start_link(ProfileId, Key, UploadOpts) when is_list(Key) -> - gen_statem:start_link(?MODULE, {profile, ProfileId, Key, UploadOpts}, []). + gen_server:start_link(?MODULE, {profile, ProfileId, Key, UploadOpts}, []). -spec write(pid(), iodata()) -> ok_or_error(term()). write(Pid, WriteData) -> @@ -72,7 +49,7 @@ write(Pid, WriteData) -> -spec write(pid(), iodata(), timeout()) -> ok_or_error(term()). write(Pid, WriteData, Timeout) -> - gen_statem:call(Pid, {write, wrap(WriteData)}, Timeout). + gen_server:call(Pid, {write, wrap(WriteData)}, Timeout). -spec complete(pid()) -> ok_or_error(term()). complete(Pid) -> @@ -80,7 +57,7 @@ complete(Pid) -> -spec complete(pid(), timeout()) -> ok_or_error(term()). complete(Pid, Timeout) -> - gen_statem:call(Pid, complete, Timeout). + gen_server:call(Pid, complete, Timeout). -spec abort(pid()) -> ok_or_error(term()). abort(Pid) -> @@ -88,7 +65,7 @@ abort(Pid) -> -spec abort(pid(), timeout()) -> ok_or_error(term()). abort(Pid, Timeout) -> - gen_statem:call(Pid, abort, Timeout). + gen_server:call(Pid, abort, Timeout). -spec shutdown(pid()) -> ok. shutdown(Pid) -> @@ -99,231 +76,73 @@ shutdown(Pid) -> %% gen_statem callbacks %%-------------------------------------------------------------------- -callback_mode() -> handle_event_function. - init({profile, ProfileId, Key, UploadOpts}) -> + _ = process_flag(trap_exit, true), {Bucket, ClientConfig, BaseOpts, UploaderConfig} = emqx_s3_profile_conf:checkout_config(ProfileId), - Upload = #{ - profile_id => ProfileId, - client => client(Bucket, ClientConfig), - key => Key, - upload_opts => maps:merge(BaseOpts, UploadOpts) - }, - init({upload, UploaderConfig, Upload}); -init({upload, Config, Upload}) -> - process_flag(trap_exit, true), - {ok, upload_not_started, Upload#{ - buffer => [], - buffer_size => 0, - min_part_size => maps:get(min_part_size, Config, ?DEFAULT_MIN_PART_SIZE), - max_part_size => maps:get(max_part_size, Config, ?DEFAULT_MAX_PART_SIZE), - upload_id => undefined, - etags => [], - part_number => 1 - }}. + Upload = emqx_s3_upload:new( + client(Bucket, ClientConfig), + Key, + maps:merge(BaseOpts, UploadOpts), + UploaderConfig + ), + {ok, #{profile_id => ProfileId, upload => Upload}}. -handle_event({call, From}, {write, WriteDataWrapped}, State, Data0) -> +-spec handle_call(_Call, gen_server:from(), data()) -> + {reply, _Result, data()} | {stop, _Reason, _Result, data()}. +handle_call({write, WriteDataWrapped}, _From, St0 = #{upload := U0}) -> WriteData = unwrap(WriteDataWrapped), - case is_valid_part(WriteData, Data0) of - true -> - handle_write(State, From, WriteData, Data0); - false -> - {keep_state_and_data, {reply, From, {error, {too_large, iolist_size(WriteData)}}}} + case emqx_s3_upload:append(WriteData, U0) of + {ok, U1} -> + handle_write(St0#{upload := U1}); + {error, _} = Error -> + {reply, Error, St0} end; -handle_event({call, From}, complete, upload_not_started, Data0) -> - case put_object(Data0) of +handle_call(complete, _From, St0 = #{upload := U0}) -> + case emqx_s3_upload:complete(U0) of + {ok, U1} -> + {stop, normal, ok, St0#{upload := U1}}; + {error, _} = Error -> + {stop, Error, Error, St0} + end; +handle_call(abort, _From, St = #{upload := Upload}) -> + case emqx_s3_upload:abort(Upload) of ok -> - {stop_and_reply, normal, {reply, From, ok}}; + {stop, normal, ok, St}; {error, _} = Error -> - {stop_and_reply, Error, {reply, From, Error}, Data0} - end; -handle_event({call, From}, complete, upload_started, Data0) -> - case complete_upload(Data0) of - {ok, Data1} -> - {stop_and_reply, normal, {reply, From, ok}, Data1}; - {error, _} = Error -> - {stop_and_reply, Error, {reply, From, Error}, Data0} - end; -handle_event({call, From}, abort, upload_not_started, _Data) -> - {stop_and_reply, normal, {reply, From, ok}}; -handle_event({call, From}, abort, upload_started, Data0) -> - case abort_upload(Data0) of - ok -> - {stop_and_reply, normal, {reply, From, ok}}; - {error, _} = Error -> - {stop_and_reply, Error, {reply, From, Error}, Data0} + {stop, Error, Error, St} end. -handle_write(upload_not_started, From, WriteData, Data0) -> - Data1 = append_buffer(Data0, WriteData), - case maybe_start_upload(Data1) of - not_started -> - {keep_state, Data1, {reply, From, ok}}; - {started, Data2} -> - case upload_part(Data2) of - {ok, Data3} -> - {next_state, upload_started, Data3, {reply, From, ok}}; - {error, _} = Error -> - {stop_and_reply, Error, {reply, From, Error}, Data2} - end; +handle_write(St = #{upload := U0}) -> + case emqx_s3_upload:write(U0) of + {ok, U1} -> + {reply, ok, St#{upload := U1}}; + {cont, U1} -> + handle_write(St#{upload := U1}); {error, _} = Error -> - {stop_and_reply, Error, {reply, From, Error}, Data1} - end; -handle_write(upload_started, From, WriteData, Data0) -> - Data1 = append_buffer(Data0, WriteData), - case maybe_upload_part(Data1) of - {ok, Data2} -> - {keep_state, Data2, {reply, From, ok}}; - {error, _} = Error -> - {stop_and_reply, Error, {reply, From, Error}, Data1} + {stop, Error, Error, St} end. -terminate(Reason, _State, #{client := Client, upload_id := UploadId, key := Key}) when - (UploadId =/= undefined) andalso (Reason =/= normal) --> - emqx_s3_client:abort_multipart(Client, Key, UploadId); -terminate(_Reason, _State, _Data) -> - ok. +-spec handle_cast(_Cast, data()) -> {noreply, data()}. +handle_cast(_Cast, St) -> + {noreply, St}. -code_change(_OldVsn, StateName, State, _Extra) -> - {ok, StateName, State}. +-spec terminate(_Reason, data()) -> ok. +terminate(normal, _St) -> + ok; +terminate({shutdown, _}, _St) -> + ok; +terminate(_Reason, #{upload := Upload}) -> + emqx_s3_upload:abort(Upload). -format_status(#{data := #{client := Client} = Data} = Status) -> - Status#{ - data => Data#{ - client => emqx_s3_client:format(Client), - buffer => [<<"...">>] - } - }. - -format_status(_Opt, [PDict, State, #{client := Client} = Data]) -> - #{ - data => Data#{ - client => emqx_s3_client:format(Client), - buffer => [<<"...">>] - }, - state => State, - pdict => PDict - }. +format_status(#{state := State = #{upload := Upload}} = Status) -> + StateRedacted = State#{upload := emqx_s3_upload:format(Upload)}, + Status#{state := StateRedacted}. %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- --spec maybe_start_upload(data()) -> not_started | {started, data()} | {error, term()}. -maybe_start_upload(#{buffer_size := BufferSize, min_part_size := MinPartSize} = Data) -> - case BufferSize >= MinPartSize of - true -> - start_upload(Data); - false -> - not_started - end. - --spec start_upload(data()) -> {started, data()} | {error, term()}. -start_upload(#{client := Client, key := Key, upload_opts := UploadOpts} = Data) -> - case emqx_s3_client:start_multipart(Client, Key, UploadOpts) of - {ok, UploadId} -> - NewData = Data#{upload_id => UploadId}, - {started, NewData}; - {error, _} = Error -> - Error - end. - --spec maybe_upload_part(data()) -> ok_or_error(data(), term()). -maybe_upload_part(#{buffer_size := BufferSize, min_part_size := MinPartSize} = Data) -> - case BufferSize >= MinPartSize of - true -> - upload_part(Data); - false -> - {ok, Data} - end. - --spec upload_part(data()) -> ok_or_error(data(), term()). -upload_part(#{buffer_size := 0} = Data) -> - {ok, Data}; -upload_part( - #{ - client := Client, - key := Key, - upload_id := UploadId, - buffer := Buffer, - part_number := PartNumber, - etags := ETags - } = Data -) -> - case emqx_s3_client:upload_part(Client, Key, UploadId, PartNumber, Buffer) of - {ok, ETag} -> - NewData = Data#{ - buffer => [], - buffer_size => 0, - part_number => PartNumber + 1, - etags => [{PartNumber, ETag} | ETags] - }, - {ok, NewData}; - {error, _} = Error -> - Error - end. - --spec complete_upload(data()) -> ok_or_error(data(), term()). -complete_upload( - #{ - client := Client, - key := Key, - upload_id := UploadId - } = Data0 -) -> - case upload_part(Data0) of - {ok, #{etags := ETagsRev} = Data1} -> - ETags = lists:reverse(ETagsRev), - case emqx_s3_client:complete_multipart(Client, Key, UploadId, ETags) of - ok -> - {ok, Data1}; - {error, _} = Error -> - Error - end; - {error, _} = Error -> - Error - end. - --spec abort_upload(data()) -> ok_or_error(term()). -abort_upload( - #{ - client := Client, - key := Key, - upload_id := UploadId - } -) -> - case emqx_s3_client:abort_multipart(Client, Key, UploadId) of - ok -> - ok; - {error, _} = Error -> - Error - end. - --spec put_object(data()) -> ok_or_error(term()). -put_object( - #{ - client := Client, - key := Key, - upload_opts := UploadOpts, - buffer := Buffer - } -) -> - case emqx_s3_client:put_object(Client, Key, UploadOpts, Buffer) of - ok -> - ok; - {error, _} = Error -> - Error - end. - --spec append_buffer(data(), iodata()) -> data(). -append_buffer(#{buffer := Buffer, buffer_size := BufferSize} = Data, WriteData) -> - Data#{ - buffer => [Buffer, WriteData], - buffer_size => BufferSize + iolist_size(WriteData) - }. - -compile({inline, [wrap/1, unwrap/1]}). wrap(Data) -> fun() -> Data end. @@ -331,8 +150,5 @@ wrap(Data) -> unwrap(WrappedData) -> WrappedData(). -is_valid_part(WriteData, #{max_part_size := MaxPartSize, buffer_size := BufferSize}) -> - BufferSize + iolist_size(WriteData) =< MaxPartSize. - client(Bucket, Config) -> emqx_s3_client:create(Bucket, Config). diff --git a/apps/emqx_utils/src/emqx_utils_redact.erl b/apps/emqx_utils/src/emqx_utils_redact.erl index c830048a9..cdff70282 100644 --- a/apps/emqx_utils/src/emqx_utils_redact.erl +++ b/apps/emqx_utils/src/emqx_utils_redact.erl @@ -152,19 +152,24 @@ redact_v(_V) -> ?REDACT_VAL. deobfuscate(NewConf, OldConf) -> + deobfuscate(NewConf, OldConf, fun(_) -> false end). + +deobfuscate(NewConf, OldConf, IsSensitiveFun) -> maps:fold( fun(K, V, Acc) -> case maps:find(K, OldConf) of error -> - case is_redacted(K, V) of + case is_redacted(K, V, IsSensitiveFun) of %% don't put redacted value into new config true -> Acc; false -> Acc#{K => V} end; + {ok, OldV} when is_map(V), is_map(OldV), ?IS_KEY_HEADERS(K) -> + Acc#{K => deobfuscate(V, OldV, fun check_is_sensitive_header/1)}; {ok, OldV} when is_map(V), is_map(OldV) -> - Acc#{K => deobfuscate(V, OldV)}; + Acc#{K => deobfuscate(V, OldV, IsSensitiveFun)}; {ok, OldV} -> - case is_redacted(K, V) of + case is_redacted(K, V, IsSensitiveFun) of true -> Acc#{K => OldV}; _ -> @@ -280,6 +285,11 @@ deobfuscate_test() -> %% Don't have password before and should allow put non-redact-val into new config NewConf3 = #{foo => <<"bar3">>, password => <<"123456">>}, ?assertEqual(NewConf3, deobfuscate(NewConf3, #{foo => <<"bar">>})), + + HeaderConf1 = #{<<"headers">> => #{<<"Authorization">> => <<"Bearer token">>}}, + HeaderConf1Obs = #{<<"headers">> => #{<<"Authorization">> => ?REDACT_VAL}}, + ?assertEqual(HeaderConf1, deobfuscate(HeaderConf1Obs, HeaderConf1)), + ok. redact_header_test_() -> diff --git a/apps/emqx_utils/src/emqx_variform.erl b/apps/emqx_utils/src/emqx_variform.erl index 09a673851..97096559d 100644 --- a/apps/emqx_utils/src/emqx_variform.erl +++ b/apps/emqx_utils/src/emqx_variform.erl @@ -42,14 +42,7 @@ M =:= maps) ). --define(COALESCE_BADARG, - throw(#{ - reason => coalesce_badarg, - explain => - "must be an array, or a call to a function which returns an array, " - "for example: coalesce([a,b,c]) or coalesce(tokens(var,','))" - }) -). +-define(IS_EMPTY(X), (X =:= <<>> orelse X =:= "" orelse X =:= undefined)). %% @doc Render a variform expression with bindings. %% A variform expression is a template string which supports variable substitution @@ -99,6 +92,7 @@ eval_as_string(Expr, Bindings, _Opts) -> return_str(Str) when is_binary(Str) -> Str; return_str(Num) when is_integer(Num) -> integer_to_binary(Num); return_str(Num) when is_float(Num) -> float_to_binary(Num, [{decimals, 10}, compact]); +return_str(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8); return_str(Other) -> throw(#{ reason => bad_return, @@ -133,6 +127,10 @@ decompile(#{expr := Expression}) -> decompile(Expression) -> Expression. +eval(Atom, _Bindings, _Opts) when is_atom(Atom) -> + %% There is no atom literals in variform, + %% but some bif functions such as regex_match may return an atom. + atom_to_binary(Atom, utf8); eval({str, Str}, _Bindings, _Opts) -> unicode:characters_to_binary(Str); eval({integer, Num}, _Bindings, _Opts) -> @@ -145,6 +143,8 @@ eval({call, FuncNameStr, Args}, Bindings, Opts) -> {Mod, Fun} = resolve_func_name(FuncNameStr), ok = assert_func_exported(Mod, Fun, length(Args)), case {Mod, Fun} of + {?BIF_MOD, iif} -> + eval_iif(Args, Bindings, Opts); {?BIF_MOD, coalesce} -> eval_coalesce(Args, Bindings, Opts); _ -> @@ -158,19 +158,41 @@ eval_loop([H | T], Bindings, Opts) -> [eval(H, Bindings, Opts) | eval_loop(T, Bi %% coalesce treats var_unbound exception as empty string '' eval_coalesce([{array, Args}], Bindings, Opts) -> - NewArgs = [lists:map(fun(Arg) -> try_eval(Arg, Bindings, Opts) end, Args)], - call(?BIF_MOD, coalesce, NewArgs); + %% The input arg is an array + eval_coalesce_loop(Args, Bindings, Opts); eval_coalesce([Arg], Bindings, Opts) -> + %% Arg is an expression (which is expected to return an array) case try_eval(Arg, Bindings, Opts) of List when is_list(List) -> - call(?BIF_MOD, coalesce, List); + case lists:dropwhile(fun(I) -> ?IS_EMPTY(I) end, List) of + [] -> + <<>>; + [H | _] -> + H + end; <<>> -> <<>>; _ -> - ?COALESCE_BADARG + throw(#{ + reason => coalesce_badarg, + explain => "the arg expression did not yield an array" + }) end; -eval_coalesce(_Args, _Bindings, _Opts) -> - ?COALESCE_BADARG. +eval_coalesce(Args, Bindings, Opts) -> + %% It also accepts arbitrary number of args + %% equivalent to [{array, Args}] + eval_coalesce_loop(Args, Bindings, Opts). + +eval_coalesce_loop([], _Bindings, _Opts) -> + <<>>; +eval_coalesce_loop([Arg | Args], Bindings, Opts) -> + Result = try_eval(Arg, Bindings, Opts), + case ?IS_EMPTY(Result) of + true -> + eval_coalesce_loop(Args, Bindings, Opts); + false -> + Result + end. try_eval(Arg, Bindings, Opts) -> try @@ -180,6 +202,21 @@ try_eval(Arg, Bindings, Opts) -> <<>> end. +eval_iif([Cond, If, Else], Bindings, Opts) -> + CondVal = try_eval(Cond, Bindings, Opts), + case is_iif_condition_met(CondVal) of + true -> + eval(If, Bindings, Opts); + false -> + eval(Else, Bindings, Opts) + end. + +%% If iif condition expression yielded boolean, use the boolean value. +%% otherwise it's met as long as it's not an empty string. +is_iif_condition_met(true) -> true; +is_iif_condition_met(false) -> false; +is_iif_condition_met(V) -> not ?IS_EMPTY(V). + %% Some functions accept arbitrary number of arguments but implemented as /1. call(Mod, Fun, Args) -> erlang:apply(Mod, Fun, Args). @@ -237,6 +274,10 @@ resolve_var_value(VarName, Bindings, _Opts) -> }) end. +assert_func_exported(?BIF_MOD, coalesce, _Arity) -> + ok; +assert_func_exported(?BIF_MOD, iif, _Arity) -> + ok; assert_func_exported(Mod, Fun, Arity) -> ok = try_load(Mod), case erlang:function_exported(Mod, Fun, Arity) of diff --git a/apps/emqx_utils/src/emqx_variform_bif.erl b/apps/emqx_utils/src/emqx_variform_bif.erl index 5c598efbd..91bd4f9cf 100644 --- a/apps/emqx_utils/src/emqx_variform_bif.erl +++ b/apps/emqx_utils/src/emqx_variform_bif.erl @@ -58,9 +58,6 @@ %% Array functions -export([nth/2]). -%% Control functions --export([coalesce/1, coalesce/2]). - %% Random functions -export([rand_str/1, rand_int/1]). @@ -76,26 +73,16 @@ %% Hash functions -export([hash/2, hash_to_range/3, map_to_range/3]). --define(IS_EMPTY(X), (X =:= <<>> orelse X =:= "" orelse X =:= undefined)). +%% String compare functions +-export([str_comp/2, str_eq/2, str_lt/2, str_lte/2, str_gt/2, str_gte/2]). + +%% Number compare functions +-export([num_comp/2, num_eq/2, num_lt/2, num_lte/2, num_gt/2, num_gte/2]). %%------------------------------------------------------------------------------ %% String Funcs %%------------------------------------------------------------------------------ -%% @doc Return the first non-empty string -coalesce(A, B) when ?IS_EMPTY(A) andalso ?IS_EMPTY(B) -> - <<>>; -coalesce(A, B) when ?IS_EMPTY(A) -> - B; -coalesce(A, _B) -> - A. - -%% @doc Return the first non-empty string -coalesce([]) -> - <<>>; -coalesce([H | T]) -> - coalesce(H, coalesce(T)). - lower(S) when is_binary(S) -> string:lowercase(S). @@ -523,3 +510,57 @@ map_to_range(Int, Min, Max) when Min + (Int rem Range); map_to_range(_, _, _) -> throw(#{reason => badarg, function => ?FUNCTION_NAME}). + +compare(A, A) -> eq; +compare(A, B) when A < B -> lt; +compare(_A, _B) -> gt. + +%% @doc Compare two strings, returns +%% - 'eq' if they are the same. +%% - 'lt' if arg-1 is ordered before arg-2 +%% - `gt` if arg-1 is ordered after arg-2 +str_comp(A0, B0) -> + A = any_to_str(A0), + B = any_to_str(B0), + compare(A, B). + +%% @doc Return 'true' if two strings are the same, otherwise 'false'. +str_eq(A, B) -> eq =:= str_comp(A, B). + +%% @doc Return 'true' if arg-1 is ordered before arg-2, otherwise 'false'. +str_lt(A, B) -> lt =:= str_comp(A, B). + +%% @doc Return 'true' if arg-1 is ordered after arg-2, otherwise 'false'. +str_gt(A, B) -> gt =:= str_comp(A, B). + +%% @doc Return 'true' if arg-1 is not ordered after arg-2, otherwise 'false'. +str_lte(A, B) -> + R = str_comp(A, B), + R =:= lt orelse R =:= eq. + +%% @doc Return 'true' if arg-1 is not ordered bfore arg-2, otherwise 'false'. +str_gte(A, B) -> + R = str_comp(A, B), + R =:= gt orelse R =:= eq. + +num_comp(A, B) when is_number(A) andalso is_number(B) -> + compare(A, B). + +%% @doc Return 'true' if two numbers are the same, otherwise 'false'. +num_eq(A, B) -> eq =:= num_comp(A, B). + +%% @doc Return 'true' if arg-1 is ordered before arg-2, otherwise 'false'. +num_lt(A, B) -> lt =:= num_comp(A, B). + +%% @doc Return 'true' if arg-1 is ordered after arg-2, otherwise 'false'. +num_gt(A, B) -> gt =:= num_comp(A, B). + +%% @doc Return 'true' if arg-1 is not ordered after arg-2, otherwise 'false'. +num_lte(A, B) -> + R = num_comp(A, B), + R =:= lt orelse R =:= eq. + +%% @doc Return 'true' if arg-1 is not ordered bfore arg-2, otherwise 'false'. +num_gte(A, B) -> + R = num_comp(A, B), + R =:= gt orelse R =:= eq. diff --git a/apps/emqx_utils/test/emqx_variform_tests.erl b/apps/emqx_utils/test/emqx_variform_tests.erl index 5f9a13326..2e3c6c4d5 100644 --- a/apps/emqx_utils/test/emqx_variform_tests.erl +++ b/apps/emqx_utils/test/emqx_variform_tests.erl @@ -151,6 +151,9 @@ coalesce_test_() -> {"arg from other func", fun() -> ?assertEqual({ok, <<"b">>}, render("coalesce(tokens(a,','))", #{a => <<",,b,c">>})) end}, + {"arg from other func, but no result", fun() -> + ?assertEqual({ok, <<"">>}, render("coalesce(tokens(a,','))", #{a => <<",,,">>})) + end}, {"var unbound", fun() -> ?assertEqual({ok, <<>>}, render("coalesce(a)", #{})) end}, {"var unbound in call", fun() -> ?assertEqual({ok, <<>>}, render("coalesce(concat(a))", #{})) @@ -158,18 +161,70 @@ coalesce_test_() -> {"var unbound in calls", fun() -> ?assertEqual({ok, <<"c">>}, render("coalesce([any_to_str(a),any_to_str(b),'c'])", #{})) end}, - {"badarg", fun() -> - ?assertMatch( - {error, #{reason := coalesce_badarg}}, render("coalesce(a,b)", #{a => 1, b => 2}) + {"coalesce n-args", fun() -> + ?assertEqual( + {ok, <<"2">>}, render("coalesce(a,b)", #{a => <<"">>, b => 2}) ) end}, - {"badarg from return", fun() -> + {"coalesce 1-arg", fun() -> ?assertMatch( {error, #{reason := coalesce_badarg}}, render("coalesce(any_to_str(a))", #{a => 1}) ) end} ]. +compare_string_test_() -> + [ + %% Testing str_eq/2 + ?_assertEqual({ok, <<"true">>}, render("str_eq('a', 'a')", #{})), + ?_assertEqual({ok, <<"false">>}, render("str_eq('a', 'b')", #{})), + ?_assertEqual({ok, <<"true">>}, render("str_eq('', '')", #{})), + ?_assertEqual({ok, <<"false">>}, render("str_eq('a', '')", #{})), + + %% Testing str_lt/2 + ?_assertEqual({ok, <<"true">>}, render("str_lt('a', 'b')", #{})), + ?_assertEqual({ok, <<"false">>}, render("str_lt('b', 'a')", #{})), + ?_assertEqual({ok, <<"false">>}, render("str_lt('a', 'a')", #{})), + ?_assertEqual({ok, <<"false">>}, render("str_lt('', '')", #{})), + + ?_assertEqual({ok, <<"true">>}, render("str_gt('b', 'a')", #{})), + ?_assertEqual({ok, <<"false">>}, render("str_gt('a', 'b')", #{})), + ?_assertEqual({ok, <<"false">>}, render("str_gt('a', 'a')", #{})), + ?_assertEqual({ok, <<"false">>}, render("str_gt('', '')", #{})), + + ?_assertEqual({ok, <<"true">>}, render("str_lte('a', 'b')", #{})), + ?_assertEqual({ok, <<"true">>}, render("str_lte('a', 'a')", #{})), + ?_assertEqual({ok, <<"false">>}, render("str_lte('b', 'a')", #{})), + ?_assertEqual({ok, <<"true">>}, render("str_lte('', '')", #{})), + + ?_assertEqual({ok, <<"true">>}, render("str_gte('b', 'a')", #{})), + ?_assertEqual({ok, <<"true">>}, render("str_gte('a', 'a')", #{})), + ?_assertEqual({ok, <<"false">>}, render("str_gte('a', 'b')", #{})), + ?_assertEqual({ok, <<"true">>}, render("str_gte('', '')", #{})), + + ?_assertEqual({ok, <<"true">>}, render("str_gt(9, 10)", #{})) + ]. + +compare_numbers_test_() -> + [ + ?_assertEqual({ok, <<"true">>}, render("num_eq(1, 1)", #{})), + ?_assertEqual({ok, <<"false">>}, render("num_eq(2, 1)", #{})), + + ?_assertEqual({ok, <<"true">>}, render("num_lt(1, 2)", #{})), + ?_assertEqual({ok, <<"false">>}, render("num_lt(2, 2)", #{})), + + ?_assertEqual({ok, <<"true">>}, render("num_gt(2, 1)", #{})), + ?_assertEqual({ok, <<"false">>}, render("num_gt(1, 1)", #{})), + + ?_assertEqual({ok, <<"true">>}, render("num_lte(1, 1)", #{})), + ?_assertEqual({ok, <<"true">>}, render("num_lte(1, 2)", #{})), + ?_assertEqual({ok, <<"false">>}, render("num_lte(2, 1)", #{})), + + ?_assertEqual({ok, <<"true">>}, render("num_gte(2, -1)", #{})), + ?_assertEqual({ok, <<"true">>}, render("num_gte(2, 2)", #{})), + ?_assertEqual({ok, <<"false">>}, render("num_gte(-1, 2)", #{})) + ]. + syntax_error_test_() -> [ {"empty expression", fun() -> ?assertMatch(?SYNTAX_ERROR, render("", #{})) end}, @@ -218,3 +273,16 @@ to_range_badarg_test_() -> ?ASSERT_BADARG(map_to_range, "('a','1',2)"), ?ASSERT_BADARG(map_to_range, "('a',2,1)") ]. + +iif_test_() -> + %% if clientid has two words separated by a -, take the suffix, and append with `/#` + Expr1 = "iif(nth(2,tokens(clientid,'-')),concat([nth(2,tokens(clientid,'-')),'/#']),'')", + [ + ?_assertEqual({ok, <<"yes-A">>}, render("iif(a,'yes-A','no-A')", #{a => <<"x">>})), + ?_assertEqual({ok, <<"no-A">>}, render("iif(a,'yes-A','no-A')", #{})), + ?_assertEqual({ok, <<"2">>}, render("iif(str_eq(a,1),2,3)", #{a => 1})), + ?_assertEqual({ok, <<"3">>}, render("iif(str_eq(a,1),2,3)", #{a => <<"not-1">>})), + ?_assertEqual({ok, <<"3">>}, render("iif(str_eq(a,1),2,3)", #{})), + ?_assertEqual({ok, <<"">>}, render(Expr1, #{clientid => <<"a">>})), + ?_assertEqual({ok, <<"suffix/#">>}, render(Expr1, #{clientid => <<"a-suffix">>})) + ]. diff --git a/build b/build index 73b83a5b6..18880ca04 100755 --- a/build +++ b/build @@ -395,7 +395,7 @@ function is_ecr_and_enterprise() { ## Build the default docker image based on debian 12. make_docker() { - local EMQX_BUILDER_VERSION="${EMQX_BUILDER_VERSION:-5.3-2}" + local EMQX_BUILDER_VERSION="${EMQX_BUILDER_VERSION:-5.3-4}" local EMQX_BUILDER_PLATFORM="${EMQX_BUILDER_PLATFORM:-debian12}" local EMQX_BUILDER_OTP="${EMQX_BUILDER_OTP:-25.3.2-2}" local EMQX_BUILDER_ELIXIR="${EMQX_BUILDER_ELIXIR:-1.15.7}" diff --git a/changes/ce/feat-12940.en.md b/changes/ce/feat-12940.en.md new file mode 100644 index 000000000..77e626194 --- /dev/null +++ b/changes/ce/feat-12940.en.md @@ -0,0 +1,12 @@ +Add `ignore_readonly` argument to `PUT /configs` API. + +Prior to this change, EMQX would retrun 400 (BAD_REQUEST) if the raw config +included readonly root keys (`cluster`, `rpc`, and `node`). + +After this enhancement it can be called as `PUT /configs?ignore_readonly=true`, +EMQX will in this case ignore readonly root config keys, and apply the rest. + +For observability purposes, an info level message is logged if any readonly keys are dropped. + +Also fixed an exception when config has bad HOCON syntax (returns 500). +Now bad syntax will cause the API to return 400 (BAD_REQUEST). diff --git a/changes/ce/fix-12932.en.md b/changes/ce/fix-12932.en.md new file mode 100644 index 000000000..fee7078a7 --- /dev/null +++ b/changes/ce/fix-12932.en.md @@ -0,0 +1 @@ +Previously, if a HTTP action request received a 503 (Service Unavailable) status, it was marked as a failure and the request was not retried. This has now been fixed so that the request is retried a configurable number of times. diff --git a/changes/ce/fix-12948.en.md b/changes/ce/fix-12948.en.md new file mode 100644 index 000000000..71ea3004a --- /dev/null +++ b/changes/ce/fix-12948.en.md @@ -0,0 +1 @@ +Fixed an issue where sensitive HTTP header values like `Authorization` would be substituted by `******` after updating a connector. diff --git a/changes/ee/fix-12950.en.md b/changes/ee/fix-12950.en.md new file mode 100644 index 000000000..595833c49 --- /dev/null +++ b/changes/ee/fix-12950.en.md @@ -0,0 +1 @@ +Added a validation to prevent duplicated topics when configuring a message validation. diff --git a/changes/feat-12910.en.md b/changes/feat-12910.en.md new file mode 100644 index 000000000..d31fa7ef7 --- /dev/null +++ b/changes/feat-12910.en.md @@ -0,0 +1,15 @@ +Provided a configuration API endpoint for plugin functionality. +This allows users to describe the configuration struct of their plugins using AVRO schema. +During plugin runtime, the plugin's configuration can be accessed via the API. + +Added new API endpoints: +- `/plugins/:name/schema` + To get plugins avro schema and i18n config in one json object. +- `/plugins/:name/config` + To get or update plugin's own config + +Changed API endpoints: +- `/plugins/install` + Status code when succeeded change to `204`. It was `200` previously. +- `/plugins/:name/move` + Status code when succeeded change to `204`. It was `200` previously. diff --git a/deploy/charts/emqx-enterprise/Chart.yaml b/deploy/charts/emqx-enterprise/Chart.yaml index 3178cce62..2f363bdfc 100644 --- a/deploy/charts/emqx-enterprise/Chart.yaml +++ b/deploy/charts/emqx-enterprise/Chart.yaml @@ -14,8 +14,8 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 5.6.1 +version: 5.7.0-alpha.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 5.6.1 +appVersion: 5.7.0-alpha.1 diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index ea7bb27cc..1bcc9f30c 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -1,4 +1,4 @@ -ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.3-2:1.15.7-26.2.1-2-debian12 +ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-debian12 ARG RUN_FROM=public.ecr.aws/debian/debian:12-slim ARG SOURCE_TYPE=src # tgz diff --git a/mix.exs b/mix.exs index bbb11cc52..d77bfc99c 100644 --- a/mix.exs +++ b/mix.exs @@ -65,14 +65,14 @@ defmodule EMQXUmbrella.MixProject do # maybe forbid to fetch quicer {:emqtt, github: "emqx/emqtt", tag: "1.10.1", override: true, system_env: maybe_no_quic_env()}, - {:rulesql, github: "emqx/rulesql", tag: "0.2.0"}, + {:rulesql, github: "emqx/rulesql", tag: "0.2.1"}, {:observer_cli, "1.7.1"}, {:system_monitor, github: "ieQu1/system_monitor", tag: "3.0.3"}, {:telemetry, "1.1.0"}, # in conflict by emqtt and hocon {:getopt, "1.0.2", override: true}, {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.8", override: true}, - {:hocon, github: "emqx/hocon", tag: "0.42.1", override: true}, + {:hocon, github: "emqx/hocon", tag: "0.42.2", override: true}, {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.3", override: true}, {:esasl, github: "emqx/esasl", tag: "0.2.1"}, {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"}, @@ -101,7 +101,8 @@ defmodule EMQXUmbrella.MixProject do {:bcrypt, github: "emqx/erlang-bcrypt", tag: "0.6.2", override: true}, {:uuid, github: "okeuday/uuid", tag: "v2.0.6", override: true}, {:quickrand, github: "okeuday/quickrand", tag: "v2.0.6", override: true}, - {:ra, "2.7.3", override: true} + {:ra, "2.7.3", override: true}, + {:mimerl, "1.2.0", override: true} ] ++ emqx_apps(profile_info, version) ++ enterprise_deps(profile_info) ++ jq_dep() ++ quicer_dep() diff --git a/rebar.config b/rebar.config index e0f88893c..a0488b509 100644 --- a/rebar.config +++ b/rebar.config @@ -91,13 +91,13 @@ {replayq, {git, "https://github.com/emqx/replayq.git", {tag, "0.3.8"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.10.1"}}}, - {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.2.0"}}}, + {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.2.1"}}}, % NOTE: depends on recon 2.5.x {observer_cli, "1.7.1"}, {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}}, {getopt, "1.0.2"}, {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.8"}}}, - {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.42.1"}}}, + {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.42.2"}}}, {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.3"}}}, {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.1"}}}, {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}}, @@ -111,7 +111,8 @@ {ssl_verify_fun, "1.1.7"}, {rfc3339, {git, "https://github.com/emqx/rfc3339.git", {tag, "0.2.3"}}}, {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.2"}}}, - {ra, "2.7.3"} + {ra, "2.7.3"}, + {mimerl, "1.2.0"} ]}. {xref_ignores, diff --git a/rel/i18n/emqx_bridge_s3.hocon b/rel/i18n/emqx_bridge_s3.hocon index fe10313e0..531ba9823 100644 --- a/rel/i18n/emqx_bridge_s3.hocon +++ b/rel/i18n/emqx_bridge_s3.hocon @@ -5,19 +5,4 @@ config_connector.label: config_connector.desc: """Configuration for a connector to S3 API compatible storage service.""" -s3_upload.label: -"""S3 Simple Upload""" -s3_upload.desc: -"""Action to upload a single object to the S3 service.""" - -s3_upload_parameters.label: -"""S3 Upload action parameters""" -s3_upload_parameters.desc: -"""Set of parameters for the upload action. Action supports templates in S3 bucket name, object key and object content.""" - -s3_object_content.label: -"""S3 Object Content""" -s3_object_content.desc: -"""Content of the S3 object being uploaded. Supports templates.""" - } diff --git a/rel/i18n/emqx_bridge_s3_aggreg_upload.hocon b/rel/i18n/emqx_bridge_s3_aggreg_upload.hocon new file mode 100644 index 000000000..07239a32d --- /dev/null +++ b/rel/i18n/emqx_bridge_s3_aggreg_upload.hocon @@ -0,0 +1,64 @@ +emqx_bridge_s3_aggreg_upload { + +s3_aggregated_upload.label: +"""S3 Aggregated Upload""" +s3_aggregated_upload.desc: +"""Action that enables time-based aggregation of incoming events and uploading them to the S3 service as a single object.""" + +s3_aggregated_upload_parameters.label: +"""S3 Aggregated Upload action parameters""" +s3_aggregated_upload_parameters.desc: +"""Set of parameters for the aggregated upload action.""" + +s3_aggregation.label: +"""Aggregation parameters""" +s3_aggregation.desc: +"""Set of parameters governing the aggregation process.""" + +s3_aggregation_interval.label: +"""Time interval""" +s3_aggregation_interval.desc: +"""Amount of time events will be aggregated in a single object before uploading.""" + +s3_aggregation_max_records.label: +"""Maximum number of records""" +s3_aggregation_max_records.desc: +"""Number of records (events) allowed per each aggregated object. Each aggregated upload will contain no more than that number of events, but may contain less.
+If event rate is high enough, there obviously may be more than one aggregated upload during the same time interval. These uploads will have different, but consecutive sequence numbers, which will be a part of S3 object key.""" + +s3_aggregated_container.label: +"""Container for aggregated events""" +s3_aggregated_container.desc: +"""Settings governing the file format of an upload containing aggregated events.""" + +s3_aggregated_container_csv.label: +"""CSV container""" +s3_aggregated_container_csv.desc: +"""Records (events) will be aggregated and uploaded as a CSV file.""" + +s3_aggregated_container_csv_column_order.label: +"""CSV column order""" +s3_aggregated_container_csv_column_order.desc: +"""Event fields that will be ordered first as columns in the resulting CSV file.
+Regardless of this setting, resulting CSV will contain all the fields of aggregated events, but all the columns not explicitly mentioned here will be ordered after the ones listed here in the lexicographical order.""" + +s3_aggregated_upload_key.label: +"""S3 object key template""" +s3_aggregated_upload_key.desc: +"""Template for the S3 object key of an aggregated upload.
+Template may contain placeholders for the following variables: +
    +
  • ${action}: name of the action (required).
  • +
  • ${node}: name of the EMQX node conducting the upload (required).
  • +
  • ${datetime.{format}}: date and time when aggregation started, formatted according to the {format} string (required): +
      +
    • ${datetime.rfc3339utc}: RFC3339-formatted date and time in UTC,
    • +
    • ${datetime.rfc3339}: RFC3339-formatted date and time in local timezone,
    • +
    • ${datetime.unix}: Unix timestamp.
    • +
    +
  • +
  • ${datetime_until.{format}}: date and time when aggregation ended, with the same formatting options.
  • +
  • ${sequence}: sequence number of the aggregated upload within the same time interval (required).
  • +
+All other placeholders are considered invalid. Note that placeholders marked as required will be added as a path suffix to the S3 object key if they are missing from the template.""" +} diff --git a/rel/i18n/emqx_bridge_s3_upload.hocon b/rel/i18n/emqx_bridge_s3_upload.hocon new file mode 100644 index 000000000..7d08cfaa5 --- /dev/null +++ b/rel/i18n/emqx_bridge_s3_upload.hocon @@ -0,0 +1,18 @@ +emqx_bridge_s3_upload { + +s3_upload.label: +"""S3 Simple Upload""" +s3_upload.desc: +"""Action to upload a single object to the S3 service.""" + +s3_upload_parameters.label: +"""S3 Upload action parameters""" +s3_upload_parameters.desc: +"""Set of parameters for the upload action. Action supports templates in S3 bucket name, object key and object content.""" + +s3_object_content.label: +"""S3 Object Content""" +s3_object_content.desc: +"""Content of the S3 object being uploaded. Supports templates.""" + +} diff --git a/rel/i18n/emqx_s3_schema.hocon b/rel/i18n/emqx_s3_schema.hocon index 44f4bbc56..b8b963539 100644 --- a/rel/i18n/emqx_s3_schema.hocon +++ b/rel/i18n/emqx_s3_schema.hocon @@ -18,6 +18,13 @@ key.desc: key.label: """Object Key""" +upload_headers.label: +"""S3 object HTTP headers""" + +upload_headers.desc: +"""HTTP headers to include in the S3 object upload request.
+Useful to specify content type, content encoding, etc. of the S3 object.""" + host.desc: """The host of the S3 endpoint.""" diff --git a/rel/i18n/emqx_schema.hocon b/rel/i18n/emqx_schema.hocon index cb504694c..33e659622 100644 --- a/rel/i18n/emqx_schema.hocon +++ b/rel/i18n/emqx_schema.hocon @@ -268,9 +268,15 @@ common_ssl_opts_schema_cacerts.desc: common_ssl_opts_schema_cacerts.label: """Use System CA Certificates""" -fields_ws_opts_mqtt_path.desc: -"""WebSocket's MQTT protocol path. So the address of EMQX Broker's WebSocket is: -ws://{ip}:{port}/mqtt""" +fields_ws_opts_mqtt_path.desc: """~ + WebSocket's MQTT protocol path. By default, the full URL for the WebSocket client to connect is: + `ws://{ip}:{port}/mqtt`. + Append `/[...]` to the end of the path to make EMQX accept any subpath. + For example, specifying `mqtt/[...]` would allow clients to connect at paths like + `mqtt/org1` or `mqtt/group2`, etc. + + NOTE: An unmatched path will cause the client to be rejected immediately at the HTTP layer, + meaning it will not be traceable at the MQTT layer.""" fields_ws_opts_mqtt_path.label: """WS MQTT Path""" @@ -1192,11 +1198,18 @@ fields_mqtt_quic_listener_max_ack_delay_ms.desc: fields_mqtt_quic_listener_max_ack_delay_ms.label: """Max ack delay ms""" -base_listener_zone.desc: -"""The configuration zone to which the listener belongs.""" +base_listener_zone.desc: """~ + The configuration zone to which the listener belongs. + Clients connected to this listener will inherit zone-settings created under this zone name. -base_listener_zone.label: -"""Zone""" + A zone can override the configs under below root names: + - `mqtt` + - `force_shutdown` + - `force_gc` + - `flapping_detect` + - `session_persistence`""" + +base_listener_zone.label: "Zone" fields_mqtt_quic_listener_handshake_idle_timeout.desc: """How long a handshake can idle before it is discarded.""" diff --git a/scripts/buildx.sh b/scripts/buildx.sh index 90d33e9be..dbe20d501 100755 --- a/scripts/buildx.sh +++ b/scripts/buildx.sh @@ -9,7 +9,7 @@ ## example: ## ./scripts/buildx.sh --profile emqx --pkgtype tgz --arch arm64 \ -## --builder ghcr.io/emqx/emqx-builder/5.3-2:1.15.7-26.2.1-2-debian12 +## --builder ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-debian12 set -euo pipefail @@ -24,7 +24,7 @@ help() { echo "--arch amd64|arm64: Target arch to build the EMQX package for" echo "--src_dir : EMQX source code in this dir, default to PWD" echo "--builder : Builder image to pull" - echo " E.g. ghcr.io/emqx/emqx-builder/5.3-2:1.15.7-26.2.1-2-debian12" + echo " E.g. ghcr.io/emqx/emqx-builder/5.3-4:1.15.7-26.2.1-2-debian12" } die() { diff --git a/scripts/pr-sanity-checks.sh b/scripts/pr-sanity-checks.sh index f9bf53bf2..61265c7e5 100755 --- a/scripts/pr-sanity-checks.sh +++ b/scripts/pr-sanity-checks.sh @@ -12,7 +12,7 @@ if ! type "yq" > /dev/null; then exit 1 fi -EMQX_BUILDER_VERSION=${EMQX_BUILDER_VERSION:-5.3-2} +EMQX_BUILDER_VERSION=${EMQX_BUILDER_VERSION:-5.3-4} EMQX_BUILDER_OTP=${EMQX_BUILDER_OTP:-26.2.1-2} EMQX_BUILDER_ELIXIR=${EMQX_BUILDER_ELIXIR:-1.15.7} EMQX_BUILDER_PLATFORM=${EMQX_BUILDER_PLATFORM:-ubuntu22.04} diff --git a/scripts/rel/cut.sh b/scripts/rel/cut.sh index 95cd7a1d4..724b0cab2 100755 --- a/scripts/rel/cut.sh +++ b/scripts/rel/cut.sh @@ -129,6 +129,12 @@ rel_branch() { e5.6.*) echo 'release-56' ;; + v5.7.*) + echo 'release-57' + ;; + e5.7.*) + echo 'release-57' + ;; *) logerr "Unsupported version tag $TAG" exit 1 diff --git a/scripts/rel/sync-remotes.sh b/scripts/rel/sync-remotes.sh index 6492c90c0..430021a79 100755 --- a/scripts/rel/sync-remotes.sh +++ b/scripts/rel/sync-remotes.sh @@ -5,7 +5,7 @@ set -euo pipefail # ensure dir cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." -BASE_BRANCHES=( 'release-56' 'release-55' 'master' ) +BASE_BRANCHES=( 'release-57' 'release-56' 'release-55' 'master' ) usage() { cat <