diff --git a/apps/emqx_resource/src/schema/emqx_resource_schema.erl b/apps/emqx_resource/src/schema/emqx_resource_schema.erl index 4b36f5b89..eb0a48b06 100644 --- a/apps/emqx_resource/src/schema/emqx_resource_schema.erl +++ b/apps/emqx_resource/src/schema/emqx_resource_schema.erl @@ -23,6 +23,8 @@ -export([namespace/0, roots/0, fields/1, desc/1]). +-export([create_opts/1]). + %% range interval in ms -define(HEALTH_CHECK_INTERVAL_RANGE_MIN, 1). -define(HEALTH_CHECK_INTERVAL_RANGE_MAX, 3_600_000). @@ -43,25 +45,41 @@ fields("resource_opts") -> )} ]; fields("creation_opts") -> - [ - {buffer_mode, fun buffer_mode/1}, - {worker_pool_size, fun worker_pool_size/1}, - {health_check_interval, fun health_check_interval/1}, - {resume_interval, fun resume_interval/1}, - {metrics_flush_interval, fun metrics_flush_interval/1}, - {start_after_created, fun start_after_created/1}, - {start_timeout, fun start_timeout/1}, - {auto_restart_interval, fun auto_restart_interval/1}, - {query_mode, fun query_mode/1}, - {request_ttl, fun request_ttl/1}, - {inflight_window, fun inflight_window/1}, - {enable_batch, fun enable_batch/1}, - {batch_size, fun batch_size/1}, - {batch_time, fun batch_time/1}, - {enable_queue, fun enable_queue/1}, - {max_buffer_bytes, fun max_buffer_bytes/1}, - {buffer_seg_bytes, fun buffer_seg_bytes/1} - ]. + create_opts([]). + +create_opts(Overrides) -> + override( + [ + {buffer_mode, fun buffer_mode/1}, + {worker_pool_size, fun worker_pool_size/1}, + {health_check_interval, fun health_check_interval/1}, + {resume_interval, fun resume_interval/1}, + {metrics_flush_interval, fun metrics_flush_interval/1}, + {start_after_created, fun start_after_created/1}, + {start_timeout, fun start_timeout/1}, + {auto_restart_interval, fun auto_restart_interval/1}, + {query_mode, fun query_mode/1}, + {request_ttl, fun request_ttl/1}, + {inflight_window, fun inflight_window/1}, + {enable_batch, fun enable_batch/1}, + {batch_size, fun batch_size/1}, + {batch_time, fun batch_time/1}, + {enable_queue, fun enable_queue/1}, + {max_buffer_bytes, fun max_buffer_bytes/1}, + {buffer_seg_bytes, fun buffer_seg_bytes/1} + ], + Overrides + ). + +override([], _) -> + []; +override([{Name, Sc} | Rest], Overrides) -> + case lists:keyfind(Name, 1, Overrides) of + {Name, Override} -> + [{Name, hocon_schema:override(Sc, Override)} | override(Rest, Overrides)]; + false -> + [{Name, Sc} | override(Rest, Overrides)] + end. resource_opts_meta() -> #{ @@ -142,6 +160,7 @@ request_ttl(_) -> undefined. enable_batch(type) -> boolean(); enable_batch(required) -> false; enable_batch(default) -> true; +enable_batch(importance) -> ?IMPORTANCE_HIDDEN; enable_batch(deprecated) -> {since, "v5.0.14"}; enable_batch(desc) -> ?DESC("enable_batch"); enable_batch(_) -> undefined. @@ -169,6 +188,7 @@ batch_size(_) -> undefined. batch_time(type) -> emqx_schema:timeout_duration_ms(); batch_time(desc) -> ?DESC("batch_time"); batch_time(default) -> ?DEFAULT_BATCH_TIME_RAW; +batch_time(importance) -> ?IMPORTANCE_LOW; batch_time(required) -> false; batch_time(_) -> undefined. diff --git a/changes/ee/fix-10998.en.md b/changes/ee/fix-10998.en.md new file mode 100644 index 000000000..682727551 --- /dev/null +++ b/changes/ee/fix-10998.en.md @@ -0,0 +1,2 @@ +Do not allow `batch_size` option for MongoDB bridge resource. +MongoDB connector currently does not support batching, the `bath_size` config value is forced to be 1 if provided. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mongodb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mongodb.erl index fec5a4a7f..e115bbf37 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mongodb.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mongodb.erl @@ -37,8 +37,24 @@ fields("config") -> [ {enable, mk(boolean(), #{desc => ?DESC("enable"), default => true})}, {collection, mk(binary(), #{desc => ?DESC("collection"), default => <<"mqtt">>})}, - {payload_template, mk(binary(), #{required => false, desc => ?DESC("payload_template")})} - ] ++ emqx_resource_schema:fields("resource_opts"); + {payload_template, mk(binary(), #{required => false, desc => ?DESC("payload_template")})}, + {resource_opts, + mk( + ref(?MODULE, "creation_opts"), + #{required => true, desc => ?DESC(emqx_resource_schema, "creation_opts")} + )} + ]; +fields("creation_opts") -> + %% so far, mongodb connector does not support batching + %% but we cannot delete this field due to compatibility reasons + %% so we'll keep this field, but hide it in the docs. + emqx_resource_schema:create_opts([ + {batch_size, #{ + importance => ?IMPORTANCE_HIDDEN, + converter => fun(_, _) -> 1 end, + desc => ?DESC("batch_size") + }} + ]); fields(mongodb_rs) -> emqx_connector_mongo:fields(rs) ++ fields("config"); fields(mongodb_sharded) -> diff --git a/rel/i18n/emqx_ee_bridge_mongodb.hocon b/rel/i18n/emqx_ee_bridge_mongodb.hocon index fab371824..b703bf5a5 100644 --- a/rel/i18n/emqx_ee_bridge_mongodb.hocon +++ b/rel/i18n/emqx_ee_bridge_mongodb.hocon @@ -54,4 +54,9 @@ payload_template.desc: payload_template.label: """Payload template""" +batch_size.desc: +"""There is no batching support for MongoDB at the moment, so this config field has no effect. Internally the value is overridden to 1.""" +batch_size.label: +"""Batch Size""" + }