yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(ds): Fix schema checker warnings for DS schema

This commit is contained in:
ieQu1 2024-02-23 14:56:41 +01:00
parent dd2e35345f
commit e126393cbf
No known key found for this signature in database
GPG Key ID: 488654DF3FED6FDE
6 changed files with 90 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
apps/emqx/src/emqx_ds_schema.erl
View File

@ -76,12 +76,12 @@ schema() ->
[
{messages,
ds_schema(#{
desc => ?DESC(messages),
importance => ?IMPORTANCE_MEDIUM,
default =>
#{
<<"backend">> => builtin
}
},
importance => ?IMPORTANCE_MEDIUM,
desc => ?DESC(messages)
})}
].
@ -92,38 +92,38 @@ fields(builtin) ->
sc(
builtin,
#{
importance => ?IMPORTANCE_MEDIUM,
'readOnly' => true,
default => builtin,
desc => ?DESC(builtin)
importance => ?IMPORTANCE_MEDIUM,
desc => ?DESC(builtin_backend)
}
)},
{'_config_handler',
sc(
{module(), atom()},
#{
importance => ?IMPORTANCE_HIDDEN,
'readOnly' => true,
default => {?MODULE, translate_builtin}
default => {?MODULE, translate_builtin},
importance => ?IMPORTANCE_HIDDEN
}
)},
{data_dir,
sc(
string(),
#{
desc => ?DESC(builtin_data_dir),
mapping => "emqx_durable_storage.db_data_dir",
required => false,
importance => ?IMPORTANCE_MEDIUM
importance => ?IMPORTANCE_MEDIUM,
desc => ?DESC(builtin_data_dir)
}
)},
{n_shards,
sc(
pos_integer(),
#{
default => 16,
importance => ?IMPORTANCE_MEDIUM,
desc => ?DESC(builtin_n_shards),
default => 16
desc => ?DESC(builtin_n_shards)
}
)},
{replication_factor,
@ -134,22 +134,20 @@ fields(builtin) ->
importance => ?IMPORTANCE_HIDDEN
}
)},
{egress,
{local_write_buffer,
sc(
ref(builtin_egress),
ref(builtin_local_write_buffer),
#{
desc => ?DESC(builtin_egress),
importance => ?IMPORTANCE_MEDIUM
importance => ?IMPORTANCE_MEDIUM,
desc => ?DESC(builtin_local_write_buffer)
}
)},
{layout,
sc(
hoconsc:union([
ref(layout_builtin_wildcard_optimized), ref(layout_builtin_reference)
]),
hoconsc:union(builtin_layouts()),
#{
desc => ?DESC(builtin_layout),
importance => ?IMPORTANCE_HIDDEN,
importance => ?IMPORTANCE_MEDIUM,
default =>
#{
<<"type">> => wildcard_optimized
@ -157,7 +155,7 @@ fields(builtin) ->
}
)}
];
fields(builtin_egress) ->
fields(builtin_local_write_buffer) ->
[
{max_items,
sc(
@ -166,7 +164,7 @@ fields(builtin_egress) ->
default => 1000,
mapping => "emqx_durable_storage.egress_batch_size",
importance => ?IMPORTANCE_MEDIUM,
desc => ?DESC(egress_max_items)
desc => ?DESC(builtin_local_write_buffer_max_items)
}
)},
{flush_interval,
@ -176,7 +174,7 @@ fields(builtin_egress) ->
default => 100,
mapping => "emqx_durable_storage.egress_flush_interval",
importance => ?IMPORTANCE_MEDIUM,
desc => ?DESC(egress_flush_interval)
desc => ?DESC(builtin_local_write_buffer_flush_interval)
}
)}
];
@ -186,9 +184,9 @@ fields(layout_builtin_wildcard_optimized) ->
sc(
wildcard_optimized,
#{
desc => ?DESC(layout_wildcard_optimized),
'readOnly' => true,
default => wildcard_optimized
default => wildcard_optimized,
desc => ?DESC(layout_builtin_wildcard_optimized_type)
}
)},
{bits_per_topic_level,
@ -222,10 +220,19 @@ fields(layout_builtin_reference) ->
{type,
sc(
reference,
#{'readOnly' => true}
#{
'readOnly' => true,
importance => ?IMPORTANCE_HIDDEN
}
)}
].
desc(builtin) ->
?DESC(builtin);
desc(builtin_local_write_buffer) ->
?DESC(builtin_local_write_buffer);
desc(layout_builtin_wildcard_optimized) ->
?DESC(layout_builtin_wildcard_optimized);
desc(_) ->
undefined.
@ -242,6 +249,18 @@ ds_schema(Options) ->
Options
).
-ifndef(TEST).
builtin_layouts() ->
[ref(layout_builtin_wildcard_optimized)].
-else.
builtin_layouts() ->
%% Reference layout stores everything in one stream, so it's not
%% suitable for production use. However, it's very simple and
%% produces a very predictabale replay order, which can be useful
%% for testing and debugging:
[ref(layout_builtin_wildcard_optimized), ref(layout_builtin_reference)].
-endif.
sc(Type, Meta) -> hoconsc:mk(Type, Meta).
ref(StructName) -> hoconsc:ref(?MODULE, StructName).

8
apps/emqx/src/emqx_schema.erl
View File

@ -255,9 +255,9 @@ roots(medium) ->
ref("overload_protection"),
#{importance => ?IMPORTANCE_HIDDEN}
)},
{"durable_storage",
{durable_storage,
sc(
ref("durable_storage"),
ref(durable_storage),
#{
importance => ?IMPORTANCE_MEDIUM,
desc => ?DESC(durable_storage)
@ -1730,7 +1730,7 @@ fields("session_persistence") ->
}
)}
];
fields("durable_storage") ->
fields(durable_storage) ->
emqx_ds_schema:schema().
mqtt_listener(Bind) ->
@ -1985,6 +1985,8 @@ desc("crl_cache") ->
"Global CRL cache options.";
desc("session_persistence") ->
"Settings governing durable sessions persistence.";
desc(durable_storage) ->
?DESC(durable_storage);
desc(_) ->
undefined.

3
changes/ce/fix-12562.en.md Normal file
View File

@ -0,0 +1,3 @@
Add a new configuration root: `durable_storage`.
This configuration tree contains the settings related to the new persistent session feature.

55
rel/i18n/emqx_ds_schema.hocon
View File

@ -10,25 +10,46 @@ builtin.desc:
"""~
Builtin session storage backend utilizing embedded RocksDB key-value store.~"""
builtin_backend.label: "Backend type"
builtin_backend.desc:
"""~
Built-in backend.~"""
builtin_data_dir.label: "Database location"
builtin_data_dir.desc:
"""~
File system directory where the database is located.
By default it is equal to `node.data_dir`.~"""
By default, it is equal to `node.data_dir`.~"""
builtin_n_shards.label: "Number of shards"
builtin_n_shards.desc:
"""~
The builtin durable storage partitions data into shards.
The built-in durable storage partitions data into shards.
This configuration parameter defines the number of shards.
Please note that it takes effect only during the initialization of the durable storage database.
Changing this configuration parameter after the database has been already created won't take any effect.~"""
builtin_egress.label: "Egress configuration"
builtin_egress.desc:
builtin_local_write_buffer.label: "Local write buffer"
builtin_local_write_buffer.desc:
"""~
Configuration related to the buffering of messages from the local node to the shard leader.~"""
Configuration related to the buffering of messages sent from the local node to the shard leader.
EMQX accumulates PUBLISH messages from the local clients in a write buffer before committing them to the durable storage.
This helps to hide network latency between EMQX nodes and improves write throughput.~"""
builtin_local_write_buffer_max_items.label: "Max items"
builtin_local_write_buffer_max_items.desc:
"""~
This configuration parameter defines maximum number of messages stored in the local write buffer.~"""
builtin_local_write_buffer_flush_interval.label: "Flush interval"
builtin_local_write_buffer_flush_interval.desc:
"""~
Maximum linger time for the buffered messages.
Local write buffer will be flushed _at least_ as often as `flush_interval`.
Larger values of `flush_interval` may lead to higher throughput and better overall performance, but may increase end-to-end latency.~"""
builtin_layout.label: "Storage layout"
builtin_layout.desc:
@ -37,8 +58,8 @@ builtin_layout.desc:
Depending on the type of workload and the topic structure, different types of strategies for storing the data can be employed to maximize efficiency of reading messages from the durable storage.~"""
layout_wildcard_optimized.label: "Wildcard-optimized storage layout"
layout_wildcard_optimized.desc:
layout_builtin_wildcard_optimized.label: "Wildcard-optimized storage layout"
layout_builtin_wildcard_optimized.desc:
"""~
_Wildcard-optimized_ layout is designed to maximize the throughput of wildcard subscriptions covering large numbers of topics.
@ -47,11 +68,16 @@ layout_wildcard_optimized.desc:
This layout is efficient for non-wildcard subscriptions as well.~"""
layout_builtin_wildcard_optimized_type.label: "Layout type"
layout_builtin_wildcard_optimized_type.desc:
"""~
Wildcard-optimized layout type.~"""
wildcard_optimized_epoch_bits.label: "Epoch bits"
wildcard_optimized_epoch_bits.desc:
"""~
Wildcard-optimized layout partitions messages recorded at different times into "epochs".
Reading messages from a single epoch can be done very efficiently, so larger epochs improve the throughput of subscribers, but may increase end-to-end latency
Reading messages from a single epoch can be done very efficiently, so larger epochs improve the throughput of subscribers, but may increase end-to-end latency.
Time span covered by each epoch grows exponentially with the value of `epoch_bits`:
@ -60,17 +86,6 @@ wildcard_optimized_epoch_bits.desc:
...
- `epoch_bits = 10`: 1024 milliseconds
- `epoch_bits = 13`: ~8 seconds
....~"""
egress_max_items.label: "Max items"
egress_max_items.desc:
"""~
This configuration parameter defines maximum number of buffered messages stored in the egress buffer.~"""
egress_flush_interval.label: "Flush interval"
egress_flush_interval.desc:
"""~
Maximum linger time for the buffered messages.
Egress buffer will be flushed _at least_ as often as `flush_interval`.~"""
...~"""
}

2
rel/i18n/emqx_schema.hocon
View File

@ -1539,7 +1539,7 @@ session_persistence_enable.desc:
If enabled, sessions configured to outlive client connections, along with their corresponding messages, will be durably stored and survive broker downtime.
:::warning
This feature is currently experimental. Please don't enable it in the producation environments that contain valuable data.
This feature is currently experimental. Please don't enable it in the production environments that contain valuable data.
:::"""

2
scripts/spellcheck/dicts/emqx.txt
View File

@ -284,9 +284,11 @@ TDengine
clickhouse
FormatType
RocketMQ
RocksDB
Keyspace
OpenTSDB
saml
storages
idp
ocpp
OCPP