Merge pull request #12517 from zmstone/0213-hocon-multiline-string-with-indentation

0213 hocon multiline string with indentation
This commit is contained in:
Zaiming (Stone) Shi 2024-02-16 15:12:46 +01:00 committed by GitHub
commit 2b46cbab7a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
51 changed files with 168 additions and 709 deletions

View File

@ -30,7 +30,7 @@
{esockd, {git, "https://github.com/emqx/esockd", {tag, "5.11.1"}}},
{ekka, {git, "https://github.com/emqx/ekka", {tag, "0.18.4"}}},
{gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.3.1"}}},
{hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.40.4"}}},
{hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.41.0"}}},
{emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.3"}}},
{pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}},
{recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}},

View File

@ -42,7 +42,8 @@
-export([
api_spec/0,
paths/0,
schema/1
schema/1,
namespace/0
]).
-export([
@ -95,6 +96,8 @@
-elvis([{elvis_style, god_modules, disable}]).
namespace() -> undefined.
api_spec() ->
emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}).

View File

@ -41,7 +41,8 @@
api_spec/0,
paths/0,
schema/1,
fields/1
fields/1,
namespace/0
]).
-export([
@ -56,6 +57,9 @@
-define(TAGS, [<<"Authorization">>]).
namespace() ->
undefined.
api_spec() ->
emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}).

View File

@ -22,7 +22,8 @@
-export([
roots/0,
fields/1,
desc/1
desc/1,
namespace/0
]).
-export([
@ -65,6 +66,8 @@
roots() -> [].
namespace() -> undefined.
fields(?CONF_NS) ->
emqx_schema:authz_fields() ++ authz_fields();
fields("metrics_status_fields") ->

View File

@ -35,7 +35,8 @@
api_spec/0,
paths/0,
schema/1,
fields/1
fields/1,
namespace/0
]).
%% operation funs
@ -69,6 +70,8 @@
-define(PUT_MAP_EXAMPLE, in_put_requestBody).
-define(POST_ARRAY_EXAMPLE, in_post_requestBody).
namespace() -> undefined.
api_spec() ->
emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}).

View File

@ -13,7 +13,8 @@
examples/1,
resource_type/1,
bridge_impl_module/1,
fields/1
fields/1,
namespace/0
]).
api_schemas(Method) ->
@ -139,6 +140,8 @@ bridge_impl_module(azure_event_hub_producer) ->
bridge_impl_module(_BridgeType) ->
undefined.
namespace() -> undefined.
fields(bridges) ->
[
{hstreamdb,

View File

@ -14,7 +14,7 @@
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
%% schema
-export([roots/0, fields/1, desc/1]).
-export([roots/0, fields/1, desc/1, namespace/0]).
%% callbacks of behaviour emqx_resource
-export([
@ -56,6 +56,8 @@
%%--------------------------------------------------------------------
%% schema
namespace() -> cassandra.
roots() ->
[{config, #{type => hoconsc:ref(?MODULE, config)}}].

View File

@ -1,6 +1,6 @@
{application, emqx_bridge_clickhouse, [
{description, "EMQX Enterprise ClickHouse Bridge"},
{vsn, "0.2.4"},
{vsn, "0.2.5"},
{registered, []},
{applications, [
kernel,

View File

@ -23,7 +23,8 @@
-export([
roots/0,
fields/1,
values/1
values/1,
namespace/0
]).
%% callbacks for behaviour emqx_resource
@ -72,6 +73,8 @@
%% Configuration and default values
%%=====================================================================
namespace() -> clickhouse.
roots() ->
[{config, #{type => hoconsc:ref(?MODULE, config)}}].

View File

@ -1,6 +1,6 @@
{application, emqx_bridge_dynamo, [
{description, "EMQX Enterprise Dynamo Bridge"},
{vsn, "0.1.4"},
{vsn, "0.1.5"},
{registered, []},
{applications, [
kernel,

View File

@ -12,7 +12,7 @@
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-include_lib("hocon/include/hoconsc.hrl").
-export([roots/0, fields/1]).
-export([roots/0, fields/1, namespace/0]).
%% `emqx_resource' API
-export([
@ -32,6 +32,9 @@
%%=====================================================================
%% Hocon schema
namespace() -> dynamodka.
roots() ->
[{config, #{type => hoconsc:ref(?MODULE, config)}}].

View File

@ -115,7 +115,8 @@ action_values() ->
%% -------------------------------------------------------------------------------------------------
%% Hocon Schema Definitions
namespace() -> "bridge_rocketmq".
namespace() -> "rocketmq".
roots() -> [].

View File

@ -12,7 +12,7 @@
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-include_lib("hocon/include/hoconsc.hrl").
-export([roots/0, fields/1]).
-export([roots/0, fields/1, namespace/0]).
%% `emqx_resource' API
-export([
@ -36,6 +36,9 @@
%%=====================================================================
%% Hocon schema
namespace() -> rocketmq.
roots() ->
[{config, #{type => hoconsc:ref(?MODULE, config)}}].

View File

@ -1,6 +1,6 @@
{application, emqx_bridge_sqlserver, [
{description, "EMQX Enterprise SQL Server Bridge"},
{vsn, "0.1.5"},
{vsn, "0.1.6"},
{registered, []},
{applications, [kernel, stdlib, emqx_resource, odbc]},
{env, []},

View File

@ -24,7 +24,8 @@
%% Hocon config schema exports
-export([
roots/0,
fields/1
fields/1,
namespace/0
]).
%% callbacks for behaviour emqx_resource
@ -132,6 +133,8 @@
%% Configuration and default values
%%====================================================================
namespace() -> sqlserver.
roots() ->
[{config, #{type => hoconsc:ref(?MODULE, config)}}].

View File

@ -158,7 +158,6 @@ dump_schema(Dir, SchemaModule) ->
ok = emqx_dashboard_desc_cache:init(),
lists:foreach(
fun(Lang) ->
ok = gen_config_md(Dir, SchemaModule, Lang),
ok = gen_schema_json(Dir, SchemaModule, Lang)
end,
["en", "zh"]
@ -468,14 +467,6 @@ bridge_schema_json() ->
SchemaInfo = #{title => <<"EMQX Data Bridge API Schema">>, version => Version},
gen_api_schema_json_iodata(emqx_bridge_api, SchemaInfo).
%% TODO: remove it and also remove hocon_md.erl and friends.
%% markdown generation from schema is a failure and we are moving to an interactive
%% viewer like swagger UI.
gen_config_md(Dir, SchemaModule, Lang) ->
SchemaMdFile = filename:join([Dir, "config-" ++ Lang ++ ".md"]),
io:format(user, "===< Generating: ~s~n", [SchemaMdFile]),
ok = gen_doc(SchemaMdFile, SchemaModule, Lang).
%% @doc return the root schema module.
-spec schema_module() -> module().
schema_module() ->
@ -515,19 +506,6 @@ make_desc_resolver(Lang) ->
unicode:characters_to_binary(Desc)
end.
-spec gen_doc(file:name_all(), module(), string()) -> ok.
gen_doc(File, SchemaModule, Lang) ->
Version = emqx_release:version(),
Title =
"# " ++ emqx_release:description() ++ " Configuration\n\n" ++
"<!--" ++ Version ++ "-->",
BodyFile = filename:join([rel, "emqx_conf.template." ++ Lang ++ ".md"]),
{ok, Body} = file:read_file(BodyFile),
Resolver = make_desc_resolver(Lang),
Opts = #{title => Title, body => Body, desc_resolver => Resolver},
Doc = hocon_schema_md:gen(SchemaModule, Opts),
file:write_file(File, Doc).
gen_api_schema_json_iodata(SchemaMod, SchemaInfo) ->
emqx_dashboard_swagger:gen_api_schema_json_iodata(
SchemaMod,

View File

@ -44,12 +44,6 @@
password/0
]).
-export([roots/0, fields/1]).
roots() -> [].
fields(_) -> [].
ssl_fields() ->
[
{ssl, #{

View File

@ -15,8 +15,8 @@
-export([
api_schemas/1,
fields/1,
%%examples/1
schema_modules/0
schema_modules/0,
namespace/0
]).
resource_type(Type) when is_binary(Type) ->
@ -93,6 +93,8 @@ connector_impl_module(rabbitmq) ->
connector_impl_module(_ConnectorType) ->
undefined.
namespace() -> undefined.
fields(connectors) ->
connector_structs().

View File

@ -27,7 +27,8 @@
-export([
paths/0,
schema/1,
fields/1
fields/1,
namespace/0
]).
-export([
@ -35,6 +36,8 @@
monitor_current/2
]).
namespace() -> undefined.
api_spec() ->
emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true, translate_body => true}).

View File

@ -19,7 +19,7 @@
-behaviour(hocon_schema).
%% API
-export([paths/0, api_spec/0, schema/1, namespace/0, fields/1]).
-export([paths/0, api_spec/0, schema/1, roots/0, namespace/0, fields/1]).
-export([init_per_suite/1, end_per_suite/1]).
-export([t_in_path/1, t_in_query/1, t_in_mix/1, t_without_in/1, t_ref/1, t_public_ref/1]).
-export([t_require/1, t_query_enum/1, t_nullable/1, t_method/1, t_api_spec/1]).
@ -563,6 +563,7 @@ schema("/method/error") ->
#{operationId => test, bar => #{200 => <<"ok">>}}.
namespace() -> undefined.
roots() -> [].
fields(page) ->
[

View File

@ -17,9 +17,10 @@
-include_lib("typerefl/include/types.hrl").
-export([roots/0, fields/1]).
-export([namespace/0, roots/0, fields/1]).
-import(hoconsc, [mk/2]).
roots() -> ["root"].
namespace() -> undefined.
fields("root") ->
[

View File

@ -813,7 +813,8 @@ to_schema(Body) ->
post => #{requestBody => Body, responses => #{200 => <<"ok">>}}
}.
%% Don't warning hocon callback namespace/0 undef.
roots() -> [].
namespace() -> atom_to_list(?MODULE).
fields(good_ref) ->

View File

@ -680,6 +680,7 @@ to_schema(Object) ->
post => #{responses => #{200 => Object, 201 => Object}}
}.
rotos() -> [].
namespace() -> undefined.
fields(good_ref) ->

View File

@ -36,7 +36,8 @@
-export([
api_spec/0,
paths/0,
schema/1
schema/1,
namespace/0
]).
-export([
@ -59,6 +60,8 @@
%% minirest behaviour callbacks
%%--------------------------------------------------------------------
namespace() -> undefined.
api_spec() ->
emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}).

View File

@ -42,7 +42,8 @@
-export([
roots/0,
fields/1
fields/1,
namespace/0
]).
%% http handlers
@ -775,6 +776,8 @@ schema_client() ->
examples_client()
).
namespace() -> undefined.
roots() ->
[
stomp_client,

View File

@ -45,7 +45,8 @@
-export([
roots/0,
fields/1
fields/1,
namespace/0
]).
%% http handlers
@ -651,6 +652,9 @@ params_paging_in_qs() ->
%%--------------------------------------------------------------------
%% schemas
namespace() ->
undefined.
roots() ->
[listener].

View File

@ -9,10 +9,12 @@
-include_lib("typerefl/include/types.hrl").
-behaviour(hocon_schema).
-export([namespace/0, fields/1, desc/1]).
-export([roots/0, namespace/0, fields/1, desc/1]).
-define(NOT_EMPTY(MSG), emqx_resource_validator:not_empty(MSG)).
roots() -> [].
namespace() -> gateway.
fields(jt808) ->

View File

@ -1,6 +1,6 @@
{application, emqx_gcp_device, [
{description, "Application simplifying migration from GCP IoT Core"},
{vsn, "0.1.3"},
{vsn, "0.1.4"},
{registered, []},
{mod, {emqx_gcp_device_app, []}},
{applications, [

View File

@ -22,7 +22,8 @@
api_spec/0,
paths/0,
schema/1,
fields/1
fields/1,
namespace/0
]).
-export([
@ -62,6 +63,9 @@
%% `minirest' and `minirest_trails' API
%%-------------------------------------------------------------------------------------------------
namespace() ->
undefined.
api_spec() ->
emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}).

View File

@ -22,7 +22,7 @@
-include_lib("emqx/include/emqx.hrl").
-include_lib("typerefl/include/types.hrl").
-export([api_spec/0, paths/0, schema/1, fields/1]).
-export([api_spec/0, paths/0, schema/1, fields/1, namespace/0]).
-export([alarms/2, format_alarm/2]).
@ -31,6 +31,9 @@
%% internal export (for query)
-export([qs2ms/2]).
namespace() ->
undefined.
api_spec() ->
emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}).

View File

@ -28,7 +28,8 @@
api_spec/0,
paths/0,
schema/1,
fields/1
fields/1,
namespace/0
]).
-export([format/1]).
@ -44,6 +45,9 @@
-define(FORMAT_FUN, {?MODULE, format}).
namespace() ->
undefined.
api_spec() ->
emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true, translate_body => true}).

View File

@ -32,7 +32,8 @@
api_spec/0,
paths/0,
schema/1,
fields/1
fields/1,
namespace/0
]).
-export([
@ -83,6 +84,8 @@
message => <<"Client ID not found">>
}).
namespace() -> undefined.
api_spec() ->
emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true, translate_body => true}).

View File

@ -21,7 +21,7 @@
-include_lib("emqx/include/logger.hrl").
-include_lib("hocon/include/hoconsc.hrl").
-export([api_spec/0, paths/0, schema/1, fields/1]).
-export([api_spec/0, paths/0, schema/1, fields/1, namespace/0]).
-export([
data_export/2,
@ -48,6 +48,8 @@
})}
).
namespace() -> undefined.
api_spec() ->
emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}).

View File

@ -27,7 +27,8 @@
-export([
api_spec/0,
paths/0,
schema/1
schema/1,
namespace/0
]).
-export([
@ -42,6 +43,8 @@
%% minirest behaviour callbacks
%%--------------------------------------------------------------------
namespace() -> undefined.
api_spec() ->
emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}).

View File

@ -30,7 +30,8 @@
api_spec/0,
schema/1,
paths/0,
fields/1
fields/1,
namespace/0
]).
%% API callbacks
@ -45,6 +46,8 @@
%% API spec funcs
%%--------------------------------------------------------------------
namespace() -> undefined.
api_spec() ->
emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}).

View File

@ -32,7 +32,8 @@
api_spec/0,
paths/0,
schema/1,
fields/1
fields/1,
namespace/0
]).
-export([
@ -40,6 +41,8 @@
publish_batch/2
]).
namespace() -> undefined.
api_spec() ->
emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true, translate_body => true}).

View File

@ -34,11 +34,14 @@
api_spec/0,
paths/0,
schema/1,
fields/1
fields/1,
namespace/0
]).
-export([list/2]).
namespace() -> undefined.
api_spec() ->
emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}).

View File

@ -28,7 +28,8 @@
api_spec/0,
paths/0,
schema/1,
fields/1
fields/1,
namespace/0
]).
-export([subscriptions/2]).
@ -48,6 +49,9 @@
{<<"match_topic">>, binary}
]).
namespace() ->
undefined.
api_spec() ->
emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}).

View File

@ -28,7 +28,8 @@
api_spec/0,
paths/0,
schema/1,
fields/1
fields/1,
namespace/0
]).
-export([
@ -41,6 +42,8 @@
-define(TOPICS_QUERY_SCHEMA, [{<<"topic">>, binary}, {<<"node">>, atom}]).
-define(TAGS, [<<"Topics">>]).
namespace() -> undefined.
api_spec() ->
emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true, translate_body => true}).

View File

@ -34,7 +34,8 @@
-export([
paths/0,
fields/1,
schema/1
schema/1,
namespace/0
]).
%% for rpc
@ -55,6 +56,8 @@
-define(INVALID_TOPIC, 'INVALID_TOPIC_NAME').
-define(MESSAGE_NOT_FOUND, 'MESSAGE_NOT_FOUND').
namespace() -> undefined.
api_spec() ->
emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}).

View File

@ -47,7 +47,8 @@
api_spec/0,
paths/0,
schema/1,
fields/1
fields/1,
namespace/0
]).
-define(EXCEED_LIMIT, 'EXCEED_LIMIT').
@ -55,6 +56,8 @@
-define(TOPIC_NOT_FOUND, 'TOPIC_NOT_FOUND').
-define(BAD_REQUEST, 'BAD_REQUEST').
namespace() -> undefined.
api_spec() ->
emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}).

View File

@ -44,7 +44,7 @@
unprepare_sql/1
]).
-export([roots/0, fields/1]).
-export([roots/0, fields/1, namespace/0]).
-export([do_get_status/1]).
@ -63,6 +63,9 @@
-export_type([state/0]).
%%=====================================================================
%% Hocon schema
namespace() -> mysql.
roots() ->
[{config, #{type => hoconsc:ref(?MODULE, config)}}].

View File

@ -12,9 +12,12 @@
%% Hocon config schema exports
-export([
roots/0,
fields/1
fields/1,
namespace/0
]).
namespace() -> oracle.
roots() ->
[{config, #{type => hoconsc:ref(?REF_MODULE, config)}}].

View File

@ -23,7 +23,7 @@
-include_lib("epgsql/include/epgsql.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-export([roots/0, fields/1]).
-export([roots/0, fields/1, namespace/0]).
-behaviour(emqx_resource).
@ -71,6 +71,8 @@
%%=====================================================================
namespace() -> postgres.
roots() ->
[{config, #{type => hoconsc:ref(?MODULE, config)}}].

View File

@ -34,7 +34,8 @@
api_spec/0,
paths/0,
schema/1,
fields/1
fields/1,
namespace/0
]).
-export([
@ -50,6 +51,8 @@
-define(IS_TRUE(Val), ((Val =:= true) orelse (Val =:= <<"true">>))).
-define(IS_FALSE(Val), ((Val =:= false) orelse (Val =:= <<"false">>))).
namespace() -> undefined.
api_spec() ->
emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}).

View File

@ -32,12 +32,15 @@
api_spec/0,
paths/0,
schema/1,
fields/1
fields/1,
namespace/0
]).
-define(BAD_REQUEST, 'BAD_REQUEST').
-define(NOT_FOUND, 'NOT_FOUND').
namespace() -> undefined.
api_spec() ->
emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}).

View File

@ -0,0 +1,15 @@
Congifuration files now support multi-line string values with indentation.
Introduced the `"""~` and `~"""` to quote indented lines. For example:
```
rule_xlu4 {
sql = """~
SELECT
*
FROM
"t/#"
~"""
}
```
See [HOCON 0.41.0](https://github.com/emqx/hocon/releases/tag/0.41.0) release note for more dtails.

View File

@ -72,7 +72,7 @@ defmodule EMQXUmbrella.MixProject do
# in conflict by emqtt and hocon
{:getopt, "1.0.2", override: true},
{:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.8", override: true},
{:hocon, github: "emqx/hocon", tag: "0.40.4", override: true},
{:hocon, github: "emqx/hocon", tag: "0.41.0", override: true},
{:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.3", override: true},
{:esasl, github: "emqx/esasl", tag: "0.2.0"},
{:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"},

View File

@ -97,7 +97,7 @@
{system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}},
{getopt, "1.0.2"},
{snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.8"}}},
{hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.40.4"}}},
{hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.41.0"}}},
{emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.3"}}},
{esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}},
{jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}},

View File

@ -1,329 +0,0 @@
EMQX configuration files are in [HOCON](https://github.com/emqx/hocon) format.
HOCON, or Human-Optimized Config Object Notation is a format for human-readable data,
and a superset of JSON.
## Layered
EMQX configuration consists of two layers.
From bottom up:
1. Cluster-synced configs: `$EMQX_NODE__DATA_DIR/configs/cluster.hocon`.
2. Local node configs: `emqx.conf` + `EMQX_` prefixed environment variables.
:::tip Tip
Prior to v5.0.23 and e5.0.3, the cluster-synced configs are stored in
`cluster-override.conf` which is applied on top of the local configs.
If upgraded from an earlier version, as long as `cluster-override.conf` exists,
`cluster.hocon` will not be created, and `cluster-override.conf` will stay on
top of the overriding layers.
:::
When environment variable `$EMQX_NODE__DATA_DIR` is not set, config `node.data_dir`
is used.
The `cluster.hocon` file is overwritten at runtime when changes
are made from Dashboard, management HTTP API, or CLI. When clustered,
after EMQX restarts, it copies the file from the node which has the greatest `uptime`.
:::tip Tip
To avoid confusion, don't add the same keys in both `cluster.hocon` and `emqx.conf`.
:::
For detailed override rules, see [Config Overlay Rules](#config-overlay-rules).
## Syntax
In config file the values can be notated as JSON like objects, such as
```
node {
name = "emqx@127.0.0.1"
cookie = "mysecret"
}
```
Another equivalent representation is flat, such as
```
node.name = "127.0.0.1"
node.cookie = "mysecret"
```
This flat format is almost backward compatible with EMQX's config file format
in 4.x series (the so called 'cuttlefish' format).
It is not fully compatible because the often HOCON requires strings to be quoted,
while cuttlefish treats all characters to the right of the `=` mark as the value.
e.g. cuttlefish: `node.name = emqx@127.0.0.1`, HOCON: `node.name = "emqx@127.0.0.1"`.
Strings without special characters in them can be unquoted in HOCON too,
e.g. `foo`, `foo_bar` and `foo_bar_1`.
For more HOCON syntax, please refer to the [specification](https://github.com/lightbend/config/blob/main/HOCON.md)
## Schema
To make the HOCON objects type-safe, EMQX introduced a schema for it.
The schema defines data types, and data fields' names and metadata for config value validation
and more.
::: tip Tip
The configuration document you are reading now is generated from schema metadata.
:::
### Complex Data Types
There are 4 complex data types in EMQX's HOCON config:
1. Struct: Named using an unquoted string, followed by a predefined list of fields.
Only lowercase letters and digits are allowed in struct and field names.
Alos, only underscore can be used as word separator.
1. Map: Map is like Struct, however the fields are not predefined.
1. Union: `MemberType1 | MemberType2 | ...`
1. Array: `[ElementType]`
::: tip Tip
If map field name is a positive integer number, it is interpreted as an alternative representation of an `Array`.
For example:
```
myarray.1 = 74
myarray.2 = 75
```
will be interpreated as `myarray = [74, 75]`, which is handy when trying to override array elements.
:::
### Primitive Data Types
Complex types define data 'boxes' which may contain other complex data
or primitive values.
There are quite some different primitive types, to name a few:
* `atom()`.
* `boolean()`.
* `string()`.
* `integer()`.
* `float()`.
* `number()`.
* `binary()`, another format of string().
* `emqx_schema:duration()`, time duration, another format of integer()
* ...
::: tip Tip
The primitive types are mostly self-describing, so there is usually not a lot to document.
For types that are not so clear by their names, the field description is to be used to find the details.
:::
### Config Paths
If we consider the whole EMQX config as a tree,
to reference a primitive value, we can use a dot-separated names form string for
the path from the tree-root (always a Struct) down to the primitive values at tree-leaves.
Each segment of the dotted string is a Struct field name or Map key.
For Array elements, 1-based index is used.
below are some examples
```
node.name = "emqx.127.0.0.1"
zone.zone1.max_packet_size = "10M"
authentication.1.enable = true
```
### Environment variables
Environment variables can be used to define or override config values.
Due to the fact that dots (`.`) are not allowed in environment variables, dots are
replaced with double-underscores (`__`).
And the `EMQX_` prefix is used as the namespace.
For example `node.name` can be represented as `EMQX_NODE__NAME`
Environment variable values are parsed as HOCON values, this allows users
to even set complex values from environment variables.
For example, this environment variable sets an array value.
```
export EMQX_LISTENERS__SSL__L1__AUTHENTICATION__SSL__CIPHERS='["TLS_AES_256_GCM_SHA384"]'
```
However, this also means a string value should be quoted if it happens to contain special
characters such as `=` and `:`.
For example, a string value `"localhost:1883"` would be
parsed into object (struct): `{"localhost": 1883}`.
To keep it as a string, one should quote the value like below:
```
EMQX_BRIDGES__MQTT__MYBRIDGE__CONNECTOR_SERVER='"localhost:1883"'
```
::: tip Tip
Unknown root paths are silently discarded by EMQX, for example `EMQX_UNKNOWN_ROOT__FOOBAR` is
silently discarded because `unknown_root` is not a predefined root path.
Unknown field names in environment variables are logged as a `warning` level log, for example:
```
[warning] unknown_env_vars: ["EMQX_AUTHENTICATION__ENABLED"]
```
because the field name is `enable`, not `enabled`.
:::
### Config Overlay Rules
HOCON objects are overlaid, in general:
- Within one file, objects defined 'later' recursively override objects defined 'earlier'
- When layered, 'later' (higher layer) objects override objects defined 'earlier' (lower layer)
Below are more detailed rules.
#### Struct Fields
Later config values overwrites earlier values.
For example, in below config, the last line `debug` overwrites `error` for
console log handler's `level` config, but leaving `enable` unchanged.
```
log {
console_handler{
enable=true,
level=error
}
}
## ... more configs ...
log.console_handler.level=debug
```
#### Map Values
Maps are like structs, only the files are user-defined rather than
the config schema. For instance, `zone1` in the example below.
```
zone {
zone1 {
mqtt.max_packet_size = 1M
}
}
## The maximum packet size can be defined as above,
## then overridden as below
zone.zone1.mqtt.max_packet_size = 10M
```
#### Array Elements
Arrays in EMQX config have two different representations
* list, such as: `[1, 2, 3]`
* indexed-map, such as: `{"1"=1, "2"=2, "3"=3}`
Dot-separated paths with number in it are parsed to indexed-maps
e.g. `authentication.1={...}` is parsed as `authentication={"1": {...}}`
This feature makes it easy to override array element values. For example:
```
authentication=[{enable=true, backend="built_in_database", mechanism="password_based"}]
# we can disable this authentication provider with:
authentication.1.enable=false
```
::: warning Warning
List arrays is a full-array override, but not a recursive merge, into indexed-map arrays.
e.g.
```
authentication=[{enable=true, backend="built_in_database", mechanism="password_based"}]
## below value will replace the whole array, but not to override just one field.
authentication=[{enable=true}]
```
:::
#### TLS/SSL ciphers
Starting from v5.0.6, EMQX no longer pre-populates the ciphers list with a default
set of cipher suite names.
Instead, the default ciphers are applied at runtime when starting the listener
for servers, or when establishing a TLS connection as a client.
Below are the default ciphers selected by EMQX.
For tlsv1.3:
```
ciphers =
[ "TLS_AES_256_GCM_SHA384", "TLS_AES_128_GCM_SHA256",
"TLS_CHACHA20_POLY1305_SHA256", "TLS_AES_128_CCM_SHA256",
"TLS_AES_128_CCM_8_SHA256"
]
```
For tlsv1.2 or earlier
```
ciphers =
[ "ECDHE-ECDSA-AES256-GCM-SHA384",
"ECDHE-RSA-AES256-GCM-SHA384",
"ECDHE-ECDSA-AES256-SHA384",
"ECDHE-RSA-AES256-SHA384",
"ECDH-ECDSA-AES256-GCM-SHA384",
"ECDH-RSA-AES256-GCM-SHA384",
"ECDH-ECDSA-AES256-SHA384",
"ECDH-RSA-AES256-SHA384",
"DHE-DSS-AES256-GCM-SHA384",
"DHE-DSS-AES256-SHA256",
"AES256-GCM-SHA384",
"AES256-SHA256",
"ECDHE-ECDSA-AES128-GCM-SHA256",
"ECDHE-RSA-AES128-GCM-SHA256",
"ECDHE-ECDSA-AES128-SHA256",
"ECDHE-RSA-AES128-SHA256",
"ECDH-ECDSA-AES128-GCM-SHA256",
"ECDH-RSA-AES128-GCM-SHA256",
"ECDH-ECDSA-AES128-SHA256",
"ECDH-RSA-AES128-SHA256",
"DHE-DSS-AES128-GCM-SHA256",
"DHE-DSS-AES128-SHA256",
"AES128-GCM-SHA256",
"AES128-SHA256",
"ECDHE-ECDSA-AES256-SHA",
"ECDHE-RSA-AES256-SHA",
"DHE-DSS-AES256-SHA",
"ECDH-ECDSA-AES256-SHA",
"ECDH-RSA-AES256-SHA",
"ECDHE-ECDSA-AES128-SHA",
"ECDHE-RSA-AES128-SHA",
"DHE-DSS-AES128-SHA",
"ECDH-ECDSA-AES128-SHA",
"ECDH-RSA-AES128-SHA"
]
```
For PSK enabled listeners
```
ciphers =
[ "RSA-PSK-AES256-GCM-SHA384",
"RSA-PSK-AES256-CBC-SHA384",
"RSA-PSK-AES128-GCM-SHA256",
"RSA-PSK-AES128-CBC-SHA256",
"RSA-PSK-AES256-CBC-SHA",
"RSA-PSK-AES128-CBC-SHA",
"PSK-AES256-GCM-SHA384",
"PSK-AES128-GCM-SHA256",
"PSK-AES256-CBC-SHA384",
"PSK-AES256-CBC-SHA",
"PSK-AES128-CBC-SHA256",
"PSK-AES128-CBC-SHA"
]
```

View File

@ -1,306 +0,0 @@
EMQX的配置文件格式是 [HOCON](https://github.com/emqx/hocon) 。
HOCONHuman-Optimized Config Object Notation是一个JSON的超集非常适用于易于人类读写的配置数据存储。
## 分层结构
EMQX的配置文件可分为二层自底向上依次是
1. 集群同步配置:`$EMQX_NODE__DATA_DIR/configs/cluster.hocon`。
2. 本地节点配置:`emqx.conf` 加上 `EMQX_` 前缀的环境变量。
:::tip Tip
在 v5.0.23 或 e5.0.3 之前,集群同步配置保存在文件 `cluster-override.conf` 中,并且它覆盖在配置的最上层。
如果从之前的版本升级上来,只要 `cluster-override.conf` 文件存在,
EMQX 就不会创建 `cluster.hocon`,并且 `cluster-override.conf` 会继续覆盖在配置的最上层。
:::
如果环境变量 `$EMQX_NODE__DATA_DIR` 没有设置,那么该目录会从 `emqx.conf``node.data_dir` 配置中读取。
配置文件 `cluster.hocon` 的内容会在运行时被EMQX重写。
这些重写发生在 dashboard UI管理HTTP API或者CLI对集群配置进行修改时。
当EMQX运行在集群中时一个EMQX节点重启之后会从集群中其他节点复制该文件内容到本地。
:::tip Tip
为避免歧义,应尽量避免让 `cluster.hocon``emqx.conf` 出现配置交集。
:::
更多的重载规则,请参考下文 [配置重载规则](#配置重载规则)。
## 配置文件语法
在配置文件中值可以被记为类似JSON的对象例如
```
node {
name = "emqx@127.0.0.1"
cookie = "mysecret"
}
```
另一种等价的表示方法是扁平的,例如
```
node.name = "127.0.0.1"
node.cookie = "mysecret"
```
这种扁平格式几乎与EMQX的配置文件格式向后兼容
在4.x系列中所谓的'cuttlefish'格式)。
它并不是完全兼容因为HOCON经常要求字符串两端加上引号。
而cuttlefish把`=`符右边的所有字符都视为值。
例如cuttlefish`node.name = emqx@127.0.0.1`HOCON`node.name = "emqx@127.0.0.1"`。
没有特殊字符的字符串在HOCON中也可以不加引号。
例如:`foo``foo_bar`和`foo_bar_1`。
关于更多的HOCON语法请参考[规范](https://github.com/lightbend/config/blob/main/HOCON.md)
## Schema
为了使HOCON对象类型安全EMQX为它引入了一个schema。
该schema定义了数据类型以及数据字段的名称和元数据用于配置值的类型检查等等。
::: tip Tip
当前阅读到配置文件的文档本身就是由模式元数据生成的。
:::
### 复杂数据类型
EMQX的配置文件中有4中复杂数据结构类型它们分别是
1. Struct结构体都是有类型名称的结构体中可以有任意多个字段。
结构体和字段的名称由不带特殊字符的全小些字母组成,名称中可以带数字,但不得以数字开头,多个单词可用下划线分隔。
1. Map: Map 与 Struct结构体类似但是内部的字段不是预先定义好的。
1. Union: 联合 `MemberType1 | MemberType2 | ...`,可以理解为:“不是这个,就是那个”
1. Array: 数组 `[ElementType]`
::: tip Tip
如果Map的字段名称是纯数字它会被解释成一个数组。
例如
```
myarray.1 = 74
myarray.2 = 75
```
会被解析成 `myarray = [74, 75]`。这个用法在重载数组元素的值时候非常有用。
:::
### 原始数据类型
复杂类型定义了数据 "盒子",其中可能包含其他复杂数据或原始值。
有很多不同的原始类型,仅举几个例子。
* 原子 `atom()`
* 布尔 `boolean()`
* 字符串 `string()`
* 整形 `integer()`
* 浮点数 `float()`
* 数值 `number()`
* 二进制编码的字符串 `binary()``string()` 的另一种格式。
* 时间间隔 `emqx_schema:duration()``integer()` 的另一种格式。
* ...
::: tip Tip
原始类型的名称大多是自我描述的,所以不需要过多的注释。
但是有一些不是那么直观的数据类型,则需要配合字段的描述文档进行理解。
:::
### 配置路径
如果我们把EMQX的配置值理解成一个类似目录树的结构那么类似于文件系统中使用斜杠或反斜杠进行层级分割
EMQX使用的配置路径的层级分割符是 `'.'`
`'.'` 号分割的每一段,则是 Struct结构体的字段或 Map 的 key。
下面有几个例子:
```
node.name = "emqx.127.0.0.1"
zone.zone1.max_packet_size = "10M"
authentication.1.enable = true
```
### 环境变量重载
因为 `'.'` 分隔符不能使用于环境变量所以我们需要使用另一个分割符。EMQX选用的是双下划线 `__`
为了与其他的环境变量有所区分EMQX还增加了一个前缀 `EMQX_` 来用作环境变量命名空间。
例如 `node.name` 的重载变量名是 `EMQX_NODE__NAME`
环境变量的值,是按 HOCON 值解析的,这也使得环境变量可以用来传递复杂数据类型的值。
例如,下面这个环境变量传入一个数组类型的值。
```
export EMQX_LISTENERS__SSL__L1__AUTHENTICATION__SSL__CIPHERS='["TLS_AES_256_GCM_SHA384"]'
```
这也意味着有些带特殊字符(例如`:` 和 `=`),则需要用双引号对这个值包起来。
例如`localhost:1883` 会被解析成一个结构体 `{"localhost": 1883}`
想要把它当字符串使用时,就必需使用引号,如下:
```
EMQX_BRIDGES__MQTT__MYBRIDGE__CONNECTOR_SERVER='"localhost:1883"'
```
::: tip Tip
未定义的根路径会被EMQX忽略例如 `EMQX_UNKNOWN_ROOT__FOOBAR` 这个环境变量会被EMQX忽略
因为 `UNKNOWN_ROOT` 不是预先定义好的根路径。
对于已知的根路径未知的字段名称将被记录为warning日志比如下面这个例子。
```
[warning] unknown_env_vars: ["EMQX_AUTHENTICATION__ENABLED"]
```
这是因为正确的字段名称是 `enable`,而不是 `enabled`
:::
### 配置重载规则
HOCON的值是分层覆盖的普遍规则如下
- 在同一个文件中,后(在文件底部)定义的值,覆盖前(在文件顶部)到值。
- 当按层级覆盖时,高层级的值覆盖低层级的值。
结下来的文档将解释更详细的规则。
#### 结构体
合并覆盖规则。在如下配置中,最后一行的 `debug` 值会覆盖覆盖原先`level`字段的 `error` 值,但是 `enable` 字段保持不变。
```
log {
console_handler{
enable=true,
level=error
}
}
## 控制台日志打印先定义为 `error` 级,后被覆写成 `debug`
log.console_handler.level=debug
```
#### Map
Map与结构体类似也是合并覆盖规则。
如下例子中,`zone1` 的 `max_packet_size` 可以在文件后面覆写。
```
zone {
zone1 {
mqtt.max_packet_size = 1M
}
}
## 报文大小限制最先被设置成1MB后被覆写为10MB
zone.zone1.mqtt.max_packet_size = 10M
```
#### 数组元素
如上面介绍过EMQX配置中的数组有两种表达方式。
* 列表格式,例如: `[1, 2, 3]`
* 带下标的Map格式例如 `{"1"=1, "2"=2, "3"=3}`
点好(`'.'`)分隔到路径中的纯数字会被解析成数组下标。
例如,`authentication.1={...}` 会被解析成 `authentication={"1": {...}}`,进而进一步解析成 `authentication=[{...}]`
有了这个特性,我们就可以轻松覆写数组某个元素的值,例如:
```
authentication=[{enable=true, backend="built_in_database", mechanism="password_based"}]
# 可以用下面的方式将第一个元素的 `enable` 字段覆写
authentication.1.enable=false
```
::: warning Warning
使用列表格式是的数组将全量覆写原值,如下例:
```
authentication=[{enable=true, backend="built_in_database", mechanism="password_based"}]
## 下面这中方式会导致数组第一个元素的除了 `enable` 以外的其他字段全部丢失
authentication=[{enable=true}]
```
:::
#### TLS/SSL ciphers
从 v5.0.6 开始 EMQX 不在配置文件中详细列出所有默认的密码套件名称。
而是在配置文件中使用一个空列表,然后在运行时替换成默认的密码套件。
下面这些密码套件是 EMQX 默认支持的:
tlsv1.3:
```
ciphers =
[ "TLS_AES_256_GCM_SHA384", "TLS_AES_128_GCM_SHA256",
"TLS_CHACHA20_POLY1305_SHA256", "TLS_AES_128_CCM_SHA256",
"TLS_AES_128_CCM_8_SHA256"
]
```
tlsv1.2 或更早
```
ciphers =
[ "ECDHE-ECDSA-AES256-GCM-SHA384",
"ECDHE-RSA-AES256-GCM-SHA384",
"ECDHE-ECDSA-AES256-SHA384",
"ECDHE-RSA-AES256-SHA384",
"ECDH-ECDSA-AES256-GCM-SHA384",
"ECDH-RSA-AES256-GCM-SHA384",
"ECDH-ECDSA-AES256-SHA384",
"ECDH-RSA-AES256-SHA384",
"DHE-DSS-AES256-GCM-SHA384",
"DHE-DSS-AES256-SHA256",
"AES256-GCM-SHA384",
"AES256-SHA256",
"ECDHE-ECDSA-AES128-GCM-SHA256",
"ECDHE-RSA-AES128-GCM-SHA256",
"ECDHE-ECDSA-AES128-SHA256",
"ECDHE-RSA-AES128-SHA256",
"ECDH-ECDSA-AES128-GCM-SHA256",
"ECDH-RSA-AES128-GCM-SHA256",
"ECDH-ECDSA-AES128-SHA256",
"ECDH-RSA-AES128-SHA256",
"DHE-DSS-AES128-GCM-SHA256",
"DHE-DSS-AES128-SHA256",
"AES128-GCM-SHA256",
"AES128-SHA256",
"ECDHE-ECDSA-AES256-SHA",
"ECDHE-RSA-AES256-SHA",
"DHE-DSS-AES256-SHA",
"ECDH-ECDSA-AES256-SHA",
"ECDH-RSA-AES256-SHA",
"ECDHE-ECDSA-AES128-SHA",
"ECDHE-RSA-AES128-SHA",
"DHE-DSS-AES128-SHA",
"ECDH-ECDSA-AES128-SHA",
"ECDH-RSA-AES128-SHA"
]
```
配置 PSK 认证的监听器
```
ciphers = [
[ "RSA-PSK-AES256-GCM-SHA384",
"RSA-PSK-AES256-CBC-SHA384",
"RSA-PSK-AES128-GCM-SHA256",
"RSA-PSK-AES128-CBC-SHA256",
"RSA-PSK-AES256-CBC-SHA",
"RSA-PSK-AES128-CBC-SHA",
"PSK-AES256-GCM-SHA384",
"PSK-AES128-GCM-SHA256",
"PSK-AES256-CBC-SHA384",
"PSK-AES256-CBC-SHA",
"PSK-AES128-CBC-SHA256",
"PSK-AES128-CBC-SHA"
]
```