From a1ad6098b3912a1524540860e3f0595aee841b21 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sun, 5 Dec 2021 23:49:46 +0100 Subject: [PATCH] build: generate document to dashboard priv dir --- apps/emqx_conf/etc/emqx_conf.md | 192 +++++++++++++++++++++++++++++++ apps/emqx_conf/src/emqx_conf.erl | 11 ++ build | 15 ++- rebar.config.erl | 6 + 4 files changed, 218 insertions(+), 6 deletions(-) create mode 100644 apps/emqx_conf/etc/emqx_conf.md diff --git a/apps/emqx_conf/etc/emqx_conf.md b/apps/emqx_conf/etc/emqx_conf.md new file mode 100644 index 000000000..f68411bff --- /dev/null +++ b/apps/emqx_conf/etc/emqx_conf.md @@ -0,0 +1,192 @@ +EMQ X configuration file is in [HOCON](https://github.com/emqx/hocon) format. +HOCON, or Human-Optimized Config Object Notation is a format for human-readable data, +and a superset of JSON. + +## Syntax + +In config file the values can be notated as JSON like ojbects, such as +``` +node { + name = "emqx@127.0.0.1" + cookie = "mysecret" +} +``` + +Another equivalent representation is flat, suh as + +``` +node.name="127.0.0.1" +node.cookie="mysecret" +``` + +This flat format is almost backward compatible with EMQ X's config file format +in 4.x series (the so called 'cuttlefish' format). + +It is 'almost' compabile because the often HOCON requires strings to be quoted, +while cuttlefish treats all characters to the right of the `=` mark as the value. + +e.g. cuttlefish: `node.name = emqx@127.0.0.1`, HOCON: `node.name = "emqx@127.0.0.1"` + +Strings without special characters in them can be unquoted in HOCON too, +e.g. `foo`, `foo_bar`, `foo_bar_1`: + +For more HOCON syntax, pelase refer to the [specification](https://github.com/lightbend/config/blob/main/HOCON.md) + +## Schema + +To make the HOCON objects type-safe, EMQ X introduded a schema for it. +The schema defines data types, and data fields' names and metadata for config value validation +and more. In fact, this config document itself is generated from schema metadata. + +### Complex Data Types + +There are 4 complex data types in EMQ X's HOCON config: + +1. Struct: Named using an unquoted string, followed by a pre-defined list of fields, + fields can not start with a number, and are only allowed to use + lowercase letters and underscores as word separater. +1. Map: Map is like Struct, however the fields are not pre-defined. + 1-based index number can also be used as map keys for an alternative + representation of an Array. +1. Union: `MemberType1 | MemberType2 | ...` +1. Array: `[ElementType]` + +### Primitive Data Types + +Complex types define data 'boxes' wich may contain other complex data +or primitive values. +There are quite some different primitive types, to name a fiew: + +* `atom()` +* `boolean()` +* `string()` +* `integer()` +* `float()` +* `number()` +* `binary()` # another format of string() +* `emqx_schema:duration()` # time duration, another format of integer() +* ... + +The primitive types are mostly self-describing, some are built-in, such +as `atom()`, some are defiend in EMQ X modules, such as `emqx_schema:duration()`. + +### Config Paths + +If we consider the whole EMQ X config as a tree, +to reference a primitive value, we can use a dot-separated names form string for +the path from the tree-root (always a Struct) down to the primitive values at tree-leaves. + +Each segment of the dotted string is a Struct filed name or Map key. +For Array elements, 1-based index is used. + +below are some examples + +``` +node.name="emqx.127.0.0.1" +zone.zone1.max_packet_size="10M" +authentication.1.enable=true +``` + +### Environment varialbes + +Environment variables can be used to define or override config values. + +Due to the fact that dots (`.`) are not allowed in environment variables, dots are +replaced with double-underscores (`__`). + +And a the `EMQX_` prefix is used as the namespace. + +For example `node.name` can be represented as `EMQX_NODE__NAME` + +Environment varialbe values are parsed as hocon values, this allows users +to even set complex values from environment variables. + +For example, this environment variable sets an array value. + +``` +export EMQX_LISTENERS__SSL__L1__AUTHENTICATION__SSL__CIPHERS="[\"TLS_AES_256_GCM_SHA384\"]" +``` + +Unknown environment variables are logged as a `warning` level log, for example: + +``` +[warning] unknown_env_vars: ["EMQX_AUTHENTICATION__ENABLED"] +``` + +because the field name is `enable`, not `enabled`. + +NOTE: Unknown root keys are however silently discarded. + +### Config overlay + +HOCON values are overlayed, earlier defined values are at layers closer to the bottom. +The overall order of the overlay rules from bottom up are: + +1. `emqx.conf` the base config file +1. `EMQX_` prfixed environment variables +1. Cluster override file, the path of which is configured as `cluster_override_conf_file` in the lower layers +1. Local override file, the path of which is configured as `local_override_conf_file` in the lower layers + +Below are the rules of config value overlay. + +#### Struct Fileds + +Later config values overwrites earlier values. +For example, in below config, the last line `debug` overwrites `errro` for +console log handler's `level` config, but leaving `enable` unchanged. +``` +log { + console_handler{ + enable=true, + level=error + } +} + +## ... more configs ... + +log.console_handler.level=debug +``` + +#### Map Values + +Maps are like structs, only the files are user-defined rather than +the config schema. For instance, `zone1` in the exampele below. + +``` +zone { + zone1 { + mqtt.max_packet_size = 1M + } +} + +## The maximum packet size can be defined as above, +## then overriden as below + +zone.zone1.mqtt.max_packet_size = 10M +``` + +#### Array Elements + +Arrays in EMQ X config have two different representations + +* list, such as: `[1, 2, 3]` +* indexed-map, such as: `{"1"=1, "2"=2, "3"=3}` + +Dot-separated paths with number in it are parsed to indexed-maps +e.g. `authentication.1={...}` is parsed as `authentication={"1": {...}}` + +Indexed-map arrays can be used to override list arrays: + +``` +authentication=[{enable=true, backend="built-in-database", mechanism="password-based"}] +# we can disable this authentication provider with: +authentication.1.enable=false +``` +However, list arrays do not get recursively merged into indexed-map arrays. +e.g. + +``` +authentication=[{enable=true, backend="built-in-database", mechanism="password-based"}] +## below value will replace the whole array, but not to override just one field. +authentication=[{enable=true}] +``` diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl index c82623e72..dec07f35c 100644 --- a/apps/emqx_conf/src/emqx_conf.erl +++ b/apps/emqx_conf/src/emqx_conf.erl @@ -24,6 +24,7 @@ -export([update/3, update/4]). -export([remove/2, remove/3]). -export([reset/2, reset/3]). +-export([gen_doc/1]). %% for rpc -export([get_node_and_config/1]). @@ -123,6 +124,16 @@ reset(Node, KeyPath, Opts) when Node =:= node() -> reset(Node, KeyPath, Opts) -> rpc:call(Node, ?MODULE, reset, [KeyPath, Opts]). +-spec gen_doc(file:name_all()) -> ok. +gen_doc(File) -> + Version = emqx_release:version(), + Title = "# EMQ X " ++ Version ++ " Configuration", + BodyFile = filename:join([code:lib_dir(emqx_conf), "etc", "emqx_conf.md"]), + {ok, Body} = file:read_file(BodyFile), + Doc = hocon_schema_doc:gen(emqx_conf_schema, #{title => Title, + body => Body}), + file:write_file(File, Doc). + %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- diff --git a/build b/build index 9f9879df4..10c11d570 100755 --- a/build +++ b/build @@ -51,16 +51,17 @@ log() { echo "===< $msg" } -docgen() { +make_doc() { local libs_dir1 libs_dir2 libs_dir1="$(find "_build/default/lib/" -maxdepth 2 -name ebin -type d)" libs_dir2="$(find "_build/$PROFILE/lib/" -maxdepth 2 -name ebin -type d)" - local conf_doc_markdown - conf_doc_markdown="$(pwd)/_build/${PROFILE}/rel/emqx/etc/emqx-config-doc.md" - echo "===< Generating config document $conf_doc_markdown" + local conf_doc_md + # TODO render md as html + conf_doc_md="$(pwd)/_build/${PROFILE}/lib/emqx_dashboard/priv/config.md" + echo "===< Generating config document $conf_doc_md" # shellcheck disable=SC2086 - erl -noshell -pa $libs_dir1 $libs_dir2 -eval "file:write_file('$conf_doc_markdown', hocon_schema_doc:gen(emqx_conf_schema, \"EMQ X ${PKG_VSN} Configuration\")), halt(0)." + erl -noshell -pa $libs_dir1 $libs_dir2 -eval "ok = emqx_conf:gen_doc(\"${conf_doc_md}\"), halt(0)." } make_rel() { @@ -70,7 +71,6 @@ make_rel() { echo "gpb should not be included in the release" exit 1 fi - docgen } ## unzip previous version .zip files to _build/$PROFILE/rel/emqx/releases before making relup @@ -205,6 +205,9 @@ make_docker_testing() { log "building artifact=$ARTIFACT for profile=$PROFILE" case "$ARTIFACT" in + doc) + make_doc + ;; rel) make_rel ;; diff --git a/rebar.config.erl b/rebar.config.erl index 22949f233..e80487fc3 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -153,36 +153,42 @@ profiles() -> , {relx, relx(Vsn, cloud, bin, ce)} , {overrides, prod_overrides()} , {project_app_dirs, project_app_dirs(ce)} + , {post_hooks, [{compile, "./build emqx doc"}]} ]} , {'emqx-pkg', [ {erl_opts, prod_compile_opts()} , {relx, relx(Vsn, cloud, pkg, ce)} , {overrides, prod_overrides()} , {project_app_dirs, project_app_dirs(ce)} + , {post_hooks, [{compile, "./build emqx-pkg doc"}]} ]} , {'emqx-enterprise', [ {erl_opts, prod_compile_opts()} , {relx, relx(Vsn, cloud, bin, ee)} , {overrides, prod_overrides()} , {project_app_dirs, project_app_dirs(ee)} + , {post_hooks, [{compile, "./build emqx-enterprise doc"}]} ]} , {'emqx-enterprise-pkg', [ {erl_opts, prod_compile_opts()} , {relx, relx(Vsn, cloud, pkg, ee)} , {overrides, prod_overrides()} , {project_app_dirs, project_app_dirs(ee)} + , {post_hooks, [{compile, "./build emqx-enterprise-pkg doc"}]} ]} , {'emqx-edge', [ {erl_opts, prod_compile_opts()} , {relx, relx(Vsn, edge, bin, ce)} , {overrides, prod_overrides()} , {project_app_dirs, project_app_dirs(ce)} + , {post_hooks, [{compile, "./build emqx-edge doc"}]} ]} , {'emqx-edge-pkg', [ {erl_opts, prod_compile_opts()} , {relx, relx(Vsn, edge, pkg, ce)} , {overrides, prod_overrides()} , {project_app_dirs, project_app_dirs(ce)} + , {post_hooks, [{compile, "./build emqx-edge-pkg doc"}]} ]} , {check, [ {erl_opts, common_compile_opts()}