Merge pull request #6905 from zmstone/feat-dump-schema-json

feat: dump schema json

Makefile

@@ -34,7 +34,6 @@ all: $(REBAR) $(PROFILES)
 
 .PHONY: ensure-rebar3
 ensure-rebar3:
-	@$(SCRIPTS)/fail-on-old-otp-version.escript
 	@$(SCRIPTS)/ensure-rebar3.sh $(REBAR_VERSION)
 
 .PHONY: ensure-hex
@@ -111,6 +110,9 @@ cover: $(REBAR)
 coveralls: $(REBAR)
 	@ENABLE_COVER_COMPILE=1 $(REBAR) as test coveralls send
 
+COMMON_DEPS := $(REBAR) get-dashboard conf-segs
+ELIXIR_COMMON_DEPS := ensure-hex ensure-mix-rebar3 ensure-mix-rebar
+
 .PHONY: $(REL_PROFILES)
 $(REL_PROFILES:%=%): $(COMMON_DEPS)
 	@$(REBAR) as $(@) do release
@@ -143,7 +145,7 @@ deps-all: $(REBAR) $(PROFILES:%=deps-%)
 ## share downloads between CI steps and/or copied into containers
 ## which may not have the right credentials
 .PHONY: $(PROFILES:%=deps-%)
-$(PROFILES:%=deps-%): $(REBAR) get-dashboard
+$(PROFILES:%=deps-%): $(COMMON_DEPS)
 	@$(REBAR) as $(@:deps-%=%) get-deps
 	@rm -f rebar.lock
 
@@ -155,9 +157,6 @@ xref: $(REBAR)
 dialyzer: $(REBAR)
 	@$(REBAR) as check dialyzer
 
-COMMON_DEPS := $(REBAR) get-dashboard conf-segs
-ELIXIR_COMMON_DEPS := ensure-hex ensure-mix-rebar3 ensure-mix-rebar
-
 ## rel target is to create release package without relup
 .PHONY: $(REL_PROFILES:%=%-rel) $(PKG_PROFILES:%=%-rel)
 $(REL_PROFILES:%=%-rel) $(PKG_PROFILES:%=%-rel): $(COMMON_DEPS)
@@ -217,6 +216,7 @@ endef
 ALL_TGZS = $(REL_PROFILES)
 $(foreach zt,$(ALL_TGZS),$(eval $(call gen-docker-target-testing,$(zt))))
 
+.PHONY:
 conf-segs:
 	@scripts/merge-config.escript
 

apps/emqx/rebar.config

@@ -19,7 +19,7 @@
     , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.0"}}}
     , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.11.3"}}}
     , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.0"}}}
-    , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.23.0"}}}
+    , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.24.0"}}}
     , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}
     , {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}
     , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.16.0"}}}

apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl

@@ -71,7 +71,7 @@ fields(limiter) ->
 fields(limiter_opts) ->
     [ {global, sc(ref(rate_burst), #{})}
     , {zone, sc(map("zone name", ref(rate_burst)), #{})}
-    , {bucket, sc(map("bucket id", ref(bucket)),
+    , {bucket, sc(map("bucket_id", ref(bucket)),
                   #{desc => "token bucket"})}
     ];
 

apps/emqx_conf/src/emqx_conf.erl

@@ -24,7 +24,7 @@
 -export([update/3, update/4]).
 -export([remove/2, remove/3]).
 -export([reset/2, reset/3]).
--export([gen_doc/1]).
+-export([dump_schema/1]).
 
 %% for rpc
 -export([get_node_and_config/1]).
@@ -121,20 +121,31 @@ reset(Node, KeyPath, Opts) when Node =:= node() ->
 reset(Node, KeyPath, Opts) ->
     emqx_conf_proto_v1:reset(Node, KeyPath, Opts).
 
+%% @doc Called from build script.
+-spec dump_schema(file:name_all()) -> ok.
+dump_schema(Dir) ->
+    SchemaJsonFile = filename:join([Dir, "schema.json"]),
+    JsonMap = hocon_schema_json:gen(emqx_conf_schema),
+    IoData = jsx:encode(JsonMap, [space, {indent, 4}]),
+    io:format(user, "===< Generating: ~s~n", [SchemaJsonFile]),
+    ok = file:write_file(SchemaJsonFile, IoData),
+    SchemaMarkdownFile = filename:join([Dir, "config.md"]),
+    io:format(user, "===< Generating: ~s~n", [SchemaMarkdownFile ]),
+    ok = gen_doc(SchemaMarkdownFile).
+
+%%--------------------------------------------------------------------
+%% Internal functions
+%%--------------------------------------------------------------------
+
 -spec gen_doc(file:name_all()) -> ok.
 gen_doc(File) ->
     Version = emqx_release:version(),
     Title = "# EMQ X " ++ Version ++ " Configuration",
     BodyFile = filename:join([code:lib_dir(emqx_conf), "etc", "emqx_conf.md"]),
     {ok, Body} = file:read_file(BodyFile),
-    Doc = hocon_schema_doc:gen(emqx_conf_schema, #{title => Title,
+    Doc = hocon_schema_md:gen(emqx_conf_schema, #{title => Title, body => Body}),
-                                                   body => Body}),
     file:write_file(File, Doc).
 
-%%--------------------------------------------------------------------
-%% Internal functions
-%%--------------------------------------------------------------------
-
 check_cluster_rpc_result(Result) ->
     case Result of
         {ok, _TnxId, Res} -> Res;

apps/emqx_prometheus/rebar.config

@@ -4,7 +4,7 @@
  [ {emqx, {path, "../emqx"}},
    %% FIXME: tag this as v3.1.3
    {prometheus, {git, "https://github.com/emqx/prometheus.erl", {ref, "9994c76adca40d91a2545102230ccce2423fd8a7"}}},
-   {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.23.0"}}},
+   {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.24.0"}}},
    {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.2.11"}}}
  ]}.
 

build

@@ -69,9 +69,8 @@ make_doc() {
     libs_dir2="$("$FIND" "_build/$PROFILE/lib/" -maxdepth 2 -name ebin -type d)"
     # shellcheck disable=SC2086
     erl -noshell -pa $libs_dir1 $libs_dir2 -eval \
-        "F = filename:join(['_build', '${PROFILE}', lib, emqx_dashboard, priv, www, static, 'config.md']), \
+        "Dir = filename:join(['_build', '${PROFILE}', lib, emqx_dashboard, priv, www, static]), \
-         io:format(\"===< Generating: ~s~n\", [F]),
+         ok = emqx_conf:dump_schema(Dir), \
-         ok = emqx_conf:gen_doc(F), \
          halt(0)."
 }
 

mix.exs

@@ -68,7 +68,7 @@ defmodule EMQXUmbrella.MixProject do
       # in conflict by emqtt and hocon
       {:getopt, "1.0.2", override: true},
       {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "0.16.0", override: true},
-      {:hocon, github: "emqx/hocon", tag: "0.23.0", override: true},
+      {:hocon, github: "emqx/hocon", tag: "0.24.0", override: true},
       {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.4.1", override: true},
       {:esasl, github: "emqx/esasl", tag: "0.2.0"},
       {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"},

rebar.config

@@ -66,7 +66,7 @@
     , {system_monitor, {git, "https://github.com/k32/system_monitor", {tag, "2.2.1"}}}
     , {getopt, "1.0.2"}
     , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.16.0"}}}
-    , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.23.0"}}}
+    , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.24.0"}}}
     , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.4.1"}}}
     , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}}
     , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}}

rebar.config.erl

@@ -3,6 +3,7 @@
 -export([do/2]).
 
 do(Dir, CONFIG) ->
+    ok = assert_otp(),
     case iolist_to_binary(Dir) of
         <<".">> ->
             C1 = deps(CONFIG),
@@ -12,6 +13,22 @@ do(Dir, CONFIG) ->
             CONFIG
     end.
 
+assert_otp() ->
+    Oldest = 23,
+    Latest = 24,
+    OtpRelease = list_to_integer(erlang:system_info(otp_release)),
+    case OtpRelease < Oldest orelse OtpRelease > Latest of
+        true ->
+            io:format(standard_error, "ERROR: Erlang/OTP version ~p found. min=~p, recommended=~p~n",
+                      [OtpRelease, Oldest, Latest]),
+            halt(1);
+        false when OtpRelease =/= Latest ->
+            io:format("WARNING: Erlang/OTP version ~p found, recommended==~p~n",
+                      [OtpRelease, Latest]);
+        false ->
+            ok
+    end.
+
 bcrypt() ->
     {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {branch, "0.6.0"}}}.
 

scripts/fail-on-old-otp-version.escript

@@ -1,12 +0,0 @@
-#!/usr/bin/env escript
-
-main(_) ->
-    OtpRelease = list_to_integer(erlang:system_info(otp_release)),
-    case OtpRelease < 21 of
-        true ->
-            io:format(standard_error, "ERROR: Erlang/OTP version ~p found. required_min=21, recommended=23~n", [OtpRelease]),
-            halt(1);
-        false ->
-            ok
-    end.
-