From 1e6b3b51c0e89aa115a8596373d76e11f2dcc323 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Tue, 21 Jun 2022 19:04:17 +0800 Subject: [PATCH 1/3] feat: generate API schema files for connectors and bridges --- apps/emqx_conf/src/emqx_conf.erl | 37 ++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl index 2f11d8e09..6ac3152f1 100644 --- a/apps/emqx_conf/src/emqx_conf.erl +++ b/apps/emqx_conf/src/emqx_conf.erl @@ -145,7 +145,7 @@ dump_schema(Dir, SchemaModule, I18nFile) -> lists:foreach( fun(Lang) -> gen_config_md(Dir, I18nFile, SchemaModule, Lang), - gen_hot_conf_schema_json(Dir, I18nFile, Lang), + gen_api_schema_json(Dir, I18nFile, Lang), gen_example_conf(filename:dirname(I18nFile), I18nFile, SchemaModule, Lang) end, [en, zh] @@ -161,14 +161,32 @@ gen_schema_json(Dir, I18nFile, SchemaModule) -> IoData = jsx:encode(JsonMap, [space, {indent, 4}]), ok = file:write_file(SchemaJsonFile, IoData). -gen_hot_conf_schema_json(Dir, I18nFile, Lang) -> +gen_api_schema_json(Dir, I18nFile, Lang) -> emqx_dashboard:init_i18n(I18nFile, Lang), - JsonFile = "hot-config-schema-" ++ atom_to_list(Lang) ++ ".json", - HotConfigSchemaFile = filename:join([Dir, JsonFile]), - io:format(user, "===< Generating: ~s~n", [HotConfigSchemaFile]), - ok = gen_hot_conf_schema(HotConfigSchemaFile), + gen_api_schema_json_hotconf(Dir, Lang), + gen_api_schema_json_connector(Dir, Lang), + gen_api_schema_json_bridge(Dir, Lang), emqx_dashboard:clear_i18n(). +gen_api_schema_json_hotconf(Dir, Lang) -> + SchemaInfo = #{title => <<"EMQX Hot Conf API Schema">>, version => <<"0.1.0">>}, + File = schema_filename(Dir, "hot-config-schema-", Lang), + ok = do_gen_api_schema_json(File, emqx_mgmt_api_configs, SchemaInfo). + +gen_api_schema_json_connector(Dir, Lang) -> + SchemaInfo = #{title => <<"EMQX Connector API Schema">>, version => <<"0.1.0">>}, + File = schema_filename(Dir, "connector-api-", Lang), + ok = do_gen_api_schema_json(File, emqx_connector_api, SchemaInfo). + +gen_api_schema_json_bridge(Dir, Lang) -> + SchemaInfo = #{title => <<"EMQX Data Bridge API Schema">>, version => <<"0.1.0">>}, + File = schema_filename(Dir, "bridge-api-", Lang), + ok = do_gen_api_schema_json(File, emqx_bridge_api, SchemaInfo). + +schema_filename(Dir, Prefix, Lang) -> + Filename = Prefix ++ atom_to_list(Lang) ++ ".json", + filename:join([Dir, Filename]). + gen_config_md(Dir, I18nFile, SchemaModule, Lang0) -> Lang = atom_to_list(Lang0), SchemaMdFile = filename:join([Dir, "config-" ++ Lang ++ ".md"]), @@ -214,9 +232,10 @@ gen_example(File, SchemaModule, I18nFile, Lang) -> file:write_file(File, Example). %% Only gen hot_conf schema, not all configuration fields. -gen_hot_conf_schema(File) -> +do_gen_api_schema_json(File, SchemaMod, SchemaInfo) -> + io:format(user, "===< Generating: ~s~n", [File]), {ApiSpec0, Components0} = emqx_dashboard_swagger:spec( - emqx_mgmt_api_configs, + SchemaMod, #{schema_converter => fun hocon_schema_to_spec/2} ), ApiSpec = lists:foldl( @@ -248,7 +267,7 @@ gen_hot_conf_schema(File) -> Components = lists:foldl(fun(M, Acc) -> maps:merge(M, Acc) end, #{}, Components0), IoData = jsx:encode( #{ - info => #{title => <<"EMQX Hot Conf Schema">>, version => <<"0.1.0">>}, + info => SchemaInfo, paths => ApiSpec, components => #{schemas => Components} }, From 613a13e5e49812430ad985159a1d3e1a2d6ecc0f Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Wed, 22 Jun 2022 09:17:51 +0800 Subject: [PATCH 2/3] chore: add appup.src for emqx_conf --- apps/emqx_conf/src/emqx_conf.app.src | 2 +- apps/emqx_conf/src/emqx_conf.appup.src | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 apps/emqx_conf/src/emqx_conf.appup.src diff --git a/apps/emqx_conf/src/emqx_conf.app.src b/apps/emqx_conf/src/emqx_conf.app.src index 0e355c6d2..a4946e8cf 100644 --- a/apps/emqx_conf/src/emqx_conf.app.src +++ b/apps/emqx_conf/src/emqx_conf.app.src @@ -1,6 +1,6 @@ {application, emqx_conf, [ {description, "EMQX configuration management"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {registered, []}, {mod, {emqx_conf_app, []}}, {applications, [kernel, stdlib]}, diff --git a/apps/emqx_conf/src/emqx_conf.appup.src b/apps/emqx_conf/src/emqx_conf.appup.src new file mode 100644 index 000000000..f443b2abc --- /dev/null +++ b/apps/emqx_conf/src/emqx_conf.appup.src @@ -0,0 +1,8 @@ +%% -*- mode: erlang -*- +{"0.1.1", + [{"0.1.0", + [{load_module,emqx_conf,brutal_purge,soft_purge,[]}]}, + {<<".*">>,[]}], + [{"0.1.0", + [{load_module,emqx_conf,brutal_purge,soft_purge,[]}]}, + {<<".*">>,[]}]}. From 08207c086c623e0a36629aa3ea1c116e74104a4e Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Wed, 22 Jun 2022 10:59:11 +0800 Subject: [PATCH 3/3] fix: update integration testcases for relup --- .ci/fvt_tests/relup.lux | 50 ++++++++++++++++++++++++-- .github/workflows/run_relup_tests.yaml | 1 + 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/.ci/fvt_tests/relup.lux b/.ci/fvt_tests/relup.lux index 45065d4bb..f5b47b89e 100644 --- a/.ci/fvt_tests/relup.lux +++ b/.ci/fvt_tests/relup.lux @@ -85,6 +85,27 @@ !cp -f ../$PROFILE-$VSN-*-ubuntu20.04-amd64.tar.gz releases/ + ## 1. upgrade to the new version + !./bin/emqx install $VSN + ?Made release permanent: "$VSN" + ?SH-PROMPT + + !./bin/emqx versions |grep permanent + ?(.*)$VSN + ?SH-PROMPT + + ## 2. downgrade to the old version + !./bin/emqx install $old_vsn + ?Made release permanent:.* + ?SH-PROMPT + + !./bin/emqx versions |grep permanent | grep -qs "$old_vsn" + ?SH-PROMPT: + !echo ==$$?== + ?^==0== + ?SH-PROMPT: + + ## 3. again, upgrade to the new version !./bin/emqx install $VSN ?Made release permanent: "$VSN" ?SH-PROMPT @@ -110,6 +131,27 @@ !cp -f ../$PROFILE-$VSN-*-ubuntu20.04-amd64.tar.gz releases/ + ## 1. upgrade to the new version + !./bin/emqx install $VSN + ?Made release permanent: "$VSN" + ?SH-PROMPT + + !./bin/emqx versions |grep permanent + ?(.*)$VSN + ?SH-PROMPT + + ## 2. downgrade to the old version + !./bin/emqx install $old_vsn + ?Made release permanent:.* + ?SH-PROMPT + + !./bin/emqx versions |grep permanent | grep -qs "$old_vsn" + ?SH-PROMPT: + !echo ==$$?== + ?^==0== + ?SH-PROMPT: + + ## 3. again, upgrade to the new version !./bin/emqx install $VSN ?Made release permanent: "$VSN" ?SH-PROMPT @@ -129,18 +171,20 @@ ?Plugin\(emqx_management.*active=true\) ?SH-PROMPT +## We don't guarantee not to lose a single message! +## So even if we received 290~300 messages, we consider it as success [shell bench] ???publish complete ??SH-PROMPT: !sleep 5 ?SH-PROMPT - !curl --user admin:public --silent --show-error http://localhost:8081/api/v4/rules | jq --raw-output ".data[0].metrics[] | select(.node==\"emqx@127.0.0.1\").matched" + !curl --user admin:public --silent --show-error http://localhost:18083/api/v5/rules | jq --raw-output ".[0].node_metrics[] | select(.node==\"emqx@127.0.0.1\") | .metrics.matched" ?300 ?SH-PROMPT - !curl http://127.0.0.1:8080/counter - ???{"data":300,"code":0} + !curl --user admin:public --silent --show-error http://localhost:18083/api/v5/rules | jq --raw-output ".[0].node_metrics[] | select(.node==\"emqx@127.0.0.1\") | .metrics.\"actions.success\"" + ?\{"data":(29[0-9])|(300),"code":0\} ?SH-PROMPT [shell emqx2] diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml index b14e6a7ce..97c33f229 100644 --- a/.github/workflows/run_relup_tests.yaml +++ b/.github/workflows/run_relup_tests.yaml @@ -71,6 +71,7 @@ jobs: - name: Get old vsn run: echo "OLD_VSNS=$(emqx/scripts/relup-base-vsns.sh ${{ matrix.profile }} | xargs echo -n)" >> $GITHUB_ENV + run: echo "VSN=$(emqx/pkg-vsn.sh ${{ matrix.profile }})" >> $GITHUB_ENV - name: build emqx env: