From fa333157eb9bca2f6fa39c833578ac91b141e3e1 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Tue, 25 Oct 2022 16:55:25 +0800 Subject: [PATCH] fix: list exported json files failed with utf8 filenames --- .../src/emqx_mgmt_api_data.erl | 5 +- ...x_bridge_mqtt_data_export_import_SUITE.erl | 4 + .../emqx_webhook_data_export_import_SUITE.erl | 105 ++++++++++-------- .../430-中文.json | 52 +++++++++ .../ee430-中文.json | 98 ++++++++++++++++ changes/v4.3.22-en.md | 8 +- changes/v4.3.22-zh.md | 6 +- 7 files changed, 223 insertions(+), 55 deletions(-) create mode 100644 apps/emqx_management/test/emqx_webhook_data_export_import_SUITE_data/430-中文.json create mode 100644 apps/emqx_management/test/emqx_webhook_data_export_import_SUITE_data/ee430-中文.json diff --git a/apps/emqx_management/src/emqx_mgmt_api_data.erl b/apps/emqx_management/src/emqx_mgmt_api_data.erl index c24f4793b..815090538 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_data.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_data.erl @@ -94,7 +94,7 @@ get_list_exported() -> {ok, #file_info{size = Size, ctime = CTime = {{Y, M, D}, {H, MM, S}}}} -> CreatedAt = io_lib:format("~p-~p-~p ~p:~p:~p", [Y, M, D, H, MM, S]), Seconds = calendar:datetime_to_gregorian_seconds(CTime), - [{Seconds, [{filename, list_to_binary(File)}, + [{Seconds, [{filename, unicode:characters_to_binary(File)}, {size, Size}, {created_at, list_to_binary(CreatedAt)}, {node, node()} @@ -135,7 +135,7 @@ download(#{filename := Filename}, _Params) -> FullFilename = fullname(Filename), case file:read_file(FullFilename) of {ok, Bin} -> - {ok, #{filename => list_to_binary(Filename), + {ok, #{filename => unicode:characters_to_binary(filename:basename(FullFilename)), file => Bin}}; {error, Reason} -> minirest:return({error, Reason}) @@ -147,6 +147,7 @@ upload(Bindings, Params) -> do_upload(_Bindings, #{<<"filename">> := Filename, <<"file">> := Bin}) -> FullFilename = fullname(Filename), + ok = filelib:ensure_dir(FullFilename), case file:write_file(FullFilename, Bin) of ok -> minirest:return({ok, [{node, node()}]}); diff --git a/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl b/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl index f01eb9631..067dc0a4c 100644 --- a/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl +++ b/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl @@ -28,12 +28,16 @@ all() -> emqx_ct:all(?MODULE). init_per_suite(Cfg) -> + ekka_mnesia:start(), + ok = emqx_dashboard_admin:mnesia(boot), application:load(emqx_modules), application:load(emqx_bridge_mqtt), emqx_ct_helpers:start_apps([emqx_rule_engine, emqx_management]), + application:ensure_all_started(emqx_dashboard), Cfg. end_per_suite(Cfg) -> + application:stop(emqx_dashboard), emqx_ct_helpers:stop_apps([emqx_management, emqx_rule_engine]), Cfg. diff --git a/apps/emqx_management/test/emqx_webhook_data_export_import_SUITE.erl b/apps/emqx_management/test/emqx_webhook_data_export_import_SUITE.erl index 763c34134..5caa795c7 100644 --- a/apps/emqx_management/test/emqx_webhook_data_export_import_SUITE.erl +++ b/apps/emqx_management/test/emqx_webhook_data_export_import_SUITE.erl @@ -28,14 +28,18 @@ all() -> emqx_ct:all(?MODULE). init_per_suite(Cfg) -> + ekka_mnesia:start(), + ok = emqx_dashboard_admin:mnesia(boot), application:load(emqx_modules), application:load(emqx_web_hook), emqx_ct_helpers:start_apps([emqx_rule_engine, emqx_management]), + application:ensure_all_started(emqx_dashboard), ok = emqx_rule_registry:mnesia(boot), ok = emqx_rule_engine:load_providers(), Cfg. end_per_suite(Cfg) -> + application:stop(emqx_dashboard), emqx_ct_helpers:stop_apps([emqx_management, emqx_rule_engine]), Cfg. @@ -46,8 +50,8 @@ remove_resource(Id) -> emqx_rule_registry:remove_resource(Id), emqx_rule_registry:remove_resource_params(Id). -import(FilePath, Version) -> - ok = emqx_mgmt_data_backup:import(get_data_path() ++ "/" ++ FilePath, <<"{}">>), +import_and_check(Filename, Version) -> + {ok, #{code := 0}} = emqx_mgmt_api_data:import(#{}, [{<<"filename">>, Filename}]), lists:foreach(fun(#resource{id = Id, config = Config} = _Resource) -> case Id of <<"webhook">> -> @@ -58,34 +62,51 @@ import(FilePath, Version) -> end end, emqx_rule_registry:get_resources()). +upload_import_export_list_download(NameVsnTable) -> + lists:foreach(fun({Filename0, Vsn}) -> + Filename = unicode:characters_to_binary(Filename0), + FullPath = filename:join([get_data_path(), Filename]), + ct:pal("testing upload_import_export_list_download for file: ~ts, version: ~p", [FullPath, Vsn]), + %% upload + {ok, FileCnt} = file:read_file(FullPath), + {ok, #{code := 0}} = emqx_mgmt_api_data:upload(#{}, + [{<<"filename">>, Filename}, {<<"file">>, FileCnt}]), + %% import + ok = import_and_check(Filename, Vsn), + %% export + {ok, #{data := #{created_at := CAt, filename := FName, size := Size}}} + = emqx_mgmt_api_data:export(#{}, []), + ?assert(true, is_binary(CAt)), + ?assert(true, is_binary(FName)), + ?assert(true, is_integer(Size)), + %% list exported files + lists:foreach(fun({Seconds, Content}) -> + ?assert(true, is_integer(Seconds)), + ?assert(true, is_binary(proplists:get_value(filename, Content))), + ?assert(true, is_binary(proplists:get_value(created_at, Content))), + ?assert(true, is_integer(proplists:get_value(size, Content))) + end, emqx_mgmt_api_data:get_list_exported()), + %% download + ?assertMatch({ok, #{filename := FName}}, + emqx_mgmt_api_data:download(#{filename => FName}, [])) + end, NameVsnTable). + %%-------------------------------------------------------------------- %% Cases %%-------------------------------------------------------------------- -ifdef(EMQX_ENTERPRISE). -t_importee4010(_) -> - import("ee4010.json", ee4010), - {ok, _} = emqx_mgmt_data_backup:export(). - -t_importee410(_) -> - import("ee410.json", ee410), - {ok, _} = emqx_mgmt_data_backup:export(). - -t_importee411(_) -> - import("ee411.json", ee411), - {ok, _} = emqx_mgmt_data_backup:export(). - -t_importee420(_) -> - import("ee420.json", ee420), - {ok, _} = emqx_mgmt_data_backup:export(). - -t_importee425(_) -> - import("ee425.json", ee425), - {ok, _} = emqx_mgmt_data_backup:export(). - -t_importee430(_) -> - import("ee430.json", ee430), - {ok, _} = emqx_mgmt_data_backup:export(). +t_upload_import_export_list_download(_) -> + NameVsnTable = [ + {"ee4010.json", ee4010}, + {"ee410.json", ee410}, + {"ee411.json", ee411}, + {"ee420.json", ee420}, + {"ee425.json", ee425}, + {"ee430.json", ee430}, + {"ee430-中文.json", ee430} + ], + upload_import_export_list_download(NameVsnTable). %%-------------------------------------------------------------------- %% handle_config @@ -131,29 +152,17 @@ handle_config(_, _) -> ok. -ifndef(EMQX_ENTERPRISE). -t_import422(_) -> - import("422.json", 422), - {ok, _} = emqx_mgmt_data_backup:export(). - -t_import423(_) -> - import("423.json", 423), - {ok, _} = emqx_mgmt_data_backup:export(). - -t_import425(_) -> - import("425.json", 425), - {ok, _} = emqx_mgmt_data_backup:export(). - -t_import430(_) -> - import("430.json", 430), - {ok, _} = emqx_mgmt_data_backup:export(). - -t_import409(_) -> - import("409.json", 409), - {ok, _} = emqx_mgmt_data_backup:export(). - -t_import415(_) -> - import("415.json", 415), - {ok, _} = emqx_mgmt_data_backup:export(). +t_upload_import_export_list_download(_) -> + NameVsnTable = [ + {"422.json", 422}, + {"423.json", 423}, + {"425.json", 425}, + {"430.json", 430}, + {"430-中文.json", 430}, + {"409.json", 409}, + {"415.json", 415} + ], + upload_import_export_list_download(NameVsnTable). %%-------------------------------------------------------------------- %% handle_config diff --git a/apps/emqx_management/test/emqx_webhook_data_export_import_SUITE_data/430-中文.json b/apps/emqx_management/test/emqx_webhook_data_export_import_SUITE_data/430-中文.json new file mode 100644 index 000000000..9472152f9 --- /dev/null +++ b/apps/emqx_management/test/emqx_webhook_data_export_import_SUITE_data/430-中文.json @@ -0,0 +1,52 @@ +{ + "version": "4.3", + "rules": [], + "resources": [ + { + "id": "webhook", + "type": "web_hook", + "config": { + "cacertfile": { + "filename": "", + "file": "" + }, + "certfile": { + "filename": "", + "file": "" + }, + "keyfile": { + "filename": "", + "file": "" + }, + "connect_timeout": "5s", + "pool_size": 8, + "request_timeout": "5s", + "url": "http://www.emqx.io", + "verify": false + }, + "created_at": 1616581851001, + "description": "webhook" + } + ], + "blacklist": [], + "apps": [ + { + "id": "admin", + "secret": "public", + "name": "Default", + "desc": "Application user", + "status": true, + "expired": "undefined" + } + ], + "users": [ + { + "username": "admin", + "password": "q8v7hISIMz+iKn/ZuAaogvAxKbA=", + "tags": "administrator" + } + ], + "auth_mnesia": [], + "acl_mnesia": [], + "date": "2021-03-24 18:31:21" +} \ No newline at end of file diff --git a/apps/emqx_management/test/emqx_webhook_data_export_import_SUITE_data/ee430-中文.json b/apps/emqx_management/test/emqx_webhook_data_export_import_SUITE_data/ee430-中文.json new file mode 100644 index 000000000..36986f24b --- /dev/null +++ b/apps/emqx_management/test/emqx_webhook_data_export_import_SUITE_data/ee430-中文.json @@ -0,0 +1,98 @@ +{ + "version": "4.3", + "rules": [], + "resources": [ + { + "id": "webhook", + "type": "web_hook", + "config": { + "cacertfile": { + "filename": "", + "file": "" + }, + "certfile": { + "filename": "", + "file": "" + }, + "connect_timeout": "5s", + "keyfile": { + "filename": "", + "file": "" + }, + "pool_size": 8, + "request_timeout": "5s", + "url": "http://www.emqx.io", + "verify": false + }, + "created_at": 1618304340172, + "description": "webhook" + } + ], + "blacklist": [], + "apps": [ + { + "id": "admin", + "secret": "public", + "name": "Default", + "desc": "Application user", + "status": true, + "expired": "undefined" + } + ], + "users": [ + { + "username": "admin", + "password": "qq8hg9pOkmYiHqzi3+bcUaK2CGA=", + "tags": "administrator" + } + ], + "auth_mnesia": [], + "acl_mnesia": [], + "modules": [ + { + "id": "module:aabeddbf", + "type": "recon", + "config": {}, + "enabled": true, + "created_at": 1618304311061, + "description": "" + }, + { + "id": "module:cbe6d976", + "type": "internal_acl", + "config": { + "acl_rule_file": "etc/acl.conf" + }, + "enabled": true, + "created_at": 1618304311061, + "description": "" + }, + { + "id": "module:46375e06", + "type": "retainer", + "config": { + "storage_type": "ram", + "max_retained_messages": 0, + "max_payload_size": "1MB", + "expiry_interval": 0 + }, + "enabled": true, + "created_at": 1618304311061, + "description": "" + }, + { + "id": "module:091eb7c3", + "type": "presence", + "config": { + "qos": 0 + }, + "enabled": true, + "created_at": 1618304311061, + "description": "" + } + ], + "schemas": [], + "configs": [], + "listeners_state": [], + "date": "2021-04-13 17:59:52" +} \ No newline at end of file diff --git a/changes/v4.3.22-en.md b/changes/v4.3.22-en.md index db76d3423..88948bb3f 100644 --- a/changes/v4.3.22-en.md +++ b/changes/v4.3.22-en.md @@ -5,10 +5,12 @@ ### Bug fixes -- Improve the display of rule's 'Maximum Speed' counter to only reserve 2 decimal places. [#9185](https://github.com/emqx/emqx/pull/9185) +- Fix that after uploading a backup file with an UTF8 filename, HTTP API `GET /data/export` fails with status code 500 [#9224](https://github.com/emqx/emqx/pull/9224). + +- Improve the display of rule's 'Maximum Speed' counter to only reserve 2 decimal places [#9185](https://github.com/emqx/emqx/pull/9185). This is to avoid displaying floats like `0.30000000000000004` on the dashboard. -- Fix the issue that emqx prints too many error logs when connecting to mongodb but auth failed. [#9184](https://github.com/emqx/emqx/pull/9184) +- Fix the issue that emqx prints too many error logs when connecting to mongodb but auth failed [#9184](https://github.com/emqx/emqx/pull/9184). - Fix that after receiving publish in `idle mode` the emqx-sn gateway may panic [#9024](https://github.com/emqx/emqx/pull/9024). @@ -16,5 +18,5 @@ - Restore old `emqx_auth_jwt` module API, so the hook callback functions registered in older version will not be invalidated after hot-upgrade [#9144](https://github.com/emqx/emqx/pull/9144). -- Fixed the response status code for the `/status` endpoint [#9210](https://github.com/emqx/emqx/pull/9210). +- Fixed the response status code for the `/status` endpoint [#9210](https://github.com/emqx/emqx/pull/9210). Before the fix, it always returned `200` even if the EMQX application was not running. Now it returns `503` in that case. diff --git a/changes/v4.3.22-zh.md b/changes/v4.3.22-zh.md index e7d343db0..3c1877c27 100644 --- a/changes/v4.3.22-zh.md +++ b/changes/v4.3.22-zh.md @@ -5,10 +5,12 @@ ### 修复 -- 改进规则的 "最大执行速度" 的计数，只保留小数点之后 2 位 [#9185](https://github.com/emqx/emqx/pull/9185) +- 修复若上传的备份文件名中包含 UTF8 字符，`GET /data/export` HTTP 接口返回 500 错误 [#9224](https://github.com/emqx/emqx/pull/9224)。 + +- 改进规则的 "最大执行速度" 的计数，只保留小数点之后 2 位 [#9185](https://github.com/emqx/emqx/pull/9185)。 避免在 dashboard 上展示类似这样的浮点数：`0.30000000000000004`。 -- 修复在尝试连接 MongoDB 数据库过程中，如果认证失败会不停打印错误日志的问题。[#9184](https://github.com/emqx/emqx/pull/9184) +- 修复在尝试连接 MongoDB 数据库过程中，如果认证失败会不停打印错误日志的问题 [#9184](https://github.com/emqx/emqx/pull/9184)。 - 修复 emqx-sn 插件在“空闲”状态下收到消息发布请求时可能崩溃的情况 [#9024](https://github.com/emqx/emqx/pull/9024)。