yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #9224 from terry-xiaoyu/utf8_backup_filenames 

fix: list exported json files failed with utf8 filenames
This commit is contained in:
Xinyu Liu 2022-10-26 15:47:00 +08:00 committed by GitHub
parent ebd01da78b fa333157eb
commit 7f63912bba
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 223 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
apps/emqx_management/src/emqx_mgmt_api_data.erl
View File

@ -94,7 +94,7 @@ get_list_exported() ->
{ok, #file_info{size = Size, ctime = CTime = {{Y, M, D}, {H, MM, S}}}} ->
CreatedAt = io_lib:format("~p-~p-~p ~p:~p:~p", [Y, M, D, H, MM, S]),
Seconds = calendar:datetime_to_gregorian_seconds(CTime),
[{Seconds, [{filename, list_to_binary(File)},
[{Seconds, [{filename, unicode:characters_to_binary(File)},
{size, Size},
{created_at, list_to_binary(CreatedAt)},
{node, node()}
@ -135,7 +135,7 @@ download(#{filename := Filename}, _Params) ->
FullFilename = fullname(Filename),
case file:read_file(FullFilename) of
{ok, Bin} ->
{ok, #{filename => list_to_binary(Filename),
{ok, #{filename => unicode:characters_to_binary(filename:basename(FullFilename)),
file => Bin}};
{error, Reason} ->
minirest:return({error, Reason})
@ -147,6 +147,7 @@ upload(Bindings, Params) ->
do_upload(_Bindings, #{<<"filename">> := Filename,
<<"file">> := Bin}) ->
FullFilename = fullname(Filename),
ok = filelib:ensure_dir(FullFilename),
case file:write_file(FullFilename, Bin) of
ok ->
minirest:return({ok, [{node, node()}]});

4
apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl
View File

@ -28,12 +28,16 @@ all() ->
emqx_ct:all(?MODULE).
init_per_suite(Cfg) ->
ekka_mnesia:start(),
ok = emqx_dashboard_admin:mnesia(boot),
application:load(emqx_modules),
application:load(emqx_bridge_mqtt),
emqx_ct_helpers:start_apps([emqx_rule_engine, emqx_management]),
application:ensure_all_started(emqx_dashboard),
Cfg.
end_per_suite(Cfg) ->
application:stop(emqx_dashboard),
emqx_ct_helpers:stop_apps([emqx_management, emqx_rule_engine]),
Cfg.

105
apps/emqx_management/test/emqx_webhook_data_export_import_SUITE.erl
View File

@ -28,14 +28,18 @@ all() ->
emqx_ct:all(?MODULE).
init_per_suite(Cfg) ->
ekka_mnesia:start(),
ok = emqx_dashboard_admin:mnesia(boot),
application:load(emqx_modules),
application:load(emqx_web_hook),
emqx_ct_helpers:start_apps([emqx_rule_engine, emqx_management]),
application:ensure_all_started(emqx_dashboard),
ok = emqx_rule_registry:mnesia(boot),
ok = emqx_rule_engine:load_providers(),
Cfg.
end_per_suite(Cfg) ->
application:stop(emqx_dashboard),
emqx_ct_helpers:stop_apps([emqx_management, emqx_rule_engine]),
Cfg.
@ -46,8 +50,8 @@ remove_resource(Id) ->
emqx_rule_registry:remove_resource(Id),
emqx_rule_registry:remove_resource_params(Id).
import(FilePath, Version) ->
ok = emqx_mgmt_data_backup:import(get_data_path() ++ "/" ++ FilePath, <<"{}">>),
import_and_check(Filename, Version) ->
{ok, #{code := 0}} = emqx_mgmt_api_data:import(#{}, [{<<"filename">>, Filename}]),
lists:foreach(fun(#resource{id = Id, config = Config} = _Resource) ->
case Id of
<<"webhook">> ->
@ -58,34 +62,51 @@ import(FilePath, Version) ->
end
end, emqx_rule_registry:get_resources()).
upload_import_export_list_download(NameVsnTable) ->
lists:foreach(fun({Filename0, Vsn}) ->
Filename = unicode:characters_to_binary(Filename0),
FullPath = filename:join([get_data_path(), Filename]),
ct:pal("testing upload_import_export_list_download for file: ~ts, version: ~p", [FullPath, Vsn]),
%% upload
{ok, FileCnt} = file:read_file(FullPath),
{ok, #{code := 0}} = emqx_mgmt_api_data:upload(#{},
[{<<"filename">>, Filename}, {<<"file">>, FileCnt}]),
%% import
ok = import_and_check(Filename, Vsn),
%% export
{ok, #{data := #{created_at := CAt, filename := FName, size := Size}}}
= emqx_mgmt_api_data:export(#{}, []),
?assert(true, is_binary(CAt)),
?assert(true, is_binary(FName)),
?assert(true, is_integer(Size)),
%% list exported files
lists:foreach(fun({Seconds, Content}) ->
?assert(true, is_integer(Seconds)),
?assert(true, is_binary(proplists:get_value(filename, Content))),
?assert(true, is_binary(proplists:get_value(created_at, Content))),
?assert(true, is_integer(proplists:get_value(size, Content)))
end, emqx_mgmt_api_data:get_list_exported()),
%% download
?assertMatch({ok, #{filename := FName}},
emqx_mgmt_api_data:download(#{filename => FName}, []))
end, NameVsnTable).
%%--------------------------------------------------------------------
%% Cases
%%--------------------------------------------------------------------
-ifdef(EMQX_ENTERPRISE).
t_importee4010(_) ->
import("ee4010.json", ee4010),
{ok, _} = emqx_mgmt_data_backup:export().
t_importee410(_) ->
import("ee410.json", ee410),
{ok, _} = emqx_mgmt_data_backup:export().
t_importee411(_) ->
import("ee411.json", ee411),
{ok, _} = emqx_mgmt_data_backup:export().
t_importee420(_) ->
import("ee420.json", ee420),
{ok, _} = emqx_mgmt_data_backup:export().
t_importee425(_) ->
import("ee425.json", ee425),
{ok, _} = emqx_mgmt_data_backup:export().
t_importee430(_) ->
import("ee430.json", ee430),
{ok, _} = emqx_mgmt_data_backup:export().
t_upload_import_export_list_download(_) ->
NameVsnTable = [
{"ee4010.json", ee4010},
{"ee410.json", ee410},
{"ee411.json", ee411},
{"ee420.json", ee420},
{"ee425.json", ee425},
{"ee430.json", ee430},
{"ee430-中文.json", ee430}
],
upload_import_export_list_download(NameVsnTable).
%%--------------------------------------------------------------------
%% handle_config
@ -131,29 +152,17 @@ handle_config(_, _) -> ok.
-ifndef(EMQX_ENTERPRISE).
t_import422(_) ->
import("422.json", 422),
{ok, _} = emqx_mgmt_data_backup:export().
t_import423(_) ->
import("423.json", 423),
{ok, _} = emqx_mgmt_data_backup:export().
t_import425(_) ->
import("425.json", 425),
{ok, _} = emqx_mgmt_data_backup:export().
t_import430(_) ->
import("430.json", 430),
{ok, _} = emqx_mgmt_data_backup:export().
t_import409(_) ->
import("409.json", 409),
{ok, _} = emqx_mgmt_data_backup:export().
t_import415(_) ->
import("415.json", 415),
{ok, _} = emqx_mgmt_data_backup:export().
t_upload_import_export_list_download(_) ->
NameVsnTable = [
{"422.json", 422},
{"423.json", 423},
{"425.json", 425},
{"430.json", 430},
{"430-中文.json", 430},
{"409.json", 409},
{"415.json", 415}
],
upload_import_export_list_download(NameVsnTable).
%%--------------------------------------------------------------------
%% handle_config

52
apps/emqx_management/test/emqx_webhook_data_export_import_SUITE_data/430-中文.json Normal file
View File

@ -0,0 +1,52 @@
{
"version": "4.3",
"rules": [],
"resources": [
{
"id": "webhook",
"type": "web_hook",
"config": {
"cacertfile": {
"filename": "",
"file": ""
},
"certfile": {
"filename": "",
"file": ""
},
"keyfile": {
"filename": "",
"file": ""
},
"connect_timeout": "5s",
"pool_size": 8,
"request_timeout": "5s",
"url": "http://www.emqx.io",
"verify": false
},
"created_at": 1616581851001,
"description": "webhook"
}
],
"blacklist": [],
"apps": [
{
"id": "admin",
"secret": "public",
"name": "Default",
"desc": "Application user",
"status": true,
"expired": "undefined"
}
],
"users": [
{
"username": "admin",
"password": "q8v7hISIMz+iKn/ZuAaogvAxKbA=",
"tags": "administrator"
}
],
"auth_mnesia": [],
"acl_mnesia": [],
"date": "2021-03-24 18:31:21"
}

98
apps/emqx_management/test/emqx_webhook_data_export_import_SUITE_data/ee430-中文.json Normal file
View File

@ -0,0 +1,98 @@
{
"version": "4.3",
"rules": [],
"resources": [
{
"id": "webhook",
"type": "web_hook",
"config": {
"cacertfile": {
"filename": "",
"file": ""
},
"certfile": {
"filename": "",
"file": ""
},
"connect_timeout": "5s",
"keyfile": {
"filename": "",
"file": ""
},
"pool_size": 8,
"request_timeout": "5s",
"url": "http://www.emqx.io",
"verify": false
},
"created_at": 1618304340172,
"description": "webhook"
}
],
"blacklist": [],
"apps": [
{
"id": "admin",
"secret": "public",
"name": "Default",
"desc": "Application user",
"status": true,
"expired": "undefined"
}
],
"users": [
{
"username": "admin",
"password": "qq8hg9pOkmYiHqzi3+bcUaK2CGA=",
"tags": "administrator"
}
],
"auth_mnesia": [],
"acl_mnesia": [],
"modules": [
{
"id": "module:aabeddbf",
"type": "recon",
"config": {},
"enabled": true,
"created_at": 1618304311061,
"description": ""
},
{
"id": "module:cbe6d976",
"type": "internal_acl",
"config": {
"acl_rule_file": "etc/acl.conf"
},
"enabled": true,
"created_at": 1618304311061,
"description": ""
},
{
"id": "module:46375e06",
"type": "retainer",
"config": {
"storage_type": "ram",
"max_retained_messages": 0,
"max_payload_size": "1MB",
"expiry_interval": 0
},
"enabled": true,
"created_at": 1618304311061,
"description": ""
},
{
"id": "module:091eb7c3",
"type": "presence",
"config": {
"qos": 0
},
"enabled": true,
"created_at": 1618304311061,
"description": ""
}
],
"schemas": [],
"configs": [],
"listeners_state": [],
"date": "2021-04-13 17:59:52"
}

8
changes/v4.3.22-en.md
View File

@ -10,10 +10,12 @@
### Bug fixes
- Improve the display of rule's 'Maximum Speed' counter to only reserve 2 decimal places. [#9185](https://github.com/emqx/emqx/pull/9185)
- Fix that after uploading a backup file with an UTF8 filename, HTTP API `GET /data/export` fails with status code 500 [#9224](https://github.com/emqx/emqx/pull/9224).
- Improve the display of rule's 'Maximum Speed' counter to only reserve 2 decimal places [#9185](https://github.com/emqx/emqx/pull/9185).
This is to avoid displaying floats like `0.30000000000000004` on the dashboard.
- Fix the issue that emqx prints too many error logs when connecting to mongodb but auth failed. [#9184](https://github.com/emqx/emqx/pull/9184)
- Fix the issue that emqx prints too many error logs when connecting to mongodb but auth failed [#9184](https://github.com/emqx/emqx/pull/9184).
- Fix that after receiving publish in `idle mode` the emqx-sn gateway may panic [#9024](https://github.com/emqx/emqx/pull/9024).
@ -21,5 +23,5 @@
- Restore old `emqx_auth_jwt` module API, so the hook callback functions registered in older version will not be invalidated after hot-upgrade [#9144](https://github.com/emqx/emqx/pull/9144).
- Fixed the response status code for the `/status` endpoint [#9210](https://github.com/emqx/emqx/pull/9210).
- Fixed the response status code for the `/status` endpoint [#9210](https://github.com/emqx/emqx/pull/9210).
Before the fix, it always returned `200` even if the EMQX application was not running. Now it returns `503` in that case.

6
changes/v4.3.22-zh.md
View File

@ -10,10 +10,12 @@
### 修复
- 改进规则的 "最大执行速度" 的计数，只保留小数点之后 2 位 [#9185](https://github.com/emqx/emqx/pull/9185)
- 修复若上传的备份文件名中包含 UTF8 字符，`GET /data/export` HTTP 接口返回 500 错误 [#9224](https://github.com/emqx/emqx/pull/9224)。
- 改进规则的 "最大执行速度" 的计数，只保留小数点之后 2 位 [#9185](https://github.com/emqx/emqx/pull/9185)。
避免在 dashboard 上展示类似这样的浮点数：`0.30000000000000004`。
- 修复在尝试连接 MongoDB 数据库过程中，如果认证失败会不停打印错误日志的问题。[#9184](https://github.com/emqx/emqx/pull/9184)
- 修复在尝试连接 MongoDB 数据库过程中，如果认证失败会不停打印错误日志的问题 [#9184](https://github.com/emqx/emqx/pull/9184)。
- 修复 emqx-sn 插件在“空闲”状态下收到消息发布请求时可能崩溃的情况 [#9024](https://github.com/emqx/emqx/pull/9024)。