fix: list exported json files failed with utf8 filenames

apps/emqx_management/src/emqx_mgmt_api_data.erl

@@ -94,7 +94,7 @@ get_list_exported() ->
                         {ok, #file_info{size = Size, ctime = CTime = {{Y, M, D}, {H, MM, S}}}} ->
                             CreatedAt = io_lib:format("~p-~p-~p ~p:~p:~p", [Y, M, D, H, MM, S]),
                             Seconds = calendar:datetime_to_gregorian_seconds(CTime),
-                            [{Seconds, [{filename, list_to_binary(File)},
+                            [{Seconds, [{filename, unicode:characters_to_binary(File)},
                                         {size, Size},
                                         {created_at, list_to_binary(CreatedAt)},
                                         {node, node()}
@@ -135,7 +135,7 @@ download(#{filename := Filename}, _Params) ->
     FullFilename = fullname(Filename),
     case file:read_file(FullFilename) of
         {ok, Bin} ->
-            {ok, #{filename => list_to_binary(Filename),
+            {ok, #{filename => unicode:characters_to_binary(filename:basename(FullFilename)),
                    file => Bin}};
         {error, Reason} ->
             minirest:return({error, Reason})
@@ -147,6 +147,7 @@ upload(Bindings, Params) ->
 do_upload(_Bindings, #{<<"filename">> := Filename,
                        <<"file">> := Bin}) ->
     FullFilename = fullname(Filename),
+    ok = filelib:ensure_dir(FullFilename),
     case file:write_file(FullFilename, Bin) of
         ok ->
             minirest:return({ok, [{node, node()}]});

apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl

@@ -28,12 +28,16 @@ all() ->
     emqx_ct:all(?MODULE).
 
 init_per_suite(Cfg) ->
+    ekka_mnesia:start(),
+    ok = emqx_dashboard_admin:mnesia(boot),
     application:load(emqx_modules),
     application:load(emqx_bridge_mqtt),
     emqx_ct_helpers:start_apps([emqx_rule_engine, emqx_management]),
+    application:ensure_all_started(emqx_dashboard),
     Cfg.
 
 end_per_suite(Cfg) ->
+    application:stop(emqx_dashboard),
     emqx_ct_helpers:stop_apps([emqx_management, emqx_rule_engine]),
     Cfg.
 

apps/emqx_management/test/emqx_webhook_data_export_import_SUITE.erl

@@ -28,14 +28,18 @@ all() ->
     emqx_ct:all(?MODULE).
 
 init_per_suite(Cfg) ->
+    ekka_mnesia:start(),
+    ok = emqx_dashboard_admin:mnesia(boot),
     application:load(emqx_modules),
     application:load(emqx_web_hook),
     emqx_ct_helpers:start_apps([emqx_rule_engine, emqx_management]),
+    application:ensure_all_started(emqx_dashboard),
     ok = emqx_rule_registry:mnesia(boot),
     ok = emqx_rule_engine:load_providers(),
     Cfg.
 
 end_per_suite(Cfg) ->
+    application:stop(emqx_dashboard),
     emqx_ct_helpers:stop_apps([emqx_management, emqx_rule_engine]),
     Cfg.
 
@@ -46,8 +50,8 @@ remove_resource(Id) ->
     emqx_rule_registry:remove_resource(Id),
     emqx_rule_registry:remove_resource_params(Id).
 
-import(FilePath, Version) ->
+import_and_check(Filename, Version) ->
-    ok = emqx_mgmt_data_backup:import(get_data_path() ++ "/" ++ FilePath, <<"{}">>),
+    {ok, #{code := 0}} = emqx_mgmt_api_data:import(#{}, [{<<"filename">>, Filename}]),
     lists:foreach(fun(#resource{id = Id, config = Config} = _Resource) ->
         case Id of
             <<"webhook">> ->
@@ -58,34 +62,51 @@ import(FilePath, Version) ->
         end
     end, emqx_rule_registry:get_resources()).
 
+upload_import_export_list_download(NameVsnTable) ->
+    lists:foreach(fun({Filename0, Vsn}) ->
+            Filename = unicode:characters_to_binary(Filename0),
+            FullPath = filename:join([get_data_path(), Filename]),
+            ct:pal("testing upload_import_export_list_download for file: ~ts, version: ~p", [FullPath, Vsn]),
+            %% upload
+            {ok, FileCnt} = file:read_file(FullPath),
+            {ok, #{code := 0}} = emqx_mgmt_api_data:upload(#{},
+                [{<<"filename">>, Filename}, {<<"file">>, FileCnt}]),
+            %% import
+            ok = import_and_check(Filename, Vsn),
+            %% export
+            {ok, #{data := #{created_at := CAt, filename := FName, size := Size}}}
+                = emqx_mgmt_api_data:export(#{}, []),
+            ?assert(true, is_binary(CAt)),
+            ?assert(true, is_binary(FName)),
+            ?assert(true, is_integer(Size)),
+            %% list exported files
+            lists:foreach(fun({Seconds, Content}) ->
+                    ?assert(true, is_integer(Seconds)),
+                    ?assert(true, is_binary(proplists:get_value(filename, Content))),
+                    ?assert(true, is_binary(proplists:get_value(created_at, Content))),
+                    ?assert(true, is_integer(proplists:get_value(size, Content)))
+                end, emqx_mgmt_api_data:get_list_exported()),
+            %% download
+            ?assertMatch({ok, #{filename := FName}},
+                emqx_mgmt_api_data:download(#{filename => FName}, []))
+        end, NameVsnTable).
+
 %%--------------------------------------------------------------------
 %% Cases
 %%--------------------------------------------------------------------
 -ifdef(EMQX_ENTERPRISE).
 
-t_importee4010(_) ->
+t_upload_import_export_list_download(_) ->
-    import("ee4010.json", ee4010),
+    NameVsnTable = [
-    {ok, _} = emqx_mgmt_data_backup:export().
+        {"ee4010.json", ee4010},
-
+        {"ee410.json", ee410},
-t_importee410(_) ->
+        {"ee411.json", ee411},
-    import("ee410.json", ee410),
+        {"ee420.json", ee420},
-    {ok, _} = emqx_mgmt_data_backup:export().
+        {"ee425.json", ee425},
-
+        {"ee430.json", ee430},
-t_importee411(_) ->
+        {"ee430-中文.json", ee430}
-    import("ee411.json", ee411),
+    ],
-    {ok, _} = emqx_mgmt_data_backup:export().
+    upload_import_export_list_download(NameVsnTable).
-
-t_importee420(_) ->
-    import("ee420.json", ee420),
-    {ok, _} = emqx_mgmt_data_backup:export().
-
-t_importee425(_) ->
-    import("ee425.json", ee425),
-    {ok, _} = emqx_mgmt_data_backup:export().
-
-t_importee430(_) ->
-    import("ee430.json", ee430),
-    {ok, _} = emqx_mgmt_data_backup:export().
 
 %%--------------------------------------------------------------------
 %% handle_config
@@ -131,29 +152,17 @@ handle_config(_, _) -> ok.
 
 -ifndef(EMQX_ENTERPRISE).
 
-t_import422(_) ->
+t_upload_import_export_list_download(_) ->
-    import("422.json", 422),
+    NameVsnTable = [
-    {ok, _} = emqx_mgmt_data_backup:export().
+        {"422.json", 422},
-
+        {"423.json", 423},
-t_import423(_) ->
+        {"425.json", 425},
-    import("423.json", 423),
+        {"430.json", 430},
-    {ok, _} = emqx_mgmt_data_backup:export().
+        {"430-中文.json", 430},
-
+        {"409.json", 409},
-t_import425(_) ->
+        {"415.json", 415}
-    import("425.json", 425),
+    ],
-    {ok, _} = emqx_mgmt_data_backup:export().
+    upload_import_export_list_download(NameVsnTable).
-
-t_import430(_) ->
-    import("430.json", 430),
-    {ok, _} = emqx_mgmt_data_backup:export().
-
-t_import409(_) ->
-    import("409.json", 409),
-    {ok, _} = emqx_mgmt_data_backup:export().
-
-t_import415(_) ->
-    import("415.json", 415),
-    {ok, _} = emqx_mgmt_data_backup:export().
 
 %%--------------------------------------------------------------------
 %% handle_config

apps/emqx_management/test/emqx_webhook_data_export_import_SUITE_data/430-中文.json

@@ -0,0 +1,52 @@
+{
+    "version": "4.3",
+    "rules": [],
+    "resources": [
+        {
+            "id": "webhook",
+            "type": "web_hook",
+            "config": {
+                "cacertfile": {
+                    "filename": "",
+                    "file": ""
+                },
+                "certfile": {
+                    "filename": "",
+                    "file": ""
+                },
+                "keyfile": {
+                    "filename": "",
+                    "file": ""
+                },
+                "connect_timeout": "5s",
+                "pool_size": 8,
+                "request_timeout": "5s",
+                "url": "http://www.emqx.io",
+                "verify": false
+            },
+            "created_at": 1616581851001,
+            "description": "webhook"
+        }
+    ],
+    "blacklist": [],
+    "apps": [
+        {
+            "id": "admin",
+            "secret": "public",
+            "name": "Default",
+            "desc": "Application user",
+            "status": true,
+            "expired": "undefined"
+        }
+    ],
+    "users": [
+        {
+            "username": "admin",
+            "password": "q8v7hISIMz+iKn/ZuAaogvAxKbA=",
+            "tags": "administrator"
+        }
+    ],
+    "auth_mnesia": [],
+    "acl_mnesia": [],
+    "date": "2021-03-24 18:31:21"
+}

apps/emqx_management/test/emqx_webhook_data_export_import_SUITE_data/ee430-中文.json

@@ -0,0 +1,98 @@
+{
+    "version": "4.3",
+    "rules": [],
+    "resources": [
+        {
+            "id": "webhook",
+            "type": "web_hook",
+            "config": {
+                "cacertfile": {
+                    "filename": "",
+                    "file": ""
+                },
+                "certfile": {
+                    "filename": "",
+                    "file": ""
+                },
+                "connect_timeout": "5s",
+                "keyfile": {
+                    "filename": "",
+                    "file": ""
+                },
+                "pool_size": 8,
+                "request_timeout": "5s",
+                "url": "http://www.emqx.io",
+                "verify": false
+            },
+            "created_at": 1618304340172,
+            "description": "webhook"
+        }
+    ],
+    "blacklist": [],
+    "apps": [
+        {
+            "id": "admin",
+            "secret": "public",
+            "name": "Default",
+            "desc": "Application user",
+            "status": true,
+            "expired": "undefined"
+        }
+    ],
+    "users": [
+        {
+            "username": "admin",
+            "password": "qq8hg9pOkmYiHqzi3+bcUaK2CGA=",
+            "tags": "administrator"
+        }
+    ],
+    "auth_mnesia": [],
+    "acl_mnesia": [],
+    "modules": [
+        {
+            "id": "module:aabeddbf",
+            "type": "recon",
+            "config": {},
+            "enabled": true,
+            "created_at": 1618304311061,
+            "description": ""
+        },
+        {
+            "id": "module:cbe6d976",
+            "type": "internal_acl",
+            "config": {
+                "acl_rule_file": "etc/acl.conf"
+            },
+            "enabled": true,
+            "created_at": 1618304311061,
+            "description": ""
+        },
+        {
+            "id": "module:46375e06",
+            "type": "retainer",
+            "config": {
+                "storage_type": "ram",
+                "max_retained_messages": 0,
+                "max_payload_size": "1MB",
+                "expiry_interval": 0
+            },
+            "enabled": true,
+            "created_at": 1618304311061,
+            "description": ""
+        },
+        {
+            "id": "module:091eb7c3",
+            "type": "presence",
+            "config": {
+                "qos": 0
+            },
+            "enabled": true,
+            "created_at": 1618304311061,
+            "description": ""
+        }
+    ],
+    "schemas": [],
+    "configs": [],
+    "listeners_state": [],
+    "date": "2021-04-13 17:59:52"
+}

changes/v4.3.22-en.md

@@ -5,10 +5,12 @@
 
 ### Bug fixes
 
-- Improve the display of rule's 'Maximum Speed' counter to only reserve 2 decimal places. [#9185](https://github.com/emqx/emqx/pull/9185)
+- Fix that after uploading a backup file with an UTF8 filename, HTTP API `GET /data/export` fails with status code 500 [#9224](https://github.com/emqx/emqx/pull/9224).
+
+- Improve the display of rule's 'Maximum Speed' counter to only reserve 2 decimal places [#9185](https://github.com/emqx/emqx/pull/9185).
   This is to avoid displaying floats like `0.30000000000000004` on the dashboard.
 
-- Fix the issue that emqx prints too many error logs when connecting to mongodb but auth failed. [#9184](https://github.com/emqx/emqx/pull/9184)
+- Fix the issue that emqx prints too many error logs when connecting to mongodb but auth failed [#9184](https://github.com/emqx/emqx/pull/9184).
 
 - Fix that after receiving publish in `idle mode` the emqx-sn gateway may panic [#9024](https://github.com/emqx/emqx/pull/9024).
 

changes/v4.3.22-zh.md

@@ -5,10 +5,12 @@
 
 ### 修复
 
-- 改进规则的 "最大执行速度" 的计数，只保留小数点之后 2 位 [#9185](https://github.com/emqx/emqx/pull/9185)
+- 修复若上传的备份文件名中包含 UTF8 字符，`GET /data/export` HTTP 接口返回 500 错误 [#9224](https://github.com/emqx/emqx/pull/9224)。
+
+- 改进规则的 "最大执行速度" 的计数，只保留小数点之后 2 位 [#9185](https://github.com/emqx/emqx/pull/9185)。
   避免在 dashboard 上展示类似这样的浮点数：`0.30000000000000004`。
 
-- 修复在尝试连接 MongoDB 数据库过程中，如果认证失败会不停打印错误日志的问题。[#9184](https://github.com/emqx/emqx/pull/9184)
+- 修复在尝试连接 MongoDB 数据库过程中，如果认证失败会不停打印错误日志的问题 [#9184](https://github.com/emqx/emqx/pull/9184)。
 
 - 修复 emqx-sn 插件在“空闲”状态下收到消息发布请求时可能崩溃的情况 [#9024](https://github.com/emqx/emqx/pull/9024)。