diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src
index bd596ffd4..bc2425a55 100644
--- a/apps/emqx_management/src/emqx_management.app.src
+++ b/apps/emqx_management/src/emqx_management.app.src
@@ -2,7 +2,7 @@
 {application, emqx_management, [
     {description, "EMQX Management API and CLI"},
     % strict semver, bump manually!
-    {vsn, "5.1.0"},
+    {vsn, "5.1.1"},
     {modules, []},
     {registered, [emqx_management_sup]},
     {applications, [
diff --git a/apps/emqx_management/src/emqx_mgmt_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_data_backup.erl
index d88a4d998..2aaa014a8 100644
--- a/apps/emqx_management/src/emqx_mgmt_data_backup.erl
+++ b/apps/emqx_management/src/emqx_mgmt_data_backup.erl
@@ -315,8 +315,10 @@ do_upload(BackupFileNameStr, BackupFileContent) ->
     catch
         error:{badmatch, {error, Reason}}:Stack ->
             ?SLOG(error, #{msg => "emqx_data_upload_failed", reason => Reason, stacktrace => Stack}),
+            _ = file:delete(FilePath),
             {error, Reason};
         Class:Reason:Stack ->
+            _ = file:delete(FilePath),
             ?SLOG(error, #{
                 msg => "emqx_data_upload_failed",
                 exception => Class,
diff --git a/apps/emqx_management/test/emqx_mgmt_api_data_backup_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_data_backup_SUITE.erl
index e94de971d..6a580fd57 100644
--- a/apps/emqx_management/test/emqx_mgmt_api_data_backup_SUITE.erl
+++ b/apps/emqx_management/test/emqx_mgmt_api_data_backup_SUITE.erl
@@ -199,7 +199,11 @@ upload_backup_test(Config, BackupName) ->
     ?assertEqual(ok, upload_backup(?NODE3_PORT, Auth, UploadFile)),
     %% This file was specially forged to pass upload validation bat fail on import
     ?assertEqual(ok, upload_backup(?NODE2_PORT, Auth, BadImportFile)),
-    ?assertEqual({error, bad_request}, upload_backup(?NODE1_PORT, Auth, BadUploadFile)).
+    ?assertEqual({error, bad_request}, upload_backup(?NODE1_PORT, Auth, BadUploadFile)),
+    %% Invalid file must not be kept
+    ?assertMatch(
+        {error, {_, 404, _}}, backup_file_op(get, ?NODE1_PORT, Auth, ?BAD_UPLOAD_BACKUP, [])
+    ).
 
 import_backup_test(Config, BackupName) ->
     Auth = ?config(auth, Config),
diff --git a/changes/ce/fix-12759.en.md b/changes/ce/fix-12759.en.md
new file mode 100644
index 000000000..2906bd17e
--- /dev/null
+++ b/changes/ce/fix-12759.en.md
@@ -0,0 +1 @@
+Do not save invalid uploaded backup files.