diff --git a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl
index e62923bc2..046e23300 100644
--- a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl
@@ -112,8 +112,13 @@ listeners(post, #{bindings := #{name := Name0}, body := LConf}) ->
 
 listeners_insta(delete, #{bindings := #{name := Name0, id := ListenerId}}) ->
     with_gateway(Name0, fun(_GwName, _) ->
-        ok = emqx_gateway_http:remove_listener(ListenerId),
-        {204}
+        case emqx_gateway_conf:listener(ListenerId) of
+            {ok, _Listener} ->
+                ok = emqx_gateway_http:remove_listener(ListenerId),
+                {204};
+            {error, not_found} ->
+                return_http_error(404, "Listener not found")
+        end
     end);
 listeners_insta(get, #{bindings := #{name := Name0, id := ListenerId}}) ->
     with_gateway(Name0, fun(_GwName, _) ->
diff --git a/apps/emqx_gateway/src/emqx_gateway_http.erl b/apps/emqx_gateway/src/emqx_gateway_http.erl
index 58c201c75..2186ac3d7 100644
--- a/apps/emqx_gateway/src/emqx_gateway_http.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_http.erl
@@ -550,7 +550,7 @@ with_gateway(GwName0, Fun) ->
             return_http_error(400, [K, " is required"]);
         %% Exceptions from emqx_gateway_utils:parse_listener_id/1
         error:{invalid_listener_id, Id} ->
-            return_http_error(400, ["Invalid listener id: ", Id]);
+            return_http_error(404, ["Listener not found: ", Id]);
         %% Exceptions from emqx:get_config/1
         error:{config_not_found, Path0} ->
             Path = lists:concat(
diff --git a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl
index f1cfd26d0..e486c8c16 100644
--- a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl
+++ b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl
@@ -409,6 +409,7 @@ t_listeners_tcp(_) ->
 
     {204, _} = request(delete, "/gateways/stomp/listeners/stomp:tcp:def"),
     {404, _} = request(get, "/gateways/stomp/listeners/stomp:tcp:def"),
+    {404, _} = request(delete, "/gateways/stomp/listeners/stomp:tcp:def"),
     ok.
 
 t_listeners_max_conns(_) ->
@@ -480,9 +481,19 @@ t_listeners_authn(_) ->
     {200, ConfResp3} = request(get, Path),
     assert_confs(AuthConf2, ConfResp3),
 
+    {404, _} = request(get, Path ++ "/users/not_exists"),
+    {404, _} = request(delete, Path ++ "/users/not_exists"),
+
     {204, _} = request(delete, Path),
     %% FIXME: 204?
     {204, _} = request(get, Path),
+
+    BadPath = "/gateways/stomp/listeners/stomp:tcp:not_exists/authentication/users/foo",
+    {404, _} = request(get, BadPath),
+    {404, _} = request(delete, BadPath),
+
+    {404, _} = request(get, "/gateways/stomp/listeners/not_exists"),
+    {404, _} = request(delete, "/gateways/stomp/listeners/not_exists"),
     ok.
 
 t_listeners_authn_data_mgmt(_) ->