diff --git a/apps/emqx_machine/src/emqx_machine_terminator.erl b/apps/emqx_machine/src/emqx_machine_terminator.erl
index 7120cc19b..77c53a64d 100644
--- a/apps/emqx_machine/src/emqx_machine_terminator.erl
+++ b/apps/emqx_machine/src/emqx_machine_terminator.erl
@@ -41,7 +41,7 @@
 -define(DO_IT, graceful_shutdown).
 
 %% @doc This API is called to shutdown the Erlang VM by RPC call from remote shell node.
-%% The shutown of apps is delegated to a to a process instead of doing it in the RPC spawned
+%% The shutdown of apps is delegated to a to a process instead of doing it in the RPC spawned
 %% process which has a remote group leader.
 start_link() ->
     {ok, _} = gen_server:start_link({local, ?TERMINATOR}, ?MODULE, [], []).
@@ -87,8 +87,9 @@ handle_cast(_Cast, State) ->
 
 handle_call(?DO_IT, _From, State) ->
     try
-        emqx_machine_boot:stop_apps(),
-        emqx_machine_boot:stop_port_apps()
+        %% stop port apps before stopping other apps.
+        emqx_machine_boot:stop_port_apps(),
+        emqx_machine_boot:stop_apps()
     catch
         C:E:St ->
             Apps = [element(1, A) || A <- application:which_applications()],
diff --git a/deploy/packages/emqx.service b/deploy/packages/emqx.service
index 2dbe550bc..ce342b632 100644
--- a/deploy/packages/emqx.service
+++ b/deploy/packages/emqx.service
@@ -22,8 +22,10 @@ LimitNOFILE=1048576
 
 # ExecStop is commented out so systemd will send a SIGTERM when 'systemctl stop'.
 # SIGTERM is handled by EMQX and it then performs a graceful shutdown
-# It's better than command 'emqx stop' because it needs to ping the node
-# ExecStop=/bin/bash /usr/bin/emqx stop
+# emqx stop will ping node, always return 0 to make sure next command will be executed
+ExecStop=/bin/bash -c '/usr/bin/emqx stop; exit 0'
+# If the process is still running, force kill it
+ExecStop=/bin/bash -c 'if [ ps -p $MAINPID >/dev/null 2>&1 ]; then kill -15 $MAINPID; fi'
 
 # Wait long enough before force kill for graceful shutdown
 TimeoutStopSec=120s