From 193f2ab7bdb22af345eed63aa5d52eb5e1650253 Mon Sep 17 00:00:00 2001
From: Rory Z <rory-z@outlook.com>
Date: Thu, 8 Dec 2022 15:31:04 +0800
Subject: [PATCH 01/18] fix(helm): chart using incorrect secret values with
 custom credentials

---
 changes/v4.4.12-en.md                    |  2 ++
 changes/v4.4.12-zh.md                    |  2 ++
 deploy/charts/emqx/templates/secret.yaml | 12 ++----------
 3 files changed, 6 insertions(+), 10 deletions(-)

diff --git a/changes/v4.4.12-en.md b/changes/v4.4.12-en.md
index d2bc93ca9..f0e683f8a 100644
--- a/changes/v4.4.12-en.md
+++ b/changes/v4.4.12-en.md
@@ -11,3 +11,5 @@
 - Trigger `message.dropped` hook when QoS2 message is resend by client with a same packet id, or 'awaiting_rel' queue is full [#9486](https://github.com/emqx/emqx/pull/9486).
 
 - When dashboard creates a new user, the password length must match 3-32 and the format is `^[A-Za-z0-9]+[A-Za-z0-9-_]*$` [#9475](https://github.com/emqx/emqx-enterprise/pull/9475).
+
+- Fixed EMQX Helm Chart using incorrect secret values when custom credentials are provided [#9508](https://github.com/emqx/emqx/pull/9508).
diff --git a/changes/v4.4.12-zh.md b/changes/v4.4.12-zh.md
index aec0cbc4e..31637a90d 100644
--- a/changes/v4.4.12-zh.md
+++ b/changes/v4.4.12-zh.md
@@ -11,3 +11,5 @@
 - 当 QoS2 消息被重发(使用相同 Packet ID)，或当 'awaiting_rel' 队列已满时，触发消息丢弃钩子(`message.dropped`)及计数器 [#9486](https://github.com/emqx/emqx-enterprise/pull/9486)。
 
 - 当控制台创建新用户时，密码长度必须在 3-32 之间，且格式为 `^[A-Za-z0-9]+[A-Za-z0-9-_]*$` [#9475](https://github.com/emqx/emqx-enterprise/pull/9475)。
+
+- 修复了 EMQX Helm Chart 中当用户使用自定义的用户名和密码时，创建的 Secret 资源不正确问题 [#9508](https://github.com/emqx/emqx/pull/9508).
diff --git a/deploy/charts/emqx/templates/secret.yaml b/deploy/charts/emqx/templates/secret.yaml
index fcc1b2f3d..46a3b1cbb 100644
--- a/deploy/charts/emqx/templates/secret.yaml
+++ b/deploy/charts/emqx/templates/secret.yaml
@@ -6,14 +6,6 @@ metadata:
   namespace: {{ .Release.Namespace }}
 type: kubernetes.io/basic-auth
 stringData:
-  {{- if not (empty .Values.emqxConfig.EMQX_MANAGEMENT__DEFAULT_APPLICATION__ID) }}
-  username: admin
-  {{- else }}
-  username: {{ .Values.emqxConfig.EMQX_MANAGEMENT__DEFAULT_APPLICATION__ID }}
-  {{- end }}
-  {{- if not (empty .Values.emqxConfig.EMQX_MANAGEMENT__DEFAULT_APPLICATION__SECRET) }}
-  password: public
-  {{- else }}
-  password: {{ .Values.emqxConfig.EMQX_MANAGEMENT__DEFAULT_APPLICATION__SECRET}}
-  {{- end }}
+  username: {{ .Values.emqxConfig.EMQX_MANAGEMENT__DEFAULT_APPLICATION__ID | default "admin" }}
+  password: {{ .Values.emqxConfig.EMQX_MANAGEMENT__DEFAULT_APPLICATION__SECRET | default "public" }}
 {{- end }}
\ No newline at end of file

From fe26c6fa404c0366bed0d19bc2d1a998388508e8 Mon Sep 17 00:00:00 2001
From: Rory Z <rory-z@outlook.com>
Date: Thu, 8 Dec 2022 11:20:58 +0800
Subject: [PATCH 02/18] fix(helm): fix helm value explain error

Fixed EMQX Helm Chart can not set JSON type value for EMQX configuration items
---
 changes/v4.4.12-en.md                           |  2 ++
 changes/v4.4.12-zh.md                           |  4 +++-
 deploy/charts/emqx/templates/configmap.env.yaml | 11 +++++++----
 deploy/charts/emqx/values.yaml                  |  2 +-
 4 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/changes/v4.4.12-en.md b/changes/v4.4.12-en.md
index f0e683f8a..f344de986 100644
--- a/changes/v4.4.12-en.md
+++ b/changes/v4.4.12-en.md
@@ -13,3 +13,5 @@
 - When dashboard creates a new user, the password length must match 3-32 and the format is `^[A-Za-z0-9]+[A-Za-z0-9-_]*$` [#9475](https://github.com/emqx/emqx-enterprise/pull/9475).
 
 - Fixed EMQX Helm Chart using incorrect secret values when custom credentials are provided [#9508](https://github.com/emqx/emqx/pull/9508).
+
+- Fixed EMQX Helm Chart can not set JSON type value for EMQX Broker configuration items [#9504](https://github.com/emqx/emqx/pull/9504).
diff --git a/changes/v4.4.12-zh.md b/changes/v4.4.12-zh.md
index 31637a90d..d46b846e1 100644
--- a/changes/v4.4.12-zh.md
+++ b/changes/v4.4.12-zh.md
@@ -12,4 +12,6 @@
 
 - 当控制台创建新用户时，密码长度必须在 3-32 之间，且格式为 `^[A-Za-z0-9]+[A-Za-z0-9-_]*$` [#9475](https://github.com/emqx/emqx-enterprise/pull/9475)。
 
-- 修复了 EMQX Helm Chart 中当用户使用自定义的用户名和密码时，创建的 Secret 资源不正确问题 [#9508](https://github.com/emqx/emqx/pull/9508).
+- 修复了 EMQX Helm Chart 中当用户使用自定义的用户名和密码时，创建的 Secret 资源不正确问题 [#9508](https://github.com/emqx/emqx/pull/9508)。
+
+- 修复 EMQX Helm Chart 无法配置 value 为 JSON 类型的 EMQX Broker 配置项 [#9504](https://github.com/emqx/emqx/pull/9504)。
diff --git a/deploy/charts/emqx/templates/configmap.env.yaml b/deploy/charts/emqx/templates/configmap.env.yaml
index c1abb4fdb..652e65e73 100644
--- a/deploy/charts/emqx/templates/configmap.env.yaml
+++ b/deploy/charts/emqx/templates/configmap.env.yaml
@@ -17,17 +17,20 @@ data:
   EMQX_CLUSTER__K8S__SERVICE_NAME:  {{ include "emqx.fullname" . }}-headless
   EMQX_CLUSTER__K8S__NAMESPACE: {{ .Release.Namespace }}
   EMQX_CLUSTER__K8S__ADDRESS_TYPE: "hostname"
-  EMQX_CLUSTER__K8S__SUFFIX: "svc.cluster.local"  
+  EMQX_CLUSTER__K8S__SUFFIX: "svc.cluster.local"
   {{- else if eq (.Values.emqxConfig.EMQX_CLUSTER__DISCOVERY)  "dns"  }}
-  EMQX_CLUSTER__DNS__NAME: "{{ .Release.Name }}-headless.{{ .Release.Namespace }}.svc.cluster.local"
   EMQX_CLUSTER__DNS__APP: {{ .Release.Name }}
+  EMQX_CLUSTER__DNS__NAME: "{{ include "emqx.fullname" . }}-headless.{{ .Release.Namespace }}.svc.cluster.local"
   EMQX_CLUSTER__DNS__TYPE: "srv"
   {{- end -}}
   {{- range $index, $value := .Values.emqxConfig }}
   {{- if $value }}
   {{- $key := (regexReplaceAllLiteral "\\." (regexReplaceAllLiteral "EMQX[_\\.]" (upper (trimAll " " $index)) "") "__") }}
+  {{- if or (kindIs "map" $value) (kindIs "slice" $value) }}
+  {{ print "EMQX_" $key }}: {{ tpl (printf "%q" (toJson $value)) $ }}
+  {{- else }}
   {{ print "EMQX_" $key }}: "{{ tpl (printf "%v" $value) $ }}"
   {{- end }}
-  {{- end}}
-
+  {{- end }}
+  {{- end }}
 {{- end }}
diff --git a/deploy/charts/emqx/values.yaml b/deploy/charts/emqx/values.yaml
index e69d42d97..eb8bcc97a 100644
--- a/deploy/charts/emqx/values.yaml
+++ b/deploy/charts/emqx/values.yaml
@@ -72,7 +72,7 @@ initContainers: {}
 
 ## EMQX configuration item, see the documentation (https://hub.docker.com/r/emqx/emqx)
 emqxConfig:
-  ## Cluster discovery by k8s
+  ## Cluster discovery by 'k8s' or 'dns'
   EMQX_CLUSTER__DISCOVERY: "k8s"
   EMQX_MANAGEMENT__DEFAULT_APPLICATION__ID: "admin"
   EMQX_MANAGEMENT__DEFAULT_APPLICATION__SECRET: "public"

From af789b26d31c2442992658eb034cc02499e5ef62 Mon Sep 17 00:00:00 2001
From: Rory Z <rory-z@outlook.com>
Date: Mon, 12 Dec 2022 15:45:51 +0800
Subject: [PATCH 03/18] chore: helm chart support externalTrafficPolicy

Fix https://github.com/emqx/emqx/issues/9515
---
 changes/v4.4.12-en.md                     | 2 ++
 changes/v4.4.12-zh.md                     | 2 ++
 deploy/charts/emqx/README.md              | 1 +
 deploy/charts/emqx/templates/service.yaml | 3 +++
 deploy/charts/emqx/values.yaml            | 6 ++++++
 5 files changed, 14 insertions(+)

diff --git a/changes/v4.4.12-en.md b/changes/v4.4.12-en.md
index f344de986..e767d929c 100644
--- a/changes/v4.4.12-en.md
+++ b/changes/v4.4.12-en.md
@@ -4,6 +4,8 @@
 
 - Add more PSK ciphers support [#9493](https://github.com/emqx/emqx/pull/9493).
 
+- Users can define the `externalTrafficPolicy` of service in EMQX Helm Chart [#9527](https://github.com/emqx/emqx/pull/9527).
+
 ### Bug Fixes
 
 - Fixed load bootstrap file when no bootstrap user in `mqtt_app` [#9474](https://github.com/emqx/emqx/pull/9474).
diff --git a/changes/v4.4.12-zh.md b/changes/v4.4.12-zh.md
index d46b846e1..9c532c6b6 100644
--- a/changes/v4.4.12-zh.md
+++ b/changes/v4.4.12-zh.md
@@ -4,6 +4,8 @@
 
 - 支持更多的 PSK 密码套件[#9493](https://github.com/emqx/emqx/pull/9493)。
 
+- 用户可以在 EMQX Helm Chart 中自定义 service 资源的 `externalTrafficPolicy` [#9527](https://github.com/emqx/emqx/pull/9527)。
+
 ### 修复
 
 - 修复 mqtt_app 表内没有 boostrap user 里未导入用户的问题 [#9474](https://github.com/emqx/emqx/pull/9474).
diff --git a/deploy/charts/emqx/README.md b/deploy/charts/emqx/README.md
index 2c00d86ec..2882f9e0d 100644
--- a/deploy/charts/emqx/README.md
+++ b/deploy/charts/emqx/README.md
@@ -68,6 +68,7 @@ Parameter  | Description | Default Value
 `service.loadBalancerIP`  | loadBalancerIP for Service |	`nil`
 `service.loadBalancerSourceRanges` |	Address(es) that are allowed when service is LoadBalancer |	`[]`
 `service.externalIPs` |	ExternalIPs for the service |	`[]`
+`service.externalTrafficPolicy` |	External Traffic Policy for the service |	`Cluster`
 `service.annotations` |	Service annotations (evaluated as a template) |	`{}`
 `ingress.dashboard.enabled` |	Enable ingress for EMQX Dashboard |	false
 `ingress.dashboard.ingressClassName` |	Set the ingress class for EMQX Dashboard
diff --git a/deploy/charts/emqx/templates/service.yaml b/deploy/charts/emqx/templates/service.yaml
index 8a1a6a212..0f374b334 100644
--- a/deploy/charts/emqx/templates/service.yaml
+++ b/deploy/charts/emqx/templates/service.yaml
@@ -14,6 +14,9 @@ metadata:
   {{- end }}
 spec:
   type: {{ .Values.service.type }}
+  {{- if or (eq .Values.service.type "LoadBalancer") (eq .Values.service.type "NodePort") }}
+  externalTrafficPolicy: {{ .Values.service.externalTrafficPolicy | default "Cluster" }}
+  {{- end }}
   {{- if eq .Values.service.type "LoadBalancer" }}
   {{- if .Values.service.loadBalancerIP }}
   loadBalancerIP: {{ .Values.service.loadBalancerIP }}
diff --git a/deploy/charts/emqx/values.yaml b/deploy/charts/emqx/values.yaml
index eb8bcc97a..1df94e42d 100644
--- a/deploy/charts/emqx/values.yaml
+++ b/deploy/charts/emqx/values.yaml
@@ -174,6 +174,12 @@ service:
   ## Set the ExternalIPs
   ##
   externalIPs: []
+  ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints.
+  ## There are two available options: Cluster (default) and Local.
+  ## Cluster obscures the client source IP and may cause a second hop to another node, but should have good overall load-spreading.
+  ## Local preserves the client source IP and avoids a second hop for LoadBalancer and NodePort type Services, but risks potentially imbalanced traffic spreading.
+  ##
+  externalTrafficPolicy: "Cluster"
   ## Provide any additional annotations which may be required. Evaluated as a template
   ##
   annotations: {}

From 781c8949b3917bac9997575cab6a786dc6cea750 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Wed, 14 Dec 2022 13:52:24 +0800
Subject: [PATCH 04/18] fix: tmp probing resources are not removed after
 testing resources

---
 .../src/emqx_rule_engine_api.erl              | 107 ++++++++++++++++--
 .../src/emqx_rule_engine_sup.erl              |  33 +++++-
 2 files changed, 132 insertions(+), 8 deletions(-)

diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl
index e53d80f02..49d89cc46 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl
@@ -16,6 +16,8 @@
 
 -module(emqx_rule_engine_api).
 
+-behaviour(gen_server).
+
 -include("rule_engine.hrl").
 -include_lib("emqx/include/logger.hrl").
 
@@ -156,6 +158,17 @@
             descr  => "List all events with detailed info"
            }).
 
+-export([start_link/0]).
+
+% gen_server Callbacks
+-export([ init/1
+        , handle_call/3
+        , handle_cast/2
+        , handle_info/2
+        , terminate/2
+        , code_change/3
+        ]).
+
 -export([ create_rule/2
         , update_rule/2
         , list_rules/2
@@ -207,10 +220,90 @@
             <<"Bad Arguments: ", R0/binary>>
         end).
 
+-define(T_CALL, 30000).
+
+start_link() ->
+    %% The caller process (the cowboy process serves the HTTP request) may times out and dies
+    %% before some time-consuming operations complete, e.g. creating rules/resources or testing
+    %% the connectivity on unreachable resources.
+    %% To avoid this problem, we delegate the operations to a gen_server.
+    gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+create_rule(_Bindings, Params) ->
+    delegate_call({create_rule, _Bindings, Params}).
+
+update_rule(_Bindings, Params) ->
+    delegate_call({update_rule, _Bindings, Params}).
+
+delete_rule(_Bindings, Params) ->
+    delegate_call({delete_rule, _Bindings, Params}).
+
+create_resource(_Bindings, Params) ->
+    delegate_call({create_resource, _Bindings, Params}).
+
+update_resource(_Bindings, Params) ->
+    delegate_call({update_resource, _Bindings, Params}).
+
+start_resource(_Bindings, Params) ->
+    delegate_call({start_resource, _Bindings, Params}).
+
+delete_resource(_Bindings, Params) ->
+    delegate_call({delete_resource, _Bindings, Params}).
+
+%% delegate API calls to a single process.
+delegate_call(Req) ->
+    gen_server:call(?MODULE, Req, ?T_CALL).
+
+%%------------------------------------------------------------------------------
+%% gen_server callbacks
+%%------------------------------------------------------------------------------
+
+init([]) ->
+    {ok, #{}}.
+
+handle_call({create_rule, _Bindings, Params}, _From, State) ->
+    {reply, delegate_create_rule(_Bindings, Params), State};
+
+handle_call({update_rule, _Bindings, Params}, _From, State) ->
+    {reply, delegate_update_rule(_Bindings, Params), State};
+
+handle_call({delete_rule, _Bindings, Params}, _From, State) ->
+    {reply, delegate_delete_rule(_Bindings, Params), State};
+
+handle_call({create_resource, _Bindings, Params}, _From, State) ->
+    {reply, delegate_create_resource(_Bindings, Params), State};
+
+handle_call({start_resource, _Bindings, Params}, _From, State) ->
+    {reply, delegate_start_resource(_Bindings, Params), State};
+
+handle_call({update_resource, _Bindings, Params}, _From, State) ->
+    {reply, delegate_update_resource(_Bindings, Params), State};
+
+handle_call({delete_resource, _Bindings, Params}, _From, State) ->
+    {reply, delegate_delete_resource(_Bindings, Params), State};
+
+handle_call(Req, _From, State) ->
+    ?LOG(error, "unexpected call: ~p", [Req]),
+    {reply, ignored, State}.
+
+handle_cast(Msg, State) ->
+    ?LOG(error, "unexpected cast: ~p", [Msg]),
+    {noreply, State}.
+
+handle_info(Info, State) ->
+    ?LOG(error, "unexpected info: ~p", [Info]),
+    {noreply, State}.
+
+terminate(_Reason, _State) ->
+    ok.
+
+code_change(_OldVsn, State, _Extra) ->
+    {ok, State}.
+
 %%------------------------------------------------------------------------------
 %% Rules API
 %%------------------------------------------------------------------------------
-create_rule(_Bindings, Params) ->
+delegate_create_rule(_Bindings, Params) ->
     if_test(fun() -> test_rule_sql(Params) end,
             fun() -> do_create_rule(Params) end,
             Params).
@@ -250,7 +343,7 @@ do_create_rule2(ParsedParams) ->
             return({error, 400, ?ERR_BADARGS(Reason)})
     end.
 
-update_rule(#{id := Id0}, Params) ->
+delegate_update_rule(#{id := Id0}, Params) ->
     Id = urldecode(Id0),
     case parse_rule_params(Params, #{id => Id}) of
         {ok, ParsedParams} ->
@@ -280,7 +373,7 @@ show_rule(#{id := Id0}, _Params) ->
     Id = urldecode(Id0),
     reply_with(fun emqx_rule_registry:get_rule/1, Id).
 
-delete_rule(#{id := Id0}, _Params) ->
+delegate_delete_rule(#{id := Id0}, _Params) ->
     Id = urldecode(Id0),
     ok = emqx_rule_engine:delete_rule(Id),
     return(ok).
@@ -309,7 +402,7 @@ show_action(#{name := Name}, _Params) ->
 %%------------------------------------------------------------------------------
 %% Resources API
 %%------------------------------------------------------------------------------
-create_resource(#{}, Params) ->
+delegate_create_resource(#{}, Params) ->
     case parse_resource_params(Params) of
         {ok, ParsedParams} ->
             if_test(fun() -> do_create_resource(test_resource, maps:without([id], ParsedParams)) end,
@@ -382,7 +475,7 @@ get_resource_status(#{id := Id0}, _Params) ->
             return({error, 400, ?ERR_NO_RESOURCE(Id)})
     end.
 
-start_resource(#{id := Id0}, _Params) ->
+delegate_start_resource(#{id := Id0}, _Params) ->
     Id = urldecode(Id0),
     case emqx_rule_engine:start_resource(Id) of
         ok ->
@@ -394,7 +487,7 @@ start_resource(#{id := Id0}, _Params) ->
             return({error, 400, ?ERR_BADARGS(Reason)})
     end.
 
-update_resource(#{id := Id0}, NewParams) ->
+delegate_update_resource(#{id := Id0}, NewParams) ->
     Id = urldecode(Id0),
     P1 = case proplists:get_value(<<"description">>, NewParams) of
         undefined -> #{};
@@ -419,7 +512,7 @@ update_resource(#{id := Id0}, NewParams) ->
             return({error, 400, ?ERR_BADARGS(Reason)})
     end.
 
-delete_resource(#{id := Id0}, _Params) ->
+delegate_delete_resource(#{id := Id0}, _Params) ->
     Id = urldecode(Id0),
     case emqx_rule_engine:delete_resource(Id) of
         ok -> return(ok);
diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_sup.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_sup.erl
index 087dbcbfb..92205abb6 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_engine_sup.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine_sup.erl
@@ -24,6 +24,8 @@
 
 -export([ start_locker/0
         , start_jwt_sup/0
+        , ensure_api_delegator_started/0
+        , ensure_api_delegator_stopped/0
         ]).
 
 -export([init/1]).
@@ -58,7 +60,8 @@ init([]) ->
                 type => worker,
                 modules => [emqx_rule_monitor]},
     JWTSup = jwt_sup_child_spec(),
-    {ok, {SupFlags, [Registry, Metrics, Monitor, JWTSup]}}.
+    API = api_delegator_sup_spec(),
+    {ok, {SupFlags, [Registry, Metrics, Monitor, JWTSup, API]}}.
 
 start_locker() ->
     Locker = #{id => emqx_rule_locker,
@@ -97,3 +100,31 @@ ensure_table(Name, Opts) ->
         error:badarg ->
             ok
     end.
+
+%% This is called by the emqx_rule_engine.appup.src when release upgrade
+ensure_api_delegator_started() ->
+    case supervisor:start_child(?MODULE, api_delegator_sup_spec()) of
+        {ok, _} -> ok;
+        {error, already_present} -> ok;
+        {error, {already_started, _Pid}} -> ok;
+        {error, _} = Err -> throw({failed_to_start_ensure_api, Err})
+    end.
+
+%% This is called by the emqx_rule_engine.appup.src when release downgrade
+ensure_api_delegator_stopped() ->
+    case supervisor:terminate_child(?MODULE, emqx_rule_engine_api) of
+        ok ->
+            %% don't crash if delete failed
+            supervisor:delete_child(?MODULE, emqx_rule_engine_api);
+        {error, not_found} -> ok
+    end.
+
+api_delegator_sup_spec() ->
+    #{
+        id => emqx_rule_engine_api,
+        start => {emqx_rule_engine_api, start_link, []},
+        restart => permanent,
+        shutdown => 5000,
+        type => worker,
+        modules => [emqx_rule_engine_api]
+    }.

From 7c420b39de8f2cb2ce287dad3550796006670b2a Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Wed, 14 Dec 2022 13:52:34 +0800
Subject: [PATCH 05/18] chore: update emqx_rule_engine.appup.src

---
 .../src/emqx_rule_engine.appup.src            | 52 ++++++++++++++-----
 1 file changed, 39 insertions(+), 13 deletions(-)

diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src
index b74843a2f..4b392eb01 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src
@@ -3,6 +3,9 @@
 {VSN,
   [{"4.4.11",[
     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
+    {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]},
+    {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
+    {apply,{emqx_rule_engine_sup,ensure_api_delegator_started,[]}},
     {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
    ]},
    {"4.4.10",
@@ -11,6 +14,8 @@
      {add_module,emqx_rule_engine_jwt_sup},
      {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
+     {apply,{emqx_rule_engine_sup,ensure_api_delegator_started,[]}},
      {apply,{emqx_rule_engine_sup,start_jwt_sup,[]}},
      {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]},
@@ -19,7 +24,6 @@
      {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]},
    {"4.4.9",
     [{add_module,emqx_rule_engine_jwt},
@@ -27,6 +31,8 @@
      {add_module,emqx_rule_engine_jwt_sup},
      {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
+     {apply,{emqx_rule_engine_sup,ensure_api_delegator_started,[]}},
      {apply,{emqx_rule_engine_sup,start_jwt_sup,[]}},
      {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]},
@@ -35,7 +41,6 @@
      {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]},
    {"4.4.8",
@@ -44,6 +49,8 @@
      {add_module,emqx_rule_engine_jwt_sup},
      {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
+     {apply,{emqx_rule_engine_sup,ensure_api_delegator_started,[]}},
      {apply,{emqx_rule_engine_sup,start_jwt_sup,[]}},
      {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]},
@@ -51,7 +58,6 @@
      {load_module,emqx_rule_engine_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]},
@@ -61,6 +67,8 @@
      {add_module,emqx_rule_engine_jwt_worker},
      {add_module,emqx_rule_engine_jwt_sup},
      {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
+     {apply,{emqx_rule_engine_sup,ensure_api_delegator_started,[]}},
      {apply,{emqx_rule_engine_sup,start_jwt_sup,[]}},
      {load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_events,brutal_purge,soft_purge,[]},
@@ -68,7 +76,6 @@
      {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
@@ -79,6 +86,8 @@
      {add_module,emqx_rule_engine_jwt_worker},
      {add_module,emqx_rule_engine_jwt_sup},
      {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
+     {apply,{emqx_rule_engine_sup,ensure_api_delegator_started,[]}},
      {apply,{emqx_rule_engine_sup,start_jwt_sup,[]}},
      {load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_events,brutal_purge,soft_purge,[]},
@@ -91,13 +100,14 @@
      {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
+     {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}]},
    {"4.4.4",
     [{add_module,emqx_rule_engine_jwt},
      {add_module,emqx_rule_engine_jwt_worker},
      {add_module,emqx_rule_engine_jwt_sup},
      {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
+     {apply,{emqx_rule_engine_sup,ensure_api_delegator_started,[]}},
      {apply,{emqx_rule_engine_sup,start_jwt_sup,[]}},
      {load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_events,brutal_purge,soft_purge,[]},
@@ -105,7 +115,6 @@
      {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
@@ -117,6 +126,8 @@
      {add_module,emqx_rule_engine_jwt_worker},
      {add_module,emqx_rule_engine_jwt_sup},
      {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
+     {apply,{emqx_rule_engine_sup,ensure_api_delegator_started,[]}},
      {apply,{emqx_rule_engine_sup,start_jwt_sup,[]}},
      {load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_app,brutal_purge,soft_purge,[]},
@@ -131,13 +142,14 @@
      {add_module,emqx_rule_date},
      {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
+     {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]},
    {"4.4.2",
     [{add_module,emqx_rule_engine_jwt},
      {add_module,emqx_rule_engine_jwt_worker},
      {add_module,emqx_rule_engine_jwt_sup},
      {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
+     {apply,{emqx_rule_engine_sup,ensure_api_delegator_started,[]}},
      {apply,{emqx_rule_engine_sup,start_jwt_sup,[]}},
      {load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_app,brutal_purge,soft_purge,[]},
@@ -153,13 +165,14 @@
      {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
      {add_module,emqx_rule_date},
      {load_module,emqx_rule_events,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]},
    {"4.4.1",
     [{add_module,emqx_rule_engine_jwt},
      {add_module,emqx_rule_engine_jwt_worker},
      {add_module,emqx_rule_engine_jwt_sup},
      {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
+     {apply,{emqx_rule_engine_sup,ensure_api_delegator_started,[]}},
      {apply,{emqx_rule_engine_sup,start_jwt_sup,[]}},
      {load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_app,brutal_purge,soft_purge,[]},
@@ -171,7 +184,6 @@
      {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_events,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
@@ -182,6 +194,8 @@
      {add_module,emqx_rule_engine_jwt_worker},
      {add_module,emqx_rule_engine_jwt_sup},
      {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
+     {apply,{emqx_rule_engine_sup,ensure_api_delegator_started,[]}},
      {apply,{emqx_rule_engine_sup,start_jwt_sup,[]}},
      {load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_app,brutal_purge,soft_purge,[]},
@@ -197,15 +211,18 @@
      {update,emqx_rule_metrics,{advanced,["4.4.0"]}},
      {load_module,emqx_rule_events,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
-     {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
+     {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
   [{"4.4.11",[
     {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
+    {apply,{emqx_rule_engine_sup,ensure_api_delegator_stopped,[]}},
+    {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]},
+    {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
     {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}
    ]},
    {"4.4.10",
     [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
+     {apply,{emqx_rule_engine_sup,ensure_api_delegator_stopped,[]}},
      {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_events,brutal_purge,soft_purge,[]},
@@ -223,6 +240,7 @@
      {delete_module,emqx_rule_engine_jwt}]},
    {"4.4.9",
     [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
+     {apply,{emqx_rule_engine_sup,ensure_api_delegator_stopped,[]}},
      {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_events,brutal_purge,soft_purge,[]},
@@ -241,6 +259,7 @@
      {delete_module,emqx_rule_engine_jwt}]},
    {"4.4.8",
     [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]},
+     {apply,{emqx_rule_engine_sup,ensure_api_delegator_stopped,[]}},
      {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_events,brutal_purge,soft_purge,[]},
@@ -260,6 +279,7 @@
      {delete_module,emqx_rule_engine_jwt}]},
    {<<"4\\.4\\.[6-7]">>,
     [{load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]},
+     {apply,{emqx_rule_engine_sup,ensure_api_delegator_stopped,[]}},
      {load_module,emqx_rule_events,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]},
@@ -279,6 +299,7 @@
      {delete_module,emqx_rule_engine_jwt}]},
    {"4.4.5",
     [{load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]},
+     {apply,{emqx_rule_engine_sup,ensure_api_delegator_stopped,[]}},
      {load_module,emqx_rule_events,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]},
@@ -299,6 +320,7 @@
      {delete_module,emqx_rule_engine_jwt}]},
    {"4.4.4",
     [{load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]},
+     {apply,{emqx_rule_engine_sup,ensure_api_delegator_stopped,[]}},
      {load_module,emqx_rule_events,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]},
@@ -319,6 +341,7 @@
      {delete_module,emqx_rule_engine_jwt}]},
    {"4.4.3",
     [{load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]},
+     {apply,{emqx_rule_engine_sup,ensure_api_delegator_stopped,[]}},
      {load_module,emqx_rule_engine_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]},
@@ -341,6 +364,7 @@
      {delete_module,emqx_rule_engine_jwt}]},
    {"4.4.2",
     [{load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]},
+     {apply,{emqx_rule_engine_sup,ensure_api_delegator_stopped,[]}},
      {load_module,emqx_rule_engine_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
@@ -364,6 +388,7 @@
      {delete_module,emqx_rule_engine_jwt}]},
    {"4.4.1",
     [{load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]},
+     {apply,{emqx_rule_engine_sup,ensure_api_delegator_stopped,[]}},
      {load_module,emqx_rule_engine_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
@@ -387,6 +412,7 @@
      {delete_module,emqx_rule_engine_jwt}]},
    {"4.4.0",
     [{load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]},
+     {apply,{emqx_rule_engine_sup,ensure_api_delegator_stopped,[]}},
      {load_module,emqx_rule_engine_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_engine_sup,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},

From 920a9fd45573e1eb0cdad5e964ce9ccd4d822275 Mon Sep 17 00:00:00 2001
From: Shawn <506895667@qq.com>
Date: Wed, 14 Dec 2022 13:53:07 +0800
Subject: [PATCH 06/18] chore: update the change logs

---
 changes/v4.4.12-en.md | 2 ++
 changes/v4.4.12-zh.md | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/changes/v4.4.12-en.md b/changes/v4.4.12-en.md
index e767d929c..0cba88202 100644
--- a/changes/v4.4.12-en.md
+++ b/changes/v4.4.12-en.md
@@ -17,3 +17,5 @@
 - Fixed EMQX Helm Chart using incorrect secret values when custom credentials are provided [#9508](https://github.com/emqx/emqx/pull/9508).
 
 - Fixed EMQX Helm Chart can not set JSON type value for EMQX Broker configuration items [#9504](https://github.com/emqx/emqx/pull/9504).
+
+- When resource creation is too slow, there may be some temporary probing connections left [#9539](https://github.com/emqx/emqx/pull/9539).
diff --git a/changes/v4.4.12-zh.md b/changes/v4.4.12-zh.md
index 9c532c6b6..fcef10e48 100644
--- a/changes/v4.4.12-zh.md
+++ b/changes/v4.4.12-zh.md
@@ -17,3 +17,5 @@
 - 修复了 EMQX Helm Chart 中当用户使用自定义的用户名和密码时，创建的 Secret 资源不正确问题 [#9508](https://github.com/emqx/emqx/pull/9508)。
 
 - 修复 EMQX Helm Chart 无法配置 value 为 JSON 类型的 EMQX Broker 配置项 [#9504](https://github.com/emqx/emqx/pull/9504)。
+
+- 当创建资源过慢的情况下，有可能会残留一些用来探活的临时的连接 [#9539](https://github.com/emqx/emqx/pull/9539)。

From 97e4929abba10913cc9e1773d0720fb472e5a7e1 Mon Sep 17 00:00:00 2001
From: Ivan Dyachkov <dev@dyachkov.org>
Date: Wed, 7 Dec 2022 14:55:09 +0100
Subject: [PATCH 07/18] ci: build amzn2 packages

also upgrade emqx-builder to 4.4-23
---
 .github/workflows/apps_version_check.yaml     |  2 +-
 .github/workflows/build_packages.yaml         | 42 +++++++------------
 .github/workflows/build_slim_packages.yaml    |  4 +-
 .github/workflows/check_deps_integrity.yaml   |  2 +-
 .github/workflows/release.yaml                |  2 +-
 .../workflows/run_acl_migration_tests.yaml    |  2 +-
 .github/workflows/run_fvt_tests.yaml          |  6 +--
 .github/workflows/run_test_cases.yaml         |  8 ++--
 8 files changed, 27 insertions(+), 41 deletions(-)

diff --git a/.github/workflows/apps_version_check.yaml b/.github/workflows/apps_version_check.yaml
index 7875d3001..b2f2b6618 100644
--- a/.github/workflows/apps_version_check.yaml
+++ b/.github/workflows/apps_version_check.yaml
@@ -13,7 +13,7 @@ jobs:
         os:
         - ubuntu20.04
 
-    container: ghcr.io/emqx/emqx-builder/4.4-20:${{ matrix.erl_otp }}-${{ matrix.os }}
+    container: ghcr.io/emqx/emqx-builder/4.4-23:${{ matrix.erl_otp }}-${{ matrix.os }}
 
     steps:
       - uses: actions/checkout@v2
diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml
index 8b19bad44..55f6dac87 100644
--- a/.github/workflows/build_packages.yaml
+++ b/.github/workflows/build_packages.yaml
@@ -162,32 +162,18 @@ jobs:
           - debian9
           - el8
           - el7
-          # - raspbian10 #armv6l is too slow to emulate
-        exclude:
-        - os: raspbian9
-          arch: amd64
-        - os: raspbian10
-          arch: amd64
-        - os: raspbian9
-          profile: emqx
-        - os: raspbian10
-          profile: emqx
-        - os: raspbian9
-          profile: emqx-ee
-        - os: raspbian10
-          profile: emqx-ee
-
+          - amzn2
     defaults:
       run:
         shell: bash
 
     steps:
-    - uses: docker/setup-buildx-action@v1
-    - uses: docker/setup-qemu-action@v1
+    - uses: docker/setup-qemu-action@v2
       with:
         image: tonistiigi/binfmt:latest
         platforms: all
-    - uses: actions/download-artifact@v2
+    - uses: docker/setup-buildx-action@v2
+    - uses: actions/download-artifact@v3
       with:
         name: source
         path: .
@@ -206,8 +192,8 @@ jobs:
           --profile "${PROFILE}" \
           --pkgtype "${PACKAGE}" \
           --arch "${ARCH}" \
-          --builder "ghcr.io/emqx/emqx-builder/4.4-20:${OTP}-${SYSTEM}"
-    - uses: actions/upload-artifact@v1
+          --builder "ghcr.io/emqx/emqx-builder/4.4-23:${OTP}-${SYSTEM}"
+    - uses: actions/upload-artifact@v3
       with:
         name: ${{ matrix.profile }}
         path: source/_packages/${{ matrix.profile }}/
@@ -233,17 +219,17 @@ jobs:
             registry: 'public.ecr.aws'
 
     steps:
-    - uses: actions/download-artifact@v2
+    - uses: actions/download-artifact@v3
       with:
         name: source
         path: .
     - name: unzip source code
       run: unzip -q source.zip
-    - uses: docker/setup-buildx-action@v1
-    - uses: docker/setup-qemu-action@v1
+    - uses: docker/setup-qemu-action@v3
       with:
         image: tonistiigi/binfmt:latest
         platforms: all
+    - uses: docker/setup-buildx-action@v2
     - uses: aws-actions/configure-aws-credentials@v1
       if: matrix.registry == 'public.ecr.aws'
       with:
@@ -253,7 +239,7 @@ jobs:
     - name: Docker login to aws ecr
       if: matrix.registry == 'public.ecr.aws'
       run: aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws
-    - uses: docker/login-action@v1
+    - uses: docker/login-action@v2
       if: matrix.registry == 'docker.io'
       with:
         username: ${{ secrets.DOCKER_HUB_USER }}
@@ -271,7 +257,7 @@ jobs:
           type=match,pattern=[v|e](.*),group=1
         labels:
           org.opencontainers.image.otp.version=${{ matrix.otp }}
-    - uses: docker/build-push-action@v2
+    - uses: docker/build-push-action@v3
       if: matrix.profile != 'emqx-ee'
       with:
         ## only push when stable tag and rc tag
@@ -282,12 +268,12 @@ jobs:
         tags: ${{ steps.meta.outputs.tags }}
         labels: ${{ steps.meta.outputs.labels }}
         build-args: |
-          BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-20:${{ matrix.otp }}-alpine3.15.1
+          BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-23:${{ matrix.otp }}-alpine3.15.1
           RUN_FROM=alpine:3.15.1
           EMQX_NAME=${{ matrix.profile }}
         file: source/deploy/docker/Dockerfile
         context: source
-    - uses: docker/build-push-action@v2
+    - uses: docker/build-push-action@v3
       if: matrix.profile == 'emqx-ee'
       with:
         ## only push when stable tag and rc tag
@@ -298,7 +284,7 @@ jobs:
         tags: ${{ steps.meta.outputs.tags }}
         labels: ${{ steps.meta.outputs.labels }}
         build-args: |
-          BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-20:${{ matrix.otp }}-alpine3.15.1
+          BUILD_FROM=ghcr.io/emqx/emqx-builder/4.4-23:${{ matrix.otp }}-alpine3.15.1
           RUN_FROM=alpine:3.15.1
           EMQX_NAME=${{ matrix.profile }}
         file: source/deploy/docker/Dockerfile.enterprise
diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml
index a9998cd91..a4a1a0f4a 100644
--- a/.github/workflows/build_slim_packages.yaml
+++ b/.github/workflows/build_slim_packages.yaml
@@ -20,7 +20,7 @@ jobs:
           - 24.3.4.2-1
         os:
           - ubuntu20.04
-          - el8
+          - el7
         runs-on:
           - aws-amd64
           - ubuntu-20.04
@@ -32,7 +32,7 @@ jobs:
           - runs-on: aws-amd64
             use-self-hosted: false
 
-    container: ghcr.io/emqx/emqx-builder/4.4-20:${{ matrix.otp }}-${{ matrix.os }}
+    container: ghcr.io/emqx/emqx-builder/4.4-23:${{ matrix.otp }}-${{ matrix.os }}
 
     steps:
     - uses: AutoModality/action-clean@v1
diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml
index 7db081000..eeff09c01 100644
--- a/.github/workflows/check_deps_integrity.yaml
+++ b/.github/workflows/check_deps_integrity.yaml
@@ -5,7 +5,7 @@ on: [pull_request]
 jobs:
   check_deps_integrity:
     runs-on: ubuntu-20.04
-    container: ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-ubuntu20.04
+    container: ghcr.io/emqx/emqx-builder/4.4-23:24.3.4.2-1-ubuntu20.04
 
     steps:
       - uses: actions/checkout@v2
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index 14df0776f..694aee796 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -7,7 +7,7 @@ on:
 jobs:
   prepare:
     runs-on: ubuntu-20.04
-    container: ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-ubuntu20.04
+    container: ghcr.io/emqx/emqx-builder/4.4-23:24.3.4.2-1-ubuntu20.04
 
     outputs:
       profiles: ${{ steps.detect-profiles.outputs.profiles}}
diff --git a/.github/workflows/run_acl_migration_tests.yaml b/.github/workflows/run_acl_migration_tests.yaml
index 65078ca65..a69ba6482 100644
--- a/.github/workflows/run_acl_migration_tests.yaml
+++ b/.github/workflows/run_acl_migration_tests.yaml
@@ -5,7 +5,7 @@ on: workflow_dispatch
 jobs:
     test:
         runs-on: ubuntu-20.04
-        container: ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-ubuntu20.04
+        container: ghcr.io/emqx/emqx-builder/4.4-23:24.3.4.2-1-ubuntu20.04
         strategy:
             fail-fast: true
         env:
diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml
index 2750be7f4..7078dfb5b 100644
--- a/.github/workflows/run_fvt_tests.yaml
+++ b/.github/workflows/run_fvt_tests.yaml
@@ -200,7 +200,7 @@ jobs:
 
     relup_test_plan:
         runs-on: ubuntu-20.04
-        container: ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-ubuntu20.04
+        container: ghcr.io/emqx/emqx-builder/4.4-23:24.3.4.2-1-ubuntu20.04
         outputs:
           profile: ${{ steps.profile-and-versions.outputs.profile }}
           vsn: ${{ steps.profile-and-versions.outputs.vsn }}
@@ -251,7 +251,7 @@ jobs:
              otp:
              - 24.3.4.2-1
         runs-on: ubuntu-20.04
-        container: ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-ubuntu20.04
+        container: ghcr.io/emqx/emqx-builder/4.4-23:24.3.4.2-1-ubuntu20.04
         defaults:
           run:
             shell: bash
@@ -288,7 +288,7 @@ jobs:
           - relup_test_plan
           - relup_test_build
         runs-on: ubuntu-20.04
-        container: ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-ubuntu20.04
+        container: ghcr.io/emqx/emqx-builder/4.4-23:24.3.4.2-1-ubuntu20.04
         strategy:
           fail-fast: false
           matrix:
diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml
index 910c3a77c..5612562a6 100644
--- a/.github/workflows/run_test_cases.yaml
+++ b/.github/workflows/run_test_cases.yaml
@@ -12,7 +12,7 @@ on:
 jobs:
   prepare:
     runs-on: aws-amd64
-    container: ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-ubuntu20.04
+    container: ghcr.io/emqx/emqx-builder/4.4-23:24.3.4.2-1-ubuntu20.04
     outputs:
       fast_ct_apps: ${{ steps.run_find_apps.outputs.fast_ct_apps }}
       docker_ct_apps: ${{ steps.run_find_apps.outputs.docker_ct_apps }}
@@ -58,7 +58,7 @@ jobs:
   eunit_and_proper:
     needs: prepare
     runs-on: aws-amd64
-    container: ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-ubuntu20.04
+    container: ghcr.io/emqx/emqx-builder/4.4-23:24.3.4.2-1-ubuntu20.04
     strategy:
       fail-fast: false
       matrix:
@@ -86,7 +86,7 @@ jobs:
   fast_ct:
     needs: prepare
     runs-on: ${{ matrix.runs-on }}
-    container: ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-ubuntu20.04
+    container: ghcr.io/emqx/emqx-builder/4.4-23:24.3.4.2-1-ubuntu20.04
     strategy:
       fail-fast: false
       matrix:
@@ -249,7 +249,7 @@ jobs:
       - fast_ct
       - docker_ct
     runs-on: aws-amd64
-    container: ghcr.io/emqx/emqx-builder/4.4-20:24.3.4.2-1-ubuntu20.04
+    container: ghcr.io/emqx/emqx-builder/4.4-23:24.3.4.2-1-ubuntu20.04
     steps:
       - uses: AutoModality/action-clean@v1
       - uses: actions/download-artifact@v3

From f1a55bc052b8d0ec51d99d1ec5376c82ed0fe780 Mon Sep 17 00:00:00 2001
From: Ivan Dyachkov <dev@dyachkov.org>
Date: Wed, 14 Dec 2022 10:08:29 +0100
Subject: [PATCH 08/18] build: fix relup on amzn2

---
 build                           | 8 ++++++++
 scripts/get-distro.sh           | 6 +++++-
 scripts/relup-base-vsns.escript | 5 +++++
 3 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/build b/build
index 23e5cef49..f0adb79d0 100755
--- a/build
+++ b/build
@@ -195,6 +195,14 @@ make_zip() {
                     ;;
             esac
             ;;
+        amzn2)
+            case "$PKG_VSN" in
+                4.4.12*)
+                    # this is the first version for amzn2, no relup
+                    has_relup='no'
+                    ;;
+            esac
+            ;;
     esac
     # shellcheck disable=SC2207
     bases=($(relup_db base-vsns "$PKG_VSN"))
diff --git a/scripts/get-distro.sh b/scripts/get-distro.sh
index 4bf81afb3..064a25b11 100755
--- a/scripts/get-distro.sh
+++ b/scripts/get-distro.sh
@@ -14,7 +14,11 @@ case "$UNAME" in
         SYSTEM="${DIST}${VERSION_ID}"
         ;;
     Linux)
-        if grep -q -i 'rhel' /etc/*-release; then
+        # /etc/os-release on amazon linux 2 contains both rhel and centos strings
+        if grep -q -i 'amzn' /etc/*-release; then
+            DIST='amzn'
+            VERSION_ID="$(sed -n '/^VERSION_ID=/p' /etc/os-release | sed -r 's/VERSION_ID=(.*)/\1/g' | sed 's/"//g')"
+        elif grep -q -i 'rhel' /etc/*-release; then
             DIST='el'
             VERSION_ID="$(rpm --eval '%{rhel}')"
         elif grep -q -i 'centos' /etc/*-release; then
diff --git a/scripts/relup-base-vsns.escript b/scripts/relup-base-vsns.escript
index 157f604cb..ccbf94168 100755
--- a/scripts/relup-base-vsns.escript
+++ b/scripts/relup-base-vsns.escript
@@ -171,6 +171,11 @@ filter_froms(Froms0, AvailableVersionsIndex) ->
                   fun(Vsn) ->
                           not lists:member(Vsn, [<<"4.4.0">>, <<"4.4.1">>])
                   end, Froms0);
+            %% amzn2 is introduced since v4.4.12 and e4.4.12
+            %% exclude tags before them
+            "amzn2" ->
+                Excluded = [list_to_binary(["4.4.", integer_to_list(X)]) || X <- lists:seq(0,11)],
+                lists:filter(fun(Vsn) -> not lists:member(Vsn, Excluded) end, Froms0);
             _ ->
                 Froms0
         end,

From 682f917d6d6458a7de7cdb3caae605cd2ebd55ce Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 14 Dec 2022 08:50:03 +0100
Subject: [PATCH 09/18] docs: add change log for adding amazon linux 2

---
 changes/v4.4.12-en.md | 8 ++++++--
 changes/v4.4.12-zh.md | 8 ++++++--
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/changes/v4.4.12-en.md b/changes/v4.4.12-en.md
index 0cba88202..be0ae4ebd 100644
--- a/changes/v4.4.12-en.md
+++ b/changes/v4.4.12-en.md
@@ -1,4 +1,8 @@
-### Enhancements
+# v4.4.12
+
+Added Amazon Linxu 2 (amzn2) package release [#9498](https://github.com/emqx/emqx/pull/9498).
+
+## Enhancements
 
 - Upgrade http client library `ehttpc` from `0.2.1` to `0.4.2` [#9456](https://github.com/emqx/emqx/pull/9456).
 
@@ -6,7 +10,7 @@
 
 - Users can define the `externalTrafficPolicy` of service in EMQX Helm Chart [#9527](https://github.com/emqx/emqx/pull/9527).
 
-### Bug Fixes
+## Bug Fixes
 
 - Fixed load bootstrap file when no bootstrap user in `mqtt_app` [#9474](https://github.com/emqx/emqx/pull/9474).
 
diff --git a/changes/v4.4.12-zh.md b/changes/v4.4.12-zh.md
index fcef10e48..c08a009db 100644
--- a/changes/v4.4.12-zh.md
+++ b/changes/v4.4.12-zh.md
@@ -1,4 +1,8 @@
-### 增强
+# v4.4.12
+
+发布 Amazon Linux 2 安装包 [#9498](https://github.com/emqx/emqx/pull/9498)。
+
+## 增强
 
 - HTTP 客户端库 `ehttpc` 从 `0.2.1` 升级到 `0.4.2` [#9456](https://github.com/emqx/emqx/pull/9456)。
 
@@ -6,7 +10,7 @@
 
 - 用户可以在 EMQX Helm Chart 中自定义 service 资源的 `externalTrafficPolicy` [#9527](https://github.com/emqx/emqx/pull/9527)。
 
-### 修复
+## 修复
 
 - 修复 mqtt_app 表内没有 boostrap user 里未导入用户的问题 [#9474](https://github.com/emqx/emqx/pull/9474).
 

From 4dc4e0039e59e477be125bdea419601d456d477a Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Thu, 15 Dec 2022 18:01:42 +0100
Subject: [PATCH 10/18] docs: fix change logs

---
 changes/v4.4.11-en.md | 4 ++--
 changes/v4.4.12-en.md | 4 ++--
 changes/v4.4.12-zh.md | 3 ++-
 3 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/changes/v4.4.11-en.md b/changes/v4.4.11-en.md
index b8876c342..c4537b6a7 100644
--- a/changes/v4.4.11-en.md
+++ b/changes/v4.4.11-en.md
@@ -18,9 +18,9 @@
 - Added support for specifying custom modules for custom authentication [#9297](https://github.com/emqx/emqx/pull/9297).
   To support simple authentication rules, it is no longer necessary to implement a full-blown plugin.
 
-- Added a JWT management for Rule-Engin, for creating and refreshing JWT tokens in rule engine actions [#9241](https://github.com/emqx/emqx/pull/9241).
+- Added a JWT management for Rule-Engine, for creating and refreshing JWT tokens in rule engine actions [#9241](https://github.com/emqx/emqx/pull/9241).
   This feature is so far only used in EMQX Enterprise Google PubSub integration.
-  Can be used as webhook integration's JWT authenticationa against the webhook service endpoint.
+  Can be used as webhook integration's JWT authentication against the webhook service endpoint.
 
 - Update `gen_rpc` dependency to 3.0.0.
 
diff --git a/changes/v4.4.12-en.md b/changes/v4.4.12-en.md
index be0ae4ebd..5c927ca05 100644
--- a/changes/v4.4.12-en.md
+++ b/changes/v4.4.12-en.md
@@ -1,9 +1,9 @@
 # v4.4.12
 
-Added Amazon Linxu 2 (amzn2) package release [#9498](https://github.com/emqx/emqx/pull/9498).
-
 ## Enhancements
 
+- Added Amazon Linxu 2 (amzn2) package release [#9498](https://github.com/emqx/emqx/pull/9498).
+
 - Upgrade http client library `ehttpc` from `0.2.1` to `0.4.2` [#9456](https://github.com/emqx/emqx/pull/9456).
 
 - Add more PSK ciphers support [#9493](https://github.com/emqx/emqx/pull/9493).
diff --git a/changes/v4.4.12-zh.md b/changes/v4.4.12-zh.md
index c08a009db..1b0336a2f 100644
--- a/changes/v4.4.12-zh.md
+++ b/changes/v4.4.12-zh.md
@@ -1,9 +1,10 @@
 # v4.4.12
 
-发布 Amazon Linux 2 安装包 [#9498](https://github.com/emqx/emqx/pull/9498)。
 
 ## 增强
 
+- 发布 Amazon Linux 2 安装包 [#9498](https://github.com/emqx/emqx/pull/9498)。
+
 - HTTP 客户端库 `ehttpc` 从 `0.2.1` 升级到 `0.4.2` [#9456](https://github.com/emqx/emqx/pull/9456)。
 
 - 支持更多的 PSK 密码套件[#9493](https://github.com/emqx/emqx/pull/9493)。

From 40f4d326a06cd036fa0cd16eeb3a13fa6c587479 Mon Sep 17 00:00:00 2001
From: Ivan Dyachkov <dev@dyachkov.org>
Date: Thu, 15 Dec 2022 09:57:16 +0100
Subject: [PATCH 11/18] ci: build macos arm64 packages

---
 .github/workflows/build_packages.yaml      | 3 +++
 .github/workflows/build_slim_packages.yaml | 3 +++
 changes/v4.4.12-en.md                      | 2 ++
 changes/v4.4.12-zh.md                      | 2 ++
 4 files changed, 10 insertions(+)

diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml
index 55f6dac87..902154638 100644
--- a/.github/workflows/build_packages.yaml
+++ b/.github/workflows/build_packages.yaml
@@ -110,9 +110,12 @@ jobs:
           - 24.3.4.2-1
         os:
           - macos-11
+          - macos-12-arm64
     runs-on: ${{ matrix.os }}
 
     steps:
+    - uses: emqx/self-hosted-cleanup-action@v1.0.3
+      if: matrix.os == macos-12-arm64
     - uses: actions/download-artifact@v3
       with:
         name: source
diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml
index a4a1a0f4a..2e7b658cf 100644
--- a/.github/workflows/build_slim_packages.yaml
+++ b/.github/workflows/build_slim_packages.yaml
@@ -120,8 +120,11 @@ jobs:
           - 24.3.4.2-1
         os:
           - macos-11
+          - macos-12-arm64
     runs-on: ${{ matrix.os }}
     steps:
+    - uses: emqx/self-hosted-cleanup-action@v1.0.3
+      if: matrix.os == macos-12-arm64
     - uses: actions/checkout@v3
       with:
           fetch-depth: 0 # clone full git history
diff --git a/changes/v4.4.12-en.md b/changes/v4.4.12-en.md
index 5c927ca05..529f6a467 100644
--- a/changes/v4.4.12-en.md
+++ b/changes/v4.4.12-en.md
@@ -10,6 +10,8 @@
 
 - Users can define the `externalTrafficPolicy` of service in EMQX Helm Chart [#9527](https://github.com/emqx/emqx/pull/9527).
 
+- Start building MacOS packages for Apple Silicon M1/M2 [#9558](https://github.com/emqx/emqx/pull/9558).
+
 ## Bug Fixes
 
 - Fixed load bootstrap file when no bootstrap user in `mqtt_app` [#9474](https://github.com/emqx/emqx/pull/9474).
diff --git a/changes/v4.4.12-zh.md b/changes/v4.4.12-zh.md
index 1b0336a2f..d986aca93 100644
--- a/changes/v4.4.12-zh.md
+++ b/changes/v4.4.12-zh.md
@@ -11,6 +11,8 @@
 
 - 用户可以在 EMQX Helm Chart 中自定义 service 资源的 `externalTrafficPolicy` [#9527](https://github.com/emqx/emqx/pull/9527)。
 
+- 新增 Apple Silicon M1/M2 安装包发布 [#9558](https://github.com/emqx/emqx/pull/9558)。
+
 ## 修复
 
 - 修复 mqtt_app 表内没有 boostrap user 里未导入用户的问题 [#9474](https://github.com/emqx/emqx/pull/9474).

From be20082ce505b0a88d78a82d0b07a653512adfd2 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Fri, 16 Dec 2022 10:50:38 +0100
Subject: [PATCH 12/18] ci: revert of docker/setup-qemu-action@v3 back to v2

---
 .github/workflows/build_packages.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml
index 55f6dac87..8167ca602 100644
--- a/.github/workflows/build_packages.yaml
+++ b/.github/workflows/build_packages.yaml
@@ -225,7 +225,7 @@ jobs:
         path: .
     - name: unzip source code
       run: unzip -q source.zip
-    - uses: docker/setup-qemu-action@v3
+    - uses: docker/setup-qemu-action@v2
       with:
         image: tonistiigi/binfmt:latest
         platforms: all

From 3698150fc5dbbd6ff49a82e28855f6049a2dc74c Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Fri, 16 Dec 2022 11:01:00 +0100
Subject: [PATCH 13/18] ci: use alternatives for python2 python3 switch

---
 .ci/build_packages/tests.sh | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/.ci/build_packages/tests.sh b/.ci/build_packages/tests.sh
index a51885c1e..3182cd414 100755
--- a/.ci/build_packages/tests.sh
+++ b/.ci/build_packages/tests.sh
@@ -116,7 +116,20 @@ emqx_test(){
             fi
         ;;
         "rpm")
-            yum install -y "${PACKAGE_PATH}/${packagename}"
+            # yum wants python2
+            if [[ "${SYSTEM:-}" == "amzn2" ]]; then
+                alternatives --list | grep python && alternatives --set python /usr/bin/python2
+            fi
+            YUM_RES="$(yum install -y "${PACKAGE_PATH}/${packagename}"| tee /dev/null)"
+            if [[ $YUM_RES =~ "Failed" ]]; then
+               echo "yum install failed"
+               exit 1
+            fi
+            # restore python3
+            if [[ "${SYSTEM:-}" == "amzn2" ]]; then
+                alternatives --list | grep python && alternatives --set python /usr/bin/python3
+            fi
+
             if ! rpm -q "${PROFILE}" | grep -q "${PROFILE}"; then
                 echo "package install error"
                 exit 1

From 4bc333812fb60aec83e7b3f12a8b64abee95a6fa Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Fri, 16 Dec 2022 11:06:14 +0100
Subject: [PATCH 14/18] chore: bump version to v4.4.12-alpha.2

---
 include/emqx_release.hrl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl
index 7fa90615f..929adf705 100644
--- a/include/emqx_release.hrl
+++ b/include/emqx_release.hrl
@@ -29,7 +29,7 @@
 
 -ifndef(EMQX_ENTERPRISE).
 
--define(EMQX_RELEASE, {opensource, "4.4.12-alpha.1"}).
+-define(EMQX_RELEASE, {opensource, "4.4.12-alpha.2"}).
 
 -else.
 

From 22c27cb45f0ba49d63644ed9c953a802ce572d85 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Thu, 15 Dec 2022 22:33:05 +0100
Subject: [PATCH 15/18] feat(cover): add covertool

---
 rebar.config               |   3 +-
 rebar.config.erl           |   1 +
 scripts/rel_otp_apps.eterm |   1 +
 scripts/xref_check.eterm   |   4 +
 src/emqx_cover.erl         | 150 +++++++++++++++++++++++++++++++++----
 5 files changed, 142 insertions(+), 17 deletions(-)

diff --git a/rebar.config b/rebar.config
index 1bcfb752c..64e9f2dec 100644
--- a/rebar.config
+++ b/rebar.config
@@ -29,7 +29,7 @@
 
 {cover_opts, [verbose]}.
 {cover_export_enabled, true}.
-{cover_excl_mods, [emqx_exproto_pb, emqx_exhook_pb]}.
+{cover_excl_mods, [emqx_exproto_pb, emqx_exhook_pb, emqx_cover]}.
 
 {provider_hooks, [{pre, [{release, {relup_helper, gen_appups}}]}]}.
 
@@ -40,6 +40,7 @@
 {deps,
     [ {gpb, "4.11.2"} %% gpb only used to build, but not for release, pin it here to avoid fetching a wrong version due to rebar plugins scattered in all the deps
     , {redbug, "2.0.7"}
+    , {covertool, {git, "https://github.com/zmstone/covertool", {tag, "2.0.4.1"}}}
     , {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.4.2"}}}
     , {gun, {git, "https://github.com/emqx/gun", {tag, "1.3.8"}}}
     , {eredis_cluster, {git, "https://github.com/emqx/eredis_cluster", {tag, "0.7.4"}}}
diff --git a/rebar.config.erl b/rebar.config.erl
index dc8d66fe0..9485bb38e 100644
--- a/rebar.config.erl
+++ b/rebar.config.erl
@@ -253,6 +253,7 @@ relx_apps(ReleaseType) ->
     , {ekka, load}
     , {emqx_plugin_libs, load}
     , observer_cli
+    , {covertool, load}
     ]
     ++ [emqx_modules || not is_enterprise()]
     ++ [emqx_license || is_enterprise()]
diff --git a/scripts/rel_otp_apps.eterm b/scripts/rel_otp_apps.eterm
index 0e6753997..b639221aa 100644
--- a/scripts/rel_otp_apps.eterm
+++ b/scripts/rel_otp_apps.eterm
@@ -14,4 +14,5 @@
 , {mnesia, load}
 , xmerl
 , tools
+, covertool % this is not really a otp app, but we don't need to worry about relup of it
 ].
diff --git a/scripts/xref_check.eterm b/scripts/xref_check.eterm
index 493cb1645..5725cdbba 100644
--- a/scripts/xref_check.eterm
+++ b/scripts/xref_check.eterm
@@ -5,6 +5,7 @@
    , excl_apps =>
          [ observer
          , redbug
+         , covertool
          ]
    , excl_mods =>
          [ hipe_unified_loader
@@ -14,6 +15,7 @@
          , basho_bench_driver_erldis
          , release_handler
          , cuttlefish_rebar_plugin
+         , emqx_cover
          ]
    , filters =>
          [{{coap_client,channel_apply,3},{coap_dtls_socket,close,1}},
@@ -239,6 +241,7 @@
    , analysis => undefined_functions
    , excl_apps =>
          [ observer
+         , covertool
          ]
    , excl_mods =>
          [ systools
@@ -246,6 +249,7 @@
          , release_handler
          , systools_relup
          , cuttlefish_rebar_plugin
+         , emqx_cover
          ]
    , filters =>
          [{'Elixir.Atom',to_string,1},
diff --git a/src/emqx_cover.erl b/src/emqx_cover.erl
index f407d0b7d..629707775 100644
--- a/src/emqx_cover.erl
+++ b/src/emqx_cover.erl
@@ -18,16 +18,68 @@
 %% It is used to collect coverage data when running blackbox test
 -module(emqx_cover).
 
+-include_lib("covertool/include/covertool.hrl").
+
+-ifdef(EMQX_ENTERPRISE).
+-define(OUTPUT_APPNAME, 'EMQX Enterprise').
+-else.
+-define(OUTPUT_APPNAME, 'EMQX').
+-endif.
+
 -export([start/0,
          start/1,
-         export_and_stop/1
+         abort/0,
+         export_and_stop/1,
+         lookup_source/1
         ]).
 
+%% This is a ETS table to keep a mapping of module name (atom) to
+%% .erl file path (relative path from project root)
+%% We needed this ETS table because the source file information
+%% is missing from the .beam metadata sicne we are using 'deterministic'
+%% compile flag.
+-define(SRC, emqx_cover_module_src).
+
+%% @doc Start cover.
+%% All emqx_ modules will be cover-compiled, this may cause
+%% some excessive RAM consumption and result in warning logs.
 start() ->
     start(#{}).
 
+%% @doc Start cover.
+%% All emqx_ modules will be cover-compiled, this may cause
+%% some excessive RAM consumption and result in warning logs.
+%% Supported options:
+%% - project_root: the directory to search for .erl source code
+%% - debug_secret_file: only applicable to EMQX Enterprise
 start(Opts) ->
-    ok = maybe_set_secret(),
+    ok = abort(),
+    %% spawn a ets table owner
+    %% this implementation is kept dead-simple
+    %% because there is no concurrency requirement
+    Parent = self(),
+    {Pid, Ref} =
+        erlang:spawn_monitor(
+          fun() ->
+                  true = register(?SRC, self()),
+                  _ = ets:new(?SRC, [named_table, public]),
+                  _ = Parent ! {started, self()},
+                  receive
+                      stop ->
+                          ok
+                  end
+          end),
+    receive
+        {started, Pid} ->
+            ok;
+        {'DOWN', Ref, process, Pid, Reason} ->
+            throw({failed_to_start, Reason})
+    after
+        1000 ->
+            throw({failed_to_start, timeout})
+    end,
+    Modules = modules(Opts),
+    ok = maybe_set_secret(Opts),
     case cover:start() of
         {ok, _Pid} ->
             ok;
@@ -36,28 +88,91 @@ start(Opts) ->
         Other ->
             throw(Other)
     end,
-    ok = cover_compile(Opts).
-
-export_and_stop(Path) ->
-    ok = cover:export(Path),
-    _ = cover:stop(),
+    ok = cover_compile(Modules),
+    io:format("cover-compiled ~p modules~n", [length(Modules)]),
+    ok = build_source_mapping(Opts, sets:from_list(Modules, [{version, 2}])),
+    CachedModulesCount = ets:info(?SRC, size),
+    io:format("source-cached ~p modules~n", [CachedModulesCount]),
     ok.
 
-maybe_set_secret() ->
-    case os:getenv("EMQX_DEBUG_SECRET_FILE") of
-        false ->
-            ok;
+%% @doc Abort cover data collection without exporting.
+abort() ->
+    _ = cover:stop(),
+    case whereis(?SRC) of
+        undefined -> ok;
+        Pid -> exit(Pid, kill)
+    end,
+    ok.
+
+%% @doc Export coverage report (xml) format.
+%% e.g. `emqx_cover:export_and_stop("/tmp/cover.xml").'
+export_and_stop(Path) when is_list(Path) ->
+    ProjectRoot = get_project_root(),
+    Config = #config{appname = ?OUTPUT_APPNAME,
+                     sources = [ProjectRoot],
+                     output = Path,
+                     lookup_source = fun ?MODULE:lookup_source/1
+                    },
+    covertool:generate_report(Config, cover:modules()).
+
+build_source_mapping(Opts, Modules) ->
+    Default = os_env("EMQX_PROJECT_ROOT"),
+    case maps:get(project_root, Opts, Default) of
+        "" ->
+            io:format(standard_error, "EMQX_PROJECT_ROOT is not set", []),
+            throw(emqx_project_root_undefined);
+        Dir ->
+            ok = put_project_root(Dir),
+            ok = do_build_source_mapping(Dir, Modules)
+    end.
+
+get_project_root() ->
+    [{_, Dir}] = ets:lookup(?SRC, {root, ?OUTPUT_APPNAME}),
+    Dir.
+
+put_project_root(Dir) ->
+    _ = ets:insert(?SRC, {{root, ?OUTPUT_APPNAME}, Dir}),
+    ok.
+
+do_build_source_mapping(Dir, Modules) ->
+    All = filelib:wildcard("**/*.erl", Dir),
+    lists:foreach(
+      fun(Path) ->
+              ModuleNameStr = filename:basename(Path, ".erl"),
+              Module = list_to_atom(ModuleNameStr),
+              case sets:is_element(Module, Modules) of
+                  true ->
+                      ets:insert(?SRC, {Module, Path});
+                  false ->
+                      ok
+              end
+      end, All),
+    ok.
+
+lookup_source(Module) ->
+    case ets:lookup(?SRC, Module) of
+        [{_, Path}] ->
+            Path;
+        [] ->
+            false
+    end.
+
+maybe_set_secret(Opts) ->
+    Default = os_env("EMQX_DEBUG_SECRET_FILE"),
+    case maps:get(debug_secret_file, Opts, Default) of
         "" ->
             ok;
         File ->
             ok = emqx:set_debug_secret(File)
     end.
 
-cover_compile(_Opts) ->
+modules(_Opts) ->
     %% TODO better filter based on Opts,
     %% e.g. we may want to see coverage info for ehttpc
     Filter = fun is_emqx_module/1,
-    Modules = find_modules(Filter),
+    find_modules(Filter).
+
+cover_compile(Modules) ->
     Results = cover:compile_beam(Modules),
     Errors = lists:filter(fun({ok, _}) -> false;
                              (_) -> true
@@ -66,14 +181,14 @@ cover_compile(_Opts) ->
         [] ->
             ok;
         _ ->
-            io:format(user, "failed_to_cover_compile:~n~p~n", [Errors]),
+            io:format("failed_to_cover_compile:~n~p~n", [Errors]),
             throw(failed_to_cover_compile)
     end.
 
 find_modules(Filter) ->
     All = code:all_loaded(),
-    F = fun({M, _BeamPath}) -> Filter(M) end,
-    lists:filter(F, All).
+    F = fun({M, _BeamPath}) -> Filter(M) andalso {true, M} end,
+    lists:filtermap(F, All).
 
 is_emqx_module(?MODULE) ->
     %% do not cover-compile self
@@ -85,3 +200,6 @@ is_emqx_module(Module) ->
         _ ->
             false
     end.
+
+os_env(Name) ->
+    os:getenv(Name, "").

From 13b14aafa561c6efb7ec7eb651490bc1c31bf7bf Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Mon, 19 Dec 2022 09:23:05 +0100
Subject: [PATCH 16/18] fix(emqx_cover): wait for DOWN message before new spawn

---
 src/emqx_cover.erl | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/emqx_cover.erl b/src/emqx_cover.erl
index 629707775..62a306f9b 100644
--- a/src/emqx_cover.erl
+++ b/src/emqx_cover.erl
@@ -100,7 +100,13 @@ abort() ->
     _ = cover:stop(),
     case whereis(?SRC) of
         undefined -> ok;
-        Pid -> exit(Pid, kill)
+        Pid ->
+            Ref = monitor(process, Pid),
+            exit(Pid, kill),
+            receive
+                {'DOWN', Ref, process, Pid, _} ->
+                    ok
+            end
     end,
     ok.
 

From 30b1c5f26d5d9f8e58a36e6b4bfd0a9eb514fa97 Mon Sep 17 00:00:00 2001
From: Ivan Dyachkov <dev@dyachkov.org>
Date: Mon, 19 Dec 2022 09:49:28 +0100
Subject: [PATCH 17/18] chore: fail build if apple notarytool failed

---
 .github/workflows/build_slim_packages.yaml |  2 +-
 build                                      | 16 ++++++++++++----
 scripts/macos-sign-binaries.sh             | 22 ++++++++++++++++++----
 3 files changed, 31 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml
index 2e7b658cf..2793d9074 100644
--- a/.github/workflows/build_slim_packages.yaml
+++ b/.github/workflows/build_slim_packages.yaml
@@ -124,7 +124,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
     - uses: emqx/self-hosted-cleanup-action@v1.0.3
-      if: matrix.os == macos-12-arm64
+      if: ${{ matrix.os == 'macos-12-arm64' }}
     - uses: actions/checkout@v3
       with:
           fetch-depth: 0 # clone full git history
diff --git a/build b/build
index f0adb79d0..e740ff3b3 100755
--- a/build
+++ b/build
@@ -236,10 +236,18 @@ make_zip() {
                     #   --apple-id <apple id> \
                     #   --password <apple id password>
                     #   --team-id <apple team id> <submission-id>
-                xcrun notarytool submit \
-                      --apple-id "${APPLE_ID}" \
-                      --password "${APPLE_ID_PASSWORD}" \
-                      --team-id "${APPLE_TEAM_ID}" "${target_zip}" --wait
+                echo 'Submitting the package for notarization to Apple (normally takes about a minute)'
+                notarytool_output="$(xcrun notarytool submit \
+                                           --apple-id "${APPLE_ID}" \
+                                           --password "${APPLE_ID_PASSWORD}" \
+                                           --team-id "${APPLE_TEAM_ID}" "${target_zip}" \
+                                           --no-progress \
+                                           --wait)"
+                echo "$notarytool_output"
+                echo "$notarytool_output" | grep -q 'status: Accepted' || {
+                    echo 'Notarization failed';
+                    exit 1;
+                }
             fi
             # sha256sum may not be available on macos
             openssl dgst -sha256 "${target_zip}" | cut -d ' ' -f 2  > "${target_zip}.sha256"
diff --git a/scripts/macos-sign-binaries.sh b/scripts/macos-sign-binaries.sh
index 384744b2a..135730694 100755
--- a/scripts/macos-sign-binaries.sh
+++ b/scripts/macos-sign-binaries.sh
@@ -43,10 +43,24 @@ done
 security -v list-keychains -s "${keychain_names[@]}" "${KEYCHAIN}"
 
 # known runtime executables and binaries
-codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/erts-*/bin/{beam.smp,dyn_erl,epmd,erl,erl_call,erl_child_setup,erlexec,escript,heart,inet_gethost,run_erl,to_erl}
-codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/runtime_tools-*/priv/lib/{dyntrace.so,trace_ip_drv.so,trace_file_drv.so}
-codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/os_mon-*/priv/bin/{cpu_sup,memsup}
+codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime \
+         "${REL_DIR}"/erts-*/bin/{beam.smp,dyn_erl,epmd,erl,erl_call,erl_child_setup,erlexec,escript,heart,inet_gethost,run_erl,to_erl}
+codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime \
+         "${REL_DIR}"/lib/runtime_tools-*/priv/lib/{dyntrace.so,trace_ip_drv.so,trace_file_drv.so}
+codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime \
+         "${REL_DIR}"/lib/os_mon-*/priv/bin/{cpu_sup,memsup}
 # other files from runtime and dependencies
-for f in asn1rt_nif.so bcrypt_nif.so crypto.so otp_test_engine.so crypto_callback.so jiffy.so crc32cer_nif.so sasl_auth.so snappyer.so odbcserver; do
+for f in \
+        asn1rt_nif.so \
+        bcrypt_nif.so \
+        crypto.so \
+        otp_test_engine.so \
+        crypto_callback.so \
+        jiffy.so \
+        crc32cer_nif.so \
+        sasl_auth.so \
+        snappyer.so \
+        odbcserver \
+    ; do
     find "${REL_DIR}"/lib/ -name "$f" -exec codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime {} \;
 done

From c776afd86a2d876aff5d0bfcc1cd14f34feecaa3 Mon Sep 17 00:00:00 2001
From: Ivan Dyachkov <dev@dyachkov.org>
Date: Mon, 19 Dec 2022 09:57:03 +0100
Subject: [PATCH 18/18] ci: backport patch for otp setup on self hosted macos
 from master

---
 .github/actions/package-macos/action.yaml  | 32 ++++++++++++++++------
 .github/workflows/build_slim_packages.yaml |  2 +-
 build                                      | 10 +++++++
 scripts/relup-base-vsns.escript            | 18 ++++++++----
 4 files changed, 46 insertions(+), 16 deletions(-)

diff --git a/.github/actions/package-macos/action.yaml b/.github/actions/package-macos/action.yaml
index 0624eb525..18afbd495 100644
--- a/.github/actions/package-macos/action.yaml
+++ b/.github/actions/package-macos/action.yaml
@@ -23,27 +23,38 @@ inputs:
 runs:
   using: composite
   steps:
-    - name: prepare
+    - id: prepare
       shell: bash
       run: |
         brew update
-        brew install curl zip unzip gnu-sed coreutils unixodbc freetds openssl@1.1
+        brew install curl zip unzip gnu-sed coreutils autoconf automake cmake unixodbc freetds openssl@1.1
         echo "/usr/local/opt/bison/bin" >> $GITHUB_PATH
         echo "/usr/local/bin" >> $GITHUB_PATH
+        OTP_SOURCE_PATH="$HOME/src/otp-${{ inputs.otp }}"
+        OTP_INSTALL_PATH="$HOME/otp/${{ inputs.otp }}"
+        echo "OTP_SOURCE_PATH=$OTP_SOURCE_PATH" >> $GITHUB_OUTPUT
+        echo "OTP_INSTALL_PATH=$OTP_INSTALL_PATH" >> $GITHUB_OUTPUT
+        mkdir -p "$OTP_INSTALL_PATH"
     - uses: actions/cache@v3
       id: cache
       with:
-        path: /opt/erlang/${{ inputs.otp }}
+        path: ${{ steps.prepare.outputs.OTP_INSTALL_PATH }}
         key: otp-install-${{ inputs.otp }}-${{ inputs.os }}-static-ssl-disable-hipe-disable-jit
     - name: build erlang
       if: steps.cache.outputs.cache-hit != 'true'
       shell: bash
       run: |
-        git clone --depth 1 --branch OTP-${{ inputs.otp }} https://github.com/emqx/otp.git $HOME/otp-${{ inputs.otp }}
-        cd $HOME/otp-${{ inputs.otp }}
-        ./configure --disable-dynamic-ssl-lib --with-ssl=/usr/local/opt/openssl@1.1 --disable-hipe --disable-jit --prefix=/opt/erlang/${{ inputs.otp }}
+        OTP_SOURCE_PATH="${{ steps.prepare.outputs.OTP_SOURCE_PATH }}"
+        OTP_INSTALL_PATH="${{ steps.prepare.outputs.OTP_INSTALL_PATH }}"
+        if [ -d "$OTP_SOURCE_PATH" ]; then
+          rm -rf "$OTP_SOURCE_PATH"
+        fi
+        git clone --depth 1 --branch OTP-${{ inputs.otp }} https://github.com/emqx/otp.git "$OTP_SOURCE_PATH"
+        cd "$OTP_SOURCE_PATH"
+        ./configure --disable-dynamic-ssl-lib --with-ssl=$(brew --prefix openssl@1.1) --disable-hipe --disable-jit --prefix="$OTP_INSTALL_PATH"
         make -j$(nproc)
-        sudo make install
+        rm -rf "$OTP_INSTALL_PATH"
+        make install
     - name: build
       env:
         AUTO_INSTALL_BUILD_DEPS: 1
@@ -56,13 +67,16 @@ runs:
         APPLE_DEVELOPER_ID_BUNDLE_PASSWORD: ${{ inputs.apple_developer_id_bundle_password }}
       shell: bash
       run: |
-        export PATH="/opt/erlang/${{ inputs.otp }}/bin:$PATH"
+        export PATH="${{ steps.prepare.outputs.OTP_INSTALL_PATH }}/bin:$PATH"
         make ensure-rebar3
-        sudo cp rebar3 /usr/local/bin/rebar3
+        mkdir -p $HOME/bin
+        cp rebar3 $HOME/bin/rebar3
+        export PATH="$HOME/bin:$PATH"
         make ${EMQX_NAME}-zip
     - name: test
       shell: bash
       run: |
+        export PATH="${{ steps.prepare.outputs.OTP_INSTALL_PATH }}/bin:$PATH"
         pkg_name=$(basename _packages/${EMQX_NAME}/${EMQX_NAME}-*.zip)
         unzip -q _packages/${EMQX_NAME}/$pkg_name
         # test with a spaces in path
diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml
index 2793d9074..205066a07 100644
--- a/.github/workflows/build_slim_packages.yaml
+++ b/.github/workflows/build_slim_packages.yaml
@@ -149,5 +149,5 @@ jobs:
         path: ./rebar3.crashdump
     - uses: actions/upload-artifact@v3
       with:
-        name: macos
+        name: ${{ matrix.os }}
         path: _packages/**/*.zip
diff --git a/build b/build
index e740ff3b3..d3c9164eb 100755
--- a/build
+++ b/build
@@ -203,6 +203,16 @@ make_zip() {
                     ;;
             esac
             ;;
+        macos*)
+            case "$PKG_VSN" in
+                4.4.12*)
+                    # this is the first version for macos arm (M1/M2), no relup
+                    if [ "$ARCH" = arm64 ]; then
+                        has_relup='no'
+                    fi
+                    ;;
+            esac
+            ;;
     esac
     # shellcheck disable=SC2207
     bases=($(relup_db base-vsns "$PKG_VSN"))
diff --git a/scripts/relup-base-vsns.escript b/scripts/relup-base-vsns.escript
index ccbf94168..e99c88a9f 100755
--- a/scripts/relup-base-vsns.escript
+++ b/scripts/relup-base-vsns.escript
@@ -162,21 +162,26 @@ filter_froms(Froms0, AvailableVersionsIndex) ->
     Froms1 =
         case get_system() of
             %% we do not support relup for windows
-            "windows" ->
+            {"windows", _} ->
                 [];
             %% debian11 is introduced since v4.4.2 and e4.4.2
             %% exclude tags before them
-            "debian11" ->
+            {"debian11", _} ->
                 lists:filter(
                   fun(Vsn) ->
                           not lists:member(Vsn, [<<"4.4.0">>, <<"4.4.1">>])
                   end, Froms0);
             %% amzn2 is introduced since v4.4.12 and e4.4.12
             %% exclude tags before them
-            "amzn2" ->
+            {"amzn2", _} ->
                 Excluded = [list_to_binary(["4.4.", integer_to_list(X)]) || X <- lists:seq(0,11)],
                 lists:filter(fun(Vsn) -> not lists:member(Vsn, Excluded) end, Froms0);
-            _ ->
+            %% macos arm64 (M1/M2) packages are introduced since v4.4.12 and e4.4.12
+            %% exclude tags before them
+            {"macos" ++ _, "aarch64" ++ _} ->
+                Excluded = [list_to_binary(["4.4.", integer_to_list(X)]) || X <- lists:seq(0,11)],
+                lists:filter(fun(Vsn) -> not lists:member(Vsn, Excluded) end, Froms0);
+            {_, _} ->
                 Froms0
         end,
     lists:filter(
@@ -184,11 +189,12 @@ filter_froms(Froms0, AvailableVersionsIndex) ->
       Froms1).
 
 get_system() ->
+    Arch = erlang:system_info(system_architecture),
     case os:getenv("SYSTEM") of
         false ->
-            string:trim(os:cmd("./scripts/get-distro.sh"));
+            {string:trim(os:cmd("./scripts/get-distro.sh")), Arch};
         System ->
-            System
+            {System, Arch}
     end.
 
 %% assumes that's X.Y.Z, without pre-releases