Merge tag 'v4.3.15-rc.3' into merge-main-v4.3-into-v4.4

CHANGES-4.3.md

@@ -30,6 +30,7 @@ File format:
 * Add rule-engine function float2str/2, user can specify the float output precision [#7991]
 
 ### Bug fixes
+
 * List subscription topic (/api/v4/subscriptions), the result do not match with multiple conditions.
 * SSL closed error bug fixed for redis client.
 * Fix mqtt-sn client disconnected due to re-send a duplicated qos2 message
@@ -40,6 +41,8 @@ File format:
   [ekka-158](https://github.com/emqx/ekka/pull/158)
 * Add regular expression check ^[0-9A-Za-z_\-]+$ for node name [#7979]
 * Fix `node_dump` variable sourcing. [#8026]
+* Fix heap size is growing too fast when trace large message.
+* Support customized timestamp format of the log messages.
 
 ## v4.3.14
 

apps/emqx_rule_engine/src/emqx_rule_engine.erl

@@ -656,11 +656,11 @@ init_action(Module, OnCreate, ActionInstId, Params) ->
                 #action_instance_params{id = ActionInstId, params = Params, apply = Apply})
     end.
 
-clear_resource(_Module, undefined, Type, ResId) ->
+clear_resource(_Module, undefined, ResId, Type) ->
     Name = alarm_name_of_resource_down(Type, ResId),
     _ = emqx_alarm:deactivate(Name),
     ok = emqx_rule_registry:remove_resource_params(ResId);
-clear_resource(Module, Destroy, Type, ResId) ->
+clear_resource(Module, Destroy, ResId, Type) ->
     Name = alarm_name_of_resource_down(Type, ResId),
     _ = emqx_alarm:deactivate(Name),
     case emqx_rule_registry:find_resource_params(ResId) of

apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl

@@ -336,7 +336,7 @@ t_clean_resource_alarms(_Config) ->
               config => #{},
               description => <<"debug resource">>}),
     ?assert(true, is_binary(ResId)),
-    Name = emqx_rule_engine:alarm_name_of_resource_down(built_in, ResId),
+    Name = emqx_rule_engine:alarm_name_of_resource_down(ResId, built_in),
     _ = emqx_alarm:activate(Name, #{id => ResId, type => built_in}),
     AlarmExist = fun(#{name := AName}) -> AName == Name end,
     Len = length(lists:filter(AlarmExist, emqx_alarm:get_alarms())),

deploy/charts/emqx/README.md

@@ -88,6 +88,8 @@ Parameter  | Description | Default Value
 `ingress.wss.hosts` | Ingress hosts for EMQX WSS |    `wss.emqx.local`
 `ingress.wss.tls` | Ingress tls for EMQX WSS |	`[]`
 `ingress.wss.annotations` | Ingress annotations for EMQX WSS |	`{}`
+| `metrics.enable` | If set to true, [prometheus-operator](https://github.com/prometheus-operator/prometheus-operator) needs to be installed, and [emqx_prometheus](https://github.com/emqx/emqx/tree/main-v4.4/apps/emqx_prometheus) needs to enable | false |
+| `metrics.type` | Now we only supported "prometheus" | "prometheus" |
 `extraEnv` | Aditional container env vars | `[]`
 `extraEnvFrom` | Aditional container env from vars (eg. [config map](https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/), [secrets](https://kubernetes.io/docs/concepts/configuration/secret/) | `[]`
 `extraArgs` | Additional container executable arguments | `[]`

deploy/charts/emqx/service-monitor.yaml

@@ -0,0 +1,39 @@
+{{- if and (.Values.metrics.enabled) (eq .Values.metrics.type "prometheus") }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ include "emqx.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: {{ include "emqx.name" . }}
+    helm.sh/chart: {{ include "emqx.chart" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+  {{- if .Values.service.annotations }}
+  annotations:
+    {{ toYaml .Values.service.annotations | indent 4 }}
+  {{- end }}
+spec:
+  endpoints:
+  - interval: 10s
+    port: mgmt 
+    scheme: http
+    path: /api/v4/emqx_prometheus
+    params:
+      type:
+        - prometheus
+    basicAuth:
+      password:
+        name: {{ include "emqx.fullname" . }}-basic-auth
+        key: password
+      username:
+        name: {{ include "emqx.fullname" . }}-basic-auth
+        key: username
+  jobLabel: {{ .Release.Name }}-scraping
+  namespaceSelector:
+    matchNames:
+      -  {{ .Release.Namespace }}
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ include "emqx.name" . }}
+{{- end }}

deploy/charts/emqx/templates/secret.yaml

@@ -0,0 +1,19 @@
+{{- if .Values.metrics.enabled }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "emqx.fullname" . }}-basic-auth
+  namespace: {{ .Release.Namespace }}
+type: kubernetes.io/basic-auth
+stringData:
+  {{- if not (empty .Values.emqxConfig.EMQX_MANAGEMENT__DEFAULT_APPLICATION__ID) }}
+  username: admin
+  {{- else }}
+  username: {{ .Values.emqxConfig.EMQX_MANAGEMENT__DEFAULT_APPLICATION__ID }}
+  {{- end }}
+  {{- if not (empty .Values.emqxConfig.EMQX_MANAGEMENT__DEFAULT_APPLICATION__SECRET) }}
+  password: public
+  {{- else }}
+  password: {{ .Values.emqxConfig.EMQX_MANAGEMENT__DEFAULT_APPLICATION__SECRET}}
+  {{- end }}
+{{- end }}

deploy/charts/emqx/values.yaml

@@ -91,6 +91,8 @@ emqxConfig:
   EMQX_CLUSTER__K8S__SUFFIX: "svc.cluster.local"
   ## if EMQX_CLUSTER__K8S__ADDRESS_TYPE eq dns
   # EMQX_CLUSTER__K8S__SUFFIX: "pod.cluster.local"
+  EMQX_MANAGEMENT__DEFAULT_APPLICATION__ID: "admin"
+  EMQX_MANAGEMENT__DEFAULT_APPLICATION__SECRET: "public"
 
 ## --------------------------------------------------------------------
 ##  [ACL](https://docs.emqx.io/broker/latest/en/advanced/acl-file.html)
@@ -263,3 +265,7 @@ containerSecurityContext:
   enabled: true
   runAsNonRoot: true
   runAsUser: 1000
+
+metrics:
+  enabled: false
+  type: prometheus

src/emqx.appup.src

@@ -4,6 +4,7 @@
   [{"4.4.3",
     [{add_module,emqx_calendar},
      {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]},
+     {load_module,emqx_packet,brutal_purge,soft_purge,[]},
      {load_module,emqx_session,brutal_purge,soft_purge,[]},
      {load_module,emqx_misc,brutal_purge,soft_purge,[]},
      {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]},
@@ -21,6 +22,7 @@
    {"4.4.2",
     [{add_module,emqx_calendar},
      {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]},
+     {load_module,emqx_packet,brutal_purge,soft_purge,[]},
      {load_module,emqx_session,brutal_purge,soft_purge,[]},
      {load_module,emqx_misc,brutal_purge,soft_purge,[]},
      {load_module,emqx_access_rule,brutal_purge,soft_purge,[]},
@@ -143,6 +145,7 @@
    {"4.4.1",
     [{add_module,emqx_calendar},
      {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]},
+     {load_module,emqx_packet,brutal_purge,soft_purge,[]},
      {load_module,emqx_access_rule,brutal_purge,soft_purge,[]},
      {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]},
      {load_module,emqx_alarm,brutal_purge,soft_purge,[]},

src/emqx_alarm_handler.erl

@@ -61,7 +61,7 @@ handle_event({set_alarm, {process_memory_high_watermark, Pid}}, State) ->
                                                      high_watermark => emqx_os_mon:get_procmem_high_watermark()}),
     {ok, State};
 handle_event({clear_alarm, process_memory_high_watermark}, State) ->
-    emqx_alarm:ensure_deactivate(high_process_memory_usage),
+    emqx_alarm:ensure_deactivated(high_process_memory_usage),
     {ok, State};
 
 handle_event(_, State) ->

src/emqx_packet.erl

@@ -427,18 +427,18 @@ format_header(#mqtt_packet_header{type = Type,
                                   dup = Dup,
                                   qos = QoS,
                                   retain = Retain}, S) ->
-    S1 = case S == undefined of
-             true -> <<>>;
-             false -> [", ", S]
-         end,
-    io_lib:format("~s(Q~p, R~p, D~p~s)", [type_name(Type), QoS, i(Retain), i(Dup), S1]).
+    Header = io_lib:format("~s(Q~p, R~p, D~p", [type_name(Type), QoS, i(Retain), i(Dup)]),
+    case S == undefined of
+        true -> [Header, ")"];
+        false -> [Header, S, ")"]
+    end.
 
 format_variable(undefined, _) ->
     undefined;
 format_variable(Variable, undefined) ->
     format_variable(Variable);
 format_variable(Variable, Payload) ->
-    io_lib:format("~s, Payload=~0p", [format_variable(Variable), Payload]).
+    [format_variable(Variable), ", Payload=", Payload].
 
 format_variable(#mqtt_packet_connect{
                  proto_ver    = ProtoVer,