Merge branch 'main-v4.3' into trace-large-msg-format-as-iolist

CHANGES-4.3.md

@@ -30,6 +30,7 @@ File format:
 * Add rule-engine function float2str/2, user can specify the float output precision [#7991]
 
 ### Bug fixes
+
 * List subscription topic (/api/v4/subscriptions), the result do not match with multiple conditions.
 * SSL closed error bug fixed for redis client.
 * Fix mqtt-sn client disconnected due to re-send a duplicated qos2 message
@@ -41,6 +42,7 @@ File format:
 * Add regular expression check ^[0-9A-Za-z_\-]+$ for node name [#7979]
 * Fix `node_dump` variable sourcing. [#8026]
 * Fix heap size is growing too fast when trace large message.
+* Support customized timestamp format of the log messages.
 
 ## v4.3.14
 

apps/emqx_rule_engine/src/emqx_rule_engine.appup.src

@@ -2,7 +2,11 @@
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
   [{"4.3.9",
-    [{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_events,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
      {add_module,emqx_rule_date},
@@ -136,7 +140,11 @@
      {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
   [{"4.3.9",
-    [{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_metrics,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]},
+     {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_events,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_funcs,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_maps,brutal_purge,soft_purge,[]},

apps/emqx_rule_engine/src/emqx_rule_engine.erl

@@ -656,11 +656,11 @@ init_action(Module, OnCreate, ActionInstId, Params) ->
                 #action_instance_params{id = ActionInstId, params = Params, apply = Apply})
     end.
 
-clear_resource(_Module, undefined, Type, ResId) ->
+clear_resource(_Module, undefined, ResId, Type) ->
     Name = alarm_name_of_resource_down(Type, ResId),
     _ = emqx_alarm:deactivate(Name),
     ok = emqx_rule_registry:remove_resource_params(ResId);
-clear_resource(Module, Destroy, Type, ResId) ->
+clear_resource(Module, Destroy, ResId, Type) ->
     Name = alarm_name_of_resource_down(Type, ResId),
     _ = emqx_alarm:deactivate(Name),
     case emqx_rule_registry:find_resource_params(ResId) of

apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl

@@ -334,7 +334,7 @@ t_clean_resource_alarms(_Config) ->
               config => #{},
               description => <<"debug resource">>}),
     ?assert(true, is_binary(ResId)),
-    Name = emqx_rule_engine:alarm_name_of_resource_down(built_in, ResId),
+    Name = emqx_rule_engine:alarm_name_of_resource_down(ResId, built_in),
     _ = emqx_alarm:activate(Name, #{id => ResId, type => built_in}),
     AlarmExist = fun(#{name := AName}) -> AName == Name end,
     Len = length(lists:filter(AlarmExist, emqx_alarm:get_alarms())),

deploy/charts/emqx/README.md

@@ -88,6 +88,8 @@ Parameter  | Description | Default Value
 `ingress.wss.hosts` | Ingress hosts for EMQX WSS |    `wss.emqx.local`
 `ingress.wss.tls` | Ingress tls for EMQX WSS |	`[]`
 `ingress.wss.annotations` | Ingress annotations for EMQX WSS |	`{}`
+| `metrics.enable` | If set to true, [prometheus-operator](https://github.com/prometheus-operator/prometheus-operator) needs to be installed, and [emqx_prometheus](https://github.com/emqx/emqx/tree/main-v4.4/apps/emqx_prometheus) needs to enable | false |
+| `metrics.type` | Now we only supported "prometheus" | "prometheus" |
 `extraEnv` | Aditional container env vars | `[]`
 `extraEnvFrom` | Aditional container env from vars (eg. [config map](https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/), [secrets](https://kubernetes.io/docs/concepts/configuration/secret/) | `[]`
 `extraArgs` | Additional container executable arguments | `[]`

deploy/charts/emqx/service-monitor.yaml

@@ -0,0 +1,39 @@
+{{- if and (.Values.metrics.enabled) (eq .Values.metrics.type "prometheus") }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ include "emqx.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: {{ include "emqx.name" . }}
+    helm.sh/chart: {{ include "emqx.chart" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+  {{- if .Values.service.annotations }}
+  annotations:
+    {{ toYaml .Values.service.annotations | indent 4 }}
+  {{- end }}
+spec:
+  endpoints:
+  - interval: 10s
+    port: mgmt 
+    scheme: http
+    path: /api/v4/emqx_prometheus
+    params:
+      type:
+        - prometheus
+    basicAuth:
+      password:
+        name: {{ include "emqx.fullname" . }}-basic-auth
+        key: password
+      username:
+        name: {{ include "emqx.fullname" . }}-basic-auth
+        key: username
+  jobLabel: {{ .Release.Name }}-scraping
+  namespaceSelector:
+    matchNames:
+      -  {{ .Release.Namespace }}
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ include "emqx.name" . }}
+{{- end }}

deploy/charts/emqx/templates/secret.yaml

@@ -0,0 +1,19 @@
+{{- if .Values.metrics.enabled }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "emqx.fullname" . }}-basic-auth
+  namespace: {{ .Release.Namespace }}
+type: kubernetes.io/basic-auth
+stringData:
+  {{- if not (empty .Values.emqxConfig.EMQX_MANAGEMENT__DEFAULT_APPLICATION__ID) }}
+  username: admin
+  {{- else }}
+  username: {{ .Values.emqxConfig.EMQX_MANAGEMENT__DEFAULT_APPLICATION__ID }}
+  {{- end }}
+  {{- if not (empty .Values.emqxConfig.EMQX_MANAGEMENT__DEFAULT_APPLICATION__SECRET) }}
+  password: public
+  {{- else }}
+  password: {{ .Values.emqxConfig.EMQX_MANAGEMENT__DEFAULT_APPLICATION__SECRET}}
+  {{- end }}
+{{- end }}

deploy/charts/emqx/values.yaml

@@ -70,6 +70,8 @@ initContainers: {}
 
 ## EMQX configuration item, see the documentation (https://hub.docker.com/r/emqx/emqx)
 emqxConfig:
+  EMQX_MANAGEMENT__DEFAULT_APPLICATION__ID: "admin"
+  EMQX_MANAGEMENT__DEFAULT_APPLICATION__SECRET: "public"
   EMQX_CLUSTER__K8S__APISERVER: "https://kubernetes.default.svc:443"
   ## The address type is used to extract host from k8s service.
   ## Value: ip | dns | hostname
@@ -248,3 +250,7 @@ containerSecurityContext:
   enabled: true
   runAsNonRoot: true
   runAsUser: 1000
+
+metrics:
+  enabled: false
+  type: prometheus

src/emqx.appup.src

@@ -2,7 +2,11 @@
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
   [{"4.3.15",
-    [{add_module,emqx_calendar},
+    [{load_module,emqx,brutal_purge,soft_purge,[]},
+     {load_module,emqx_plugins,brutal_purge,soft_purge,[]},
+     {load_module,emqx_sys,brutal_purge,soft_purge,[]},
+     {load_module,emqx_hooks,brutal_purge,soft_purge,[]},
+     {add_module,emqx_calendar},
      {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]},
      {load_module,emqx_packet,brutal_purge,soft_purge,[]},
      {load_module,emqx_session,brutal_purge,soft_purge,[]},
@@ -542,7 +546,11 @@
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
   [{"4.3.15",
-    [{delete_module,emqx_calendar},
+    [{load_module,emqx,brutal_purge,soft_purge,[]},
+     {load_module,emqx_plugins,brutal_purge,soft_purge,[]},
+     {load_module,emqx_sys,brutal_purge,soft_purge,[]},
+     {load_module,emqx_hooks,brutal_purge,soft_purge,[]},
+     {delete_module,emqx_calendar},
      {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]},
      {load_module,emqx_packet,brutal_purge,soft_purge,[]},
      {load_module,emqx_alarm,brutal_purge,soft_purge,[]},