diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index c23c061ed..ab8786b53 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -30,6 +30,7 @@ File format: * Add rule-engine function float2str/2, user can specify the float output precision [#7991] ### Bug fixes + * List subscription topic (/api/v4/subscriptions), the result do not match with multiple conditions. * SSL closed error bug fixed for redis client. * Fix mqtt-sn client disconnected due to re-send a duplicated qos2 message @@ -40,6 +41,8 @@ File format: [ekka-158](https://github.com/emqx/ekka/pull/158) * Add regular expression check ^[0-9A-Za-z_\-]+$ for node name [#7979] * Fix `node_dump` variable sourcing. [#8026] +* Fix heap size is growing too fast when trace large message. +* Support customized timestamp format of the log messages. ## v4.3.14 diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.erl b/apps/emqx_rule_engine/src/emqx_rule_engine.erl index 312fc9e7d..923f78920 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.erl @@ -656,11 +656,11 @@ init_action(Module, OnCreate, ActionInstId, Params) -> #action_instance_params{id = ActionInstId, params = Params, apply = Apply}) end. -clear_resource(_Module, undefined, Type, ResId) -> +clear_resource(_Module, undefined, ResId, Type) -> Name = alarm_name_of_resource_down(Type, ResId), _ = emqx_alarm:deactivate(Name), ok = emqx_rule_registry:remove_resource_params(ResId); -clear_resource(Module, Destroy, Type, ResId) -> +clear_resource(Module, Destroy, ResId, Type) -> Name = alarm_name_of_resource_down(Type, ResId), _ = emqx_alarm:deactivate(Name), case emqx_rule_registry:find_resource_params(ResId) of diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index 6900f76aa..c871ad0e2 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -336,7 +336,7 @@ t_clean_resource_alarms(_Config) -> config => #{}, description => <<"debug resource">>}), ?assert(true, is_binary(ResId)), - Name = emqx_rule_engine:alarm_name_of_resource_down(built_in, ResId), + Name = emqx_rule_engine:alarm_name_of_resource_down(ResId, built_in), _ = emqx_alarm:activate(Name, #{id => ResId, type => built_in}), AlarmExist = fun(#{name := AName}) -> AName == Name end, Len = length(lists:filter(AlarmExist, emqx_alarm:get_alarms())), diff --git a/deploy/charts/emqx/README.md b/deploy/charts/emqx/README.md index 935624769..8461c4ef6 100644 --- a/deploy/charts/emqx/README.md +++ b/deploy/charts/emqx/README.md @@ -88,6 +88,8 @@ Parameter | Description | Default Value `ingress.wss.hosts` | Ingress hosts for EMQX WSS | `wss.emqx.local` `ingress.wss.tls` | Ingress tls for EMQX WSS | `[]` `ingress.wss.annotations` | Ingress annotations for EMQX WSS | `{}` +| `metrics.enable` | If set to true, [prometheus-operator](https://github.com/prometheus-operator/prometheus-operator) needs to be installed, and [emqx_prometheus](https://github.com/emqx/emqx/tree/main-v4.4/apps/emqx_prometheus) needs to enable | false | +| `metrics.type` | Now we only supported "prometheus" | "prometheus" | `extraEnv` | Aditional container env vars | `[]` `extraEnvFrom` | Aditional container env from vars (eg. [config map](https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/), [secrets](https://kubernetes.io/docs/concepts/configuration/secret/) | `[]` `extraArgs` | Additional container executable arguments | `[]` diff --git a/deploy/charts/emqx/service-monitor.yaml b/deploy/charts/emqx/service-monitor.yaml new file mode 100644 index 000000000..9988bcb1f --- /dev/null +++ b/deploy/charts/emqx/service-monitor.yaml @@ -0,0 +1,39 @@ +{{- if and (.Values.metrics.enabled) (eq .Values.metrics.type "prometheus") }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "emqx.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ include "emqx.name" . }} + helm.sh/chart: {{ include "emqx.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + {{- if .Values.service.annotations }} + annotations: + {{ toYaml .Values.service.annotations | indent 4 }} + {{- end }} +spec: + endpoints: + - interval: 10s + port: mgmt + scheme: http + path: /api/v4/emqx_prometheus + params: + type: + - prometheus + basicAuth: + password: + name: {{ include "emqx.fullname" . }}-basic-auth + key: password + username: + name: {{ include "emqx.fullname" . }}-basic-auth + key: username + jobLabel: {{ .Release.Name }}-scraping + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "emqx.name" . }} +{{- end }} \ No newline at end of file diff --git a/deploy/charts/emqx/templates/secret.yaml b/deploy/charts/emqx/templates/secret.yaml new file mode 100644 index 000000000..fcc1b2f3d --- /dev/null +++ b/deploy/charts/emqx/templates/secret.yaml @@ -0,0 +1,19 @@ +{{- if .Values.metrics.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "emqx.fullname" . }}-basic-auth + namespace: {{ .Release.Namespace }} +type: kubernetes.io/basic-auth +stringData: + {{- if not (empty .Values.emqxConfig.EMQX_MANAGEMENT__DEFAULT_APPLICATION__ID) }} + username: admin + {{- else }} + username: {{ .Values.emqxConfig.EMQX_MANAGEMENT__DEFAULT_APPLICATION__ID }} + {{- end }} + {{- if not (empty .Values.emqxConfig.EMQX_MANAGEMENT__DEFAULT_APPLICATION__SECRET) }} + password: public + {{- else }} + password: {{ .Values.emqxConfig.EMQX_MANAGEMENT__DEFAULT_APPLICATION__SECRET}} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/deploy/charts/emqx/values.yaml b/deploy/charts/emqx/values.yaml index 69ed2424e..efae3bfea 100644 --- a/deploy/charts/emqx/values.yaml +++ b/deploy/charts/emqx/values.yaml @@ -91,6 +91,8 @@ emqxConfig: EMQX_CLUSTER__K8S__SUFFIX: "svc.cluster.local" ## if EMQX_CLUSTER__K8S__ADDRESS_TYPE eq dns # EMQX_CLUSTER__K8S__SUFFIX: "pod.cluster.local" + EMQX_MANAGEMENT__DEFAULT_APPLICATION__ID: "admin" + EMQX_MANAGEMENT__DEFAULT_APPLICATION__SECRET: "public" ## -------------------------------------------------------------------- ## [ACL](https://docs.emqx.io/broker/latest/en/advanced/acl-file.html) @@ -263,3 +265,7 @@ containerSecurityContext: enabled: true runAsNonRoot: true runAsUser: 1000 + +metrics: + enabled: false + type: prometheus diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index b5765abf3..85b37e4c1 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.4.4-rc.2"}). +-define(EMQX_RELEASE, {opensource, "4.4.4-rc.3"}). -else. diff --git a/scripts/buildx.sh b/scripts/buildx.sh index 3337383eb..f15069f86 100755 --- a/scripts/buildx.sh +++ b/scripts/buildx.sh @@ -76,5 +76,6 @@ docker run -i --rm \ -v "$(pwd)":/emqx \ --workdir /emqx \ --platform="linux/$ARCH" \ + --user root \ "$BUILDER" \ - bash -euc "git config --global --add safe.directory /emqx && chown -R root:root ./_build && make ${PROFILE}-${PKGTYPE} && .ci/build_packages/tests.sh $PROFILE $PKGTYPE" + bash -euc "git config --global --add safe.directory /emqx && chown -R root:root _build && make ${PROFILE}-${PKGTYPE} && .ci/build_packages/tests.sh $PROFILE $PKGTYPE" diff --git a/src/emqx.appup.src b/src/emqx.appup.src index a949e87db..c7cdd3e60 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -4,6 +4,7 @@ [{"4.4.3", [{add_module,emqx_calendar}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_packet,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, @@ -21,6 +22,7 @@ {"4.4.2", [{add_module,emqx_calendar}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_packet,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, @@ -40,7 +42,8 @@ {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_relup}]}, {"4.4.1", - [{add_module,emqx_calendar}, + [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, + {add_module,emqx_calendar}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, @@ -70,7 +73,8 @@ {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {add_module,emqx_relup}]}, {"4.4.0", - [{add_module,emqx_calendar}, + [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, + {add_module,emqx_calendar}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, @@ -105,7 +109,8 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.4.3", - [{add_module,emqx_calendar}, + [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, + {delete_module,emqx_calendar}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, @@ -121,7 +126,8 @@ {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_relup}]}, {"4.4.2", - [{add_module,emqx_calendar}, + [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, + {delete_module,emqx_calendar}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, @@ -141,8 +147,9 @@ {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_relup}]}, {"4.4.1", - [{add_module,emqx_calendar}, + [{delete_module,emqx_calendar}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_packet,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, @@ -170,7 +177,8 @@ {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {delete_module,emqx_relup}]}, {"4.4.0", - [{add_module,emqx_calendar}, + [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, + {delete_module,emqx_calendar}, {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, diff --git a/src/emqx_alarm_handler.erl b/src/emqx_alarm_handler.erl index 1520a231b..0ccf3363a 100644 --- a/src/emqx_alarm_handler.erl +++ b/src/emqx_alarm_handler.erl @@ -61,7 +61,7 @@ handle_event({set_alarm, {process_memory_high_watermark, Pid}}, State) -> high_watermark => emqx_os_mon:get_procmem_high_watermark()}), {ok, State}; handle_event({clear_alarm, process_memory_high_watermark}, State) -> - emqx_alarm:ensure_deactivate(high_process_memory_usage), + emqx_alarm:ensure_deactivated(high_process_memory_usage), {ok, State}; handle_event(_, State) -> diff --git a/src/emqx_packet.erl b/src/emqx_packet.erl index c60c06811..ecf913543 100644 --- a/src/emqx_packet.erl +++ b/src/emqx_packet.erl @@ -427,18 +427,18 @@ format_header(#mqtt_packet_header{type = Type, dup = Dup, qos = QoS, retain = Retain}, S) -> - S1 = case S == undefined of - true -> <<>>; - false -> [", ", S] - end, - io_lib:format("~s(Q~p, R~p, D~p~s)", [type_name(Type), QoS, i(Retain), i(Dup), S1]). + Header = io_lib:format("~s(Q~p, R~p, D~p", [type_name(Type), QoS, i(Retain), i(Dup)]), + case S == undefined of + true -> [Header, ")"]; + false -> [Header, S, ")"] + end. format_variable(undefined, _) -> undefined; format_variable(Variable, undefined) -> format_variable(Variable); format_variable(Variable, Payload) -> - io_lib:format("~s, Payload=~0p", [format_variable(Variable), Payload]). + [format_variable(Variable), ", Payload=", Payload]. format_variable(#mqtt_packet_connect{ proto_ver = ProtoVer,