From 74776153242a78750c528d022dd144be0bb2e40c Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 9 Dec 2021 17:15:23 +0800 Subject: [PATCH 1/9] fix(emqx_sn): appup.src miss 4.3.2 --- apps/emqx_sn/src/emqx_sn.appup.src | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/emqx_sn/src/emqx_sn.appup.src b/apps/emqx_sn/src/emqx_sn.appup.src index 0fbd16b1b..caf715adc 100644 --- a/apps/emqx_sn/src/emqx_sn.appup.src +++ b/apps/emqx_sn/src/emqx_sn.appup.src @@ -5,7 +5,8 @@ {load_module, emqx_sn_registry, brutal_purge, soft_purge, []} ]}, {"4.3.2", [ - {load_module, emqx_sn_gateway, brutal_purge, soft_purge, []} + {load_module, emqx_sn_gateway, brutal_purge, soft_purge, []}, + {load_module, emqx_sn_registry, brutal_purge, soft_purge, []} ]}, {<<"4\\.3\\.[0-1]">>, [ {restart_application, emqx_sn} @@ -16,7 +17,8 @@ {load_module, emqx_sn_registry, brutal_purge, soft_purge, []} ]}, {"4.3.2", [ - {load_module, emqx_sn_gateway, brutal_purge, soft_purge, []} + {load_module, emqx_sn_gateway, brutal_purge, soft_purge, []}, + {load_module, emqx_sn_registry, brutal_purge, soft_purge, []} ]}, {<<"4\\.3\\.[0-1]">>, [ {restart_application, emqx_sn} From aa04df4d1b02b7cea9b10e76f5ded6e903b42806 Mon Sep 17 00:00:00 2001 From: zhouzb Date: Thu, 9 Dec 2021 18:40:45 +0800 Subject: [PATCH 2/9] chore(webhook): update appup --- .../emqx_web_hook/src/emqx_web_hook.appup.src | 30 ++++++++----------- 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/apps/emqx_web_hook/src/emqx_web_hook.appup.src b/apps/emqx_web_hook/src/emqx_web_hook.appup.src index 5c801d24a..c55204cbf 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook.appup.src +++ b/apps/emqx_web_hook/src/emqx_web_hook.appup.src @@ -1,34 +1,28 @@ %% -*- mode: erlang -*- {VSN, - [{"4.3.7", - [{apply,{application,stop,[emqx_web_hook]}}, - {load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, - {"4.3.5", - [{load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, - {<<"4.3.[0-2]">>, + [{<<"4\\.3\\.[0-2]">>, [{apply,{application,stop,[emqx_web_hook]}}, {load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}, {load_module,emqx_web_hook,brutal_purge,soft_purge,[]}, {load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, - {<<"4.3.[3-4]">>, + {<<"4\\.3\\.[3-5]">>, [{load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}, {load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, + {<<"4\\.3\\.[6-7]">>, + [{apply,{application,stop,[emqx_web_hook]}}, + {load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.7", - [{apply,{application,stop,[emqx_web_hook]}}, - {load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, - {"4.3.5", - [{load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, - {<<"4.3.[0-2]">>, + [{<<"4\\.3\\.[0-2]">>, [{apply,{application,stop,[emqx_web_hook]}}, {load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}, {load_module,emqx_web_hook,brutal_purge,soft_purge,[]}, {load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, - {<<"4.3.[3-4]">>, + {<<"4\\.3\\.[3-5]">>, [{load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}, {load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, + {<<"4\\.3\\.[6-7]">>, + [{apply,{application,stop,[emqx_web_hook]}}, + {load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}]}. From b6ffacc062cb3fd9762912b3f13079dc239b3f5f Mon Sep 17 00:00:00 2001 From: zhouzb Date: Thu, 9 Dec 2021 18:46:13 +0800 Subject: [PATCH 3/9] chore(appup): fix appup for webhook --- apps/emqx_web_hook/src/emqx_web_hook.appup.src | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/apps/emqx_web_hook/src/emqx_web_hook.appup.src b/apps/emqx_web_hook/src/emqx_web_hook.appup.src index c55204cbf..c70a652c5 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook.appup.src +++ b/apps/emqx_web_hook/src/emqx_web_hook.appup.src @@ -5,10 +5,7 @@ {load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}, {load_module,emqx_web_hook,brutal_purge,soft_purge,[]}, {load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, - {<<"4\\.3\\.[3-5]">>, - [{load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, - {<<"4\\.3\\.[6-7]">>, + {<<"4\\.3\\.[3-7]">>, [{apply,{application,stop,[emqx_web_hook]}}, {load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}, {load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, @@ -18,10 +15,7 @@ {load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}, {load_module,emqx_web_hook,brutal_purge,soft_purge,[]}, {load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, - {<<"4\\.3\\.[3-5]">>, - [{load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, - {<<"4\\.3\\.[6-7]">>, + {<<"4\\.3\\.[3-7]">>, [{apply,{application,stop,[emqx_web_hook]}}, {load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}, {load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, From db8131d2a533a04aa8ff2cb5e279b67096c64171 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 9 Dec 2021 19:58:14 +0800 Subject: [PATCH 4/9] revert(emqx_conf): zone.external.max_topic_levels revert part of changes in 289415b5aac36b1229b5fa3feccec90d2cf54e8e --- etc/emqx.conf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/etc/emqx.conf b/etc/emqx.conf index ae5c33508..94416783f 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -772,8 +772,7 @@ zone.external.force_gc_policy = 16000|16MB ## Maximum topic levels allowed. 0 means no limit. ## ## Value: Number [0-65535] -## Default: 7 -zone.external.max_topic_levels = 7 +## zone.external.max_topic_levels = 7 ## Maximum QoS allowed. ## From 5c497975a628ad145d7487fc647356337b1e9374 Mon Sep 17 00:00:00 2001 From: Spycsh <757407490@qq.com> Date: Fri, 3 Dec 2021 15:50:55 +0100 Subject: [PATCH 5/9] chore: fix main-v4.3 support for networking.k8s.io/v1 --- deploy/charts/emqx/templates/ingress.yaml | 34 +++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/deploy/charts/emqx/templates/ingress.yaml b/deploy/charts/emqx/templates/ingress.yaml index f527e41f4..c6d7f7fa1 100644 --- a/deploy/charts/emqx/templates/ingress.yaml +++ b/deploy/charts/emqx/templates/ingress.yaml @@ -1,5 +1,7 @@ {{- if .Values.ingress.dashboard.enabled -}} -{{- if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} apiVersion: networking.k8s.io/v1beta1 {{- else -}} apiVersion: extensions/v1beta1 @@ -17,15 +19,28 @@ metadata: {{- toYaml .Values.ingress.dashboard.annotations | nindent 4 }} {{- end }} spec: +{{- if and .Values.ingress.dashboard.ingressClassName (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.dashboard.ingressClassName }} +{{- end }} rules: {{- range $host := .Values.ingress.dashboard.hosts }} - host: {{ $host }} http: paths: - path: / + {{- if (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: ImplementationSpecific + {{- end }} backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ include "emqx.fullname" $ }} + port: + number: {{ $.Values.service.dashboard }} + {{- else }} serviceName: {{ include "emqx.fullname" $ }} servicePort: {{ $.Values.service.dashboard }} + {{- end }} {{- end -}} {{- if .Values.ingress.dashboard.tls }} tls: @@ -34,7 +49,9 @@ spec: --- {{- end }} {{- if .Values.ingress.mgmt.enabled -}} -{{- if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} apiVersion: networking.k8s.io/v1beta1 {{- else -}} apiVersion: extensions/v1beta1 @@ -52,15 +69,28 @@ metadata: {{- toYaml .Values.ingress.mgmt.annotations | nindent 4 }} {{- end }} spec: +{{- if and .Values.ingress.mgmt.ingressClassName (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.mgmt.ingressClassName }} +{{- end }} rules: {{- range $host := .Values.ingress.mgmt.hosts }} - host: {{ $host }} http: paths: - path: / + {{- if (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: ImplementationSpecific + {{- end }} backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ include "emqx.fullname" $ }} + port: + number: {{ $.Values.service.mgmt }} + {{- else }} serviceName: {{ include "emqx.fullname" $ }} servicePort: {{ $.Values.service.mgmt }} + {{- end }} {{- end -}} {{- if .Values.ingress.mgmt.tls }} tls: From b9acf94fd7dc297cfe07d41cf9c2c6491368e49e Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 9 Dec 2021 16:05:50 +0800 Subject: [PATCH 6/9] feat(frame): better log for proxy_protocol config disabled --- src/emqx_connection.erl | 7 ++++++- src/emqx_frame.erl | 14 ++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/emqx_connection.erl b/src/emqx_connection.erl index 5dfb3e220..ef12f6bcf 100644 --- a/src/emqx_connection.erl +++ b/src/emqx_connection.erl @@ -629,10 +629,15 @@ parse_incoming(Data, Packets, State = #state{parse_state = ParseState}) -> NState = State#state{parse_state = NParseState}, parse_incoming(Rest, [Packet|Packets], NState) catch + error:proxy_protocol_config_disabled:_Stk -> + ?LOG(error, + "~nMalformed packet, " + "please check proxy_protocol config for specific listeners and zones~n"), + {[{frame_error, proxy_protocol_config_disabled} | Packets], State}; error:Reason:Stk -> ?LOG(error, "~nParse failed for ~0p~n~0p~nFrame data:~0p", [Reason, Stk, Data]), - {[{frame_error, Reason}|Packets], State} + {[{frame_error, Reason} | Packets], State} end. -compile({inline, [next_incoming_msgs/1]}). diff --git a/src/emqx_frame.erl b/src/emqx_frame.erl index 79ac9da35..0722d996d 100644 --- a/src/emqx_frame.erl +++ b/src/emqx_frame.erl @@ -71,6 +71,13 @@ -define(MULTIPLIER_MAX, 16#200000). +%% proxy_protocol v1 header human readable +-define(PPV1_PROXY, "PROXY "). +-define(PPV1_PROXY_UNKNOWN, "PROXY UNKNOWN"). +%% proxy_protocol v2 header signature: +%% 16#0D,16#0A, 16#0D,16#0A,16#00,16#0D,16#0A,16#51,16#55,16#49,16#54,16#0A +-define(PPV2_HEADER_SIG, "\r\n\r\n\0\r\nQUIT\n"). + -dialyzer({no_match, [serialize_utf8_string/2]}). %%-------------------------------------------------------------------- @@ -100,6 +107,13 @@ parse(Bin) -> -spec(parse(binary(), parse_state()) -> parse_result()). parse(<<>>, {none, Options}) -> {more, {none, Options}}; +parse(<>, {none, _Options}) + when IPVer =:= <<"TCP4 ">> orelse IPVer =:= <<"TCP6 ">> -> + error(proxy_protocol_config_disabled); +parse(<>, {none, _Options}) -> + error(proxy_protocol_config_disabled); +parse(<>, {none, _Options}) -> + error(proxy_protocol_config_disabled); parse(<>, {none, Options = #{strict_mode := StrictMode}}) -> %% Validate header if strict mode. From 3ecc79e1bca3c156a106cda7160ce4081e40df29 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 9 Dec 2021 16:35:18 +0800 Subject: [PATCH 7/9] test(frame): test packet proxy_protocol config disabled --- src/emqx_frame.erl | 4 ++++ test/emqx_frame_SUITE.erl | 12 ++++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/emqx_frame.erl b/src/emqx_frame.erl index 0722d996d..17daf9809 100644 --- a/src/emqx_frame.erl +++ b/src/emqx_frame.erl @@ -80,6 +80,10 @@ -dialyzer({no_match, [serialize_utf8_string/2]}). +-ifdef(TEST). +-export([parse_variable_byte_integer/1]). +-endif. + %%-------------------------------------------------------------------- %% Init Parse State %%-------------------------------------------------------------------- diff --git a/test/emqx_frame_SUITE.erl b/test/emqx_frame_SUITE.erl index 591c75bdc..4b43c8506 100644 --- a/test/emqx_frame_SUITE.erl +++ b/test/emqx_frame_SUITE.erl @@ -43,7 +43,9 @@ groups() -> [{parse, [parallel], [t_parse_cont, t_parse_frame_too_large, - t_parse_frame_malformed_variable_byte_integer + t_parse_frame_malformed_variable_byte_integer, + t_parse_frame_variable_byte_integer, + t_parse_frame_proxy_protocol %% proxy_protocol_config_disabled packet. ]}, {connect, [parallel], [t_serialize_parse_v3_connect, @@ -142,6 +144,13 @@ t_parse_frame_variable_byte_integer(_) -> ?catch_error(malformed_variable_byte_integer, emqx_frame:parse_variable_byte_integer(Bin)). +t_parse_frame_proxy_protocol(_) -> + BinList = [ <<"PROXY TCP4 ">>, <<"PROXY TCP6 ">>, <<"PROXY UNKNOWN">> + , <<"\r\n\r\n\0\r\nQUIT\n">>], + [?assertError( proxy_protocol_config_disabled + , emqx_frame:parse(Bin)) + || Bin <- BinList]. + t_serialize_parse_v3_connect(_) -> Bin = <<16,37,0,6,77,81,73,115,100,112,3,2,0,60,0,23,109,111,115, 113,112,117, 98,47,49,48,52,53,49,45,105,77,97,99,46,108, @@ -555,4 +564,3 @@ parse_to_packet(Bin, Opts) -> Packet. payload(Len) -> iolist_to_binary(lists:duplicate(Len, 1)). - From bad314ae0875a1868ab692def7ec4de9e4779596 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 9 Dec 2021 19:45:24 +0800 Subject: [PATCH 8/9] chore(gitignore): update gitignore --- .gitignore | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.gitignore b/.gitignore index 218b22261..bbb4edebb 100644 --- a/.gitignore +++ b/.gitignore @@ -47,3 +47,15 @@ dist.zip scripts/git-token etc/*.seg _upgrade_base/ +erlang_ls.config +.els_cache/ +# VSCode files +.vs/ +.vscode/ +# Emacs Backup files +*~ +# Emacs temporary files +.#* +*# +# For direnv +.envrc From 24060285c5a4c1941766b0ff1b97fdc9a0b52ac4 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Fri, 10 Dec 2021 14:15:09 +0800 Subject: [PATCH 9/9] chore(appup): update appup.src --- src/emqx.appup.src | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index c510b3043..cea35e90d 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -3,12 +3,14 @@ [{"4.3.11", [{load_module,emqx_vm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}]}, {"4.3.10", [{load_module,emqx_vm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}]}, {"4.3.9", [{load_module,emqx_vm,brutal_purge,soft_purge,[]}, @@ -194,12 +196,14 @@ [{"4.3.11", [{load_module,emqx_vm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}]}, {"4.3.10", [{load_module,emqx_vm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}]}, {"4.3.9", [{load_module,emqx_vm,brutal_purge,soft_purge,[]},