From 74e97541540408a1218ba38c02f62d803fae0d87 Mon Sep 17 00:00:00 2001 From: Rory Z Date: Fri, 22 Apr 2022 09:57:39 +0800 Subject: [PATCH 01/12] feat(helm): the pathType of ingress can be made configurable via values --- deploy/charts/emqx/README.md | 4 ++++ deploy/charts/emqx/templates/ingress.dashboard.yaml | 2 +- deploy/charts/emqx/templates/ingress.mgmt.yaml | 2 +- deploy/charts/emqx/templates/ingress.wss.yaml | 2 +- deploy/charts/emqx/values.yaml | 3 +++ 5 files changed, 10 insertions(+), 3 deletions(-) diff --git a/deploy/charts/emqx/README.md b/deploy/charts/emqx/README.md index 54d267415..935624769 100644 --- a/deploy/charts/emqx/README.md +++ b/deploy/charts/emqx/README.md @@ -70,18 +70,21 @@ Parameter | Description | Default Value `ingress.dashboard.enabled` | Enable ingress for EMQX Dashboard | false `ingress.dashboard.ingressClassName` | Set the ingress class for EMQX Dashboard `ingress.dashboard.path` | Ingress path for EMQX Dashboard | `/` +`ingress.dashboard.pathType` | Ingress pathType for EMQX Dashboard | `ImplementationSpecific` `ingress.dashboard.hosts` | Ingress hosts for EMQX Mgmt API | dashboard.emqx.local `ingress.dashboard.tls` | Ingress tls for EMQX Mgmt API | `[]` `ingress.dashboard.annotations` | Ingress annotations for EMQX Mgmt API | `{}` `ingress.mgmt.enabled` | Enable ingress for EMQX Mgmt API | `false` `ingress.mqtt.ingressClassName` | Set the ingress class for EMQX Mgmt API | `nil` `ingress.mgmt.path` | Ingress path for EMQX Mgmt API | `/` +`ingress.mgmt.pathType` | Ingress pathType for EMQX Mgmt API | `ImplementationSpecific` `ingress.mgmt.hosts` | Ingress hosts for EMQX Mgmt API | `api.emqx.local` `ingress.mgmt.tls` | Ingress tls for EMQX Mgmt API | `[]` `ingress.mgmt.annotations` | Ingress annotations for EMQX Mgmt API | `{}` `ingress.wss.enabled` | Enable ingress for EMQX Mgmt API | `false` `ingress.wss.ingressClassName` | Set the ingress class for EMQX Mgmt API | `nil` `ingress.wss.path` | Ingress path for EMQX WSS | `/` +`ingress.wss.pathType` | Ingress pathType for EMQX WSS | `ImplementationSpecific` `ingress.wss.hosts` | Ingress hosts for EMQX WSS | `wss.emqx.local` `ingress.wss.tls` | Ingress tls for EMQX WSS | `[]` `ingress.wss.annotations` | Ingress annotations for EMQX WSS | `{}` @@ -122,6 +125,7 @@ ingress: nginx.ingress.kubernetes.io/use-proxy-protocol: "false" nginx.ingress.kubernetes.io/proxy-protocol-header-timeout: "5s" path: /mqtt + pathType: ImplementationSpecific hosts: - myhost.example.com tls: diff --git a/deploy/charts/emqx/templates/ingress.dashboard.yaml b/deploy/charts/emqx/templates/ingress.dashboard.yaml index 9ea06cdc4..5a8bbd4a9 100644 --- a/deploy/charts/emqx/templates/ingress.dashboard.yaml +++ b/deploy/charts/emqx/templates/ingress.dashboard.yaml @@ -34,7 +34,7 @@ spec: paths: - path: / {{- if (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: ImplementationSpecific + pathType: {{ .Values.ingress.dashboard.pathType | default "ImplementationSpecific" }} {{- end }} backend: {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} diff --git a/deploy/charts/emqx/templates/ingress.mgmt.yaml b/deploy/charts/emqx/templates/ingress.mgmt.yaml index c9abe79a9..6c3811f77 100644 --- a/deploy/charts/emqx/templates/ingress.mgmt.yaml +++ b/deploy/charts/emqx/templates/ingress.mgmt.yaml @@ -34,7 +34,7 @@ spec: paths: - path: {{ $.Values.ingress.mgmt.path | default "/" }} {{- if (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: ImplementationSpecific + pathType: {{ .Values.ingress.mgmt.pathType | default "ImplementationSpecific" }} {{- end }} backend: {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} diff --git a/deploy/charts/emqx/templates/ingress.wss.yaml b/deploy/charts/emqx/templates/ingress.wss.yaml index 7c7b395a5..ec74889bc 100644 --- a/deploy/charts/emqx/templates/ingress.wss.yaml +++ b/deploy/charts/emqx/templates/ingress.wss.yaml @@ -34,7 +34,7 @@ spec: paths: - path: {{ $.Values.ingress.wss.path | default "/mqtt" }} {{- if (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: ImplementationSpecific + pathType: {{ .Values.ingress.wss.pathType | default "ImplementationSpecific" }} {{- end }} backend: {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} diff --git a/deploy/charts/emqx/values.yaml b/deploy/charts/emqx/values.yaml index d78f19a8e..0f273671c 100644 --- a/deploy/charts/emqx/values.yaml +++ b/deploy/charts/emqx/values.yaml @@ -195,6 +195,7 @@ ingress: # kubernetes.io/ingress.class: nginx # kubernetes.io/tls-acme: "true" path: / + pathType: ImplementationSpecific hosts: - dashboard.emqx.local tls: [] @@ -206,6 +207,7 @@ ingress: # kubernetes.io/ingress.class: nginx # kubernetes.io/tls-acme: "true" path: / + pathType: ImplementationSpecific hosts: - api.emqx.local tls: [] @@ -228,6 +230,7 @@ ingress: # nginx.ingress.kubernetes.io/use-proxy-protocol: "false" # nginx.ingress.kubernetes.io/proxy-protocol-header-timeout: "5s" path: /mqtt + pathType: ImplementationSpecific # path: /wss(\/.*)? hosts: - wss.emqx.local From 10bca7f6b553919f58630288e6d9fe678c5858b5 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 20 Apr 2022 16:53:50 +0800 Subject: [PATCH 02/12] fix: topic filter with qos/clientid/share --- .../src/emqx_mgmt_api_subscriptions.erl | 33 +++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl index 6b35d7a0a..484309dc2 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl @@ -67,7 +67,8 @@ list(Bindings, Params) when map_size(Bindings) == 0 -> Topic0 -> Topic = emqx_mgmt_util:urldecode(Topic0), Data = emqx_mgmt:list_subscriptions_via_topic(Topic, ?format_fun), - minirest:return({ok, add_meta(Params, Data)}) + FilterData = filter_subscriptions(Data, Params), + minirest:return({ok, add_meta(Params, FilterData)}) end; list(#{node := Node} = Bindings, Params) -> @@ -85,7 +86,8 @@ list(#{node := Node} = Bindings, Params) -> Topic0 -> Topic = emqx_mgmt_util:urldecode(Topic0), Data = emqx_mgmt:list_subscriptions_via_topic(Node, Topic, ?format_fun), - minirest:return({ok, add_meta(Params, Data)}) + FilterData = filter_subscriptions(Data, Params), + minirest:return({ok, add_meta(Params, FilterData)}) end. add_meta(Params, List) -> @@ -169,3 +171,30 @@ update_ms(share, X, {{Pid, Topic}, Opts}) -> {{Pid, Topic}, Opts#{share => X}}; update_ms(qos, X, {{Pid, Topic}, Opts}) -> {{Pid, Topic}, Opts#{qos => X}}. + +filter_subscriptions(Data0, Params) -> + Data1 = filter_by_key(qos, qos(Params), Data0), + Data2 = filter_by_key(clientid, proplists:get_value(<<"clientid">>, Params), Data1), + case proplists:get_value(<<"share">>, Params) of + undefined -> Data2; + Share -> + Prefix = filename:join([<<"$share">>, Share]), + Size = byte_size(Prefix), + lists:filter(fun(#{topic := Topic}) -> + case Topic of + <> -> true; + _ -> false + end + end, + Data2) + end. + +qos(Params) -> + case proplists:get_value(<<"qos">>, Params) of + undefined -> undefined; + Qos when is_integer(Qos) -> Qos; + Qos when is_binary(Qos) -> binary_to_integer(Qos) + end. + +filter_by_key(_Key, undefined, List) -> List; +filter_by_key(Key, Value, List) -> lists:filter(fun(E) -> Value =:= maps:get(Key, E) end, List). From 5d8539c01a9e9a7311a017d546080d04e46a83e1 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sat, 23 Apr 2022 13:27:02 +0200 Subject: [PATCH 03/12] fix(bin/emqx): make sed replace \n work in macos 10 --- bin/emqx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bin/emqx b/bin/emqx index ac459b6f7..bcb7622a0 100755 --- a/bin/emqx +++ b/bin/emqx @@ -356,7 +356,10 @@ generate_config() { ## transform a single line args list like '-config ... -args_file ... -vm_args ...' to lines and get path for each file respectively ## NOTE: the -args_file and -vm_args are the same file passed twice because args_file is used by beam, but not possible to get at runtime ## by calling init:get_arguments/0 - lines="$(echo "$CUTTLEFISH_OUTPUT" | tail -1 | sed 's/-config/\nconfig=/g' | sed 's/-args_file/\nargs_file=/g' | sed 's/-vm_args/\nvm_args=/g')" + lines="$(echo "$CUTTLEFISH_OUTPUT" | tail -1 \ + | sed -e $'s/-config/\\\nconfig=/g' \ + | sed -e $'s/-args_file/\\\nargs_file=/g' \ + | sed -e $'s/-vm_args/\\\nvm_args=/g')" CONFIG_FILE="$(trim "$(echo -e "$lines" | grep 'config=' | sed 's/config=//g')")" CUTTLE_GEN_ARG_FILE="$(trim "$(echo -e "$lines" | grep 'vm_args=' | sed 's/vm_args=//g')")" From 29e453383a5d48b5288eab4f2a420f6fab02d718 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Sun, 24 Apr 2022 16:48:12 +0800 Subject: [PATCH 04/12] chore: update changelog --- CHANGES-4.3.md | 5 +++++ apps/emqx_management/src/emqx_management.app.src | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index 623e2d399..7c618417a 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -10,6 +10,11 @@ File format: - One list item per change topic Change log ends with a list of github PRs +## v4.3.15 + +### Bug fixes +* List subscription topic (/api/v4/subscriptions), the result do not match with multiple conditions. + ## v4.3.14 ### Enhancements diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index bee65781a..64f44637a 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -1,6 +1,6 @@ {application, emqx_management, [{description, "EMQ X Management API and CLI"}, - {vsn, "4.3.11"}, % strict semver, bump manually! + {vsn, "4.3.12"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel,stdlib,minirest]}, From 4789f1f97ba5b2c6e41553faf997308925bb61d5 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Sun, 24 Apr 2022 17:37:37 +0800 Subject: [PATCH 05/12] chore: bump management to 4.3.13 --- apps/emqx_management/src/emqx_management.app.src | 2 +- apps/emqx_management/src/emqx_management.appup.src | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index 64f44637a..e203c7a56 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -1,6 +1,6 @@ {application, emqx_management, [{description, "EMQ X Management API and CLI"}, - {vsn, "4.3.12"}, % strict semver, bump manually! + {vsn, "4.3.13"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel,stdlib,minirest]}, diff --git a/apps/emqx_management/src/emqx_management.appup.src b/apps/emqx_management/src/emqx_management.appup.src index 7fa9b6d6f..9265913e6 100644 --- a/apps/emqx_management/src/emqx_management.appup.src +++ b/apps/emqx_management/src/emqx_management.appup.src @@ -1,13 +1,13 @@ %% -*- mode: erlang -*- {VSN, - [ {<<"4\\.3\\.([0-9]|1[0-1])">>, + [ {<<"4\\.3\\.([0-9]|1[0-2])">>, [ {apply,{minirest,stop_http,['http:management']}}, {apply,{minirest,stop_http,['https:management']}}, {restart_application, emqx_management} ]}, {<<".*">>, []} ], - [ {<<"4\\.3\\.([0-9]|1[0-1])">>, + [ {<<"4\\.3\\.([0-9]|1[0-2])">>, [ {apply,{minirest,stop_http,['http:management']}}, {apply,{minirest,stop_http,['https:management']}}, {restart_application, emqx_management} From 0dd48e30b6adba294976f4a2a979d163e952daeb Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 25 Apr 2022 11:31:29 +0200 Subject: [PATCH 06/12] fix(emqx_mgmt_auth): randomise all bytes in app secret prior to this change, app secrets are uuids which is not completely random. --- CHANGES-4.3.md | 5 +++-- apps/emqx_management/src/emqx_mgmt_auth.erl | 7 +++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index 391d7770a..297adc1ab 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -17,9 +17,10 @@ File format: * Made possible for EMQX to boot from a Linux directory which has white spaces in its path. * Add support for JWT authorization [#7596] Now MQTT clients may be authorized with respect to a specific claim containing publish/subscribe topic whitelists. - +* Better randomisation of app screts (changed from timestamp seeded sha hash (uuid) to crypto:strong_rand_bytes) + ### Bug fixes -* List subscription topic (/api/v4/subscriptions), the result do not match with multiple conditions. +* List subscription topic (/api/v4/subscriptions), the result do not match with multiple conditions. ## v4.3.14 diff --git a/apps/emqx_management/src/emqx_mgmt_auth.erl b/apps/emqx_management/src/emqx_mgmt_auth.erl index 5413cf95a..6eca989cf 100644 --- a/apps/emqx_management/src/emqx_mgmt_auth.erl +++ b/apps/emqx_management/src/emqx_mgmt_auth.erl @@ -138,8 +138,11 @@ generate_appsecret_if_need(InSecrt) when is_binary(InSecrt), byte_size(InSecrt) generate_appsecret_if_need(_) -> AppConf = application:get_env(?APP, application, []), case proplists:get_value(default_secret, AppConf) of - undefined -> emqx_guid:to_base62(emqx_guid:gen()); - Secret when is_binary(Secret) -> Secret + undefined -> + Random = crypto:strong_rand_bytes(32), + emqx_base62:encode(Random); + Secret when is_binary(Secret) -> + Secret end. -spec(get_appsecret(appid()) -> {appsecret() | undefined}). From 8385b3cca4062e93e2c42688e8d2e5dd0bd95466 Mon Sep 17 00:00:00 2001 From: Georgy Sychev Date: Mon, 25 Apr 2022 13:47:23 +0400 Subject: [PATCH 07/12] chore(eredis_cluster): bump eredis version to 0.7.2 --- CHANGES-4.3.md | 1 + rebar.config | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index 391d7770a..ae725c520 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -21,6 +21,7 @@ File format: ### Bug fixes * List subscription topic (/api/v4/subscriptions), the result do not match with multiple conditions. +* SSL closed error bug fixed ## v4.3.14 diff --git a/rebar.config b/rebar.config index 0a7faf889..af247fa90 100644 --- a/rebar.config +++ b/rebar.config @@ -40,7 +40,7 @@ {deps, [ {gpb, "4.11.2"} %% gpb only used to build, but not for release, pin it here to avoid fetching a wrong version due to rebar plugins scattered in all the deps , {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.2.0"}}} - , {eredis_cluster, {git, "https://github.com/emqx/eredis_cluster", {tag, "0.7.1"}}} + , {eredis_cluster, {git, "https://github.com/emqx/eredis_cluster", {tag, "0.7.2"}}} , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.8.2"}}} From 1bae610c3db0591977f8e8cd3d1b009769f99021 Mon Sep 17 00:00:00 2001 From: gsychev <79104934+gsychev@users.noreply.github.com> Date: Tue, 26 Apr 2022 18:37:20 +0000 Subject: [PATCH 08/12] chore(eredis): changelog Co-authored-by: Zaiming (Stone) Shi --- CHANGES-4.3.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index ae725c520..d8c235712 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -21,7 +21,7 @@ File format: ### Bug fixes * List subscription topic (/api/v4/subscriptions), the result do not match with multiple conditions. -* SSL closed error bug fixed +* SSL closed error bug fixed for redis client. ## v4.3.14 From 4f45670725cefa80b5f523de3f7ac7aeb80ce19a Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 26 Apr 2022 21:02:45 +0200 Subject: [PATCH 09/12] fix: add 2 seconds delay penalty for bad logins --- lib-ce/emqx_dashboard/src/emqx_dashboard_admin.erl | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard_admin.erl b/lib-ce/emqx_dashboard/src/emqx_dashboard_admin.erl index 73aa0c614..a4504fe29 100644 --- a/lib-ce/emqx_dashboard/src/emqx_dashboard_admin.erl +++ b/lib-ce/emqx_dashboard/src/emqx_dashboard_admin.erl @@ -183,13 +183,21 @@ check(Username, Password) -> case lookup_user(Username) of [#mqtt_admin{password = PwdHash}] -> case is_valid_pwd(PwdHash, Password) of - true -> ok; - false -> {error, <<"Username/Password error">>} + true -> + ok; + false -> + ok = bad_login_penalty(), + {error, <<"Username/Password error">>} end; [] -> + ok = bad_login_penalty(), {error, <<"Username/Password error">>} end. +bad_login_penalty() -> + timer:sleep(2000), + ok. + is_valid_pwd(<>, Password) -> Hash =:= md5_hash(Salt, Password). From 0cfeca9d9018cf88e930fd9f93cf53dc95835f6a Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 28 Apr 2022 11:09:25 -0300 Subject: [PATCH 10/12] fix(update_appup): clean untracked nested git directories Since the script copies the current workdir into the temporary directory for the previous build, when we latter try to clean the directory, `git clean -fdx` refuses to clean the folders of some dependencies: ``` + git clean -fdx Skipping repository _build/default/lib/lc ``` So, if the branch contains a dependency with changed version, it'll not be picked up by the script, as both versions will be identical and hence have no `.appup` difference. --- scripts/update-appup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/update-appup.sh b/scripts/update-appup.sh index 55bcc0122..9a099b025 100755 --- a/scripts/update-appup.sh +++ b/scripts/update-appup.sh @@ -101,7 +101,7 @@ else git fetch "$REMOTE" fi git reset --hard - git clean -fdx + git clean -ffdx git checkout "${PREV_TAG}" make "$PROFILE" popd From 6e404f7220207270468af045c2d941dfd6bb5aae Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 29 Apr 2022 07:38:59 +0200 Subject: [PATCH 11/12] fix: use /bin/bash in systemd service file Otherwise ubuntu may have troulbe to boot --- deploy/packages/emqx.service | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/packages/emqx.service b/deploy/packages/emqx.service index def74a1a4..35be2ad61 100644 --- a/deploy/packages/emqx.service +++ b/deploy/packages/emqx.service @@ -11,10 +11,10 @@ Environment=HOME=/var/lib/emqx # Must use a 'bash' wrap for some OS # errno=13 'Permission denied' # Cannot create FIFO ... for writing -ExecStart=bash /usr/bin/emqx start +ExecStart=/bin/bash /usr/bin/emqx start LimitNOFILE=1048576 -ExecStop=bash /usr/bin/emqx stop +ExecStop=/bin/bash /usr/bin/emqx stop Restart=on-failure # When clustered, give the peers enough time to get this node's 'DOWN' event From 47377e07085860fe59fe0462f56532d8887b818d Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 29 Apr 2022 10:14:23 +0200 Subject: [PATCH 12/12] ci: rename apps_vsn_check for main-v4.x so it's a different name for master branch and 4.x branches --- .github/workflows/apps_version_check.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/apps_version_check.yaml b/.github/workflows/apps_version_check.yaml index 7dfb4009f..2ef18934f 100644 --- a/.github/workflows/apps_version_check.yaml +++ b/.github/workflows/apps_version_check.yaml @@ -3,7 +3,7 @@ name: Check Apps Version on: [pull_request] jobs: - check_apps_version: + check_apps_version_4_x: runs-on: ubuntu-20.04 strategy: