Merge remote-tracking branch 'origin/main-v4.3' into main-v4.4

.github/workflows/apps_version_check.yaml

@@ -3,7 +3,7 @@ name: Check Apps Version
 on: [pull_request]
 
 jobs:
-  check_apps_version:
+  check_apps_version_4_x:
     runs-on: ubuntu-20.04
 
     strategy:

CHANGES-4.3.md

@@ -17,6 +17,12 @@ File format:
 * Made possible for EMQX to boot from a Linux directory which has white spaces in its path.
 * Add support for JWT authorization [#7596]
   Now MQTT clients may be authorized with respect to a specific claim containing publish/subscribe topic whitelists.
+* Better randomisation of app screts (changed from timestamp seeded sha hash (uuid) to crypto:strong_rand_bytes)
+
+### Bug fixes
+* List subscription topic (/api/v4/subscriptions), the result do not match with multiple conditions.
+
+* SSL closed error bug fixed for redis client.
 
 ## v4.3.14
 

apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl

@@ -67,7 +67,8 @@ list(Bindings, Params) when map_size(Bindings) == 0 ->
         Topic0 ->
             Topic = emqx_mgmt_util:urldecode(Topic0),
             Data = emqx_mgmt:list_subscriptions_via_topic(Topic, ?format_fun),
-            minirest:return({ok, add_meta(Params, Data)})
+            FilterData = filter_subscriptions(Data, Params),
+            minirest:return({ok, add_meta(Params, FilterData)})
     end;
 
 list(#{node := Node} = Bindings, Params) ->
@@ -85,7 +86,8 @@ list(#{node := Node} = Bindings, Params) ->
         Topic0 ->
             Topic = emqx_mgmt_util:urldecode(Topic0),
             Data = emqx_mgmt:list_subscriptions_via_topic(Node, Topic, ?format_fun),
-            minirest:return({ok, add_meta(Params, Data)})
+            FilterData = filter_subscriptions(Data, Params),
+            minirest:return({ok, add_meta(Params, FilterData)})
     end.
 
 add_meta(Params, List) ->
@@ -169,3 +171,30 @@ update_ms(share, X, {{Pid, Topic}, Opts}) ->
     {{Pid, Topic}, Opts#{share => X}};
 update_ms(qos, X, {{Pid, Topic}, Opts}) ->
     {{Pid, Topic}, Opts#{qos => X}}.
+
+filter_subscriptions(Data0, Params) ->
+    Data1 = filter_by_key(qos, qos(Params), Data0),
+    Data2 = filter_by_key(clientid, proplists:get_value(<<"clientid">>, Params), Data1),
+    case proplists:get_value(<<"share">>, Params) of
+        undefined -> Data2;
+        Share ->
+            Prefix = filename:join([<<"$share">>, Share]),
+            Size = byte_size(Prefix),
+            lists:filter(fun(#{topic := Topic}) ->
+                case Topic of
+                    <<Prefix:Size/binary, _/binary>> -> true;
+                    _ -> false
+                end
+                         end,
+                Data2)
+    end.
+
+qos(Params) ->
+    case proplists:get_value(<<"qos">>, Params) of
+        undefined -> undefined;
+        Qos when is_integer(Qos) -> Qos;
+        Qos when is_binary(Qos) -> binary_to_integer(Qos)
+    end.
+
+filter_by_key(_Key, undefined, List) -> List;
+filter_by_key(Key, Value, List) -> lists:filter(fun(E) -> Value =:= maps:get(Key, E) end, List).

apps/emqx_management/src/emqx_mgmt_auth.erl

@@ -138,8 +138,11 @@ generate_appsecret_if_need(InSecrt) when is_binary(InSecrt), byte_size(InSecrt)
 generate_appsecret_if_need(_) ->
     AppConf = application:get_env(?APP, application, []),
     case proplists:get_value(default_secret,  AppConf) of
-       undefined -> emqx_guid:to_base62(emqx_guid:gen());
+       undefined ->
-       Secret when is_binary(Secret) -> Secret
+            Random = crypto:strong_rand_bytes(32),
+            emqx_base62:encode(Random);
+       Secret when is_binary(Secret) ->
+            Secret
     end.
 
 -spec(get_appsecret(appid()) -> {appsecret() | undefined}).

bin/emqx

@@ -362,9 +362,9 @@ generate_config() {
     ## NOTE: the -args_file and -vm_args are the same file passed twice because args_file is used by beam, but not possible to get at runtime
     ## by calling init:get_arguments/0
     lines="$(echo "$CUTTLEFISH_OUTPUT" | tail -1 \
-                | sed -e $'s/-config/\\\nconfig=/g' \
+                    | sed -e $'s/-config/\\\nconfig=/g' \
-                | sed -e $'s/-args_file/\\\nargs_file=/g' \
+                    | sed -e $'s/-args_file/\\\nargs_file=/g' \
-                | sed -e $'s/-vm_args/\\\nvm_args=/g')"
+                    | sed -e $'s/-vm_args/\\\nvm_args=/g')"
     CONFIG_FILE="$(trim "$(echo -e "$lines" | grep 'config=' | sed 's/config=//g')")"
     CUTTLE_GEN_ARG_FILE="$(trim "$(echo -e "$lines" | grep 'vm_args=' | sed 's/vm_args=//g')")"
 

deploy/charts/emqx/README.md

@@ -70,18 +70,21 @@ Parameter  | Description | Default Value
 `ingress.dashboard.enabled` |	Enable ingress for EMQX Dashboard |	false
 `ingress.dashboard.ingressClassName` |	Set the ingress class for EMQX Dashboard
 `ingress.dashboard.path` | Ingress path for EMQX Dashboard |	`/`
+`ingress.dashboard.pathType` | Ingress pathType for EMQX Dashboard |	`ImplementationSpecific`
 `ingress.dashboard.hosts` | Ingress hosts for EMQX Mgmt API |	dashboard.emqx.local
 `ingress.dashboard.tls` | Ingress tls for EMQX Mgmt API |	`[]`
 `ingress.dashboard.annotations` | Ingress annotations for EMQX Mgmt API |	`{}`
 `ingress.mgmt.enabled` |	Enable ingress for EMQX Mgmt API |	`false`
 `ingress.mqtt.ingressClassName` |	Set the ingress class for EMQX Mgmt API | `nil`
 `ingress.mgmt.path` | Ingress path for EMQX Mgmt API | `/`
+`ingress.mgmt.pathType` | Ingress pathType for EMQX Mgmt API |	`ImplementationSpecific`
 `ingress.mgmt.hosts` | Ingress hosts for EMQX Mgmt API |	`api.emqx.local`
 `ingress.mgmt.tls` | Ingress tls for EMQX Mgmt API |	`[]`
 `ingress.mgmt.annotations` | Ingress annotations for EMQX Mgmt API |	`{}`
 `ingress.wss.enabled` |	Enable ingress for EMQX Mgmt API |	`false`
 `ingress.wss.ingressClassName` |	Set the ingress class for EMQX Mgmt API | `nil`
 `ingress.wss.path` | Ingress path for EMQX WSS |	`/`
+`ingress.wss.pathType` | Ingress pathType for EMQX WSS |	`ImplementationSpecific`
 `ingress.wss.hosts` | Ingress hosts for EMQX WSS |    `wss.emqx.local`
 `ingress.wss.tls` | Ingress tls for EMQX WSS |	`[]`
 `ingress.wss.annotations` | Ingress annotations for EMQX WSS |	`{}`
@@ -122,6 +125,7 @@ ingress:
       nginx.ingress.kubernetes.io/use-proxy-protocol: "false"
       nginx.ingress.kubernetes.io/proxy-protocol-header-timeout: "5s"
     path: /mqtt
+    pathType: ImplementationSpecific
     hosts:
     - myhost.example.com
     tls:

deploy/charts/emqx/templates/ingress.dashboard.yaml

@@ -34,7 +34,7 @@ spec:
       paths:
       - path: /
         {{- if (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }}
-        pathType: ImplementationSpecific
+        pathType: {{ .Values.ingress.dashboard.pathType | default "ImplementationSpecific" }}
         {{- end }}
         backend:
           {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}

deploy/charts/emqx/templates/ingress.mgmt.yaml

@@ -34,7 +34,7 @@ spec:
       paths:
       - path: {{ $.Values.ingress.mgmt.path | default "/" }}
         {{- if (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }}
-        pathType: ImplementationSpecific
+        pathType: {{ .Values.ingress.mgmt.pathType | default "ImplementationSpecific" }}
         {{- end }}
         backend:
           {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}

deploy/charts/emqx/templates/ingress.wss.yaml

@@ -34,7 +34,7 @@ spec:
       paths:
       - path: {{ $.Values.ingress.wss.path | default "/mqtt" }}
         {{- if (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }}
-        pathType: ImplementationSpecific
+        pathType: {{ .Values.ingress.wss.pathType | default "ImplementationSpecific" }}
         {{- end }}
         backend:
           {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}

deploy/charts/emqx/values.yaml

@@ -210,6 +210,7 @@ ingress:
       # kubernetes.io/ingress.class: nginx
       # kubernetes.io/tls-acme: "true"
     path: /
+    pathType: ImplementationSpecific
     hosts:
     - dashboard.emqx.local
     tls: []
@@ -221,6 +222,7 @@ ingress:
       # kubernetes.io/ingress.class: nginx
       # kubernetes.io/tls-acme: "true"
     path: /
+    pathType: ImplementationSpecific
     hosts:
     - api.emqx.local
     tls: []
@@ -243,6 +245,7 @@ ingress:
       # nginx.ingress.kubernetes.io/use-proxy-protocol: "false"
       # nginx.ingress.kubernetes.io/proxy-protocol-header-timeout: "5s"
     path: /mqtt
+    pathType: ImplementationSpecific
     # path: /wss(\/.*)?
     hosts:
     - wss.emqx.local

deploy/packages/emqx.service

@@ -11,10 +11,10 @@ Environment=HOME=/var/lib/emqx
 # Must use a 'bash' wrap for some OS
 # errno=13 'Permission denied'
 # Cannot create FIFO ... for writing
-ExecStart=bash /usr/bin/emqx start
+ExecStart=/bin/bash /usr/bin/emqx start
 
 LimitNOFILE=1048576
-ExecStop=bash /usr/bin/emqx stop
+ExecStop=/bin/bash /usr/bin/emqx stop
 Restart=on-failure
 
 # When clustered, give the peers enough time to get this node's 'DOWN' event

lib-ce/emqx_dashboard/src/emqx_dashboard_admin.erl

@@ -183,13 +183,21 @@ check(Username, Password) ->
     case lookup_user(Username) of
         [#mqtt_admin{password = PwdHash}] ->
             case is_valid_pwd(PwdHash, Password) of
-                true  -> ok;
+                true  ->
-                false -> {error, <<"Username/Password error">>}
+                    ok;
+                false ->
+                    ok = bad_login_penalty(),
+                    {error, <<"Username/Password error">>}
             end;
         [] ->
+            ok = bad_login_penalty(),
             {error, <<"Username/Password error">>}
     end.
 
+bad_login_penalty() ->
+    timer:sleep(2000),
+    ok.
+
 is_valid_pwd(<<Salt:4/binary, Hash/binary>>, Password) ->
     Hash =:= md5_hash(Salt, Password).
 

rebar.config

@@ -40,7 +40,7 @@
 {deps,
     [ {gpb, "4.11.2"} %% gpb only used to build, but not for release, pin it here to avoid fetching a wrong version due to rebar plugins scattered in all the deps
     , {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.2.0"}}}
-    , {eredis_cluster, {git, "https://github.com/emqx/eredis_cluster", {tag, "0.7.1"}}}
+    , {eredis_cluster, {git, "https://github.com/emqx/eredis_cluster", {tag, "0.7.2"}}}
     , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}}
     , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}
     , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}}

scripts/update-appup.sh

@@ -102,7 +102,7 @@ else
         git fetch "$REMOTE"
     fi
     git reset --hard
-    git clean -fdx
+    git clean -ffdx
     git checkout "${PREV_TAG}"
     make "$PROFILE"
     popd