Merge pull request #12872 from zmstone/0412-support-variform-for-client_attrs

0412 use variform for client_attrs

apps/emqx/src/emqx_channel.erl

@@ -251,7 +251,7 @@ init(
             MP -> MP
         end,
     ListenerId = emqx_listeners:listener_id(Type, Listener),
-    ClientInfo0 = set_peercert_infos(
+    ClientInfo = set_peercert_infos(
         Peercert,
         #{
             zone => Zone,
@@ -269,8 +269,6 @@ init(
         },
         Zone
     ),
-    AttrExtractionConfig = get_mqtt_conf(Zone, client_attrs_init),
-    ClientInfo = initialize_client_attrs_from_cert(AttrExtractionConfig, ClientInfo0, Peercert),
     {NClientInfo, NConnInfo} = take_ws_cookie(ClientInfo, ConnInfo),
     #channel{
         conninfo = NConnInfo,
@@ -1575,7 +1573,7 @@ enrich_client(ConnPkt, Channel = #channel{clientinfo = ClientInfo}) ->
             fun maybe_username_as_clientid/2,
             fun maybe_assign_clientid/2,
             %% attr init should happen after clientid and username assign
-            fun maybe_set_client_initial_attr/2
+            fun maybe_set_client_initial_attrs/2
         ],
         ConnPkt,
         ClientInfo
@@ -1587,47 +1585,6 @@ enrich_client(ConnPkt, Channel = #channel{clientinfo = ClientInfo}) ->
             {error, ReasonCode, Channel#channel{clientinfo = NClientInfo}}
     end.
 
-initialize_client_attrs_from_cert(
-    #{
-        extract_from := From,
-        extract_regexp := Regexp,
-        extract_as := AttrName
-    },
-    ClientInfo,
-    Peercert
-) when From =:= cn orelse From =:= dn ->
-    case extract_client_attr_from_cert(From, Regexp, Peercert) of
-        {ok, Value} ->
-            ?SLOG(
-                debug,
-                #{
-                    msg => "client_attr_init_from_cert",
-                    extracted_as => AttrName,
-                    extracted_value => Value
-                }
-            ),
-            ClientInfo#{client_attrs => #{AttrName => Value}};
-        _ ->
-            ClientInfo#{client_attrs => #{}}
-    end;
-initialize_client_attrs_from_cert(_, ClientInfo, _Peercert) ->
-    ClientInfo.
-
-extract_client_attr_from_cert(cn, Regexp, Peercert) ->
-    CN = esockd_peercert:common_name(Peercert),
-    re_extract(CN, Regexp);
-extract_client_attr_from_cert(dn, Regexp, Peercert) ->
-    DN = esockd_peercert:subject(Peercert),
-    re_extract(DN, Regexp).
-
-re_extract(Str, Regexp) when is_binary(Str) ->
-    case re:run(Str, Regexp, [{capture, all_but_first, list}]) of
-        {match, [_ | _] = List} -> {ok, iolist_to_binary(List)};
-        _ -> nomatch
-    end;
-re_extract(_NotStr, _Regexp) ->
-    ignored.
-
 set_username(
     #mqtt_packet_connect{username = Username},
     ClientInfo = #{username := undefined}
@@ -1668,75 +1625,50 @@ maybe_assign_clientid(#mqtt_packet_connect{clientid = <<>>}, ClientInfo) ->
 maybe_assign_clientid(#mqtt_packet_connect{clientid = ClientId}, ClientInfo) ->
     {ok, ClientInfo#{clientid => ClientId}}.
 
-maybe_set_client_initial_attr(ConnPkt, #{zone := Zone} = ClientInfo0) ->
+get_client_attrs_init_config(Zone) ->
-    Config = get_mqtt_conf(Zone, client_attrs_init),
+    get_mqtt_conf(Zone, client_attrs_init, []).
-    ClientInfo = initialize_client_attrs_from_user_property(Config, ConnPkt, ClientInfo0),
-    Attrs = maps:get(client_attrs, ClientInfo, #{}),
-    case extract_attr_from_clientinfo(Config, ClientInfo) of
-        {ok, Value} ->
-            #{extract_as := Name} = Config,
-            ?SLOG(
-                debug,
-                #{
-                    msg => "client_attr_init_from_clientinfo",
-                    extracted_as => Name,
-                    extracted_value => Value
-                }
-            ),
-            {ok, ClientInfo#{client_attrs => Attrs#{Name => Value}}};
-        _ ->
-            {ok, ClientInfo}
-    end.
 
-initialize_client_attrs_from_user_property(
+maybe_set_client_initial_attrs(ConnPkt, #{zone := Zone} = ClientInfo) ->
-    #{
+    Inits = get_client_attrs_init_config(Zone),
-        extract_from := user_property,
+    UserProperty = get_user_property_as_map(ConnPkt),
-        extract_as := PropertyKey
+    {ok, initialize_client_attrs(Inits, ClientInfo#{user_property => UserProperty})}.
-    },
-    ConnPkt,
-    ClientInfo
-) ->
-    case extract_client_attr_from_user_property(ConnPkt, PropertyKey) of
-        {ok, Value} ->
-            ?SLOG(
-                debug,
-                #{
-                    msg => "client_attr_init_from_user_property",
-                    extracted_as => PropertyKey,
-                    extracted_value => Value
-                }
-            ),
-            ClientInfo#{client_attrs => #{PropertyKey => Value}};
-        _ ->
-            ClientInfo
-    end;
-initialize_client_attrs_from_user_property(_, _ConnInfo, ClientInfo) ->
-    ClientInfo.
 
-extract_client_attr_from_user_property(
+initialize_client_attrs(Inits, ClientInfo) ->
-    #mqtt_packet_connect{properties = #{'User-Property' := UserProperty}}, PropertyKey
+    lists:foldl(
-) ->
+        fun(#{expression := Variform, set_as_attr := Name}, Acc) ->
-    case lists:keyfind(PropertyKey, 1, UserProperty) of
+            Attrs = maps:get(client_attrs, ClientInfo, #{}),
-        {_, Value} ->
+            case emqx_variform:render(Variform, ClientInfo) of
-            {ok, Value};
+                {ok, Value} ->
-        _ ->
+                    ?SLOG(
-            not_found
+                        debug,
-    end;
+                        #{
-extract_client_attr_from_user_property(_ConnPkt, _PropertyKey) ->
+                            msg => "client_attr_initialized",
-    ignored.
+                            set_as_attr => Name,
+                            attr_value => Value
+                        }
+                    ),
+                    Acc#{client_attrs => Attrs#{Name => Value}};
+                {error, Reason} ->
+                    ?SLOG(
+                        warning,
+                        #{
+                            msg => "client_attr_initialization_failed",
+                            reason => Reason
+                        }
+                    ),
+                    Acc
+            end
+        end,
+        ClientInfo,
+        Inits
+    ).
 
-extract_attr_from_clientinfo(#{extract_from := clientid, extract_regexp := Regexp}, #{
+get_user_property_as_map(#mqtt_packet_connect{properties = #{'User-Property' := UserProperty}}) when
-    clientid := ClientId
+    is_list(UserProperty)
-}) ->
-    re_extract(ClientId, Regexp);
-extract_attr_from_clientinfo(#{extract_from := username, extract_regexp := Regexp}, #{
-    username := Username
-}) when
-    Username =/= undefined
 ->
-    re_extract(Username, Regexp);
+    maps:from_list(UserProperty);
-extract_attr_from_clientinfo(_Config, _CLientInfo) ->
+get_user_property_as_map(_) ->
-    ignored.
+    #{}.
 
 fix_mountpoint(#{mountpoint := undefined} = ClientInfo) ->
     ClientInfo;

apps/emqx/src/emqx_schema.erl

@@ -1734,20 +1734,38 @@ fields(durable_storage) ->
     emqx_ds_schema:schema();
 fields("client_attrs_init") ->
     [
-        {extract_from,
+        {expression,
             sc(
-                hoconsc:enum([clientid, username, cn, dn, user_property]),
+                typerefl:alias("string", any()),
-                #{desc => ?DESC("client_attrs_init_extract_from")}
+                #{
+                    desc => ?DESC("client_attrs_init_expression"),
+                    converter => fun compile_variform/2
+                }
             )},
-        {extract_regexp, sc(binary(), #{desc => ?DESC("client_attrs_init_extract_regexp")})},
+        {set_as_attr,
-        {extract_as,
             sc(binary(), #{
-                default => <<"alias">>,
+                desc => ?DESC("client_attrs_init_set_as_attr"),
-                desc => ?DESC("client_attrs_init_extract_as"),
                 validator => fun restricted_string/1
             })}
     ].
 
+compile_variform(undefined, _Opts) ->
+    undefined;
+compile_variform(Expression, #{make_serializable := true}) ->
+    case is_binary(Expression) of
+        true ->
+            Expression;
+        false ->
+            emqx_variform:decompile(Expression)
+    end;
+compile_variform(Expression, _Opts) ->
+    case emqx_variform:compile(Expression) of
+        {ok, Compiled} ->
+            Compiled;
+        {error, Reason} ->
+            throw(#{expression => Expression, reason => Reason})
+    end.
+
 restricted_string(Str) ->
     case emqx_utils:is_restricted_str(Str) of
         true -> ok;
@@ -3552,9 +3570,9 @@ mqtt_general() ->
             )},
         {"client_attrs_init",
             sc(
-                hoconsc:union([disabled, ref("client_attrs_init")]),
+                hoconsc:array(ref("client_attrs_init")),
                 #{
-                    default => disabled,
+                    default => [],
                     desc => ?DESC("client_attrs_init")
                 }
             )}

apps/emqx/test/emqx_client_SUITE.erl

@@ -395,13 +395,14 @@ t_certdn_as_alias(_) ->
 
 test_cert_extraction_as_alias(Which) ->
     %% extract the first two chars
-    Re = <<"^(..).*$">>,
     ClientId = iolist_to_binary(["ClientIdFor_", atom_to_list(Which)]),
-    emqx_config:put_zone_conf(default, [mqtt, client_attrs_init], #{
+    {ok, Compiled} = emqx_variform:compile("substr(" ++ atom_to_list(Which) ++ ",0,2)"),
-        extract_from => Which,
+    emqx_config:put_zone_conf(default, [mqtt, client_attrs_init], [
-        extract_regexp => Re,
+        #{
-        extract_as => <<"alias">>
+            expression => Compiled,
-    }),
+            set_as_attr => <<"alias">>
+        }
+    ]),
     SslConf = emqx_common_test_helpers:client_mtls('tlsv1.2'),
     {ok, Client} = emqtt:start_link([
         {clientid, ClientId}, {port, 8883}, {ssl, true}, {ssl_opts, SslConf}
@@ -416,10 +417,13 @@ test_cert_extraction_as_alias(Which) ->
 
 t_client_attr_from_user_property(_Config) ->
     ClientId = atom_to_binary(?FUNCTION_NAME),
-    emqx_config:put_zone_conf(default, [mqtt, client_attrs_init], #{
+    {ok, Compiled} = emqx_variform:compile("user_property.group"),
-        extract_from => user_property,
+    emqx_config:put_zone_conf(default, [mqtt, client_attrs_init], [
-        extract_as => <<"group">>
+        #{
-    }),
+            expression => Compiled,
+            set_as_attr => <<"group">>
+        }
+    ]),
     SslConf = emqx_common_test_helpers:client_mtls('tlsv1.3'),
     {ok, Client} = emqtt:start_link([
         {clientid, ClientId},

apps/emqx/test/emqx_config_SUITE.erl

@@ -454,7 +454,7 @@ zone_global_defaults() ->
                 upgrade_qos => false,
                 use_username_as_clientid => false,
                 wildcard_subscription => true,
-                client_attrs_init => disabled
+                client_attrs_init => []
             },
         overload_protection =>
             #{

apps/emqx/test/emqx_listeners_SUITE.erl

@@ -150,11 +150,13 @@ t_client_attr_as_mountpoint(_Config) ->
         <<"limiter">> => #{},
         <<"mountpoint">> => <<"groups/${client_attrs.ns}/">>
     },
-    emqx_config:put_zone_conf(default, [mqtt, client_attrs_init], #{
+    {ok, Compiled} = emqx_variform:compile("nth(1,tokens(clientid,'-'))"),
-        extract_from => clientid,
+    emqx_config:put_zone_conf(default, [mqtt, client_attrs_init], [
-        extract_regexp => <<"^(.+)-.+$">>,
+        #{
-        extract_as => <<"ns">>
+            expression => Compiled,
-    }),
+            set_as_attr => <<"ns">>
+        }
+    ]),
     emqx_logger:set_log_level(debug),
     with_listener(tcp, attr_as_moutpoint, ListenerConf, fun() ->
         {ok, Client} = emqtt:start_link(#{
@@ -170,7 +172,7 @@ t_client_attr_as_mountpoint(_Config) ->
         ?assertMatch([_], emqx_router:match_routes(MatchTopic)),
         emqtt:stop(Client)
     end),
-    emqx_config:put_zone_conf(default, [mqtt, client_attrs_init], disabled),
+    emqx_config:put_zone_conf(default, [mqtt, client_attrs_init], []),
     ok.
 
 t_current_conns_tcp(_Config) ->

apps/emqx_auth/test/emqx_authz/emqx_authz_SUITE.erl

@@ -557,12 +557,14 @@ t_publish_last_will_testament_denied_topic(_Config) ->
 
 t_alias_prefix(_Config) ->
     {ok, _} = emqx_authz:update(?CMD_REPLACE, [?SOURCE_FILE_CLIENT_ATTR]),
-    ExtractSuffix = <<"^.*-(.*)$">>,
+    %% '^.*-(.*)$': extract the suffix after the last '-'
-    emqx_config:put_zone_conf(default, [mqtt, client_attrs_init], #{
+    {ok, Compiled} = emqx_variform:compile("concat(regex_extract(clientid,'^.*-(.*)$'))"),
-        extract_from => clientid,
+    emqx_config:put_zone_conf(default, [mqtt, client_attrs_init], [
-        extract_regexp => ExtractSuffix,
+        #{
-        extract_as => <<"alias">>
+            expression => Compiled,
-    }),
+            set_as_attr => <<"alias">>
+        }
+    ]),
     ClientId = <<"org1-name2">>,
     SubTopic = <<"name2/#">>,
     SubTopicNotAllowed = <<"name3/#">>,
@@ -572,7 +574,7 @@ t_alias_prefix(_Config) ->
     ?assertMatch({ok, _, [?RC_NOT_AUTHORIZED]}, emqtt:subscribe(C, SubTopicNotAllowed)),
     unlink(C),
     emqtt:stop(C),
-    emqx_config:put_zone_conf(default, [mqtt, client_attrs_init], disalbed),
+    emqx_config:put_zone_conf(default, [mqtt, client_attrs_init], []),
     ok.
 
 %% client is allowed by ACL to publish to its LWT topic, is connected,

apps/emqx_dashboard/src/emqx_dashboard_swagger.erl

@@ -661,7 +661,7 @@ trans_desc(Init, Hocon, Func, Name, Options) ->
             Spec1 = trans_label(Spec0, Hocon, Name, Options),
             case Spec1 of
                 #{description := _} -> Spec1;
-                _ -> Spec1#{description => <<Name/binary, " Description">>}
+                _ -> Spec1
             end
     end.
 

apps/emqx_rule_engine/src/emqx_rule_funcs.erl

@@ -202,7 +202,8 @@
 -export([
     md5/1,
     sha/1,
-    sha256/1
+    sha256/1,
+    hash/2
 ]).
 
 %% zip Funcs
@@ -710,24 +711,11 @@ map(Map = #{}) ->
 map(Data) ->
     error(badarg, [Data]).
 
-bin2hexstr(Bin) when is_binary(Bin) ->
+bin2hexstr(Bin) ->
-    emqx_utils:bin_to_hexstr(Bin, upper);
+    emqx_variform_bif:bin2hexstr(Bin).
-%% If Bin is a bitstring which is not divisible by 8, we pad it and then do the
-%% conversion
-bin2hexstr(Bin) when is_bitstring(Bin), (8 - (bit_size(Bin) rem 8)) >= 4 ->
-    PadSize = 8 - (bit_size(Bin) rem 8),
-    Padding = <<0:PadSize>>,
-    BinToConvert = <<Padding/bitstring, Bin/bitstring>>,
-    <<_FirstByte:8, HexStr/binary>> = emqx_utils:bin_to_hexstr(BinToConvert, upper),
-    HexStr;
-bin2hexstr(Bin) when is_bitstring(Bin) ->
-    PadSize = 8 - (bit_size(Bin) rem 8),
-    Padding = <<0:PadSize>>,
-    BinToConvert = <<Padding/bitstring, Bin/bitstring>>,
-    emqx_utils:bin_to_hexstr(BinToConvert, upper).
 
-hexstr2bin(Str) when is_binary(Str) ->
+hexstr2bin(Str) ->
-    emqx_utils:hexstr_to_bin(Str).
+    emqx_variform_bif:hexstr2bin(Str).
 
 %%------------------------------------------------------------------------------
 %% NULL Funcs
@@ -771,66 +759,66 @@ is_array(_) -> false.
 %% String Funcs
 %%------------------------------------------------------------------------------
 
-coalesce(List) -> emqx_variform_str:coalesce(List).
+coalesce(List) -> emqx_variform_bif:coalesce(List).
 
-coalesce(A, B) -> emqx_variform_str:coalesce(A, B).
+coalesce(A, B) -> emqx_variform_bif:coalesce(A, B).
 
-lower(S) -> emqx_variform_str:lower(S).
+lower(S) -> emqx_variform_bif:lower(S).
 
-ltrim(S) -> emqx_variform_str:ltrim(S).
+ltrim(S) -> emqx_variform_bif:ltrim(S).
 
-reverse(S) -> emqx_variform_str:reverse(S).
+reverse(S) -> emqx_variform_bif:reverse(S).
 
-rtrim(S) -> emqx_variform_str:rtrim(S).
+rtrim(S) -> emqx_variform_bif:rtrim(S).
 
-strlen(S) -> emqx_variform_str:strlen(S).
+strlen(S) -> emqx_variform_bif:strlen(S).
 
-substr(S, Start) -> emqx_variform_str:substr(S, Start).
+substr(S, Start) -> emqx_variform_bif:substr(S, Start).
 
-substr(S, Start, Length) -> emqx_variform_str:substr(S, Start, Length).
+substr(S, Start, Length) -> emqx_variform_bif:substr(S, Start, Length).
 
-trim(S) -> emqx_variform_str:trim(S).
+trim(S) -> emqx_variform_bif:trim(S).
 
-upper(S) -> emqx_variform_str:upper(S).
+upper(S) -> emqx_variform_bif:upper(S).
 
-split(S, P) -> emqx_variform_str:split(S, P).
+split(S, P) -> emqx_variform_bif:split(S, P).
 
-split(S, P, Position) -> emqx_variform_str:split(S, P, Position).
+split(S, P, Position) -> emqx_variform_bif:split(S, P, Position).
 
-tokens(S, Separators) -> emqx_variform_str:tokens(S, Separators).
+tokens(S, Separators) -> emqx_variform_bif:tokens(S, Separators).
 
-tokens(S, Separators, NoCRLF) -> emqx_variform_str:tokens(S, Separators, NoCRLF).
+tokens(S, Separators, NoCRLF) -> emqx_variform_bif:tokens(S, Separators, NoCRLF).
 
-concat(S1, S2) -> emqx_variform_str:concat(S1, S2).
+concat(S1, S2) -> emqx_variform_bif:concat(S1, S2).
 
-concat(List) -> emqx_variform_str:concat(List).
+concat(List) -> emqx_variform_bif:concat(List).
 
-sprintf_s(Format, Args) -> emqx_variform_str:sprintf_s(Format, Args).
+sprintf_s(Format, Args) -> emqx_variform_bif:sprintf_s(Format, Args).
 
-pad(S, Len) -> emqx_variform_str:pad(S, Len).
+pad(S, Len) -> emqx_variform_bif:pad(S, Len).
 
-pad(S, Len, Position) -> emqx_variform_str:pad(S, Len, Position).
+pad(S, Len, Position) -> emqx_variform_bif:pad(S, Len, Position).
 
-pad(S, Len, Position, Char) -> emqx_variform_str:pad(S, Len, Position, Char).
+pad(S, Len, Position, Char) -> emqx_variform_bif:pad(S, Len, Position, Char).
 
-replace(SrcStr, Pattern, RepStr) -> emqx_variform_str:replace(SrcStr, Pattern, RepStr).
+replace(SrcStr, Pattern, RepStr) -> emqx_variform_bif:replace(SrcStr, Pattern, RepStr).
 
 replace(SrcStr, Pattern, RepStr, Position) ->
-    emqx_variform_str:replace(SrcStr, Pattern, RepStr, Position).
+    emqx_variform_bif:replace(SrcStr, Pattern, RepStr, Position).
 
-regex_match(Str, RE) -> emqx_variform_str:regex_match(Str, RE).
+regex_match(Str, RE) -> emqx_variform_bif:regex_match(Str, RE).
 
-regex_replace(SrcStr, RE, RepStr) -> emqx_variform_str:regex_replace(SrcStr, RE, RepStr).
+regex_replace(SrcStr, RE, RepStr) -> emqx_variform_bif:regex_replace(SrcStr, RE, RepStr).
 
-ascii(Char) -> emqx_variform_str:ascii(Char).
+ascii(Char) -> emqx_variform_bif:ascii(Char).
 
-find(S, P) -> emqx_variform_str:find(S, P).
+find(S, P) -> emqx_variform_bif:find(S, P).
 
-find(S, P, Position) -> emqx_variform_str:find(S, P, Position).
+find(S, P, Position) -> emqx_variform_bif:find(S, P, Position).
 
-join_to_string(Str) -> emqx_variform_str:join_to_string(Str).
+join_to_string(Str) -> emqx_variform_bif:join_to_string(Str).
 
-join_to_string(Sep, List) -> emqx_variform_str:join_to_string(Sep, List).
+join_to_string(Sep, List) -> emqx_variform_bif:join_to_string(Sep, List).
 
 join_to_sql_values_string(List) ->
     QuotedList =
@@ -878,7 +866,7 @@ jq(FilterProgram, JSONBin) ->
         ])
     ).
 
-unescape(Str) -> emqx_variform_str:unescape(Str).
+unescape(Str) -> emqx_variform_bif:unescape(Str).
 
 %%------------------------------------------------------------------------------
 %% Array Funcs
@@ -1001,7 +989,7 @@ sha256(S) when is_binary(S) ->
     hash(sha256, S).
 
 hash(Type, Data) ->
-    emqx_utils:bin_to_hexstr(crypto:hash(Type, Data), lower).
+    emqx_variform_bif:hash(Type, Data).
 
 %%------------------------------------------------------------------------------
 %% gzip Funcs

apps/emqx_utils/src/emqx_variform.erl

@@ -28,14 +28,35 @@
     erase_allowed_module/1,
     erase_allowed_modules/1
 ]).
+
 -export([render/2, render/3]).
+-export([compile/1, decompile/1]).
+
+-export_type([compiled/0]).
+
+-type compiled() :: #{expr := string(), form := term()}.
+-define(BIF_MOD, emqx_variform_bif).
+-define(IS_ALLOWED_MOD(M),
+    (M =:= ?BIF_MOD orelse
+        M =:= lists orelse
+        M =:= maps)
+).
+
+-define(COALESCE_BADARG,
+    throw(#{
+        reason => coalesce_badarg,
+        explain =>
+            "must be an array, or a call to a function which returns an array, "
+            "for example: coalesce([a,b,c]) or coalesce(tokens(var,','))"
+    })
+).
 
 %% @doc Render a variform expression with bindings.
 %% A variform expression is a template string which supports variable substitution
 %% and function calls.
 %%
 %% The function calls are in the form of `module.function(arg1, arg2, ...)` where `module`
-%% is optional, and if not provided, the function is assumed to be in the `emqx_variform_str` module.
+%% is optional, and if not provided, the function is assumed to be in the `emqx_variform_bif` module.
 %% Both module and function must be existing atoms, and only whitelisted functions are allowed.
 %%
 %% A function arg can be a constant string or a number.
@@ -49,18 +70,54 @@
 %%
 %% For unresolved variables, empty string (but not "undefined") is used.
 %% In case of runtime exeption, an error is returned.
+%% In case of unbound variable is referenced, error is returned.
 -spec render(string(), map()) -> {ok, binary()} | {error, term()}.
 render(Expression, Bindings) ->
     render(Expression, Bindings, #{}).
 
-render(Expression, Bindings, Opts) when is_binary(Expression) ->
+render(#{form := Form}, Bindings, Opts) ->
-    render(unicode:characters_to_list(Expression), Bindings, Opts);
+    eval_as_string(Form, Bindings, Opts);
 render(Expression, Bindings, Opts) ->
+    case compile(Expression) of
+        {ok, Compiled} ->
+            render(Compiled, Bindings, Opts);
+        {error, Reason} ->
+            {error, Reason}
+    end.
+
+eval_as_string(Expr, Bindings, _Opts) ->
+    try
+        {ok, return_str(eval(Expr, Bindings, #{}))}
+    catch
+        throw:Reason ->
+            {error, Reason};
+        C:E:S ->
+            {error, #{exception => C, reason => E, stack_trace => S}}
+    end.
+
+%% Force the expression to return binary string.
+return_str(Str) when is_binary(Str) -> Str;
+return_str(Num) when is_integer(Num) -> integer_to_binary(Num);
+return_str(Num) when is_float(Num) -> float_to_binary(Num, [{decimals, 10}, compact]);
+return_str(Other) ->
+    throw(#{
+        reason => bad_return,
+        expected => string,
+        got => Other
+    }).
+
+%% @doc Compile varifom expression.
+-spec compile(string() | binary() | compiled()) -> {ok, compiled()} | {error, any()}.
+compile(#{form := _} = Compiled) ->
+    {ok, Compiled};
+compile(Expression) when is_binary(Expression) ->
+    compile(unicode:characters_to_list(Expression));
+compile(Expression) ->
     case emqx_variform_scan:string(Expression) of
         {ok, Tokens, _Line} ->
             case emqx_variform_parser:parse(Tokens) of
-                {ok, Expr} ->
+                {ok, Form} ->
-                    eval_as_string(Expr, Bindings, Opts);
+                    {ok, #{expr => Expression, form => Form}};
                 {error, {_, emqx_variform_parser, Msg}} ->
                     %% syntax error
                     {error, lists:flatten(Msg)};
@@ -71,40 +128,59 @@ render(Expression, Bindings, Opts) ->
             {error, Reason}
     end.
 
-eval_as_string(Expr, Bindings, _Opts) ->
+decompile(#{expr := Expression}) ->
-    try
+    Expression;
-        {ok, str(eval(Expr, Bindings))}
+decompile(Expression) ->
-    catch
+    Expression.
-        throw:Reason ->
-            {error, Reason};
-        C:E:S ->
-            {error, #{exception => C, reason => E, stack_trace => S}}
-    end.
 
-eval({str, Str}, _Bindings) ->
+eval({str, Str}, _Bindings, _Opts) ->
-    str(Str);
+    unicode:characters_to_binary(Str);
-eval({integer, Num}, _Bindings) ->
+eval({integer, Num}, _Bindings, _Opts) ->
     Num;
-eval({float, Num}, _Bindings) ->
+eval({float, Num}, _Bindings, _Opts) ->
     Num;
-eval({array, Args}, Bindings) ->
+eval({array, Args}, Bindings, Opts) ->
-    eval(Args, Bindings);
+    eval_loop(Args, Bindings, Opts);
-eval({call, FuncNameStr, Args}, Bindings) ->
+eval({call, FuncNameStr, Args}, Bindings, Opts) ->
     {Mod, Fun} = resolve_func_name(FuncNameStr),
     ok = assert_func_exported(Mod, Fun, length(Args)),
-    call(Mod, Fun, eval(Args, Bindings));
+    case {Mod, Fun} of
-eval({var, VarName}, Bindings) ->
+        {?BIF_MOD, coalesce} ->
-    resolve_var_value(VarName, Bindings);
+            eval_coalesce(Args, Bindings, Opts);
-eval([Arg | Args], Bindings) ->
+        _ ->
-    [eval(Arg, Bindings) | eval(Args, Bindings)];
+            call(Mod, Fun, eval_loop(Args, Bindings, Opts))
-eval([], _Bindings) ->
+    end;
-    [].
+eval({var, VarName}, Bindings, Opts) ->
+    resolve_var_value(VarName, Bindings, Opts).
+
+eval_loop([], _, _) -> [];
+eval_loop([H | T], Bindings, Opts) -> [eval(H, Bindings, Opts) | eval_loop(T, Bindings, Opts)].
+
+%% coalesce treats var_unbound exception as empty string ''
+eval_coalesce([{array, Args}], Bindings, Opts) ->
+    NewArgs = [lists:map(fun(Arg) -> try_eval(Arg, Bindings, Opts) end, Args)],
+    call(?BIF_MOD, coalesce, NewArgs);
+eval_coalesce([Arg], Bindings, Opts) ->
+    case try_eval(Arg, Bindings, Opts) of
+        List when is_list(List) ->
+            call(?BIF_MOD, coalesce, List);
+        <<>> ->
+            <<>>;
+        _ ->
+            ?COALESCE_BADARG
+    end;
+eval_coalesce(_Args, _Bindings, _Opts) ->
+    ?COALESCE_BADARG.
+
+try_eval(Arg, Bindings, Opts) ->
+    try
+        eval(Arg, Bindings, Opts)
+    catch
+        throw:#{reason := var_unbound} ->
+            <<>>
+    end.
 
 %% Some functions accept arbitrary number of arguments but implemented as /1.
-call(emqx_variform_str, concat, Args) ->
-    str(emqx_variform_str:concat(Args));
-call(emqx_variform_str, coalesce, Args) ->
-    str(emqx_variform_str:coalesce(Args));
 call(Mod, Fun, Args) ->
     erlang:apply(Mod, Fun, Args).
 
@@ -144,23 +220,23 @@ resolve_func_name(FuncNameStr) ->
                             function => Fun
                         })
                 end,
-            {emqx_variform_str, FuncName};
+            {?BIF_MOD, FuncName};
         _ ->
             throw(#{reason => invalid_function_reference, function => FuncNameStr})
     end.
 
-resolve_var_value(VarName, Bindings) ->
+%% _Opts can be extended in the future. For example, unbound var as 'undfeined'
+resolve_var_value(VarName, Bindings, _Opts) ->
     case emqx_template:lookup_var(split(VarName), Bindings) of
         {ok, Value} ->
             Value;
         {error, _Reason} ->
-            <<>>
+            throw(#{
+                var_name => VarName,
+                reason => var_unbound
+            })
     end.
 
-assert_func_exported(emqx_variform_str, concat, _Arity) ->
-    ok;
-assert_func_exported(emqx_variform_str, coalesce, _Arity) ->
-    ok;
 assert_func_exported(Mod, Fun, Arity) ->
     ok = try_load(Mod),
     case erlang:function_exported(Mod, Fun, Arity) of
@@ -187,7 +263,7 @@ try_load(Mod) ->
             ok
     end.
 
-assert_module_allowed(emqx_variform_str) ->
+assert_module_allowed(Mod) when ?IS_ALLOWED_MOD(Mod) ->
     ok;
 assert_module_allowed(Mod) ->
     Allowed = get_allowed_modules(),
@@ -220,8 +296,5 @@ erase_allowed_modules(Modules) when is_list(Modules) ->
 get_allowed_modules() ->
     persistent_term:get({emqx_variform, allowed_modules}, []).
 
-str(Value) ->
-    emqx_utils_conv:bin(Value).
-
 split(VarName) ->
     lists:map(fun erlang:iolist_to_binary/1, string:tokens(VarName, ".")).

apps/emqx_utils/src/emqx_variform_bif.erl

@@ -14,13 +14,11 @@
 %% limitations under the License.
 %%--------------------------------------------------------------------
 
-%% Predefined functions string templating
+%% Predefined functions for variform expressions.
--module(emqx_variform_str).
+-module(emqx_variform_bif).
 
 %% String Funcs
 -export([
-    coalesce/1,
-    coalesce/2,
     lower/1,
     ltrim/1,
     ltrim/2,
@@ -47,15 +45,37 @@
     replace/4,
     regex_match/2,
     regex_replace/3,
+    regex_extract/2,
     ascii/1,
     find/2,
     find/3,
     join_to_string/1,
     join_to_string/2,
     unescape/1,
-    nth/2
+    any_to_str/1
 ]).
 
+%% Array functions
+-export([nth/2]).
+
+%% Control functions
+-export([coalesce/1, coalesce/2]).
+
+%% Random functions
+-export([rand_str/1, rand_int/1]).
+
+%% Schema-less encod/decode
+-export([
+    bin2hexstr/1,
+    hexstr2bin/1,
+    int2hexstr/1,
+    base64_encode/1,
+    base64_decode/1
+]).
+
+%% Hash functions
+-export([hash/2, hash_to_range/3, map_to_range/3]).
+
 -define(IS_EMPTY(X), (X =:= <<>> orelse X =:= "" orelse X =:= undefined)).
 
 %%------------------------------------------------------------------------------
@@ -143,8 +163,10 @@ tokens(S, Separators, <<"nocrlf">>) ->
 concat(S1, S2) ->
     concat([S1, S2]).
 
+%% @doc Concatenate a list of strings.
+%% NOTE: it converts non-string elements to Erlang term literals for backward compatibility
 concat(List) ->
-    unicode:characters_to_binary(lists:map(fun str/1, List), unicode).
+    unicode:characters_to_binary(lists:map(fun any_to_str/1, List), unicode).
 
 sprintf_s(Format, Args) when is_list(Args) ->
     erlang:iolist_to_binary(io_lib:format(binary_to_list(Format), Args)).
@@ -190,6 +212,22 @@ regex_match(Str, RE) ->
 regex_replace(SrcStr, RE, RepStr) ->
     re:replace(SrcStr, RE, RepStr, [global, {return, binary}]).
 
+%% @doc Searches the string Str for patterns specified by Regexp.
+%% If matches are found, it returns a list of all captured groups from these matches.
+%% If no matches are found or there are no groups captured, it returns an empty list.
+%% This function can be used to extract parts of a string based on a regular expression,
+%% excluding the complete match itself.
+%% Examples:
+%%  ("Number: 12345", "(\\d+)") -> [<<"12345">>]
+%%  ("Hello, world!", "(\\w+)") -> [<<"Hello">>, <<"world">>]
+%%  ("No numbers here!", "(\\d+)") -> []
+%%  ("Date: 2021-05-20", "(\\d{4})-(\\d{2})-(\\d{2})") -> [<<"2021">>, <<"05">>, <<"20">>]
+regex_extract(Str, Regexp) ->
+    case re:run(Str, Regexp, [{capture, all_but_first, list}]) of
+        {match, [_ | _] = L} -> lists:map(fun erlang:iolist_to_binary/1, L);
+        _ -> []
+    end.
+
 ascii(Char) when is_binary(Char) ->
     [FirstC | _] = binary_to_list(Char),
     FirstC.
@@ -212,7 +250,7 @@ join_to_string(List) when is_list(List) ->
     join_to_string(<<", ">>, List).
 
 join_to_string(Sep, List) when is_list(List), is_binary(Sep) ->
-    iolist_to_binary(lists:join(Sep, [str(Item) || Item <- List])).
+    iolist_to_binary(lists:join(Sep, [any_to_str(Item) || Item <- List])).
 
 unescape(Bin) when is_binary(Bin) ->
     UnicodeList = unicode:characters_to_list(Bin, utf8),
@@ -364,5 +402,124 @@ is_hex_digit(_) -> false.
 %% Data Type Conversion Funcs
 %%------------------------------------------------------------------------------
 
-str(Data) ->
+any_to_str(Data) ->
     emqx_utils_conv:bin(Data).
+
+%%------------------------------------------------------------------------------
+%% Random functions
+%%------------------------------------------------------------------------------
+
+%% @doc Make a random string with urlsafe-base64 charset.
+rand_str(Length) when is_integer(Length) andalso Length > 0 ->
+    RawBytes = erlang:ceil((Length * 3) / 4),
+    RandomData = rand:bytes(RawBytes),
+    urlsafe(binary:part(base64_encode(RandomData), 0, Length));
+rand_str(_) ->
+    throw(#{reason => badarg, function => ?FUNCTION_NAME}).
+
+%% @doc Make a random integer in the range `[1, N]`.
+rand_int(N) when is_integer(N) andalso N >= 1 ->
+    rand:uniform(N);
+rand_int(N) ->
+    throw(#{reason => badarg, function => ?FUNCTION_NAME, expected => "positive integer", got => N}).
+
+%% TODO: call base64:encode(Bin, #{mode => urlsafe, padding => false})
+%% when oldest OTP to support is 26 or newer.
+urlsafe(Str0) ->
+    Str = replace(Str0, <<"+">>, <<"-">>),
+    replace(Str, <<"/">>, <<"_">>).
+
+%%------------------------------------------------------------------------------
+%% Data encoding
+%%------------------------------------------------------------------------------
+
+%% @doc Encode an integer to hex string. e.g. 15 as 'f'
+int2hexstr(Int) ->
+    erlang:integer_to_binary(Int, 16).
+
+%% @doc Encode bytes in hex string format.
+bin2hexstr(Bin) when is_binary(Bin) ->
+    emqx_utils:bin_to_hexstr(Bin, upper);
+%% If Bin is a bitstring which is not divisible by 8, we pad it and then do the
+%% conversion
+bin2hexstr(Bin) when is_bitstring(Bin), (8 - (bit_size(Bin) rem 8)) >= 4 ->
+    PadSize = 8 - (bit_size(Bin) rem 8),
+    Padding = <<0:PadSize>>,
+    BinToConvert = <<Padding/bitstring, Bin/bitstring>>,
+    <<_FirstByte:8, HexStr/binary>> = emqx_utils:bin_to_hexstr(BinToConvert, upper),
+    HexStr;
+bin2hexstr(Bin) when is_bitstring(Bin) ->
+    PadSize = 8 - (bit_size(Bin) rem 8),
+    Padding = <<0:PadSize>>,
+    BinToConvert = <<Padding/bitstring, Bin/bitstring>>,
+    emqx_utils:bin_to_hexstr(BinToConvert, upper).
+
+%% @doc Decode hex string into its original bytes.
+hexstr2bin(Str) when is_binary(Str) ->
+    emqx_utils:hexstr_to_bin(Str).
+
+%% @doc Encode any bytes to base64.
+base64_encode(Bin) ->
+    base64:encode(Bin).
+
+%% @doc Decode base64 encoded string.
+base64_decode(Bin) ->
+    base64:decode(Bin).
+
+%%------------------------------------------------------------------------------
+%% Hash functions
+%%------------------------------------------------------------------------------
+
+%% @doc Hash with all available algorithm provided by crypto module.
+%% Return hex format string.
+%% - md4 | md5
+%% - sha (sha1)
+%% - sha224 | sha256 | sha384 | sha512
+%% - sha3_224 | sha3_256 | sha3_384 | sha3_512
+%% - shake128 | shake256
+%% - blake2b | blake2s
+hash(<<"sha1">>, Bin) ->
+    hash(sha, Bin);
+hash(Algorithm, Bin) when is_binary(Algorithm) ->
+    Type =
+        try
+            binary_to_existing_atom(Algorithm)
+        catch
+            _:_ ->
+                throw(#{
+                    reason => unknown_hash_algorithm,
+                    algorithm => Algorithm
+                })
+        end,
+    hash(Type, Bin);
+hash(Type, Bin) when is_atom(Type) ->
+    %% lower is for backward compatibility
+    emqx_utils:bin_to_hexstr(crypto:hash(Type, Bin), lower).
+
+%% @doc Hash binary data to an integer within a specified range [Min, Max]
+hash_to_range(Bin, Min, Max) when
+    is_binary(Bin) andalso
+        size(Bin) > 0 andalso
+        is_integer(Min) andalso
+        is_integer(Max) andalso
+        Min =< Max
+->
+    Hash = hash(sha256, Bin),
+    HashNum = binary_to_integer(Hash, 16),
+    map_to_range(HashNum, Min, Max);
+hash_to_range(_, _, _) ->
+    throw(#{reason => badarg, function => ?FUNCTION_NAME}).
+
+map_to_range(Bin, Min, Max) when is_binary(Bin) andalso size(Bin) > 0 ->
+    HashNum = binary:decode_unsigned(Bin),
+    map_to_range(HashNum, Min, Max);
+map_to_range(Int, Min, Max) when
+    is_integer(Int) andalso
+        is_integer(Min) andalso
+        is_integer(Max) andalso
+        Min =< Max
+->
+    Range = Max - Min + 1,
+    Min + (Int rem Range);
+map_to_range(_, _, _) ->
+    throw(#{reason => badarg, function => ?FUNCTION_NAME}).

apps/emqx_utils/test/emqx_variform_bif_tests.erl

@@ -0,0 +1,74 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+%% Most of the functions are tested as rule-engine string funcs
+-module(emqx_variform_bif_tests).
+
+-compile(export_all).
+-compile(nowarn_export_all).
+
+-include_lib("eunit/include/eunit.hrl").
+
+regex_extract_test_() ->
+    [
+        ?_assertEqual([<<"12345">>], regex_extract("Order number: 12345", "(\\d+)")),
+        ?_assertEqual(
+            [<<"Hello">>, <<"world">>], regex_extract("Hello, world!", "(\\w+).*\s(\\w+)")
+        ),
+        ?_assertEqual([], regex_extract("No numbers here!", "(\\d+)")),
+        ?_assertEqual(
+            [<<"2021">>, <<"05">>, <<"20">>],
+            regex_extract("Date: 2021-05-20", "(\\d{4})-(\\d{2})-(\\d{2})")
+        ),
+        ?_assertEqual([<<"Hello">>], regex_extract("Hello, world!", "(Hello)")),
+        ?_assertEqual(
+            [<<"12">>, <<"34">>], regex_extract("Items: 12, Price: 34", "(\\d+).*\s(\\d+)")
+        ),
+        ?_assertEqual(
+            [<<"john.doe@example.com">>],
+            regex_extract("Contact: john.doe@example.com", "([\\w\\.]+@[\\w\\.]+)")
+        ),
+        ?_assertEqual([], regex_extract("Just some text, nothing more.", "([A-Z]\\d{3})")),
+        ?_assertEqual(
+            [<<"admin">>, <<"1234">>],
+            regex_extract("User: admin, Pass: 1234", "User: (\\w+), Pass: (\\d+)")
+        ),
+        ?_assertEqual([], regex_extract("", "(\\d+)")),
+        ?_assertEqual([], regex_extract("$$$###!!!", "(\\d+)")),
+        ?_assertEqual([<<"23.1">>], regex_extract("Erlang 23.1 version", "(\\d+\\.\\d+)")),
+        ?_assertEqual(
+            [<<"192.168.1.1">>],
+            regex_extract("Server IP: 192.168.1.1 at port 8080", "(\\d+\\.\\d+\\.\\d+\\.\\d+)")
+        )
+    ].
+
+regex_extract(Str, RegEx) ->
+    emqx_variform_bif:regex_extract(Str, RegEx).
+
+rand_str_test() ->
+    ?assertEqual(3, size(emqx_variform_bif:rand_str(3))),
+    ?assertThrow(#{reason := badarg}, size(emqx_variform_bif:rand_str(0))).
+
+rand_int_test() ->
+    N = emqx_variform_bif:rand_int(10),
+    ?assert(N =< 10 andalso N >= 1),
+    ?assertThrow(#{reason := badarg}, emqx_variform_bif:rand_int(0)),
+    ?assertThrow(#{reason := badarg}, emqx_variform_bif:rand_int(-1)).
+
+base64_encode_decode_test() ->
+    RandBytes = crypto:strong_rand_bytes(100),
+    Encoded = emqx_variform_bif:base64_encode(RandBytes),
+    ?assertEqual(RandBytes, emqx_variform_bif:base64_decode(Encoded)).

apps/emqx_utils/test/emqx_variform_tests.erl

@@ -27,14 +27,16 @@ redner_test_() ->
     [
         {"direct var reference", fun() -> ?assertEqual({ok, <<"1">>}, render("a", #{a => 1})) end},
         {"concat strings", fun() ->
-            ?assertEqual({ok, <<"a,b">>}, render("concat('a',',','b')", #{}))
+            ?assertEqual({ok, <<"a,b">>}, render("concat(['a',',','b'])", #{}))
+        end},
+        {"concat empty string", fun() ->
+            ?assertEqual({ok, <<"">>}, render("concat([''])", #{}))
         end},
-        {"concat empty string", fun() -> ?assertEqual({ok, <<"">>}, render("concat('')", #{})) end},
         {"tokens 1st", fun() ->
             ?assertEqual({ok, <<"a">>}, render("nth(1,tokens(var, ','))", #{var => <<"a,b">>}))
         end},
-        {"unknown var as empty str", fun() ->
+        {"unknown var return error", fun() ->
-            ?assertEqual({ok, <<>>}, render("var", #{}))
+            ?assertMatch({error, #{reason := var_unbound}}, render("var", #{}))
         end},
         {"out of range nth index", fun() ->
             ?assertEqual({ok, <<>>}, render("nth(2, tokens(var, ','))", #{var => <<"a">>}))
@@ -97,7 +99,7 @@ unknown_func_test_() ->
         {"unknown function in a known module", fun() ->
             ?assertMatch(
                 {error, #{reason := unknown_variform_function}},
-                render("emqx_variform_str.nonexistingatom__(a)", #{})
+                render("emqx_variform_bif.nonexistingatom__(a)", #{})
             )
         end},
         {"invalid func reference", fun() ->
@@ -133,19 +135,39 @@ inject_allowed_module_test() ->
 
 coalesce_test_() ->
     [
-        {"coalesce first", fun() ->
+        {"first", fun() ->
-            ?assertEqual({ok, <<"a">>}, render("coalesce('a','b')", #{}))
+            ?assertEqual({ok, <<"a">>}, render("coalesce(['a','b'])", #{}))
         end},
-        {"coalesce second", fun() ->
+        {"second", fun() ->
-            ?assertEqual({ok, <<"b">>}, render("coalesce('', 'b')", #{}))
+            ?assertEqual({ok, <<"b">>}, render("coalesce(['', 'b'])", #{}))
         end},
-        {"coalesce first var", fun() ->
+        {"first var", fun() ->
-            ?assertEqual({ok, <<"a">>}, render("coalesce(a,b)", #{a => <<"a">>, b => <<"b">>}))
+            ?assertEqual({ok, <<"a">>}, render("coalesce([a,b])", #{a => <<"a">>, b => <<"b">>}))
         end},
-        {"coalesce second var", fun() ->
+        {"second var", fun() ->
-            ?assertEqual({ok, <<"b">>}, render("coalesce(a,b)", #{b => <<"b">>}))
+            ?assertEqual({ok, <<"b">>}, render("coalesce([a,b])", #{b => <<"b">>}))
         end},
-        {"coalesce empty", fun() -> ?assertEqual({ok, <<>>}, render("coalesce(a,b)", #{})) end}
+        {"empty", fun() -> ?assertEqual({ok, <<>>}, render("coalesce([a,b])", #{})) end},
+        {"arg from other func", fun() ->
+            ?assertEqual({ok, <<"b">>}, render("coalesce(tokens(a,','))", #{a => <<",,b,c">>}))
+        end},
+        {"var unbound", fun() -> ?assertEqual({ok, <<>>}, render("coalesce(a)", #{})) end},
+        {"var unbound in call", fun() ->
+            ?assertEqual({ok, <<>>}, render("coalesce(concat(a))", #{}))
+        end},
+        {"var unbound in calls", fun() ->
+            ?assertEqual({ok, <<"c">>}, render("coalesce([any_to_str(a),any_to_str(b),'c'])", #{}))
+        end},
+        {"badarg", fun() ->
+            ?assertMatch(
+                {error, #{reason := coalesce_badarg}}, render("coalesce(a,b)", #{a => 1, b => 2})
+            )
+        end},
+        {"badarg from return", fun() ->
+            ?assertMatch(
+                {error, #{reason := coalesce_badarg}}, render("coalesce(any_to_str(a))", #{a => 1})
+            )
+        end}
     ].
 
 syntax_error_test_() ->
@@ -160,3 +182,39 @@ syntax_error_test_() ->
 
 render(Expression, Bindings) ->
     emqx_variform:render(Expression, Bindings).
+
+hash_pick_test() ->
+    lists:foreach(
+        fun(_) ->
+            {ok, Res} = render("nth(hash_to_range(rand_str(10),1,5),[1,2,3,4,5])", #{}),
+            ?assert(Res >= <<"1">> andalso Res =< <<"5">>)
+        end,
+        lists:seq(1, 100)
+    ).
+
+map_to_range_pick_test() ->
+    lists:foreach(
+        fun(_) ->
+            {ok, Res} = render("nth(map_to_range(rand_str(10),1,5),[1,2,3,4,5])", #{}),
+            ?assert(Res >= <<"1">> andalso Res =< <<"5">>)
+        end,
+        lists:seq(1, 100)
+    ).
+
+-define(ASSERT_BADARG(FUNC, ARGS),
+    ?_assertEqual(
+        {error, #{reason => badarg, function => FUNC}},
+        render(atom_to_list(FUNC) ++ ARGS, #{})
+    )
+).
+
+to_range_badarg_test_() ->
+    [
+        ?ASSERT_BADARG(hash_to_range, "(1,1,2)"),
+        ?ASSERT_BADARG(hash_to_range, "('',1,2)"),
+        ?ASSERT_BADARG(hash_to_range, "('a','1',2)"),
+        ?ASSERT_BADARG(hash_to_range, "('a',2,1)"),
+        ?ASSERT_BADARG(map_to_range, "('',1,2)"),
+        ?ASSERT_BADARG(map_to_range, "('a','1',2)"),
+        ?ASSERT_BADARG(map_to_range, "('a',2,1)")
+    ].

changes/ce/feat-12872.en.md

@@ -7,8 +7,8 @@ an MQTT connection.
 
 ### Initialization of `client_attrs`
 
-- The `client_attrs` field can be initially populated based on the configuration from one of the
+- The `client_attrs` fields can be initially populated from one of the
-  following sources:
+  following `clientinfo` fields:
   - `cn`: The common name from the TLS client's certificate.
   - `dn`: The distinguished name from the TLS client's certificate, that is, the certificate "Subject".
   - `clientid`: The MQTT client ID provided by the client.

rel/i18n/emqx_schema.hocon

@@ -1575,47 +1575,37 @@ client_attrs_init {
   label: "Client Attributes Initialization"
   desc: """~
     Specify how to initialize client attributes.
-    One initial client attribute can be initialized as `client_attrs.NAME`,
+    Each client attribute can be initialized as `client_attrs.{NAME}`,
-    where `NAME` is the name of the attribute specified in the config `extract_as`.
+    where `{NAME}` is the name of the attribute specified in the config field `set_as_attr`.
     The initialized client attribute will be stored in the `client_attrs` property with the specified name,
     and can be used as a placeholder in a template for authentication and authorization.
-    For example, use `${client_attrs.alias}` to render an HTTP POST body when `extract_as = alias`,
+    For example, use `${client_attrs.alias}` to render an HTTP POST body when `set_as_attr = alias`,
     or render listener config `moutpoint = devices/${client_attrs.alias}/` to initialize a per-client topic namespace."""
 }
 
-client_attrs_init_extract_from {
+client_attrs_init_expression {
-  label: "Client Property to Extract Attribute"
-  desc: """~
-    Specify from which client property the client attribute should be extracted.
-
-    Supported values:
-    - `clientid`: Extract from the client ID.
-    - `username`: Extract from the username.
-    - `cn`: Extract from the Common Name (CN) field of the client certificate.
-    - `dn`: Extract from the Distinguished Name (DN) field of the client certificate.
-    - `user_property`: Extract from the user property sent in the MQTT v5 `CONNECT` packet.
-      In this case, `extract_regexp` is not applicable, and `extract_as` should be the user property key.
-
-    NOTE: this extraction happens **after** `clientid` or `username` is initialized
-    from `peer_cert_as_clientid` or `peer_cert_as_username` config."""
-}
-
-client_attrs_init_extract_regexp {
   label: "Client Attribute Extraction Regular Expression"
   desc: """~
-    The regular expression to extract a client attribute from the client property specified by `client_attrs_init.extract_from` config.
+    A one line expression to evaluate a set of predefined string functions (like in the rule engine SQL statements).
-    The expression should match the entire client property value, and capturing groups are concatenated to make the client attribute.
+    The expression can be a function call with nested calls as its arguments, or direct variable reference.
-    For example if the client attribute is the first part of the client ID delimited by a dash, the regular expression would be `^(.+?)-.*$`.
+    So far, it does not provide user-defined variable binding (like `var a=1`) or user-defined functions.
-    Note that failure to match the regular expression will result in the client attribute being absent but not an empty string.
+    As an example, to extract the prefix of client ID delimited by a dot: `nth(1, tokens(clientid, '.'))`.
-    Note also that currently only printable ASCII characters are allowed as input for the regular expression extraction."""
+
+    The variables pre-bound variables are:
+    - `cn`: Client's TLS certificate common name.
+    - `dn`: Client's TLS certificate distinguished name (the subject).
+    - `clientid`: MQTT Client ID.
+    - `username`: MQTT Client's username.
+    - `user_property.{NAME}`: User properties in the CONNECT packet.
+
+    You can read more about variform expressions in EMQX docs."""
 }
 
-client_attrs_init_extract_as {
+client_attrs_init_set_as_attr {
   label: "Name The Extracted Attribute"
   desc: """~
-    The name of the client attribute extracted from the client property specified by `client_attrs_init.extract_from` config.
+    The name of the client attribute extracted from the client data.
-    The extracted attribute will be stored in the `client_attrs` property with this name.
+    The extracted attribute will be stored in the `client_attrs` property with this name."""
-    In case `extract_from = user_property`, this should be the key of the user property."""
 }
 
 }

scripts/spellcheck/dicts/emqx.txt

@@ -259,6 +259,7 @@ uplink
 url
 utc
 util
+variform
 ver
 vm
 vsn