Merge pull request #12872 from zmstone/0412-support-variform-for-client_attrs

0412 use variform for client_attrs
This commit is contained in:
Zaiming (Stone) Shi 2024-04-16 13:29:56 +02:00 committed by GitHub
commit 4d38a8fd44
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
15 changed files with 590 additions and 291 deletions

View File

@ -251,7 +251,7 @@ init(
MP -> MP
end,
ListenerId = emqx_listeners:listener_id(Type, Listener),
ClientInfo0 = set_peercert_infos(
ClientInfo = set_peercert_infos(
Peercert,
#{
zone => Zone,
@ -269,8 +269,6 @@ init(
},
Zone
),
AttrExtractionConfig = get_mqtt_conf(Zone, client_attrs_init),
ClientInfo = initialize_client_attrs_from_cert(AttrExtractionConfig, ClientInfo0, Peercert),
{NClientInfo, NConnInfo} = take_ws_cookie(ClientInfo, ConnInfo),
#channel{
conninfo = NConnInfo,
@ -1575,7 +1573,7 @@ enrich_client(ConnPkt, Channel = #channel{clientinfo = ClientInfo}) ->
fun maybe_username_as_clientid/2,
fun maybe_assign_clientid/2,
%% attr init should happen after clientid and username assign
fun maybe_set_client_initial_attr/2
fun maybe_set_client_initial_attrs/2
],
ConnPkt,
ClientInfo
@ -1587,47 +1585,6 @@ enrich_client(ConnPkt, Channel = #channel{clientinfo = ClientInfo}) ->
{error, ReasonCode, Channel#channel{clientinfo = NClientInfo}}
end.
initialize_client_attrs_from_cert(
#{
extract_from := From,
extract_regexp := Regexp,
extract_as := AttrName
},
ClientInfo,
Peercert
) when From =:= cn orelse From =:= dn ->
case extract_client_attr_from_cert(From, Regexp, Peercert) of
{ok, Value} ->
?SLOG(
debug,
#{
msg => "client_attr_init_from_cert",
extracted_as => AttrName,
extracted_value => Value
}
),
ClientInfo#{client_attrs => #{AttrName => Value}};
_ ->
ClientInfo#{client_attrs => #{}}
end;
initialize_client_attrs_from_cert(_, ClientInfo, _Peercert) ->
ClientInfo.
extract_client_attr_from_cert(cn, Regexp, Peercert) ->
CN = esockd_peercert:common_name(Peercert),
re_extract(CN, Regexp);
extract_client_attr_from_cert(dn, Regexp, Peercert) ->
DN = esockd_peercert:subject(Peercert),
re_extract(DN, Regexp).
re_extract(Str, Regexp) when is_binary(Str) ->
case re:run(Str, Regexp, [{capture, all_but_first, list}]) of
{match, [_ | _] = List} -> {ok, iolist_to_binary(List)};
_ -> nomatch
end;
re_extract(_NotStr, _Regexp) ->
ignored.
set_username(
#mqtt_packet_connect{username = Username},
ClientInfo = #{username := undefined}
@ -1668,75 +1625,50 @@ maybe_assign_clientid(#mqtt_packet_connect{clientid = <<>>}, ClientInfo) ->
maybe_assign_clientid(#mqtt_packet_connect{clientid = ClientId}, ClientInfo) ->
{ok, ClientInfo#{clientid => ClientId}}.
maybe_set_client_initial_attr(ConnPkt, #{zone := Zone} = ClientInfo0) ->
Config = get_mqtt_conf(Zone, client_attrs_init),
ClientInfo = initialize_client_attrs_from_user_property(Config, ConnPkt, ClientInfo0),
Attrs = maps:get(client_attrs, ClientInfo, #{}),
case extract_attr_from_clientinfo(Config, ClientInfo) of
{ok, Value} ->
#{extract_as := Name} = Config,
?SLOG(
debug,
#{
msg => "client_attr_init_from_clientinfo",
extracted_as => Name,
extracted_value => Value
}
),
{ok, ClientInfo#{client_attrs => Attrs#{Name => Value}}};
_ ->
{ok, ClientInfo}
end.
get_client_attrs_init_config(Zone) ->
get_mqtt_conf(Zone, client_attrs_init, []).
initialize_client_attrs_from_user_property(
#{
extract_from := user_property,
extract_as := PropertyKey
},
ConnPkt,
ClientInfo
) ->
case extract_client_attr_from_user_property(ConnPkt, PropertyKey) of
{ok, Value} ->
?SLOG(
debug,
#{
msg => "client_attr_init_from_user_property",
extracted_as => PropertyKey,
extracted_value => Value
}
),
ClientInfo#{client_attrs => #{PropertyKey => Value}};
_ ->
ClientInfo
end;
initialize_client_attrs_from_user_property(_, _ConnInfo, ClientInfo) ->
ClientInfo.
maybe_set_client_initial_attrs(ConnPkt, #{zone := Zone} = ClientInfo) ->
Inits = get_client_attrs_init_config(Zone),
UserProperty = get_user_property_as_map(ConnPkt),
{ok, initialize_client_attrs(Inits, ClientInfo#{user_property => UserProperty})}.
extract_client_attr_from_user_property(
#mqtt_packet_connect{properties = #{'User-Property' := UserProperty}}, PropertyKey
) ->
case lists:keyfind(PropertyKey, 1, UserProperty) of
{_, Value} ->
{ok, Value};
_ ->
not_found
end;
extract_client_attr_from_user_property(_ConnPkt, _PropertyKey) ->
ignored.
initialize_client_attrs(Inits, ClientInfo) ->
lists:foldl(
fun(#{expression := Variform, set_as_attr := Name}, Acc) ->
Attrs = maps:get(client_attrs, ClientInfo, #{}),
case emqx_variform:render(Variform, ClientInfo) of
{ok, Value} ->
?SLOG(
debug,
#{
msg => "client_attr_initialized",
set_as_attr => Name,
attr_value => Value
}
),
Acc#{client_attrs => Attrs#{Name => Value}};
{error, Reason} ->
?SLOG(
warning,
#{
msg => "client_attr_initialization_failed",
reason => Reason
}
),
Acc
end
end,
ClientInfo,
Inits
).
extract_attr_from_clientinfo(#{extract_from := clientid, extract_regexp := Regexp}, #{
clientid := ClientId
}) ->
re_extract(ClientId, Regexp);
extract_attr_from_clientinfo(#{extract_from := username, extract_regexp := Regexp}, #{
username := Username
}) when
Username =/= undefined
get_user_property_as_map(#mqtt_packet_connect{properties = #{'User-Property' := UserProperty}}) when
is_list(UserProperty)
->
re_extract(Username, Regexp);
extract_attr_from_clientinfo(_Config, _CLientInfo) ->
ignored.
maps:from_list(UserProperty);
get_user_property_as_map(_) ->
#{}.
fix_mountpoint(#{mountpoint := undefined} = ClientInfo) ->
ClientInfo;

View File

@ -1734,20 +1734,38 @@ fields(durable_storage) ->
emqx_ds_schema:schema();
fields("client_attrs_init") ->
[
{extract_from,
{expression,
sc(
hoconsc:enum([clientid, username, cn, dn, user_property]),
#{desc => ?DESC("client_attrs_init_extract_from")}
typerefl:alias("string", any()),
#{
desc => ?DESC("client_attrs_init_expression"),
converter => fun compile_variform/2
}
)},
{extract_regexp, sc(binary(), #{desc => ?DESC("client_attrs_init_extract_regexp")})},
{extract_as,
{set_as_attr,
sc(binary(), #{
default => <<"alias">>,
desc => ?DESC("client_attrs_init_extract_as"),
desc => ?DESC("client_attrs_init_set_as_attr"),
validator => fun restricted_string/1
})}
].
compile_variform(undefined, _Opts) ->
undefined;
compile_variform(Expression, #{make_serializable := true}) ->
case is_binary(Expression) of
true ->
Expression;
false ->
emqx_variform:decompile(Expression)
end;
compile_variform(Expression, _Opts) ->
case emqx_variform:compile(Expression) of
{ok, Compiled} ->
Compiled;
{error, Reason} ->
throw(#{expression => Expression, reason => Reason})
end.
restricted_string(Str) ->
case emqx_utils:is_restricted_str(Str) of
true -> ok;
@ -3552,9 +3570,9 @@ mqtt_general() ->
)},
{"client_attrs_init",
sc(
hoconsc:union([disabled, ref("client_attrs_init")]),
hoconsc:array(ref("client_attrs_init")),
#{
default => disabled,
default => [],
desc => ?DESC("client_attrs_init")
}
)}

View File

@ -395,13 +395,14 @@ t_certdn_as_alias(_) ->
test_cert_extraction_as_alias(Which) ->
%% extract the first two chars
Re = <<"^(..).*$">>,
ClientId = iolist_to_binary(["ClientIdFor_", atom_to_list(Which)]),
emqx_config:put_zone_conf(default, [mqtt, client_attrs_init], #{
extract_from => Which,
extract_regexp => Re,
extract_as => <<"alias">>
}),
{ok, Compiled} = emqx_variform:compile("substr(" ++ atom_to_list(Which) ++ ",0,2)"),
emqx_config:put_zone_conf(default, [mqtt, client_attrs_init], [
#{
expression => Compiled,
set_as_attr => <<"alias">>
}
]),
SslConf = emqx_common_test_helpers:client_mtls('tlsv1.2'),
{ok, Client} = emqtt:start_link([
{clientid, ClientId}, {port, 8883}, {ssl, true}, {ssl_opts, SslConf}
@ -416,10 +417,13 @@ test_cert_extraction_as_alias(Which) ->
t_client_attr_from_user_property(_Config) ->
ClientId = atom_to_binary(?FUNCTION_NAME),
emqx_config:put_zone_conf(default, [mqtt, client_attrs_init], #{
extract_from => user_property,
extract_as => <<"group">>
}),
{ok, Compiled} = emqx_variform:compile("user_property.group"),
emqx_config:put_zone_conf(default, [mqtt, client_attrs_init], [
#{
expression => Compiled,
set_as_attr => <<"group">>
}
]),
SslConf = emqx_common_test_helpers:client_mtls('tlsv1.3'),
{ok, Client} = emqtt:start_link([
{clientid, ClientId},

View File

@ -454,7 +454,7 @@ zone_global_defaults() ->
upgrade_qos => false,
use_username_as_clientid => false,
wildcard_subscription => true,
client_attrs_init => disabled
client_attrs_init => []
},
overload_protection =>
#{

View File

@ -150,11 +150,13 @@ t_client_attr_as_mountpoint(_Config) ->
<<"limiter">> => #{},
<<"mountpoint">> => <<"groups/${client_attrs.ns}/">>
},
emqx_config:put_zone_conf(default, [mqtt, client_attrs_init], #{
extract_from => clientid,
extract_regexp => <<"^(.+)-.+$">>,
extract_as => <<"ns">>
}),
{ok, Compiled} = emqx_variform:compile("nth(1,tokens(clientid,'-'))"),
emqx_config:put_zone_conf(default, [mqtt, client_attrs_init], [
#{
expression => Compiled,
set_as_attr => <<"ns">>
}
]),
emqx_logger:set_log_level(debug),
with_listener(tcp, attr_as_moutpoint, ListenerConf, fun() ->
{ok, Client} = emqtt:start_link(#{
@ -170,7 +172,7 @@ t_client_attr_as_mountpoint(_Config) ->
?assertMatch([_], emqx_router:match_routes(MatchTopic)),
emqtt:stop(Client)
end),
emqx_config:put_zone_conf(default, [mqtt, client_attrs_init], disabled),
emqx_config:put_zone_conf(default, [mqtt, client_attrs_init], []),
ok.
t_current_conns_tcp(_Config) ->

View File

@ -557,12 +557,14 @@ t_publish_last_will_testament_denied_topic(_Config) ->
t_alias_prefix(_Config) ->
{ok, _} = emqx_authz:update(?CMD_REPLACE, [?SOURCE_FILE_CLIENT_ATTR]),
ExtractSuffix = <<"^.*-(.*)$">>,
emqx_config:put_zone_conf(default, [mqtt, client_attrs_init], #{
extract_from => clientid,
extract_regexp => ExtractSuffix,
extract_as => <<"alias">>
}),
%% '^.*-(.*)$': extract the suffix after the last '-'
{ok, Compiled} = emqx_variform:compile("concat(regex_extract(clientid,'^.*-(.*)$'))"),
emqx_config:put_zone_conf(default, [mqtt, client_attrs_init], [
#{
expression => Compiled,
set_as_attr => <<"alias">>
}
]),
ClientId = <<"org1-name2">>,
SubTopic = <<"name2/#">>,
SubTopicNotAllowed = <<"name3/#">>,
@ -572,7 +574,7 @@ t_alias_prefix(_Config) ->
?assertMatch({ok, _, [?RC_NOT_AUTHORIZED]}, emqtt:subscribe(C, SubTopicNotAllowed)),
unlink(C),
emqtt:stop(C),
emqx_config:put_zone_conf(default, [mqtt, client_attrs_init], disalbed),
emqx_config:put_zone_conf(default, [mqtt, client_attrs_init], []),
ok.
%% client is allowed by ACL to publish to its LWT topic, is connected,

View File

@ -661,7 +661,7 @@ trans_desc(Init, Hocon, Func, Name, Options) ->
Spec1 = trans_label(Spec0, Hocon, Name, Options),
case Spec1 of
#{description := _} -> Spec1;
_ -> Spec1#{description => <<Name/binary, " Description">>}
_ -> Spec1
end
end.

View File

@ -202,7 +202,8 @@
-export([
md5/1,
sha/1,
sha256/1
sha256/1,
hash/2
]).
%% zip Funcs
@ -710,24 +711,11 @@ map(Map = #{}) ->
map(Data) ->
error(badarg, [Data]).
bin2hexstr(Bin) when is_binary(Bin) ->
emqx_utils:bin_to_hexstr(Bin, upper);
%% If Bin is a bitstring which is not divisible by 8, we pad it and then do the
%% conversion
bin2hexstr(Bin) when is_bitstring(Bin), (8 - (bit_size(Bin) rem 8)) >= 4 ->
PadSize = 8 - (bit_size(Bin) rem 8),
Padding = <<0:PadSize>>,
BinToConvert = <<Padding/bitstring, Bin/bitstring>>,
<<_FirstByte:8, HexStr/binary>> = emqx_utils:bin_to_hexstr(BinToConvert, upper),
HexStr;
bin2hexstr(Bin) when is_bitstring(Bin) ->
PadSize = 8 - (bit_size(Bin) rem 8),
Padding = <<0:PadSize>>,
BinToConvert = <<Padding/bitstring, Bin/bitstring>>,
emqx_utils:bin_to_hexstr(BinToConvert, upper).
bin2hexstr(Bin) ->
emqx_variform_bif:bin2hexstr(Bin).
hexstr2bin(Str) when is_binary(Str) ->
emqx_utils:hexstr_to_bin(Str).
hexstr2bin(Str) ->
emqx_variform_bif:hexstr2bin(Str).
%%------------------------------------------------------------------------------
%% NULL Funcs
@ -771,66 +759,66 @@ is_array(_) -> false.
%% String Funcs
%%------------------------------------------------------------------------------
coalesce(List) -> emqx_variform_str:coalesce(List).
coalesce(List) -> emqx_variform_bif:coalesce(List).
coalesce(A, B) -> emqx_variform_str:coalesce(A, B).
coalesce(A, B) -> emqx_variform_bif:coalesce(A, B).
lower(S) -> emqx_variform_str:lower(S).
lower(S) -> emqx_variform_bif:lower(S).
ltrim(S) -> emqx_variform_str:ltrim(S).
ltrim(S) -> emqx_variform_bif:ltrim(S).
reverse(S) -> emqx_variform_str:reverse(S).
reverse(S) -> emqx_variform_bif:reverse(S).
rtrim(S) -> emqx_variform_str:rtrim(S).
rtrim(S) -> emqx_variform_bif:rtrim(S).
strlen(S) -> emqx_variform_str:strlen(S).
strlen(S) -> emqx_variform_bif:strlen(S).
substr(S, Start) -> emqx_variform_str:substr(S, Start).
substr(S, Start) -> emqx_variform_bif:substr(S, Start).
substr(S, Start, Length) -> emqx_variform_str:substr(S, Start, Length).
substr(S, Start, Length) -> emqx_variform_bif:substr(S, Start, Length).
trim(S) -> emqx_variform_str:trim(S).
trim(S) -> emqx_variform_bif:trim(S).
upper(S) -> emqx_variform_str:upper(S).
upper(S) -> emqx_variform_bif:upper(S).
split(S, P) -> emqx_variform_str:split(S, P).
split(S, P) -> emqx_variform_bif:split(S, P).
split(S, P, Position) -> emqx_variform_str:split(S, P, Position).
split(S, P, Position) -> emqx_variform_bif:split(S, P, Position).
tokens(S, Separators) -> emqx_variform_str:tokens(S, Separators).
tokens(S, Separators) -> emqx_variform_bif:tokens(S, Separators).
tokens(S, Separators, NoCRLF) -> emqx_variform_str:tokens(S, Separators, NoCRLF).
tokens(S, Separators, NoCRLF) -> emqx_variform_bif:tokens(S, Separators, NoCRLF).
concat(S1, S2) -> emqx_variform_str:concat(S1, S2).
concat(S1, S2) -> emqx_variform_bif:concat(S1, S2).
concat(List) -> emqx_variform_str:concat(List).
concat(List) -> emqx_variform_bif:concat(List).
sprintf_s(Format, Args) -> emqx_variform_str:sprintf_s(Format, Args).
sprintf_s(Format, Args) -> emqx_variform_bif:sprintf_s(Format, Args).
pad(S, Len) -> emqx_variform_str:pad(S, Len).
pad(S, Len) -> emqx_variform_bif:pad(S, Len).
pad(S, Len, Position) -> emqx_variform_str:pad(S, Len, Position).
pad(S, Len, Position) -> emqx_variform_bif:pad(S, Len, Position).
pad(S, Len, Position, Char) -> emqx_variform_str:pad(S, Len, Position, Char).
pad(S, Len, Position, Char) -> emqx_variform_bif:pad(S, Len, Position, Char).
replace(SrcStr, Pattern, RepStr) -> emqx_variform_str:replace(SrcStr, Pattern, RepStr).
replace(SrcStr, Pattern, RepStr) -> emqx_variform_bif:replace(SrcStr, Pattern, RepStr).
replace(SrcStr, Pattern, RepStr, Position) ->
emqx_variform_str:replace(SrcStr, Pattern, RepStr, Position).
emqx_variform_bif:replace(SrcStr, Pattern, RepStr, Position).
regex_match(Str, RE) -> emqx_variform_str:regex_match(Str, RE).
regex_match(Str, RE) -> emqx_variform_bif:regex_match(Str, RE).
regex_replace(SrcStr, RE, RepStr) -> emqx_variform_str:regex_replace(SrcStr, RE, RepStr).
regex_replace(SrcStr, RE, RepStr) -> emqx_variform_bif:regex_replace(SrcStr, RE, RepStr).
ascii(Char) -> emqx_variform_str:ascii(Char).
ascii(Char) -> emqx_variform_bif:ascii(Char).
find(S, P) -> emqx_variform_str:find(S, P).
find(S, P) -> emqx_variform_bif:find(S, P).
find(S, P, Position) -> emqx_variform_str:find(S, P, Position).
find(S, P, Position) -> emqx_variform_bif:find(S, P, Position).
join_to_string(Str) -> emqx_variform_str:join_to_string(Str).
join_to_string(Str) -> emqx_variform_bif:join_to_string(Str).
join_to_string(Sep, List) -> emqx_variform_str:join_to_string(Sep, List).
join_to_string(Sep, List) -> emqx_variform_bif:join_to_string(Sep, List).
join_to_sql_values_string(List) ->
QuotedList =
@ -878,7 +866,7 @@ jq(FilterProgram, JSONBin) ->
])
).
unescape(Str) -> emqx_variform_str:unescape(Str).
unescape(Str) -> emqx_variform_bif:unescape(Str).
%%------------------------------------------------------------------------------
%% Array Funcs
@ -1001,7 +989,7 @@ sha256(S) when is_binary(S) ->
hash(sha256, S).
hash(Type, Data) ->
emqx_utils:bin_to_hexstr(crypto:hash(Type, Data), lower).
emqx_variform_bif:hash(Type, Data).
%%------------------------------------------------------------------------------
%% gzip Funcs

View File

@ -28,14 +28,35 @@
erase_allowed_module/1,
erase_allowed_modules/1
]).
-export([render/2, render/3]).
-export([compile/1, decompile/1]).
-export_type([compiled/0]).
-type compiled() :: #{expr := string(), form := term()}.
-define(BIF_MOD, emqx_variform_bif).
-define(IS_ALLOWED_MOD(M),
(M =:= ?BIF_MOD orelse
M =:= lists orelse
M =:= maps)
).
-define(COALESCE_BADARG,
throw(#{
reason => coalesce_badarg,
explain =>
"must be an array, or a call to a function which returns an array, "
"for example: coalesce([a,b,c]) or coalesce(tokens(var,','))"
})
).
%% @doc Render a variform expression with bindings.
%% A variform expression is a template string which supports variable substitution
%% and function calls.
%%
%% The function calls are in the form of `module.function(arg1, arg2, ...)` where `module`
%% is optional, and if not provided, the function is assumed to be in the `emqx_variform_str` module.
%% is optional, and if not provided, the function is assumed to be in the `emqx_variform_bif` module.
%% Both module and function must be existing atoms, and only whitelisted functions are allowed.
%%
%% A function arg can be a constant string or a number.
@ -49,18 +70,54 @@
%%
%% For unresolved variables, empty string (but not "undefined") is used.
%% In case of runtime exeption, an error is returned.
%% In case of unbound variable is referenced, error is returned.
-spec render(string(), map()) -> {ok, binary()} | {error, term()}.
render(Expression, Bindings) ->
render(Expression, Bindings, #{}).
render(Expression, Bindings, Opts) when is_binary(Expression) ->
render(unicode:characters_to_list(Expression), Bindings, Opts);
render(#{form := Form}, Bindings, Opts) ->
eval_as_string(Form, Bindings, Opts);
render(Expression, Bindings, Opts) ->
case compile(Expression) of
{ok, Compiled} ->
render(Compiled, Bindings, Opts);
{error, Reason} ->
{error, Reason}
end.
eval_as_string(Expr, Bindings, _Opts) ->
try
{ok, return_str(eval(Expr, Bindings, #{}))}
catch
throw:Reason ->
{error, Reason};
C:E:S ->
{error, #{exception => C, reason => E, stack_trace => S}}
end.
%% Force the expression to return binary string.
return_str(Str) when is_binary(Str) -> Str;
return_str(Num) when is_integer(Num) -> integer_to_binary(Num);
return_str(Num) when is_float(Num) -> float_to_binary(Num, [{decimals, 10}, compact]);
return_str(Other) ->
throw(#{
reason => bad_return,
expected => string,
got => Other
}).
%% @doc Compile varifom expression.
-spec compile(string() | binary() | compiled()) -> {ok, compiled()} | {error, any()}.
compile(#{form := _} = Compiled) ->
{ok, Compiled};
compile(Expression) when is_binary(Expression) ->
compile(unicode:characters_to_list(Expression));
compile(Expression) ->
case emqx_variform_scan:string(Expression) of
{ok, Tokens, _Line} ->
case emqx_variform_parser:parse(Tokens) of
{ok, Expr} ->
eval_as_string(Expr, Bindings, Opts);
{ok, Form} ->
{ok, #{expr => Expression, form => Form}};
{error, {_, emqx_variform_parser, Msg}} ->
%% syntax error
{error, lists:flatten(Msg)};
@ -71,40 +128,59 @@ render(Expression, Bindings, Opts) ->
{error, Reason}
end.
eval_as_string(Expr, Bindings, _Opts) ->
try
{ok, str(eval(Expr, Bindings))}
catch
throw:Reason ->
{error, Reason};
C:E:S ->
{error, #{exception => C, reason => E, stack_trace => S}}
end.
decompile(#{expr := Expression}) ->
Expression;
decompile(Expression) ->
Expression.
eval({str, Str}, _Bindings) ->
str(Str);
eval({integer, Num}, _Bindings) ->
eval({str, Str}, _Bindings, _Opts) ->
unicode:characters_to_binary(Str);
eval({integer, Num}, _Bindings, _Opts) ->
Num;
eval({float, Num}, _Bindings) ->
eval({float, Num}, _Bindings, _Opts) ->
Num;
eval({array, Args}, Bindings) ->
eval(Args, Bindings);
eval({call, FuncNameStr, Args}, Bindings) ->
eval({array, Args}, Bindings, Opts) ->
eval_loop(Args, Bindings, Opts);
eval({call, FuncNameStr, Args}, Bindings, Opts) ->
{Mod, Fun} = resolve_func_name(FuncNameStr),
ok = assert_func_exported(Mod, Fun, length(Args)),
call(Mod, Fun, eval(Args, Bindings));
eval({var, VarName}, Bindings) ->
resolve_var_value(VarName, Bindings);
eval([Arg | Args], Bindings) ->
[eval(Arg, Bindings) | eval(Args, Bindings)];
eval([], _Bindings) ->
[].
case {Mod, Fun} of
{?BIF_MOD, coalesce} ->
eval_coalesce(Args, Bindings, Opts);
_ ->
call(Mod, Fun, eval_loop(Args, Bindings, Opts))
end;
eval({var, VarName}, Bindings, Opts) ->
resolve_var_value(VarName, Bindings, Opts).
eval_loop([], _, _) -> [];
eval_loop([H | T], Bindings, Opts) -> [eval(H, Bindings, Opts) | eval_loop(T, Bindings, Opts)].
%% coalesce treats var_unbound exception as empty string ''
eval_coalesce([{array, Args}], Bindings, Opts) ->
NewArgs = [lists:map(fun(Arg) -> try_eval(Arg, Bindings, Opts) end, Args)],
call(?BIF_MOD, coalesce, NewArgs);
eval_coalesce([Arg], Bindings, Opts) ->
case try_eval(Arg, Bindings, Opts) of
List when is_list(List) ->
call(?BIF_MOD, coalesce, List);
<<>> ->
<<>>;
_ ->
?COALESCE_BADARG
end;
eval_coalesce(_Args, _Bindings, _Opts) ->
?COALESCE_BADARG.
try_eval(Arg, Bindings, Opts) ->
try
eval(Arg, Bindings, Opts)
catch
throw:#{reason := var_unbound} ->
<<>>
end.
%% Some functions accept arbitrary number of arguments but implemented as /1.
call(emqx_variform_str, concat, Args) ->
str(emqx_variform_str:concat(Args));
call(emqx_variform_str, coalesce, Args) ->
str(emqx_variform_str:coalesce(Args));
call(Mod, Fun, Args) ->
erlang:apply(Mod, Fun, Args).
@ -144,23 +220,23 @@ resolve_func_name(FuncNameStr) ->
function => Fun
})
end,
{emqx_variform_str, FuncName};
{?BIF_MOD, FuncName};
_ ->
throw(#{reason => invalid_function_reference, function => FuncNameStr})
end.
resolve_var_value(VarName, Bindings) ->
%% _Opts can be extended in the future. For example, unbound var as 'undfeined'
resolve_var_value(VarName, Bindings, _Opts) ->
case emqx_template:lookup_var(split(VarName), Bindings) of
{ok, Value} ->
Value;
{error, _Reason} ->
<<>>
throw(#{
var_name => VarName,
reason => var_unbound
})
end.
assert_func_exported(emqx_variform_str, concat, _Arity) ->
ok;
assert_func_exported(emqx_variform_str, coalesce, _Arity) ->
ok;
assert_func_exported(Mod, Fun, Arity) ->
ok = try_load(Mod),
case erlang:function_exported(Mod, Fun, Arity) of
@ -187,7 +263,7 @@ try_load(Mod) ->
ok
end.
assert_module_allowed(emqx_variform_str) ->
assert_module_allowed(Mod) when ?IS_ALLOWED_MOD(Mod) ->
ok;
assert_module_allowed(Mod) ->
Allowed = get_allowed_modules(),
@ -220,8 +296,5 @@ erase_allowed_modules(Modules) when is_list(Modules) ->
get_allowed_modules() ->
persistent_term:get({emqx_variform, allowed_modules}, []).
str(Value) ->
emqx_utils_conv:bin(Value).
split(VarName) ->
lists:map(fun erlang:iolist_to_binary/1, string:tokens(VarName, ".")).

View File

@ -14,13 +14,11 @@
%% limitations under the License.
%%--------------------------------------------------------------------
%% Predefined functions string templating
-module(emqx_variform_str).
%% Predefined functions for variform expressions.
-module(emqx_variform_bif).
%% String Funcs
-export([
coalesce/1,
coalesce/2,
lower/1,
ltrim/1,
ltrim/2,
@ -47,15 +45,37 @@
replace/4,
regex_match/2,
regex_replace/3,
regex_extract/2,
ascii/1,
find/2,
find/3,
join_to_string/1,
join_to_string/2,
unescape/1,
nth/2
any_to_str/1
]).
%% Array functions
-export([nth/2]).
%% Control functions
-export([coalesce/1, coalesce/2]).
%% Random functions
-export([rand_str/1, rand_int/1]).
%% Schema-less encod/decode
-export([
bin2hexstr/1,
hexstr2bin/1,
int2hexstr/1,
base64_encode/1,
base64_decode/1
]).
%% Hash functions
-export([hash/2, hash_to_range/3, map_to_range/3]).
-define(IS_EMPTY(X), (X =:= <<>> orelse X =:= "" orelse X =:= undefined)).
%%------------------------------------------------------------------------------
@ -143,8 +163,10 @@ tokens(S, Separators, <<"nocrlf">>) ->
concat(S1, S2) ->
concat([S1, S2]).
%% @doc Concatenate a list of strings.
%% NOTE: it converts non-string elements to Erlang term literals for backward compatibility
concat(List) ->
unicode:characters_to_binary(lists:map(fun str/1, List), unicode).
unicode:characters_to_binary(lists:map(fun any_to_str/1, List), unicode).
sprintf_s(Format, Args) when is_list(Args) ->
erlang:iolist_to_binary(io_lib:format(binary_to_list(Format), Args)).
@ -190,6 +212,22 @@ regex_match(Str, RE) ->
regex_replace(SrcStr, RE, RepStr) ->
re:replace(SrcStr, RE, RepStr, [global, {return, binary}]).
%% @doc Searches the string Str for patterns specified by Regexp.
%% If matches are found, it returns a list of all captured groups from these matches.
%% If no matches are found or there are no groups captured, it returns an empty list.
%% This function can be used to extract parts of a string based on a regular expression,
%% excluding the complete match itself.
%% Examples:
%% ("Number: 12345", "(\\d+)") -> [<<"12345">>]
%% ("Hello, world!", "(\\w+)") -> [<<"Hello">>, <<"world">>]
%% ("No numbers here!", "(\\d+)") -> []
%% ("Date: 2021-05-20", "(\\d{4})-(\\d{2})-(\\d{2})") -> [<<"2021">>, <<"05">>, <<"20">>]
regex_extract(Str, Regexp) ->
case re:run(Str, Regexp, [{capture, all_but_first, list}]) of
{match, [_ | _] = L} -> lists:map(fun erlang:iolist_to_binary/1, L);
_ -> []
end.
ascii(Char) when is_binary(Char) ->
[FirstC | _] = binary_to_list(Char),
FirstC.
@ -212,7 +250,7 @@ join_to_string(List) when is_list(List) ->
join_to_string(<<", ">>, List).
join_to_string(Sep, List) when is_list(List), is_binary(Sep) ->
iolist_to_binary(lists:join(Sep, [str(Item) || Item <- List])).
iolist_to_binary(lists:join(Sep, [any_to_str(Item) || Item <- List])).
unescape(Bin) when is_binary(Bin) ->
UnicodeList = unicode:characters_to_list(Bin, utf8),
@ -364,5 +402,124 @@ is_hex_digit(_) -> false.
%% Data Type Conversion Funcs
%%------------------------------------------------------------------------------
str(Data) ->
any_to_str(Data) ->
emqx_utils_conv:bin(Data).
%%------------------------------------------------------------------------------
%% Random functions
%%------------------------------------------------------------------------------
%% @doc Make a random string with urlsafe-base64 charset.
rand_str(Length) when is_integer(Length) andalso Length > 0 ->
RawBytes = erlang:ceil((Length * 3) / 4),
RandomData = rand:bytes(RawBytes),
urlsafe(binary:part(base64_encode(RandomData), 0, Length));
rand_str(_) ->
throw(#{reason => badarg, function => ?FUNCTION_NAME}).
%% @doc Make a random integer in the range `[1, N]`.
rand_int(N) when is_integer(N) andalso N >= 1 ->
rand:uniform(N);
rand_int(N) ->
throw(#{reason => badarg, function => ?FUNCTION_NAME, expected => "positive integer", got => N}).
%% TODO: call base64:encode(Bin, #{mode => urlsafe, padding => false})
%% when oldest OTP to support is 26 or newer.
urlsafe(Str0) ->
Str = replace(Str0, <<"+">>, <<"-">>),
replace(Str, <<"/">>, <<"_">>).
%%------------------------------------------------------------------------------
%% Data encoding
%%------------------------------------------------------------------------------
%% @doc Encode an integer to hex string. e.g. 15 as 'f'
int2hexstr(Int) ->
erlang:integer_to_binary(Int, 16).
%% @doc Encode bytes in hex string format.
bin2hexstr(Bin) when is_binary(Bin) ->
emqx_utils:bin_to_hexstr(Bin, upper);
%% If Bin is a bitstring which is not divisible by 8, we pad it and then do the
%% conversion
bin2hexstr(Bin) when is_bitstring(Bin), (8 - (bit_size(Bin) rem 8)) >= 4 ->
PadSize = 8 - (bit_size(Bin) rem 8),
Padding = <<0:PadSize>>,
BinToConvert = <<Padding/bitstring, Bin/bitstring>>,
<<_FirstByte:8, HexStr/binary>> = emqx_utils:bin_to_hexstr(BinToConvert, upper),
HexStr;
bin2hexstr(Bin) when is_bitstring(Bin) ->
PadSize = 8 - (bit_size(Bin) rem 8),
Padding = <<0:PadSize>>,
BinToConvert = <<Padding/bitstring, Bin/bitstring>>,
emqx_utils:bin_to_hexstr(BinToConvert, upper).
%% @doc Decode hex string into its original bytes.
hexstr2bin(Str) when is_binary(Str) ->
emqx_utils:hexstr_to_bin(Str).
%% @doc Encode any bytes to base64.
base64_encode(Bin) ->
base64:encode(Bin).
%% @doc Decode base64 encoded string.
base64_decode(Bin) ->
base64:decode(Bin).
%%------------------------------------------------------------------------------
%% Hash functions
%%------------------------------------------------------------------------------
%% @doc Hash with all available algorithm provided by crypto module.
%% Return hex format string.
%% - md4 | md5
%% - sha (sha1)
%% - sha224 | sha256 | sha384 | sha512
%% - sha3_224 | sha3_256 | sha3_384 | sha3_512
%% - shake128 | shake256
%% - blake2b | blake2s
hash(<<"sha1">>, Bin) ->
hash(sha, Bin);
hash(Algorithm, Bin) when is_binary(Algorithm) ->
Type =
try
binary_to_existing_atom(Algorithm)
catch
_:_ ->
throw(#{
reason => unknown_hash_algorithm,
algorithm => Algorithm
})
end,
hash(Type, Bin);
hash(Type, Bin) when is_atom(Type) ->
%% lower is for backward compatibility
emqx_utils:bin_to_hexstr(crypto:hash(Type, Bin), lower).
%% @doc Hash binary data to an integer within a specified range [Min, Max]
hash_to_range(Bin, Min, Max) when
is_binary(Bin) andalso
size(Bin) > 0 andalso
is_integer(Min) andalso
is_integer(Max) andalso
Min =< Max
->
Hash = hash(sha256, Bin),
HashNum = binary_to_integer(Hash, 16),
map_to_range(HashNum, Min, Max);
hash_to_range(_, _, _) ->
throw(#{reason => badarg, function => ?FUNCTION_NAME}).
map_to_range(Bin, Min, Max) when is_binary(Bin) andalso size(Bin) > 0 ->
HashNum = binary:decode_unsigned(Bin),
map_to_range(HashNum, Min, Max);
map_to_range(Int, Min, Max) when
is_integer(Int) andalso
is_integer(Min) andalso
is_integer(Max) andalso
Min =< Max
->
Range = Max - Min + 1,
Min + (Int rem Range);
map_to_range(_, _, _) ->
throw(#{reason => badarg, function => ?FUNCTION_NAME}).

View File

@ -0,0 +1,74 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
%% Most of the functions are tested as rule-engine string funcs
-module(emqx_variform_bif_tests).
-compile(export_all).
-compile(nowarn_export_all).
-include_lib("eunit/include/eunit.hrl").
regex_extract_test_() ->
[
?_assertEqual([<<"12345">>], regex_extract("Order number: 12345", "(\\d+)")),
?_assertEqual(
[<<"Hello">>, <<"world">>], regex_extract("Hello, world!", "(\\w+).*\s(\\w+)")
),
?_assertEqual([], regex_extract("No numbers here!", "(\\d+)")),
?_assertEqual(
[<<"2021">>, <<"05">>, <<"20">>],
regex_extract("Date: 2021-05-20", "(\\d{4})-(\\d{2})-(\\d{2})")
),
?_assertEqual([<<"Hello">>], regex_extract("Hello, world!", "(Hello)")),
?_assertEqual(
[<<"12">>, <<"34">>], regex_extract("Items: 12, Price: 34", "(\\d+).*\s(\\d+)")
),
?_assertEqual(
[<<"john.doe@example.com">>],
regex_extract("Contact: john.doe@example.com", "([\\w\\.]+@[\\w\\.]+)")
),
?_assertEqual([], regex_extract("Just some text, nothing more.", "([A-Z]\\d{3})")),
?_assertEqual(
[<<"admin">>, <<"1234">>],
regex_extract("User: admin, Pass: 1234", "User: (\\w+), Pass: (\\d+)")
),
?_assertEqual([], regex_extract("", "(\\d+)")),
?_assertEqual([], regex_extract("$$$###!!!", "(\\d+)")),
?_assertEqual([<<"23.1">>], regex_extract("Erlang 23.1 version", "(\\d+\\.\\d+)")),
?_assertEqual(
[<<"192.168.1.1">>],
regex_extract("Server IP: 192.168.1.1 at port 8080", "(\\d+\\.\\d+\\.\\d+\\.\\d+)")
)
].
regex_extract(Str, RegEx) ->
emqx_variform_bif:regex_extract(Str, RegEx).
rand_str_test() ->
?assertEqual(3, size(emqx_variform_bif:rand_str(3))),
?assertThrow(#{reason := badarg}, size(emqx_variform_bif:rand_str(0))).
rand_int_test() ->
N = emqx_variform_bif:rand_int(10),
?assert(N =< 10 andalso N >= 1),
?assertThrow(#{reason := badarg}, emqx_variform_bif:rand_int(0)),
?assertThrow(#{reason := badarg}, emqx_variform_bif:rand_int(-1)).
base64_encode_decode_test() ->
RandBytes = crypto:strong_rand_bytes(100),
Encoded = emqx_variform_bif:base64_encode(RandBytes),
?assertEqual(RandBytes, emqx_variform_bif:base64_decode(Encoded)).

View File

@ -27,14 +27,16 @@ redner_test_() ->
[
{"direct var reference", fun() -> ?assertEqual({ok, <<"1">>}, render("a", #{a => 1})) end},
{"concat strings", fun() ->
?assertEqual({ok, <<"a,b">>}, render("concat('a',',','b')", #{}))
?assertEqual({ok, <<"a,b">>}, render("concat(['a',',','b'])", #{}))
end},
{"concat empty string", fun() ->
?assertEqual({ok, <<"">>}, render("concat([''])", #{}))
end},
{"concat empty string", fun() -> ?assertEqual({ok, <<"">>}, render("concat('')", #{})) end},
{"tokens 1st", fun() ->
?assertEqual({ok, <<"a">>}, render("nth(1,tokens(var, ','))", #{var => <<"a,b">>}))
end},
{"unknown var as empty str", fun() ->
?assertEqual({ok, <<>>}, render("var", #{}))
{"unknown var return error", fun() ->
?assertMatch({error, #{reason := var_unbound}}, render("var", #{}))
end},
{"out of range nth index", fun() ->
?assertEqual({ok, <<>>}, render("nth(2, tokens(var, ','))", #{var => <<"a">>}))
@ -97,7 +99,7 @@ unknown_func_test_() ->
{"unknown function in a known module", fun() ->
?assertMatch(
{error, #{reason := unknown_variform_function}},
render("emqx_variform_str.nonexistingatom__(a)", #{})
render("emqx_variform_bif.nonexistingatom__(a)", #{})
)
end},
{"invalid func reference", fun() ->
@ -133,19 +135,39 @@ inject_allowed_module_test() ->
coalesce_test_() ->
[
{"coalesce first", fun() ->
?assertEqual({ok, <<"a">>}, render("coalesce('a','b')", #{}))
{"first", fun() ->
?assertEqual({ok, <<"a">>}, render("coalesce(['a','b'])", #{}))
end},
{"coalesce second", fun() ->
?assertEqual({ok, <<"b">>}, render("coalesce('', 'b')", #{}))
{"second", fun() ->
?assertEqual({ok, <<"b">>}, render("coalesce(['', 'b'])", #{}))
end},
{"coalesce first var", fun() ->
?assertEqual({ok, <<"a">>}, render("coalesce(a,b)", #{a => <<"a">>, b => <<"b">>}))
{"first var", fun() ->
?assertEqual({ok, <<"a">>}, render("coalesce([a,b])", #{a => <<"a">>, b => <<"b">>}))
end},
{"coalesce second var", fun() ->
?assertEqual({ok, <<"b">>}, render("coalesce(a,b)", #{b => <<"b">>}))
{"second var", fun() ->
?assertEqual({ok, <<"b">>}, render("coalesce([a,b])", #{b => <<"b">>}))
end},
{"coalesce empty", fun() -> ?assertEqual({ok, <<>>}, render("coalesce(a,b)", #{})) end}
{"empty", fun() -> ?assertEqual({ok, <<>>}, render("coalesce([a,b])", #{})) end},
{"arg from other func", fun() ->
?assertEqual({ok, <<"b">>}, render("coalesce(tokens(a,','))", #{a => <<",,b,c">>}))
end},
{"var unbound", fun() -> ?assertEqual({ok, <<>>}, render("coalesce(a)", #{})) end},
{"var unbound in call", fun() ->
?assertEqual({ok, <<>>}, render("coalesce(concat(a))", #{}))
end},
{"var unbound in calls", fun() ->
?assertEqual({ok, <<"c">>}, render("coalesce([any_to_str(a),any_to_str(b),'c'])", #{}))
end},
{"badarg", fun() ->
?assertMatch(
{error, #{reason := coalesce_badarg}}, render("coalesce(a,b)", #{a => 1, b => 2})
)
end},
{"badarg from return", fun() ->
?assertMatch(
{error, #{reason := coalesce_badarg}}, render("coalesce(any_to_str(a))", #{a => 1})
)
end}
].
syntax_error_test_() ->
@ -160,3 +182,39 @@ syntax_error_test_() ->
render(Expression, Bindings) ->
emqx_variform:render(Expression, Bindings).
hash_pick_test() ->
lists:foreach(
fun(_) ->
{ok, Res} = render("nth(hash_to_range(rand_str(10),1,5),[1,2,3,4,5])", #{}),
?assert(Res >= <<"1">> andalso Res =< <<"5">>)
end,
lists:seq(1, 100)
).
map_to_range_pick_test() ->
lists:foreach(
fun(_) ->
{ok, Res} = render("nth(map_to_range(rand_str(10),1,5),[1,2,3,4,5])", #{}),
?assert(Res >= <<"1">> andalso Res =< <<"5">>)
end,
lists:seq(1, 100)
).
-define(ASSERT_BADARG(FUNC, ARGS),
?_assertEqual(
{error, #{reason => badarg, function => FUNC}},
render(atom_to_list(FUNC) ++ ARGS, #{})
)
).
to_range_badarg_test_() ->
[
?ASSERT_BADARG(hash_to_range, "(1,1,2)"),
?ASSERT_BADARG(hash_to_range, "('',1,2)"),
?ASSERT_BADARG(hash_to_range, "('a','1',2)"),
?ASSERT_BADARG(hash_to_range, "('a',2,1)"),
?ASSERT_BADARG(map_to_range, "('',1,2)"),
?ASSERT_BADARG(map_to_range, "('a','1',2)"),
?ASSERT_BADARG(map_to_range, "('a',2,1)")
].

View File

@ -7,8 +7,8 @@ an MQTT connection.
### Initialization of `client_attrs`
- The `client_attrs` field can be initially populated based on the configuration from one of the
following sources:
- The `client_attrs` fields can be initially populated from one of the
following `clientinfo` fields:
- `cn`: The common name from the TLS client's certificate.
- `dn`: The distinguished name from the TLS client's certificate, that is, the certificate "Subject".
- `clientid`: The MQTT client ID provided by the client.

View File

@ -1575,47 +1575,37 @@ client_attrs_init {
label: "Client Attributes Initialization"
desc: """~
Specify how to initialize client attributes.
One initial client attribute can be initialized as `client_attrs.NAME`,
where `NAME` is the name of the attribute specified in the config `extract_as`.
Each client attribute can be initialized as `client_attrs.{NAME}`,
where `{NAME}` is the name of the attribute specified in the config field `set_as_attr`.
The initialized client attribute will be stored in the `client_attrs` property with the specified name,
and can be used as a placeholder in a template for authentication and authorization.
For example, use `${client_attrs.alias}` to render an HTTP POST body when `extract_as = alias`,
For example, use `${client_attrs.alias}` to render an HTTP POST body when `set_as_attr = alias`,
or render listener config `moutpoint = devices/${client_attrs.alias}/` to initialize a per-client topic namespace."""
}
client_attrs_init_extract_from {
label: "Client Property to Extract Attribute"
desc: """~
Specify from which client property the client attribute should be extracted.
Supported values:
- `clientid`: Extract from the client ID.
- `username`: Extract from the username.
- `cn`: Extract from the Common Name (CN) field of the client certificate.
- `dn`: Extract from the Distinguished Name (DN) field of the client certificate.
- `user_property`: Extract from the user property sent in the MQTT v5 `CONNECT` packet.
In this case, `extract_regexp` is not applicable, and `extract_as` should be the user property key.
NOTE: this extraction happens **after** `clientid` or `username` is initialized
from `peer_cert_as_clientid` or `peer_cert_as_username` config."""
}
client_attrs_init_extract_regexp {
client_attrs_init_expression {
label: "Client Attribute Extraction Regular Expression"
desc: """~
The regular expression to extract a client attribute from the client property specified by `client_attrs_init.extract_from` config.
The expression should match the entire client property value, and capturing groups are concatenated to make the client attribute.
For example if the client attribute is the first part of the client ID delimited by a dash, the regular expression would be `^(.+?)-.*$`.
Note that failure to match the regular expression will result in the client attribute being absent but not an empty string.
Note also that currently only printable ASCII characters are allowed as input for the regular expression extraction."""
A one line expression to evaluate a set of predefined string functions (like in the rule engine SQL statements).
The expression can be a function call with nested calls as its arguments, or direct variable reference.
So far, it does not provide user-defined variable binding (like `var a=1`) or user-defined functions.
As an example, to extract the prefix of client ID delimited by a dot: `nth(1, tokens(clientid, '.'))`.
The variables pre-bound variables are:
- `cn`: Client's TLS certificate common name.
- `dn`: Client's TLS certificate distinguished name (the subject).
- `clientid`: MQTT Client ID.
- `username`: MQTT Client's username.
- `user_property.{NAME}`: User properties in the CONNECT packet.
You can read more about variform expressions in EMQX docs."""
}
client_attrs_init_extract_as {
client_attrs_init_set_as_attr {
label: "Name The Extracted Attribute"
desc: """~
The name of the client attribute extracted from the client property specified by `client_attrs_init.extract_from` config.
The extracted attribute will be stored in the `client_attrs` property with this name.
In case `extract_from = user_property`, this should be the key of the user property."""
The name of the client attribute extracted from the client data.
The extracted attribute will be stored in the `client_attrs` property with this name."""
}
}

View File

@ -259,6 +259,7 @@ uplink
url
utc
util
variform
ver
vm
vsn