feat(tpl): factor out loose json concept into a separate module

Which is called `emqx_jsonish`. Also introduce an _access module_
abstraction to extract information from such data during rendering.
This commit is contained in:
Andrew Mayorov 2023-10-23 15:42:58 +07:00
parent 69cfa740ea
commit 02c1bd70b6
No known key found for this signature in database
GPG Key ID: 2837C62ACFBFED5D
6 changed files with 295 additions and 125 deletions

View File

@ -232,11 +232,10 @@ parse_user_properties(_) ->
undefined. undefined.
render_template(Template, Bindings) -> render_template(Template, Bindings) ->
Opts = #{var_lookup => fun emqx_template:lookup_loose_json/2}, emqx_template:render(Template, {emqx_jsonish, Bindings}).
emqx_template:render(Template, Bindings, Opts).
render_simple_var([{var, _Name, Accessor}], Data, Default) -> render_simple_var([{var, _Name, Accessor}], Data, Default) ->
case emqx_template:lookup_loose_json(Accessor, Data) of case emqx_jsonish:lookup(Accessor, Data) of
{ok, Var} -> Var; {ok, Var} -> Var;
%% cannot find the variable from Data %% cannot find the variable from Data
{error, _} -> Default {error, _} -> Default

View File

@ -0,0 +1,71 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_jsonish).
-export([lookup/2]).
-export_type([t/0]).
%% @doc Either a map or a JSON serial.
%% Think of it as a kind of lazily parsed and/or constructed JSON.
-type t() :: propmap() | serial().
%% @doc JSON in serialized form.
-type serial() :: binary().
-type propmap() :: #{prop() => value()}.
-type prop() :: atom() | binary().
-type value() :: scalar() | [scalar() | propmap()] | t().
-type scalar() :: atom() | unicode:chardata() | number().
%%
%% @doc Lookup a value in the JSON-ish map accessible through the given accessor.
%% If accessor implies drilling down into a binary, it will be treated as JSON serial.
%% Failure to parse the binary as JSON will result in an _invalid type_ error.
%% Nested JSON is NOT parsed recursively.
-spec lookup(emqx_template:accessor(), t()) ->
{ok, value()}
| {error, undefined | {_Location :: non_neg_integer(), _InvalidType :: atom()}}.
lookup(Var, Jsonish) ->
lookup(0, _Decoded = false, Var, Jsonish).
lookup(_, _, [], Value) ->
{ok, Value};
lookup(Loc, Decoded, [Prop | Rest], Jsonish) when is_map(Jsonish) ->
case emqx_template:lookup(Prop, Jsonish) of
{ok, Value} ->
lookup(Loc + 1, Decoded, Rest, Value);
{error, Reason} ->
{error, Reason}
end;
lookup(Loc, _Decoded = false, Rest, Json) when is_binary(Json) ->
try emqx_utils_json:decode(Json) of
Value ->
% NOTE: This is intentional, we don't want to parse nested JSON.
lookup(Loc, true, Rest, Value)
catch
error:_ ->
{error, {Loc, binary}}
end;
lookup(Loc, _, _, Invalid) ->
{error, {Loc, type_name(Invalid)}}.
type_name(Term) when is_atom(Term) -> atom;
type_name(Term) when is_number(Term) -> number;
type_name(Term) when is_binary(Term) -> binary;
type_name(Term) when is_list(Term) -> list.

View File

@ -29,7 +29,8 @@
-export([render_strict/3]). -export([render_strict/3]).
-export([lookup_var/2]). -export([lookup_var/2]).
-export([lookup_loose_json/2]). -export([lookup/2]).
-export([to_string/1]). -export([to_string/1]).
-export_type([t/0]). -export_type([t/0]).
@ -38,6 +39,10 @@
-export_type([placeholder/0]). -export_type([placeholder/0]).
-export_type([varname/0]). -export_type([varname/0]).
-export_type([bindings/0]). -export_type([bindings/0]).
-export_type([accessor/0]).
-export_type([context/0]).
-export_type([render_opts/0]).
-type t() :: str() | {'$tpl', deeptpl()}. -type t() :: str() | {'$tpl', deeptpl()}.
@ -70,19 +75,22 @@
fun((Value :: term()) -> unicode:chardata()) fun((Value :: term()) -> unicode:chardata())
| fun((varname(), Value :: term()) -> unicode:chardata()). | fun((varname(), Value :: term()) -> unicode:chardata()).
-type var_lookup() ::
fun((accessor(), bindings()) -> {ok, binding()} | {error, reason()}).
-type parse_opts() :: #{ -type parse_opts() :: #{
strip_double_quote => boolean() strip_double_quote => boolean()
}. }.
-type render_opts() :: #{ -type render_opts() :: #{
var_trans => var_trans(), var_trans => var_trans()
var_lookup => var_lookup()
}. }.
-define(PH_VAR_THIS, '$this'). -type context() ::
%% Map with (potentially nested) bindings.
bindings()
%% Arbitrary term accessible via an access module with `lookup/2` function.
| {_AccessModule :: module(), _Bindings}.
%% Access module API
-callback lookup(accessor(), _Bindings) -> {ok, _Value} | {error, reason()}.
-define(RE_PLACEHOLDER, "\\$\\{[.]?([a-zA-Z0-9._]*)\\}"). -define(RE_PLACEHOLDER, "\\$\\{[.]?([a-zA-Z0-9._]*)\\}").
-define(RE_ESCAPE, "\\$\\{(\\$)\\}"). -define(RE_ESCAPE, "\\$\\{(\\$)\\}").
@ -130,7 +138,7 @@ prepend(Head, To) ->
parse_accessor(Var) -> parse_accessor(Var) ->
case string:split(Var, <<".">>, all) of case string:split(Var, <<".">>, all) of
[<<>>] -> [<<>>] ->
?PH_VAR_THIS; [];
Name -> Name ->
Name Name
end. end.
@ -180,18 +188,18 @@ render_placeholder(Name) ->
%% If one or more placeholders are not found in bindings, an error is returned. %% If one or more placeholders are not found in bindings, an error is returned.
%% By default, all binding values are converted to strings using `to_string/1` %% By default, all binding values are converted to strings using `to_string/1`
%% function. Option `var_trans` can be used to override this behaviour. %% function. Option `var_trans` can be used to override this behaviour.
-spec render(t(), bindings()) -> -spec render(t(), context()) ->
{term(), [_Error :: {varname(), reason()}]}. {term(), [_Error :: {varname(), reason()}]}.
render(Template, Bindings) -> render(Template, Context) ->
render(Template, Bindings, #{}). render(Template, Context, #{}).
-spec render(t(), bindings(), render_opts()) -> -spec render(t(), context(), render_opts()) ->
{term(), [_Error :: {varname(), undefined}]}. {term(), [_Error :: {varname(), undefined}]}.
render(Template, Bindings, Opts) when is_list(Template) -> render(Template, Context, Opts) when is_list(Template) ->
lists:mapfoldl( lists:mapfoldl(
fun fun
({var, Name, Accessor}, EAcc) -> ({var, Name, Accessor}, EAcc) ->
{String, Errors} = render_binding(Name, Accessor, Bindings, Opts), {String, Errors} = render_binding(Name, Accessor, Context, Opts),
{String, Errors ++ EAcc}; {String, Errors ++ EAcc};
(String, EAcc) -> (String, EAcc) ->
{String, EAcc} {String, EAcc}
@ -199,11 +207,11 @@ render(Template, Bindings, Opts) when is_list(Template) ->
[], [],
Template Template
); );
render({'$tpl', Template}, Bindings, Opts) -> render({'$tpl', Template}, Context, Opts) ->
render_deep(Template, Bindings, Opts). render_deep(Template, Context, Opts).
render_binding(Name, Accessor, Bindings, Opts) -> render_binding(Name, Accessor, Context, Opts) ->
case lookup_value(Accessor, Bindings, Opts) of case lookup_value(Accessor, Context) of
{ok, Value} -> {ok, Value} ->
{render_value(Name, Value, Opts), []}; {render_value(Name, Value, Opts), []};
{error, Reason} -> {error, Reason} ->
@ -213,9 +221,9 @@ render_binding(Name, Accessor, Bindings, Opts) ->
{render_value(Name, undefined, Opts), [{Name, Reason}]} {render_value(Name, undefined, Opts), [{Name, Reason}]}
end. end.
lookup_value(Accessor, Bindings, #{var_lookup := LookupFun}) -> lookup_value(Accessor, {AccessMod, Bindings}) ->
LookupFun(Accessor, Bindings); AccessMod:lookup(Accessor, Bindings);
lookup_value(Accessor, Bindings, #{}) -> lookup_value(Accessor, Bindings) ->
lookup_var(Accessor, Bindings). lookup_var(Accessor, Bindings).
render_value(_Name, Value, #{var_trans := TransFun}) when is_function(TransFun, 1) -> render_value(_Name, Value, #{var_trans := TransFun}) when is_function(TransFun, 1) ->
@ -228,19 +236,19 @@ render_value(_Name, Value, #{}) ->
%% @doc Render a template with given bindings. %% @doc Render a template with given bindings.
%% Behaves like `render/2`, but raises an error exception if one or more placeholders %% Behaves like `render/2`, but raises an error exception if one or more placeholders
%% are not found in the bindings. %% are not found in the bindings.
-spec render_strict(t(), bindings()) -> -spec render_strict(t(), context()) ->
term(). term().
render_strict(Template, Bindings) -> render_strict(Template, Context) ->
render_strict(Template, Bindings, #{}). render_strict(Template, Context, #{}).
-spec render_strict(t(), bindings(), render_opts()) -> -spec render_strict(t(), context(), render_opts()) ->
term(). term().
render_strict(Template, Bindings, Opts) -> render_strict(Template, Context, Opts) ->
case render(Template, Bindings, Opts) of case render(Template, Context, Opts) of
{Render, []} -> {Render, []} ->
Render; Render;
{_, Errors = [_ | _]} -> {_, Errors = [_ | _]} ->
error(Errors, [unparse(Template), Bindings]) error(Errors, [unparse(Template), Context])
end. end.
%% @doc Parse an arbitrary Erlang term into a "deep" template. %% @doc Parse an arbitrary Erlang term into a "deep" template.
@ -275,30 +283,30 @@ parse_deep_term(Term, Opts) when is_binary(Term) ->
parse_deep_term(Term, _Opts) -> parse_deep_term(Term, _Opts) ->
Term. Term.
render_deep(Template, Bindings, Opts) when is_map(Template) -> render_deep(Template, Context, Opts) when is_map(Template) ->
maps:fold( maps:fold(
fun(KT, VT, {Acc, Errors}) -> fun(KT, VT, {Acc, Errors}) ->
{K, KErrors} = render_deep(KT, Bindings, Opts), {K, KErrors} = render_deep(KT, Context, Opts),
{V, VErrors} = render_deep(VT, Bindings, Opts), {V, VErrors} = render_deep(VT, Context, Opts),
{Acc#{K => V}, KErrors ++ VErrors ++ Errors} {Acc#{K => V}, KErrors ++ VErrors ++ Errors}
end, end,
{#{}, []}, {#{}, []},
Template Template
); );
render_deep({list, Template}, Bindings, Opts) when is_list(Template) -> render_deep({list, Template}, Context, Opts) when is_list(Template) ->
lists:mapfoldr( lists:mapfoldr(
fun(T, Errors) -> fun(T, Errors) ->
{E, VErrors} = render_deep(T, Bindings, Opts), {E, VErrors} = render_deep(T, Context, Opts),
{E, VErrors ++ Errors} {E, VErrors ++ Errors}
end, end,
[], [],
Template Template
); );
render_deep({tuple, Template}, Bindings, Opts) when is_list(Template) -> render_deep({tuple, Template}, Context, Opts) when is_list(Template) ->
{Term, Errors} = render_deep({list, Template}, Bindings, Opts), {Term, Errors} = render_deep({list, Template}, Context, Opts),
{list_to_tuple(Term), Errors}; {list_to_tuple(Term), Errors};
render_deep(Template, Bindings, Opts) when is_list(Template) -> render_deep(Template, Context, Opts) when is_list(Template) ->
{String, Errors} = render(Template, Bindings, Opts), {String, Errors} = render(Template, Context, Opts),
{unicode:characters_to_binary(String), Errors}; {unicode:characters_to_binary(String), Errors};
render_deep(Term, _Bindings, _Opts) -> render_deep(Term, _Bindings, _Opts) ->
{Term, []}. {Term, []}.
@ -331,7 +339,7 @@ unparse_deep(Term) ->
lookup_var(Var, Bindings) -> lookup_var(Var, Bindings) ->
lookup_var(0, Var, Bindings). lookup_var(0, Var, Bindings).
lookup_var(_, Var, Value) when Var == ?PH_VAR_THIS orelse Var == [] -> lookup_var(_, [], Value) ->
{ok, Value}; {ok, Value};
lookup_var(Loc, [Prop | Rest], Bindings) when is_map(Bindings) -> lookup_var(Loc, [Prop | Rest], Bindings) when is_map(Bindings) ->
case lookup(Prop, Bindings) of case lookup(Prop, Bindings) of
@ -343,35 +351,6 @@ lookup_var(Loc, [Prop | Rest], Bindings) when is_map(Bindings) ->
lookup_var(Loc, _, Invalid) -> lookup_var(Loc, _, Invalid) ->
{error, {Loc, type_name(Invalid)}}. {error, {Loc, type_name(Invalid)}}.
%% @doc Lookup a variable in the bindings accessible through the accessor.
%% Additionally to `lookup_var/2` behavior, this function also tries to parse any
%% binary as JSON to a map if accessor needs to go deeper into it.
-spec lookup_loose_json(accessor(), bindings() | binary()) ->
{ok, binding()} | {error, reason()}.
lookup_loose_json(Var, Bindings) ->
lookup_loose_json(0, Var, Bindings).
lookup_loose_json(_, Var, Value) when Var == ?PH_VAR_THIS orelse Var == [] ->
{ok, Value};
lookup_loose_json(Loc, [Prop | Rest], Bindings) when is_map(Bindings) ->
case lookup(Prop, Bindings) of
{ok, Value} ->
lookup_loose_json(Loc + 1, Rest, Value);
{error, Reason} ->
{error, Reason}
end;
lookup_loose_json(Loc, Rest, Json) when is_binary(Json) ->
try emqx_utils_json:decode(Json) of
Bindings ->
% NOTE: This is intentional, we don't want to parse nested JSON.
lookup_var(Loc, Rest, Bindings)
catch
error:_ ->
{error, {Loc, binary}}
end;
lookup_loose_json(Loc, _, Invalid) ->
{error, {Loc, type_name(Invalid)}}.
type_name(Term) when is_atom(Term) -> atom; type_name(Term) when is_atom(Term) -> atom;
type_name(Term) when is_number(Term) -> number; type_name(Term) when is_number(Term) -> number;
type_name(Term) when is_binary(Term) -> binary; type_name(Term) when is_binary(Term) -> binary;

View File

@ -27,9 +27,9 @@
-export_type([row_template/0]). -export_type([row_template/0]).
-type template() :: emqx_template:t(). -type template() :: emqx_template:str().
-type row_template() :: [emqx_template:placeholder()]. -type row_template() :: [emqx_template:placeholder()].
-type bindings() :: emqx_template:bindings(). -type context() :: emqx_template:context().
-type values() :: [emqx_utils_sql:value()]. -type values() :: [emqx_utils_sql:value()].
@ -62,19 +62,19 @@ parse(String, Opts) ->
%% @doc Render an SQL statement template given a set of bindings. %% @doc Render an SQL statement template given a set of bindings.
%% Interpolation generally follows the SQL syntax, strings are escaped according to the %% Interpolation generally follows the SQL syntax, strings are escaped according to the
%% `escaping` option. %% `escaping` option.
-spec render(template(), bindings(), render_opts()) -> -spec render(template(), context(), render_opts()) ->
{unicode:chardata(), [_Error]}. {unicode:chardata(), [_Error]}.
render(Template, Bindings, Opts) -> render(Template, Context, Opts) ->
emqx_template:render(Template, Bindings, #{ emqx_template:render(Template, Context, #{
var_trans => fun(Value) -> emqx_utils_sql:to_sql_string(Value, Opts) end var_trans => fun(Value) -> emqx_utils_sql:to_sql_string(Value, Opts) end
}). }).
%% @doc Render an SQL statement template given a set of bindings. %% @doc Render an SQL statement template given a set of bindings.
%% Errors are raised if any placeholders are not bound. %% Errors are raised if any placeholders are not bound.
-spec render_strict(template(), bindings(), render_opts()) -> -spec render_strict(template(), context(), render_opts()) ->
unicode:chardata(). unicode:chardata().
render_strict(Template, Bindings, Opts) -> render_strict(Template, Context, Opts) ->
emqx_template:render_strict(Template, Bindings, #{ emqx_template:render_strict(Template, Context, #{
var_trans => fun(Value) -> emqx_utils_sql:to_sql_string(Value, Opts) end var_trans => fun(Value) -> emqx_utils_sql:to_sql_string(Value, Opts) end
}). }).
@ -124,14 +124,14 @@ mk_replace(':n', N) ->
%% An _SQL value_ is a vaguely defined concept here, it is something that's considered %% An _SQL value_ is a vaguely defined concept here, it is something that's considered
%% compatible with the protocol of the database being used. See the definition of %% compatible with the protocol of the database being used. See the definition of
%% `emqx_utils_sql:value()` for more details. %% `emqx_utils_sql:value()` for more details.
-spec render_prepstmt(template(), bindings()) -> -spec render_prepstmt(template(), context()) ->
{values(), [_Error]}. {values(), [_Error]}.
render_prepstmt(Template, Bindings) -> render_prepstmt(Template, Context) ->
Opts = #{var_trans => fun emqx_utils_sql:to_sql_value/1}, Opts = #{var_trans => fun emqx_utils_sql:to_sql_value/1},
emqx_template:render(Template, Bindings, Opts). emqx_template:render(Template, Context, Opts).
-spec render_prepstmt_strict(template(), bindings()) -> -spec render_prepstmt_strict(template(), context()) ->
values(). values().
render_prepstmt_strict(Template, Bindings) -> render_prepstmt_strict(Template, Context) ->
Opts = #{var_trans => fun emqx_utils_sql:to_sql_value/1}, Opts = #{var_trans => fun emqx_utils_sql:to_sql_value/1},
emqx_template:render_strict(Template, Bindings, Opts). emqx_template:render_strict(Template, Context, Opts).

View File

@ -0,0 +1,97 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_jsonish_tests).
-include_lib("eunit/include/eunit.hrl").
prop_prio_test_() ->
[
?_assertEqual(
{ok, 42},
emqx_jsonish:lookup([<<"foo">>], #{<<"foo">> => 42, foo => 1337})
),
?_assertEqual(
{ok, 1337},
emqx_jsonish:lookup([<<"foo">>], #{foo => 1337})
)
].
undefined_test() ->
?assertEqual(
{error, undefined},
emqx_jsonish:lookup([<<"foo">>], #{})
).
undefined_deep_test() ->
?assertEqual(
{error, undefined},
emqx_jsonish:lookup([<<"foo">>, <<"bar">>], #{})
).
undefined_deep_json_test() ->
?assertEqual(
{error, undefined},
emqx_jsonish:lookup(
[<<"foo">>, <<"bar">>, <<"baz">>],
<<"{\"foo\":{\"bar\":{\"no\":{}}}}">>
)
).
invalid_type_test() ->
?assertEqual(
{error, {0, number}},
emqx_jsonish:lookup([<<"foo">>], <<"42">>)
).
invalid_type_deep_test() ->
?assertEqual(
{error, {2, atom}},
emqx_jsonish:lookup([<<"foo">>, <<"bar">>, <<"tuple">>], #{foo => #{bar => baz}})
).
decode_json_test() ->
?assertEqual(
{ok, 42},
emqx_jsonish:lookup([<<"foo">>, <<"bar">>], <<"{\"foo\":{\"bar\":42}}">>)
).
decode_json_deep_test() ->
?assertEqual(
{ok, 42},
emqx_jsonish:lookup([<<"foo">>, <<"bar">>], #{<<"foo">> => <<"{\"bar\": 42}">>})
).
decode_json_invalid_type_test() ->
?assertEqual(
{error, {1, list}},
emqx_jsonish:lookup([<<"foo">>, <<"bar">>], #{<<"foo">> => <<"[1,2,3]">>})
).
decode_no_json_test() ->
?assertEqual(
{error, {1, binary}},
emqx_jsonish:lookup([<<"foo">>, <<"bar">>], #{<<"foo">> => <<0, 1, 2, 3>>})
).
decode_json_no_nested_test() ->
?assertEqual(
{error, {2, binary}},
emqx_jsonish:lookup(
[<<"foo">>, <<"bar">>, <<"baz">>],
#{<<"foo">> => <<"{\"bar\":\"{\\\"baz\\\":42}\"}">>}
)
).

View File

@ -25,7 +25,7 @@
all() -> emqx_common_test_helpers:all(?MODULE). all() -> emqx_common_test_helpers:all(?MODULE).
t_render(_) -> t_render(_) ->
Bindings = #{ Context = #{
a => <<"1">>, a => <<"1">>,
b => 1, b => 1,
c => 1.0, c => 1.0,
@ -38,15 +38,15 @@ t_render(_) ->
), ),
?assertEqual( ?assertEqual(
{<<"a:1,b:1,c:1.0,d:{\"d1\":\"hi\"},d1:hi,l:[0,1,1000],u:utf-8 is ǝɹǝɥ"/utf8>>, []}, {<<"a:1,b:1,c:1.0,d:{\"d1\":\"hi\"},d1:hi,l:[0,1,1000],u:utf-8 is ǝɹǝɥ"/utf8>>, []},
render_string(Template, Bindings) render_string(Template, Context)
). ).
t_render_var_trans(_) -> t_render_var_trans(_) ->
Bindings = #{a => <<"1">>, b => 1, c => #{prop => 1.0}}, Context = #{a => <<"1">>, b => 1, c => #{prop => 1.0}},
Template = emqx_template:parse(<<"a:${a},b:${b},c:${c.prop}">>), Template = emqx_template:parse(<<"a:${a},b:${b},c:${c.prop}">>),
{String, Errors} = emqx_template:render( {String, Errors} = emqx_template:render(
Template, Template,
Bindings, Context,
#{var_trans => fun(Name, _) -> "<" ++ Name ++ ">" end} #{var_trans => fun(Name, _) -> "<" ++ Name ++ ">" end}
), ),
?assertEqual( ?assertEqual(
@ -55,7 +55,7 @@ t_render_var_trans(_) ->
). ).
t_render_path(_) -> t_render_path(_) ->
Bindings = #{d => #{d1 => <<"hi">>}}, Context = #{d => #{d1 => <<"hi">>}},
Template = emqx_template:parse(<<"d.d1:${d.d1}">>), Template = emqx_template:parse(<<"d.d1:${d.d1}">>),
?assertEqual( ?assertEqual(
ok, ok,
@ -63,11 +63,11 @@ t_render_path(_) ->
), ),
?assertEqual( ?assertEqual(
{<<"d.d1:hi">>, []}, {<<"d.d1:hi">>, []},
render_string(Template, Bindings) render_string(Template, Context)
). ).
t_render_custom_ph(_) -> t_render_custom_ph(_) ->
Bindings = #{a => <<"a">>, b => <<"b">>}, Context = #{a => <<"a">>, b => <<"b">>},
Template = emqx_template:parse(<<"a:${a},b:${b}">>), Template = emqx_template:parse(<<"a:${a},b:${b}">>),
?assertEqual( ?assertEqual(
{error, [{"b", disallowed}]}, {error, [{"b", disallowed}]},
@ -75,21 +75,21 @@ t_render_custom_ph(_) ->
), ),
?assertEqual( ?assertEqual(
<<"a:a,b:b">>, <<"a:a,b:b">>,
render_strict_string(Template, Bindings) render_strict_string(Template, Context)
). ).
t_render_this(_) -> t_render_this(_) ->
Bindings = #{a => <<"a">>, b => [1, 2, 3]}, Context = #{a => <<"a">>, b => [1, 2, 3]},
Template = emqx_template:parse(<<"this:${} / also:${.}">>), Template = emqx_template:parse(<<"this:${} / also:${.}">>),
?assertEqual(ok, emqx_template:validate(["."], Template)), ?assertEqual(ok, emqx_template:validate(["."], Template)),
?assertEqual( ?assertEqual(
% NOTE: order of the keys in the JSON object depends on the JSON encoder % NOTE: order of the keys in the JSON object depends on the JSON encoder
<<"this:{\"b\":[1,2,3],\"a\":\"a\"} / also:{\"b\":[1,2,3],\"a\":\"a\"}">>, <<"this:{\"b\":[1,2,3],\"a\":\"a\"} / also:{\"b\":[1,2,3],\"a\":\"a\"}">>,
render_strict_string(Template, Bindings) render_strict_string(Template, Context)
). ).
t_render_missing_bindings(_) -> t_render_missing_bindings(_) ->
Bindings = #{no => #{}, c => #{<<"c1">> => 42}}, Context = #{no => #{}, c => #{<<"c1">> => 42}},
Template = emqx_template:parse( Template = emqx_template:parse(
<<"a:${a},b:${b},c:${c.c1.c2},d:${d.d1},e:${no.such_atom_i_swear}">> <<"a:${a},b:${b},c:${c.c1.c2},d:${d.d1},e:${no.such_atom_i_swear}">>
), ),
@ -101,7 +101,7 @@ t_render_missing_bindings(_) ->
{"b", undefined}, {"b", undefined},
{"a", undefined} {"a", undefined}
]}, ]},
render_string(Template, Bindings) render_string(Template, Context)
), ),
?assertError( ?assertError(
[ [
@ -111,7 +111,21 @@ t_render_missing_bindings(_) ->
{"b", undefined}, {"b", undefined},
{"a", undefined} {"a", undefined}
], ],
render_strict_string(Template, Bindings) render_strict_string(Template, Context)
).
t_render_custom_bindings(_) ->
_ = erlang:put(a, <<"foo">>),
_ = erlang:put(b, #{<<"bar">> => #{atom => 42}}),
Template = emqx_template:parse(
<<"a:${a},b:${b.bar.atom},c:${c},oops:${b.bar.atom.oops}">>
),
?assertEqual(
{<<"a:foo,b:42,c:undefined,oops:undefined">>, [
{"b.bar.atom.oops", {2, number}},
{"c", undefined}
]},
render_string(Template, {?MODULE, []})
). ).
t_unparse(_) -> t_unparse(_) ->
@ -141,33 +155,33 @@ t_const(_) ->
). ).
t_render_partial_ph(_) -> t_render_partial_ph(_) ->
Bindings = #{a => <<"1">>, b => 1, c => 1.0, d => #{d1 => <<"hi">>}}, Context = #{a => <<"1">>, b => 1, c => 1.0, d => #{d1 => <<"hi">>}},
Template = emqx_template:parse(<<"a:$a,b:b},c:{c},d:${d">>), Template = emqx_template:parse(<<"a:$a,b:b},c:{c},d:${d">>),
?assertEqual( ?assertEqual(
<<"a:$a,b:b},c:{c},d:${d">>, <<"a:$a,b:b},c:{c},d:${d">>,
render_strict_string(Template, Bindings) render_strict_string(Template, Context)
). ).
t_parse_escaped(_) -> t_parse_escaped(_) ->
Bindings = #{a => <<"1">>, b => 1, c => "VAR"}, Context = #{a => <<"1">>, b => 1, c => "VAR"},
Template = emqx_template:parse(<<"a:${a},b:${$}{b},c:${$}{${c}},lit:${$}{$}">>), Template = emqx_template:parse(<<"a:${a},b:${$}{b},c:${$}{${c}},lit:${$}{$}">>),
?assertEqual( ?assertEqual(
<<"a:1,b:${b},c:${VAR},lit:${$}">>, <<"a:1,b:${b},c:${VAR},lit:${$}">>,
render_strict_string(Template, Bindings) render_strict_string(Template, Context)
). ).
t_parse_escaped_dquote(_) -> t_parse_escaped_dquote(_) ->
Bindings = #{a => <<"1">>, b => 1}, Context = #{a => <<"1">>, b => 1},
Template = emqx_template:parse(<<"a:\"${a}\",b:\"${$}{b}\"">>, #{ Template = emqx_template:parse(<<"a:\"${a}\",b:\"${$}{b}\"">>, #{
strip_double_quote => true strip_double_quote => true
}), }),
?assertEqual( ?assertEqual(
<<"a:1,b:\"${b}\"">>, <<"a:1,b:\"${b}\"">>,
render_strict_string(Template, Bindings) render_strict_string(Template, Context)
). ).
t_parse_sql_prepstmt(_) -> t_parse_sql_prepstmt(_) ->
Bindings = #{a => <<"1">>, b => 1, c => 1.0, d => #{d1 => <<"hi">>}}, Context = #{a => <<"1">>, b => 1, c => 1.0, d => #{d1 => <<"hi">>}},
{PrepareStatement, RowTemplate} = {PrepareStatement, RowTemplate} =
emqx_template_sql:parse_prepstmt(<<"a:${a},b:${b},c:${c},d:${d}">>, #{ emqx_template_sql:parse_prepstmt(<<"a:${a},b:${b},c:${c},d:${d}">>, #{
parameters => '?' parameters => '?'
@ -175,11 +189,11 @@ t_parse_sql_prepstmt(_) ->
?assertEqual(<<"a:?,b:?,c:?,d:?">>, bin(PrepareStatement)), ?assertEqual(<<"a:?,b:?,c:?,d:?">>, bin(PrepareStatement)),
?assertEqual( ?assertEqual(
{[<<"1">>, 1, 1.0, <<"{\"d1\":\"hi\"}">>], _Errors = []}, {[<<"1">>, 1, 1.0, <<"{\"d1\":\"hi\"}">>], _Errors = []},
emqx_template_sql:render_prepstmt(RowTemplate, Bindings) emqx_template_sql:render_prepstmt(RowTemplate, Context)
). ).
t_parse_sql_prepstmt_n(_) -> t_parse_sql_prepstmt_n(_) ->
Bindings = #{a => undefined, b => true, c => atom, d => #{d1 => 42.1337}}, Context = #{a => undefined, b => true, c => atom, d => #{d1 => 42.1337}},
{PrepareStatement, RowTemplate} = {PrepareStatement, RowTemplate} =
emqx_template_sql:parse_prepstmt(<<"a:${a},b:${b},c:${c},d:${d}">>, #{ emqx_template_sql:parse_prepstmt(<<"a:${a},b:${b},c:${c},d:${d}">>, #{
parameters => '$n' parameters => '$n'
@ -187,7 +201,7 @@ t_parse_sql_prepstmt_n(_) ->
?assertEqual(<<"a:$1,b:$2,c:$3,d:$4">>, bin(PrepareStatement)), ?assertEqual(<<"a:$1,b:$2,c:$3,d:$4">>, bin(PrepareStatement)),
?assertEqual( ?assertEqual(
[null, true, <<"atom">>, <<"{\"d1\":42.1337}">>], [null, true, <<"atom">>, <<"{\"d1\":42.1337}">>],
emqx_template_sql:render_prepstmt_strict(RowTemplate, Bindings) emqx_template_sql:render_prepstmt_strict(RowTemplate, Context)
). ).
t_parse_sql_prepstmt_colon(_) -> t_parse_sql_prepstmt_colon(_) ->
@ -198,14 +212,14 @@ t_parse_sql_prepstmt_colon(_) ->
?assertEqual(<<"a=:1,b=:2,c=:3,d=:4">>, bin(PrepareStatement)). ?assertEqual(<<"a=:1,b=:2,c=:3,d=:4">>, bin(PrepareStatement)).
t_parse_sql_prepstmt_partial_ph(_) -> t_parse_sql_prepstmt_partial_ph(_) ->
Bindings = #{a => <<"1">>, b => 1, c => 1.0, d => #{d1 => <<"hi">>}}, Context = #{a => <<"1">>, b => 1, c => 1.0, d => #{d1 => <<"hi">>}},
{PrepareStatement, RowTemplate} = {PrepareStatement, RowTemplate} =
emqx_template_sql:parse_prepstmt(<<"a:$a,b:b},c:{c},d:${d">>, #{parameters => '?'}), emqx_template_sql:parse_prepstmt(<<"a:$a,b:b},c:{c},d:${d">>, #{parameters => '?'}),
?assertEqual(<<"a:$a,b:b},c:{c},d:${d">>, bin(PrepareStatement)), ?assertEqual(<<"a:$a,b:b},c:{c},d:${d">>, bin(PrepareStatement)),
?assertEqual([], emqx_template_sql:render_prepstmt_strict(RowTemplate, Bindings)). ?assertEqual([], emqx_template_sql:render_prepstmt_strict(RowTemplate, Context)).
t_render_sql(_) -> t_render_sql(_) ->
Bindings = #{ Context = #{
a => <<"1">>, a => <<"1">>,
b => 1, b => 1,
c => 1.0, c => 1.0,
@ -216,17 +230,17 @@ t_render_sql(_) ->
Template = emqx_template:parse(<<"a:${a},b:${b},c:${c},d:${d},n:${n},u:${u}">>), Template = emqx_template:parse(<<"a:${a},b:${b},c:${c},d:${d},n:${n},u:${u}">>),
?assertMatch( ?assertMatch(
{_String, _Errors = []}, {_String, _Errors = []},
emqx_template_sql:render(Template, Bindings, #{}) emqx_template_sql:render(Template, Context, #{})
), ),
?assertEqual( ?assertEqual(
<<"a:'1',b:1,c:1.0,d:'{\"d1\":\"hi\"}',n:NULL,u:'utf8\\'s cool 🐸'"/utf8>>, <<"a:'1',b:1,c:1.0,d:'{\"d1\":\"hi\"}',n:NULL,u:'utf8\\'s cool 🐸'"/utf8>>,
bin(emqx_template_sql:render_strict(Template, Bindings, #{})) bin(emqx_template_sql:render_strict(Template, Context, #{}))
). ).
t_render_mysql(_) -> t_render_mysql(_) ->
%% with apostrophes %% with apostrophes
%% https://github.com/emqx/emqx/issues/4135 %% https://github.com/emqx/emqx/issues/4135
Bindings = #{ Context = #{
a => <<"1''2">>, a => <<"1''2">>,
b => 1, b => 1,
c => 1.0, c => 1.0,
@ -245,13 +259,13 @@ t_render_mysql(_) ->
"e:'\\\\\\0💩',f:0x6E6F6E2D75746638DCC900,g:'utf8\\'s cool 🐸',"/utf8, "e:'\\\\\\0💩',f:0x6E6F6E2D75746638DCC900,g:'utf8\\'s cool 🐸',"/utf8,
"h:'imgood'" "h:'imgood'"
>>, >>,
bin(emqx_template_sql:render_strict(Template, Bindings, #{escaping => mysql})) bin(emqx_template_sql:render_strict(Template, Context, #{escaping => mysql}))
). ).
t_render_cql(_) -> t_render_cql(_) ->
%% with apostrophes for cassandra %% with apostrophes for cassandra
%% https://github.com/emqx/emqx/issues/4148 %% https://github.com/emqx/emqx/issues/4148
Bindings = #{ Context = #{
a => <<"1''2">>, a => <<"1''2">>,
b => 1, b => 1,
c => 1.0, c => 1.0,
@ -260,7 +274,7 @@ t_render_cql(_) ->
Template = emqx_template:parse(<<"a:${a},b:${b},c:${c},d:${d}">>), Template = emqx_template:parse(<<"a:${a},b:${b},c:${c},d:${d}">>),
?assertEqual( ?assertEqual(
<<"a:'1''''2',b:1,c:1.0,d:'{\"d1\":\"someone''s phone\"}'">>, <<"a:'1''''2',b:1,c:1.0,d:'{\"d1\":\"someone''s phone\"}'">>,
bin(emqx_template_sql:render_strict(Template, Bindings, #{escaping => cql})) bin(emqx_template_sql:render_strict(Template, Context, #{escaping => cql}))
). ).
t_render_sql_custom_ph(_) -> t_render_sql_custom_ph(_) ->
@ -273,7 +287,7 @@ t_render_sql_custom_ph(_) ->
?assertEqual(<<"a:$1,b:$2">>, bin(PrepareStatement)). ?assertEqual(<<"a:$1,b:$2">>, bin(PrepareStatement)).
t_render_sql_strip_double_quote(_) -> t_render_sql_strip_double_quote(_) ->
Bindings = #{a => <<"a">>, b => <<"b">>}, Context = #{a => <<"a">>, b => <<"b">>},
%% no strip_double_quote option: "${key}" -> "value" %% no strip_double_quote option: "${key}" -> "value"
{PrepareStatement1, RowTemplate1} = emqx_template_sql:parse_prepstmt( {PrepareStatement1, RowTemplate1} = emqx_template_sql:parse_prepstmt(
@ -283,7 +297,7 @@ t_render_sql_strip_double_quote(_) ->
?assertEqual(<<"a:\"$1\",b:\"$2\"">>, bin(PrepareStatement1)), ?assertEqual(<<"a:\"$1\",b:\"$2\"">>, bin(PrepareStatement1)),
?assertEqual( ?assertEqual(
[<<"a">>, <<"b">>], [<<"a">>, <<"b">>],
emqx_template_sql:render_prepstmt_strict(RowTemplate1, Bindings) emqx_template_sql:render_prepstmt_strict(RowTemplate1, Context)
), ),
%% strip_double_quote = true: "${key}" -> value %% strip_double_quote = true: "${key}" -> value
@ -294,11 +308,11 @@ t_render_sql_strip_double_quote(_) ->
?assertEqual(<<"a:$1,b:$2">>, bin(PrepareStatement2)), ?assertEqual(<<"a:$1,b:$2">>, bin(PrepareStatement2)),
?assertEqual( ?assertEqual(
[<<"a">>, <<"b">>], [<<"a">>, <<"b">>],
emqx_template_sql:render_prepstmt_strict(RowTemplate2, Bindings) emqx_template_sql:render_prepstmt_strict(RowTemplate2, Context)
). ).
t_render_tmpl_deep(_) -> t_render_tmpl_deep(_) ->
Bindings = #{a => <<"1">>, b => 1, c => 1.0, d => #{d1 => <<"hi">>}}, Context = #{a => <<"1">>, b => 1, c => 1.0, d => #{d1 => <<"hi">>}},
Template = emqx_template:parse_deep( Template = emqx_template:parse_deep(
#{<<"${a}">> => [<<"$${b}">>, "c", 2, 3.0, '${d}', {[<<"${c}">>, <<"${$}{d}">>], 0}]} #{<<"${a}">> => [<<"$${b}">>, "c", 2, 3.0, '${d}', {[<<"${c}">>, <<"${$}{d}">>], 0}]}
@ -311,7 +325,7 @@ t_render_tmpl_deep(_) ->
?assertEqual( ?assertEqual(
#{<<"1">> => [<<"$1">>, "c", 2, 3.0, '${d}', {[<<"1.0">>, <<"${d}">>], 0}]}, #{<<"1">> => [<<"$1">>, "c", 2, 3.0, '${d}', {[<<"1.0">>, <<"${d}">>], 0}]},
emqx_template:render_strict(Template, Bindings) emqx_template:render_strict(Template, Context)
). ).
t_unparse_tmpl_deep(_) -> t_unparse_tmpl_deep(_) ->
@ -321,12 +335,22 @@ t_unparse_tmpl_deep(_) ->
%% %%
render_string(Template, Bindings) -> render_string(Template, Context) ->
{String, Errors} = emqx_template:render(Template, Bindings), {String, Errors} = emqx_template:render(Template, Context),
{bin(String), Errors}. {bin(String), Errors}.
render_strict_string(Template, Bindings) -> render_strict_string(Template, Context) ->
bin(emqx_template:render_strict(Template, Bindings)). bin(emqx_template:render_strict(Template, Context)).
bin(String) -> bin(String) ->
unicode:characters_to_binary(String). unicode:characters_to_binary(String).
%% Access module API
lookup([], _) ->
{error, undefined};
lookup([Prop | Rest], _) ->
case erlang:get(binary_to_atom(Prop)) of
undefined -> {error, undefined};
Value -> emqx_template:lookup_var(Rest, Value)
end.