From 28d55d72ca0738b923328d3deedb06ffee0b6aad Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 18 Apr 2023 13:03:02 +0300 Subject: [PATCH 01/21] feat(tpl): split `emqx_placeholder` into a couple of modules Located under `emqx_connector` for now. The APIs of the new modules are slightly different from the old ones. The new ones are more explicit in terms of error handling. Also copy the according testsuite from `emqx_plugin_libs` and enrich it for better coverage. --- .../emqx_connector/src/emqx_connector_sql.erl | 159 ++++++++ .../src/emqx_connector_template.erl | 351 ++++++++++++++++++ .../src/emqx_connector_template_sql.erl | 135 +++++++ .../src/emqx_connector_utils.erl | 35 -- .../test/emqx_connector_template_SUITE.erl | 323 ++++++++++++++++ 5 files changed, 968 insertions(+), 35 deletions(-) create mode 100644 apps/emqx_connector/src/emqx_connector_sql.erl create mode 100644 apps/emqx_connector/src/emqx_connector_template.erl create mode 100644 apps/emqx_connector/src/emqx_connector_template_sql.erl delete mode 100644 apps/emqx_connector/src/emqx_connector_utils.erl create mode 100644 apps/emqx_connector/test/emqx_connector_template_SUITE.erl diff --git a/apps/emqx_connector/src/emqx_connector_sql.erl b/apps/emqx_connector/src/emqx_connector_sql.erl new file mode 100644 index 000000000..be0b220e6 --- /dev/null +++ b/apps/emqx_connector/src/emqx_connector_sql.erl @@ -0,0 +1,159 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_connector_sql). + +-export([get_statement_type/1]). +-export([parse_insert/1]). + +-export([to_sql_value/1]). +-export([to_sql_string/2]). + +-export([escape_sql/1]). +-export([escape_cql/1]). +-export([escape_mysql/1]). + +-export_type([value/0]). + +-type statement_type() :: select | insert | delete. +-type value() :: null | binary() | number() | boolean() | [value()]. + +-dialyzer({no_improper_lists, [escape_mysql/4, escape_prepend/4]}). + +-spec get_statement_type(iodata()) -> statement_type() | {error, unknown}. +get_statement_type(Query) -> + KnownTypes = #{ + <<"select">> => select, + <<"insert">> => insert, + <<"delete">> => delete + }, + case re:run(Query, <<"^\\s*([a-zA-Z]+)">>, [{capture, all_but_first, binary}]) of + {match, [Token]} -> + maps:get(string:lowercase(Token), KnownTypes, {error, unknown}); + _ -> + {error, unknown} + end. + +%% @doc Parse an INSERT SQL statement into its INSERT part and the VALUES part. +%% SQL = <<"INSERT INTO \"abc\" (c1, c2, c3) VALUES (${a}, ${b}, ${c.prop})">> +%% {ok, {<<"INSERT INTO \"abc\" (c1, c2, c3)">>, <<"(${a}, ${b}, ${c.prop})">>}} +-spec parse_insert(iodata()) -> + {ok, {_Statement :: binary(), _Rows :: binary()}} | {error, not_insert_sql}. +parse_insert(SQL) -> + case re:split(SQL, "((?i)values)", [{return, binary}]) of + [Part1, _, Part3] -> + case string:trim(Part1, leading) of + <<"insert", _/binary>> = InsertSQL -> + {ok, {InsertSQL, Part3}}; + <<"INSERT", _/binary>> = InsertSQL -> + {ok, {InsertSQL, Part3}}; + _ -> + {error, not_insert_sql} + end; + _ -> + {error, not_insert_sql} + end. + +%% @doc Convert an Erlang term to a value that can be used primarily in +%% prepared SQL statements. +-spec to_sql_value(term()) -> value(). +to_sql_value(undefined) -> null; +to_sql_value(List) when is_list(List) -> List; +to_sql_value(Bin) when is_binary(Bin) -> Bin; +to_sql_value(Num) when is_number(Num) -> Num; +to_sql_value(Bool) when is_boolean(Bool) -> Bool; +to_sql_value(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8); +to_sql_value(Map) when is_map(Map) -> emqx_utils_json:encode(Map). + +%% @doc Convert an Erlang term to a string that can be interpolated in literal +%% SQL statements. The value is escaped if necessary. +-spec to_sql_string(term(), Options) -> iodata() when + Options :: #{ + escaping => cql | mysql | sql + }. +to_sql_string(String, #{escaping := mysql}) when is_binary(String) -> + try + escape_mysql(String) + catch + throw:invalid_utf8 -> + [<<"0x">>, binary:encode_hex(String)] + end; +to_sql_string(Term, #{escaping := mysql}) -> + maybe_escape(Term, fun escape_mysql/1); +to_sql_string(Term, #{escaping := cql}) -> + maybe_escape(Term, fun escape_cql/1); +to_sql_string(Term, #{}) -> + maybe_escape(Term, fun escape_sql/1). + +-spec maybe_escape(_Value, fun((binary()) -> iodata())) -> iodata(). +maybe_escape(undefined, _EscapeFun) -> + <<"NULL">>; +maybe_escape(Str, EscapeFun) when is_binary(Str) -> + EscapeFun(Str); +maybe_escape(Str, EscapeFun) when is_list(Str) -> + case unicode:characters_to_binary(Str) of + Bin when is_binary(Bin) -> + EscapeFun(Bin); + Otherwise -> + error(Otherwise) + end; +maybe_escape(Val, EscapeFun) when is_atom(Val) orelse is_map(Val) -> + EscapeFun(emqx_connector_template:to_string(Val)); +maybe_escape(Val, _EscapeFun) -> + emqx_connector_template:to_string(Val). + +-spec escape_sql(binary()) -> iodata(). +escape_sql(S) -> + % NOTE + % This is a bit misleading: currently, escaping logic in `escape_sql/1` likely + % won't work with pgsql since it does not support C-style escapes by default. + % https://www.postgresql.org/docs/14/sql-syntax-lexical.html#SQL-SYNTAX-CONSTANTS + ES = binary:replace(S, [<<"\\">>, <<"'">>], <<"\\">>, [global, {insert_replaced, 1}]), + [$', ES, $']. + +-spec escape_cql(binary()) -> iodata(). +escape_cql(S) -> + ES = binary:replace(S, <<"'">>, <<"'">>, [global, {insert_replaced, 1}]), + [$', ES, $']. + +-spec escape_mysql(binary()) -> iodata(). +escape_mysql(S0) -> + % https://dev.mysql.com/doc/refman/8.0/en/string-literals.html + [$', escape_mysql(S0, 0, 0, S0), $']. + +%% NOTE +%% This thing looks more complicated than needed because it's optimized for as few +%% intermediate memory (re)allocations as possible. +escape_mysql(<<$', Rest/binary>>, I, Run, Src) -> + escape_prepend(I, Run, Src, [<<"\\'">> | escape_mysql(Rest, I + Run + 1, 0, Src)]); +escape_mysql(<<$\\, Rest/binary>>, I, Run, Src) -> + escape_prepend(I, Run, Src, [<<"\\\\">> | escape_mysql(Rest, I + Run + 1, 0, Src)]); +escape_mysql(<<0, Rest/binary>>, I, Run, Src) -> + escape_prepend(I, Run, Src, [<<"\\0">> | escape_mysql(Rest, I + Run + 1, 0, Src)]); +escape_mysql(<<_/utf8, Rest/binary>> = S, I, Run, Src) -> + CWidth = byte_size(S) - byte_size(Rest), + escape_mysql(Rest, I, Run + CWidth, Src); +escape_mysql(<<>>, 0, _, Src) -> + Src; +escape_mysql(<<>>, I, Run, Src) -> + binary:part(Src, I, Run); +escape_mysql(_, _I, _Run, _Src) -> + throw(invalid_utf8). + +escape_prepend(_RunI, 0, _Src, Tail) -> + Tail; +escape_prepend(I, Run, Src, Tail) -> + [binary:part(Src, I, Run) | Tail]. diff --git a/apps/emqx_connector/src/emqx_connector_template.erl b/apps/emqx_connector/src/emqx_connector_template.erl new file mode 100644 index 000000000..c346d4289 --- /dev/null +++ b/apps/emqx_connector/src/emqx_connector_template.erl @@ -0,0 +1,351 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_connector_template). + +-include_lib("emqx/include/emqx_placeholder.hrl"). + +-export([parse/1]). +-export([parse/2]). +-export([parse_deep/1]). +-export([parse_deep/2]). +-export([validate/2]). +-export([trivial/1]). +-export([unparse/1]). +-export([render/2]). +-export([render/3]). +-export([render_strict/2]). +-export([render_strict/3]). + +-export([to_string/1]). + +-export_type([t/0]). +-export_type([str/0]). +-export_type([deep/0]). +-export_type([placeholder/0]). +-export_type([bindings/0]). + +-type t() :: str() | {'$tpl', deeptpl()}. + +-type str() :: [unicode:chardata() | placeholder()]. +-type deep() :: {'$tpl', deeptpl()}. + +-type deeptpl() :: + t() + | #{deeptpl() => deeptpl()} + | {list, [deeptpl()]} + | {tuple, [deeptpl()]} + | scalar() + | function() + | pid() + | port() + | reference(). + +-type placeholder() :: {var, var()}. +-type var() :: _Name :: [binary()]. + +-type scalar() :: atom() | unicode:chardata() | number(). +-type binding() :: scalar() | list(scalar()) | bindings(). +-type bindings() :: #{atom() | binary() => binding()}. + +-type var_trans() :: + fun((Value :: term()) -> unicode:chardata()) + | fun((var(), Value :: term()) -> unicode:chardata()). + +-type parse_opts() :: #{ + strip_double_quote => boolean() +}. + +-type render_opts() :: #{ + var_trans => var_trans() +}. + +-define(RE_PLACEHOLDER, "\\$(\\$?)\\{[.]?([a-zA-Z0-9._]*)\\}"). + +%% @doc Parse a unicode string into a template. +%% String might contain zero or more of placeholders in the form of `${var}`, +%% where `var` is a _location_ (possibly deeply nested) of some value in the +%% bindings map. +%% String might contain special escaped form `$${...}` which interpreted as a +%% literal `${...}`. +-spec parse(String :: unicode:chardata()) -> + t(). +parse(String) -> + parse(String, #{}). + +-spec parse(String :: unicode:chardata(), parse_opts()) -> + t(). +parse(String, Opts) -> + RE = + case Opts of + #{strip_double_quote := true} -> + <<"((?|" ?RE_PLACEHOLDER "|\"" ?RE_PLACEHOLDER "\"))">>; + #{} -> + <<"(" ?RE_PLACEHOLDER ")">> + end, + Splits = re:split(String, RE, [{return, binary}, group, trim, unicode]), + Components = lists:flatmap(fun parse_split/1, Splits), + Components. + +parse_split([Part, _PH, <<>>, Var]) -> + % Regular placeholder + prepend(Part, [{var, parse_var(Var)}]); +parse_split([Part, _PH = <>, <<"$">>, _]) -> + % Escaped literal, take all but the second byte, which is always `$`. + % Important to make a whole token starting with `$` so the `unparse/11` + % function can distinguish escaped literals. + prepend(Part, [<>]); +parse_split([Tail]) -> + [Tail]. + +prepend(<<>>, To) -> + To; +prepend(Head, To) -> + [Head | To]. + +parse_var(Var) -> + case string:split(Var, <<".">>, all) of + [<<>>] -> + ?PH_VAR_THIS; + Name -> + % TODO: lowercase? + Name + end. + +-spec validate([var() | binary()], t()) -> + ok | {error, [_Error :: {var(), disallowed}]}. +validate(AllowedIn, Template) -> + Allowed = [try_parse_var(V) || V <- AllowedIn], + {_, Errors} = render(Template, #{}), + {Used, _} = lists:unzip(Errors), + case lists:usort(Used) -- Allowed of + [] -> + ok; + Disallowed -> + {error, [{Var, disallowed} || Var <- Disallowed]} + end. + +try_parse_var(Var) when is_binary(Var) -> + parse_var(Var); +try_parse_var(Name) when is_list(Name) -> + Name. + +-spec trivial(t()) -> + boolean(). +trivial(Template) -> + validate([], Template) == ok. + +-spec unparse(t()) -> + unicode:chardata(). +unparse({'$tpl', Template}) -> + unparse_deep(Template); +unparse(Template) -> + lists:map(fun unparse_part/1, Template). + +unparse_part({var, Name}) -> + render_placeholder(Name); +unparse_part(Part = <<"${", _/binary>>) -> + <<"$", Part/binary>>; +unparse_part(Part) -> + Part. + +render_placeholder(Name) -> + "${" ++ lists:join($., Name) ++ "}". + +%% @doc Render a template with given bindings. +%% Returns a term with all placeholders replaced with values from bindings. +%% If one or more placeholders are not found in bindings, an error is returned. +%% By default, all binding values are converted to strings using `to_string/1` +%% function. Option `var_trans` can be used to override this behaviour. +-spec render(t(), bindings()) -> + {term(), [_Error :: {var(), undefined}]}. +render(Template, Bindings) -> + render(Template, Bindings, #{}). + +-spec render(t(), bindings(), render_opts()) -> + {term(), [_Error :: {var(), undefined}]}. +render(Template, Bindings, Opts) when is_list(Template) -> + lists:mapfoldl( + fun + ({var, Name}, EAcc) -> + {String, Errors} = render_binding(Name, Bindings, Opts), + {String, Errors ++ EAcc}; + (String, EAcc) -> + {String, EAcc} + end, + [], + Template + ); +render({'$tpl', Template}, Bindings, Opts) -> + render_deep(Template, Bindings, Opts). + +render_binding(Name, Bindings, Opts) -> + case lookup_var(Name, Bindings) of + {ok, Value} -> + {render_value(Name, Value, Opts), []}; + {error, Reason} -> + % TODO + % Currently, it's not possible to distinguish between a missing value + % and an atom `undefined` in `TransFun`. + {render_value(Name, undefined, Opts), [{Name, Reason}]} + end. + +render_value(_Name, Value, #{var_trans := TransFun}) when is_function(TransFun, 1) -> + TransFun(Value); +render_value(Name, Value, #{var_trans := TransFun}) when is_function(TransFun, 2) -> + TransFun(Name, Value); +render_value(_Name, Value, #{}) -> + to_string(Value). + +-spec render_strict(t(), bindings()) -> + unicode:chardata(). +render_strict(Template, Bindings) -> + render_strict(Template, Bindings, #{}). + +-spec render_strict(t(), bindings(), render_opts()) -> + unicode:chardata(). +render_strict(Template, Bindings, Opts) -> + case render(Template, Bindings, Opts) of + {String, []} -> + String; + {_, Errors = [_ | _]} -> + error(Errors, [unicode:characters_to_list(unparse(Template)), Bindings]) + end. + +%% @doc Parse an arbitrary Erlang term into a "deep" template. +%% Any binaries nested in the term are treated as string templates, while +%% lists are not analyzed for "printability" and are treated as nested terms. +%% The result is a usual template, and can be fed to other functions in this +%% module. +-spec parse_deep(unicode:chardata()) -> + t(). +parse_deep(Term) -> + parse_deep(Term, #{}). + +-spec parse_deep(unicode:chardata(), parse_opts()) -> + t(). +parse_deep(Term, Opts) -> + {'$tpl', parse_deep_term(Term, Opts)}. + +parse_deep_term(Term, Opts) when is_map(Term) -> + maps:fold( + fun(K, V, Acc) -> + Acc#{parse_deep_term(K, Opts) => parse_deep_term(V, Opts)} + end, + #{}, + Term + ); +parse_deep_term(Term, Opts) when is_list(Term) -> + {list, [parse_deep_term(E, Opts) || E <- Term]}; +parse_deep_term(Term, Opts) when is_tuple(Term) -> + {tuple, [parse_deep_term(E, Opts) || E <- tuple_to_list(Term)]}; +parse_deep_term(Term, Opts) when is_binary(Term) -> + parse(Term, Opts); +parse_deep_term(Term, _Opts) -> + Term. + +render_deep(Template, Bindings, Opts) when is_map(Template) -> + maps:fold( + fun(KT, VT, {Acc, Errors}) -> + {K, KErrors} = render_deep(KT, Bindings, Opts), + {V, VErrors} = render_deep(VT, Bindings, Opts), + {Acc#{K => V}, KErrors ++ VErrors ++ Errors} + end, + {#{}, []}, + Template + ); +render_deep({list, Template}, Bindings, Opts) when is_list(Template) -> + lists:mapfoldr( + fun(T, Errors) -> + {E, VErrors} = render_deep(T, Bindings, Opts), + {E, VErrors ++ Errors} + end, + [], + Template + ); +render_deep({tuple, Template}, Bindings, Opts) when is_list(Template) -> + {Term, Errors} = render_deep({list, Template}, Bindings, Opts), + {list_to_tuple(Term), Errors}; +render_deep(Template, Bindings, Opts) when is_list(Template) -> + {String, Errors} = render(Template, Bindings, Opts), + {unicode:characters_to_binary(String), Errors}; +render_deep(Term, _Bindings, _Opts) -> + {Term, []}. + +unparse_deep(Template) when is_map(Template) -> + maps:fold( + fun(K, V, Acc) -> + Acc#{unparse_deep(K) => unparse_deep(V)} + end, + #{}, + Template + ); +unparse_deep({list, Template}) when is_list(Template) -> + [unparse_deep(E) || E <- Template]; +unparse_deep({tuple, Template}) when is_list(Template) -> + list_to_tuple(unparse_deep({list, Template})); +unparse_deep(Template) when is_list(Template) -> + unicode:characters_to_binary(unparse(Template)); +unparse_deep(Term) -> + Term. + +%% + +-spec lookup_var(var(), bindings()) -> + {ok, binding()} | {error, undefined}. +lookup_var(?PH_VAR_THIS, Value) -> + {ok, Value}; +lookup_var([], Value) -> + {ok, Value}; +lookup_var([Prop | Rest], Bindings) -> + case lookup(Prop, Bindings) of + {ok, Value} -> + lookup_var(Rest, Value); + {error, Reason} -> + {error, Reason} + end. + +-spec lookup(Prop :: binary(), bindings()) -> + {ok, binding()} | {error, undefined}. +lookup(Prop, Bindings) when is_binary(Prop) -> + case maps:get(Prop, Bindings, undefined) of + undefined -> + try + {ok, maps:get(binary_to_existing_atom(Prop, utf8), Bindings)} + catch + error:{badkey, _} -> + {error, undefined}; + error:badarg -> + {error, undefined} + end; + Value -> + {ok, Value} + end. + +-spec to_string(binding()) -> + unicode:chardata(). +to_string(undefined) -> + []; +to_string(Bin) when is_binary(Bin) -> Bin; +to_string(Num) when is_integer(Num) -> integer_to_binary(Num); +to_string(Num) when is_float(Num) -> float_to_binary(Num, [{decimals, 10}, compact]); +to_string(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8); +to_string(Map) when is_map(Map) -> emqx_utils_json:encode(Map); +to_string(List) when is_list(List) -> + case io_lib:printable_unicode_list(List) of + true -> List; + false -> emqx_utils_json:encode(List) + end. diff --git a/apps/emqx_connector/src/emqx_connector_template_sql.erl b/apps/emqx_connector/src/emqx_connector_template_sql.erl new file mode 100644 index 000000000..0febfe575 --- /dev/null +++ b/apps/emqx_connector/src/emqx_connector_template_sql.erl @@ -0,0 +1,135 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_connector_template_sql). + +-export([parse/1]). +-export([parse/2]). +-export([render/3]). +-export([render_strict/3]). + +-export([parse_prepstmt/2]). +-export([render_prepstmt/2]). +-export([render_prepstmt_strict/2]). + +-export_type([row_template/0]). + +-type template() :: emqx_connector_template:t(). +-type row_template() :: [emqx_connector_template:placeholder()]. +-type bindings() :: emqx_connector_template:bindings(). + +-type values() :: [emqx_connector_sql:value()]. + +-type parse_opts() :: #{ + parameters => '$n' | '?', + % Inherited from `emqx_connector_template:parse_opts()` + strip_double_quote => boolean() +}. + +-type render_opts() :: #{ + escaping => mysql | cql | sql +}. + +-define(TEMPLATE_PARSE_OPTS, [strip_double_quote]). + +%% + +%% @doc Parse an SQL statement string with zero or more placeholders into a template. +-spec parse(unicode:chardata()) -> + template(). +parse(String) -> + parse(String, #{}). + +%% @doc Parse an SQL statement string with zero or more placeholders into a template. +-spec parse(unicode:chardata(), parse_opts()) -> + template(). +parse(String, Opts) -> + emqx_connector_template:parse(String, Opts). + +%% @doc Render an SQL statement template given a set of bindings. +%% Interpolation generally follows the SQL syntax, strings are escaped according to the +%% `escaping` option. +-spec render(template(), bindings(), render_opts()) -> + {unicode:chardata(), [_Error]}. +render(Template, Bindings, Opts) -> + emqx_connector_template:render(Template, Bindings, #{ + var_trans => fun(Value) -> emqx_connector_sql:to_sql_string(Value, Opts) end + }). + +%% @doc Render an SQL statement template given a set of bindings. +%% Errors are raised if any placeholders are not bound. +-spec render_strict(template(), bindings(), render_opts()) -> + unicode:chardata(). +render_strict(Template, Bindings, Opts) -> + emqx_connector_template:render_strict(Template, Bindings, #{ + var_trans => fun(Value) -> emqx_connector_sql:to_sql_string(Value, Opts) end + }). + +%% @doc Parse an SQL statement string into a prepared statement and a row template. +%% The row template is a template for a row of SQL values to be inserted to a database +%% during the execution of the prepared statement. +%% Example: +%% ``` +%% {Statement, RowTemplate} = emqx_connector_template_sql:parse_prepstmt( +%% "INSERT INTO table (id, name, age) VALUES (${id}, ${name}, 42)", +%% #{parameters => '$n'} +%% ), +%% Statement = <<"INSERT INTO table (id, name, age) VALUES ($1, $2, 42)">>, +%% RowTemplate = [{var, [...]}, ...] +%% ``` +-spec parse_prepstmt(unicode:chardata(), parse_opts()) -> + {unicode:chardata(), row_template()}. +parse_prepstmt(String, Opts) -> + Template = emqx_connector_template:parse(String, maps:with(?TEMPLATE_PARSE_OPTS, Opts)), + Statement = mk_prepared_statement(Template, Opts), + Placeholders = [Placeholder || Placeholder = {var, _} <- Template], + {Statement, Placeholders}. + +mk_prepared_statement(Template, Opts) -> + ParameterFormat = maps:get(parameters, Opts, '?'), + {Statement, _} = + lists:mapfoldl( + fun + ({var, _}, Acc) -> + mk_replace(ParameterFormat, Acc); + (String, Acc) -> + {String, Acc} + end, + 1, + Template + ), + Statement. + +mk_replace('?', Acc) -> + {"?", Acc}; +mk_replace('$n', N) -> + {"$" ++ integer_to_list(N), N + 1}. + +%% @doc Render a row template into a list of SQL values. +%% An _SQL value_ is a vaguely defined concept here, it is something that's considered +%% compatible with the protocol of the database being used. See the definition of +%% `emqx_connector_sql:value()` for more details. +-spec render_prepstmt(template(), bindings()) -> + {values(), [_Error]}. +render_prepstmt(Template, Bindings) -> + Opts = #{var_trans => fun emqx_connector_sql:to_sql_value/1}, + emqx_connector_template:render(Template, Bindings, Opts). + +-spec render_prepstmt_strict(template(), bindings()) -> + values(). +render_prepstmt_strict(Template, Bindings) -> + Opts = #{var_trans => fun emqx_connector_sql:to_sql_value/1}, + emqx_connector_template:render_strict(Template, Bindings, Opts). diff --git a/apps/emqx_connector/src/emqx_connector_utils.erl b/apps/emqx_connector/src/emqx_connector_utils.erl deleted file mode 100644 index 6000f6be5..000000000 --- a/apps/emqx_connector/src/emqx_connector_utils.erl +++ /dev/null @@ -1,35 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_connector_utils). - --export([split_insert_sql/1]). - -%% SQL = <<"INSERT INTO \"abc\" (c1,c2,c3) VALUES (${1}, ${1}, ${1})">> -split_insert_sql(SQL) -> - case re:split(SQL, "((?i)values)", [{return, binary}]) of - [Part1, _, Part3] -> - case string:trim(Part1, leading) of - <<"insert", _/binary>> = InsertSQL -> - {ok, {InsertSQL, Part3}}; - <<"INSERT", _/binary>> = InsertSQL -> - {ok, {InsertSQL, Part3}}; - _ -> - {error, not_insert_sql} - end; - _ -> - {error, not_insert_sql} - end. diff --git a/apps/emqx_connector/test/emqx_connector_template_SUITE.erl b/apps/emqx_connector/test/emqx_connector_template_SUITE.erl new file mode 100644 index 000000000..666fbfa58 --- /dev/null +++ b/apps/emqx_connector/test/emqx_connector_template_SUITE.erl @@ -0,0 +1,323 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_connector_template_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("emqx/include/emqx_placeholder.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +all() -> emqx_common_test_helpers:all(?MODULE). + +t_render(_) -> + Bindings = #{ + a => <<"1">>, + b => 1, + c => 1.0, + d => #{<<"d1">> => <<"hi">>}, + l => [0, 1, 1000], + u => "utf-8 is ǝɹǝɥ" + }, + Template = emqx_connector_template:parse( + <<"a:${a},b:${b},c:${c},d:${d},d1:${d.d1},l:${l},u:${u}">> + ), + ?assertEqual( + {<<"a:1,b:1,c:1.0,d:{\"d1\":\"hi\"},d1:hi,l:[0,1,1000],u:utf-8 is ǝɹǝɥ"/utf8>>, []}, + render_string(Template, Bindings) + ). + +t_render_var_trans(_) -> + Bindings = #{a => <<"1">>, b => 1, c => #{prop => 1.0}}, + Template = emqx_connector_template:parse(<<"a:${a},b:${b},c:${c.prop}">>), + {String, Errors} = emqx_connector_template:render( + Template, + Bindings, + #{var_trans => fun(Name, _) -> "<" ++ lists:join($., Name) ++ ">" end} + ), + ?assertEqual( + {<<"a:,b:,c:">>, []}, + {bin(String), Errors} + ). + +t_render_path(_) -> + Bindings = #{d => #{d1 => <<"hi">>}}, + Template = emqx_connector_template:parse(<<"d.d1:${d.d1}">>), + ?assertEqual( + ok, + emqx_connector_template:validate([<<"d.d1">>], Template) + ), + ?assertEqual( + {<<"d.d1:hi">>, []}, + render_string(Template, Bindings) + ). + +t_render_custom_ph(_) -> + Bindings = #{a => <<"a">>, b => <<"b">>}, + Template = emqx_connector_template:parse(<<"a:${a},b:${b}">>), + ?assertEqual( + {error, [{[<<"b">>], disallowed}]}, + emqx_connector_template:validate([<<"a">>], Template) + ), + ?assertEqual( + <<"a:a,b:b">>, + render_strict_string(Template, Bindings) + ). + +t_render_this(_) -> + Bindings = #{a => <<"a">>, b => [1, 2, 3]}, + Template = emqx_connector_template:parse(<<"this:${} / also:${.}">>), + ?assertEqual(ok, emqx_connector_template:validate([?PH_VAR_THIS], Template)), + ?assertEqual( + % NOTE: order of the keys in the JSON object depends on the JSON encoder + <<"this:{\"b\":[1,2,3],\"a\":\"a\"} / also:{\"b\":[1,2,3],\"a\":\"a\"}">>, + render_strict_string(Template, Bindings) + ). + +t_render_missing_bindings(_) -> + Bindings = #{no => #{}}, + Template = emqx_connector_template:parse( + <<"a:${a},b:${b},c:${c},d:${d.d1},e:${no.such_atom_i_swear}">> + ), + ?assertEqual( + {<<"a:,b:,c:,d:,e:">>, [ + {[<<"no">>, <<"such_atom_i_swear">>], undefined}, + {[<<"d">>, <<"d1">>], undefined}, + {[<<"c">>], undefined}, + {[<<"b">>], undefined}, + {[<<"a">>], undefined} + ]}, + render_string(Template, Bindings) + ), + ?assertError( + [ + {[<<"no">>, <<"such_atom_i_swear">>], undefined}, + {[<<"d">>, <<"d1">>], undefined}, + {[<<"c">>], undefined}, + {[<<"b">>], undefined}, + {[<<"a">>], undefined} + ], + render_strict_string(Template, Bindings) + ). + +t_unparse(_) -> + TString = <<"a:${a},b:${b},c:$${c},d:{${d.d1}}">>, + Template = emqx_connector_template:parse(TString), + ?assertEqual( + TString, + unicode:characters_to_binary(emqx_connector_template:unparse(Template)) + ). + +t_trivial(_) -> + ?assertEqual( + true, + emqx_connector_template:trivial(emqx_connector_template:parse(<<"">>)) + ), + ?assertEqual( + false, + emqx_connector_template:trivial(emqx_connector_template:parse(<<"a:${a},b:${b},c:$${c}">>)) + ), + ?assertEqual( + true, + emqx_connector_template:trivial( + emqx_connector_template:parse(<<"a:$${a},b:$${b},c:$${c}">>) + ) + ). + +t_render_partial_ph(_) -> + Bindings = #{a => <<"1">>, b => 1, c => 1.0, d => #{d1 => <<"hi">>}}, + Template = emqx_connector_template:parse(<<"a:$a,b:b},c:{c},d:${d">>), + ?assertEqual( + <<"a:$a,b:b},c:{c},d:${d">>, + render_strict_string(Template, Bindings) + ). + +t_parse_escaped(_) -> + Bindings = #{a => <<"1">>, b => 1}, + Template = emqx_connector_template:parse(<<"a:${a},b:$${b}">>), + ?assertEqual( + <<"a:1,b:${b}">>, + render_strict_string(Template, Bindings) + ). + +t_parse_escaped_dquote(_) -> + Bindings = #{a => <<"1">>, b => 1}, + Template = emqx_connector_template:parse(<<"a:\"${a}\",b:\"$${b}\"">>, #{ + strip_double_quote => true + }), + ?assertEqual( + <<"a:1,b:\"${b}\"">>, + render_strict_string(Template, Bindings) + ). + +t_parse_sql_prepstmt(_) -> + Bindings = #{a => <<"1">>, b => 1, c => 1.0, d => #{d1 => <<"hi">>}}, + {PrepareStatement, RowTemplate} = + emqx_connector_template_sql:parse_prepstmt(<<"a:${a},b:${b},c:${c},d:${d}">>, #{ + parameters => '?' + }), + ?assertEqual(<<"a:?,b:?,c:?,d:?">>, bin(PrepareStatement)), + ?assertEqual( + {[<<"1">>, 1, 1.0, <<"{\"d1\":\"hi\"}">>], _Errors = []}, + emqx_connector_template_sql:render_prepstmt(RowTemplate, Bindings) + ). + +t_parse_sql_prepstmt_n(_) -> + Bindings = #{a => undefined, b => true, c => atom, d => #{d1 => 42.1337}}, + {PrepareStatement, RowTemplate} = + emqx_connector_template_sql:parse_prepstmt(<<"a:${a},b:${b},c:${c},d:${d}">>, #{ + parameters => '$n' + }), + ?assertEqual(<<"a:$1,b:$2,c:$3,d:$4">>, bin(PrepareStatement)), + ?assertEqual( + [null, true, <<"atom">>, <<"{\"d1\":42.1337}">>], + emqx_connector_template_sql:render_prepstmt_strict(RowTemplate, Bindings) + ). + +t_parse_sql_prepstmt_partial_ph(_) -> + Bindings = #{a => <<"1">>, b => 1, c => 1.0, d => #{d1 => <<"hi">>}}, + {PrepareStatement, RowTemplate} = + emqx_connector_template_sql:parse_prepstmt(<<"a:$a,b:b},c:{c},d:${d">>, #{parameters => '?'}), + ?assertEqual(<<"a:$a,b:b},c:{c},d:${d">>, bin(PrepareStatement)), + ?assertEqual([], emqx_connector_template_sql:render_prepstmt_strict(RowTemplate, Bindings)). + +t_render_sql(_) -> + Bindings = #{ + a => <<"1">>, + b => 1, + c => 1.0, + d => #{d1 => <<"hi">>}, + n => undefined, + u => "utf8's cool 🐸" + }, + Template = emqx_connector_template:parse(<<"a:${a},b:${b},c:${c},d:${d},n:${n},u:${u}">>), + ?assertMatch( + {_String, _Errors = []}, + emqx_connector_template_sql:render(Template, Bindings, #{}) + ), + ?assertEqual( + <<"a:'1',b:1,c:1.0,d:'{\"d1\":\"hi\"}',n:NULL,u:'utf8\\'s cool 🐸'"/utf8>>, + bin(emqx_connector_template_sql:render_strict(Template, Bindings, #{})) + ). + +t_render_mysql(_) -> + %% with apostrophes + %% https://github.com/emqx/emqx/issues/4135 + Bindings = #{ + a => <<"1''2">>, + b => 1, + c => 1.0, + d => #{d1 => <<"someone's phone">>}, + e => <<$\\, 0, "💩"/utf8>>, + f => <<"non-utf8", 16#DCC900:24>>, + g => "utf8's cool 🐸", + h => imgood + }, + Template = emqx_connector_template_sql:parse( + <<"a:${a},b:${b},c:${c},d:${d},e:${e},f:${f},g:${g},h:${h}">> + ), + ?assertEqual( + << + "a:'1\\'\\'2',b:1,c:1.0,d:'{\"d1\":\"someone\\'s phone\"}'," + "e:'\\\\\\0💩',f:0x6E6F6E2D75746638DCC900,g:'utf8\\'s cool 🐸',"/utf8, + "h:'imgood'" + >>, + bin(emqx_connector_template_sql:render_strict(Template, Bindings, #{escaping => mysql})) + ). + +t_render_cql(_) -> + %% with apostrophes for cassandra + %% https://github.com/emqx/emqx/issues/4148 + Bindings = #{ + a => <<"1''2">>, + b => 1, + c => 1.0, + d => #{d1 => <<"someone's phone">>} + }, + Template = emqx_connector_template:parse(<<"a:${a},b:${b},c:${c},d:${d}">>), + ?assertEqual( + <<"a:'1''''2',b:1,c:1.0,d:'{\"d1\":\"someone''s phone\"}'">>, + bin(emqx_connector_template_sql:render_strict(Template, Bindings, #{escaping => cql})) + ). + +t_render_sql_custom_ph(_) -> + {PrepareStatement, RowTemplate} = + emqx_connector_template_sql:parse_prepstmt(<<"a:${a},b:${b}">>, #{parameters => '$n'}), + ?assertEqual( + {error, [{[<<"b">>], disallowed}]}, + emqx_connector_template:validate([<<"a">>], RowTemplate) + ), + ?assertEqual(<<"a:$1,b:$2">>, bin(PrepareStatement)). + +t_render_sql_strip_double_quote(_) -> + Bindings = #{a => <<"a">>, b => <<"b">>}, + + %% no strip_double_quote option: "${key}" -> "value" + {PrepareStatement1, RowTemplate1} = emqx_connector_template_sql:parse_prepstmt( + <<"a:\"${a}\",b:\"${b}\"">>, + #{parameters => '$n'} + ), + ?assertEqual(<<"a:\"$1\",b:\"$2\"">>, bin(PrepareStatement1)), + ?assertEqual( + [<<"a">>, <<"b">>], + emqx_connector_template_sql:render_prepstmt_strict(RowTemplate1, Bindings) + ), + + %% strip_double_quote = true: "${key}" -> value + {PrepareStatement2, RowTemplate2} = emqx_connector_template_sql:parse_prepstmt( + <<"a:\"${a}\",b:\"${b}\"">>, + #{parameters => '$n', strip_double_quote => true} + ), + ?assertEqual(<<"a:$1,b:$2">>, bin(PrepareStatement2)), + ?assertEqual( + [<<"a">>, <<"b">>], + emqx_connector_template_sql:render_prepstmt_strict(RowTemplate2, Bindings) + ). + +t_render_tmpl_deep(_) -> + Bindings = #{a => <<"1">>, b => 1, c => 1.0, d => #{d1 => <<"hi">>}}, + + Template = emqx_connector_template:parse_deep( + #{<<"${a}">> => [<<"${b}">>, "c", 2, 3.0, '${d}', {[<<"${c}">>, <<"$${d}">>], 0}]} + ), + + ?assertEqual( + {error, [{V, disallowed} || V <- [[<<"b">>], [<<"c">>]]]}, + emqx_connector_template:validate([<<"a">>], Template) + ), + + ?assertEqual( + #{<<"1">> => [<<"1">>, "c", 2, 3.0, '${d}', {[<<"1.0">>, <<"${d}">>], 0}]}, + emqx_connector_template:render_strict(Template, Bindings) + ). + +t_unparse_tmpl_deep(_) -> + Term = #{<<"${a}">> => [<<"$${b}">>, "c", 2, 3.0, '${d}', {[<<"${c}">>], 0}]}, + Template = emqx_connector_template:parse_deep(Term), + ?assertEqual(Term, emqx_connector_template:unparse(Template)). + +%% + +render_string(Template, Bindings) -> + {String, Errors} = emqx_connector_template:render(Template, Bindings), + {bin(String), Errors}. + +render_strict_string(Template, Bindings) -> + bin(emqx_connector_template:render_strict(Template, Bindings)). + +bin(String) -> + unicode:characters_to_binary(String). From 35902dc72db829d2ff0c4de4206aae04841533cb Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 18 Apr 2023 15:34:38 +0300 Subject: [PATCH 02/21] feat(tpl): switch basic connectors to `emqx_connector_template` Also avoid `filename:join/2` in HTTP connector since it's both OS specific and an overkill. --- .../src/emqx_bridge_http_connector.erl | 120 ++++----- .../test/emqx_bridge_http_connector_tests.erl | 3 +- .../test/emqx_bridge_mysql_SUITE.erl | 30 +-- .../test/emqx_bridge_pgsql_SUITE.erl | 3 +- apps/emqx_mysql/src/emqx_mysql.erl | 248 +++++++++--------- apps/emqx_postgresql/src/emqx_postgresql.erl | 200 +++++++------- 6 files changed, 280 insertions(+), 324 deletions(-) diff --git a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl index 5d1b1947c..869f081fb 100644 --- a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl +++ b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl @@ -479,61 +479,47 @@ preprocess_request( } = Req ) -> #{ - method => emqx_placeholder:preproc_tmpl(to_bin(Method)), - path => emqx_placeholder:preproc_tmpl(Path), - body => maybe_preproc_tmpl(body, Req), - headers => wrap_auth_header(preproc_headers(Headers)), + method => parse_template(to_bin(Method)), + path => parse_template(Path), + body => maybe_parse_template(body, Req), + headers => parse_headers(Headers), request_timeout => maps:get(request_timeout, Req, ?DEFAULT_REQUEST_TIMEOUT_MS), max_retries => maps:get(max_retries, Req, 2) }. -preproc_headers(Headers) when is_map(Headers) -> +parse_headers(Headers) when is_map(Headers) -> maps:fold( - fun(K, V, Acc) -> - [ - { - emqx_placeholder:preproc_tmpl(to_bin(K)), - emqx_placeholder:preproc_tmpl(to_bin(V)) - } - | Acc - ] - end, + fun(K, V, Acc) -> [parse_header(K, V) | Acc] end, [], Headers ); -preproc_headers(Headers) when is_list(Headers) -> +parse_headers(Headers) when is_list(Headers) -> lists:map( - fun({K, V}) -> - { - emqx_placeholder:preproc_tmpl(to_bin(K)), - emqx_placeholder:preproc_tmpl(to_bin(V)) - } - end, + fun({K, V}) -> parse_header(K, V) end, Headers ). -wrap_auth_header(Headers) -> - lists:map(fun maybe_wrap_auth_header/1, Headers). +parse_header(K, V) -> + KStr = to_bin(K), + VTpl = parse_template(to_bin(V)), + {parse_template(KStr), maybe_wrap_auth_header(KStr, VTpl)}. -maybe_wrap_auth_header({[{str, Key}] = StrKey, Val}) -> - {_, MaybeWrapped} = maybe_wrap_auth_header({Key, Val}), - {StrKey, MaybeWrapped}; -maybe_wrap_auth_header({Key, Val} = Header) when - is_binary(Key), (size(Key) =:= 19 orelse size(Key) =:= 13) +maybe_wrap_auth_header(Key, VTpl) when + (byte_size(Key) =:= 19 orelse byte_size(Key) =:= 13) -> %% We check the size of potential keys in the guard above and consider only %% those that match the number of characters of either "Authorization" or %% "Proxy-Authorization". case try_bin_to_lower(Key) of <<"authorization">> -> - {Key, emqx_secret:wrap(Val)}; + emqx_secret:wrap(VTpl); <<"proxy-authorization">> -> - {Key, emqx_secret:wrap(Val)}; + emqx_secret:wrap(VTpl); _Other -> - Header + VTpl end; -maybe_wrap_auth_header(Header) -> - Header. +maybe_wrap_auth_header(_Key, VTpl) -> + VTpl. try_bin_to_lower(Bin) -> try iolist_to_binary(string:lowercase(Bin)) of @@ -542,46 +528,57 @@ try_bin_to_lower(Bin) -> _:_ -> Bin end. -maybe_preproc_tmpl(Key, Conf) -> +maybe_parse_template(Key, Conf) -> case maps:get(Key, Conf, undefined) of undefined -> undefined; - Val -> emqx_placeholder:preproc_tmpl(Val) + Val -> parse_template(Val) end. +parse_template(String) -> + emqx_connector_template:parse(String). + process_request( #{ - method := MethodTks, - path := PathTks, - body := BodyTks, - headers := HeadersTks, + method := MethodTemplate, + path := PathTemplate, + body := BodyTemplate, + headers := HeadersTemplate, request_timeout := ReqTimeout } = Conf, Msg ) -> Conf#{ - method => make_method(emqx_placeholder:proc_tmpl(MethodTks, Msg)), - path => emqx_placeholder:proc_tmpl(PathTks, Msg), - body => process_request_body(BodyTks, Msg), - headers => proc_headers(HeadersTks, Msg), + method => make_method(render_template_string(MethodTemplate, Msg)), + path => unicode:characters_to_list(render_template(PathTemplate, Msg)), + body => render_request_body(BodyTemplate, Msg), + headers => render_headers(HeadersTemplate, Msg), request_timeout => ReqTimeout }. -process_request_body(undefined, Msg) -> +render_request_body(undefined, Msg) -> emqx_utils_json:encode(Msg); -process_request_body(BodyTks, Msg) -> - emqx_placeholder:proc_tmpl(BodyTks, Msg). +render_request_body(BodyTks, Msg) -> + render_template(BodyTks, Msg). -proc_headers(HeaderTks, Msg) -> +render_headers(HeaderTks, Msg) -> lists:map( fun({K, V}) -> { - emqx_placeholder:proc_tmpl(K, Msg), - emqx_placeholder:proc_tmpl(emqx_secret:unwrap(V), Msg) + render_template_string(K, Msg), + render_template_string(emqx_secret:unwrap(V), Msg) } end, HeaderTks ). +render_template(Template, Msg) -> + % NOTE: ignoring errors here, missing variables will be rendered as `"undefined"`. + {String, _Errors} = emqx_connector_template:render(Template, Msg), + String. + +render_template_string(Template, Msg) -> + unicode:characters_to_binary(render_template(Template, Msg)). + make_method(M) when M == <<"POST">>; M == <<"post">> -> post; make_method(M) when M == <<"PUT">>; M == <<"put">> -> put; make_method(M) when M == <<"GET">>; M == <<"get">> -> get; @@ -716,8 +713,6 @@ maybe_retry(Result, _Context, ReplyFunAndArgs) -> emqx_resource:apply_reply_fun(ReplyFunAndArgs, Result). %% The HOCON schema system may generate sensitive keys with this format -is_sensitive_key([{str, StringKey}]) -> - is_sensitive_key(StringKey); is_sensitive_key(Atom) when is_atom(Atom) -> is_sensitive_key(erlang:atom_to_binary(Atom)); is_sensitive_key(Bin) when is_binary(Bin), (size(Bin) =:= 19 orelse size(Bin) =:= 13) -> @@ -742,25 +737,19 @@ redact(Data) -> %% and we also can't know the body format and where the sensitive data will be %% so the easy way to keep data security is redacted the whole body redact_request({Path, Headers}) -> - {Path, redact(Headers)}; + {Path, Headers}; redact_request({Path, Headers, _Body}) -> - {Path, redact(Headers), <<"******">>}. + {Path, Headers, <<"******">>}. -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). redact_test_() -> - TestData1 = [ - {<<"content-type">>, <<"application/json">>}, - {<<"Authorization">>, <<"Basic YWxhZGRpbjpvcGVuc2VzYW1l">>} - ], - - TestData2 = #{ - headers => - [ - {[{str, <<"content-type">>}], [{str, <<"application/json">>}]}, - {[{str, <<"Authorization">>}], [{str, <<"Basic YWxhZGRpbjpvcGVuc2VzYW1l">>}]} - ] + TestData = #{ + headers => [ + {<<"content-type">>, <<"application/json">>}, + {<<"Authorization">>, <<"Basic YWxhZGRpbjpvcGVuc2VzYW1l">>} + ] }, [ ?_assert(is_sensitive_key(<<"Authorization">>)), @@ -770,8 +759,7 @@ redact_test_() -> ?_assert(is_sensitive_key('PrOxy-authoRizaTion')), ?_assertNot(is_sensitive_key(<<"Something">>)), ?_assertNot(is_sensitive_key(89)), - ?_assertNotEqual(TestData1, redact(TestData1)), - ?_assertNotEqual(TestData2, redact(TestData2)) + ?_assertNotEqual(TestData, redact(TestData)) ]. join_paths_test_() -> diff --git a/apps/emqx_bridge_http/test/emqx_bridge_http_connector_tests.erl b/apps/emqx_bridge_http/test/emqx_bridge_http_connector_tests.erl index 6b5c2b0cd..1de210260 100644 --- a/apps/emqx_bridge_http/test/emqx_bridge_http_connector_tests.erl +++ b/apps/emqx_bridge_http/test/emqx_bridge_http_connector_tests.erl @@ -83,7 +83,8 @@ is_wrapped(Secret) when is_function(Secret) -> is_wrapped(_Other) -> false. -untmpl([{_, V} | _]) -> V. +untmpl(Tpl) -> + iolist_to_binary(emqx_connector_template:render_strict(Tpl, #{})). is_unwrapped_headers(Headers) -> lists:all(fun is_unwrapped_header/1, Headers). diff --git a/apps/emqx_bridge_mysql/test/emqx_bridge_mysql_SUITE.erl b/apps/emqx_bridge_mysql/test/emqx_bridge_mysql_SUITE.erl index 3ed40e903..2eeccfd77 100644 --- a/apps/emqx_bridge_mysql/test/emqx_bridge_mysql_SUITE.erl +++ b/apps/emqx_bridge_mysql/test/emqx_bridge_mysql_SUITE.erl @@ -565,8 +565,6 @@ t_simple_sql_query(Config) -> ok. t_missing_data(Config) -> - BatchSize = ?config(batch_size, Config), - IsBatch = BatchSize > 1, ?assertMatch( {ok, _}, create_bridge(Config) @@ -577,27 +575,13 @@ t_missing_data(Config) -> ), send_message(Config, #{}), {ok, [Event]} = snabbkaffe:receive_events(SRef), - case IsBatch of - true -> - ?assertMatch( - #{ - result := - {error, - {unrecoverable_error, - {1292, _, <<"Truncated incorrect DOUBLE value: 'undefined'">>}}} - }, - Event - ); - false -> - ?assertMatch( - #{ - result := - {error, - {unrecoverable_error, {1048, _, <<"Column 'arrived' cannot be null">>}}} - }, - Event - ) - end, + ?assertMatch( + #{ + result := + {error, {unrecoverable_error, {1048, _, <<"Column 'arrived' cannot be null">>}}} + }, + Event + ), ok. t_bad_sql_parameter(Config) -> diff --git a/apps/emqx_bridge_pgsql/test/emqx_bridge_pgsql_SUITE.erl b/apps/emqx_bridge_pgsql/test/emqx_bridge_pgsql_SUITE.erl index cd79db43d..156d4bd16 100644 --- a/apps/emqx_bridge_pgsql/test/emqx_bridge_pgsql_SUITE.erl +++ b/apps/emqx_bridge_pgsql/test/emqx_bridge_pgsql_SUITE.erl @@ -324,6 +324,7 @@ connect_and_drop_table(Config) -> connect_and_clear_table(Config) -> Con = connect_direct_pgsql(Config), + _ = epgsql:squery(Con, ?SQL_CREATE_TABLE), {ok, _} = epgsql:squery(Con, ?SQL_DELETE), ok = epgsql:close(Con). @@ -668,7 +669,7 @@ t_missing_table(Config) -> ok end, fun(Trace) -> - ?assertMatch([_, _, _], ?of_kind(pgsql_undefined_table, Trace)), + ?assertMatch([_], ?of_kind(pgsql_undefined_table, Trace)), ok end ), diff --git a/apps/emqx_mysql/src/emqx_mysql.erl b/apps/emqx_mysql/src/emqx_mysql.erl index 4440bcfbb..927c9d067 100644 --- a/apps/emqx_mysql/src/emqx_mysql.erl +++ b/apps/emqx_mysql/src/emqx_mysql.erl @@ -46,16 +46,12 @@ default_port => ?MYSQL_DEFAULT_PORT }). --type prepares() :: #{atom() => binary()}. --type params_tokens() :: #{atom() => list()}. --type sqls() :: #{atom() => binary()}. +-type template() :: {unicode:chardata(), emqx_connector_template:str()}. -type state() :: #{ pool_name := binary(), - prepare_statement := prepares(), - params_tokens := params_tokens(), - batch_inserts := sqls(), - batch_params_tokens := params_tokens() + prepares := ok | {error, _}, + templates := #{{atom(), batch | prepstmt} => template()} }. %%===================================================================== @@ -154,13 +150,13 @@ on_query(InstId, {TypeOrKey, SQLOrKey, Params}, State) -> on_query( InstId, {TypeOrKey, SQLOrKey, Params, Timeout}, - #{pool_name := PoolName, prepare_statement := Prepares} = State + State ) -> MySqlFunction = mysql_function(TypeOrKey), {SQLOrKey2, Data} = proc_sql_params(TypeOrKey, SQLOrKey, Params, State), case on_sql_query(InstId, MySqlFunction, SQLOrKey2, Data, Timeout, State) of {error, not_prepared} -> - case maybe_prepare_sql(SQLOrKey2, Prepares, PoolName) of + case maybe_prepare_sql(SQLOrKey2, State) of ok -> ?tp( mysql_connector_on_query_prepared_sql, @@ -187,23 +183,27 @@ on_query( on_batch_query( InstId, - BatchReq, - #{batch_inserts := Inserts, batch_params_tokens := ParamsTokens} = State + BatchReq = [{Key, _} | _], + #{query_templates := Templates} = State ) -> - case hd(BatchReq) of - {Key, _} -> - case maps:get(Key, Inserts, undefined) of - undefined -> - {error, {unrecoverable_error, batch_select_not_implemented}}; - InsertSQL -> - Tokens = maps:get(Key, ParamsTokens), - on_batch_insert(InstId, BatchReq, InsertSQL, Tokens, State) - end; - Request -> - LogMeta = #{connector => InstId, first_request => Request, state => State}, - ?SLOG(error, LogMeta#{msg => "invalid request"}), - {error, {unrecoverable_error, invalid_request}} - end. + case maps:get({Key, batch}, Templates, undefined) of + undefined -> + {error, {unrecoverable_error, batch_select_not_implemented}}; + Template -> + on_batch_insert(InstId, BatchReq, Template, State) + end; +on_batch_query( + InstId, + BatchReq, + State +) -> + ?SLOG(error, #{ + msg => "invalid request", + connector => InstId, + request => BatchReq, + state => State + }), + {error, {unrecoverable_error, invalid_request}}. mysql_function(sql) -> query; @@ -222,8 +222,8 @@ on_get_status(_InstId, #{pool_name := PoolName} = State) -> {ok, NState} -> %% return new state with prepared statements {connected, NState}; - {error, {undefined_table, NState}} -> - {disconnected, NState, unhealthy_target}; + {error, undefined_table} -> + {disconnected, State, unhealthy_target}; {error, _Reason} -> %% do not log error, it is logged in prepare_sql_to_conn connecting @@ -238,8 +238,8 @@ do_get_status(Conn) -> do_check_prepares( #{ pool_name := PoolName, - prepare_statement := #{send_message := SQL} - } = State + templates := #{{send_message, prepstmt} := SQL} + } ) -> % it's already connected. Verify if target table still exists Workers = [Worker || {_WorkerName, Worker} <- ecpool:workers(PoolName)], @@ -250,7 +250,7 @@ do_check_prepares( {ok, Conn} -> case mysql:prepare(Conn, get_status, SQL) of {error, {1146, _, _}} -> - {error, {undefined_table, State}}; + {error, undefined_table}; {ok, Statement} -> mysql:unprepare(Conn, Statement); _ -> @@ -265,17 +265,14 @@ do_check_prepares( ok, Workers ); -do_check_prepares(#{prepare_statement := Statement}) when is_map(Statement) -> +do_check_prepares(#{prepares := ok}) -> ok; -do_check_prepares(State = #{pool_name := PoolName, prepare_statement := {error, Prepares}}) -> +do_check_prepares(#{prepares := {error, _}} = State) -> %% retry to prepare - case prepare_sql(Prepares, PoolName) of + case prepare_sql(State) of ok -> %% remove the error - {ok, State#{prepare_statement => Prepares}}; - {error, undefined_table} -> - %% indicate the error - {error, {undefined_table, State#{prepare_statement => {error, Prepares}}}}; + {ok, State#{prepares => ok}}; {error, Reason} -> {error, Reason} end. @@ -285,41 +282,44 @@ do_check_prepares(State = #{pool_name := PoolName, prepare_statement := {error, connect(Options) -> mysql:start_link(Options). -init_prepare(State = #{prepare_statement := Prepares, pool_name := PoolName}) -> - case maps:size(Prepares) of +init_prepare(State = #{query_templates := Templates}) -> + case maps:size(Templates) of 0 -> - State; + State#{prepares => ok}; _ -> - case prepare_sql(Prepares, PoolName) of + case prepare_sql(State) of ok -> - State; + State#{prepares => ok}; {error, Reason} -> - LogMeta = #{msg => <<"mysql_init_prepare_statement_failed">>, reason => Reason}, - ?SLOG(error, LogMeta), + ?SLOG(error, #{ + msg => <<"MySQL init prepare statement failed">>, + reason => Reason + }), %% mark the prepare_statement as failed - State#{prepare_statement => {error, Prepares}} + State#{prepares => {error, Reason}} end end. -maybe_prepare_sql(SQLOrKey, Prepares, PoolName) -> - case maps:is_key(SQLOrKey, Prepares) of - true -> prepare_sql(Prepares, PoolName); +maybe_prepare_sql(SQLOrKey, State = #{query_templates := Templates}) -> + case maps:is_key({SQLOrKey, prepstmt}, Templates) of + true -> prepare_sql(State); false -> {error, {unrecoverable_error, prepared_statement_invalid}} end. -prepare_sql(Prepares, PoolName) when is_map(Prepares) -> - prepare_sql(maps:to_list(Prepares), PoolName); -prepare_sql(Prepares, PoolName) -> - case do_prepare_sql(Prepares, PoolName) of +prepare_sql(#{query_templates := Templates, pool_name := PoolName}) -> + prepare_sql(maps:to_list(Templates), PoolName). + +prepare_sql(Templates, PoolName) -> + case do_prepare_sql(Templates, PoolName) of ok -> %% prepare for reconnect - ecpool:add_reconnect_callback(PoolName, {?MODULE, prepare_sql_to_conn, [Prepares]}), + ecpool:add_reconnect_callback(PoolName, {?MODULE, prepare_sql_to_conn, [Templates]}), ok; {error, R} -> {error, R} end. -do_prepare_sql(Prepares, PoolName) -> +do_prepare_sql(Templates, PoolName) -> Conns = [ begin @@ -328,33 +328,30 @@ do_prepare_sql(Prepares, PoolName) -> end || {_Name, Worker} <- ecpool:workers(PoolName) ], - prepare_sql_to_conn_list(Conns, Prepares). + prepare_sql_to_conn_list(Conns, Templates). -prepare_sql_to_conn_list([], _PrepareList) -> +prepare_sql_to_conn_list([], _Templates) -> ok; -prepare_sql_to_conn_list([Conn | ConnList], PrepareList) -> - case prepare_sql_to_conn(Conn, PrepareList) of +prepare_sql_to_conn_list([Conn | ConnList], Templates) -> + case prepare_sql_to_conn(Conn, Templates) of ok -> - prepare_sql_to_conn_list(ConnList, PrepareList); + prepare_sql_to_conn_list(ConnList, Templates); {error, R} -> %% rollback - Fun = fun({Key, _}) -> - _ = unprepare_sql_to_conn(Conn, Key), - ok - end, - lists:foreach(Fun, PrepareList), + _ = [unprepare_sql_to_conn(Conn, Template) || Template <- Templates], {error, R} end. -prepare_sql_to_conn(Conn, []) when is_pid(Conn) -> ok; -prepare_sql_to_conn(Conn, [{Key, SQL} | PrepareList]) when is_pid(Conn) -> - LogMeta = #{msg => "mysql_prepare_statement", name => Key, prepare_sql => SQL}, +prepare_sql_to_conn(_Conn, []) -> + ok; +prepare_sql_to_conn(Conn, [{{Key, prepstmt}, {SQL, _RowTemplate}} | Rest]) -> + LogMeta = #{msg => "MySQL Prepare Statement", name => Key, prepare_sql => SQL}, ?SLOG(info, LogMeta), _ = unprepare_sql_to_conn(Conn, Key), case mysql:prepare(Conn, Key, SQL) of {ok, _Key} -> ?SLOG(info, LogMeta#{result => success}), - prepare_sql_to_conn(Conn, PrepareList); + prepare_sql_to_conn(Conn, Rest); {error, {1146, _, _} = Reason} -> %% Target table is not created ?tp(mysql_undefined_table, #{}), @@ -365,84 +362,85 @@ prepare_sql_to_conn(Conn, [{Key, SQL} | PrepareList]) when is_pid(Conn) -> % syntax failures. Retrying syntax failures is not very productive. ?SLOG(error, LogMeta#{result => failed, reason => Reason}), {error, Reason} - end. + end; +prepare_sql_to_conn(Conn, [{_Key, _Template} | Rest]) -> + prepare_sql_to_conn(Conn, Rest). -unprepare_sql_to_conn(Conn, PrepareSqlKey) -> - mysql:unprepare(Conn, PrepareSqlKey). +unprepare_sql_to_conn(Conn, {{Key, prepstmt}, _}) -> + mysql:unprepare(Conn, Key); +unprepare_sql_to_conn(Conn, Key) when is_atom(Key) -> + mysql:unprepare(Conn, Key); +unprepare_sql_to_conn(_Conn, _) -> + ok. parse_prepare_sql(Config) -> - SQL = - case maps:get(prepare_statement, Config, undefined) of - undefined -> - case maps:get(sql, Config, undefined) of - undefined -> #{}; - Template -> #{send_message => Template} - end; - Any -> - Any + Queries = + case Config of + #{prepare_statement := Qs} -> + Qs; + #{sql := Query} -> + #{send_message => Query}; + _ -> + #{} end, - parse_prepare_sql(maps:to_list(SQL), #{}, #{}, #{}, #{}). + Templates = maps:fold(fun parse_prepare_sql/3, #{}, Queries), + #{query_templates => Templates}. -parse_prepare_sql([{Key, H} | _] = L, Prepares, Tokens, BatchInserts, BatchTks) -> - {PrepareSQL, ParamsTokens} = emqx_placeholder:preproc_sql(H), - parse_batch_prepare_sql( - L, Prepares#{Key => PrepareSQL}, Tokens#{Key => ParamsTokens}, BatchInserts, BatchTks - ); -parse_prepare_sql([], Prepares, Tokens, BatchInserts, BatchTks) -> - #{ - prepare_statement => Prepares, - params_tokens => Tokens, - batch_inserts => BatchInserts, - batch_params_tokens => BatchTks - }. +parse_prepare_sql(Key, Query, Acc) -> + Template = emqx_connector_template_sql:parse_prepstmt(Query, #{parameters => '?'}), + AccNext = Acc#{{Key, prepstmt} => Template}, + parse_batch_sql(Key, Query, AccNext). -parse_batch_prepare_sql([{Key, H} | T], Prepares, Tokens, BatchInserts, BatchTks) -> - case emqx_utils_sql:get_statement_type(H) of - select -> - parse_prepare_sql(T, Prepares, Tokens, BatchInserts, BatchTks); +parse_batch_sql(Key, Query, Acc) -> + case emqx_connector_sql:get_statement_type(Query) of insert -> - case emqx_utils_sql:parse_insert(H) of - {ok, {InsertSQL, Params}} -> - ParamsTks = emqx_placeholder:preproc_tmpl(Params), - parse_prepare_sql( - T, - Prepares, - Tokens, - BatchInserts#{Key => InsertSQL}, - BatchTks#{Key => ParamsTks} - ); + case emqx_connector_sql:parse_insert(Query) of + {ok, {Insert, Params}} -> + RowTemplate = emqx_connector_template_sql:parse(Params), + Acc#{{Key, batch} => {Insert, RowTemplate}}; {error, Reason} -> - ?SLOG(error, #{msg => "split_sql_failed", sql => H, reason => Reason}), - parse_prepare_sql(T, Prepares, Tokens, BatchInserts, BatchTks) + ?SLOG(error, #{ + msg => "parse insert sql statement failed", + sql => Query, + reason => Reason + }), + Acc end; - Type when is_atom(Type) -> - ?SLOG(error, #{msg => "detect_sql_type_unsupported", sql => H, type => Type}), - parse_prepare_sql(T, Prepares, Tokens, BatchInserts, BatchTks); - {error, Reason} -> - ?SLOG(error, #{msg => "detect_sql_type_failed", sql => H, reason => Reason}), - parse_prepare_sql(T, Prepares, Tokens, BatchInserts, BatchTks) + select -> + Acc; + Otherwise -> + ?SLOG(error, #{ + msg => "invalid sql statement type", + sql => Query, + type => Otherwise + }), + Acc end. proc_sql_params(query, SQLOrKey, Params, _State) -> {SQLOrKey, Params}; proc_sql_params(prepared_query, SQLOrKey, Params, _State) -> {SQLOrKey, Params}; -proc_sql_params(TypeOrKey, SQLOrData, Params, #{params_tokens := ParamsTokens}) -> - case maps:get(TypeOrKey, ParamsTokens, undefined) of +proc_sql_params(TypeOrKey, SQLOrData, Params, #{query_templates := Templates}) -> + case maps:get({TypeOrKey, prepstmt}, Templates, undefined) of undefined -> {SQLOrData, Params}; - Tokens -> - {TypeOrKey, emqx_placeholder:proc_sql(Tokens, SQLOrData)} + {_InsertPart, RowTemplate} -> + % NOTE: ignoring errors here, missing variables are set to `null`. + {Row, _Errors} = emqx_connector_template_sql:render_prepstmt(RowTemplate, SQLOrData), + {TypeOrKey, Row} end. -on_batch_insert(InstId, BatchReqs, InsertPart, Tokens, State) -> - ValuesPart = lists:join($,, [ - emqx_placeholder:proc_param_str(Tokens, Msg, fun emqx_placeholder:quote_mysql/1) - || {_, Msg} <- BatchReqs - ]), - Query = [InsertPart, <<" values ">> | ValuesPart], +on_batch_insert(InstId, BatchReqs, {InsertPart, RowTemplate}, State) -> + Rows = [render_row(RowTemplate, Msg) || {_, Msg} <- BatchReqs], + Query = [InsertPart, <<" values ">> | lists:join($,, Rows)], on_sql_query(InstId, query, Query, no_params, default_timeout, State). +render_row(RowTemplate, Data) -> + % NOTE: ignoring errors here, missing variables are set to "NULL". + {Row, _Errors} = emqx_connector_template_sql:render(RowTemplate, Data, #{escaping => mysql}), + Row. + on_sql_query( InstId, SQLFunc, diff --git a/apps/emqx_postgresql/src/emqx_postgresql.erl b/apps/emqx_postgresql/src/emqx_postgresql.erl index dc6447536..71ba93b9b 100644 --- a/apps/emqx_postgresql/src/emqx_postgresql.erl +++ b/apps/emqx_postgresql/src/emqx_postgresql.erl @@ -52,15 +52,12 @@ default_port => ?PGSQL_DEFAULT_PORT }). --type prepares() :: #{atom() => binary()}. --type params_tokens() :: #{atom() => list()}. - +-type template() :: {unicode:chardata(), emqx_connector_template_sql:row_template()}. -type state() :: #{ pool_name := binary(), - prepare_sql := prepares(), - params_tokens := params_tokens(), - prepare_statement := epgsql:statement() + query_templates := #{binary() => template()}, + prepares := #{binary() => epgsql:statement()} | {error, _} }. %% FIXME: add `{error, sync_required}' to `epgsql:execute_batch' @@ -142,7 +139,7 @@ on_start( State = parse_prepare_sql(Config), case emqx_resource_pool:start(InstId, ?MODULE, Options ++ SslOpts) of ok -> - {ok, init_prepare(State#{pool_name => InstId, prepare_statement => #{}})}; + {ok, init_prepare(State#{pool_name => InstId, prepares => #{}})}; {error, Reason} -> ?tp( pgsql_connector_start_failed, @@ -189,55 +186,50 @@ pgsql_query_type(_) -> on_batch_query( InstId, - BatchReq, - #{pool_name := PoolName, params_tokens := Tokens, prepare_statement := Sts} = State + [{Key, _} = Request | _] = BatchReq, + #{pool_name := PoolName, query_templates := Templates, prepares := PrepStatements} = State ) -> - case BatchReq of - [{Key, _} = Request | _] -> - BinKey = to_bin(Key), - case maps:get(BinKey, Tokens, undefined) of - undefined -> - Log = #{ - connector => InstId, - first_request => Request, - state => State, - msg => "batch_prepare_not_implemented" - }, - ?SLOG(error, Log), - {error, {unrecoverable_error, batch_prepare_not_implemented}}; - TokenList -> - {_, Datas} = lists:unzip(BatchReq), - Datas2 = [emqx_placeholder:proc_sql(TokenList, Data) || Data <- Datas], - St = maps:get(BinKey, Sts), - case on_sql_query(InstId, PoolName, execute_batch, St, Datas2) of - {error, _Error} = Result -> - handle_result(Result); - {_Column, Results} -> - handle_batch_result(Results, 0) - end - end; - _ -> + BinKey = to_bin(Key), + case maps:get(BinKey, Templates, undefined) of + undefined -> Log = #{ connector => InstId, - request => BatchReq, + first_request => Request, state => State, - msg => "invalid_request" + msg => "batch prepare not implemented" }, ?SLOG(error, Log), - {error, {unrecoverable_error, invalid_request}} - end. + {error, {unrecoverable_error, batch_prepare_not_implemented}}; + {_Statement, RowTemplate} -> + PrepStatement = maps:get(BinKey, PrepStatements), + Rows = [render_prepare_sql_row(RowTemplate, Data) || {_Key, Data} <- BatchReq], + case on_sql_query(InstId, PoolName, execute_batch, PrepStatement, Rows) of + {error, _Error} = Result -> + handle_result(Result); + {_Column, Results} -> + handle_batch_result(Results, 0) + end + end; +on_batch_query(InstId, BatchReq, State) -> + ?SLOG(error, #{ + connector => InstId, + request => BatchReq, + state => State, + msg => "invalid request" + }), + {error, {unrecoverable_error, invalid_request}}. proc_sql_params(query, SQLOrKey, Params, _State) -> {SQLOrKey, Params}; proc_sql_params(prepared_query, SQLOrKey, Params, _State) -> {SQLOrKey, Params}; -proc_sql_params(TypeOrKey, SQLOrData, Params, #{params_tokens := ParamsTokens}) -> +proc_sql_params(TypeOrKey, SQLOrData, Params, #{query_templates := Templates}) -> Key = to_bin(TypeOrKey), - case maps:get(Key, ParamsTokens, undefined) of + case maps:get(Key, Templates, undefined) of undefined -> {SQLOrData, Params}; - Tokens -> - {Key, emqx_placeholder:proc_sql(Tokens, SQLOrData)} + {_Statement, RowTemplate} -> + {Key, render_prepare_sql_row(RowTemplate, SQLOrData)} end. on_sql_query(InstId, PoolName, Type, NameOrSQL, Data) -> @@ -297,9 +289,9 @@ on_get_status(_InstId, #{pool_name := PoolName} = State) -> {ok, NState} -> %% return new state with prepared statements {connected, NState}; - {error, {undefined_table, NState}} -> + {error, undefined_table} -> %% return new state indicating that we are connected but the target table is not created - {disconnected, NState, unhealthy_target}; + {disconnected, State, unhealthy_target}; {error, _Reason} -> %% do not log error, it is logged in prepare_sql_to_conn connecting @@ -314,8 +306,8 @@ do_get_status(Conn) -> do_check_prepares( #{ pool_name := PoolName, - prepare_sql := #{<<"send_message">> := SQL} - } = State + query_templates := #{<<"send_message">> := {SQL, _RowTemplate}} + } ) -> WorkerPids = [Worker || {_WorkerName, Worker} <- ecpool:workers(PoolName)], case validate_table_existence(WorkerPids, SQL) of @@ -324,19 +316,16 @@ do_check_prepares( {error, undefined_table} -> {error, {undefined_table, State}} end; -do_check_prepares(#{prepare_sql := Prepares}) when is_map(Prepares) -> +do_check_prepares(#{prepares := Prepares}) when is_map(Prepares) -> ok; -do_check_prepares(State = #{pool_name := PoolName, prepare_sql := {error, Prepares}}) -> +do_check_prepares(#{prepares := {error, _}} = State) -> %% retry to prepare - case prepare_sql(Prepares, PoolName) of - {ok, Sts} -> + case prepare_sql(State) of + {ok, PrepStatements} -> %% remove the error - {ok, State#{prepare_sql => Prepares, prepare_statement := Sts}}; - {error, undefined_table} -> - %% indicate the error - {error, {undefined_table, State#{prepare_sql => {error, Prepares}}}}; - Error -> - {error, Error} + {ok, State#{prepares := PrepStatements}}; + {error, Reason} -> + {error, Reason} end. -spec validate_table_existence([pid()], binary()) -> ok | {error, undefined_table}. @@ -426,69 +415,63 @@ conn_opts([_Opt | Opts], Acc) -> conn_opts(Opts, Acc). parse_prepare_sql(Config) -> - SQL = - case maps:get(prepare_statement, Config, undefined) of - undefined -> - case maps:get(sql, Config, undefined) of - undefined -> #{}; - Template -> #{<<"send_message">> => Template} - end; - Any -> - Any + Queries = + case Config of + #{prepare_statement := Qs} -> + Qs; + #{sql := Query} -> + #{<<"send_message">> => Query}; + #{} -> + #{} end, - parse_prepare_sql(maps:to_list(SQL), #{}, #{}). + Templates = maps:fold(fun parse_prepare_sql/3, #{}, Queries), + #{query_templates => Templates}. -parse_prepare_sql([{Key, H} | T], Prepares, Tokens) -> - {PrepareSQL, ParamsTokens} = emqx_placeholder:preproc_sql(H, '$n'), - parse_prepare_sql( - T, Prepares#{Key => PrepareSQL}, Tokens#{Key => ParamsTokens} - ); -parse_prepare_sql([], Prepares, Tokens) -> - #{ - prepare_sql => Prepares, - params_tokens => Tokens - }. +parse_prepare_sql(Key, Query, Acc) -> + Template = emqx_connector_template_sql:parse_prepstmt(Query, #{parameters => '$n'}), + Acc#{Key => Template}. -init_prepare(State = #{prepare_sql := Prepares, pool_name := PoolName}) -> - case maps:size(Prepares) of - 0 -> - State; - _ -> - case prepare_sql(Prepares, PoolName) of - {ok, Sts} -> - State#{prepare_statement := Sts}; - Error -> - LogMsg = - maps:merge( - #{msg => <<"postgresql_init_prepare_statement_failed">>}, - translate_to_log_context(Error) - ), - ?SLOG(error, LogMsg), - %% mark the prepare_sql as failed - State#{prepare_sql => {error, Prepares}} - end +render_prepare_sql_row(RowTemplate, Data) -> + % NOTE: ignoring errors here, missing variables will be replaced with `null`. + {Row, _Errors} = emqx_connector_template_sql:render_prepstmt(RowTemplate, Data), + Row. + +init_prepare(State = #{query_templates := Templates}) when map_size(Templates) == 0 -> + State; +init_prepare(State = #{}) -> + case prepare_sql(State) of + {ok, PrepStatements} -> + State#{prepares => PrepStatements}; + Error -> + ?SLOG(error, maps:merge( + #{msg => <<"postgresql_init_prepare_statement_failed">>}, + translate_to_log_context(Error) + )), + %% mark the prepares failed + State#{prepares => Error} end. -prepare_sql(Prepares, PoolName) when is_map(Prepares) -> - prepare_sql(maps:to_list(Prepares), PoolName); -prepare_sql(Prepares, PoolName) -> - case do_prepare_sql(Prepares, PoolName) of +prepare_sql(#{query_templates := Templates, pool_name := PoolName}) -> + prepare_sql(maps:to_list(Templates), PoolName). + +prepare_sql(Templates, PoolName) -> + case do_prepare_sql(Templates, PoolName) of {ok, _Sts} = Ok -> %% prepare for reconnect - ecpool:add_reconnect_callback(PoolName, {?MODULE, prepare_sql_to_conn, [Prepares]}), + ecpool:add_reconnect_callback(PoolName, {?MODULE, prepare_sql_to_conn, [Templates]}), Ok; Error -> Error end. -do_prepare_sql(Prepares, PoolName) -> - do_prepare_sql(ecpool:workers(PoolName), Prepares, #{}). +do_prepare_sql(Templates, PoolName) -> + do_prepare_sql(ecpool:workers(PoolName), Templates, #{}). -do_prepare_sql([{_Name, Worker} | T], Prepares, _LastSts) -> +do_prepare_sql([{_Name, Worker} | Rest], Templates, _LastSts) -> {ok, Conn} = ecpool_worker:client(Worker), - case prepare_sql_to_conn(Conn, Prepares) of + case prepare_sql_to_conn(Conn, Templates) of {ok, Sts} -> - do_prepare_sql(T, Prepares, Sts); + do_prepare_sql(Rest, Templates, Sts); Error -> Error end; @@ -498,13 +481,14 @@ do_prepare_sql([], _Prepares, LastSts) -> prepare_sql_to_conn(Conn, Prepares) -> prepare_sql_to_conn(Conn, Prepares, #{}). -prepare_sql_to_conn(Conn, [], Statements) when is_pid(Conn) -> {ok, Statements}; -prepare_sql_to_conn(Conn, [{Key, SQL} | PrepareList], Statements) when is_pid(Conn) -> - LogMeta = #{msg => "postgresql_prepare_statement", name => Key, prepare_sql => SQL}, +prepare_sql_to_conn(Conn, [], Statements) when is_pid(Conn) -> + {ok, Statements}; +prepare_sql_to_conn(Conn, [{Key, {SQL, _RowTemplate}} | Rest], Statements) when is_pid(Conn) -> + LogMeta = #{msg => "PostgreSQL Prepare Statement", name => Key, sql => SQL}, ?SLOG(info, LogMeta), case epgsql:parse2(Conn, Key, SQL, []) of {ok, Statement} -> - prepare_sql_to_conn(Conn, PrepareList, Statements#{Key => Statement}); + prepare_sql_to_conn(Conn, Rest, Statements#{Key => Statement}); {error, {error, error, _, undefined_table, _, _} = Error} -> %% Target table is not created ?tp(pgsql_undefined_table, #{}), From 0538a77700d56e1da90e0127ce7de2f867402e23 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 18 Apr 2023 15:21:35 +0300 Subject: [PATCH 03/21] feat(tpl): use `emqx_connector_template` in `emqx_authn`, `emqx_authz` This slightly changes semantics: now the attempt to create authenticator with illegal bindings in templates will fail, instead of treating them as literals. The runtime behaviour on the other hand should be the same. --- apps/emqx/include/emqx_placeholder.hrl | 80 +++++++++++-------- .../src/emqx_authn/emqx_authn_utils.erl | 80 +++++++++++-------- .../src/emqx_authz/emqx_authz_rule.erl | 18 ++--- .../src/emqx_authz/emqx_authz_utils.erl | 64 +++++++++------ .../test/emqx_authz/emqx_authz_rule_SUITE.erl | 6 +- apps/emqx_auth_http/src/emqx_authz_http.erl | 22 ++--- .../src/emqx_authz_mongodb.erl | 10 +-- apps/emqx_auth_mysql/src/emqx_authz_mysql.erl | 10 +-- .../src/emqx_authz_postgresql.erl | 10 +-- apps/emqx_auth_redis/src/emqx_authz_redis.erl | 10 +-- .../src/emqx_connector_template.erl | 8 +- 11 files changed, 178 insertions(+), 140 deletions(-) diff --git a/apps/emqx/include/emqx_placeholder.hrl b/apps/emqx/include/emqx_placeholder.hrl index 7b2ce6c6b..1db80c72d 100644 --- a/apps/emqx/include/emqx_placeholder.hrl +++ b/apps/emqx/include/emqx_placeholder.hrl @@ -19,67 +19,79 @@ -define(PH_VAR_THIS, <<"$_THIS_">>). --define(PH(Type), <<"${", Type/binary, "}">>). +-define(PH(Var), <<"${" Var "}">>). %% action: publish/subscribe --define(PH_ACTION, <<"${action}">>). +-define(VAR_ACTION, "action"). +-define(PH_ACTION, ?PH(?VAR_ACTION)). %% cert --define(PH_CERT_SUBJECT, <<"${cert_subject}">>). --define(PH_CERT_CN_NAME, <<"${cert_common_name}">>). +-define(VAR_CERT_SUBJECT, "cert_subject"). +-define(VAR_CERT_CN_NAME, "cert_common_name"). +-define(PH_CERT_SUBJECT, ?PH(?VAR_CERT_SUBJECT)). +-define(PH_CERT_CN_NAME, ?PH(?VAR_CERT_CN_NAME)). %% MQTT --define(PH_PASSWORD, <<"${password}">>). --define(PH_CLIENTID, <<"${clientid}">>). --define(PH_FROM_CLIENTID, <<"${from_clientid}">>). --define(PH_USERNAME, <<"${username}">>). --define(PH_FROM_USERNAME, <<"${from_username}">>). --define(PH_TOPIC, <<"${topic}">>). +-define(VAR_PASSWORD, "password"). +-define(VAR_CLIENTID, "clientid"). +-define(VAR_USERNAME, "username"). +-define(VAR_TOPIC, "topic"). +-define(PH_PASSWORD, ?PH(?VAR_PASSWORD)). +-define(PH_CLIENTID, ?PH(?VAR_CLIENTID)). +-define(PH_FROM_CLIENTID, ?PH("from_clientid")). +-define(PH_USERNAME, ?PH(?VAR_USERNAME)). +-define(PH_FROM_USERNAME, ?PH("from_username")). +-define(PH_TOPIC, ?PH(?VAR_TOPIC)). %% MQTT payload --define(PH_PAYLOAD, <<"${payload}">>). +-define(PH_PAYLOAD, ?PH("payload")). %% client IPAddress --define(PH_PEERHOST, <<"${peerhost}">>). +-define(VAR_PEERHOST, "peerhost"). +-define(PH_PEERHOST, ?PH(?VAR_PEERHOST)). %% ip & port --define(PH_HOST, <<"${host}">>). --define(PH_PORT, <<"${port}">>). +-define(PH_HOST, ?PH("host")). +-define(PH_PORT, ?PH("port")). %% Enumeration of message QoS 0,1,2 --define(PH_QOS, <<"${qos}">>). --define(PH_FLAGS, <<"${flags}">>). +-define(VAR_QOS, "qos"). +-define(PH_QOS, ?PH(?VAR_QOS)). +-define(PH_FLAGS, ?PH("flags")). %% Additional data related to process within the MQTT message --define(PH_HEADERS, <<"${headers}">>). +-define(PH_HEADERS, ?PH("headers")). %% protocol name --define(PH_PROTONAME, <<"${proto_name}">>). +-define(VAR_PROTONAME, "proto_name"). +-define(PH_PROTONAME, ?PH(?VAR_PROTONAME)). %% protocol version --define(PH_PROTOVER, <<"${proto_ver}">>). +-define(PH_PROTOVER, ?PH("proto_ver")). %% MQTT keepalive interval --define(PH_KEEPALIVE, <<"${keepalive}">>). +-define(PH_KEEPALIVE, ?PH("keepalive")). %% MQTT clean_start --define(PH_CLEAR_START, <<"${clean_start}">>). +-define(PH_CLEAR_START, ?PH("clean_start")). %% MQTT Session Expiration time --define(PH_EXPIRY_INTERVAL, <<"${expiry_interval}">>). +-define(PH_EXPIRY_INTERVAL, ?PH("expiry_interval")). %% Time when PUBLISH message reaches Broker (ms) --define(PH_PUBLISH_RECEIVED_AT, <<"${publish_received_at}">>). +-define(PH_PUBLISH_RECEIVED_AT, ?PH("publish_received_at")). %% Mountpoint for bridging messages --define(PH_MOUNTPOINT, <<"${mountpoint}">>). +-define(VAR_MOUNTPOINT, "mountpoint"). +-define(PH_MOUNTPOINT, ?PH(?VAR_MOUNTPOINT)). %% IPAddress and Port of terminal --define(PH_PEERNAME, <<"${peername}">>). +-define(PH_PEERNAME, ?PH("peername")). %% IPAddress and Port listened by emqx --define(PH_SOCKNAME, <<"${sockname}">>). +-define(PH_SOCKNAME, ?PH("sockname")). %% whether it is MQTT bridge connection --define(PH_IS_BRIDGE, <<"${is_bridge}">>). +-define(PH_IS_BRIDGE, ?PH("is_bridge")). %% Terminal connection completion time (s) --define(PH_CONNECTED_AT, <<"${connected_at}">>). +-define(PH_CONNECTED_AT, ?PH("connected_at")). %% Event trigger time(millisecond) --define(PH_TIMESTAMP, <<"${timestamp}">>). +-define(PH_TIMESTAMP, ?PH("timestamp")). %% Terminal disconnection completion time (s) --define(PH_DISCONNECTED_AT, <<"${disconnected_at}">>). +-define(PH_DISCONNECTED_AT, ?PH("disconnected_at")). --define(PH_NODE, <<"${node}">>). --define(PH_REASON, <<"${reason}">>). +-define(PH_NODE, ?PH("node")). +-define(PH_REASON, ?PH("reason")). --define(PH_ENDPOINT_NAME, <<"${endpoint_name}">>). --define(PH_RETAIN, <<"${retain}">>). +-define(PH_ENDPOINT_NAME, ?PH("endpoint_name")). +-define(VAR_RETAIN, "retain"). +-define(PH_RETAIN, ?PH(?VAR_RETAIN)). %% sync change these place holder with binary def. -define(PH_S_ACTION, "${action}"). diff --git a/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl b/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl index a9d672922..9be3e24d6 100644 --- a/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl +++ b/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl @@ -45,12 +45,12 @@ ]). -define(AUTHN_PLACEHOLDERS, [ - ?PH_USERNAME, - ?PH_CLIENTID, - ?PH_PASSWORD, - ?PH_PEERHOST, - ?PH_CERT_SUBJECT, - ?PH_CERT_CN_NAME + <>, + <>, + <>, + <>, + <>, + <> ]). -define(DEFAULT_RESOURCE_OPTS, #{ @@ -107,48 +107,62 @@ check_password_from_selected_map(Algorithm, Selected, Password) -> end. parse_deep(Template) -> - emqx_placeholder:preproc_tmpl_deep(Template, #{placeholders => ?AUTHN_PLACEHOLDERS}). + Result = emqx_connector_template:parse_deep(Template), + ok = emqx_connector_template:validate(?AUTHN_PLACEHOLDERS, Result), + Result. parse_str(Template) -> - emqx_placeholder:preproc_tmpl(Template, #{placeholders => ?AUTHN_PLACEHOLDERS}). + Result = emqx_connector_template:parse(Template), + ok = emqx_connector_template:validate(?AUTHN_PLACEHOLDERS, Result), + Result. parse_sql(Template, ReplaceWith) -> - emqx_placeholder:preproc_sql( + {Statement, Result} = emqx_connector_template_sql:parse_prepstmt( Template, - #{ - replace_with => ReplaceWith, - placeholders => ?AUTHN_PLACEHOLDERS, - strip_double_quote => true - } - ). + #{parameters => ReplaceWith, strip_double_quote => true} + ), + ok = emqx_connector_template:validate(?AUTHN_PLACEHOLDERS, Result), + {Statement, Result}. render_deep(Template, Credential) -> - emqx_placeholder:proc_tmpl_deep( + % NOTE + % Ignoring errors here, undefined bindings will be replaced with empty string. + {Term, _Errors} = emqx_connector_template:render( Template, mapping_credential(Credential), - #{return => full_binary, var_trans => fun handle_var/2} - ). + #{var_trans => fun handle_var/2} + ), + Term. render_str(Template, Credential) -> - emqx_placeholder:proc_tmpl( + % NOTE + % Ignoring errors here, undefined bindings will be replaced with empty string. + {String, _Errors} = emqx_connector_template:render( Template, mapping_credential(Credential), - #{return => full_binary, var_trans => fun handle_var/2} - ). + #{var_trans => fun handle_var/2} + ), + unicode:characters_to_binary(String). render_urlencoded_str(Template, Credential) -> - emqx_placeholder:proc_tmpl( + % NOTE + % Ignoring errors here, undefined bindings will be replaced with empty string. + {String, _Errors} = emqx_connector_template:render( Template, mapping_credential(Credential), - #{return => full_binary, var_trans => fun urlencode_var/2} - ). + #{var_trans => fun urlencode_var/2} + ), + unicode:characters_to_binary(String). render_sql_params(ParamList, Credential) -> - emqx_placeholder:proc_tmpl( + % NOTE + % Ignoring errors here, undefined bindings will be replaced with empty string. + {Row, _Errors} = emqx_connector_template:render( ParamList, mapping_credential(Credential), - #{return => rawlist, var_trans => fun handle_sql_var/2} - ). + #{var_trans => fun handle_sql_var/2} + ), + Row. is_superuser(#{<<"is_superuser">> := Value}) -> #{is_superuser => to_bool(Value)}; @@ -272,19 +286,19 @@ without_password(Credential, [Name | Rest]) -> urlencode_var(Var, Value) -> emqx_http_lib:uri_encode(handle_var(Var, Value)). -handle_var(_Name, undefined) -> +handle_var(_, undefined) -> <<>>; handle_var([<<"peerhost">>], PeerHost) -> - emqx_placeholder:bin(inet:ntoa(PeerHost)); + emqx_connector_template:to_string(inet:ntoa(PeerHost)); handle_var(_, Value) -> - emqx_placeholder:bin(Value). + emqx_connector_template:to_string(Value). -handle_sql_var(_Name, undefined) -> +handle_sql_var(_, undefined) -> <<>>; handle_sql_var([<<"peerhost">>], PeerHost) -> - emqx_placeholder:bin(inet:ntoa(PeerHost)); + emqx_connector_sql:to_sql_value(inet:ntoa(PeerHost)); handle_sql_var(_, Value) -> - emqx_placeholder:sql_data(Value). + emqx_connector_sql:to_sql_value(Value). mapping_credential(C = #{cn := CN, dn := DN}) -> C#{cert_common_name => CN, cert_subject => DN}; diff --git a/apps/emqx_auth/src/emqx_authz/emqx_authz_rule.erl b/apps/emqx_auth/src/emqx_authz/emqx_authz_rule.erl index 6e13cac91..9cf79ba88 100644 --- a/apps/emqx_auth/src/emqx_authz/emqx_authz_rule.erl +++ b/apps/emqx_auth/src/emqx_authz/emqx_authz_rule.erl @@ -183,19 +183,15 @@ compile_topic(<<"eq ", Topic/binary>>) -> compile_topic({eq, Topic}) -> {eq, emqx_topic:words(bin(Topic))}; compile_topic(Topic) -> - TopicBin = bin(Topic), - case - emqx_placeholder:preproc_tmpl( - TopicBin, - #{placeholders => [?PH_USERNAME, ?PH_CLIENTID]} - ) - of - [{str, _}] -> emqx_topic:words(TopicBin); - Tokens -> {pattern, Tokens} + Template = emqx_connector_template:parse(Topic), + ok = emqx_connector_template:validate([<>, <>], Template), + case emqx_connector_template:trivial(Template) of + true -> emqx_topic:words(bin(Topic)); + false -> {pattern, Template} end. bin(L) when is_list(L) -> - list_to_binary(L); + unicode:characters_to_binary(L); bin(B) when is_binary(B) -> B. @@ -307,7 +303,7 @@ match_who(_, _) -> match_topics(_ClientInfo, _Topic, []) -> false; match_topics(ClientInfo, Topic, [{pattern, PatternFilter} | Filters]) -> - TopicFilter = emqx_placeholder:proc_tmpl(PatternFilter, ClientInfo), + TopicFilter = bin(emqx_connector_template:render_strict(PatternFilter, ClientInfo)), match_topic(emqx_topic:words(Topic), emqx_topic:words(TopicFilter)) orelse match_topics(ClientInfo, Topic, Filters); match_topics(ClientInfo, Topic, [TopicFilter | Filters]) -> diff --git a/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl b/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl index 3a0d4f1a1..e5aeab21b 100644 --- a/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl +++ b/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl @@ -108,48 +108,62 @@ update_config(Path, ConfigRequest) -> }). parse_deep(Template, PlaceHolders) -> - emqx_placeholder:preproc_tmpl_deep(Template, #{placeholders => PlaceHolders}). + Result = emqx_connector_template:parse_deep(Template), + ok = emqx_connector_template:validate(PlaceHolders, Result), + Result. parse_str(Template, PlaceHolders) -> - emqx_placeholder:preproc_tmpl(Template, #{placeholders => PlaceHolders}). + Result = emqx_connector_template:parse(Template), + ok = emqx_connector_template:validate(PlaceHolders, Result), + Result. parse_sql(Template, ReplaceWith, PlaceHolders) -> - emqx_placeholder:preproc_sql( + {Statement, Result} = emqx_connector_template_sql:parse_prepstmt( Template, - #{ - replace_with => ReplaceWith, - placeholders => PlaceHolders, - strip_double_quote => true - } - ). + #{parameters => ReplaceWith, strip_double_quote => true} + ), + ok = emqx_connector_template:validate(PlaceHolders, Result), + {Statement, Result}. render_deep(Template, Values) -> - emqx_placeholder:proc_tmpl_deep( + % NOTE + % Ignoring errors here, undefined bindings will be replaced with empty string. + {Term, _Errors} = emqx_connector_template:render( Template, client_vars(Values), - #{return => full_binary, var_trans => fun handle_var/2} - ). + #{var_trans => fun handle_var/2} + ), + Term. render_str(Template, Values) -> - emqx_placeholder:proc_tmpl( + % NOTE + % Ignoring errors here, undefined bindings will be replaced with empty string. + {String, _Errors} = emqx_connector_template:render( Template, client_vars(Values), - #{return => full_binary, var_trans => fun handle_var/2} - ). + #{var_trans => fun handle_var/2} + ), + unicode:characters_to_binary(String). render_urlencoded_str(Template, Values) -> - emqx_placeholder:proc_tmpl( + % NOTE + % Ignoring errors here, undefined bindings will be replaced with empty string. + {String, _Errors} = emqx_connector_template:render( Template, client_vars(Values), - #{return => full_binary, var_trans => fun urlencode_var/2} - ). + #{var_trans => fun urlencode_var/2} + ), + unicode:characters_to_binary(String). render_sql_params(ParamList, Values) -> - emqx_placeholder:proc_tmpl( + % NOTE + % Ignoring errors here, undefined bindings will be replaced with empty string. + {Row, _Errors} = emqx_connector_template:render( ParamList, client_vars(Values), - #{return => rawlist, var_trans => fun handle_sql_var/2} - ). + #{var_trans => fun handle_sql_var/2} + ), + Row. -spec parse_http_resp_body(binary(), binary()) -> allow | deny | ignore | error. parse_http_resp_body(<<"application/x-www-form-urlencoded", _/binary>>, Body) -> @@ -218,19 +232,19 @@ convert_client_var(Other) -> Other. urlencode_var(Var, Value) -> emqx_http_lib:uri_encode(handle_var(Var, Value)). -handle_var(_Name, undefined) -> +handle_var(_, undefined) -> <<>>; handle_var([<<"peerhost">>], IpAddr) -> inet_parse:ntoa(IpAddr); handle_var(_Name, Value) -> - emqx_placeholder:bin(Value). + emqx_connector_template:to_string(Value). -handle_sql_var(_Name, undefined) -> +handle_sql_var(_, undefined) -> <<>>; handle_sql_var([<<"peerhost">>], IpAddr) -> inet_parse:ntoa(IpAddr); handle_sql_var(_Name, Value) -> - emqx_placeholder:sql_data(Value). + emqx_connector_sql:to_sql_value(Value). bin(A) when is_atom(A) -> atom_to_binary(A, utf8); bin(L) when is_list(L) -> list_to_binary(L); diff --git a/apps/emqx_auth/test/emqx_authz/emqx_authz_rule_SUITE.erl b/apps/emqx_auth/test/emqx_authz/emqx_authz_rule_SUITE.erl index b34e4fb00..bca21cd8d 100644 --- a/apps/emqx_auth/test/emqx_authz/emqx_authz_rule_SUITE.erl +++ b/apps/emqx_auth/test/emqx_authz/emqx_authz_rule_SUITE.erl @@ -67,6 +67,10 @@ set_special_configs(_App) -> ok. t_compile(_) -> + % NOTE + % Some of the following testcase are relying on the internal representation of + % `emqx_connector_template:t()`. If the internal representation is changed, these + % testcases may fail. ?assertEqual({deny, all, all, [['#']]}, emqx_authz_rule:compile({deny, all})), ?assertEqual( @@ -116,7 +120,7 @@ t_compile(_) -> ?assertEqual( {allow, {username, {eq, <<"test">>}}, publish, [ - {pattern, [{str, <<"t/foo">>}, {var, [<<"username">>]}, {str, <<"boo">>}]} + {pattern, [<<"t/foo">>, {var, [<<"username">>]}, <<"boo">>]} ]}, emqx_authz_rule:compile({allow, {username, "test"}, publish, ["t/foo${username}boo"]}) ), diff --git a/apps/emqx_auth_http/src/emqx_authz_http.erl b/apps/emqx_auth_http/src/emqx_authz_http.erl index ed7051bb6..2ab76f305 100644 --- a/apps/emqx_auth_http/src/emqx_authz_http.erl +++ b/apps/emqx_auth_http/src/emqx_authz_http.erl @@ -39,20 +39,20 @@ -endif. -define(PLACEHOLDERS, [ - ?PH_USERNAME, - ?PH_CLIENTID, - ?PH_PEERHOST, - ?PH_PROTONAME, - ?PH_MOUNTPOINT, - ?PH_TOPIC, - ?PH_ACTION, - ?PH_CERT_SUBJECT, - ?PH_CERT_CN_NAME + <>, + <>, + <>, + <>, + <>, + <>, + <>, + <>, + <> ]). -define(PLACEHOLDERS_FOR_RICH_ACTIONS, [ - ?PH_QOS, - ?PH_RETAIN + <>, + <> ]). description() -> diff --git a/apps/emqx_auth_mongodb/src/emqx_authz_mongodb.erl b/apps/emqx_auth_mongodb/src/emqx_authz_mongodb.erl index 3b235ad2c..97a5fa3a6 100644 --- a/apps/emqx_auth_mongodb/src/emqx_authz_mongodb.erl +++ b/apps/emqx_auth_mongodb/src/emqx_authz_mongodb.erl @@ -36,11 +36,11 @@ -endif. -define(PLACEHOLDERS, [ - ?PH_USERNAME, - ?PH_CLIENTID, - ?PH_PEERHOST, - ?PH_CERT_CN_NAME, - ?PH_CERT_SUBJECT + <>, + <>, + <>, + <>, + <> ]). description() -> diff --git a/apps/emqx_auth_mysql/src/emqx_authz_mysql.erl b/apps/emqx_auth_mysql/src/emqx_authz_mysql.erl index 4ca71e332..e87d2afa2 100644 --- a/apps/emqx_auth_mysql/src/emqx_authz_mysql.erl +++ b/apps/emqx_auth_mysql/src/emqx_authz_mysql.erl @@ -38,11 +38,11 @@ -endif. -define(PLACEHOLDERS, [ - ?PH_USERNAME, - ?PH_CLIENTID, - ?PH_PEERHOST, - ?PH_CERT_CN_NAME, - ?PH_CERT_SUBJECT + <>, + <>, + <>, + <>, + <> ]). description() -> diff --git a/apps/emqx_auth_postgresql/src/emqx_authz_postgresql.erl b/apps/emqx_auth_postgresql/src/emqx_authz_postgresql.erl index b930f77e4..645fff293 100644 --- a/apps/emqx_auth_postgresql/src/emqx_authz_postgresql.erl +++ b/apps/emqx_auth_postgresql/src/emqx_authz_postgresql.erl @@ -38,11 +38,11 @@ -endif. -define(PLACEHOLDERS, [ - ?PH_USERNAME, - ?PH_CLIENTID, - ?PH_PEERHOST, - ?PH_CERT_CN_NAME, - ?PH_CERT_SUBJECT + <>, + <>, + <>, + <>, + <> ]). description() -> diff --git a/apps/emqx_auth_redis/src/emqx_authz_redis.erl b/apps/emqx_auth_redis/src/emqx_authz_redis.erl index 9b69f508a..7ac893da1 100644 --- a/apps/emqx_auth_redis/src/emqx_authz_redis.erl +++ b/apps/emqx_auth_redis/src/emqx_authz_redis.erl @@ -36,11 +36,11 @@ -endif. -define(PLACEHOLDERS, [ - ?PH_CERT_CN_NAME, - ?PH_CERT_SUBJECT, - ?PH_PEERHOST, - ?PH_CLIENTID, - ?PH_USERNAME + <>, + <>, + <>, + <>, + <> ]). description() -> diff --git a/apps/emqx_connector/src/emqx_connector_template.erl b/apps/emqx_connector/src/emqx_connector_template.erl index c346d4289..4f583573c 100644 --- a/apps/emqx_connector/src/emqx_connector_template.erl +++ b/apps/emqx_connector/src/emqx_connector_template.erl @@ -153,7 +153,7 @@ trivial(Template) -> unparse({'$tpl', Template}) -> unparse_deep(Template); unparse(Template) -> - lists:map(fun unparse_part/1, Template). + unicode:characters_to_list(lists:map(fun unparse_part/1, Template)). unparse_part({var, Name}) -> render_placeholder(Name); @@ -222,7 +222,7 @@ render_strict(Template, Bindings, Opts) -> {String, []} -> String; {_, Errors = [_ | _]} -> - error(Errors, [unicode:characters_to_list(unparse(Template)), Bindings]) + error(Errors, [unparse(Template), Bindings]) end. %% @doc Parse an arbitrary Erlang term into a "deep" template. @@ -306,9 +306,7 @@ unparse_deep(Term) -> -spec lookup_var(var(), bindings()) -> {ok, binding()} | {error, undefined}. -lookup_var(?PH_VAR_THIS, Value) -> - {ok, Value}; -lookup_var([], Value) -> +lookup_var(Var, Value) when Var == ?PH_VAR_THIS orelse Var == [] -> {ok, Value}; lookup_var([Prop | Rest], Bindings) -> case lookup(Prop, Bindings) of From e1bca5844f40cc6adf4ecd6a4ec56f964e607185 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 18 Apr 2023 17:07:23 +0300 Subject: [PATCH 04/21] feat(tpl): use `emqx_connector_template` in `emqx_prometheus` app --- apps/emqx_prometheus/src/emqx_prometheus.erl | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/apps/emqx_prometheus/src/emqx_prometheus.erl b/apps/emqx_prometheus/src/emqx_prometheus.erl index e9030d3ed..fa9a39cc6 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus.erl @@ -24,7 +24,6 @@ -include("emqx_prometheus.hrl"). --include_lib("prometheus/include/prometheus.hrl"). -include_lib("prometheus/include/prometheus_model.hrl"). -include_lib("emqx/include/logger.hrl"). @@ -114,16 +113,12 @@ handle_info(_Msg, State) -> push_to_push_gateway(Uri, Headers, JobName) when is_list(Headers) -> [Name, Ip] = string:tokens(atom_to_list(node()), "@"), - JobName1 = emqx_placeholder:preproc_tmpl(JobName), - JobName2 = binary_to_list( - emqx_placeholder:proc_tmpl( - JobName1, - #{<<"name">> => Name, <<"host">> => Ip} - ) + JobName1 = emqx_connector_template:render_strict( + emqx_connector_template:parse(JobName), + #{<<"name">> => Name, <<"host">> => Ip} ), - - Url = lists:concat([Uri, "/metrics/job/", JobName2]), Data = prometheus_text_format:format(), + Url = lists:concat([Uri, "/metrics/job/", unicode:characters_to_list(JobName1)]), case httpc:request(post, {Url, Headers, "text/plain", Data}, ?HTTP_OPTIONS, []) of {ok, {{"HTTP/1.1", 200, _}, _RespHeaders, _RespBody}} -> ok; From b812f9af5a481f8927fe6f25404eec143a73f06b Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 18 Apr 2023 17:42:39 +0300 Subject: [PATCH 05/21] feat(tpl): use `emqx_connector_template` in `emqx_rule_engine` app --- .../src/emqx_connector_template.erl | 1 + .../src/emqx_rule_actions.erl | 103 +++++++++--------- .../test/emqx_rule_engine_SUITE.erl | 5 +- 3 files changed, 57 insertions(+), 52 deletions(-) diff --git a/apps/emqx_connector/src/emqx_connector_template.erl b/apps/emqx_connector/src/emqx_connector_template.erl index 4f583573c..bb26edec1 100644 --- a/apps/emqx_connector/src/emqx_connector_template.erl +++ b/apps/emqx_connector/src/emqx_connector_template.erl @@ -30,6 +30,7 @@ -export([render_strict/2]). -export([render_strict/3]). +-export([lookup_var/2]). -export([to_string/1]). -export_type([t/0]). diff --git a/apps/emqx_rule_engine/src/emqx_rule_actions.erl b/apps/emqx_rule_engine/src/emqx_rule_actions.erl index 276f8d0e0..bb9966b4a 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_actions.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_actions.erl @@ -65,23 +65,18 @@ pre_process_action_args( qos := QoS, retain := Retain, payload := Payload, - mqtt_properties := MQTTPropertiesTemplate0, - user_properties := UserPropertiesTemplate + mqtt_properties := MQTTProperties, + user_properties := UserProperties } = Args ) -> - MQTTPropertiesTemplate = - maps:map( - fun(_Key, V) -> emqx_placeholder:preproc_tmpl(V) end, - MQTTPropertiesTemplate0 - ), Args#{ preprocessed_tmpl => #{ - topic => emqx_placeholder:preproc_tmpl(Topic), - qos => preproc_vars(QoS), - retain => preproc_vars(Retain), - payload => emqx_placeholder:preproc_tmpl(Payload), - mqtt_properties => MQTTPropertiesTemplate, - user_properties => preproc_user_properties(UserPropertiesTemplate) + topic => emqx_connector_template:parse(Topic), + qos => parse_vars(QoS), + retain => parse_vars(Retain), + payload => parse_payload(Payload), + mqtt_properties => parse_mqtt_properties(MQTTProperties), + user_properties => parse_user_properties(UserProperties) } }; pre_process_action_args(_, Args) -> @@ -114,25 +109,27 @@ republish( #{metadata := #{rule_id := RuleId}} = Env, #{ preprocessed_tmpl := #{ - qos := QoSTks, - retain := RetainTks, - topic := TopicTks, - payload := PayloadTks, + qos := QoSTemplate, + retain := RetainTemplate, + topic := TopicTemplate, + payload := PayloadTemplate, mqtt_properties := MQTTPropertiesTemplate, - user_properties := UserPropertiesTks + user_properties := UserPropertiesTemplate } } ) -> - Topic = emqx_placeholder:proc_tmpl(TopicTks, Selected), - Payload = format_msg(PayloadTks, Selected), - QoS = replace_simple_var(QoSTks, Selected, 0), - Retain = replace_simple_var(RetainTks, Selected, false), + Topic = unicode:characters_to_binary( + emqx_connector_template:render_strict(TopicTemplate, Selected) + ), + Payload = emqx_connector_template:render_strict(PayloadTemplate, Selected), + QoS = render_simple_var(QoSTemplate, Selected, 0), + Retain = render_simple_var(RetainTemplate, Selected, false), %% 'flags' is set for message re-publishes or message related %% events such as message.acked and message.dropped Flags0 = maps:get(flags, Env, #{}), Flags = Flags0#{retain => Retain}, - PubProps0 = format_pub_props(UserPropertiesTks, Selected, Env), - MQTTProps = format_mqtt_properties(MQTTPropertiesTemplate, Selected, Env), + PubProps0 = render_pub_props(UserPropertiesTemplate, Selected, Env), + MQTTProps = render_mqtt_properties(MQTTPropertiesTemplate, Selected, Env), PubProps = maps:merge(PubProps0, MQTTProps), ?TRACE( "RULE", @@ -203,58 +200,66 @@ safe_publish(RuleId, Topic, QoS, Flags, Payload, PubProps) -> _ = emqx_broker:safe_publish(Msg), emqx_metrics:inc_msg(Msg). -preproc_vars(Data) when is_binary(Data) -> - emqx_placeholder:preproc_tmpl(Data); -preproc_vars(Data) -> - Data. +parse_vars(Data) when is_binary(Data) -> + emqx_connector_template:parse(Data); +parse_vars(Data) -> + {const, Data}. -preproc_user_properties(<<"${pub_props.'User-Property'}">>) -> +parse_mqtt_properties(MQTTPropertiesTemplate) -> + maps:map( + fun(_Key, V) -> emqx_connector_template:parse(V) end, + MQTTPropertiesTemplate + ). + +parse_user_properties(<<"${pub_props.'User-Property'}">>) -> %% keep the original %% avoid processing this special variable because %% we do not want to force users to select the value %% the value will be taken from Env.pub_props directly ?ORIGINAL_USER_PROPERTIES; -preproc_user_properties(<<"${", _/binary>> = V) -> +parse_user_properties(<<"${", _/binary>> = V) -> %% use a variable - emqx_placeholder:preproc_tmpl(V); -preproc_user_properties(_) -> + emqx_connector_template:parse(V); +parse_user_properties(_) -> %% invalid, discard undefined. -replace_simple_var(Tokens, Data, Default) when is_list(Tokens) -> - [Var] = emqx_placeholder:proc_tmpl(Tokens, Data, #{return => rawlist}), - case Var of +render_simple_var([{var, Name}], Data, Default) -> + case emqx_connector_template:lookup_var(Name, Data) of + {ok, Var} -> Var; %% cannot find the variable from Data - undefined -> Default; - _ -> Var + {error, _} -> Default end; -replace_simple_var(Val, _Data, _Default) -> +render_simple_var({const, Val}, _Data, _Default) -> Val. -format_msg([], Selected) -> - emqx_utils_json:encode(Selected); -format_msg(Tokens, Selected) -> - emqx_placeholder:proc_tmpl(Tokens, Selected). +parse_payload(Payload) -> + case string:is_empty(Payload) of + false -> emqx_connector_template:parse(Payload); + true -> emqx_connector_template:parse("${.}") + end. -format_pub_props(UserPropertiesTks, Selected, Env) -> +render_pub_props(UserPropertiesTemplate, Selected, Env) -> UserProperties = - case UserPropertiesTks of + case UserPropertiesTemplate of ?ORIGINAL_USER_PROPERTIES -> maps:get('User-Property', maps:get(pub_props, Env, #{}), #{}); undefined -> #{}; _ -> - replace_simple_var(UserPropertiesTks, Selected, #{}) + render_simple_var(UserPropertiesTemplate, Selected, #{}) end, #{'User-Property' => UserProperties}. -format_mqtt_properties(MQTTPropertiesTemplate, Selected, Env) -> +render_mqtt_properties(MQTTPropertiesTemplate, Selected, Env) -> #{metadata := #{rule_id := RuleId}} = Env, - MQTTProperties0 = + MQTTProperties = maps:fold( fun(K, Template, Acc) -> try - V = emqx_placeholder:proc_tmpl(Template, Selected), + V = unicode:characters_to_binary( + emqx_connector_template:render_strict(Template, Selected) + ), Acc#{K => V} catch Kind:Error -> @@ -275,7 +280,7 @@ format_mqtt_properties(MQTTPropertiesTemplate, Selected, Env) -> #{}, MQTTPropertiesTemplate ), - coerce_properties_values(MQTTProperties0, Env). + coerce_properties_values(MQTTProperties, Env). ensure_int(B) when is_binary(B) -> try diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index 00ca68264..fcb04f9b3 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -1364,14 +1364,13 @@ t_sqlselect_inject_props(_Config) -> actions => [Repub] } ), - Props = user_properties(#{<<"inject_key">> => <<"inject_val">>}), {ok, Client} = emqtt:start_link([{username, <<"emqx">>}, {proto_ver, v5}]), {ok, _} = emqtt:connect(Client), {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), emqtt:publish(Client, <<"t1">>, #{}, <<"{\"x\":1}">>, [{qos, 0}]), receive - {publish, #{topic := T, payload := Payload, properties := Props2}} -> - ?assertEqual(Props, Props2), + {publish, #{topic := T, payload := Payload, properties := Props}} -> + ?assertEqual(user_properties(#{<<"inject_key">> => <<"inject_val">>}), Props), ?assertEqual(<<"t2">>, T), ?assertEqual(<<"{\"x\":1}">>, Payload) after 2000 -> From 49f5325c6768eae83d0e6e398a05e74328b234a8 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Wed, 26 Apr 2023 22:45:24 +0300 Subject: [PATCH 06/21] feat(tpl): unify validations / errors var representations --- .../src/emqx_authz/emqx_authz_rule.erl | 2 +- .../test/emqx_authz/emqx_authz_rule_SUITE.erl | 10 ++-- apps/emqx_auth_http/src/emqx_authz_http.erl | 18 ++++---- .../src/emqx_authz_mongodb.erl | 10 ++-- apps/emqx_auth_mysql/src/emqx_authz_mysql.erl | 10 ++-- .../src/emqx_authz_postgresql.erl | 10 ++-- apps/emqx_auth_redis/src/emqx_authz_redis.erl | 10 ++-- .../src/emqx_connector_template.erl | 46 +++++++++---------- .../src/emqx_connector_template_sql.erl | 6 +-- .../test/emqx_connector_template_SUITE.erl | 40 ++++++++-------- .../src/emqx_rule_actions.erl | 4 +- 11 files changed, 81 insertions(+), 85 deletions(-) diff --git a/apps/emqx_auth/src/emqx_authz/emqx_authz_rule.erl b/apps/emqx_auth/src/emqx_authz/emqx_authz_rule.erl index 9cf79ba88..6f5369aec 100644 --- a/apps/emqx_auth/src/emqx_authz/emqx_authz_rule.erl +++ b/apps/emqx_auth/src/emqx_authz/emqx_authz_rule.erl @@ -184,7 +184,7 @@ compile_topic({eq, Topic}) -> {eq, emqx_topic:words(bin(Topic))}; compile_topic(Topic) -> Template = emqx_connector_template:parse(Topic), - ok = emqx_connector_template:validate([<>, <>], Template), + ok = emqx_connector_template:validate([?VAR_USERNAME, ?VAR_CLIENTID], Template), case emqx_connector_template:trivial(Template) of true -> emqx_topic:words(bin(Topic)); false -> {pattern, Template} diff --git a/apps/emqx_auth/test/emqx_authz/emqx_authz_rule_SUITE.erl b/apps/emqx_auth/test/emqx_authz/emqx_authz_rule_SUITE.erl index bca21cd8d..5031daff6 100644 --- a/apps/emqx_auth/test/emqx_authz/emqx_authz_rule_SUITE.erl +++ b/apps/emqx_auth/test/emqx_authz/emqx_authz_rule_SUITE.erl @@ -78,13 +78,13 @@ t_compile(_) -> emqx_authz_rule:compile({allow, {ipaddr, "127.0.0.1"}, all, [{eq, "#"}, {eq, "+"}]}) ), - ?assertEqual( + ?assertMatch( {allow, {ipaddrs, [ {{127, 0, 0, 1}, {127, 0, 0, 1}, 32}, {{192, 168, 1, 0}, {192, 168, 1, 255}, 24} ]}, - subscribe, [{pattern, [{var, [<<"clientid">>]}]}]}, + subscribe, [{pattern, [{var, "clientid", [_]}]}]}, emqx_authz_rule:compile( {allow, {ipaddrs, ["127.0.0.1", "192.168.1.0/24"]}, subscribe, [?PH_S_CLIENTID]} ) @@ -106,7 +106,7 @@ t_compile(_) -> {clientid, {re_pattern, _, _, _, _}} ]}, publish, [ - {pattern, [{var, [<<"username">>]}]}, {pattern, [{var, [<<"clientid">>]}]} + {pattern, [{var, "username", [_]}]}, {pattern, [{var, "clientid", [_]}]} ]}, emqx_authz_rule:compile( {allow, @@ -118,9 +118,9 @@ t_compile(_) -> ) ), - ?assertEqual( + ?assertMatch( {allow, {username, {eq, <<"test">>}}, publish, [ - {pattern, [<<"t/foo">>, {var, [<<"username">>]}, <<"boo">>]} + {pattern, [<<"t/foo">>, {var, "username", [_]}, <<"boo">>]} ]}, emqx_authz_rule:compile({allow, {username, "test"}, publish, ["t/foo${username}boo"]}) ), diff --git a/apps/emqx_auth_http/src/emqx_authz_http.erl b/apps/emqx_auth_http/src/emqx_authz_http.erl index 2ab76f305..bbb2bf9b5 100644 --- a/apps/emqx_auth_http/src/emqx_authz_http.erl +++ b/apps/emqx_auth_http/src/emqx_authz_http.erl @@ -39,15 +39,15 @@ -endif. -define(PLACEHOLDERS, [ - <>, - <>, - <>, - <>, - <>, - <>, - <>, - <>, - <> + ?VAR_USERNAME, + ?VAR_CLIENTID, + ?VAR_PEERHOST, + ?VAR_PROTONAME, + ?VAR_MOUNTPOINT, + ?VAR_TOPIC, + ?VAR_ACTION, + ?VAR_CERT_SUBJECT, + ?VAR_CERT_CN_NAME ]). -define(PLACEHOLDERS_FOR_RICH_ACTIONS, [ diff --git a/apps/emqx_auth_mongodb/src/emqx_authz_mongodb.erl b/apps/emqx_auth_mongodb/src/emqx_authz_mongodb.erl index 97a5fa3a6..35ac3a41b 100644 --- a/apps/emqx_auth_mongodb/src/emqx_authz_mongodb.erl +++ b/apps/emqx_auth_mongodb/src/emqx_authz_mongodb.erl @@ -36,11 +36,11 @@ -endif. -define(PLACEHOLDERS, [ - <>, - <>, - <>, - <>, - <> + ?VAR_USERNAME, + ?VAR_CLIENTID, + ?VAR_PEERHOST, + ?VAR_CERT_CN_NAME, + ?VAR_CERT_SUBJECT ]). description() -> diff --git a/apps/emqx_auth_mysql/src/emqx_authz_mysql.erl b/apps/emqx_auth_mysql/src/emqx_authz_mysql.erl index e87d2afa2..a6d71d1ca 100644 --- a/apps/emqx_auth_mysql/src/emqx_authz_mysql.erl +++ b/apps/emqx_auth_mysql/src/emqx_authz_mysql.erl @@ -38,11 +38,11 @@ -endif. -define(PLACEHOLDERS, [ - <>, - <>, - <>, - <>, - <> + ?VAR_USERNAME, + ?VAR_CLIENTID, + ?VAR_PEERHOST, + ?VAR_CERT_CN_NAME, + ?VAR_CERT_SUBJECT ]). description() -> diff --git a/apps/emqx_auth_postgresql/src/emqx_authz_postgresql.erl b/apps/emqx_auth_postgresql/src/emqx_authz_postgresql.erl index 645fff293..b538bd95e 100644 --- a/apps/emqx_auth_postgresql/src/emqx_authz_postgresql.erl +++ b/apps/emqx_auth_postgresql/src/emqx_authz_postgresql.erl @@ -38,11 +38,11 @@ -endif. -define(PLACEHOLDERS, [ - <>, - <>, - <>, - <>, - <> + ?VAR_USERNAME, + ?VAR_CLIENTID, + ?VAR_PEERHOST, + ?VAR_CERT_CN_NAME, + ?VAR_CERT_SUBJECT ]). description() -> diff --git a/apps/emqx_auth_redis/src/emqx_authz_redis.erl b/apps/emqx_auth_redis/src/emqx_authz_redis.erl index 7ac893da1..eb63804b9 100644 --- a/apps/emqx_auth_redis/src/emqx_authz_redis.erl +++ b/apps/emqx_auth_redis/src/emqx_authz_redis.erl @@ -36,11 +36,11 @@ -endif. -define(PLACEHOLDERS, [ - <>, - <>, - <>, - <>, - <> + ?VAR_CERT_CN_NAME, + ?VAR_CERT_SUBJECT, + ?VAR_PEERHOST, + ?VAR_CLIENTID, + ?VAR_USERNAME ]). description() -> diff --git a/apps/emqx_connector/src/emqx_connector_template.erl b/apps/emqx_connector/src/emqx_connector_template.erl index bb26edec1..221cc5e86 100644 --- a/apps/emqx_connector/src/emqx_connector_template.erl +++ b/apps/emqx_connector/src/emqx_connector_template.erl @@ -37,6 +37,7 @@ -export_type([str/0]). -export_type([deep/0]). -export_type([placeholder/0]). +-export_type([varname/0]). -export_type([bindings/0]). -type t() :: str() | {'$tpl', deeptpl()}. @@ -55,8 +56,9 @@ | port() | reference(). --type placeholder() :: {var, var()}. --type var() :: _Name :: [binary()]. +-type placeholder() :: {var, varname(), accessor()}. +-type accessor() :: [binary()]. +-type varname() :: string(). -type scalar() :: atom() | unicode:chardata() | number(). -type binding() :: scalar() | list(scalar()) | bindings(). @@ -64,7 +66,7 @@ -type var_trans() :: fun((Value :: term()) -> unicode:chardata()) - | fun((var(), Value :: term()) -> unicode:chardata()). + | fun((varname(), Value :: term()) -> unicode:chardata()). -type parse_opts() :: #{ strip_double_quote => boolean() @@ -103,7 +105,7 @@ parse(String, Opts) -> parse_split([Part, _PH, <<>>, Var]) -> % Regular placeholder - prepend(Part, [{var, parse_var(Var)}]); + prepend(Part, [{var, unicode:characters_to_list(Var), parse_accessor(Var)}]); parse_split([Part, _PH = <>, <<"$">>, _]) -> % Escaped literal, take all but the second byte, which is always `$`. % Important to make a whole token starting with `$` so the `unparse/11` @@ -117,7 +119,7 @@ prepend(<<>>, To) -> prepend(Head, To) -> [Head | To]. -parse_var(Var) -> +parse_accessor(Var) -> case string:split(Var, <<".">>, all) of [<<>>] -> ?PH_VAR_THIS; @@ -126,10 +128,9 @@ parse_var(Var) -> Name end. --spec validate([var() | binary()], t()) -> - ok | {error, [_Error :: {var(), disallowed}]}. -validate(AllowedIn, Template) -> - Allowed = [try_parse_var(V) || V <- AllowedIn], +-spec validate([varname()], t()) -> + ok | {error, [_Error :: {varname(), disallowed}]}. +validate(Allowed, Template) -> {_, Errors} = render(Template, #{}), {Used, _} = lists:unzip(Errors), case lists:usort(Used) -- Allowed of @@ -139,11 +140,6 @@ validate(AllowedIn, Template) -> {error, [{Var, disallowed} || Var <- Disallowed]} end. -try_parse_var(Var) when is_binary(Var) -> - parse_var(Var); -try_parse_var(Name) when is_list(Name) -> - Name. - -spec trivial(t()) -> boolean(). trivial(Template) -> @@ -156,7 +152,7 @@ unparse({'$tpl', Template}) -> unparse(Template) -> unicode:characters_to_list(lists:map(fun unparse_part/1, Template)). -unparse_part({var, Name}) -> +unparse_part({var, Name, _Accessor}) -> render_placeholder(Name); unparse_part(Part = <<"${", _/binary>>) -> <<"$", Part/binary>>; @@ -164,7 +160,7 @@ unparse_part(Part) -> Part. render_placeholder(Name) -> - "${" ++ lists:join($., Name) ++ "}". + "${" ++ Name ++ "}". %% @doc Render a template with given bindings. %% Returns a term with all placeholders replaced with values from bindings. @@ -172,17 +168,17 @@ render_placeholder(Name) -> %% By default, all binding values are converted to strings using `to_string/1` %% function. Option `var_trans` can be used to override this behaviour. -spec render(t(), bindings()) -> - {term(), [_Error :: {var(), undefined}]}. + {term(), [_Error :: {varname(), undefined}]}. render(Template, Bindings) -> render(Template, Bindings, #{}). -spec render(t(), bindings(), render_opts()) -> - {term(), [_Error :: {var(), undefined}]}. + {term(), [_Error :: {varname(), undefined}]}. render(Template, Bindings, Opts) when is_list(Template) -> lists:mapfoldl( fun - ({var, Name}, EAcc) -> - {String, Errors} = render_binding(Name, Bindings, Opts), + ({var, Name, Accessor}, EAcc) -> + {String, Errors} = render_binding(Name, Accessor, Bindings, Opts), {String, Errors ++ EAcc}; (String, EAcc) -> {String, EAcc} @@ -193,8 +189,8 @@ render(Template, Bindings, Opts) when is_list(Template) -> render({'$tpl', Template}, Bindings, Opts) -> render_deep(Template, Bindings, Opts). -render_binding(Name, Bindings, Opts) -> - case lookup_var(Name, Bindings) of +render_binding(Name, Accessor, Bindings, Opts) -> + case lookup_var(Accessor, Bindings) of {ok, Value} -> {render_value(Name, Value, Opts), []}; {error, Reason} -> @@ -231,12 +227,12 @@ render_strict(Template, Bindings, Opts) -> %% lists are not analyzed for "printability" and are treated as nested terms. %% The result is a usual template, and can be fed to other functions in this %% module. --spec parse_deep(unicode:chardata()) -> +-spec parse_deep(term()) -> t(). parse_deep(Term) -> parse_deep(Term, #{}). --spec parse_deep(unicode:chardata(), parse_opts()) -> +-spec parse_deep(term(), parse_opts()) -> t(). parse_deep(Term, Opts) -> {'$tpl', parse_deep_term(Term, Opts)}. @@ -305,7 +301,7 @@ unparse_deep(Term) -> %% --spec lookup_var(var(), bindings()) -> +-spec lookup_var(accessor(), bindings()) -> {ok, binding()} | {error, undefined}. lookup_var(Var, Value) when Var == ?PH_VAR_THIS orelse Var == [] -> {ok, Value}; diff --git a/apps/emqx_connector/src/emqx_connector_template_sql.erl b/apps/emqx_connector/src/emqx_connector_template_sql.erl index 0febfe575..e95ecde42 100644 --- a/apps/emqx_connector/src/emqx_connector_template_sql.erl +++ b/apps/emqx_connector/src/emqx_connector_template_sql.erl @@ -88,14 +88,14 @@ render_strict(Template, Bindings, Opts) -> %% #{parameters => '$n'} %% ), %% Statement = <<"INSERT INTO table (id, name, age) VALUES ($1, $2, 42)">>, -%% RowTemplate = [{var, [...]}, ...] +%% RowTemplate = [{var, "...", [...]}, ...] %% ``` -spec parse_prepstmt(unicode:chardata(), parse_opts()) -> {unicode:chardata(), row_template()}. parse_prepstmt(String, Opts) -> Template = emqx_connector_template:parse(String, maps:with(?TEMPLATE_PARSE_OPTS, Opts)), Statement = mk_prepared_statement(Template, Opts), - Placeholders = [Placeholder || Placeholder = {var, _} <- Template], + Placeholders = [Placeholder || Placeholder <- Template, element(1, Placeholder) == var], {Statement, Placeholders}. mk_prepared_statement(Template, Opts) -> @@ -103,7 +103,7 @@ mk_prepared_statement(Template, Opts) -> {Statement, _} = lists:mapfoldl( fun - ({var, _}, Acc) -> + (Var, Acc) when element(1, Var) == var -> mk_replace(ParameterFormat, Acc); (String, Acc) -> {String, Acc} diff --git a/apps/emqx_connector/test/emqx_connector_template_SUITE.erl b/apps/emqx_connector/test/emqx_connector_template_SUITE.erl index 666fbfa58..998baae37 100644 --- a/apps/emqx_connector/test/emqx_connector_template_SUITE.erl +++ b/apps/emqx_connector/test/emqx_connector_template_SUITE.erl @@ -47,7 +47,7 @@ t_render_var_trans(_) -> {String, Errors} = emqx_connector_template:render( Template, Bindings, - #{var_trans => fun(Name, _) -> "<" ++ lists:join($., Name) ++ ">" end} + #{var_trans => fun(Name, _) -> "<" ++ Name ++ ">" end} ), ?assertEqual( {<<"a:,b:,c:">>, []}, @@ -59,7 +59,7 @@ t_render_path(_) -> Template = emqx_connector_template:parse(<<"d.d1:${d.d1}">>), ?assertEqual( ok, - emqx_connector_template:validate([<<"d.d1">>], Template) + emqx_connector_template:validate(["d.d1"], Template) ), ?assertEqual( {<<"d.d1:hi">>, []}, @@ -70,8 +70,8 @@ t_render_custom_ph(_) -> Bindings = #{a => <<"a">>, b => <<"b">>}, Template = emqx_connector_template:parse(<<"a:${a},b:${b}">>), ?assertEqual( - {error, [{[<<"b">>], disallowed}]}, - emqx_connector_template:validate([<<"a">>], Template) + {error, [{"b", disallowed}]}, + emqx_connector_template:validate(["a"], Template) ), ?assertEqual( <<"a:a,b:b">>, @@ -81,7 +81,7 @@ t_render_custom_ph(_) -> t_render_this(_) -> Bindings = #{a => <<"a">>, b => [1, 2, 3]}, Template = emqx_connector_template:parse(<<"this:${} / also:${.}">>), - ?assertEqual(ok, emqx_connector_template:validate([?PH_VAR_THIS], Template)), + ?assertEqual(ok, emqx_connector_template:validate(["."], Template)), ?assertEqual( % NOTE: order of the keys in the JSON object depends on the JSON encoder <<"this:{\"b\":[1,2,3],\"a\":\"a\"} / also:{\"b\":[1,2,3],\"a\":\"a\"}">>, @@ -95,21 +95,21 @@ t_render_missing_bindings(_) -> ), ?assertEqual( {<<"a:,b:,c:,d:,e:">>, [ - {[<<"no">>, <<"such_atom_i_swear">>], undefined}, - {[<<"d">>, <<"d1">>], undefined}, - {[<<"c">>], undefined}, - {[<<"b">>], undefined}, - {[<<"a">>], undefined} + {"no.such_atom_i_swear", undefined}, + {"d.d1", undefined}, + {"c", undefined}, + {"b", undefined}, + {"a", undefined} ]}, render_string(Template, Bindings) ), ?assertError( [ - {[<<"no">>, <<"such_atom_i_swear">>], undefined}, - {[<<"d">>, <<"d1">>], undefined}, - {[<<"c">>], undefined}, - {[<<"b">>], undefined}, - {[<<"a">>], undefined} + {"no.such_atom_i_swear", undefined}, + {"d.d1", undefined}, + {"c", undefined}, + {"b", undefined}, + {"a", undefined} ], render_strict_string(Template, Bindings) ). @@ -256,10 +256,10 @@ t_render_cql(_) -> t_render_sql_custom_ph(_) -> {PrepareStatement, RowTemplate} = - emqx_connector_template_sql:parse_prepstmt(<<"a:${a},b:${b}">>, #{parameters => '$n'}), + emqx_connector_template_sql:parse_prepstmt(<<"a:${a},b:${b.c}">>, #{parameters => '$n'}), ?assertEqual( - {error, [{[<<"b">>], disallowed}]}, - emqx_connector_template:validate([<<"a">>], RowTemplate) + {error, [{"b.c", disallowed}]}, + emqx_connector_template:validate(["a"], RowTemplate) ), ?assertEqual(<<"a:$1,b:$2">>, bin(PrepareStatement)). @@ -296,8 +296,8 @@ t_render_tmpl_deep(_) -> ), ?assertEqual( - {error, [{V, disallowed} || V <- [[<<"b">>], [<<"c">>]]]}, - emqx_connector_template:validate([<<"a">>], Template) + {error, [{V, disallowed} || V <- ["b", "c"]]}, + emqx_connector_template:validate(["a"], Template) ), ?assertEqual( diff --git a/apps/emqx_rule_engine/src/emqx_rule_actions.erl b/apps/emqx_rule_engine/src/emqx_rule_actions.erl index bb9966b4a..fa677ce78 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_actions.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_actions.erl @@ -224,8 +224,8 @@ parse_user_properties(_) -> %% invalid, discard undefined. -render_simple_var([{var, Name}], Data, Default) -> - case emqx_connector_template:lookup_var(Name, Data) of +render_simple_var([{var, _Name, Accessor}], Data, Default) -> + case emqx_connector_template:lookup_var(Accessor, Data) of {ok, Var} -> Var; %% cannot find the variable from Data {error, _} -> Default From 49fba40ee7a402ce07fc6620ea035b21eea0573e Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 27 Apr 2023 13:53:17 +0300 Subject: [PATCH 07/21] fix(tpl): ensure backward compat with authz / authn templates This commit leans heavy into discouraging the former approach where only part of placeholders were interpolated, depending on `placeholders` option. --- .../src/emqx_authn/emqx_authn_utils.erl | 97 +++++++++++++------ .../src/emqx_authz/emqx_authz_rule.erl | 3 +- .../src/emqx_authz/emqx_authz_utils.erl | 85 +++++++++++----- apps/emqx_auth_http/src/emqx_authz_http.erl | 26 ++--- .../test/emqx_authn_http_SUITE.erl | 53 ++++++++-- .../test/emqx_authz_http_SUITE.erl | 61 ++++++++++++ .../src/emqx_authz_mongodb.erl | 6 +- apps/emqx_auth_mysql/src/emqx_authz_mysql.erl | 6 +- .../src/emqx_authz_postgresql.erl | 6 +- apps/emqx_auth_redis/src/emqx_authz_redis.erl | 4 +- 10 files changed, 260 insertions(+), 87 deletions(-) diff --git a/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl b/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl index 9be3e24d6..d9b20a47c 100644 --- a/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl +++ b/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl @@ -18,6 +18,7 @@ -include_lib("emqx/include/emqx_placeholder.hrl"). -include_lib("emqx_authn.hrl"). +-include_lib("snabbkaffe/include/trace.hrl"). -export([ create_resource/3, @@ -44,13 +45,13 @@ default_headers_no_content_type/0 ]). --define(AUTHN_PLACEHOLDERS, [ - <>, - <>, - <>, - <>, - <>, - <> +-define(ALLOWED_VARS, [ + ?VAR_USERNAME, + ?VAR_CLIENTID, + ?VAR_PASSWORD, + ?VAR_PEERHOST, + ?VAR_CERT_SUBJECT, + ?VAR_CERT_CN_NAME ]). -define(DEFAULT_RESOURCE_OPTS, #{ @@ -108,21 +109,55 @@ check_password_from_selected_map(Algorithm, Selected, Password) -> parse_deep(Template) -> Result = emqx_connector_template:parse_deep(Template), - ok = emqx_connector_template:validate(?AUTHN_PLACEHOLDERS, Result), - Result. + handle_disallowed_placeholders(Result, {deep, Template}). parse_str(Template) -> Result = emqx_connector_template:parse(Template), - ok = emqx_connector_template:validate(?AUTHN_PLACEHOLDERS, Result), - Result. + handle_disallowed_placeholders(Result, {string, Template}). parse_sql(Template, ReplaceWith) -> {Statement, Result} = emqx_connector_template_sql:parse_prepstmt( Template, #{parameters => ReplaceWith, strip_double_quote => true} ), - ok = emqx_connector_template:validate(?AUTHN_PLACEHOLDERS, Result), - {Statement, Result}. + {Statement, handle_disallowed_placeholders(Result, {string, Template})}. + +handle_disallowed_placeholders(Template, Source) -> + case emqx_connector_template:validate(?ALLOWED_VARS, Template) of + ok -> + Template; + {error, Disallowed} -> + ?tp(warning, "authn_template_invalid", #{ + template => Source, + reason => Disallowed, + allowed => #{placeholders => ?ALLOWED_VARS}, + notice => + "Disallowed placeholders will be rendered as is." + " However, consider using `$${...}` escaping for literal `${...}` where" + " needed to avoid unexpected results." + }), + Result = prerender_disallowed_placeholders(Template), + case Source of + {string, _} -> + emqx_connector_template:parse(Result); + {deep, _} -> + emqx_connector_template:parse_deep(Result) + end + end. + +prerender_disallowed_placeholders(Template) -> + {Result, _} = emqx_connector_template:render(Template, #{}, #{ + var_trans => fun(Name, _) -> + % NOTE + % Rendering disallowed placeholders in escaped form, which will then + % parse as a literal string. + case lists:member(Name, ?ALLOWED_VARS) of + true -> "${" ++ Name ++ "}"; + false -> "$${" ++ Name ++ "}" + end + end + }), + Result. render_deep(Template, Credential) -> % NOTE @@ -130,7 +165,7 @@ render_deep(Template, Credential) -> {Term, _Errors} = emqx_connector_template:render( Template, mapping_credential(Credential), - #{var_trans => fun handle_var/2} + #{var_trans => fun to_string/2} ), Term. @@ -140,7 +175,7 @@ render_str(Template, Credential) -> {String, _Errors} = emqx_connector_template:render( Template, mapping_credential(Credential), - #{var_trans => fun handle_var/2} + #{var_trans => fun to_string/2} ), unicode:characters_to_binary(String). @@ -150,7 +185,7 @@ render_urlencoded_str(Template, Credential) -> {String, _Errors} = emqx_connector_template:render( Template, mapping_credential(Credential), - #{var_trans => fun urlencode_var/2} + #{var_trans => fun to_urlencoded_string/2} ), unicode:characters_to_binary(String). @@ -160,7 +195,7 @@ render_sql_params(ParamList, Credential) -> {Row, _Errors} = emqx_connector_template:render( ParamList, mapping_credential(Credential), - #{var_trans => fun handle_sql_var/2} + #{var_trans => fun to_sql_valaue/2} ), Row. @@ -283,22 +318,24 @@ without_password(Credential, [Name | Rest]) -> without_password(Credential, Rest) end. -urlencode_var(Var, Value) -> - emqx_http_lib:uri_encode(handle_var(Var, Value)). +to_urlencoded_string(Name, Value) -> + emqx_http_lib:uri_encode(to_string(Name, Value)). -handle_var(_, undefined) -> - <<>>; -handle_var([<<"peerhost">>], PeerHost) -> - emqx_connector_template:to_string(inet:ntoa(PeerHost)); -handle_var(_, Value) -> - emqx_connector_template:to_string(Value). +to_string(Name, Value) -> + emqx_connector_template:to_string(render_var(Name, Value)). -handle_sql_var(_, undefined) -> +to_sql_valaue(Name, Value) -> + emqx_connector_sql:to_sql_value(render_var(Name, Value)). + +render_var(_, undefined) -> + % NOTE + % Any allowed but undefined binding will be replaced with empty string, even when + % rendering SQL values. <<>>; -handle_sql_var([<<"peerhost">>], PeerHost) -> - emqx_connector_sql:to_sql_value(inet:ntoa(PeerHost)); -handle_sql_var(_, Value) -> - emqx_connector_sql:to_sql_value(Value). +render_var(?VAR_PEERHOST, Value) -> + inet:ntoa(Value); +render_var(_Name, Value) -> + Value. mapping_credential(C = #{cn := CN, dn := DN}) -> C#{cert_common_name => CN, cert_subject => DN}; diff --git a/apps/emqx_auth/src/emqx_authz/emqx_authz_rule.erl b/apps/emqx_auth/src/emqx_authz/emqx_authz_rule.erl index 6f5369aec..1e7a49855 100644 --- a/apps/emqx_auth/src/emqx_authz/emqx_authz_rule.erl +++ b/apps/emqx_auth/src/emqx_authz/emqx_authz_rule.erl @@ -183,8 +183,7 @@ compile_topic(<<"eq ", Topic/binary>>) -> compile_topic({eq, Topic}) -> {eq, emqx_topic:words(bin(Topic))}; compile_topic(Topic) -> - Template = emqx_connector_template:parse(Topic), - ok = emqx_connector_template:validate([?VAR_USERNAME, ?VAR_CLIENTID], Template), + Template = emqx_authz_utils:parse_str(Topic, [?VAR_USERNAME, ?VAR_CLIENTID]), case emqx_connector_template:trivial(Template) of true -> emqx_topic:words(bin(Topic)); false -> {pattern, Template} diff --git a/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl b/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl index e5aeab21b..bd7b353a5 100644 --- a/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl +++ b/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl @@ -16,7 +16,9 @@ -module(emqx_authz_utils). +-include_lib("emqx/include/emqx_placeholder.hrl"). -include_lib("emqx_authz.hrl"). +-include_lib("snabbkaffe/include/trace.hrl"). -export([ cleanup_resources/0, @@ -109,21 +111,56 @@ update_config(Path, ConfigRequest) -> parse_deep(Template, PlaceHolders) -> Result = emqx_connector_template:parse_deep(Template), - ok = emqx_connector_template:validate(PlaceHolders, Result), - Result. + handle_disallowed_placeholders(Result, {deep, Template}, PlaceHolders). parse_str(Template, PlaceHolders) -> Result = emqx_connector_template:parse(Template), - ok = emqx_connector_template:validate(PlaceHolders, Result), - Result. + handle_disallowed_placeholders(Result, {string, Template}, PlaceHolders). parse_sql(Template, ReplaceWith, PlaceHolders) -> {Statement, Result} = emqx_connector_template_sql:parse_prepstmt( Template, #{parameters => ReplaceWith, strip_double_quote => true} ), - ok = emqx_connector_template:validate(PlaceHolders, Result), - {Statement, Result}. + FResult = handle_disallowed_placeholders(Result, {string, Template}, PlaceHolders), + {Statement, FResult}. + +handle_disallowed_placeholders(Template, Source, Allowed) -> + case emqx_connector_template:validate(Allowed, Template) of + ok -> + Template; + {error, Disallowed} -> + ?tp(warning, "authz_template_invalid", #{ + template => Source, + reason => Disallowed, + allowed => #{placeholders => Allowed}, + notice => + "Disallowed placeholders will be rendered as is." + " However, consider using `$${...}` escaping for literal `${...}` where" + " needed to avoid unexpected results." + }), + Result = prerender_disallowed_placeholders(Template, Allowed), + case Source of + {string, _} -> + emqx_connector_template:parse(Result); + {deep, _} -> + emqx_connector_template:parse_deep(Result) + end + end. + +prerender_disallowed_placeholders(Template, Allowed) -> + {Result, _} = emqx_connector_template:render(Template, #{}, #{ + var_trans => fun(Name, _) -> + % NOTE + % Rendering disallowed placeholders in escaped form, which will then + % parse as a literal string. + case lists:member(Name, Allowed) of + true -> "${" ++ Name ++ "}"; + false -> "$${" ++ Name ++ "}" + end + end + }), + Result. render_deep(Template, Values) -> % NOTE @@ -131,7 +168,7 @@ render_deep(Template, Values) -> {Term, _Errors} = emqx_connector_template:render( Template, client_vars(Values), - #{var_trans => fun handle_var/2} + #{var_trans => fun to_string/2} ), Term. @@ -141,7 +178,7 @@ render_str(Template, Values) -> {String, _Errors} = emqx_connector_template:render( Template, client_vars(Values), - #{var_trans => fun handle_var/2} + #{var_trans => fun to_string/2} ), unicode:characters_to_binary(String). @@ -151,7 +188,7 @@ render_urlencoded_str(Template, Values) -> {String, _Errors} = emqx_connector_template:render( Template, client_vars(Values), - #{var_trans => fun urlencode_var/2} + #{var_trans => fun to_urlencoded_string/2} ), unicode:characters_to_binary(String). @@ -161,7 +198,7 @@ render_sql_params(ParamList, Values) -> {Row, _Errors} = emqx_connector_template:render( ParamList, client_vars(Values), - #{var_trans => fun handle_sql_var/2} + #{var_trans => fun to_sql_value/2} ), Row. @@ -229,22 +266,24 @@ convert_client_var({dn, DN}) -> {cert_subject, DN}; convert_client_var({protocol, Proto}) -> {proto_name, Proto}; convert_client_var(Other) -> Other. -urlencode_var(Var, Value) -> - emqx_http_lib:uri_encode(handle_var(Var, Value)). +to_urlencoded_string(Name, Value) -> + emqx_http_lib:uri_encode(to_string(Name, Value)). -handle_var(_, undefined) -> - <<>>; -handle_var([<<"peerhost">>], IpAddr) -> - inet_parse:ntoa(IpAddr); -handle_var(_Name, Value) -> - emqx_connector_template:to_string(Value). +to_string(Name, Value) -> + emqx_connector_template:to_string(render_var(Name, Value)). -handle_sql_var(_, undefined) -> +to_sql_value(Name, Value) -> + emqx_connector_sql:to_sql_value(render_var(Name, Value)). + +render_var(_, undefined) -> + % NOTE + % Any allowed but undefined binding will be replaced with empty string, even when + % rendering SQL values. <<>>; -handle_sql_var([<<"peerhost">>], IpAddr) -> - inet_parse:ntoa(IpAddr); -handle_sql_var(_Name, Value) -> - emqx_connector_sql:to_sql_value(Value). +render_var(?VAR_PEERHOST, Value) -> + inet:ntoa(Value); +render_var(_Name, Value) -> + Value. bin(A) when is_atom(A) -> atom_to_binary(A, utf8); bin(L) when is_list(L) -> list_to_binary(L); diff --git a/apps/emqx_auth_http/src/emqx_authz_http.erl b/apps/emqx_auth_http/src/emqx_authz_http.erl index bbb2bf9b5..04f76b4c9 100644 --- a/apps/emqx_auth_http/src/emqx_authz_http.erl +++ b/apps/emqx_auth_http/src/emqx_authz_http.erl @@ -38,7 +38,7 @@ -compile(nowarn_export_all). -endif. --define(PLACEHOLDERS, [ +-define(ALLOWED_VARS, [ ?VAR_USERNAME, ?VAR_CLIENTID, ?VAR_PEERHOST, @@ -50,9 +50,9 @@ ?VAR_CERT_CN_NAME ]). --define(PLACEHOLDERS_FOR_RICH_ACTIONS, [ - <>, - <> +-define(ALLOWED_VARS_RICH_ACTIONS, [ + ?VAR_QOS, + ?VAR_RETAIN ]). description() -> @@ -157,14 +157,14 @@ parse_config( method => Method, base_url => BaseUrl, headers => Headers, - base_path_templete => emqx_authz_utils:parse_str(Path, placeholders()), + base_path_templete => emqx_authz_utils:parse_str(Path, allowed_vars()), base_query_template => emqx_authz_utils:parse_deep( cow_qs:parse_qs(to_bin(Query)), - placeholders() + allowed_vars() ), body_template => emqx_authz_utils:parse_deep( maps:to_list(maps:get(body, Conf, #{})), - placeholders() + allowed_vars() ), request_timeout => ReqTimeout, %% pool_type default value `random` @@ -260,10 +260,10 @@ to_bin(B) when is_binary(B) -> B; to_bin(L) when is_list(L) -> list_to_binary(L); to_bin(X) -> X. -placeholders() -> - placeholders(emqx_authz:feature_available(rich_actions)). +allowed_vars() -> + allowed_vars(emqx_authz:feature_available(rich_actions)). -placeholders(true) -> - ?PLACEHOLDERS ++ ?PLACEHOLDERS_FOR_RICH_ACTIONS; -placeholders(false) -> - ?PLACEHOLDERS. +allowed_vars(true) -> + ?ALLOWED_VARS ++ ?ALLOWED_VARS_RICH_ACTIONS; +allowed_vars(false) -> + ?ALLOWED_VARS. diff --git a/apps/emqx_auth_http/test/emqx_authn_http_SUITE.erl b/apps/emqx_auth_http/test/emqx_authn_http_SUITE.erl index 577b3b638..e307b5bbf 100644 --- a/apps/emqx_auth_http/test/emqx_authn_http_SUITE.erl +++ b/apps/emqx_auth_http/test/emqx_authn_http_SUITE.erl @@ -27,7 +27,7 @@ -define(PATH, [?CONF_NS_ATOM]). -define(HTTP_PORT, 32333). --define(HTTP_PATH, "/auth"). +-define(HTTP_PATH, "/auth/[...]"). -define(CREDENTIALS, #{ clientid => <<"clienta">>, username => <<"plain">>, @@ -146,8 +146,12 @@ t_authenticate(_Config) -> test_user_auth(#{ handler := Handler, config_params := SpecificConfgParams, - result := Result + result := Expect }) -> + Result = perform_user_auth(SpecificConfgParams, Handler, ?CREDENTIALS), + ?assertEqual(Expect, Result). + +perform_user_auth(SpecificConfgParams, Handler, Credentials) -> AuthConfig = maps:merge(raw_http_auth_config(), SpecificConfgParams), {ok, _} = emqx:update_config( @@ -157,21 +161,21 @@ test_user_auth(#{ ok = emqx_authn_http_test_server:set_handler(Handler), - ?assertEqual(Result, emqx_access_control:authenticate(?CREDENTIALS)), + Result = emqx_access_control:authenticate(Credentials), emqx_authn_test_lib:delete_authenticators( [authentication], ?GLOBAL - ). + ), + + Result. t_authenticate_path_placeholders(_Config) -> - ok = emqx_authn_http_test_server:stop(), - {ok, _} = emqx_authn_http_test_server:start_link(?HTTP_PORT, <<"/[...]">>), ok = emqx_authn_http_test_server:set_handler( fun(Req0, State) -> Req = case cowboy_req:path(Req0) of - <<"/my/p%20ath//us%20er/auth//">> -> + <<"/auth/p%20ath//us%20er/auth//">> -> cowboy_req:reply( 200, #{<<"content-type">> => <<"application/json">>}, @@ -193,7 +197,7 @@ t_authenticate_path_placeholders(_Config) -> AuthConfig = maps:merge( raw_http_auth_config(), #{ - <<"url">> => <<"http://127.0.0.1:32333/my/p%20ath//${username}/auth//">>, + <<"url">> => <<"http://127.0.0.1:32333/auth/p%20ath//${username}/auth//">>, <<"body">> => #{} } ), @@ -255,6 +259,39 @@ t_no_value_for_placeholder(_Config) -> ?GLOBAL ). +t_disallowed_placeholders_preserved(_Config) -> + Config = #{ + <<"method">> => <<"post">>, + <<"headers">> => #{<<"content-type">> => <<"application/json">>}, + <<"body">> => #{ + <<"username">> => ?PH_USERNAME, + <<"password">> => ?PH_PASSWORD, + <<"this">> => <<"${whatisthis}">> + } + }, + Handler = fun(Req0, State) -> + {ok, Body, Req1} = cowboy_req:read_body(Req0), + #{ + <<"username">> := <<"plain">>, + <<"password">> := <<"plain">>, + <<"this">> := <<"${whatisthis}">> + } = emqx_utils_json:decode(Body), + Req = cowboy_req:reply( + 200, + #{<<"content-type">> => <<"application/json">>}, + emqx_utils_json:encode(#{result => allow, is_superuser => false}), + Req1 + ), + {ok, Req, State} + end, + ?assertMatch({ok, _}, perform_user_auth(Config, Handler, ?CREDENTIALS)), + + % NOTE: disallowed placeholder left intact, which makes the URL invalid + ConfigUrl = Config#{ + <<"url">> => <<"http://127.0.0.1:32333/auth/${whatisthis}">> + }, + ?assertMatch({error, _}, perform_user_auth(ConfigUrl, Handler, ?CREDENTIALS)). + t_destroy(_Config) -> AuthConfig = raw_http_auth_config(), diff --git a/apps/emqx_auth_http/test/emqx_authz_http_SUITE.erl b/apps/emqx_auth_http/test/emqx_authz_http_SUITE.erl index e56e25f5f..845259e78 100644 --- a/apps/emqx_auth_http/test/emqx_authz_http_SUITE.erl +++ b/apps/emqx_auth_http/test/emqx_authz_http_SUITE.erl @@ -494,6 +494,67 @@ t_no_value_for_placeholder(_Config) -> emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>) ). +t_disallowed_placeholders_preserved(_Config) -> + ok = setup_handler_and_config( + fun(Req0, State) -> + {ok, Body, Req1} = cowboy_req:read_body(Req0), + ?assertMatch( + #{ + <<"cname">> := <<>>, + <<"usertypo">> := <<"${usertypo}">> + }, + emqx_utils_json:decode(Body) + ), + {ok, ?AUTHZ_HTTP_RESP(allow, Req1), State} + end, + #{ + <<"method">> => <<"post">>, + <<"body">> => #{ + <<"cname">> => ?PH_CERT_CN_NAME, + <<"usertypo">> => <<"${usertypo}">> + } + } + ), + + ClientInfo = #{ + clientid => <<"client id">>, + username => <<"user name">>, + peerhost => {127, 0, 0, 1}, + protocol => <<"MQTT">>, + zone => default, + listener => {tcp, default} + }, + + ?assertEqual( + allow, + emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>) + ). + +t_disallowed_placeholders_path(_Config) -> + ok = setup_handler_and_config( + fun(Req, State) -> + {ok, ?AUTHZ_HTTP_RESP(allow, Req), State} + end, + #{ + <<"url">> => <<"http://127.0.0.1:33333/authz/use%20rs/${typo}">> + } + ), + + ClientInfo = #{ + clientid => <<"client id">>, + username => <<"user name">>, + peerhost => {127, 0, 0, 1}, + protocol => <<"MQTT">>, + zone => default, + listener => {tcp, default} + }, + + % % NOTE: disallowed placeholder left intact, which makes the URL invalid + ?assertEqual( + deny, + emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>) + ). + t_create_replace(_Config) -> ClientInfo = #{ clientid => <<"clientid">>, diff --git a/apps/emqx_auth_mongodb/src/emqx_authz_mongodb.erl b/apps/emqx_auth_mongodb/src/emqx_authz_mongodb.erl index 35ac3a41b..fdeb9d542 100644 --- a/apps/emqx_auth_mongodb/src/emqx_authz_mongodb.erl +++ b/apps/emqx_auth_mongodb/src/emqx_authz_mongodb.erl @@ -35,7 +35,7 @@ -compile(nowarn_export_all). -endif. --define(PLACEHOLDERS, [ +-define(ALLOWED_VARS, [ ?VAR_USERNAME, ?VAR_CLIENTID, ?VAR_PEERHOST, @@ -49,11 +49,11 @@ description() -> create(#{filter := Filter} = Source) -> ResourceId = emqx_authz_utils:make_resource_id(?MODULE), {ok, _Data} = emqx_authz_utils:create_resource(ResourceId, emqx_mongodb, Source), - FilterTemp = emqx_authz_utils:parse_deep(Filter, ?PLACEHOLDERS), + FilterTemp = emqx_authz_utils:parse_deep(Filter, ?ALLOWED_VARS), Source#{annotations => #{id => ResourceId}, filter_template => FilterTemp}. update(#{filter := Filter} = Source) -> - FilterTemp = emqx_authz_utils:parse_deep(Filter, ?PLACEHOLDERS), + FilterTemp = emqx_authz_utils:parse_deep(Filter, ?ALLOWED_VARS), case emqx_authz_utils:update_resource(emqx_mongodb, Source) of {error, Reason} -> error({load_config_error, Reason}); diff --git a/apps/emqx_auth_mysql/src/emqx_authz_mysql.erl b/apps/emqx_auth_mysql/src/emqx_authz_mysql.erl index a6d71d1ca..8c9e54ee1 100644 --- a/apps/emqx_auth_mysql/src/emqx_authz_mysql.erl +++ b/apps/emqx_auth_mysql/src/emqx_authz_mysql.erl @@ -37,7 +37,7 @@ -compile(nowarn_export_all). -endif. --define(PLACEHOLDERS, [ +-define(ALLOWED_VARS, [ ?VAR_USERNAME, ?VAR_CLIENTID, ?VAR_PEERHOST, @@ -49,14 +49,14 @@ description() -> "AuthZ with Mysql". create(#{query := SQL} = Source0) -> - {PrepareSQL, TmplToken} = emqx_authz_utils:parse_sql(SQL, '?', ?PLACEHOLDERS), + {PrepareSQL, TmplToken} = emqx_authz_utils:parse_sql(SQL, '?', ?ALLOWED_VARS), ResourceId = emqx_authz_utils:make_resource_id(?MODULE), Source = Source0#{prepare_statement => #{?PREPARE_KEY => PrepareSQL}}, {ok, _Data} = emqx_authz_utils:create_resource(ResourceId, emqx_mysql, Source), Source#{annotations => #{id => ResourceId, tmpl_token => TmplToken}}. update(#{query := SQL} = Source0) -> - {PrepareSQL, TmplToken} = emqx_authz_utils:parse_sql(SQL, '?', ?PLACEHOLDERS), + {PrepareSQL, TmplToken} = emqx_authz_utils:parse_sql(SQL, '?', ?ALLOWED_VARS), Source = Source0#{prepare_statement => #{?PREPARE_KEY => PrepareSQL}}, case emqx_authz_utils:update_resource(emqx_mysql, Source) of {error, Reason} -> diff --git a/apps/emqx_auth_postgresql/src/emqx_authz_postgresql.erl b/apps/emqx_auth_postgresql/src/emqx_authz_postgresql.erl index b538bd95e..14b7598a6 100644 --- a/apps/emqx_auth_postgresql/src/emqx_authz_postgresql.erl +++ b/apps/emqx_auth_postgresql/src/emqx_authz_postgresql.erl @@ -37,7 +37,7 @@ -compile(nowarn_export_all). -endif. --define(PLACEHOLDERS, [ +-define(ALLOWED_VARS, [ ?VAR_USERNAME, ?VAR_CLIENTID, ?VAR_PEERHOST, @@ -49,7 +49,7 @@ description() -> "AuthZ with PostgreSQL". create(#{query := SQL0} = Source) -> - {SQL, PlaceHolders} = emqx_authz_utils:parse_sql(SQL0, '$n', ?PLACEHOLDERS), + {SQL, PlaceHolders} = emqx_authz_utils:parse_sql(SQL0, '$n', ?ALLOWED_VARS), ResourceID = emqx_authz_utils:make_resource_id(emqx_postgresql), {ok, _Data} = emqx_authz_utils:create_resource( ResourceID, @@ -59,7 +59,7 @@ create(#{query := SQL0} = Source) -> Source#{annotations => #{id => ResourceID, placeholders => PlaceHolders}}. update(#{query := SQL0, annotations := #{id := ResourceID}} = Source) -> - {SQL, PlaceHolders} = emqx_authz_utils:parse_sql(SQL0, '$n', ?PLACEHOLDERS), + {SQL, PlaceHolders} = emqx_authz_utils:parse_sql(SQL0, '$n', ?ALLOWED_VARS), case emqx_authz_utils:update_resource( emqx_postgresql, diff --git a/apps/emqx_auth_redis/src/emqx_authz_redis.erl b/apps/emqx_auth_redis/src/emqx_authz_redis.erl index eb63804b9..ca4a11742 100644 --- a/apps/emqx_auth_redis/src/emqx_authz_redis.erl +++ b/apps/emqx_auth_redis/src/emqx_authz_redis.erl @@ -35,7 +35,7 @@ -compile(nowarn_export_all). -endif. --define(PLACEHOLDERS, [ +-define(ALLOWED_VARS, [ ?VAR_CERT_CN_NAME, ?VAR_CERT_SUBJECT, ?VAR_PEERHOST, @@ -133,7 +133,7 @@ parse_cmd(Query) -> case emqx_redis_command:split(Query) of {ok, Cmd} -> ok = validate_cmd(Cmd), - emqx_authz_utils:parse_deep(Cmd, ?PLACEHOLDERS); + emqx_authz_utils:parse_deep(Cmd, ?ALLOWED_VARS); {error, Reason} -> error({invalid_redis_cmd, Reason, Query}) end. From dfb7faf6a8a8a011c07f429b595c55011b7c3bb2 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Wed, 3 May 2023 18:53:18 +0300 Subject: [PATCH 08/21] fix(tpl): ensure rough backward compat in `emqx_prometheus` But also emit warning when job name template renders with errors. --- apps/emqx_prometheus/src/emqx_prometheus.app.src | 2 +- apps/emqx_prometheus/src/emqx_prometheus.erl | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/apps/emqx_prometheus/src/emqx_prometheus.app.src b/apps/emqx_prometheus/src/emqx_prometheus.app.src index c4abbec27..4631fec8b 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.app.src +++ b/apps/emqx_prometheus/src/emqx_prometheus.app.src @@ -2,7 +2,7 @@ {application, emqx_prometheus, [ {description, "Prometheus for EMQX"}, % strict semver, bump manually! - {vsn, "5.0.16"}, + {vsn, "5.0.17"}, {modules, []}, {registered, [emqx_prometheus_sup]}, {applications, [kernel, stdlib, prometheus, emqx, emqx_management]}, diff --git a/apps/emqx_prometheus/src/emqx_prometheus.erl b/apps/emqx_prometheus/src/emqx_prometheus.erl index fa9a39cc6..41dec9ee9 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus.erl @@ -113,10 +113,18 @@ handle_info(_Msg, State) -> push_to_push_gateway(Uri, Headers, JobName) when is_list(Headers) -> [Name, Ip] = string:tokens(atom_to_list(node()), "@"), - JobName1 = emqx_connector_template:render_strict( + % NOTE: allowing errors here to keep rough backward compatibility + {JobName1, Errors} = emqx_connector_template:render( emqx_connector_template:parse(JobName), #{<<"name">> => Name, <<"host">> => Ip} ), + _ = + Errors == [] orelse + ?SLOG(warning, #{ + msg => "prometheus_job_name_template_invalid", + errors => Errors, + template => JobName + }), Data = prometheus_text_format:format(), Url = lists:concat([Uri, "/metrics/job/", unicode:characters_to_list(JobName1)]), case httpc:request(post, {Url, Headers, "text/plain", Data}, ?HTTP_OPTIONS, []) of From 7bb995f0c6a329b4c3375f821e5de272d34e1507 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 28 Apr 2023 12:03:55 +0300 Subject: [PATCH 09/21] feat(tpl): support `:n` SQL parameters --- apps/emqx_connector/src/emqx_connector_template_sql.erl | 6 ++++-- apps/emqx_connector/test/emqx_connector_template_SUITE.erl | 7 +++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/apps/emqx_connector/src/emqx_connector_template_sql.erl b/apps/emqx_connector/src/emqx_connector_template_sql.erl index e95ecde42..90d79415e 100644 --- a/apps/emqx_connector/src/emqx_connector_template_sql.erl +++ b/apps/emqx_connector/src/emqx_connector_template_sql.erl @@ -34,7 +34,7 @@ -type values() :: [emqx_connector_sql:value()]. -type parse_opts() :: #{ - parameters => '$n' | '?', + parameters => '$n' | ':n' | '?', % Inherited from `emqx_connector_template:parse_opts()` strip_double_quote => boolean() }. @@ -116,7 +116,9 @@ mk_prepared_statement(Template, Opts) -> mk_replace('?', Acc) -> {"?", Acc}; mk_replace('$n', N) -> - {"$" ++ integer_to_list(N), N + 1}. + {"$" ++ integer_to_list(N), N + 1}; +mk_replace(':n', N) -> + {":" ++ integer_to_list(N), N + 1}. %% @doc Render a row template into a list of SQL values. %% An _SQL value_ is a vaguely defined concept here, it is something that's considered diff --git a/apps/emqx_connector/test/emqx_connector_template_SUITE.erl b/apps/emqx_connector/test/emqx_connector_template_SUITE.erl index 998baae37..562e0e550 100644 --- a/apps/emqx_connector/test/emqx_connector_template_SUITE.erl +++ b/apps/emqx_connector/test/emqx_connector_template_SUITE.erl @@ -188,6 +188,13 @@ t_parse_sql_prepstmt_n(_) -> emqx_connector_template_sql:render_prepstmt_strict(RowTemplate, Bindings) ). +t_parse_sql_prepstmt_colon(_) -> + {PrepareStatement, _RowTemplate} = + emqx_connector_template_sql:parse_prepstmt(<<"a=${a},b=${b},c=${c},d=${d}">>, #{ + parameters => ':n' + }), + ?assertEqual(<<"a=:1,b=:2,c=:3,d=:4">>, bin(PrepareStatement)). + t_parse_sql_prepstmt_partial_ph(_) -> Bindings = #{a => <<"1">>, b => 1, c => 1.0, d => #{d1 => <<"hi">>}}, {PrepareStatement, RowTemplate} = From a9693eada7e3832480e1b32f9e37c93d2b1539bc Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 4 May 2023 16:47:36 +0300 Subject: [PATCH 10/21] fix(tpl): rename `trivial` -> `is_const` This is clearer. Former naming was a bit misleading. --- apps/emqx_auth/src/emqx_authz/emqx_authz_rule.erl | 2 +- apps/emqx_connector/src/emqx_connector_template.erl | 9 +++++---- .../test/emqx_connector_template_SUITE.erl | 8 ++++---- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/apps/emqx_auth/src/emqx_authz/emqx_authz_rule.erl b/apps/emqx_auth/src/emqx_authz/emqx_authz_rule.erl index 1e7a49855..160f9cac4 100644 --- a/apps/emqx_auth/src/emqx_authz/emqx_authz_rule.erl +++ b/apps/emqx_auth/src/emqx_authz/emqx_authz_rule.erl @@ -184,7 +184,7 @@ compile_topic({eq, Topic}) -> {eq, emqx_topic:words(bin(Topic))}; compile_topic(Topic) -> Template = emqx_authz_utils:parse_str(Topic, [?VAR_USERNAME, ?VAR_CLIENTID]), - case emqx_connector_template:trivial(Template) of + case emqx_connector_template:is_const(Template) of true -> emqx_topic:words(bin(Topic)); false -> {pattern, Template} end. diff --git a/apps/emqx_connector/src/emqx_connector_template.erl b/apps/emqx_connector/src/emqx_connector_template.erl index 221cc5e86..e966f4244 100644 --- a/apps/emqx_connector/src/emqx_connector_template.erl +++ b/apps/emqx_connector/src/emqx_connector_template.erl @@ -23,7 +23,7 @@ -export([parse_deep/1]). -export([parse_deep/2]). -export([validate/2]). --export([trivial/1]). +-export([is_const/1]). -export([unparse/1]). -export([render/2]). -export([render/3]). @@ -124,7 +124,6 @@ parse_accessor(Var) -> [<<>>] -> ?PH_VAR_THIS; Name -> - % TODO: lowercase? Name end. @@ -140,9 +139,11 @@ validate(Allowed, Template) -> {error, [{Var, disallowed} || Var <- Disallowed]} end. --spec trivial(t()) -> +%% @doc Check if a template is constant with respect to rendering, i.e. does not +%% contain any placeholders. +-spec is_const(t()) -> boolean(). -trivial(Template) -> +is_const(Template) -> validate([], Template) == ok. -spec unparse(t()) -> diff --git a/apps/emqx_connector/test/emqx_connector_template_SUITE.erl b/apps/emqx_connector/test/emqx_connector_template_SUITE.erl index 562e0e550..41f637226 100644 --- a/apps/emqx_connector/test/emqx_connector_template_SUITE.erl +++ b/apps/emqx_connector/test/emqx_connector_template_SUITE.erl @@ -122,18 +122,18 @@ t_unparse(_) -> unicode:characters_to_binary(emqx_connector_template:unparse(Template)) ). -t_trivial(_) -> +t_const(_) -> ?assertEqual( true, - emqx_connector_template:trivial(emqx_connector_template:parse(<<"">>)) + emqx_connector_template:is_const(emqx_connector_template:parse(<<"">>)) ), ?assertEqual( false, - emqx_connector_template:trivial(emqx_connector_template:parse(<<"a:${a},b:${b},c:$${c}">>)) + emqx_connector_template:is_const(emqx_connector_template:parse(<<"a:${a},b:${b},c:$${c}">>)) ), ?assertEqual( true, - emqx_connector_template:trivial( + emqx_connector_template:is_const( emqx_connector_template:parse(<<"a:$${a},b:$${b},c:$${c}">>) ) ). From 1fcdfe991ca0e0403f94c740347ac00bb4b8f877 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 4 May 2023 17:00:27 +0300 Subject: [PATCH 11/21] fix(tpl): add few missing `@doc`s + correct typespecs --- .../src/emqx_connector_template.erl | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/apps/emqx_connector/src/emqx_connector_template.erl b/apps/emqx_connector/src/emqx_connector_template.erl index e966f4244..6f9688347 100644 --- a/apps/emqx_connector/src/emqx_connector_template.erl +++ b/apps/emqx_connector/src/emqx_connector_template.erl @@ -127,6 +127,9 @@ parse_accessor(Var) -> Name end. +%% @doc Validate a template against a set of allowed variables. +%% If the given template contains any variable not in the allowed set, an error +%% is returned. -spec validate([varname()], t()) -> ok | {error, [_Error :: {varname(), disallowed}]}. validate(Allowed, Template) -> @@ -146,8 +149,9 @@ validate(Allowed, Template) -> is_const(Template) -> validate([], Template) == ok. +%% @doc Restore original term from a parsed template. -spec unparse(t()) -> - unicode:chardata(). + term(). unparse({'$tpl', Template}) -> unparse_deep(Template); unparse(Template) -> @@ -208,17 +212,20 @@ render_value(Name, Value, #{var_trans := TransFun}) when is_function(TransFun, 2 render_value(_Name, Value, #{}) -> to_string(Value). +%% @doc Render a template with given bindings. +%% Behaves like `render/2`, but raises an error exception if one or more placeholders +%% are not found in the bindings. -spec render_strict(t(), bindings()) -> - unicode:chardata(). + term(). render_strict(Template, Bindings) -> render_strict(Template, Bindings, #{}). -spec render_strict(t(), bindings(), render_opts()) -> - unicode:chardata(). + term(). render_strict(Template, Bindings, Opts) -> case render(Template, Bindings, Opts) of - {String, []} -> - String; + {Render, []} -> + Render; {_, Errors = [_ | _]} -> error(Errors, [unparse(Template), Bindings]) end. From f689d6c233575a1755efbd80a7d6abc1cff5d296 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 4 May 2023 19:08:03 +0300 Subject: [PATCH 12/21] fix(tpl): ensure backward compat in `emqx_rule_engine` Missing bindings in string templates will be rendered as "undefined", as before. Rendering still assumes that missing binding with implicit default (`undefined`) is an error. This will also restore complete backward compat in `emqx_prometheus`. --- .../src/emqx_connector_template.erl | 2 -- .../test/emqx_connector_template_SUITE.erl | 2 +- .../src/emqx_rule_actions.erl | 9 ++++--- .../test/emqx_rule_engine_SUITE.erl | 27 +++++++++++++++++++ 4 files changed, 33 insertions(+), 7 deletions(-) diff --git a/apps/emqx_connector/src/emqx_connector_template.erl b/apps/emqx_connector/src/emqx_connector_template.erl index 6f9688347..72062fc2c 100644 --- a/apps/emqx_connector/src/emqx_connector_template.erl +++ b/apps/emqx_connector/src/emqx_connector_template.erl @@ -340,8 +340,6 @@ lookup(Prop, Bindings) when is_binary(Prop) -> -spec to_string(binding()) -> unicode:chardata(). -to_string(undefined) -> - []; to_string(Bin) when is_binary(Bin) -> Bin; to_string(Num) when is_integer(Num) -> integer_to_binary(Num); to_string(Num) when is_float(Num) -> float_to_binary(Num, [{decimals, 10}, compact]); diff --git a/apps/emqx_connector/test/emqx_connector_template_SUITE.erl b/apps/emqx_connector/test/emqx_connector_template_SUITE.erl index 41f637226..b6784ea54 100644 --- a/apps/emqx_connector/test/emqx_connector_template_SUITE.erl +++ b/apps/emqx_connector/test/emqx_connector_template_SUITE.erl @@ -94,7 +94,7 @@ t_render_missing_bindings(_) -> <<"a:${a},b:${b},c:${c},d:${d.d1},e:${no.such_atom_i_swear}">> ), ?assertEqual( - {<<"a:,b:,c:,d:,e:">>, [ + {<<"a:undefined,b:undefined,c:undefined,d:undefined,e:undefined">>, [ {"no.such_atom_i_swear", undefined}, {"d.d1", undefined}, {"c", undefined}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_actions.erl b/apps/emqx_rule_engine/src/emqx_rule_actions.erl index fa677ce78..7473572c8 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_actions.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_actions.erl @@ -118,10 +118,11 @@ republish( } } ) -> - Topic = unicode:characters_to_binary( - emqx_connector_template:render_strict(TopicTemplate, Selected) - ), - Payload = emqx_connector_template:render_strict(PayloadTemplate, Selected), + % NOTE: rendering missing bindings as string "undefined" + {TopicString, _Errors1} = emqx_connector_template:render(TopicTemplate, Selected), + {PayloadString, _Errors2} = emqx_connector_template:render(PayloadTemplate, Selected), + Topic = iolist_to_binary(TopicString), + Payload = iolist_to_binary(PayloadString), QoS = render_simple_var(QoSTemplate, Selected, 0), Retain = render_simple_var(RetainTemplate, Selected, false), %% 'flags' is set for message re-publishes or message related diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index fcb04f9b3..41fec48ee 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -81,6 +81,7 @@ groups() -> t_sqlselect_3, t_sqlselect_message_publish_event_keep_original_props_1, t_sqlselect_message_publish_event_keep_original_props_2, + t_sqlselect_missing_template_vars_render_as_undefined, t_sqlparse_event_1, t_sqlparse_event_2, t_sqlparse_event_3, @@ -1946,6 +1947,32 @@ t_sqlselect_as_put(_Config) -> PayloadMap2 ). +t_sqlselect_missing_template_vars_render_as_undefined(_Config) -> + SQL = <<"SELECT * FROM \"$events/client_connected\"">>, + Repub = republish_action(<<"t2">>, <<"${clientid}:${missing.var}">>), + {ok, TopicRule} = emqx_rule_engine:create_rule( + #{ + sql => SQL, + id => ?TMP_RULEID, + actions => [Repub] + } + ), + {ok, Client1} = emqtt:start_link([{clientid, <<"sub-01">>}]), + {ok, _} = emqtt:connect(Client1), + {ok, _, _} = emqtt:subscribe(Client1, <<"t2">>), + {ok, Client2} = emqtt:start_link([{clientid, <<"pub-02">>}]), + {ok, _} = emqtt:connect(Client2), + emqtt:publish(Client2, <<"foo/bar/1">>, <<>>), + receive + {publish, Msg} -> + ?assertMatch(#{topic := <<"t2">>, payload := <<"pub-02:undefined">>}, Msg) + after 2000 -> + ct:fail(wait_for_t2) + end, + emqtt:stop(Client2), + emqtt:stop(Client1), + delete_rule(TopicRule). + t_sqlparse_event_1(_Config) -> Sql = "select topic as tp " From 343b679741949db12f462145e983d1c089eb13c5 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 5 May 2023 10:24:47 +0300 Subject: [PATCH 13/21] feat(tpl): make escaping mechanism more foolproof Treat "${$}" as literal "$". This allows to template express strings, for example, of the form "${some_var_value}" where `some_var_value` is interpolated from bindings. --- .../src/emqx_authn/emqx_authn_utils.erl | 4 ++-- .../src/emqx_authz/emqx_authz_utils.erl | 4 ++-- .../src/emqx_connector_template.erl | 24 +++++++++---------- .../test/emqx_connector_template_SUITE.erl | 22 +++++++++-------- 4 files changed, 28 insertions(+), 26 deletions(-) diff --git a/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl b/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl index d9b20a47c..0a938eafb 100644 --- a/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl +++ b/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl @@ -133,7 +133,7 @@ handle_disallowed_placeholders(Template, Source) -> allowed => #{placeholders => ?ALLOWED_VARS}, notice => "Disallowed placeholders will be rendered as is." - " However, consider using `$${...}` escaping for literal `${...}` where" + " However, consider using `${$}` escaping for literal `$` where" " needed to avoid unexpected results." }), Result = prerender_disallowed_placeholders(Template), @@ -153,7 +153,7 @@ prerender_disallowed_placeholders(Template) -> % parse as a literal string. case lists:member(Name, ?ALLOWED_VARS) of true -> "${" ++ Name ++ "}"; - false -> "$${" ++ Name ++ "}" + false -> "${$}{" ++ Name ++ "}" end end }), diff --git a/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl b/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl index bd7b353a5..444955504 100644 --- a/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl +++ b/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl @@ -136,7 +136,7 @@ handle_disallowed_placeholders(Template, Source, Allowed) -> allowed => #{placeholders => Allowed}, notice => "Disallowed placeholders will be rendered as is." - " However, consider using `$${...}` escaping for literal `${...}` where" + " However, consider using `${$}` escaping for literal `$` where" " needed to avoid unexpected results." }), Result = prerender_disallowed_placeholders(Template, Allowed), @@ -156,7 +156,7 @@ prerender_disallowed_placeholders(Template, Allowed) -> % parse as a literal string. case lists:member(Name, Allowed) of true -> "${" ++ Name ++ "}"; - false -> "$${" ++ Name ++ "}" + false -> "${$}{" ++ Name ++ "}" end end }), diff --git a/apps/emqx_connector/src/emqx_connector_template.erl b/apps/emqx_connector/src/emqx_connector_template.erl index 72062fc2c..619dbd6ec 100644 --- a/apps/emqx_connector/src/emqx_connector_template.erl +++ b/apps/emqx_connector/src/emqx_connector_template.erl @@ -42,7 +42,7 @@ -type t() :: str() | {'$tpl', deeptpl()}. --type str() :: [unicode:chardata() | placeholder()]. +-type str() :: [iodata() | byte() | placeholder()]. -type deep() :: {'$tpl', deeptpl()}. -type deeptpl() :: @@ -76,7 +76,8 @@ var_trans => var_trans() }. --define(RE_PLACEHOLDER, "\\$(\\$?)\\{[.]?([a-zA-Z0-9._]*)\\}"). +-define(RE_PLACEHOLDER, "\\$\\{[.]?([a-zA-Z0-9._]*)\\}"). +-define(RE_ESCAPE, "\\$\\{(\\$)\\}"). %% @doc Parse a unicode string into a template. %% String might contain zero or more of placeholders in the form of `${var}`, @@ -95,22 +96,21 @@ parse(String, Opts) -> RE = case Opts of #{strip_double_quote := true} -> - <<"((?|" ?RE_PLACEHOLDER "|\"" ?RE_PLACEHOLDER "\"))">>; + <<"((?|" ?RE_PLACEHOLDER "|\"" ?RE_PLACEHOLDER "\")|" ?RE_ESCAPE ")">>; #{} -> - <<"(" ?RE_PLACEHOLDER ")">> + <<"(" ?RE_PLACEHOLDER "|" ?RE_ESCAPE ")">> end, Splits = re:split(String, RE, [{return, binary}, group, trim, unicode]), Components = lists:flatmap(fun parse_split/1, Splits), Components. -parse_split([Part, _PH, <<>>, Var]) -> +parse_split([Part, _PH, Var, <<>>]) -> % Regular placeholder prepend(Part, [{var, unicode:characters_to_list(Var), parse_accessor(Var)}]); -parse_split([Part, _PH = <>, <<"$">>, _]) -> - % Escaped literal, take all but the second byte, which is always `$`. - % Important to make a whole token starting with `$` so the `unparse/11` - % function can distinguish escaped literals. - prepend(Part, [<>]); +parse_split([Part, _Escape, <<>>, <<"$">>]) -> + % Escaped literal `$`. + % Use single char as token so the `unparse/1` function can distinguish escaped `$`. + prepend(Part, [$$]); parse_split([Tail]) -> [Tail]. @@ -159,8 +159,8 @@ unparse(Template) -> unparse_part({var, Name, _Accessor}) -> render_placeholder(Name); -unparse_part(Part = <<"${", _/binary>>) -> - <<"$", Part/binary>>; +unparse_part($$) -> + <<"${$}">>; unparse_part(Part) -> Part. diff --git a/apps/emqx_connector/test/emqx_connector_template_SUITE.erl b/apps/emqx_connector/test/emqx_connector_template_SUITE.erl index b6784ea54..3700caa96 100644 --- a/apps/emqx_connector/test/emqx_connector_template_SUITE.erl +++ b/apps/emqx_connector/test/emqx_connector_template_SUITE.erl @@ -115,7 +115,7 @@ t_render_missing_bindings(_) -> ). t_unparse(_) -> - TString = <<"a:${a},b:${b},c:$${c},d:{${d.d1}}">>, + TString = <<"a:${a},b:${b},c:$${c},d:{${d.d1}},e:${$}{e},lit:${$}{$}">>, Template = emqx_connector_template:parse(TString), ?assertEqual( TString, @@ -129,12 +129,14 @@ t_const(_) -> ), ?assertEqual( false, - emqx_connector_template:is_const(emqx_connector_template:parse(<<"a:${a},b:${b},c:$${c}">>)) + emqx_connector_template:is_const( + emqx_connector_template:parse(<<"a:${a},b:${b},c:${$}{c}">>) + ) ), ?assertEqual( true, emqx_connector_template:is_const( - emqx_connector_template:parse(<<"a:$${a},b:$${b},c:$${c}">>) + emqx_connector_template:parse(<<"a:${$}{a},b:${$}{b}">>) ) ). @@ -147,16 +149,16 @@ t_render_partial_ph(_) -> ). t_parse_escaped(_) -> - Bindings = #{a => <<"1">>, b => 1}, - Template = emqx_connector_template:parse(<<"a:${a},b:$${b}">>), + Bindings = #{a => <<"1">>, b => 1, c => "VAR"}, + Template = emqx_connector_template:parse(<<"a:${a},b:${$}{b},c:${$}{${c}},lit:${$}{$}">>), ?assertEqual( - <<"a:1,b:${b}">>, + <<"a:1,b:${b},c:${VAR},lit:${$}">>, render_strict_string(Template, Bindings) ). t_parse_escaped_dquote(_) -> Bindings = #{a => <<"1">>, b => 1}, - Template = emqx_connector_template:parse(<<"a:\"${a}\",b:\"$${b}\"">>, #{ + Template = emqx_connector_template:parse(<<"a:\"${a}\",b:\"${$}{b}\"">>, #{ strip_double_quote => true }), ?assertEqual( @@ -299,7 +301,7 @@ t_render_tmpl_deep(_) -> Bindings = #{a => <<"1">>, b => 1, c => 1.0, d => #{d1 => <<"hi">>}}, Template = emqx_connector_template:parse_deep( - #{<<"${a}">> => [<<"${b}">>, "c", 2, 3.0, '${d}', {[<<"${c}">>, <<"$${d}">>], 0}]} + #{<<"${a}">> => [<<"$${b}">>, "c", 2, 3.0, '${d}', {[<<"${c}">>, <<"${$}{d}">>], 0}]} ), ?assertEqual( @@ -308,12 +310,12 @@ t_render_tmpl_deep(_) -> ), ?assertEqual( - #{<<"1">> => [<<"1">>, "c", 2, 3.0, '${d}', {[<<"1.0">>, <<"${d}">>], 0}]}, + #{<<"1">> => [<<"$1">>, "c", 2, 3.0, '${d}', {[<<"1.0">>, <<"${d}">>], 0}]}, emqx_connector_template:render_strict(Template, Bindings) ). t_unparse_tmpl_deep(_) -> - Term = #{<<"${a}">> => [<<"$${b}">>, "c", 2, 3.0, '${d}', {[<<"${c}">>], 0}]}, + Term = #{<<"${a}">> => [<<"$${b}">>, "c", 2, 3.0, '${d}', {[<<"${c}">>], <<"${$}{d}">>, 0}]}, Template = emqx_connector_template:parse_deep(Term), ?assertEqual(Term, emqx_connector_template:unparse(Template)). From 8e4585d64fab2c72b62e77351f1a0d074f580d30 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 14 Jul 2023 18:40:11 +0200 Subject: [PATCH 14/21] chore: move template modules to `emqx_utils` Even though most of the time these modules will be used by connectors, there are exceptions (namely, `emqx_rule_engine`). Besides, they are general enough to land there, more so given that `emqx_placeholder` is already there. --- .../src/emqx_authn/emqx_authn_utils.erl | 26 +-- .../src/emqx_authz/emqx_authz_rule.erl | 4 +- .../src/emqx_authz/emqx_authz_utils.erl | 26 +-- .../test/emqx_authz/emqx_authz_rule_SUITE.erl | 4 +- .../src/emqx_bridge_http_connector.erl | 4 +- .../test/emqx_bridge_http_connector_tests.erl | 2 +- .../emqx_connector/src/emqx_connector_sql.erl | 159 ------------------ apps/emqx_mysql/src/emqx_mysql.erl | 14 +- apps/emqx_postgresql/src/emqx_postgresql.erl | 6 +- apps/emqx_prometheus/src/emqx_prometheus.erl | 4 +- .../src/emqx_rule_actions.erl | 20 +-- .../src/emqx_template.erl} | 6 +- .../src/emqx_template_sql.erl} | 36 ++-- apps/emqx_utils/src/emqx_utils_sql.erl | 10 +- .../test/emqx_template_SUITE.erl} | 96 +++++------ 15 files changed, 130 insertions(+), 287 deletions(-) delete mode 100644 apps/emqx_connector/src/emqx_connector_sql.erl rename apps/{emqx_connector/src/emqx_connector_template.erl => emqx_utils/src/emqx_template.erl} (99%) rename apps/{emqx_connector/src/emqx_connector_template_sql.erl => emqx_utils/src/emqx_template_sql.erl} (77%) rename apps/{emqx_connector/test/emqx_connector_template_SUITE.erl => emqx_utils/test/emqx_template_SUITE.erl} (69%) diff --git a/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl b/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl index 0a938eafb..f782e0e6c 100644 --- a/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl +++ b/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl @@ -108,22 +108,22 @@ check_password_from_selected_map(Algorithm, Selected, Password) -> end. parse_deep(Template) -> - Result = emqx_connector_template:parse_deep(Template), + Result = emqx_template:parse_deep(Template), handle_disallowed_placeholders(Result, {deep, Template}). parse_str(Template) -> - Result = emqx_connector_template:parse(Template), + Result = emqx_template:parse(Template), handle_disallowed_placeholders(Result, {string, Template}). parse_sql(Template, ReplaceWith) -> - {Statement, Result} = emqx_connector_template_sql:parse_prepstmt( + {Statement, Result} = emqx_template_sql:parse_prepstmt( Template, #{parameters => ReplaceWith, strip_double_quote => true} ), {Statement, handle_disallowed_placeholders(Result, {string, Template})}. handle_disallowed_placeholders(Template, Source) -> - case emqx_connector_template:validate(?ALLOWED_VARS, Template) of + case emqx_template:validate(?ALLOWED_VARS, Template) of ok -> Template; {error, Disallowed} -> @@ -139,14 +139,14 @@ handle_disallowed_placeholders(Template, Source) -> Result = prerender_disallowed_placeholders(Template), case Source of {string, _} -> - emqx_connector_template:parse(Result); + emqx_template:parse(Result); {deep, _} -> - emqx_connector_template:parse_deep(Result) + emqx_template:parse_deep(Result) end end. prerender_disallowed_placeholders(Template) -> - {Result, _} = emqx_connector_template:render(Template, #{}, #{ + {Result, _} = emqx_template:render(Template, #{}, #{ var_trans => fun(Name, _) -> % NOTE % Rendering disallowed placeholders in escaped form, which will then @@ -162,7 +162,7 @@ prerender_disallowed_placeholders(Template) -> render_deep(Template, Credential) -> % NOTE % Ignoring errors here, undefined bindings will be replaced with empty string. - {Term, _Errors} = emqx_connector_template:render( + {Term, _Errors} = emqx_template:render( Template, mapping_credential(Credential), #{var_trans => fun to_string/2} @@ -172,7 +172,7 @@ render_deep(Template, Credential) -> render_str(Template, Credential) -> % NOTE % Ignoring errors here, undefined bindings will be replaced with empty string. - {String, _Errors} = emqx_connector_template:render( + {String, _Errors} = emqx_template:render( Template, mapping_credential(Credential), #{var_trans => fun to_string/2} @@ -182,7 +182,7 @@ render_str(Template, Credential) -> render_urlencoded_str(Template, Credential) -> % NOTE % Ignoring errors here, undefined bindings will be replaced with empty string. - {String, _Errors} = emqx_connector_template:render( + {String, _Errors} = emqx_template:render( Template, mapping_credential(Credential), #{var_trans => fun to_urlencoded_string/2} @@ -192,7 +192,7 @@ render_urlencoded_str(Template, Credential) -> render_sql_params(ParamList, Credential) -> % NOTE % Ignoring errors here, undefined bindings will be replaced with empty string. - {Row, _Errors} = emqx_connector_template:render( + {Row, _Errors} = emqx_template:render( ParamList, mapping_credential(Credential), #{var_trans => fun to_sql_valaue/2} @@ -322,10 +322,10 @@ to_urlencoded_string(Name, Value) -> emqx_http_lib:uri_encode(to_string(Name, Value)). to_string(Name, Value) -> - emqx_connector_template:to_string(render_var(Name, Value)). + emqx_template:to_string(render_var(Name, Value)). to_sql_valaue(Name, Value) -> - emqx_connector_sql:to_sql_value(render_var(Name, Value)). + emqx_utils_sql:to_sql_value(render_var(Name, Value)). render_var(_, undefined) -> % NOTE diff --git a/apps/emqx_auth/src/emqx_authz/emqx_authz_rule.erl b/apps/emqx_auth/src/emqx_authz/emqx_authz_rule.erl index 160f9cac4..ad6dec56b 100644 --- a/apps/emqx_auth/src/emqx_authz/emqx_authz_rule.erl +++ b/apps/emqx_auth/src/emqx_authz/emqx_authz_rule.erl @@ -184,7 +184,7 @@ compile_topic({eq, Topic}) -> {eq, emqx_topic:words(bin(Topic))}; compile_topic(Topic) -> Template = emqx_authz_utils:parse_str(Topic, [?VAR_USERNAME, ?VAR_CLIENTID]), - case emqx_connector_template:is_const(Template) of + case emqx_template:is_const(Template) of true -> emqx_topic:words(bin(Topic)); false -> {pattern, Template} end. @@ -302,7 +302,7 @@ match_who(_, _) -> match_topics(_ClientInfo, _Topic, []) -> false; match_topics(ClientInfo, Topic, [{pattern, PatternFilter} | Filters]) -> - TopicFilter = bin(emqx_connector_template:render_strict(PatternFilter, ClientInfo)), + TopicFilter = bin(emqx_template:render_strict(PatternFilter, ClientInfo)), match_topic(emqx_topic:words(Topic), emqx_topic:words(TopicFilter)) orelse match_topics(ClientInfo, Topic, Filters); match_topics(ClientInfo, Topic, [TopicFilter | Filters]) -> diff --git a/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl b/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl index 444955504..a17a563ae 100644 --- a/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl +++ b/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl @@ -110,15 +110,15 @@ update_config(Path, ConfigRequest) -> }). parse_deep(Template, PlaceHolders) -> - Result = emqx_connector_template:parse_deep(Template), + Result = emqx_template:parse_deep(Template), handle_disallowed_placeholders(Result, {deep, Template}, PlaceHolders). parse_str(Template, PlaceHolders) -> - Result = emqx_connector_template:parse(Template), + Result = emqx_template:parse(Template), handle_disallowed_placeholders(Result, {string, Template}, PlaceHolders). parse_sql(Template, ReplaceWith, PlaceHolders) -> - {Statement, Result} = emqx_connector_template_sql:parse_prepstmt( + {Statement, Result} = emqx_template_sql:parse_prepstmt( Template, #{parameters => ReplaceWith, strip_double_quote => true} ), @@ -126,7 +126,7 @@ parse_sql(Template, ReplaceWith, PlaceHolders) -> {Statement, FResult}. handle_disallowed_placeholders(Template, Source, Allowed) -> - case emqx_connector_template:validate(Allowed, Template) of + case emqx_template:validate(Allowed, Template) of ok -> Template; {error, Disallowed} -> @@ -142,14 +142,14 @@ handle_disallowed_placeholders(Template, Source, Allowed) -> Result = prerender_disallowed_placeholders(Template, Allowed), case Source of {string, _} -> - emqx_connector_template:parse(Result); + emqx_template:parse(Result); {deep, _} -> - emqx_connector_template:parse_deep(Result) + emqx_template:parse_deep(Result) end end. prerender_disallowed_placeholders(Template, Allowed) -> - {Result, _} = emqx_connector_template:render(Template, #{}, #{ + {Result, _} = emqx_template:render(Template, #{}, #{ var_trans => fun(Name, _) -> % NOTE % Rendering disallowed placeholders in escaped form, which will then @@ -165,7 +165,7 @@ prerender_disallowed_placeholders(Template, Allowed) -> render_deep(Template, Values) -> % NOTE % Ignoring errors here, undefined bindings will be replaced with empty string. - {Term, _Errors} = emqx_connector_template:render( + {Term, _Errors} = emqx_template:render( Template, client_vars(Values), #{var_trans => fun to_string/2} @@ -175,7 +175,7 @@ render_deep(Template, Values) -> render_str(Template, Values) -> % NOTE % Ignoring errors here, undefined bindings will be replaced with empty string. - {String, _Errors} = emqx_connector_template:render( + {String, _Errors} = emqx_template:render( Template, client_vars(Values), #{var_trans => fun to_string/2} @@ -185,7 +185,7 @@ render_str(Template, Values) -> render_urlencoded_str(Template, Values) -> % NOTE % Ignoring errors here, undefined bindings will be replaced with empty string. - {String, _Errors} = emqx_connector_template:render( + {String, _Errors} = emqx_template:render( Template, client_vars(Values), #{var_trans => fun to_urlencoded_string/2} @@ -195,7 +195,7 @@ render_urlencoded_str(Template, Values) -> render_sql_params(ParamList, Values) -> % NOTE % Ignoring errors here, undefined bindings will be replaced with empty string. - {Row, _Errors} = emqx_connector_template:render( + {Row, _Errors} = emqx_template:render( ParamList, client_vars(Values), #{var_trans => fun to_sql_value/2} @@ -270,10 +270,10 @@ to_urlencoded_string(Name, Value) -> emqx_http_lib:uri_encode(to_string(Name, Value)). to_string(Name, Value) -> - emqx_connector_template:to_string(render_var(Name, Value)). + emqx_template:to_string(render_var(Name, Value)). to_sql_value(Name, Value) -> - emqx_connector_sql:to_sql_value(render_var(Name, Value)). + emqx_utils_sql:to_sql_value(render_var(Name, Value)). render_var(_, undefined) -> % NOTE diff --git a/apps/emqx_auth/test/emqx_authz/emqx_authz_rule_SUITE.erl b/apps/emqx_auth/test/emqx_authz/emqx_authz_rule_SUITE.erl index 5031daff6..d81a93038 100644 --- a/apps/emqx_auth/test/emqx_authz/emqx_authz_rule_SUITE.erl +++ b/apps/emqx_auth/test/emqx_authz/emqx_authz_rule_SUITE.erl @@ -69,8 +69,8 @@ set_special_configs(_App) -> t_compile(_) -> % NOTE % Some of the following testcase are relying on the internal representation of - % `emqx_connector_template:t()`. If the internal representation is changed, these - % testcases may fail. + % `emqx_template:t()`. If the internal representation is changed, these testcases + % may fail. ?assertEqual({deny, all, all, [['#']]}, emqx_authz_rule:compile({deny, all})), ?assertEqual( diff --git a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl index 869f081fb..88f55af52 100644 --- a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl +++ b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl @@ -535,7 +535,7 @@ maybe_parse_template(Key, Conf) -> end. parse_template(String) -> - emqx_connector_template:parse(String). + emqx_template:parse(String). process_request( #{ @@ -573,7 +573,7 @@ render_headers(HeaderTks, Msg) -> render_template(Template, Msg) -> % NOTE: ignoring errors here, missing variables will be rendered as `"undefined"`. - {String, _Errors} = emqx_connector_template:render(Template, Msg), + {String, _Errors} = emqx_template:render(Template, Msg), String. render_template_string(Template, Msg) -> diff --git a/apps/emqx_bridge_http/test/emqx_bridge_http_connector_tests.erl b/apps/emqx_bridge_http/test/emqx_bridge_http_connector_tests.erl index 1de210260..4f5e2929c 100644 --- a/apps/emqx_bridge_http/test/emqx_bridge_http_connector_tests.erl +++ b/apps/emqx_bridge_http/test/emqx_bridge_http_connector_tests.erl @@ -84,7 +84,7 @@ is_wrapped(_Other) -> false. untmpl(Tpl) -> - iolist_to_binary(emqx_connector_template:render_strict(Tpl, #{})). + iolist_to_binary(emqx_template:render_strict(Tpl, #{})). is_unwrapped_headers(Headers) -> lists:all(fun is_unwrapped_header/1, Headers). diff --git a/apps/emqx_connector/src/emqx_connector_sql.erl b/apps/emqx_connector/src/emqx_connector_sql.erl deleted file mode 100644 index be0b220e6..000000000 --- a/apps/emqx_connector/src/emqx_connector_sql.erl +++ /dev/null @@ -1,159 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_connector_sql). - --export([get_statement_type/1]). --export([parse_insert/1]). - --export([to_sql_value/1]). --export([to_sql_string/2]). - --export([escape_sql/1]). --export([escape_cql/1]). --export([escape_mysql/1]). - --export_type([value/0]). - --type statement_type() :: select | insert | delete. --type value() :: null | binary() | number() | boolean() | [value()]. - --dialyzer({no_improper_lists, [escape_mysql/4, escape_prepend/4]}). - --spec get_statement_type(iodata()) -> statement_type() | {error, unknown}. -get_statement_type(Query) -> - KnownTypes = #{ - <<"select">> => select, - <<"insert">> => insert, - <<"delete">> => delete - }, - case re:run(Query, <<"^\\s*([a-zA-Z]+)">>, [{capture, all_but_first, binary}]) of - {match, [Token]} -> - maps:get(string:lowercase(Token), KnownTypes, {error, unknown}); - _ -> - {error, unknown} - end. - -%% @doc Parse an INSERT SQL statement into its INSERT part and the VALUES part. -%% SQL = <<"INSERT INTO \"abc\" (c1, c2, c3) VALUES (${a}, ${b}, ${c.prop})">> -%% {ok, {<<"INSERT INTO \"abc\" (c1, c2, c3)">>, <<"(${a}, ${b}, ${c.prop})">>}} --spec parse_insert(iodata()) -> - {ok, {_Statement :: binary(), _Rows :: binary()}} | {error, not_insert_sql}. -parse_insert(SQL) -> - case re:split(SQL, "((?i)values)", [{return, binary}]) of - [Part1, _, Part3] -> - case string:trim(Part1, leading) of - <<"insert", _/binary>> = InsertSQL -> - {ok, {InsertSQL, Part3}}; - <<"INSERT", _/binary>> = InsertSQL -> - {ok, {InsertSQL, Part3}}; - _ -> - {error, not_insert_sql} - end; - _ -> - {error, not_insert_sql} - end. - -%% @doc Convert an Erlang term to a value that can be used primarily in -%% prepared SQL statements. --spec to_sql_value(term()) -> value(). -to_sql_value(undefined) -> null; -to_sql_value(List) when is_list(List) -> List; -to_sql_value(Bin) when is_binary(Bin) -> Bin; -to_sql_value(Num) when is_number(Num) -> Num; -to_sql_value(Bool) when is_boolean(Bool) -> Bool; -to_sql_value(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8); -to_sql_value(Map) when is_map(Map) -> emqx_utils_json:encode(Map). - -%% @doc Convert an Erlang term to a string that can be interpolated in literal -%% SQL statements. The value is escaped if necessary. --spec to_sql_string(term(), Options) -> iodata() when - Options :: #{ - escaping => cql | mysql | sql - }. -to_sql_string(String, #{escaping := mysql}) when is_binary(String) -> - try - escape_mysql(String) - catch - throw:invalid_utf8 -> - [<<"0x">>, binary:encode_hex(String)] - end; -to_sql_string(Term, #{escaping := mysql}) -> - maybe_escape(Term, fun escape_mysql/1); -to_sql_string(Term, #{escaping := cql}) -> - maybe_escape(Term, fun escape_cql/1); -to_sql_string(Term, #{}) -> - maybe_escape(Term, fun escape_sql/1). - --spec maybe_escape(_Value, fun((binary()) -> iodata())) -> iodata(). -maybe_escape(undefined, _EscapeFun) -> - <<"NULL">>; -maybe_escape(Str, EscapeFun) when is_binary(Str) -> - EscapeFun(Str); -maybe_escape(Str, EscapeFun) when is_list(Str) -> - case unicode:characters_to_binary(Str) of - Bin when is_binary(Bin) -> - EscapeFun(Bin); - Otherwise -> - error(Otherwise) - end; -maybe_escape(Val, EscapeFun) when is_atom(Val) orelse is_map(Val) -> - EscapeFun(emqx_connector_template:to_string(Val)); -maybe_escape(Val, _EscapeFun) -> - emqx_connector_template:to_string(Val). - --spec escape_sql(binary()) -> iodata(). -escape_sql(S) -> - % NOTE - % This is a bit misleading: currently, escaping logic in `escape_sql/1` likely - % won't work with pgsql since it does not support C-style escapes by default. - % https://www.postgresql.org/docs/14/sql-syntax-lexical.html#SQL-SYNTAX-CONSTANTS - ES = binary:replace(S, [<<"\\">>, <<"'">>], <<"\\">>, [global, {insert_replaced, 1}]), - [$', ES, $']. - --spec escape_cql(binary()) -> iodata(). -escape_cql(S) -> - ES = binary:replace(S, <<"'">>, <<"'">>, [global, {insert_replaced, 1}]), - [$', ES, $']. - --spec escape_mysql(binary()) -> iodata(). -escape_mysql(S0) -> - % https://dev.mysql.com/doc/refman/8.0/en/string-literals.html - [$', escape_mysql(S0, 0, 0, S0), $']. - -%% NOTE -%% This thing looks more complicated than needed because it's optimized for as few -%% intermediate memory (re)allocations as possible. -escape_mysql(<<$', Rest/binary>>, I, Run, Src) -> - escape_prepend(I, Run, Src, [<<"\\'">> | escape_mysql(Rest, I + Run + 1, 0, Src)]); -escape_mysql(<<$\\, Rest/binary>>, I, Run, Src) -> - escape_prepend(I, Run, Src, [<<"\\\\">> | escape_mysql(Rest, I + Run + 1, 0, Src)]); -escape_mysql(<<0, Rest/binary>>, I, Run, Src) -> - escape_prepend(I, Run, Src, [<<"\\0">> | escape_mysql(Rest, I + Run + 1, 0, Src)]); -escape_mysql(<<_/utf8, Rest/binary>> = S, I, Run, Src) -> - CWidth = byte_size(S) - byte_size(Rest), - escape_mysql(Rest, I, Run + CWidth, Src); -escape_mysql(<<>>, 0, _, Src) -> - Src; -escape_mysql(<<>>, I, Run, Src) -> - binary:part(Src, I, Run); -escape_mysql(_, _I, _Run, _Src) -> - throw(invalid_utf8). - -escape_prepend(_RunI, 0, _Src, Tail) -> - Tail; -escape_prepend(I, Run, Src, Tail) -> - [binary:part(Src, I, Run) | Tail]. diff --git a/apps/emqx_mysql/src/emqx_mysql.erl b/apps/emqx_mysql/src/emqx_mysql.erl index 927c9d067..e052b9b89 100644 --- a/apps/emqx_mysql/src/emqx_mysql.erl +++ b/apps/emqx_mysql/src/emqx_mysql.erl @@ -46,7 +46,7 @@ default_port => ?MYSQL_DEFAULT_PORT }). --type template() :: {unicode:chardata(), emqx_connector_template:str()}. +-type template() :: {unicode:chardata(), emqx_template:str()}. -type state() :: #{ pool_name := binary(), @@ -387,16 +387,16 @@ parse_prepare_sql(Config) -> #{query_templates => Templates}. parse_prepare_sql(Key, Query, Acc) -> - Template = emqx_connector_template_sql:parse_prepstmt(Query, #{parameters => '?'}), + Template = emqx_template_sql:parse_prepstmt(Query, #{parameters => '?'}), AccNext = Acc#{{Key, prepstmt} => Template}, parse_batch_sql(Key, Query, AccNext). parse_batch_sql(Key, Query, Acc) -> - case emqx_connector_sql:get_statement_type(Query) of + case emqx_utils_sql:get_statement_type(Query) of insert -> - case emqx_connector_sql:parse_insert(Query) of + case emqx_utils_sql:parse_insert(Query) of {ok, {Insert, Params}} -> - RowTemplate = emqx_connector_template_sql:parse(Params), + RowTemplate = emqx_template_sql:parse(Params), Acc#{{Key, batch} => {Insert, RowTemplate}}; {error, Reason} -> ?SLOG(error, #{ @@ -427,7 +427,7 @@ proc_sql_params(TypeOrKey, SQLOrData, Params, #{query_templates := Templates}) - {SQLOrData, Params}; {_InsertPart, RowTemplate} -> % NOTE: ignoring errors here, missing variables are set to `null`. - {Row, _Errors} = emqx_connector_template_sql:render_prepstmt(RowTemplate, SQLOrData), + {Row, _Errors} = emqx_template_sql:render_prepstmt(RowTemplate, SQLOrData), {TypeOrKey, Row} end. @@ -438,7 +438,7 @@ on_batch_insert(InstId, BatchReqs, {InsertPart, RowTemplate}, State) -> render_row(RowTemplate, Data) -> % NOTE: ignoring errors here, missing variables are set to "NULL". - {Row, _Errors} = emqx_connector_template_sql:render(RowTemplate, Data, #{escaping => mysql}), + {Row, _Errors} = emqx_template_sql:render(RowTemplate, Data, #{escaping => mysql}), Row. on_sql_query( diff --git a/apps/emqx_postgresql/src/emqx_postgresql.erl b/apps/emqx_postgresql/src/emqx_postgresql.erl index 71ba93b9b..3f7b43c79 100644 --- a/apps/emqx_postgresql/src/emqx_postgresql.erl +++ b/apps/emqx_postgresql/src/emqx_postgresql.erl @@ -52,7 +52,7 @@ default_port => ?PGSQL_DEFAULT_PORT }). --type template() :: {unicode:chardata(), emqx_connector_template_sql:row_template()}. +-type template() :: {unicode:chardata(), emqx_template_sql:row_template()}. -type state() :: #{ pool_name := binary(), @@ -428,12 +428,12 @@ parse_prepare_sql(Config) -> #{query_templates => Templates}. parse_prepare_sql(Key, Query, Acc) -> - Template = emqx_connector_template_sql:parse_prepstmt(Query, #{parameters => '$n'}), + Template = emqx_template_sql:parse_prepstmt(Query, #{parameters => '$n'}), Acc#{Key => Template}. render_prepare_sql_row(RowTemplate, Data) -> % NOTE: ignoring errors here, missing variables will be replaced with `null`. - {Row, _Errors} = emqx_connector_template_sql:render_prepstmt(RowTemplate, Data), + {Row, _Errors} = emqx_template_sql:render_prepstmt(RowTemplate, Data), Row. init_prepare(State = #{query_templates := Templates}) when map_size(Templates) == 0 -> diff --git a/apps/emqx_prometheus/src/emqx_prometheus.erl b/apps/emqx_prometheus/src/emqx_prometheus.erl index 41dec9ee9..a242931c4 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus.erl @@ -114,8 +114,8 @@ handle_info(_Msg, State) -> push_to_push_gateway(Uri, Headers, JobName) when is_list(Headers) -> [Name, Ip] = string:tokens(atom_to_list(node()), "@"), % NOTE: allowing errors here to keep rough backward compatibility - {JobName1, Errors} = emqx_connector_template:render( - emqx_connector_template:parse(JobName), + {JobName1, Errors} = emqx_template:render( + emqx_template:parse(JobName), #{<<"name">> => Name, <<"host">> => Ip} ), _ = diff --git a/apps/emqx_rule_engine/src/emqx_rule_actions.erl b/apps/emqx_rule_engine/src/emqx_rule_actions.erl index 7473572c8..7a8b2520c 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_actions.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_actions.erl @@ -71,7 +71,7 @@ pre_process_action_args( ) -> Args#{ preprocessed_tmpl => #{ - topic => emqx_connector_template:parse(Topic), + topic => emqx_template:parse(Topic), qos => parse_vars(QoS), retain => parse_vars(Retain), payload => parse_payload(Payload), @@ -119,8 +119,8 @@ republish( } ) -> % NOTE: rendering missing bindings as string "undefined" - {TopicString, _Errors1} = emqx_connector_template:render(TopicTemplate, Selected), - {PayloadString, _Errors2} = emqx_connector_template:render(PayloadTemplate, Selected), + {TopicString, _Errors1} = emqx_template:render(TopicTemplate, Selected), + {PayloadString, _Errors2} = emqx_template:render(PayloadTemplate, Selected), Topic = iolist_to_binary(TopicString), Payload = iolist_to_binary(PayloadString), QoS = render_simple_var(QoSTemplate, Selected, 0), @@ -202,13 +202,13 @@ safe_publish(RuleId, Topic, QoS, Flags, Payload, PubProps) -> emqx_metrics:inc_msg(Msg). parse_vars(Data) when is_binary(Data) -> - emqx_connector_template:parse(Data); + emqx_template:parse(Data); parse_vars(Data) -> {const, Data}. parse_mqtt_properties(MQTTPropertiesTemplate) -> maps:map( - fun(_Key, V) -> emqx_connector_template:parse(V) end, + fun(_Key, V) -> emqx_template:parse(V) end, MQTTPropertiesTemplate ). @@ -220,13 +220,13 @@ parse_user_properties(<<"${pub_props.'User-Property'}">>) -> ?ORIGINAL_USER_PROPERTIES; parse_user_properties(<<"${", _/binary>> = V) -> %% use a variable - emqx_connector_template:parse(V); + emqx_template:parse(V); parse_user_properties(_) -> %% invalid, discard undefined. render_simple_var([{var, _Name, Accessor}], Data, Default) -> - case emqx_connector_template:lookup_var(Accessor, Data) of + case emqx_template:lookup_var(Accessor, Data) of {ok, Var} -> Var; %% cannot find the variable from Data {error, _} -> Default @@ -236,8 +236,8 @@ render_simple_var({const, Val}, _Data, _Default) -> parse_payload(Payload) -> case string:is_empty(Payload) of - false -> emqx_connector_template:parse(Payload); - true -> emqx_connector_template:parse("${.}") + false -> emqx_template:parse(Payload); + true -> emqx_template:parse("${.}") end. render_pub_props(UserPropertiesTemplate, Selected, Env) -> @@ -259,7 +259,7 @@ render_mqtt_properties(MQTTPropertiesTemplate, Selected, Env) -> fun(K, Template, Acc) -> try V = unicode:characters_to_binary( - emqx_connector_template:render_strict(Template, Selected) + emqx_template:render_strict(Template, Selected) ), Acc#{K => V} catch diff --git a/apps/emqx_connector/src/emqx_connector_template.erl b/apps/emqx_utils/src/emqx_template.erl similarity index 99% rename from apps/emqx_connector/src/emqx_connector_template.erl rename to apps/emqx_utils/src/emqx_template.erl index 619dbd6ec..deb25d807 100644 --- a/apps/emqx_connector/src/emqx_connector_template.erl +++ b/apps/emqx_utils/src/emqx_template.erl @@ -14,9 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_connector_template). - --include_lib("emqx/include/emqx_placeholder.hrl"). +-module(emqx_template). -export([parse/1]). -export([parse/2]). @@ -76,6 +74,8 @@ var_trans => var_trans() }. +-define(PH_VAR_THIS, '$this'). + -define(RE_PLACEHOLDER, "\\$\\{[.]?([a-zA-Z0-9._]*)\\}"). -define(RE_ESCAPE, "\\$\\{(\\$)\\}"). diff --git a/apps/emqx_connector/src/emqx_connector_template_sql.erl b/apps/emqx_utils/src/emqx_template_sql.erl similarity index 77% rename from apps/emqx_connector/src/emqx_connector_template_sql.erl rename to apps/emqx_utils/src/emqx_template_sql.erl index 90d79415e..f215cd868 100644 --- a/apps/emqx_connector/src/emqx_connector_template_sql.erl +++ b/apps/emqx_utils/src/emqx_template_sql.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_connector_template_sql). +-module(emqx_template_sql). -export([parse/1]). -export([parse/2]). @@ -27,15 +27,15 @@ -export_type([row_template/0]). --type template() :: emqx_connector_template:t(). --type row_template() :: [emqx_connector_template:placeholder()]. --type bindings() :: emqx_connector_template:bindings(). +-type template() :: emqx_template:t(). +-type row_template() :: [emqx_template:placeholder()]. +-type bindings() :: emqx_template:bindings(). --type values() :: [emqx_connector_sql:value()]. +-type values() :: [emqx_utils_sql:value()]. -type parse_opts() :: #{ parameters => '$n' | ':n' | '?', - % Inherited from `emqx_connector_template:parse_opts()` + % Inherited from `emqx_template:parse_opts()` strip_double_quote => boolean() }. @@ -57,7 +57,7 @@ parse(String) -> -spec parse(unicode:chardata(), parse_opts()) -> template(). parse(String, Opts) -> - emqx_connector_template:parse(String, Opts). + emqx_template:parse(String, Opts). %% @doc Render an SQL statement template given a set of bindings. %% Interpolation generally follows the SQL syntax, strings are escaped according to the @@ -65,8 +65,8 @@ parse(String, Opts) -> -spec render(template(), bindings(), render_opts()) -> {unicode:chardata(), [_Error]}. render(Template, Bindings, Opts) -> - emqx_connector_template:render(Template, Bindings, #{ - var_trans => fun(Value) -> emqx_connector_sql:to_sql_string(Value, Opts) end + emqx_template:render(Template, Bindings, #{ + var_trans => fun(Value) -> emqx_utils_sql:to_sql_string(Value, Opts) end }). %% @doc Render an SQL statement template given a set of bindings. @@ -74,8 +74,8 @@ render(Template, Bindings, Opts) -> -spec render_strict(template(), bindings(), render_opts()) -> unicode:chardata(). render_strict(Template, Bindings, Opts) -> - emqx_connector_template:render_strict(Template, Bindings, #{ - var_trans => fun(Value) -> emqx_connector_sql:to_sql_string(Value, Opts) end + emqx_template:render_strict(Template, Bindings, #{ + var_trans => fun(Value) -> emqx_utils_sql:to_sql_string(Value, Opts) end }). %% @doc Parse an SQL statement string into a prepared statement and a row template. @@ -83,7 +83,7 @@ render_strict(Template, Bindings, Opts) -> %% during the execution of the prepared statement. %% Example: %% ``` -%% {Statement, RowTemplate} = emqx_connector_template_sql:parse_prepstmt( +%% {Statement, RowTemplate} = emqx_template_sql:parse_prepstmt( %% "INSERT INTO table (id, name, age) VALUES (${id}, ${name}, 42)", %% #{parameters => '$n'} %% ), @@ -93,7 +93,7 @@ render_strict(Template, Bindings, Opts) -> -spec parse_prepstmt(unicode:chardata(), parse_opts()) -> {unicode:chardata(), row_template()}. parse_prepstmt(String, Opts) -> - Template = emqx_connector_template:parse(String, maps:with(?TEMPLATE_PARSE_OPTS, Opts)), + Template = emqx_template:parse(String, maps:with(?TEMPLATE_PARSE_OPTS, Opts)), Statement = mk_prepared_statement(Template, Opts), Placeholders = [Placeholder || Placeholder <- Template, element(1, Placeholder) == var], {Statement, Placeholders}. @@ -123,15 +123,15 @@ mk_replace(':n', N) -> %% @doc Render a row template into a list of SQL values. %% An _SQL value_ is a vaguely defined concept here, it is something that's considered %% compatible with the protocol of the database being used. See the definition of -%% `emqx_connector_sql:value()` for more details. +%% `emqx_utils_sql:value()` for more details. -spec render_prepstmt(template(), bindings()) -> {values(), [_Error]}. render_prepstmt(Template, Bindings) -> - Opts = #{var_trans => fun emqx_connector_sql:to_sql_value/1}, - emqx_connector_template:render(Template, Bindings, Opts). + Opts = #{var_trans => fun emqx_utils_sql:to_sql_value/1}, + emqx_template:render(Template, Bindings, Opts). -spec render_prepstmt_strict(template(), bindings()) -> values(). render_prepstmt_strict(Template, Bindings) -> - Opts = #{var_trans => fun emqx_connector_sql:to_sql_value/1}, - emqx_connector_template:render_strict(Template, Bindings, Opts). + Opts = #{var_trans => fun emqx_utils_sql:to_sql_value/1}, + emqx_template:render_strict(Template, Bindings, Opts). diff --git a/apps/emqx_utils/src/emqx_utils_sql.erl b/apps/emqx_utils/src/emqx_utils_sql.erl index 3caed6b62..12aac6464 100644 --- a/apps/emqx_utils/src/emqx_utils_sql.erl +++ b/apps/emqx_utils/src/emqx_utils_sql.erl @@ -80,7 +80,7 @@ to_sql_value(Map) when is_map(Map) -> emqx_utils_json:encode(Map). %% @doc Convert an Erlang term to a string that can be interpolated in literal %% SQL statements. The value is escaped if necessary. --spec to_sql_string(term(), Options) -> iodata() when +-spec to_sql_string(term(), Options) -> unicode:chardata() when Options :: #{ escaping => cql | mysql | sql }. @@ -98,7 +98,9 @@ to_sql_string(Term, #{escaping := cql}) -> to_sql_string(Term, #{}) -> maybe_escape(Term, fun escape_sql/1). --spec maybe_escape(_Value, fun((binary()) -> iodata())) -> iodata(). +-spec maybe_escape(_Value, fun((binary()) -> iodata())) -> unicode:chardata(). +maybe_escape(undefined, _EscapeFun) -> + <<"NULL">>; maybe_escape(Str, EscapeFun) when is_binary(Str) -> EscapeFun(Str); maybe_escape(Str, EscapeFun) when is_list(Str) -> @@ -109,9 +111,9 @@ maybe_escape(Str, EscapeFun) when is_list(Str) -> error(Otherwise) end; maybe_escape(Val, EscapeFun) when is_atom(Val) orelse is_map(Val) -> - EscapeFun(emqx_utils_conv:bin(Val)); + EscapeFun(emqx_template:to_string(Val)); maybe_escape(Val, _EscapeFun) -> - emqx_utils_conv:bin(Val). + emqx_template:to_string(Val). -spec escape_sql(binary()) -> iodata(). escape_sql(S) -> diff --git a/apps/emqx_connector/test/emqx_connector_template_SUITE.erl b/apps/emqx_utils/test/emqx_template_SUITE.erl similarity index 69% rename from apps/emqx_connector/test/emqx_connector_template_SUITE.erl rename to apps/emqx_utils/test/emqx_template_SUITE.erl index 3700caa96..657c3c94f 100644 --- a/apps/emqx_connector/test/emqx_connector_template_SUITE.erl +++ b/apps/emqx_utils/test/emqx_template_SUITE.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_connector_template_SUITE). +-module(emqx_template_SUITE). -compile(export_all). -compile(nowarn_export_all). @@ -33,7 +33,7 @@ t_render(_) -> l => [0, 1, 1000], u => "utf-8 is ǝɹǝɥ" }, - Template = emqx_connector_template:parse( + Template = emqx_template:parse( <<"a:${a},b:${b},c:${c},d:${d},d1:${d.d1},l:${l},u:${u}">> ), ?assertEqual( @@ -43,8 +43,8 @@ t_render(_) -> t_render_var_trans(_) -> Bindings = #{a => <<"1">>, b => 1, c => #{prop => 1.0}}, - Template = emqx_connector_template:parse(<<"a:${a},b:${b},c:${c.prop}">>), - {String, Errors} = emqx_connector_template:render( + Template = emqx_template:parse(<<"a:${a},b:${b},c:${c.prop}">>), + {String, Errors} = emqx_template:render( Template, Bindings, #{var_trans => fun(Name, _) -> "<" ++ Name ++ ">" end} @@ -56,10 +56,10 @@ t_render_var_trans(_) -> t_render_path(_) -> Bindings = #{d => #{d1 => <<"hi">>}}, - Template = emqx_connector_template:parse(<<"d.d1:${d.d1}">>), + Template = emqx_template:parse(<<"d.d1:${d.d1}">>), ?assertEqual( ok, - emqx_connector_template:validate(["d.d1"], Template) + emqx_template:validate(["d.d1"], Template) ), ?assertEqual( {<<"d.d1:hi">>, []}, @@ -68,10 +68,10 @@ t_render_path(_) -> t_render_custom_ph(_) -> Bindings = #{a => <<"a">>, b => <<"b">>}, - Template = emqx_connector_template:parse(<<"a:${a},b:${b}">>), + Template = emqx_template:parse(<<"a:${a},b:${b}">>), ?assertEqual( {error, [{"b", disallowed}]}, - emqx_connector_template:validate(["a"], Template) + emqx_template:validate(["a"], Template) ), ?assertEqual( <<"a:a,b:b">>, @@ -80,8 +80,8 @@ t_render_custom_ph(_) -> t_render_this(_) -> Bindings = #{a => <<"a">>, b => [1, 2, 3]}, - Template = emqx_connector_template:parse(<<"this:${} / also:${.}">>), - ?assertEqual(ok, emqx_connector_template:validate(["."], Template)), + Template = emqx_template:parse(<<"this:${} / also:${.}">>), + ?assertEqual(ok, emqx_template:validate(["."], Template)), ?assertEqual( % NOTE: order of the keys in the JSON object depends on the JSON encoder <<"this:{\"b\":[1,2,3],\"a\":\"a\"} / also:{\"b\":[1,2,3],\"a\":\"a\"}">>, @@ -90,7 +90,7 @@ t_render_this(_) -> t_render_missing_bindings(_) -> Bindings = #{no => #{}}, - Template = emqx_connector_template:parse( + Template = emqx_template:parse( <<"a:${a},b:${b},c:${c},d:${d.d1},e:${no.such_atom_i_swear}">> ), ?assertEqual( @@ -116,33 +116,33 @@ t_render_missing_bindings(_) -> t_unparse(_) -> TString = <<"a:${a},b:${b},c:$${c},d:{${d.d1}},e:${$}{e},lit:${$}{$}">>, - Template = emqx_connector_template:parse(TString), + Template = emqx_template:parse(TString), ?assertEqual( TString, - unicode:characters_to_binary(emqx_connector_template:unparse(Template)) + unicode:characters_to_binary(emqx_template:unparse(Template)) ). t_const(_) -> ?assertEqual( true, - emqx_connector_template:is_const(emqx_connector_template:parse(<<"">>)) + emqx_template:is_const(emqx_template:parse(<<"">>)) ), ?assertEqual( false, - emqx_connector_template:is_const( - emqx_connector_template:parse(<<"a:${a},b:${b},c:${$}{c}">>) + emqx_template:is_const( + emqx_template:parse(<<"a:${a},b:${b},c:${$}{c}">>) ) ), ?assertEqual( true, - emqx_connector_template:is_const( - emqx_connector_template:parse(<<"a:${$}{a},b:${$}{b}">>) + emqx_template:is_const( + emqx_template:parse(<<"a:${$}{a},b:${$}{b}">>) ) ). t_render_partial_ph(_) -> Bindings = #{a => <<"1">>, b => 1, c => 1.0, d => #{d1 => <<"hi">>}}, - Template = emqx_connector_template:parse(<<"a:$a,b:b},c:{c},d:${d">>), + Template = emqx_template:parse(<<"a:$a,b:b},c:{c},d:${d">>), ?assertEqual( <<"a:$a,b:b},c:{c},d:${d">>, render_strict_string(Template, Bindings) @@ -150,7 +150,7 @@ t_render_partial_ph(_) -> t_parse_escaped(_) -> Bindings = #{a => <<"1">>, b => 1, c => "VAR"}, - Template = emqx_connector_template:parse(<<"a:${a},b:${$}{b},c:${$}{${c}},lit:${$}{$}">>), + Template = emqx_template:parse(<<"a:${a},b:${$}{b},c:${$}{${c}},lit:${$}{$}">>), ?assertEqual( <<"a:1,b:${b},c:${VAR},lit:${$}">>, render_strict_string(Template, Bindings) @@ -158,7 +158,7 @@ t_parse_escaped(_) -> t_parse_escaped_dquote(_) -> Bindings = #{a => <<"1">>, b => 1}, - Template = emqx_connector_template:parse(<<"a:\"${a}\",b:\"${$}{b}\"">>, #{ + Template = emqx_template:parse(<<"a:\"${a}\",b:\"${$}{b}\"">>, #{ strip_double_quote => true }), ?assertEqual( @@ -169,30 +169,30 @@ t_parse_escaped_dquote(_) -> t_parse_sql_prepstmt(_) -> Bindings = #{a => <<"1">>, b => 1, c => 1.0, d => #{d1 => <<"hi">>}}, {PrepareStatement, RowTemplate} = - emqx_connector_template_sql:parse_prepstmt(<<"a:${a},b:${b},c:${c},d:${d}">>, #{ + emqx_template_sql:parse_prepstmt(<<"a:${a},b:${b},c:${c},d:${d}">>, #{ parameters => '?' }), ?assertEqual(<<"a:?,b:?,c:?,d:?">>, bin(PrepareStatement)), ?assertEqual( {[<<"1">>, 1, 1.0, <<"{\"d1\":\"hi\"}">>], _Errors = []}, - emqx_connector_template_sql:render_prepstmt(RowTemplate, Bindings) + emqx_template_sql:render_prepstmt(RowTemplate, Bindings) ). t_parse_sql_prepstmt_n(_) -> Bindings = #{a => undefined, b => true, c => atom, d => #{d1 => 42.1337}}, {PrepareStatement, RowTemplate} = - emqx_connector_template_sql:parse_prepstmt(<<"a:${a},b:${b},c:${c},d:${d}">>, #{ + emqx_template_sql:parse_prepstmt(<<"a:${a},b:${b},c:${c},d:${d}">>, #{ parameters => '$n' }), ?assertEqual(<<"a:$1,b:$2,c:$3,d:$4">>, bin(PrepareStatement)), ?assertEqual( [null, true, <<"atom">>, <<"{\"d1\":42.1337}">>], - emqx_connector_template_sql:render_prepstmt_strict(RowTemplate, Bindings) + emqx_template_sql:render_prepstmt_strict(RowTemplate, Bindings) ). t_parse_sql_prepstmt_colon(_) -> {PrepareStatement, _RowTemplate} = - emqx_connector_template_sql:parse_prepstmt(<<"a=${a},b=${b},c=${c},d=${d}">>, #{ + emqx_template_sql:parse_prepstmt(<<"a=${a},b=${b},c=${c},d=${d}">>, #{ parameters => ':n' }), ?assertEqual(<<"a=:1,b=:2,c=:3,d=:4">>, bin(PrepareStatement)). @@ -200,9 +200,9 @@ t_parse_sql_prepstmt_colon(_) -> t_parse_sql_prepstmt_partial_ph(_) -> Bindings = #{a => <<"1">>, b => 1, c => 1.0, d => #{d1 => <<"hi">>}}, {PrepareStatement, RowTemplate} = - emqx_connector_template_sql:parse_prepstmt(<<"a:$a,b:b},c:{c},d:${d">>, #{parameters => '?'}), + emqx_template_sql:parse_prepstmt(<<"a:$a,b:b},c:{c},d:${d">>, #{parameters => '?'}), ?assertEqual(<<"a:$a,b:b},c:{c},d:${d">>, bin(PrepareStatement)), - ?assertEqual([], emqx_connector_template_sql:render_prepstmt_strict(RowTemplate, Bindings)). + ?assertEqual([], emqx_template_sql:render_prepstmt_strict(RowTemplate, Bindings)). t_render_sql(_) -> Bindings = #{ @@ -213,14 +213,14 @@ t_render_sql(_) -> n => undefined, u => "utf8's cool 🐸" }, - Template = emqx_connector_template:parse(<<"a:${a},b:${b},c:${c},d:${d},n:${n},u:${u}">>), + Template = emqx_template:parse(<<"a:${a},b:${b},c:${c},d:${d},n:${n},u:${u}">>), ?assertMatch( {_String, _Errors = []}, - emqx_connector_template_sql:render(Template, Bindings, #{}) + emqx_template_sql:render(Template, Bindings, #{}) ), ?assertEqual( <<"a:'1',b:1,c:1.0,d:'{\"d1\":\"hi\"}',n:NULL,u:'utf8\\'s cool 🐸'"/utf8>>, - bin(emqx_connector_template_sql:render_strict(Template, Bindings, #{})) + bin(emqx_template_sql:render_strict(Template, Bindings, #{})) ). t_render_mysql(_) -> @@ -236,7 +236,7 @@ t_render_mysql(_) -> g => "utf8's cool 🐸", h => imgood }, - Template = emqx_connector_template_sql:parse( + Template = emqx_template_sql:parse( <<"a:${a},b:${b},c:${c},d:${d},e:${e},f:${f},g:${g},h:${h}">> ), ?assertEqual( @@ -245,7 +245,7 @@ t_render_mysql(_) -> "e:'\\\\\\0💩',f:0x6E6F6E2D75746638DCC900,g:'utf8\\'s cool 🐸',"/utf8, "h:'imgood'" >>, - bin(emqx_connector_template_sql:render_strict(Template, Bindings, #{escaping => mysql})) + bin(emqx_template_sql:render_strict(Template, Bindings, #{escaping => mysql})) ). t_render_cql(_) -> @@ -257,18 +257,18 @@ t_render_cql(_) -> c => 1.0, d => #{d1 => <<"someone's phone">>} }, - Template = emqx_connector_template:parse(<<"a:${a},b:${b},c:${c},d:${d}">>), + Template = emqx_template:parse(<<"a:${a},b:${b},c:${c},d:${d}">>), ?assertEqual( <<"a:'1''''2',b:1,c:1.0,d:'{\"d1\":\"someone''s phone\"}'">>, - bin(emqx_connector_template_sql:render_strict(Template, Bindings, #{escaping => cql})) + bin(emqx_template_sql:render_strict(Template, Bindings, #{escaping => cql})) ). t_render_sql_custom_ph(_) -> {PrepareStatement, RowTemplate} = - emqx_connector_template_sql:parse_prepstmt(<<"a:${a},b:${b.c}">>, #{parameters => '$n'}), + emqx_template_sql:parse_prepstmt(<<"a:${a},b:${b.c}">>, #{parameters => '$n'}), ?assertEqual( {error, [{"b.c", disallowed}]}, - emqx_connector_template:validate(["a"], RowTemplate) + emqx_template:validate(["a"], RowTemplate) ), ?assertEqual(<<"a:$1,b:$2">>, bin(PrepareStatement)). @@ -276,57 +276,57 @@ t_render_sql_strip_double_quote(_) -> Bindings = #{a => <<"a">>, b => <<"b">>}, %% no strip_double_quote option: "${key}" -> "value" - {PrepareStatement1, RowTemplate1} = emqx_connector_template_sql:parse_prepstmt( + {PrepareStatement1, RowTemplate1} = emqx_template_sql:parse_prepstmt( <<"a:\"${a}\",b:\"${b}\"">>, #{parameters => '$n'} ), ?assertEqual(<<"a:\"$1\",b:\"$2\"">>, bin(PrepareStatement1)), ?assertEqual( [<<"a">>, <<"b">>], - emqx_connector_template_sql:render_prepstmt_strict(RowTemplate1, Bindings) + emqx_template_sql:render_prepstmt_strict(RowTemplate1, Bindings) ), %% strip_double_quote = true: "${key}" -> value - {PrepareStatement2, RowTemplate2} = emqx_connector_template_sql:parse_prepstmt( + {PrepareStatement2, RowTemplate2} = emqx_template_sql:parse_prepstmt( <<"a:\"${a}\",b:\"${b}\"">>, #{parameters => '$n', strip_double_quote => true} ), ?assertEqual(<<"a:$1,b:$2">>, bin(PrepareStatement2)), ?assertEqual( [<<"a">>, <<"b">>], - emqx_connector_template_sql:render_prepstmt_strict(RowTemplate2, Bindings) + emqx_template_sql:render_prepstmt_strict(RowTemplate2, Bindings) ). t_render_tmpl_deep(_) -> Bindings = #{a => <<"1">>, b => 1, c => 1.0, d => #{d1 => <<"hi">>}}, - Template = emqx_connector_template:parse_deep( + Template = emqx_template:parse_deep( #{<<"${a}">> => [<<"$${b}">>, "c", 2, 3.0, '${d}', {[<<"${c}">>, <<"${$}{d}">>], 0}]} ), ?assertEqual( {error, [{V, disallowed} || V <- ["b", "c"]]}, - emqx_connector_template:validate(["a"], Template) + emqx_template:validate(["a"], Template) ), ?assertEqual( #{<<"1">> => [<<"$1">>, "c", 2, 3.0, '${d}', {[<<"1.0">>, <<"${d}">>], 0}]}, - emqx_connector_template:render_strict(Template, Bindings) + emqx_template:render_strict(Template, Bindings) ). t_unparse_tmpl_deep(_) -> Term = #{<<"${a}">> => [<<"$${b}">>, "c", 2, 3.0, '${d}', {[<<"${c}">>], <<"${$}{d}">>, 0}]}, - Template = emqx_connector_template:parse_deep(Term), - ?assertEqual(Term, emqx_connector_template:unparse(Template)). + Template = emqx_template:parse_deep(Term), + ?assertEqual(Term, emqx_template:unparse(Template)). %% render_string(Template, Bindings) -> - {String, Errors} = emqx_connector_template:render(Template, Bindings), + {String, Errors} = emqx_template:render(Template, Bindings), {bin(String), Errors}. render_strict_string(Template, Bindings) -> - bin(emqx_connector_template:render_strict(Template, Bindings)). + bin(emqx_template:render_strict(Template, Bindings)). bin(String) -> unicode:characters_to_binary(String). From 75cc66378667372da7c7ef0ebfcc6f284050d5a4 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 14 Jul 2023 18:43:12 +0200 Subject: [PATCH 15/21] chore(ruleeng): streamline application dependencies --- apps/emqx_rule_engine/src/emqx_rule_engine.app.src | 11 ++++++++++- mix.exs | 1 + 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index c353742ae..cad752886 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -5,7 +5,16 @@ {vsn, "5.0.28"}, {modules, []}, {registered, [emqx_rule_engine_sup, emqx_rule_engine]}, - {applications, [kernel, stdlib, rulesql, getopt, emqx_ctl, uuid]}, + {applications, [ + kernel, + stdlib, + rulesql, + getopt, + uuid, + emqx, + emqx_utils, + emqx_ctl + ]}, {mod, {emqx_rule_engine_app, []}}, {env, []}, {licenses, ["Apache-2.0"]}, diff --git a/mix.exs b/mix.exs index 3817b5121..409c29924 100644 --- a/mix.exs +++ b/mix.exs @@ -338,6 +338,7 @@ defmodule EMQXUmbrella.MixProject do :emqx_management, :emqx_retainer, :emqx_prometheus, + :emqx_rule_engine, :emqx_auto_subscribe, :emqx_slow_subs, :emqx_plugins, From 69cfa740ea9775dfa2302a2d589c593c06cff7e9 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 20 Oct 2023 20:36:44 +0700 Subject: [PATCH 16/21] fix(ruleeng): ensure full backward compatibility --- .../src/emqx_rule_actions.erl | 52 +++++++------- apps/emqx_utils/src/emqx_template.erl | 72 ++++++++++++++++--- apps/emqx_utils/test/emqx_template_SUITE.erl | 8 +-- 3 files changed, 95 insertions(+), 37 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_actions.erl b/apps/emqx_rule_engine/src/emqx_rule_actions.erl index 7a8b2520c..96eb4a789 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_actions.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_actions.erl @@ -72,8 +72,8 @@ pre_process_action_args( Args#{ preprocessed_tmpl => #{ topic => emqx_template:parse(Topic), - qos => parse_vars(QoS), - retain => parse_vars(Retain), + qos => parse_simple_var(QoS), + retain => parse_simple_var(Retain), payload => parse_payload(Payload), mqtt_properties => parse_mqtt_properties(MQTTProperties), user_properties => parse_user_properties(UserProperties) @@ -119,8 +119,8 @@ republish( } ) -> % NOTE: rendering missing bindings as string "undefined" - {TopicString, _Errors1} = emqx_template:render(TopicTemplate, Selected), - {PayloadString, _Errors2} = emqx_template:render(PayloadTemplate, Selected), + {TopicString, _Errors1} = render_template(TopicTemplate, Selected), + {PayloadString, _Errors2} = render_template(PayloadTemplate, Selected), Topic = iolist_to_binary(TopicString), Payload = iolist_to_binary(PayloadString), QoS = render_simple_var(QoSTemplate, Selected, 0), @@ -201,11 +201,17 @@ safe_publish(RuleId, Topic, QoS, Flags, Payload, PubProps) -> _ = emqx_broker:safe_publish(Msg), emqx_metrics:inc_msg(Msg). -parse_vars(Data) when is_binary(Data) -> +parse_simple_var(Data) when is_binary(Data) -> emqx_template:parse(Data); -parse_vars(Data) -> +parse_simple_var(Data) -> {const, Data}. +parse_payload(Payload) -> + case string:is_empty(Payload) of + false -> emqx_template:parse(Payload); + true -> emqx_template:parse("${.}") + end. + parse_mqtt_properties(MQTTPropertiesTemplate) -> maps:map( fun(_Key, V) -> emqx_template:parse(V) end, @@ -225,8 +231,12 @@ parse_user_properties(_) -> %% invalid, discard undefined. +render_template(Template, Bindings) -> + Opts = #{var_lookup => fun emqx_template:lookup_loose_json/2}, + emqx_template:render(Template, Bindings, Opts). + render_simple_var([{var, _Name, Accessor}], Data, Default) -> - case emqx_template:lookup_var(Accessor, Data) of + case emqx_template:lookup_loose_json(Accessor, Data) of {ok, Var} -> Var; %% cannot find the variable from Data {error, _} -> Default @@ -234,12 +244,6 @@ render_simple_var([{var, _Name, Accessor}], Data, Default) -> render_simple_var({const, Val}, _Data, _Default) -> Val. -parse_payload(Payload) -> - case string:is_empty(Payload) of - false -> emqx_template:parse(Payload); - true -> emqx_template:parse("${.}") - end. - render_pub_props(UserPropertiesTemplate, Selected, Env) -> UserProperties = case UserPropertiesTemplate of @@ -257,26 +261,24 @@ render_mqtt_properties(MQTTPropertiesTemplate, Selected, Env) -> MQTTProperties = maps:fold( fun(K, Template, Acc) -> - try - V = unicode:characters_to_binary( - emqx_template:render_strict(Template, Selected) - ), - Acc#{K => V} - catch - Kind:Error -> + {V, Errors} = render_template(Template, Selected), + NAcc = Acc#{K => iolist_to_binary(V)}, + case Errors of + [] -> + ok; + Errors -> ?SLOG( debug, #{ msg => "bad_mqtt_property_value_ignored", rule_id => RuleId, - exception => Kind, - reason => Error, + reason => Errors, property => K, selected => Selected } - ), - Acc - end + ) + end, + NAcc end, #{}, MQTTPropertiesTemplate diff --git a/apps/emqx_utils/src/emqx_template.erl b/apps/emqx_utils/src/emqx_template.erl index deb25d807..43d9158de 100644 --- a/apps/emqx_utils/src/emqx_template.erl +++ b/apps/emqx_utils/src/emqx_template.erl @@ -29,6 +29,7 @@ -export([render_strict/3]). -export([lookup_var/2]). +-export([lookup_loose_json/2]). -export([to_string/1]). -export_type([t/0]). @@ -62,16 +63,23 @@ -type binding() :: scalar() | list(scalar()) | bindings(). -type bindings() :: #{atom() | binary() => binding()}. +-type reason() :: undefined | {location(), _InvalidType :: atom()}. +-type location() :: non_neg_integer(). + -type var_trans() :: fun((Value :: term()) -> unicode:chardata()) | fun((varname(), Value :: term()) -> unicode:chardata()). +-type var_lookup() :: + fun((accessor(), bindings()) -> {ok, binding()} | {error, reason()}). + -type parse_opts() :: #{ strip_double_quote => boolean() }. -type render_opts() :: #{ - var_trans => var_trans() + var_trans => var_trans(), + var_lookup => var_lookup() }. -define(PH_VAR_THIS, '$this'). @@ -173,7 +181,7 @@ render_placeholder(Name) -> %% By default, all binding values are converted to strings using `to_string/1` %% function. Option `var_trans` can be used to override this behaviour. -spec render(t(), bindings()) -> - {term(), [_Error :: {varname(), undefined}]}. + {term(), [_Error :: {varname(), reason()}]}. render(Template, Bindings) -> render(Template, Bindings, #{}). @@ -195,7 +203,7 @@ render({'$tpl', Template}, Bindings, Opts) -> render_deep(Template, Bindings, Opts). render_binding(Name, Accessor, Bindings, Opts) -> - case lookup_var(Accessor, Bindings) of + case lookup_value(Accessor, Bindings, Opts) of {ok, Value} -> {render_value(Name, Value, Opts), []}; {error, Reason} -> @@ -205,6 +213,11 @@ render_binding(Name, Accessor, Bindings, Opts) -> {render_value(Name, undefined, Opts), [{Name, Reason}]} end. +lookup_value(Accessor, Bindings, #{var_lookup := LookupFun}) -> + LookupFun(Accessor, Bindings); +lookup_value(Accessor, Bindings, #{}) -> + lookup_var(Accessor, Bindings). + render_value(_Name, Value, #{var_trans := TransFun}) when is_function(TransFun, 1) -> TransFun(Value); render_value(Name, Value, #{var_trans := TransFun}) when is_function(TransFun, 2) -> @@ -309,17 +322,60 @@ unparse_deep(Term) -> %% +%% @doc Lookup a variable in the bindings accessible through the accessor. +%% Lookup is "loose" in the sense that atom and binary keys in the bindings are +%% treated equally. This is useful for both hand-crafted and JSON-like bindings. +%% This is the default lookup function used by rendering functions. -spec lookup_var(accessor(), bindings()) -> - {ok, binding()} | {error, undefined}. -lookup_var(Var, Value) when Var == ?PH_VAR_THIS orelse Var == [] -> + {ok, binding()} | {error, reason()}. +lookup_var(Var, Bindings) -> + lookup_var(0, Var, Bindings). + +lookup_var(_, Var, Value) when Var == ?PH_VAR_THIS orelse Var == [] -> {ok, Value}; -lookup_var([Prop | Rest], Bindings) -> +lookup_var(Loc, [Prop | Rest], Bindings) when is_map(Bindings) -> case lookup(Prop, Bindings) of {ok, Value} -> - lookup_var(Rest, Value); + lookup_var(Loc + 1, Rest, Value); {error, Reason} -> {error, Reason} - end. + end; +lookup_var(Loc, _, Invalid) -> + {error, {Loc, type_name(Invalid)}}. + +%% @doc Lookup a variable in the bindings accessible through the accessor. +%% Additionally to `lookup_var/2` behavior, this function also tries to parse any +%% binary as JSON to a map if accessor needs to go deeper into it. +-spec lookup_loose_json(accessor(), bindings() | binary()) -> + {ok, binding()} | {error, reason()}. +lookup_loose_json(Var, Bindings) -> + lookup_loose_json(0, Var, Bindings). + +lookup_loose_json(_, Var, Value) when Var == ?PH_VAR_THIS orelse Var == [] -> + {ok, Value}; +lookup_loose_json(Loc, [Prop | Rest], Bindings) when is_map(Bindings) -> + case lookup(Prop, Bindings) of + {ok, Value} -> + lookup_loose_json(Loc + 1, Rest, Value); + {error, Reason} -> + {error, Reason} + end; +lookup_loose_json(Loc, Rest, Json) when is_binary(Json) -> + try emqx_utils_json:decode(Json) of + Bindings -> + % NOTE: This is intentional, we don't want to parse nested JSON. + lookup_var(Loc, Rest, Bindings) + catch + error:_ -> + {error, {Loc, binary}} + end; +lookup_loose_json(Loc, _, Invalid) -> + {error, {Loc, type_name(Invalid)}}. + +type_name(Term) when is_atom(Term) -> atom; +type_name(Term) when is_number(Term) -> number; +type_name(Term) when is_binary(Term) -> binary; +type_name(Term) when is_list(Term) -> list. -spec lookup(Prop :: binary(), bindings()) -> {ok, binding()} | {error, undefined}. diff --git a/apps/emqx_utils/test/emqx_template_SUITE.erl b/apps/emqx_utils/test/emqx_template_SUITE.erl index 657c3c94f..22ffff47c 100644 --- a/apps/emqx_utils/test/emqx_template_SUITE.erl +++ b/apps/emqx_utils/test/emqx_template_SUITE.erl @@ -89,15 +89,15 @@ t_render_this(_) -> ). t_render_missing_bindings(_) -> - Bindings = #{no => #{}}, + Bindings = #{no => #{}, c => #{<<"c1">> => 42}}, Template = emqx_template:parse( - <<"a:${a},b:${b},c:${c},d:${d.d1},e:${no.such_atom_i_swear}">> + <<"a:${a},b:${b},c:${c.c1.c2},d:${d.d1},e:${no.such_atom_i_swear}">> ), ?assertEqual( {<<"a:undefined,b:undefined,c:undefined,d:undefined,e:undefined">>, [ {"no.such_atom_i_swear", undefined}, {"d.d1", undefined}, - {"c", undefined}, + {"c.c1.c2", {2, number}}, {"b", undefined}, {"a", undefined} ]}, @@ -107,7 +107,7 @@ t_render_missing_bindings(_) -> [ {"no.such_atom_i_swear", undefined}, {"d.d1", undefined}, - {"c", undefined}, + {"c.c1.c2", {2, number}}, {"b", undefined}, {"a", undefined} ], From 02c1bd70b68b8e443aafc10a2185809c71de6e4a Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 23 Oct 2023 15:42:58 +0700 Subject: [PATCH 17/21] feat(tpl): factor out loose json concept into a separate module Which is called `emqx_jsonish`. Also introduce an _access module_ abstraction to extract information from such data during rendering. --- .../src/emqx_rule_actions.erl | 5 +- apps/emqx_utils/src/emqx_jsonish.erl | 71 +++++++++++ apps/emqx_utils/src/emqx_template.erl | 113 +++++++----------- apps/emqx_utils/src/emqx_template_sql.erl | 28 ++--- apps/emqx_utils/test/emqx_jsonish_tests.erl | 97 +++++++++++++++ apps/emqx_utils/test/emqx_template_SUITE.erl | 106 +++++++++------- 6 files changed, 295 insertions(+), 125 deletions(-) create mode 100644 apps/emqx_utils/src/emqx_jsonish.erl create mode 100644 apps/emqx_utils/test/emqx_jsonish_tests.erl diff --git a/apps/emqx_rule_engine/src/emqx_rule_actions.erl b/apps/emqx_rule_engine/src/emqx_rule_actions.erl index 96eb4a789..d0810eb84 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_actions.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_actions.erl @@ -232,11 +232,10 @@ parse_user_properties(_) -> undefined. render_template(Template, Bindings) -> - Opts = #{var_lookup => fun emqx_template:lookup_loose_json/2}, - emqx_template:render(Template, Bindings, Opts). + emqx_template:render(Template, {emqx_jsonish, Bindings}). render_simple_var([{var, _Name, Accessor}], Data, Default) -> - case emqx_template:lookup_loose_json(Accessor, Data) of + case emqx_jsonish:lookup(Accessor, Data) of {ok, Var} -> Var; %% cannot find the variable from Data {error, _} -> Default diff --git a/apps/emqx_utils/src/emqx_jsonish.erl b/apps/emqx_utils/src/emqx_jsonish.erl new file mode 100644 index 000000000..b2d92c7fc --- /dev/null +++ b/apps/emqx_utils/src/emqx_jsonish.erl @@ -0,0 +1,71 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_jsonish). + +-export([lookup/2]). + +-export_type([t/0]). + +%% @doc Either a map or a JSON serial. +%% Think of it as a kind of lazily parsed and/or constructed JSON. +-type t() :: propmap() | serial(). + +%% @doc JSON in serialized form. +-type serial() :: binary(). + +-type propmap() :: #{prop() => value()}. +-type prop() :: atom() | binary(). +-type value() :: scalar() | [scalar() | propmap()] | t(). +-type scalar() :: atom() | unicode:chardata() | number(). + +%% + +%% @doc Lookup a value in the JSON-ish map accessible through the given accessor. +%% If accessor implies drilling down into a binary, it will be treated as JSON serial. +%% Failure to parse the binary as JSON will result in an _invalid type_ error. +%% Nested JSON is NOT parsed recursively. +-spec lookup(emqx_template:accessor(), t()) -> + {ok, value()} + | {error, undefined | {_Location :: non_neg_integer(), _InvalidType :: atom()}}. +lookup(Var, Jsonish) -> + lookup(0, _Decoded = false, Var, Jsonish). + +lookup(_, _, [], Value) -> + {ok, Value}; +lookup(Loc, Decoded, [Prop | Rest], Jsonish) when is_map(Jsonish) -> + case emqx_template:lookup(Prop, Jsonish) of + {ok, Value} -> + lookup(Loc + 1, Decoded, Rest, Value); + {error, Reason} -> + {error, Reason} + end; +lookup(Loc, _Decoded = false, Rest, Json) when is_binary(Json) -> + try emqx_utils_json:decode(Json) of + Value -> + % NOTE: This is intentional, we don't want to parse nested JSON. + lookup(Loc, true, Rest, Value) + catch + error:_ -> + {error, {Loc, binary}} + end; +lookup(Loc, _, _, Invalid) -> + {error, {Loc, type_name(Invalid)}}. + +type_name(Term) when is_atom(Term) -> atom; +type_name(Term) when is_number(Term) -> number; +type_name(Term) when is_binary(Term) -> binary; +type_name(Term) when is_list(Term) -> list. diff --git a/apps/emqx_utils/src/emqx_template.erl b/apps/emqx_utils/src/emqx_template.erl index 43d9158de..1ccc74c50 100644 --- a/apps/emqx_utils/src/emqx_template.erl +++ b/apps/emqx_utils/src/emqx_template.erl @@ -29,7 +29,8 @@ -export([render_strict/3]). -export([lookup_var/2]). --export([lookup_loose_json/2]). +-export([lookup/2]). + -export([to_string/1]). -export_type([t/0]). @@ -38,6 +39,10 @@ -export_type([placeholder/0]). -export_type([varname/0]). -export_type([bindings/0]). +-export_type([accessor/0]). + +-export_type([context/0]). +-export_type([render_opts/0]). -type t() :: str() | {'$tpl', deeptpl()}. @@ -70,19 +75,22 @@ fun((Value :: term()) -> unicode:chardata()) | fun((varname(), Value :: term()) -> unicode:chardata()). --type var_lookup() :: - fun((accessor(), bindings()) -> {ok, binding()} | {error, reason()}). - -type parse_opts() :: #{ strip_double_quote => boolean() }. -type render_opts() :: #{ - var_trans => var_trans(), - var_lookup => var_lookup() + var_trans => var_trans() }. --define(PH_VAR_THIS, '$this'). +-type context() :: + %% Map with (potentially nested) bindings. + bindings() + %% Arbitrary term accessible via an access module with `lookup/2` function. + | {_AccessModule :: module(), _Bindings}. + +%% Access module API +-callback lookup(accessor(), _Bindings) -> {ok, _Value} | {error, reason()}. -define(RE_PLACEHOLDER, "\\$\\{[.]?([a-zA-Z0-9._]*)\\}"). -define(RE_ESCAPE, "\\$\\{(\\$)\\}"). @@ -130,7 +138,7 @@ prepend(Head, To) -> parse_accessor(Var) -> case string:split(Var, <<".">>, all) of [<<>>] -> - ?PH_VAR_THIS; + []; Name -> Name end. @@ -180,18 +188,18 @@ render_placeholder(Name) -> %% If one or more placeholders are not found in bindings, an error is returned. %% By default, all binding values are converted to strings using `to_string/1` %% function. Option `var_trans` can be used to override this behaviour. --spec render(t(), bindings()) -> +-spec render(t(), context()) -> {term(), [_Error :: {varname(), reason()}]}. -render(Template, Bindings) -> - render(Template, Bindings, #{}). +render(Template, Context) -> + render(Template, Context, #{}). --spec render(t(), bindings(), render_opts()) -> +-spec render(t(), context(), render_opts()) -> {term(), [_Error :: {varname(), undefined}]}. -render(Template, Bindings, Opts) when is_list(Template) -> +render(Template, Context, Opts) when is_list(Template) -> lists:mapfoldl( fun ({var, Name, Accessor}, EAcc) -> - {String, Errors} = render_binding(Name, Accessor, Bindings, Opts), + {String, Errors} = render_binding(Name, Accessor, Context, Opts), {String, Errors ++ EAcc}; (String, EAcc) -> {String, EAcc} @@ -199,11 +207,11 @@ render(Template, Bindings, Opts) when is_list(Template) -> [], Template ); -render({'$tpl', Template}, Bindings, Opts) -> - render_deep(Template, Bindings, Opts). +render({'$tpl', Template}, Context, Opts) -> + render_deep(Template, Context, Opts). -render_binding(Name, Accessor, Bindings, Opts) -> - case lookup_value(Accessor, Bindings, Opts) of +render_binding(Name, Accessor, Context, Opts) -> + case lookup_value(Accessor, Context) of {ok, Value} -> {render_value(Name, Value, Opts), []}; {error, Reason} -> @@ -213,9 +221,9 @@ render_binding(Name, Accessor, Bindings, Opts) -> {render_value(Name, undefined, Opts), [{Name, Reason}]} end. -lookup_value(Accessor, Bindings, #{var_lookup := LookupFun}) -> - LookupFun(Accessor, Bindings); -lookup_value(Accessor, Bindings, #{}) -> +lookup_value(Accessor, {AccessMod, Bindings}) -> + AccessMod:lookup(Accessor, Bindings); +lookup_value(Accessor, Bindings) -> lookup_var(Accessor, Bindings). render_value(_Name, Value, #{var_trans := TransFun}) when is_function(TransFun, 1) -> @@ -228,19 +236,19 @@ render_value(_Name, Value, #{}) -> %% @doc Render a template with given bindings. %% Behaves like `render/2`, but raises an error exception if one or more placeholders %% are not found in the bindings. --spec render_strict(t(), bindings()) -> +-spec render_strict(t(), context()) -> term(). -render_strict(Template, Bindings) -> - render_strict(Template, Bindings, #{}). +render_strict(Template, Context) -> + render_strict(Template, Context, #{}). --spec render_strict(t(), bindings(), render_opts()) -> +-spec render_strict(t(), context(), render_opts()) -> term(). -render_strict(Template, Bindings, Opts) -> - case render(Template, Bindings, Opts) of +render_strict(Template, Context, Opts) -> + case render(Template, Context, Opts) of {Render, []} -> Render; {_, Errors = [_ | _]} -> - error(Errors, [unparse(Template), Bindings]) + error(Errors, [unparse(Template), Context]) end. %% @doc Parse an arbitrary Erlang term into a "deep" template. @@ -275,30 +283,30 @@ parse_deep_term(Term, Opts) when is_binary(Term) -> parse_deep_term(Term, _Opts) -> Term. -render_deep(Template, Bindings, Opts) when is_map(Template) -> +render_deep(Template, Context, Opts) when is_map(Template) -> maps:fold( fun(KT, VT, {Acc, Errors}) -> - {K, KErrors} = render_deep(KT, Bindings, Opts), - {V, VErrors} = render_deep(VT, Bindings, Opts), + {K, KErrors} = render_deep(KT, Context, Opts), + {V, VErrors} = render_deep(VT, Context, Opts), {Acc#{K => V}, KErrors ++ VErrors ++ Errors} end, {#{}, []}, Template ); -render_deep({list, Template}, Bindings, Opts) when is_list(Template) -> +render_deep({list, Template}, Context, Opts) when is_list(Template) -> lists:mapfoldr( fun(T, Errors) -> - {E, VErrors} = render_deep(T, Bindings, Opts), + {E, VErrors} = render_deep(T, Context, Opts), {E, VErrors ++ Errors} end, [], Template ); -render_deep({tuple, Template}, Bindings, Opts) when is_list(Template) -> - {Term, Errors} = render_deep({list, Template}, Bindings, Opts), +render_deep({tuple, Template}, Context, Opts) when is_list(Template) -> + {Term, Errors} = render_deep({list, Template}, Context, Opts), {list_to_tuple(Term), Errors}; -render_deep(Template, Bindings, Opts) when is_list(Template) -> - {String, Errors} = render(Template, Bindings, Opts), +render_deep(Template, Context, Opts) when is_list(Template) -> + {String, Errors} = render(Template, Context, Opts), {unicode:characters_to_binary(String), Errors}; render_deep(Term, _Bindings, _Opts) -> {Term, []}. @@ -331,7 +339,7 @@ unparse_deep(Term) -> lookup_var(Var, Bindings) -> lookup_var(0, Var, Bindings). -lookup_var(_, Var, Value) when Var == ?PH_VAR_THIS orelse Var == [] -> +lookup_var(_, [], Value) -> {ok, Value}; lookup_var(Loc, [Prop | Rest], Bindings) when is_map(Bindings) -> case lookup(Prop, Bindings) of @@ -343,35 +351,6 @@ lookup_var(Loc, [Prop | Rest], Bindings) when is_map(Bindings) -> lookup_var(Loc, _, Invalid) -> {error, {Loc, type_name(Invalid)}}. -%% @doc Lookup a variable in the bindings accessible through the accessor. -%% Additionally to `lookup_var/2` behavior, this function also tries to parse any -%% binary as JSON to a map if accessor needs to go deeper into it. --spec lookup_loose_json(accessor(), bindings() | binary()) -> - {ok, binding()} | {error, reason()}. -lookup_loose_json(Var, Bindings) -> - lookup_loose_json(0, Var, Bindings). - -lookup_loose_json(_, Var, Value) when Var == ?PH_VAR_THIS orelse Var == [] -> - {ok, Value}; -lookup_loose_json(Loc, [Prop | Rest], Bindings) when is_map(Bindings) -> - case lookup(Prop, Bindings) of - {ok, Value} -> - lookup_loose_json(Loc + 1, Rest, Value); - {error, Reason} -> - {error, Reason} - end; -lookup_loose_json(Loc, Rest, Json) when is_binary(Json) -> - try emqx_utils_json:decode(Json) of - Bindings -> - % NOTE: This is intentional, we don't want to parse nested JSON. - lookup_var(Loc, Rest, Bindings) - catch - error:_ -> - {error, {Loc, binary}} - end; -lookup_loose_json(Loc, _, Invalid) -> - {error, {Loc, type_name(Invalid)}}. - type_name(Term) when is_atom(Term) -> atom; type_name(Term) when is_number(Term) -> number; type_name(Term) when is_binary(Term) -> binary; diff --git a/apps/emqx_utils/src/emqx_template_sql.erl b/apps/emqx_utils/src/emqx_template_sql.erl index f215cd868..4e9d8f622 100644 --- a/apps/emqx_utils/src/emqx_template_sql.erl +++ b/apps/emqx_utils/src/emqx_template_sql.erl @@ -27,9 +27,9 @@ -export_type([row_template/0]). --type template() :: emqx_template:t(). +-type template() :: emqx_template:str(). -type row_template() :: [emqx_template:placeholder()]. --type bindings() :: emqx_template:bindings(). +-type context() :: emqx_template:context(). -type values() :: [emqx_utils_sql:value()]. @@ -62,19 +62,19 @@ parse(String, Opts) -> %% @doc Render an SQL statement template given a set of bindings. %% Interpolation generally follows the SQL syntax, strings are escaped according to the %% `escaping` option. --spec render(template(), bindings(), render_opts()) -> +-spec render(template(), context(), render_opts()) -> {unicode:chardata(), [_Error]}. -render(Template, Bindings, Opts) -> - emqx_template:render(Template, Bindings, #{ +render(Template, Context, Opts) -> + emqx_template:render(Template, Context, #{ var_trans => fun(Value) -> emqx_utils_sql:to_sql_string(Value, Opts) end }). %% @doc Render an SQL statement template given a set of bindings. %% Errors are raised if any placeholders are not bound. --spec render_strict(template(), bindings(), render_opts()) -> +-spec render_strict(template(), context(), render_opts()) -> unicode:chardata(). -render_strict(Template, Bindings, Opts) -> - emqx_template:render_strict(Template, Bindings, #{ +render_strict(Template, Context, Opts) -> + emqx_template:render_strict(Template, Context, #{ var_trans => fun(Value) -> emqx_utils_sql:to_sql_string(Value, Opts) end }). @@ -124,14 +124,14 @@ mk_replace(':n', N) -> %% An _SQL value_ is a vaguely defined concept here, it is something that's considered %% compatible with the protocol of the database being used. See the definition of %% `emqx_utils_sql:value()` for more details. --spec render_prepstmt(template(), bindings()) -> +-spec render_prepstmt(template(), context()) -> {values(), [_Error]}. -render_prepstmt(Template, Bindings) -> +render_prepstmt(Template, Context) -> Opts = #{var_trans => fun emqx_utils_sql:to_sql_value/1}, - emqx_template:render(Template, Bindings, Opts). + emqx_template:render(Template, Context, Opts). --spec render_prepstmt_strict(template(), bindings()) -> +-spec render_prepstmt_strict(template(), context()) -> values(). -render_prepstmt_strict(Template, Bindings) -> +render_prepstmt_strict(Template, Context) -> Opts = #{var_trans => fun emqx_utils_sql:to_sql_value/1}, - emqx_template:render_strict(Template, Bindings, Opts). + emqx_template:render_strict(Template, Context, Opts). diff --git a/apps/emqx_utils/test/emqx_jsonish_tests.erl b/apps/emqx_utils/test/emqx_jsonish_tests.erl new file mode 100644 index 000000000..c776615a1 --- /dev/null +++ b/apps/emqx_utils/test/emqx_jsonish_tests.erl @@ -0,0 +1,97 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_jsonish_tests). + +-include_lib("eunit/include/eunit.hrl"). + +prop_prio_test_() -> + [ + ?_assertEqual( + {ok, 42}, + emqx_jsonish:lookup([<<"foo">>], #{<<"foo">> => 42, foo => 1337}) + ), + ?_assertEqual( + {ok, 1337}, + emqx_jsonish:lookup([<<"foo">>], #{foo => 1337}) + ) + ]. + +undefined_test() -> + ?assertEqual( + {error, undefined}, + emqx_jsonish:lookup([<<"foo">>], #{}) + ). + +undefined_deep_test() -> + ?assertEqual( + {error, undefined}, + emqx_jsonish:lookup([<<"foo">>, <<"bar">>], #{}) + ). + +undefined_deep_json_test() -> + ?assertEqual( + {error, undefined}, + emqx_jsonish:lookup( + [<<"foo">>, <<"bar">>, <<"baz">>], + <<"{\"foo\":{\"bar\":{\"no\":{}}}}">> + ) + ). + +invalid_type_test() -> + ?assertEqual( + {error, {0, number}}, + emqx_jsonish:lookup([<<"foo">>], <<"42">>) + ). + +invalid_type_deep_test() -> + ?assertEqual( + {error, {2, atom}}, + emqx_jsonish:lookup([<<"foo">>, <<"bar">>, <<"tuple">>], #{foo => #{bar => baz}}) + ). + +decode_json_test() -> + ?assertEqual( + {ok, 42}, + emqx_jsonish:lookup([<<"foo">>, <<"bar">>], <<"{\"foo\":{\"bar\":42}}">>) + ). + +decode_json_deep_test() -> + ?assertEqual( + {ok, 42}, + emqx_jsonish:lookup([<<"foo">>, <<"bar">>], #{<<"foo">> => <<"{\"bar\": 42}">>}) + ). + +decode_json_invalid_type_test() -> + ?assertEqual( + {error, {1, list}}, + emqx_jsonish:lookup([<<"foo">>, <<"bar">>], #{<<"foo">> => <<"[1,2,3]">>}) + ). + +decode_no_json_test() -> + ?assertEqual( + {error, {1, binary}}, + emqx_jsonish:lookup([<<"foo">>, <<"bar">>], #{<<"foo">> => <<0, 1, 2, 3>>}) + ). + +decode_json_no_nested_test() -> + ?assertEqual( + {error, {2, binary}}, + emqx_jsonish:lookup( + [<<"foo">>, <<"bar">>, <<"baz">>], + #{<<"foo">> => <<"{\"bar\":\"{\\\"baz\\\":42}\"}">>} + ) + ). diff --git a/apps/emqx_utils/test/emqx_template_SUITE.erl b/apps/emqx_utils/test/emqx_template_SUITE.erl index 22ffff47c..f8355f769 100644 --- a/apps/emqx_utils/test/emqx_template_SUITE.erl +++ b/apps/emqx_utils/test/emqx_template_SUITE.erl @@ -25,7 +25,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). t_render(_) -> - Bindings = #{ + Context = #{ a => <<"1">>, b => 1, c => 1.0, @@ -38,15 +38,15 @@ t_render(_) -> ), ?assertEqual( {<<"a:1,b:1,c:1.0,d:{\"d1\":\"hi\"},d1:hi,l:[0,1,1000],u:utf-8 is ǝɹǝɥ"/utf8>>, []}, - render_string(Template, Bindings) + render_string(Template, Context) ). t_render_var_trans(_) -> - Bindings = #{a => <<"1">>, b => 1, c => #{prop => 1.0}}, + Context = #{a => <<"1">>, b => 1, c => #{prop => 1.0}}, Template = emqx_template:parse(<<"a:${a},b:${b},c:${c.prop}">>), {String, Errors} = emqx_template:render( Template, - Bindings, + Context, #{var_trans => fun(Name, _) -> "<" ++ Name ++ ">" end} ), ?assertEqual( @@ -55,7 +55,7 @@ t_render_var_trans(_) -> ). t_render_path(_) -> - Bindings = #{d => #{d1 => <<"hi">>}}, + Context = #{d => #{d1 => <<"hi">>}}, Template = emqx_template:parse(<<"d.d1:${d.d1}">>), ?assertEqual( ok, @@ -63,11 +63,11 @@ t_render_path(_) -> ), ?assertEqual( {<<"d.d1:hi">>, []}, - render_string(Template, Bindings) + render_string(Template, Context) ). t_render_custom_ph(_) -> - Bindings = #{a => <<"a">>, b => <<"b">>}, + Context = #{a => <<"a">>, b => <<"b">>}, Template = emqx_template:parse(<<"a:${a},b:${b}">>), ?assertEqual( {error, [{"b", disallowed}]}, @@ -75,21 +75,21 @@ t_render_custom_ph(_) -> ), ?assertEqual( <<"a:a,b:b">>, - render_strict_string(Template, Bindings) + render_strict_string(Template, Context) ). t_render_this(_) -> - Bindings = #{a => <<"a">>, b => [1, 2, 3]}, + Context = #{a => <<"a">>, b => [1, 2, 3]}, Template = emqx_template:parse(<<"this:${} / also:${.}">>), ?assertEqual(ok, emqx_template:validate(["."], Template)), ?assertEqual( % NOTE: order of the keys in the JSON object depends on the JSON encoder <<"this:{\"b\":[1,2,3],\"a\":\"a\"} / also:{\"b\":[1,2,3],\"a\":\"a\"}">>, - render_strict_string(Template, Bindings) + render_strict_string(Template, Context) ). t_render_missing_bindings(_) -> - Bindings = #{no => #{}, c => #{<<"c1">> => 42}}, + Context = #{no => #{}, c => #{<<"c1">> => 42}}, Template = emqx_template:parse( <<"a:${a},b:${b},c:${c.c1.c2},d:${d.d1},e:${no.such_atom_i_swear}">> ), @@ -101,7 +101,7 @@ t_render_missing_bindings(_) -> {"b", undefined}, {"a", undefined} ]}, - render_string(Template, Bindings) + render_string(Template, Context) ), ?assertError( [ @@ -111,7 +111,21 @@ t_render_missing_bindings(_) -> {"b", undefined}, {"a", undefined} ], - render_strict_string(Template, Bindings) + render_strict_string(Template, Context) + ). + +t_render_custom_bindings(_) -> + _ = erlang:put(a, <<"foo">>), + _ = erlang:put(b, #{<<"bar">> => #{atom => 42}}), + Template = emqx_template:parse( + <<"a:${a},b:${b.bar.atom},c:${c},oops:${b.bar.atom.oops}">> + ), + ?assertEqual( + {<<"a:foo,b:42,c:undefined,oops:undefined">>, [ + {"b.bar.atom.oops", {2, number}}, + {"c", undefined} + ]}, + render_string(Template, {?MODULE, []}) ). t_unparse(_) -> @@ -141,33 +155,33 @@ t_const(_) -> ). t_render_partial_ph(_) -> - Bindings = #{a => <<"1">>, b => 1, c => 1.0, d => #{d1 => <<"hi">>}}, + Context = #{a => <<"1">>, b => 1, c => 1.0, d => #{d1 => <<"hi">>}}, Template = emqx_template:parse(<<"a:$a,b:b},c:{c},d:${d">>), ?assertEqual( <<"a:$a,b:b},c:{c},d:${d">>, - render_strict_string(Template, Bindings) + render_strict_string(Template, Context) ). t_parse_escaped(_) -> - Bindings = #{a => <<"1">>, b => 1, c => "VAR"}, + Context = #{a => <<"1">>, b => 1, c => "VAR"}, Template = emqx_template:parse(<<"a:${a},b:${$}{b},c:${$}{${c}},lit:${$}{$}">>), ?assertEqual( <<"a:1,b:${b},c:${VAR},lit:${$}">>, - render_strict_string(Template, Bindings) + render_strict_string(Template, Context) ). t_parse_escaped_dquote(_) -> - Bindings = #{a => <<"1">>, b => 1}, + Context = #{a => <<"1">>, b => 1}, Template = emqx_template:parse(<<"a:\"${a}\",b:\"${$}{b}\"">>, #{ strip_double_quote => true }), ?assertEqual( <<"a:1,b:\"${b}\"">>, - render_strict_string(Template, Bindings) + render_strict_string(Template, Context) ). t_parse_sql_prepstmt(_) -> - Bindings = #{a => <<"1">>, b => 1, c => 1.0, d => #{d1 => <<"hi">>}}, + Context = #{a => <<"1">>, b => 1, c => 1.0, d => #{d1 => <<"hi">>}}, {PrepareStatement, RowTemplate} = emqx_template_sql:parse_prepstmt(<<"a:${a},b:${b},c:${c},d:${d}">>, #{ parameters => '?' @@ -175,11 +189,11 @@ t_parse_sql_prepstmt(_) -> ?assertEqual(<<"a:?,b:?,c:?,d:?">>, bin(PrepareStatement)), ?assertEqual( {[<<"1">>, 1, 1.0, <<"{\"d1\":\"hi\"}">>], _Errors = []}, - emqx_template_sql:render_prepstmt(RowTemplate, Bindings) + emqx_template_sql:render_prepstmt(RowTemplate, Context) ). t_parse_sql_prepstmt_n(_) -> - Bindings = #{a => undefined, b => true, c => atom, d => #{d1 => 42.1337}}, + Context = #{a => undefined, b => true, c => atom, d => #{d1 => 42.1337}}, {PrepareStatement, RowTemplate} = emqx_template_sql:parse_prepstmt(<<"a:${a},b:${b},c:${c},d:${d}">>, #{ parameters => '$n' @@ -187,7 +201,7 @@ t_parse_sql_prepstmt_n(_) -> ?assertEqual(<<"a:$1,b:$2,c:$3,d:$4">>, bin(PrepareStatement)), ?assertEqual( [null, true, <<"atom">>, <<"{\"d1\":42.1337}">>], - emqx_template_sql:render_prepstmt_strict(RowTemplate, Bindings) + emqx_template_sql:render_prepstmt_strict(RowTemplate, Context) ). t_parse_sql_prepstmt_colon(_) -> @@ -198,14 +212,14 @@ t_parse_sql_prepstmt_colon(_) -> ?assertEqual(<<"a=:1,b=:2,c=:3,d=:4">>, bin(PrepareStatement)). t_parse_sql_prepstmt_partial_ph(_) -> - Bindings = #{a => <<"1">>, b => 1, c => 1.0, d => #{d1 => <<"hi">>}}, + Context = #{a => <<"1">>, b => 1, c => 1.0, d => #{d1 => <<"hi">>}}, {PrepareStatement, RowTemplate} = emqx_template_sql:parse_prepstmt(<<"a:$a,b:b},c:{c},d:${d">>, #{parameters => '?'}), ?assertEqual(<<"a:$a,b:b},c:{c},d:${d">>, bin(PrepareStatement)), - ?assertEqual([], emqx_template_sql:render_prepstmt_strict(RowTemplate, Bindings)). + ?assertEqual([], emqx_template_sql:render_prepstmt_strict(RowTemplate, Context)). t_render_sql(_) -> - Bindings = #{ + Context = #{ a => <<"1">>, b => 1, c => 1.0, @@ -216,17 +230,17 @@ t_render_sql(_) -> Template = emqx_template:parse(<<"a:${a},b:${b},c:${c},d:${d},n:${n},u:${u}">>), ?assertMatch( {_String, _Errors = []}, - emqx_template_sql:render(Template, Bindings, #{}) + emqx_template_sql:render(Template, Context, #{}) ), ?assertEqual( <<"a:'1',b:1,c:1.0,d:'{\"d1\":\"hi\"}',n:NULL,u:'utf8\\'s cool 🐸'"/utf8>>, - bin(emqx_template_sql:render_strict(Template, Bindings, #{})) + bin(emqx_template_sql:render_strict(Template, Context, #{})) ). t_render_mysql(_) -> %% with apostrophes %% https://github.com/emqx/emqx/issues/4135 - Bindings = #{ + Context = #{ a => <<"1''2">>, b => 1, c => 1.0, @@ -245,13 +259,13 @@ t_render_mysql(_) -> "e:'\\\\\\0💩',f:0x6E6F6E2D75746638DCC900,g:'utf8\\'s cool 🐸',"/utf8, "h:'imgood'" >>, - bin(emqx_template_sql:render_strict(Template, Bindings, #{escaping => mysql})) + bin(emqx_template_sql:render_strict(Template, Context, #{escaping => mysql})) ). t_render_cql(_) -> %% with apostrophes for cassandra %% https://github.com/emqx/emqx/issues/4148 - Bindings = #{ + Context = #{ a => <<"1''2">>, b => 1, c => 1.0, @@ -260,7 +274,7 @@ t_render_cql(_) -> Template = emqx_template:parse(<<"a:${a},b:${b},c:${c},d:${d}">>), ?assertEqual( <<"a:'1''''2',b:1,c:1.0,d:'{\"d1\":\"someone''s phone\"}'">>, - bin(emqx_template_sql:render_strict(Template, Bindings, #{escaping => cql})) + bin(emqx_template_sql:render_strict(Template, Context, #{escaping => cql})) ). t_render_sql_custom_ph(_) -> @@ -273,7 +287,7 @@ t_render_sql_custom_ph(_) -> ?assertEqual(<<"a:$1,b:$2">>, bin(PrepareStatement)). t_render_sql_strip_double_quote(_) -> - Bindings = #{a => <<"a">>, b => <<"b">>}, + Context = #{a => <<"a">>, b => <<"b">>}, %% no strip_double_quote option: "${key}" -> "value" {PrepareStatement1, RowTemplate1} = emqx_template_sql:parse_prepstmt( @@ -283,7 +297,7 @@ t_render_sql_strip_double_quote(_) -> ?assertEqual(<<"a:\"$1\",b:\"$2\"">>, bin(PrepareStatement1)), ?assertEqual( [<<"a">>, <<"b">>], - emqx_template_sql:render_prepstmt_strict(RowTemplate1, Bindings) + emqx_template_sql:render_prepstmt_strict(RowTemplate1, Context) ), %% strip_double_quote = true: "${key}" -> value @@ -294,11 +308,11 @@ t_render_sql_strip_double_quote(_) -> ?assertEqual(<<"a:$1,b:$2">>, bin(PrepareStatement2)), ?assertEqual( [<<"a">>, <<"b">>], - emqx_template_sql:render_prepstmt_strict(RowTemplate2, Bindings) + emqx_template_sql:render_prepstmt_strict(RowTemplate2, Context) ). t_render_tmpl_deep(_) -> - Bindings = #{a => <<"1">>, b => 1, c => 1.0, d => #{d1 => <<"hi">>}}, + Context = #{a => <<"1">>, b => 1, c => 1.0, d => #{d1 => <<"hi">>}}, Template = emqx_template:parse_deep( #{<<"${a}">> => [<<"$${b}">>, "c", 2, 3.0, '${d}', {[<<"${c}">>, <<"${$}{d}">>], 0}]} @@ -311,7 +325,7 @@ t_render_tmpl_deep(_) -> ?assertEqual( #{<<"1">> => [<<"$1">>, "c", 2, 3.0, '${d}', {[<<"1.0">>, <<"${d}">>], 0}]}, - emqx_template:render_strict(Template, Bindings) + emqx_template:render_strict(Template, Context) ). t_unparse_tmpl_deep(_) -> @@ -321,12 +335,22 @@ t_unparse_tmpl_deep(_) -> %% -render_string(Template, Bindings) -> - {String, Errors} = emqx_template:render(Template, Bindings), +render_string(Template, Context) -> + {String, Errors} = emqx_template:render(Template, Context), {bin(String), Errors}. -render_strict_string(Template, Bindings) -> - bin(emqx_template:render_strict(Template, Bindings)). +render_strict_string(Template, Context) -> + bin(emqx_template:render_strict(Template, Context)). bin(String) -> unicode:characters_to_binary(String). + +%% Access module API + +lookup([], _) -> + {error, undefined}; +lookup([Prop | Rest], _) -> + case erlang:get(binary_to_atom(Prop)) of + undefined -> {error, undefined}; + Value -> emqx_template:lookup_var(Rest, Value) + end. From b5b6c3f8cc6ee7a446a7b92b917df30b4fdc0fa1 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 20 Oct 2023 18:25:47 +0700 Subject: [PATCH 18/21] fix(tpl): ensure full backward compat in basic connectors --- .../src/emqx_bridge_http_connector.erl | 2 +- .../test/emqx_bridge_mysql_SUITE.erl | 29 ++++++++++++++----- apps/emqx_mysql/src/emqx_mysql.erl | 15 +++++++--- apps/emqx_postgresql/src/emqx_postgresql.erl | 19 +++++++----- apps/emqx_utils/src/emqx_placeholder.erl | 6 ++-- apps/emqx_utils/src/emqx_template_sql.erl | 7 ++++- apps/emqx_utils/src/emqx_utils_sql.erl | 9 ++++-- apps/emqx_utils/test/emqx_template_SUITE.erl | 4 +++ 8 files changed, 64 insertions(+), 27 deletions(-) diff --git a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl index 88f55af52..b2f876d21 100644 --- a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl +++ b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl @@ -573,7 +573,7 @@ render_headers(HeaderTks, Msg) -> render_template(Template, Msg) -> % NOTE: ignoring errors here, missing variables will be rendered as `"undefined"`. - {String, _Errors} = emqx_template:render(Template, Msg), + {String, _Errors} = emqx_template:render(Template, {emqx_jsonish, Msg}), String. render_template_string(Template, Msg) -> diff --git a/apps/emqx_bridge_mysql/test/emqx_bridge_mysql_SUITE.erl b/apps/emqx_bridge_mysql/test/emqx_bridge_mysql_SUITE.erl index 2eeccfd77..a34b65ede 100644 --- a/apps/emqx_bridge_mysql/test/emqx_bridge_mysql_SUITE.erl +++ b/apps/emqx_bridge_mysql/test/emqx_bridge_mysql_SUITE.erl @@ -565,6 +565,7 @@ t_simple_sql_query(Config) -> ok. t_missing_data(Config) -> + BatchSize = ?config(batch_size, Config), ?assertMatch( {ok, _}, create_bridge(Config) @@ -575,13 +576,27 @@ t_missing_data(Config) -> ), send_message(Config, #{}), {ok, [Event]} = snabbkaffe:receive_events(SRef), - ?assertMatch( - #{ - result := - {error, {unrecoverable_error, {1048, _, <<"Column 'arrived' cannot be null">>}}} - }, - Event - ), + case BatchSize of + N when N > 1 -> + ?assertMatch( + #{ + result := + {error, + {unrecoverable_error, + {1292, _, <<"Truncated incorrect DOUBLE value: 'undefined'">>}}} + }, + Event + ); + 1 -> + ?assertMatch( + #{ + result := + {error, + {unrecoverable_error, {1048, _, <<"Column 'arrived' cannot be null">>}}} + }, + Event + ) + end, ok. t_bad_sql_parameter(Config) -> diff --git a/apps/emqx_mysql/src/emqx_mysql.erl b/apps/emqx_mysql/src/emqx_mysql.erl index e052b9b89..d8b7994ab 100644 --- a/apps/emqx_mysql/src/emqx_mysql.erl +++ b/apps/emqx_mysql/src/emqx_mysql.erl @@ -426,8 +426,12 @@ proc_sql_params(TypeOrKey, SQLOrData, Params, #{query_templates := Templates}) - undefined -> {SQLOrData, Params}; {_InsertPart, RowTemplate} -> - % NOTE: ignoring errors here, missing variables are set to `null`. - {Row, _Errors} = emqx_template_sql:render_prepstmt(RowTemplate, SQLOrData), + % NOTE + % Ignoring errors here, missing variables are set to `null`. + {Row, _Errors} = emqx_template_sql:render_prepstmt( + RowTemplate, + {emqx_jsonish, SQLOrData} + ), {TypeOrKey, Row} end. @@ -437,8 +441,11 @@ on_batch_insert(InstId, BatchReqs, {InsertPart, RowTemplate}, State) -> on_sql_query(InstId, query, Query, no_params, default_timeout, State). render_row(RowTemplate, Data) -> - % NOTE: ignoring errors here, missing variables are set to "NULL". - {Row, _Errors} = emqx_template_sql:render(RowTemplate, Data, #{escaping => mysql}), + % NOTE + % Ignoring errors here, missing variables are set to "'undefined'" due to backward + % compatibility requirements. + RenderOpts = #{escaping => mysql, undefined => <<"undefined">>}, + {Row, _Errors} = emqx_template_sql:render(RowTemplate, {emqx_jsonish, Data}, RenderOpts), Row. on_sql_query( diff --git a/apps/emqx_postgresql/src/emqx_postgresql.erl b/apps/emqx_postgresql/src/emqx_postgresql.erl index 3f7b43c79..814d8a074 100644 --- a/apps/emqx_postgresql/src/emqx_postgresql.erl +++ b/apps/emqx_postgresql/src/emqx_postgresql.erl @@ -313,8 +313,8 @@ do_check_prepares( case validate_table_existence(WorkerPids, SQL) of ok -> ok; - {error, undefined_table} -> - {error, {undefined_table, State}} + {error, Reason} -> + {error, Reason} end; do_check_prepares(#{prepares := Prepares}) when is_map(Prepares) -> ok; @@ -433,7 +433,7 @@ parse_prepare_sql(Key, Query, Acc) -> render_prepare_sql_row(RowTemplate, Data) -> % NOTE: ignoring errors here, missing variables will be replaced with `null`. - {Row, _Errors} = emqx_template_sql:render_prepstmt(RowTemplate, Data), + {Row, _Errors} = emqx_template_sql:render_prepstmt(RowTemplate, {emqx_jsonish, Data}), Row. init_prepare(State = #{query_templates := Templates}) when map_size(Templates) == 0 -> @@ -443,10 +443,13 @@ init_prepare(State = #{}) -> {ok, PrepStatements} -> State#{prepares => PrepStatements}; Error -> - ?SLOG(error, maps:merge( - #{msg => <<"postgresql_init_prepare_statement_failed">>}, - translate_to_log_context(Error) - )), + ?SLOG( + error, + maps:merge( + #{msg => <<"postgresql_init_prepare_statement_failed">>}, + translate_to_log_context(Error) + ) + ), %% mark the prepares failed State#{prepares => Error} end. @@ -484,7 +487,7 @@ prepare_sql_to_conn(Conn, Prepares) -> prepare_sql_to_conn(Conn, [], Statements) when is_pid(Conn) -> {ok, Statements}; prepare_sql_to_conn(Conn, [{Key, {SQL, _RowTemplate}} | Rest], Statements) when is_pid(Conn) -> - LogMeta = #{msg => "PostgreSQL Prepare Statement", name => Key, sql => SQL}, + LogMeta = #{msg => "postgresql_prepare_statement", name => Key, sql => SQL}, ?SLOG(info, LogMeta), case epgsql:parse2(Conn, Key, SQL, []) of {ok, Statement} -> diff --git a/apps/emqx_utils/src/emqx_placeholder.erl b/apps/emqx_utils/src/emqx_placeholder.erl index 4d386840f..90df6003b 100644 --- a/apps/emqx_utils/src/emqx_placeholder.erl +++ b/apps/emqx_utils/src/emqx_placeholder.erl @@ -249,15 +249,15 @@ bin(Val) -> emqx_utils_conv:bin(Val). -spec quote_sql(_Value) -> iolist(). quote_sql(Str) -> - emqx_utils_sql:to_sql_string(Str, #{escaping => sql}). + emqx_utils_sql:to_sql_string(Str, #{escaping => sql, undefined => <<"undefined">>}). -spec quote_cql(_Value) -> iolist(). quote_cql(Str) -> - emqx_utils_sql:to_sql_string(Str, #{escaping => cql}). + emqx_utils_sql:to_sql_string(Str, #{escaping => cql, undefined => <<"undefined">>}). -spec quote_mysql(_Value) -> iolist(). quote_mysql(Str) -> - emqx_utils_sql:to_sql_string(Str, #{escaping => mysql}). + emqx_utils_sql:to_sql_string(Str, #{escaping => mysql, undefined => <<"undefined">>}). lookup_var(Var, Value) when Var == ?PH_VAR_THIS orelse Var == [] -> Value; diff --git a/apps/emqx_utils/src/emqx_template_sql.erl b/apps/emqx_utils/src/emqx_template_sql.erl index 4e9d8f622..9b2c1d55c 100644 --- a/apps/emqx_utils/src/emqx_template_sql.erl +++ b/apps/emqx_utils/src/emqx_template_sql.erl @@ -40,7 +40,12 @@ }. -type render_opts() :: #{ - escaping => mysql | cql | sql + %% String escaping rules to use. + %% Default: `sql` (generic) + escaping => sql | mysql | cql, + %% Value to map `undefined` to, either to NULLs or to arbitrary strings. + %% Default: `null` + undefined => null | unicode:chardata() }. -define(TEMPLATE_PARSE_OPTS, [strip_double_quote]). diff --git a/apps/emqx_utils/src/emqx_utils_sql.erl b/apps/emqx_utils/src/emqx_utils_sql.erl index 12aac6464..9ce9e576d 100644 --- a/apps/emqx_utils/src/emqx_utils_sql.erl +++ b/apps/emqx_utils/src/emqx_utils_sql.erl @@ -82,8 +82,13 @@ to_sql_value(Map) when is_map(Map) -> emqx_utils_json:encode(Map). %% SQL statements. The value is escaped if necessary. -spec to_sql_string(term(), Options) -> unicode:chardata() when Options :: #{ - escaping => cql | mysql | sql + escaping => mysql | sql | cql, + undefined => null | unicode:chardata() }. +to_sql_string(undefined, #{undefined := Str} = Opts) when Str =/= null -> + to_sql_string(Str, Opts); +to_sql_string(undefined, #{}) -> + <<"NULL">>; to_sql_string(String, #{escaping := mysql}) when is_binary(String) -> try escape_mysql(String) @@ -99,8 +104,6 @@ to_sql_string(Term, #{}) -> maybe_escape(Term, fun escape_sql/1). -spec maybe_escape(_Value, fun((binary()) -> iodata())) -> unicode:chardata(). -maybe_escape(undefined, _EscapeFun) -> - <<"NULL">>; maybe_escape(Str, EscapeFun) when is_binary(Str) -> EscapeFun(Str); maybe_escape(Str, EscapeFun) when is_list(Str) -> diff --git a/apps/emqx_utils/test/emqx_template_SUITE.erl b/apps/emqx_utils/test/emqx_template_SUITE.erl index f8355f769..4dfe5de2e 100644 --- a/apps/emqx_utils/test/emqx_template_SUITE.erl +++ b/apps/emqx_utils/test/emqx_template_SUITE.erl @@ -235,6 +235,10 @@ t_render_sql(_) -> ?assertEqual( <<"a:'1',b:1,c:1.0,d:'{\"d1\":\"hi\"}',n:NULL,u:'utf8\\'s cool 🐸'"/utf8>>, bin(emqx_template_sql:render_strict(Template, Context, #{})) + ), + ?assertEqual( + <<"a:'1',b:1,c:1.0,d:'{\"d1\":\"hi\"}',n:'undefined',u:'utf8\\'s cool 🐸'"/utf8>>, + bin(emqx_template_sql:render_strict(Template, Context, #{undefined => "undefined"})) ). t_render_mysql(_) -> From e521a9f5fc9adaf3a3bbc35bf44ff26358404603 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 2 Nov 2023 15:40:20 +0700 Subject: [PATCH 19/21] fix(utils): denote `emqx_jsonish` follows access module behaviour Defined in `emqx_template`. --- apps/emqx_utils/src/emqx_jsonish.erl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/emqx_utils/src/emqx_jsonish.erl b/apps/emqx_utils/src/emqx_jsonish.erl index b2d92c7fc..ef26da1d8 100644 --- a/apps/emqx_utils/src/emqx_jsonish.erl +++ b/apps/emqx_utils/src/emqx_jsonish.erl @@ -16,6 +16,7 @@ -module(emqx_jsonish). +-behaviour(emqx_template). -export([lookup/2]). -export_type([t/0]). @@ -53,11 +54,11 @@ lookup(Loc, Decoded, [Prop | Rest], Jsonish) when is_map(Jsonish) -> {error, Reason} -> {error, Reason} end; -lookup(Loc, _Decoded = false, Rest, Json) when is_binary(Json) -> +lookup(Loc, _Decoded = false, Props, Json) when is_binary(Json) -> try emqx_utils_json:decode(Json) of Value -> % NOTE: This is intentional, we don't want to parse nested JSON. - lookup(Loc, true, Rest, Value) + lookup(Loc, true, Props, Value) catch error:_ -> {error, {Loc, binary}} From f1847fe494c18b4731890955dce974d1eb1027ae Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 2 Nov 2023 15:41:42 +0700 Subject: [PATCH 20/21] chore(tpl): drop unnecessary binding --- apps/emqx_utils/src/emqx_template.erl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/emqx_utils/src/emqx_template.erl b/apps/emqx_utils/src/emqx_template.erl index 1ccc74c50..ac330becf 100644 --- a/apps/emqx_utils/src/emqx_template.erl +++ b/apps/emqx_utils/src/emqx_template.erl @@ -117,8 +117,7 @@ parse(String, Opts) -> <<"(" ?RE_PLACEHOLDER "|" ?RE_ESCAPE ")">> end, Splits = re:split(String, RE, [{return, binary}, group, trim, unicode]), - Components = lists:flatmap(fun parse_split/1, Splits), - Components. + lists:flatmap(fun parse_split/1, Splits). parse_split([Part, _PH, Var, <<>>]) -> % Regular placeholder From 729c6edff632534cde79d74e412df0d84d0893c5 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 2 Nov 2023 17:08:28 +0700 Subject: [PATCH 21/21] chore(repub): simplify error handling in MQTT props templating --- .../src/emqx_rule_actions.erl | 78 ++++++++----------- 1 file changed, 31 insertions(+), 47 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_actions.erl b/apps/emqx_rule_engine/src/emqx_rule_actions.erl index d0810eb84..29dbc2315 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_actions.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_actions.erl @@ -255,31 +255,33 @@ render_pub_props(UserPropertiesTemplate, Selected, Env) -> end, #{'User-Property' => UserProperties}. +%% + +-define(BADPROP(K, REASON, ENV, DATA), + ?SLOG( + debug, + DATA#{ + msg => "bad_mqtt_property_value_ignored", + rule_id => emqx_utils_maps:deep_get([metadata, rule_id], ENV, undefined), + reason => REASON, + property => K + } + ) +). + render_mqtt_properties(MQTTPropertiesTemplate, Selected, Env) -> - #{metadata := #{rule_id := RuleId}} = Env, MQTTProperties = - maps:fold( - fun(K, Template, Acc) -> + maps:map( + fun(K, Template) -> {V, Errors} = render_template(Template, Selected), - NAcc = Acc#{K => iolist_to_binary(V)}, case Errors of [] -> ok; Errors -> - ?SLOG( - debug, - #{ - msg => "bad_mqtt_property_value_ignored", - rule_id => RuleId, - reason => Errors, - property => K, - selected => Selected - } - ) + ?BADPROP(K, Errors, Env, #{selected => Selected}) end, - NAcc + iolist_to_binary(V) end, - #{}, MQTTPropertiesTemplate ), coerce_properties_values(MQTTProperties, Env). @@ -294,42 +296,24 @@ ensure_int(B) when is_binary(B) -> ensure_int(I) when is_integer(I) -> I. -coerce_properties_values(MQTTProperties, #{metadata := #{rule_id := RuleId}}) -> - maps:fold( - fun(K, V0, Acc) -> +coerce_properties_values(MQTTProperties, Env) -> + maps:filtermap( + fun(K, V) -> try - V = encode_mqtt_property(K, V0), - Acc#{K => V} + {true, encode_mqtt_property(K, V)} catch - throw:bad_integer -> - ?SLOG( - debug, - #{ - msg => "bad_mqtt_property_value_ignored", - rule_id => RuleId, - reason => bad_integer, - property => K, - value => V0 - } - ), - Acc; + throw:Reason -> + ?BADPROP(K, Reason, Env, #{value => V}), + false; Kind:Reason:Stacktrace -> - ?SLOG( - debug, - #{ - msg => "bad_mqtt_property_value_ignored", - rule_id => RuleId, - exception => Kind, - reason => Reason, - property => K, - value => V0, - stacktrace => Stacktrace - } - ), - Acc + ?BADPROP(K, Reason, Env, #{ + value => V, + exception => Kind, + stacktrace => Stacktrace + }), + false end end, - #{}, MQTTProperties ).