feat(tpl): use `emqx_connector_template` in `emqx_authn`, `emqx_authz`

This slightly changes semantics: now the attempt to create authenticator
with illegal bindings in templates will fail, instead of treating them
as literals. The runtime behaviour on the other hand should be the same.

apps/emqx/include/emqx_placeholder.hrl

@@ -19,67 +19,79 @@
 
 -define(PH_VAR_THIS, <<"$_THIS_">>).
 
--define(PH(Type), <<"${", Type/binary, "}">>).
+-define(PH(Var), <<"${" Var "}">>).
 
 %% action: publish/subscribe
--define(PH_ACTION, <<"${action}">>).
+-define(VAR_ACTION, "action").
+-define(PH_ACTION, ?PH(?VAR_ACTION)).
 
 %% cert
--define(PH_CERT_SUBJECT, <<"${cert_subject}">>).
+-define(VAR_CERT_SUBJECT, "cert_subject").
--define(PH_CERT_CN_NAME, <<"${cert_common_name}">>).
+-define(VAR_CERT_CN_NAME, "cert_common_name").
+-define(PH_CERT_SUBJECT, ?PH(?VAR_CERT_SUBJECT)).
+-define(PH_CERT_CN_NAME, ?PH(?VAR_CERT_CN_NAME)).
 
 %% MQTT
--define(PH_PASSWORD, <<"${password}">>).
+-define(VAR_PASSWORD, "password").
--define(PH_CLIENTID, <<"${clientid}">>).
+-define(VAR_CLIENTID, "clientid").
--define(PH_FROM_CLIENTID, <<"${from_clientid}">>).
+-define(VAR_USERNAME, "username").
--define(PH_USERNAME, <<"${username}">>).
+-define(VAR_TOPIC, "topic").
--define(PH_FROM_USERNAME, <<"${from_username}">>).
+-define(PH_PASSWORD, ?PH(?VAR_PASSWORD)).
--define(PH_TOPIC, <<"${topic}">>).
+-define(PH_CLIENTID, ?PH(?VAR_CLIENTID)).
+-define(PH_FROM_CLIENTID, ?PH("from_clientid")).
+-define(PH_USERNAME, ?PH(?VAR_USERNAME)).
+-define(PH_FROM_USERNAME, ?PH("from_username")).
+-define(PH_TOPIC, ?PH(?VAR_TOPIC)).
 %% MQTT payload
--define(PH_PAYLOAD, <<"${payload}">>).
+-define(PH_PAYLOAD, ?PH("payload")).
 %% client IPAddress
--define(PH_PEERHOST, <<"${peerhost}">>).
+-define(VAR_PEERHOST, "peerhost").
+-define(PH_PEERHOST, ?PH(?VAR_PEERHOST)).
 %% ip & port
--define(PH_HOST, <<"${host}">>).
+-define(PH_HOST, ?PH("host")).
--define(PH_PORT, <<"${port}">>).
+-define(PH_PORT, ?PH("port")).
 %% Enumeration of message QoS 0,1,2
--define(PH_QOS, <<"${qos}">>).
+-define(VAR_QOS, "qos").
--define(PH_FLAGS, <<"${flags}">>).
+-define(PH_QOS, ?PH(?VAR_QOS)).
+-define(PH_FLAGS, ?PH("flags")).
 %% Additional data related to process within the MQTT message
--define(PH_HEADERS, <<"${headers}">>).
+-define(PH_HEADERS, ?PH("headers")).
 %% protocol name
--define(PH_PROTONAME, <<"${proto_name}">>).
+-define(VAR_PROTONAME, "proto_name").
+-define(PH_PROTONAME, ?PH(?VAR_PROTONAME)).
 %% protocol version
--define(PH_PROTOVER, <<"${proto_ver}">>).
+-define(PH_PROTOVER, ?PH("proto_ver")).
 %% MQTT keepalive interval
--define(PH_KEEPALIVE, <<"${keepalive}">>).
+-define(PH_KEEPALIVE, ?PH("keepalive")).
 %% MQTT clean_start
--define(PH_CLEAR_START, <<"${clean_start}">>).
+-define(PH_CLEAR_START, ?PH("clean_start")).
 %% MQTT Session Expiration time
--define(PH_EXPIRY_INTERVAL, <<"${expiry_interval}">>).
+-define(PH_EXPIRY_INTERVAL, ?PH("expiry_interval")).
 
 %% Time when PUBLISH message reaches Broker (ms)
--define(PH_PUBLISH_RECEIVED_AT, <<"${publish_received_at}">>).
+-define(PH_PUBLISH_RECEIVED_AT, ?PH("publish_received_at")).
 %% Mountpoint for bridging messages
--define(PH_MOUNTPOINT, <<"${mountpoint}">>).
+-define(VAR_MOUNTPOINT, "mountpoint").
+-define(PH_MOUNTPOINT, ?PH(?VAR_MOUNTPOINT)).
 %% IPAddress and Port of terminal
--define(PH_PEERNAME, <<"${peername}">>).
+-define(PH_PEERNAME, ?PH("peername")).
 %% IPAddress and Port listened by emqx
--define(PH_SOCKNAME, <<"${sockname}">>).
+-define(PH_SOCKNAME, ?PH("sockname")).
 %% whether it is MQTT bridge connection
--define(PH_IS_BRIDGE, <<"${is_bridge}">>).
+-define(PH_IS_BRIDGE, ?PH("is_bridge")).
 %% Terminal connection completion time (s)
--define(PH_CONNECTED_AT, <<"${connected_at}">>).
+-define(PH_CONNECTED_AT, ?PH("connected_at")).
 %% Event trigger time(millisecond)
--define(PH_TIMESTAMP, <<"${timestamp}">>).
+-define(PH_TIMESTAMP, ?PH("timestamp")).
 %% Terminal disconnection completion time (s)
--define(PH_DISCONNECTED_AT, <<"${disconnected_at}">>).
+-define(PH_DISCONNECTED_AT, ?PH("disconnected_at")).
 
--define(PH_NODE, <<"${node}">>).
+-define(PH_NODE, ?PH("node")).
--define(PH_REASON, <<"${reason}">>).
+-define(PH_REASON, ?PH("reason")).
 
--define(PH_ENDPOINT_NAME, <<"${endpoint_name}">>).
+-define(PH_ENDPOINT_NAME, ?PH("endpoint_name")).
--define(PH_RETAIN, <<"${retain}">>).
+-define(VAR_RETAIN, "retain").
+-define(PH_RETAIN, ?PH(?VAR_RETAIN)).
 
 %% sync change these place holder with binary def.
 -define(PH_S_ACTION, "${action}").

apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl

@@ -45,12 +45,12 @@
 ]).
 
 -define(AUTHN_PLACEHOLDERS, [
-    ?PH_USERNAME,
+    <<?VAR_USERNAME>>,
-    ?PH_CLIENTID,
+    <<?VAR_CLIENTID>>,
-    ?PH_PASSWORD,
+    <<?VAR_PASSWORD>>,
-    ?PH_PEERHOST,
+    <<?VAR_PEERHOST>>,
-    ?PH_CERT_SUBJECT,
+    <<?VAR_CERT_SUBJECT>>,
-    ?PH_CERT_CN_NAME
+    <<?VAR_CERT_CN_NAME>>
 ]).
 
 -define(DEFAULT_RESOURCE_OPTS, #{
@@ -107,48 +107,62 @@ check_password_from_selected_map(Algorithm, Selected, Password) ->
     end.
 
 parse_deep(Template) ->
-    emqx_placeholder:preproc_tmpl_deep(Template, #{placeholders => ?AUTHN_PLACEHOLDERS}).
+    Result = emqx_connector_template:parse_deep(Template),
+    ok = emqx_connector_template:validate(?AUTHN_PLACEHOLDERS, Result),
+    Result.
 
 parse_str(Template) ->
-    emqx_placeholder:preproc_tmpl(Template, #{placeholders => ?AUTHN_PLACEHOLDERS}).
+    Result = emqx_connector_template:parse(Template),
+    ok = emqx_connector_template:validate(?AUTHN_PLACEHOLDERS, Result),
+    Result.
 
 parse_sql(Template, ReplaceWith) ->
-    emqx_placeholder:preproc_sql(
+    {Statement, Result} = emqx_connector_template_sql:parse_prepstmt(
         Template,
-        #{
+        #{parameters => ReplaceWith, strip_double_quote => true}
-            replace_with => ReplaceWith,
+    ),
-            placeholders => ?AUTHN_PLACEHOLDERS,
+    ok = emqx_connector_template:validate(?AUTHN_PLACEHOLDERS, Result),
-            strip_double_quote => true
+    {Statement, Result}.
-        }
-    ).
 
 render_deep(Template, Credential) ->
-    emqx_placeholder:proc_tmpl_deep(
+    % NOTE
+    % Ignoring errors here, undefined bindings will be replaced with empty string.
+    {Term, _Errors} = emqx_connector_template:render(
         Template,
         mapping_credential(Credential),
-        #{return => full_binary, var_trans => fun handle_var/2}
+        #{var_trans => fun handle_var/2}
-    ).
+    ),
+    Term.
 
 render_str(Template, Credential) ->
-    emqx_placeholder:proc_tmpl(
+    % NOTE
+    % Ignoring errors here, undefined bindings will be replaced with empty string.
+    {String, _Errors} = emqx_connector_template:render(
         Template,
         mapping_credential(Credential),
-        #{return => full_binary, var_trans => fun handle_var/2}
+        #{var_trans => fun handle_var/2}
-    ).
+    ),
+    unicode:characters_to_binary(String).
 
 render_urlencoded_str(Template, Credential) ->
-    emqx_placeholder:proc_tmpl(
+    % NOTE
+    % Ignoring errors here, undefined bindings will be replaced with empty string.
+    {String, _Errors} = emqx_connector_template:render(
         Template,
         mapping_credential(Credential),
-        #{return => full_binary, var_trans => fun urlencode_var/2}
+        #{var_trans => fun urlencode_var/2}
-    ).
+    ),
+    unicode:characters_to_binary(String).
 
 render_sql_params(ParamList, Credential) ->
-    emqx_placeholder:proc_tmpl(
+    % NOTE
+    % Ignoring errors here, undefined bindings will be replaced with empty string.
+    {Row, _Errors} = emqx_connector_template:render(
         ParamList,
         mapping_credential(Credential),
-        #{return => rawlist, var_trans => fun handle_sql_var/2}
+        #{var_trans => fun handle_sql_var/2}
-    ).
+    ),
+    Row.
 
 is_superuser(#{<<"is_superuser">> := Value}) ->
     #{is_superuser => to_bool(Value)};
@@ -272,19 +286,19 @@ without_password(Credential, [Name | Rest]) ->
 urlencode_var(Var, Value) ->
     emqx_http_lib:uri_encode(handle_var(Var, Value)).
 
-handle_var(_Name, undefined) ->
+handle_var(_, undefined) ->
     <<>>;
 handle_var([<<"peerhost">>], PeerHost) ->
-    emqx_placeholder:bin(inet:ntoa(PeerHost));
+    emqx_connector_template:to_string(inet:ntoa(PeerHost));
 handle_var(_, Value) ->
-    emqx_placeholder:bin(Value).
+    emqx_connector_template:to_string(Value).
 
-handle_sql_var(_Name, undefined) ->
+handle_sql_var(_, undefined) ->
     <<>>;
 handle_sql_var([<<"peerhost">>], PeerHost) ->
-    emqx_placeholder:bin(inet:ntoa(PeerHost));
+    emqx_connector_sql:to_sql_value(inet:ntoa(PeerHost));
 handle_sql_var(_, Value) ->
-    emqx_placeholder:sql_data(Value).
+    emqx_connector_sql:to_sql_value(Value).
 
 mapping_credential(C = #{cn := CN, dn := DN}) ->
     C#{cert_common_name => CN, cert_subject => DN};

apps/emqx_auth/src/emqx_authz/emqx_authz_rule.erl

@@ -183,19 +183,15 @@ compile_topic(<<"eq ", Topic/binary>>) ->
 compile_topic({eq, Topic}) ->
     {eq, emqx_topic:words(bin(Topic))};
 compile_topic(Topic) ->
-    TopicBin = bin(Topic),
+    Template = emqx_connector_template:parse(Topic),
-    case
+    ok = emqx_connector_template:validate([<<?VAR_USERNAME>>, <<?VAR_CLIENTID>>], Template),
-        emqx_placeholder:preproc_tmpl(
+    case emqx_connector_template:trivial(Template) of
-            TopicBin,
+        true -> emqx_topic:words(bin(Topic));
-            #{placeholders => [?PH_USERNAME, ?PH_CLIENTID]}
+        false -> {pattern, Template}
-        )
-    of
-        [{str, _}] -> emqx_topic:words(TopicBin);
-        Tokens -> {pattern, Tokens}
     end.
 
 bin(L) when is_list(L) ->
-    list_to_binary(L);
+    unicode:characters_to_binary(L);
 bin(B) when is_binary(B) ->
     B.
 
@@ -307,7 +303,7 @@ match_who(_, _) ->
 match_topics(_ClientInfo, _Topic, []) ->
     false;
 match_topics(ClientInfo, Topic, [{pattern, PatternFilter} | Filters]) ->
-    TopicFilter = emqx_placeholder:proc_tmpl(PatternFilter, ClientInfo),
+    TopicFilter = bin(emqx_connector_template:render_strict(PatternFilter, ClientInfo)),
     match_topic(emqx_topic:words(Topic), emqx_topic:words(TopicFilter)) orelse
         match_topics(ClientInfo, Topic, Filters);
 match_topics(ClientInfo, Topic, [TopicFilter | Filters]) ->

apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl

@@ -108,48 +108,62 @@ update_config(Path, ConfigRequest) ->
     }).
 
 parse_deep(Template, PlaceHolders) ->
-    emqx_placeholder:preproc_tmpl_deep(Template, #{placeholders => PlaceHolders}).
+    Result = emqx_connector_template:parse_deep(Template),
+    ok = emqx_connector_template:validate(PlaceHolders, Result),
+    Result.
 
 parse_str(Template, PlaceHolders) ->
-    emqx_placeholder:preproc_tmpl(Template, #{placeholders => PlaceHolders}).
+    Result = emqx_connector_template:parse(Template),
+    ok = emqx_connector_template:validate(PlaceHolders, Result),
+    Result.
 
 parse_sql(Template, ReplaceWith, PlaceHolders) ->
-    emqx_placeholder:preproc_sql(
+    {Statement, Result} = emqx_connector_template_sql:parse_prepstmt(
         Template,
-        #{
+        #{parameters => ReplaceWith, strip_double_quote => true}
-            replace_with => ReplaceWith,
+    ),
-            placeholders => PlaceHolders,
+    ok = emqx_connector_template:validate(PlaceHolders, Result),
-            strip_double_quote => true
+    {Statement, Result}.
-        }
-    ).
 
 render_deep(Template, Values) ->
-    emqx_placeholder:proc_tmpl_deep(
+    % NOTE
+    % Ignoring errors here, undefined bindings will be replaced with empty string.
+    {Term, _Errors} = emqx_connector_template:render(
         Template,
         client_vars(Values),
-        #{return => full_binary, var_trans => fun handle_var/2}
+        #{var_trans => fun handle_var/2}
-    ).
+    ),
+    Term.
 
 render_str(Template, Values) ->
-    emqx_placeholder:proc_tmpl(
+    % NOTE
+    % Ignoring errors here, undefined bindings will be replaced with empty string.
+    {String, _Errors} = emqx_connector_template:render(
         Template,
         client_vars(Values),
-        #{return => full_binary, var_trans => fun handle_var/2}
+        #{var_trans => fun handle_var/2}
-    ).
+    ),
+    unicode:characters_to_binary(String).
 
 render_urlencoded_str(Template, Values) ->
-    emqx_placeholder:proc_tmpl(
+    % NOTE
+    % Ignoring errors here, undefined bindings will be replaced with empty string.
+    {String, _Errors} = emqx_connector_template:render(
         Template,
         client_vars(Values),
-        #{return => full_binary, var_trans => fun urlencode_var/2}
+        #{var_trans => fun urlencode_var/2}
-    ).
+    ),
+    unicode:characters_to_binary(String).
 
 render_sql_params(ParamList, Values) ->
-    emqx_placeholder:proc_tmpl(
+    % NOTE
+    % Ignoring errors here, undefined bindings will be replaced with empty string.
+    {Row, _Errors} = emqx_connector_template:render(
         ParamList,
         client_vars(Values),
-        #{return => rawlist, var_trans => fun handle_sql_var/2}
+        #{var_trans => fun handle_sql_var/2}
-    ).
+    ),
+    Row.
 
 -spec parse_http_resp_body(binary(), binary()) -> allow | deny | ignore | error.
 parse_http_resp_body(<<"application/x-www-form-urlencoded", _/binary>>, Body) ->
@@ -218,19 +232,19 @@ convert_client_var(Other) -> Other.
 urlencode_var(Var, Value) ->
     emqx_http_lib:uri_encode(handle_var(Var, Value)).
 
-handle_var(_Name, undefined) ->
+handle_var(_, undefined) ->
     <<>>;
 handle_var([<<"peerhost">>], IpAddr) ->
     inet_parse:ntoa(IpAddr);
 handle_var(_Name, Value) ->
-    emqx_placeholder:bin(Value).
+    emqx_connector_template:to_string(Value).
 
-handle_sql_var(_Name, undefined) ->
+handle_sql_var(_, undefined) ->
     <<>>;
 handle_sql_var([<<"peerhost">>], IpAddr) ->
     inet_parse:ntoa(IpAddr);
 handle_sql_var(_Name, Value) ->
-    emqx_placeholder:sql_data(Value).
+    emqx_connector_sql:to_sql_value(Value).
 
 bin(A) when is_atom(A) -> atom_to_binary(A, utf8);
 bin(L) when is_list(L) -> list_to_binary(L);

apps/emqx_auth/test/emqx_authz/emqx_authz_rule_SUITE.erl

@@ -67,6 +67,10 @@ set_special_configs(_App) ->
     ok.
 
 t_compile(_) ->
+    % NOTE
+    % Some of the following testcase are relying on the internal representation of
+    % `emqx_connector_template:t()`. If the internal representation is changed, these
+    % testcases may fail.
     ?assertEqual({deny, all, all, [['#']]}, emqx_authz_rule:compile({deny, all})),
 
     ?assertEqual(
@@ -116,7 +120,7 @@ t_compile(_) ->
 
     ?assertEqual(
         {allow, {username, {eq, <<"test">>}}, publish, [
-            {pattern, [{str, <<"t/foo">>}, {var, [<<"username">>]}, {str, <<"boo">>}]}
+            {pattern, [<<"t/foo">>, {var, [<<"username">>]}, <<"boo">>]}
         ]},
         emqx_authz_rule:compile({allow, {username, "test"}, publish, ["t/foo${username}boo"]})
     ),

apps/emqx_auth_http/src/emqx_authz_http.erl

@@ -39,20 +39,20 @@
 -endif.
 
 -define(PLACEHOLDERS, [
-    ?PH_USERNAME,
+    <<?VAR_USERNAME>>,
-    ?PH_CLIENTID,
+    <<?VAR_CLIENTID>>,
-    ?PH_PEERHOST,
+    <<?VAR_PEERHOST>>,
-    ?PH_PROTONAME,
+    <<?VAR_PROTONAME>>,
-    ?PH_MOUNTPOINT,
+    <<?VAR_MOUNTPOINT>>,
-    ?PH_TOPIC,
+    <<?VAR_TOPIC>>,
-    ?PH_ACTION,
+    <<?VAR_ACTION>>,
-    ?PH_CERT_SUBJECT,
+    <<?VAR_CERT_SUBJECT>>,
-    ?PH_CERT_CN_NAME
+    <<?VAR_CERT_CN_NAME>>
 ]).
 
 -define(PLACEHOLDERS_FOR_RICH_ACTIONS, [
-    ?PH_QOS,
+    <<?VAR_QOS>>,
-    ?PH_RETAIN
+    <<?VAR_RETAIN>>
 ]).
 
 description() ->

apps/emqx_auth_mongodb/src/emqx_authz_mongodb.erl

@@ -36,11 +36,11 @@
 -endif.
 
 -define(PLACEHOLDERS, [
-    ?PH_USERNAME,
+    <<?VAR_USERNAME>>,
-    ?PH_CLIENTID,
+    <<?VAR_CLIENTID>>,
-    ?PH_PEERHOST,
+    <<?VAR_PEERHOST>>,
-    ?PH_CERT_CN_NAME,
+    <<?VAR_CERT_CN_NAME>>,
-    ?PH_CERT_SUBJECT
+    <<?VAR_CERT_SUBJECT>>
 ]).
 
 description() ->

apps/emqx_auth_mysql/src/emqx_authz_mysql.erl

@@ -38,11 +38,11 @@
 -endif.
 
 -define(PLACEHOLDERS, [
-    ?PH_USERNAME,
+    <<?VAR_USERNAME>>,
-    ?PH_CLIENTID,
+    <<?VAR_CLIENTID>>,
-    ?PH_PEERHOST,
+    <<?VAR_PEERHOST>>,
-    ?PH_CERT_CN_NAME,
+    <<?VAR_CERT_CN_NAME>>,
-    ?PH_CERT_SUBJECT
+    <<?VAR_CERT_SUBJECT>>
 ]).
 
 description() ->

apps/emqx_auth_postgresql/src/emqx_authz_postgresql.erl

@@ -38,11 +38,11 @@
 -endif.
 
 -define(PLACEHOLDERS, [
-    ?PH_USERNAME,
+    <<?VAR_USERNAME>>,
-    ?PH_CLIENTID,
+    <<?VAR_CLIENTID>>,
-    ?PH_PEERHOST,
+    <<?VAR_PEERHOST>>,
-    ?PH_CERT_CN_NAME,
+    <<?VAR_CERT_CN_NAME>>,
-    ?PH_CERT_SUBJECT
+    <<?VAR_CERT_SUBJECT>>
 ]).
 
 description() ->

apps/emqx_auth_redis/src/emqx_authz_redis.erl

@@ -36,11 +36,11 @@
 -endif.
 
 -define(PLACEHOLDERS, [
-    ?PH_CERT_CN_NAME,
+    <<?VAR_CERT_CN_NAME>>,
-    ?PH_CERT_SUBJECT,
+    <<?VAR_CERT_SUBJECT>>,
-    ?PH_PEERHOST,
+    <<?VAR_PEERHOST>>,
-    ?PH_CLIENTID,
+    <<?VAR_CLIENTID>>,
-    ?PH_USERNAME
+    <<?VAR_USERNAME>>
 ]).
 
 description() ->

apps/emqx_connector/src/emqx_connector_template.erl

@@ -153,7 +153,7 @@ trivial(Template) ->
 unparse({'$tpl', Template}) ->
     unparse_deep(Template);
 unparse(Template) ->
-    lists:map(fun unparse_part/1, Template).
+    unicode:characters_to_list(lists:map(fun unparse_part/1, Template)).
 
 unparse_part({var, Name}) ->
     render_placeholder(Name);
@@ -222,7 +222,7 @@ render_strict(Template, Bindings, Opts) ->
         {String, []} ->
             String;
         {_, Errors = [_ | _]} ->
-            error(Errors, [unicode:characters_to_list(unparse(Template)), Bindings])
+            error(Errors, [unparse(Template), Bindings])
     end.
 
 %% @doc Parse an arbitrary Erlang term into a "deep" template.
@@ -306,9 +306,7 @@ unparse_deep(Term) ->
 
 -spec lookup_var(var(), bindings()) ->
     {ok, binding()} | {error, undefined}.
-lookup_var(?PH_VAR_THIS, Value) ->
+lookup_var(Var, Value) when Var == ?PH_VAR_THIS orelse Var == [] ->
-    {ok, Value};
-lookup_var([], Value) ->
     {ok, Value};
 lookup_var([Prop | Rest], Bindings) ->
     case lookup(Prop, Bindings) of