From 0538a77700d56e1da90e0127ce7de2f867402e23 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 18 Apr 2023 15:21:35 +0300 Subject: [PATCH] feat(tpl): use `emqx_connector_template` in `emqx_authn`, `emqx_authz` This slightly changes semantics: now the attempt to create authenticator with illegal bindings in templates will fail, instead of treating them as literals. The runtime behaviour on the other hand should be the same. --- apps/emqx/include/emqx_placeholder.hrl | 80 +++++++++++-------- .../src/emqx_authn/emqx_authn_utils.erl | 80 +++++++++++-------- .../src/emqx_authz/emqx_authz_rule.erl | 18 ++--- .../src/emqx_authz/emqx_authz_utils.erl | 64 +++++++++------ .../test/emqx_authz/emqx_authz_rule_SUITE.erl | 6 +- apps/emqx_auth_http/src/emqx_authz_http.erl | 22 ++--- .../src/emqx_authz_mongodb.erl | 10 +-- apps/emqx_auth_mysql/src/emqx_authz_mysql.erl | 10 +-- .../src/emqx_authz_postgresql.erl | 10 +-- apps/emqx_auth_redis/src/emqx_authz_redis.erl | 10 +-- .../src/emqx_connector_template.erl | 8 +- 11 files changed, 178 insertions(+), 140 deletions(-) diff --git a/apps/emqx/include/emqx_placeholder.hrl b/apps/emqx/include/emqx_placeholder.hrl index 7b2ce6c6b..1db80c72d 100644 --- a/apps/emqx/include/emqx_placeholder.hrl +++ b/apps/emqx/include/emqx_placeholder.hrl @@ -19,67 +19,79 @@ -define(PH_VAR_THIS, <<"$_THIS_">>). --define(PH(Type), <<"${", Type/binary, "}">>). +-define(PH(Var), <<"${" Var "}">>). %% action: publish/subscribe --define(PH_ACTION, <<"${action}">>). +-define(VAR_ACTION, "action"). +-define(PH_ACTION, ?PH(?VAR_ACTION)). %% cert --define(PH_CERT_SUBJECT, <<"${cert_subject}">>). --define(PH_CERT_CN_NAME, <<"${cert_common_name}">>). +-define(VAR_CERT_SUBJECT, "cert_subject"). +-define(VAR_CERT_CN_NAME, "cert_common_name"). +-define(PH_CERT_SUBJECT, ?PH(?VAR_CERT_SUBJECT)). +-define(PH_CERT_CN_NAME, ?PH(?VAR_CERT_CN_NAME)). %% MQTT --define(PH_PASSWORD, <<"${password}">>). --define(PH_CLIENTID, <<"${clientid}">>). --define(PH_FROM_CLIENTID, <<"${from_clientid}">>). --define(PH_USERNAME, <<"${username}">>). --define(PH_FROM_USERNAME, <<"${from_username}">>). --define(PH_TOPIC, <<"${topic}">>). +-define(VAR_PASSWORD, "password"). +-define(VAR_CLIENTID, "clientid"). +-define(VAR_USERNAME, "username"). +-define(VAR_TOPIC, "topic"). +-define(PH_PASSWORD, ?PH(?VAR_PASSWORD)). +-define(PH_CLIENTID, ?PH(?VAR_CLIENTID)). +-define(PH_FROM_CLIENTID, ?PH("from_clientid")). +-define(PH_USERNAME, ?PH(?VAR_USERNAME)). +-define(PH_FROM_USERNAME, ?PH("from_username")). +-define(PH_TOPIC, ?PH(?VAR_TOPIC)). %% MQTT payload --define(PH_PAYLOAD, <<"${payload}">>). +-define(PH_PAYLOAD, ?PH("payload")). %% client IPAddress --define(PH_PEERHOST, <<"${peerhost}">>). +-define(VAR_PEERHOST, "peerhost"). +-define(PH_PEERHOST, ?PH(?VAR_PEERHOST)). %% ip & port --define(PH_HOST, <<"${host}">>). --define(PH_PORT, <<"${port}">>). +-define(PH_HOST, ?PH("host")). +-define(PH_PORT, ?PH("port")). %% Enumeration of message QoS 0,1,2 --define(PH_QOS, <<"${qos}">>). --define(PH_FLAGS, <<"${flags}">>). +-define(VAR_QOS, "qos"). +-define(PH_QOS, ?PH(?VAR_QOS)). +-define(PH_FLAGS, ?PH("flags")). %% Additional data related to process within the MQTT message --define(PH_HEADERS, <<"${headers}">>). +-define(PH_HEADERS, ?PH("headers")). %% protocol name --define(PH_PROTONAME, <<"${proto_name}">>). +-define(VAR_PROTONAME, "proto_name"). +-define(PH_PROTONAME, ?PH(?VAR_PROTONAME)). %% protocol version --define(PH_PROTOVER, <<"${proto_ver}">>). +-define(PH_PROTOVER, ?PH("proto_ver")). %% MQTT keepalive interval --define(PH_KEEPALIVE, <<"${keepalive}">>). +-define(PH_KEEPALIVE, ?PH("keepalive")). %% MQTT clean_start --define(PH_CLEAR_START, <<"${clean_start}">>). +-define(PH_CLEAR_START, ?PH("clean_start")). %% MQTT Session Expiration time --define(PH_EXPIRY_INTERVAL, <<"${expiry_interval}">>). +-define(PH_EXPIRY_INTERVAL, ?PH("expiry_interval")). %% Time when PUBLISH message reaches Broker (ms) --define(PH_PUBLISH_RECEIVED_AT, <<"${publish_received_at}">>). +-define(PH_PUBLISH_RECEIVED_AT, ?PH("publish_received_at")). %% Mountpoint for bridging messages --define(PH_MOUNTPOINT, <<"${mountpoint}">>). +-define(VAR_MOUNTPOINT, "mountpoint"). +-define(PH_MOUNTPOINT, ?PH(?VAR_MOUNTPOINT)). %% IPAddress and Port of terminal --define(PH_PEERNAME, <<"${peername}">>). +-define(PH_PEERNAME, ?PH("peername")). %% IPAddress and Port listened by emqx --define(PH_SOCKNAME, <<"${sockname}">>). +-define(PH_SOCKNAME, ?PH("sockname")). %% whether it is MQTT bridge connection --define(PH_IS_BRIDGE, <<"${is_bridge}">>). +-define(PH_IS_BRIDGE, ?PH("is_bridge")). %% Terminal connection completion time (s) --define(PH_CONNECTED_AT, <<"${connected_at}">>). +-define(PH_CONNECTED_AT, ?PH("connected_at")). %% Event trigger time(millisecond) --define(PH_TIMESTAMP, <<"${timestamp}">>). +-define(PH_TIMESTAMP, ?PH("timestamp")). %% Terminal disconnection completion time (s) --define(PH_DISCONNECTED_AT, <<"${disconnected_at}">>). +-define(PH_DISCONNECTED_AT, ?PH("disconnected_at")). --define(PH_NODE, <<"${node}">>). --define(PH_REASON, <<"${reason}">>). +-define(PH_NODE, ?PH("node")). +-define(PH_REASON, ?PH("reason")). --define(PH_ENDPOINT_NAME, <<"${endpoint_name}">>). --define(PH_RETAIN, <<"${retain}">>). +-define(PH_ENDPOINT_NAME, ?PH("endpoint_name")). +-define(VAR_RETAIN, "retain"). +-define(PH_RETAIN, ?PH(?VAR_RETAIN)). %% sync change these place holder with binary def. -define(PH_S_ACTION, "${action}"). diff --git a/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl b/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl index a9d672922..9be3e24d6 100644 --- a/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl +++ b/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl @@ -45,12 +45,12 @@ ]). -define(AUTHN_PLACEHOLDERS, [ - ?PH_USERNAME, - ?PH_CLIENTID, - ?PH_PASSWORD, - ?PH_PEERHOST, - ?PH_CERT_SUBJECT, - ?PH_CERT_CN_NAME + <>, + <>, + <>, + <>, + <>, + <> ]). -define(DEFAULT_RESOURCE_OPTS, #{ @@ -107,48 +107,62 @@ check_password_from_selected_map(Algorithm, Selected, Password) -> end. parse_deep(Template) -> - emqx_placeholder:preproc_tmpl_deep(Template, #{placeholders => ?AUTHN_PLACEHOLDERS}). + Result = emqx_connector_template:parse_deep(Template), + ok = emqx_connector_template:validate(?AUTHN_PLACEHOLDERS, Result), + Result. parse_str(Template) -> - emqx_placeholder:preproc_tmpl(Template, #{placeholders => ?AUTHN_PLACEHOLDERS}). + Result = emqx_connector_template:parse(Template), + ok = emqx_connector_template:validate(?AUTHN_PLACEHOLDERS, Result), + Result. parse_sql(Template, ReplaceWith) -> - emqx_placeholder:preproc_sql( + {Statement, Result} = emqx_connector_template_sql:parse_prepstmt( Template, - #{ - replace_with => ReplaceWith, - placeholders => ?AUTHN_PLACEHOLDERS, - strip_double_quote => true - } - ). + #{parameters => ReplaceWith, strip_double_quote => true} + ), + ok = emqx_connector_template:validate(?AUTHN_PLACEHOLDERS, Result), + {Statement, Result}. render_deep(Template, Credential) -> - emqx_placeholder:proc_tmpl_deep( + % NOTE + % Ignoring errors here, undefined bindings will be replaced with empty string. + {Term, _Errors} = emqx_connector_template:render( Template, mapping_credential(Credential), - #{return => full_binary, var_trans => fun handle_var/2} - ). + #{var_trans => fun handle_var/2} + ), + Term. render_str(Template, Credential) -> - emqx_placeholder:proc_tmpl( + % NOTE + % Ignoring errors here, undefined bindings will be replaced with empty string. + {String, _Errors} = emqx_connector_template:render( Template, mapping_credential(Credential), - #{return => full_binary, var_trans => fun handle_var/2} - ). + #{var_trans => fun handle_var/2} + ), + unicode:characters_to_binary(String). render_urlencoded_str(Template, Credential) -> - emqx_placeholder:proc_tmpl( + % NOTE + % Ignoring errors here, undefined bindings will be replaced with empty string. + {String, _Errors} = emqx_connector_template:render( Template, mapping_credential(Credential), - #{return => full_binary, var_trans => fun urlencode_var/2} - ). + #{var_trans => fun urlencode_var/2} + ), + unicode:characters_to_binary(String). render_sql_params(ParamList, Credential) -> - emqx_placeholder:proc_tmpl( + % NOTE + % Ignoring errors here, undefined bindings will be replaced with empty string. + {Row, _Errors} = emqx_connector_template:render( ParamList, mapping_credential(Credential), - #{return => rawlist, var_trans => fun handle_sql_var/2} - ). + #{var_trans => fun handle_sql_var/2} + ), + Row. is_superuser(#{<<"is_superuser">> := Value}) -> #{is_superuser => to_bool(Value)}; @@ -272,19 +286,19 @@ without_password(Credential, [Name | Rest]) -> urlencode_var(Var, Value) -> emqx_http_lib:uri_encode(handle_var(Var, Value)). -handle_var(_Name, undefined) -> +handle_var(_, undefined) -> <<>>; handle_var([<<"peerhost">>], PeerHost) -> - emqx_placeholder:bin(inet:ntoa(PeerHost)); + emqx_connector_template:to_string(inet:ntoa(PeerHost)); handle_var(_, Value) -> - emqx_placeholder:bin(Value). + emqx_connector_template:to_string(Value). -handle_sql_var(_Name, undefined) -> +handle_sql_var(_, undefined) -> <<>>; handle_sql_var([<<"peerhost">>], PeerHost) -> - emqx_placeholder:bin(inet:ntoa(PeerHost)); + emqx_connector_sql:to_sql_value(inet:ntoa(PeerHost)); handle_sql_var(_, Value) -> - emqx_placeholder:sql_data(Value). + emqx_connector_sql:to_sql_value(Value). mapping_credential(C = #{cn := CN, dn := DN}) -> C#{cert_common_name => CN, cert_subject => DN}; diff --git a/apps/emqx_auth/src/emqx_authz/emqx_authz_rule.erl b/apps/emqx_auth/src/emqx_authz/emqx_authz_rule.erl index 6e13cac91..9cf79ba88 100644 --- a/apps/emqx_auth/src/emqx_authz/emqx_authz_rule.erl +++ b/apps/emqx_auth/src/emqx_authz/emqx_authz_rule.erl @@ -183,19 +183,15 @@ compile_topic(<<"eq ", Topic/binary>>) -> compile_topic({eq, Topic}) -> {eq, emqx_topic:words(bin(Topic))}; compile_topic(Topic) -> - TopicBin = bin(Topic), - case - emqx_placeholder:preproc_tmpl( - TopicBin, - #{placeholders => [?PH_USERNAME, ?PH_CLIENTID]} - ) - of - [{str, _}] -> emqx_topic:words(TopicBin); - Tokens -> {pattern, Tokens} + Template = emqx_connector_template:parse(Topic), + ok = emqx_connector_template:validate([<>, <>], Template), + case emqx_connector_template:trivial(Template) of + true -> emqx_topic:words(bin(Topic)); + false -> {pattern, Template} end. bin(L) when is_list(L) -> - list_to_binary(L); + unicode:characters_to_binary(L); bin(B) when is_binary(B) -> B. @@ -307,7 +303,7 @@ match_who(_, _) -> match_topics(_ClientInfo, _Topic, []) -> false; match_topics(ClientInfo, Topic, [{pattern, PatternFilter} | Filters]) -> - TopicFilter = emqx_placeholder:proc_tmpl(PatternFilter, ClientInfo), + TopicFilter = bin(emqx_connector_template:render_strict(PatternFilter, ClientInfo)), match_topic(emqx_topic:words(Topic), emqx_topic:words(TopicFilter)) orelse match_topics(ClientInfo, Topic, Filters); match_topics(ClientInfo, Topic, [TopicFilter | Filters]) -> diff --git a/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl b/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl index 3a0d4f1a1..e5aeab21b 100644 --- a/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl +++ b/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl @@ -108,48 +108,62 @@ update_config(Path, ConfigRequest) -> }). parse_deep(Template, PlaceHolders) -> - emqx_placeholder:preproc_tmpl_deep(Template, #{placeholders => PlaceHolders}). + Result = emqx_connector_template:parse_deep(Template), + ok = emqx_connector_template:validate(PlaceHolders, Result), + Result. parse_str(Template, PlaceHolders) -> - emqx_placeholder:preproc_tmpl(Template, #{placeholders => PlaceHolders}). + Result = emqx_connector_template:parse(Template), + ok = emqx_connector_template:validate(PlaceHolders, Result), + Result. parse_sql(Template, ReplaceWith, PlaceHolders) -> - emqx_placeholder:preproc_sql( + {Statement, Result} = emqx_connector_template_sql:parse_prepstmt( Template, - #{ - replace_with => ReplaceWith, - placeholders => PlaceHolders, - strip_double_quote => true - } - ). + #{parameters => ReplaceWith, strip_double_quote => true} + ), + ok = emqx_connector_template:validate(PlaceHolders, Result), + {Statement, Result}. render_deep(Template, Values) -> - emqx_placeholder:proc_tmpl_deep( + % NOTE + % Ignoring errors here, undefined bindings will be replaced with empty string. + {Term, _Errors} = emqx_connector_template:render( Template, client_vars(Values), - #{return => full_binary, var_trans => fun handle_var/2} - ). + #{var_trans => fun handle_var/2} + ), + Term. render_str(Template, Values) -> - emqx_placeholder:proc_tmpl( + % NOTE + % Ignoring errors here, undefined bindings will be replaced with empty string. + {String, _Errors} = emqx_connector_template:render( Template, client_vars(Values), - #{return => full_binary, var_trans => fun handle_var/2} - ). + #{var_trans => fun handle_var/2} + ), + unicode:characters_to_binary(String). render_urlencoded_str(Template, Values) -> - emqx_placeholder:proc_tmpl( + % NOTE + % Ignoring errors here, undefined bindings will be replaced with empty string. + {String, _Errors} = emqx_connector_template:render( Template, client_vars(Values), - #{return => full_binary, var_trans => fun urlencode_var/2} - ). + #{var_trans => fun urlencode_var/2} + ), + unicode:characters_to_binary(String). render_sql_params(ParamList, Values) -> - emqx_placeholder:proc_tmpl( + % NOTE + % Ignoring errors here, undefined bindings will be replaced with empty string. + {Row, _Errors} = emqx_connector_template:render( ParamList, client_vars(Values), - #{return => rawlist, var_trans => fun handle_sql_var/2} - ). + #{var_trans => fun handle_sql_var/2} + ), + Row. -spec parse_http_resp_body(binary(), binary()) -> allow | deny | ignore | error. parse_http_resp_body(<<"application/x-www-form-urlencoded", _/binary>>, Body) -> @@ -218,19 +232,19 @@ convert_client_var(Other) -> Other. urlencode_var(Var, Value) -> emqx_http_lib:uri_encode(handle_var(Var, Value)). -handle_var(_Name, undefined) -> +handle_var(_, undefined) -> <<>>; handle_var([<<"peerhost">>], IpAddr) -> inet_parse:ntoa(IpAddr); handle_var(_Name, Value) -> - emqx_placeholder:bin(Value). + emqx_connector_template:to_string(Value). -handle_sql_var(_Name, undefined) -> +handle_sql_var(_, undefined) -> <<>>; handle_sql_var([<<"peerhost">>], IpAddr) -> inet_parse:ntoa(IpAddr); handle_sql_var(_Name, Value) -> - emqx_placeholder:sql_data(Value). + emqx_connector_sql:to_sql_value(Value). bin(A) when is_atom(A) -> atom_to_binary(A, utf8); bin(L) when is_list(L) -> list_to_binary(L); diff --git a/apps/emqx_auth/test/emqx_authz/emqx_authz_rule_SUITE.erl b/apps/emqx_auth/test/emqx_authz/emqx_authz_rule_SUITE.erl index b34e4fb00..bca21cd8d 100644 --- a/apps/emqx_auth/test/emqx_authz/emqx_authz_rule_SUITE.erl +++ b/apps/emqx_auth/test/emqx_authz/emqx_authz_rule_SUITE.erl @@ -67,6 +67,10 @@ set_special_configs(_App) -> ok. t_compile(_) -> + % NOTE + % Some of the following testcase are relying on the internal representation of + % `emqx_connector_template:t()`. If the internal representation is changed, these + % testcases may fail. ?assertEqual({deny, all, all, [['#']]}, emqx_authz_rule:compile({deny, all})), ?assertEqual( @@ -116,7 +120,7 @@ t_compile(_) -> ?assertEqual( {allow, {username, {eq, <<"test">>}}, publish, [ - {pattern, [{str, <<"t/foo">>}, {var, [<<"username">>]}, {str, <<"boo">>}]} + {pattern, [<<"t/foo">>, {var, [<<"username">>]}, <<"boo">>]} ]}, emqx_authz_rule:compile({allow, {username, "test"}, publish, ["t/foo${username}boo"]}) ), diff --git a/apps/emqx_auth_http/src/emqx_authz_http.erl b/apps/emqx_auth_http/src/emqx_authz_http.erl index ed7051bb6..2ab76f305 100644 --- a/apps/emqx_auth_http/src/emqx_authz_http.erl +++ b/apps/emqx_auth_http/src/emqx_authz_http.erl @@ -39,20 +39,20 @@ -endif. -define(PLACEHOLDERS, [ - ?PH_USERNAME, - ?PH_CLIENTID, - ?PH_PEERHOST, - ?PH_PROTONAME, - ?PH_MOUNTPOINT, - ?PH_TOPIC, - ?PH_ACTION, - ?PH_CERT_SUBJECT, - ?PH_CERT_CN_NAME + <>, + <>, + <>, + <>, + <>, + <>, + <>, + <>, + <> ]). -define(PLACEHOLDERS_FOR_RICH_ACTIONS, [ - ?PH_QOS, - ?PH_RETAIN + <>, + <> ]). description() -> diff --git a/apps/emqx_auth_mongodb/src/emqx_authz_mongodb.erl b/apps/emqx_auth_mongodb/src/emqx_authz_mongodb.erl index 3b235ad2c..97a5fa3a6 100644 --- a/apps/emqx_auth_mongodb/src/emqx_authz_mongodb.erl +++ b/apps/emqx_auth_mongodb/src/emqx_authz_mongodb.erl @@ -36,11 +36,11 @@ -endif. -define(PLACEHOLDERS, [ - ?PH_USERNAME, - ?PH_CLIENTID, - ?PH_PEERHOST, - ?PH_CERT_CN_NAME, - ?PH_CERT_SUBJECT + <>, + <>, + <>, + <>, + <> ]). description() -> diff --git a/apps/emqx_auth_mysql/src/emqx_authz_mysql.erl b/apps/emqx_auth_mysql/src/emqx_authz_mysql.erl index 4ca71e332..e87d2afa2 100644 --- a/apps/emqx_auth_mysql/src/emqx_authz_mysql.erl +++ b/apps/emqx_auth_mysql/src/emqx_authz_mysql.erl @@ -38,11 +38,11 @@ -endif. -define(PLACEHOLDERS, [ - ?PH_USERNAME, - ?PH_CLIENTID, - ?PH_PEERHOST, - ?PH_CERT_CN_NAME, - ?PH_CERT_SUBJECT + <>, + <>, + <>, + <>, + <> ]). description() -> diff --git a/apps/emqx_auth_postgresql/src/emqx_authz_postgresql.erl b/apps/emqx_auth_postgresql/src/emqx_authz_postgresql.erl index b930f77e4..645fff293 100644 --- a/apps/emqx_auth_postgresql/src/emqx_authz_postgresql.erl +++ b/apps/emqx_auth_postgresql/src/emqx_authz_postgresql.erl @@ -38,11 +38,11 @@ -endif. -define(PLACEHOLDERS, [ - ?PH_USERNAME, - ?PH_CLIENTID, - ?PH_PEERHOST, - ?PH_CERT_CN_NAME, - ?PH_CERT_SUBJECT + <>, + <>, + <>, + <>, + <> ]). description() -> diff --git a/apps/emqx_auth_redis/src/emqx_authz_redis.erl b/apps/emqx_auth_redis/src/emqx_authz_redis.erl index 9b69f508a..7ac893da1 100644 --- a/apps/emqx_auth_redis/src/emqx_authz_redis.erl +++ b/apps/emqx_auth_redis/src/emqx_authz_redis.erl @@ -36,11 +36,11 @@ -endif. -define(PLACEHOLDERS, [ - ?PH_CERT_CN_NAME, - ?PH_CERT_SUBJECT, - ?PH_PEERHOST, - ?PH_CLIENTID, - ?PH_USERNAME + <>, + <>, + <>, + <>, + <> ]). description() -> diff --git a/apps/emqx_connector/src/emqx_connector_template.erl b/apps/emqx_connector/src/emqx_connector_template.erl index c346d4289..4f583573c 100644 --- a/apps/emqx_connector/src/emqx_connector_template.erl +++ b/apps/emqx_connector/src/emqx_connector_template.erl @@ -153,7 +153,7 @@ trivial(Template) -> unparse({'$tpl', Template}) -> unparse_deep(Template); unparse(Template) -> - lists:map(fun unparse_part/1, Template). + unicode:characters_to_list(lists:map(fun unparse_part/1, Template)). unparse_part({var, Name}) -> render_placeholder(Name); @@ -222,7 +222,7 @@ render_strict(Template, Bindings, Opts) -> {String, []} -> String; {_, Errors = [_ | _]} -> - error(Errors, [unicode:characters_to_list(unparse(Template)), Bindings]) + error(Errors, [unparse(Template), Bindings]) end. %% @doc Parse an arbitrary Erlang term into a "deep" template. @@ -306,9 +306,7 @@ unparse_deep(Term) -> -spec lookup_var(var(), bindings()) -> {ok, binding()} | {error, undefined}. -lookup_var(?PH_VAR_THIS, Value) -> - {ok, Value}; -lookup_var([], Value) -> +lookup_var(Var, Value) when Var == ?PH_VAR_THIS orelse Var == [] -> {ok, Value}; lookup_var([Prop | Rest], Bindings) -> case lookup(Prop, Bindings) of