fix: non-utf8 clientid connect event case crash if strict_mod=false

2024-04-29 12:06:26 +08:00 · 2024-04-29 12:06:26 +08:00 · ef9a63ae17
parent 002dc8541b
commit ef9a63ae17
3 changed files with 141 additions and 113 deletions
--- a/apps/emqx/src/emqx_connection.erl
+++ b/apps/emqx/src/emqx_connection.erl
@ -346,7 +346,7 @@ init_state(
        max_size => emqx_config:get_zone_conf(Zone, [mqtt, max_packet_size])
    },
    ParseState = emqx_frame:initial_parse_state(FrameOpts),
-    Serialize = emqx_frame:serialize_opts(),
+    Serialize = emqx_frame:initial_serialize_opts(FrameOpts),
    %% Init Channel
    Channel = emqx_channel:init(ConnInfo, Opts),
    GcState =
--- a/apps/emqx/src/emqx_frame.erl
+++ b/apps/emqx/src/emqx_frame.erl
@ -29,11 +29,12 @@
    parse/2,
    serialize_fun/0,
    serialize_fun/1,
-    serialize_opts/0,
+    initial_serialize_opts/1,
    serialize_opts/1,
    serialize_pkt/2,
    serialize/1,
-    serialize/2
+    serialize/2,
+    serialize/3
 ]).

 -export([describe_state/1]).
@ -84,7 +85,7 @@

 -define(MULTIPLIER_MAX, 16#200000).

-dialyzer({no_match, [serialize_utf8_string/2]}).
+-dialyzer({no_match, [serialize_utf8_string/3]}).

 %% @doc Describe state for logging.
 describe_state(?NONE(_Opts)) ->
@ -718,43 +719,51 @@ serialize_fun() -> serialize_fun(?DEFAULT_OPTIONS).

 serialize_fun(#mqtt_packet_connect{proto_ver = ProtoVer, properties = ConnProps}) ->
    MaxSize = get_property('Maximum-Packet-Size', ConnProps, ?MAX_PACKET_SIZE),
-    serialize_fun(#{version => ProtoVer, max_size => MaxSize});
-serialize_fun(#{version := Ver, max_size := MaxSize}) ->
+    serialize_fun(#{version => ProtoVer, max_size => MaxSize, strict_mode => false});
+serialize_fun(#{version := Ver, max_size := MaxSize, strict_mode := StrictMode}) ->
    fun(Packet) ->
-        IoData = serialize(Packet, Ver),
+        IoData = serialize(Packet, Ver, StrictMode),
        case is_too_large(IoData, MaxSize) of
            true -> <<>>;
            false -> IoData
        end
    end.

-serialize_opts() ->
-    ?DEFAULT_OPTIONS.
+initial_serialize_opts(Opts) ->
+    maps:merge(?DEFAULT_OPTIONS, Opts).

 serialize_opts(#mqtt_packet_connect{proto_ver = ProtoVer, properties = ConnProps}) ->
    MaxSize = get_property('Maximum-Packet-Size', ConnProps, ?MAX_PACKET_SIZE),
-    #{version => ProtoVer, max_size => MaxSize}.
+    #{version => ProtoVer, max_size => MaxSize, strict_mode => false}.

-serialize_pkt(Packet, #{version := Ver, max_size := MaxSize}) ->
-    IoData = serialize(Packet, Ver),
+serialize_pkt(Packet, #{version := Ver, max_size := MaxSize, strict_mode := StrictMode}) ->
+    IoData = serialize(Packet, Ver, StrictMode),
    case is_too_large(IoData, MaxSize) of
        true -> <<>>;
        false -> IoData
    end.

 -spec serialize(emqx_types:packet()) -> iodata().
-serialize(Packet) -> serialize(Packet, ?MQTT_PROTO_V4).
+serialize(Packet) -> serialize(Packet, ?MQTT_PROTO_V4, false).

-spec serialize(emqx_types:packet(), emqx_types:proto_ver()) -> iodata().
+serialize(Packet, Ver) -> serialize(Packet, Ver, false).
+
+-spec serialize(emqx_types:packet(), emqx_types:proto_ver(), boolean()) -> iodata().
 serialize(
    #mqtt_packet{
        header = Header,
        variable = Variable,
        payload = Payload
    },
-    Ver
+    Ver,
+    StrictMode
 ) ->
-    serialize(Header, serialize_variable(Variable, Ver), serialize_payload(Payload)).
+    serialize(
+        Header,
+        serialize_variable(Variable, Ver, StrictMode),
+        serialize_payload(Payload),
+        StrictMode
+    ).

 serialize(
    #mqtt_packet_header{
@ -764,7 +773,8 @@ serialize(
        retain = Retain
    },
    VariableBin,
-    PayloadBin
+    PayloadBin,
+    _StrictMode
 ) when
    ?CONNECT =< Type andalso Type =< ?AUTH
 ->
@ -796,7 +806,8 @@ serialize_variable(
        username = Username,
        password = Password
    },
-    _Ver
+    _Ver,
+    StrictMode
 ) ->
    [
        serialize_binary_data(ProtoName),
@ -814,20 +825,20 @@ serialize_variable(
            0:1,
            KeepAlive:16/big-unsigned-integer
        >>,
-        serialize_properties(Properties, ProtoVer),
-        serialize_utf8_string(ClientId),
+        serialize_properties(Properties, ProtoVer, StrictMode),
+        serialize_utf8_string(ClientId, StrictMode),
        case WillFlag of
            true ->
                [
-                    serialize_properties(WillProps, ProtoVer),
-                    serialize_utf8_string(WillTopic),
+                    serialize_properties(WillProps, ProtoVer, StrictMode),
+                    serialize_utf8_string(WillTopic, StrictMode),
                    serialize_binary_data(WillPayload)
                ];
            false ->
                <<>>
        end,
-        serialize_utf8_string(Username, true),
-        serialize_utf8_string(Password, true)
+        serialize_utf8_string(Username, true, StrictMode),
+        serialize_utf8_string(Password, true, StrictMode)
    ];
 serialize_variable(
    #mqtt_packet_connack{
@ -835,26 +846,28 @@ serialize_variable(
        reason_code = ReasonCode,
        properties = Properties
    },
-    Ver
+    Ver,
+    StrictMode
 ) ->
-    [AckFlags, ReasonCode, serialize_properties(Properties, Ver)];
+    [AckFlags, ReasonCode, serialize_properties(Properties, Ver, StrictMode)];
 serialize_variable(
    #mqtt_packet_publish{
        topic_name = TopicName,
        packet_id = PacketId,
        properties = Properties
    },
-    Ver
+    Ver,
+    StrictMode
 ) ->
    [
-        serialize_utf8_string(TopicName),
+        serialize_utf8_string(TopicName, StrictMode),
        case PacketId of
            undefined -> <<>>;
            _ -> <<PacketId:16/big-unsigned-integer>>
        end,
-        serialize_properties(Properties, Ver)
+        serialize_properties(Properties, Ver, StrictMode)
    ];
-serialize_variable(#mqtt_packet_puback{packet_id = PacketId}, Ver) when
+serialize_variable(#mqtt_packet_puback{packet_id = PacketId}, Ver, _StrictMode) when
    Ver == ?MQTT_PROTO_V3; Ver == ?MQTT_PROTO_V4
 ->
    <<PacketId:16/big-unsigned-integer>>;
@ -864,12 +877,13 @@ serialize_variable(
        reason_code = ReasonCode,
        properties = Properties
    },
-    Ver = ?MQTT_PROTO_V5
+    Ver = ?MQTT_PROTO_V5,
+    StrictMode
 ) ->
    [
        <<PacketId:16/big-unsigned-integer>>,
        ReasonCode,
-        serialize_properties(Properties, Ver)
+        serialize_properties(Properties, Ver, StrictMode)
    ];
 serialize_variable(
    #mqtt_packet_subscribe{
@ -877,12 +891,13 @@ serialize_variable(
        properties = Properties,
        topic_filters = TopicFilters
    },
-    Ver
+    Ver,
+    StrictMode
 ) ->
    [
        <<PacketId:16/big-unsigned-integer>>,
-        serialize_properties(Properties, Ver),
-        serialize_topic_filters(subscribe, TopicFilters, Ver)
+        serialize_properties(Properties, Ver, StrictMode),
+        serialize_topic_filters(subscribe, TopicFilters, Ver, StrictMode)
    ];
 serialize_variable(
    #mqtt_packet_suback{
@ -890,11 +905,12 @@ serialize_variable(
        properties = Properties,
        reason_codes = ReasonCodes
    },
-    Ver
+    Ver,
+    StrictMode
 ) ->
    [
        <<PacketId:16/big-unsigned-integer>>,
-        serialize_properties(Properties, Ver),
+        serialize_properties(Properties, Ver, StrictMode),
        serialize_reason_codes(ReasonCodes)
    ];
 serialize_variable(
@ -903,12 +919,13 @@ serialize_variable(
        properties = Properties,
        topic_filters = TopicFilters
    },
-    Ver
+    Ver,
+    StrictMode
 ) ->
    [
        <<PacketId:16/big-unsigned-integer>>,
-        serialize_properties(Properties, Ver),
-        serialize_topic_filters(unsubscribe, TopicFilters, Ver)
+        serialize_properties(Properties, Ver, StrictMode),
+        serialize_topic_filters(unsubscribe, TopicFilters, Ver, StrictMode)
    ];
 serialize_variable(
    #mqtt_packet_unsuback{
@ -916,14 +933,15 @@ serialize_variable(
        properties = Properties,
        reason_codes = ReasonCodes
    },
-    Ver
+    Ver,
+    StrictMode
 ) ->
    [
        <<PacketId:16/big-unsigned-integer>>,
-        serialize_properties(Properties, Ver),
+        serialize_properties(Properties, Ver, StrictMode),
        serialize_reason_codes(ReasonCodes)
    ];
-serialize_variable(#mqtt_packet_disconnect{}, Ver) when
+serialize_variable(#mqtt_packet_disconnect{}, Ver, _StrictMode) when
    Ver == ?MQTT_PROTO_V3; Ver == ?MQTT_PROTO_V4
 ->
    <<>>;
@ -932,110 +950,115 @@ serialize_variable(
        reason_code = ReasonCode,
        properties = Properties
    },
-    Ver = ?MQTT_PROTO_V5
+    Ver = ?MQTT_PROTO_V5,
+    StrictMode
 ) ->
-    [ReasonCode, serialize_properties(Properties, Ver)];
-serialize_variable(#mqtt_packet_disconnect{}, _Ver) ->
+    [ReasonCode, serialize_properties(Properties, Ver, StrictMode)];
+serialize_variable(#mqtt_packet_disconnect{}, _Ver, _StrictMode) ->
    <<>>;
 serialize_variable(
    #mqtt_packet_auth{
        reason_code = ReasonCode,
        properties = Properties
    },
-    Ver = ?MQTT_PROTO_V5
+    Ver = ?MQTT_PROTO_V5,
+    StrictMode
 ) ->
-    [ReasonCode, serialize_properties(Properties, Ver)];
-serialize_variable(PacketId, ?MQTT_PROTO_V3) when is_integer(PacketId) ->
+    [ReasonCode, serialize_properties(Properties, Ver, StrictMode)];
+serialize_variable(PacketId, ?MQTT_PROTO_V3, _StrictMode) when is_integer(PacketId) ->
    <<PacketId:16/big-unsigned-integer>>;
-serialize_variable(PacketId, ?MQTT_PROTO_V4) when is_integer(PacketId) ->
+serialize_variable(PacketId, ?MQTT_PROTO_V4, _StrictMode) when is_integer(PacketId) ->
    <<PacketId:16/big-unsigned-integer>>;
-serialize_variable(undefined, _Ver) ->
+serialize_variable(undefined, _Ver, _StrictMode) ->
    <<>>.

 serialize_payload(undefined) -> <<>>;
 serialize_payload(Bin) -> Bin.

-serialize_properties(_Props, Ver) when Ver =/= ?MQTT_PROTO_V5 ->
+serialize_properties(_Props, Ver, _StrictMode) when Ver =/= ?MQTT_PROTO_V5 ->
    <<>>;
-serialize_properties(Props, ?MQTT_PROTO_V5) ->
-    serialize_properties(Props).
+serialize_properties(Props, ?MQTT_PROTO_V5, StrictMode) ->
+    serialize_properties(Props, StrictMode).

-serialize_properties(undefined) ->
+serialize_properties(undefined, _StrictMode) ->
    <<0>>;
-serialize_properties(Props) when map_size(Props) == 0 ->
+serialize_properties(Props, _StrictMode) when map_size(Props) == 0 ->
    <<0>>;
-serialize_properties(Props) when is_map(Props) ->
-    Bin = <<<<(serialize_property(Prop, Val))/binary>> || {Prop, Val} <- maps:to_list(Props)>>,
+serialize_properties(Props, StrictMode) when is_map(Props) ->
+    Bin = <<
+        <<(serialize_property(Prop, Val, StrictMode))/binary>>
+     || {Prop, Val} <- maps:to_list(Props)
+    >>,
    [serialize_variable_byte_integer(byte_size(Bin)), Bin].

-serialize_property(_, Disabled) when Disabled =:= disabled; Disabled =:= undefined ->
+serialize_property(_, Disabled, _StrictMode) when Disabled =:= disabled; Disabled =:= undefined ->
    <<>>;
-serialize_property(internal_extra, _) ->
+serialize_property(internal_extra, _, _StrictMode) ->
    <<>>;
-serialize_property('Payload-Format-Indicator', Val) ->
+serialize_property('Payload-Format-Indicator', Val, _StrictMode) ->
    <<16#01, Val>>;
-serialize_property('Message-Expiry-Interval', Val) ->
+serialize_property('Message-Expiry-Interval', Val, _StrictMode) ->
    <<16#02, Val:32/big>>;
-serialize_property('Content-Type', Val) ->
-    <<16#03, (serialize_utf8_string(Val))/binary>>;
-serialize_property('Response-Topic', Val) ->
-    <<16#08, (serialize_utf8_string(Val))/binary>>;
-serialize_property('Correlation-Data', Val) ->
+serialize_property('Content-Type', Val, StrictMode) ->
+    <<16#03, (serialize_utf8_string(Val, StrictMode))/binary>>;
+serialize_property('Response-Topic', Val, StrictMode) ->
+    <<16#08, (serialize_utf8_string(Val, StrictMode))/binary>>;
+serialize_property('Correlation-Data', Val, _StrictMode) ->
    <<16#09, (byte_size(Val)):16, Val/binary>>;
-serialize_property('Subscription-Identifier', Val) ->
+serialize_property('Subscription-Identifier', Val, _StrictMode) ->
    <<16#0B, (serialize_variable_byte_integer(Val))/binary>>;
-serialize_property('Session-Expiry-Interval', Val) ->
+serialize_property('Session-Expiry-Interval', Val, _StrictMode) ->
    <<16#11, Val:32/big>>;
-serialize_property('Assigned-Client-Identifier', Val) ->
-    <<16#12, (serialize_utf8_string(Val))/binary>>;
-serialize_property('Server-Keep-Alive', Val) ->
+serialize_property('Assigned-Client-Identifier', Val, StrictMode) ->
+    <<16#12, (serialize_utf8_string(Val, StrictMode))/binary>>;
+serialize_property('Server-Keep-Alive', Val, _StrictMode) ->
    <<16#13, Val:16/big>>;
-serialize_property('Authentication-Method', Val) ->
-    <<16#15, (serialize_utf8_string(Val))/binary>>;
-serialize_property('Authentication-Data', Val) ->
+serialize_property('Authentication-Method', Val, StrictMode) ->
+    <<16#15, (serialize_utf8_string(Val, StrictMode))/binary>>;
+serialize_property('Authentication-Data', Val, _StrictMode) ->
    <<16#16, (iolist_size(Val)):16, Val/binary>>;
-serialize_property('Request-Problem-Information', Val) ->
+serialize_property('Request-Problem-Information', Val, _StrictMode) ->
    <<16#17, Val>>;
-serialize_property('Will-Delay-Interval', Val) ->
+serialize_property('Will-Delay-Interval', Val, _StrictMode) ->
    <<16#18, Val:32/big>>;
-serialize_property('Request-Response-Information', Val) ->
+serialize_property('Request-Response-Information', Val, _StrictMode) ->
    <<16#19, Val>>;
-serialize_property('Response-Information', Val) ->
-    <<16#1A, (serialize_utf8_string(Val))/binary>>;
-serialize_property('Server-Reference', Val) ->
-    <<16#1C, (serialize_utf8_string(Val))/binary>>;
-serialize_property('Reason-String', Val) ->
-    <<16#1F, (serialize_utf8_string(Val))/binary>>;
-serialize_property('Receive-Maximum', Val) ->
+serialize_property('Response-Information', Val, StrictMode) ->
+    <<16#1A, (serialize_utf8_string(Val, StrictMode))/binary>>;
+serialize_property('Server-Reference', Val, StrictMode) ->
+    <<16#1C, (serialize_utf8_string(Val, StrictMode))/binary>>;
+serialize_property('Reason-String', Val, StrictMode) ->
+    <<16#1F, (serialize_utf8_string(Val, StrictMode))/binary>>;
+serialize_property('Receive-Maximum', Val, _StrictMode) ->
    <<16#21, Val:16/big>>;
-serialize_property('Topic-Alias-Maximum', Val) ->
+serialize_property('Topic-Alias-Maximum', Val, _StrictMode) ->
    <<16#22, Val:16/big>>;
-serialize_property('Topic-Alias', Val) ->
+serialize_property('Topic-Alias', Val, _StrictMode) ->
    <<16#23, Val:16/big>>;
-serialize_property('Maximum-QoS', Val) ->
+serialize_property('Maximum-QoS', Val, _StrictMode) ->
    <<16#24, Val>>;
-serialize_property('Retain-Available', Val) ->
+serialize_property('Retain-Available', Val, _StrictMode) ->
    <<16#25, Val>>;
-serialize_property('User-Property', {Key, Val}) ->
-    <<16#26, (serialize_utf8_pair({Key, Val}))/binary>>;
-serialize_property('User-Property', Props) when is_list(Props) ->
+serialize_property('User-Property', {Key, Val}, StrictMode) ->
+    <<16#26, (serialize_utf8_pair(Key, Val, StrictMode))/binary>>;
+serialize_property('User-Property', Props, StrictMode) when is_list(Props) ->
    <<
-        <<(serialize_property('User-Property', {Key, Val}))/binary>>
+        <<(serialize_property('User-Property', {Key, Val}, StrictMode))/binary>>
     || {Key, Val} <- Props
    >>;
-serialize_property('Maximum-Packet-Size', Val) ->
+serialize_property('Maximum-Packet-Size', Val, _StrictMode) ->
    <<16#27, Val:32/big>>;
-serialize_property('Wildcard-Subscription-Available', Val) ->
+serialize_property('Wildcard-Subscription-Available', Val, _StrictMode) ->
    <<16#28, Val>>;
-serialize_property('Subscription-Identifier-Available', Val) ->
+serialize_property('Subscription-Identifier-Available', Val, _StrictMode) ->
    <<16#29, Val>>;
-serialize_property('Shared-Subscription-Available', Val) ->
+serialize_property('Shared-Subscription-Available', Val, _StrictMode) ->
    <<16#2A, Val>>.

-serialize_topic_filters(subscribe, TopicFilters, ?MQTT_PROTO_V5) ->
+serialize_topic_filters(subscribe, TopicFilters, ?MQTT_PROTO_V5, StrictMode) ->
    <<
        <<
-            (serialize_utf8_string(Topic))/binary,
+            (serialize_utf8_string(Topic, StrictMode))/binary,
            ?RESERVED:2,
            Rh:2,
            (flag(Rap)):1,
@ -1044,37 +1067,42 @@ serialize_topic_filters(subscribe, TopicFilters, ?MQTT_PROTO_V5) ->
        >>
     || {Topic, #{rh := Rh, rap := Rap, nl := Nl, qos := QoS}} <- TopicFilters
    >>;
-serialize_topic_filters(subscribe, TopicFilters, _Ver) ->
+serialize_topic_filters(subscribe, TopicFilters, _Ver, StrictMode) ->
    <<
-        <<(serialize_utf8_string(Topic))/binary, ?RESERVED:6, QoS:2>>
+        <<(serialize_utf8_string(Topic, StrictMode))/binary, ?RESERVED:6, QoS:2>>
     || {Topic, #{qos := QoS}} <- TopicFilters
    >>;
-serialize_topic_filters(unsubscribe, TopicFilters, _Ver) ->
-    <<<<(serialize_utf8_string(Topic))/binary>> || Topic <- TopicFilters>>.
+serialize_topic_filters(unsubscribe, TopicFilters, _Ver, StrictMode) ->
+    <<<<(serialize_utf8_string(Topic, StrictMode))/binary>> || Topic <- TopicFilters>>.

 serialize_reason_codes(undefined) ->
    <<>>;
 serialize_reason_codes(ReasonCodes) when is_list(ReasonCodes) ->
    <<<<Code>> || Code <- ReasonCodes>>.

-serialize_utf8_pair({Name, Value}) ->
-    <<(serialize_utf8_string(Name))/binary, (serialize_utf8_string(Value))/binary>>.
+serialize_utf8_pair(Name, Value, StrictMode) ->
+    <<
+        (serialize_utf8_string(Name, StrictMode))/binary,
+        (serialize_utf8_string(Value, StrictMode))/binary
+    >>.

 serialize_binary_data(Bin) ->
    [<<(byte_size(Bin)):16/big-unsigned-integer>>, Bin].

-serialize_utf8_string(undefined, false) ->
+serialize_utf8_string(undefined, false, _StrictMode) ->
    ?SERIALIZE_ERR(utf8_string_undefined);
-serialize_utf8_string(undefined, true) ->
+serialize_utf8_string(undefined, true, _StrictMode) ->
    <<>>;
-serialize_utf8_string(String, _AllowNull) ->
-    serialize_utf8_string(String).
+serialize_utf8_string(String, _AllowNull, StrictMode) ->
+    serialize_utf8_string(String, StrictMode).

-serialize_utf8_string(String) ->
+serialize_utf8_string(String, true) ->
    StringBin = unicode:characters_to_binary(String),
-    Len = byte_size(StringBin),
+    serialize_utf8_string(StringBin, false);
+serialize_utf8_string(String, false) ->
+    Len = byte_size(String),
    true = (Len =< 16#ffff),
-    <<Len:16/big, StringBin/binary>>.
+    <<Len:16/big, String/binary>>.

 serialize_remaining_len(I) ->
    serialize_variable_byte_integer(I).
--- a/apps/emqx/src/emqx_ws_connection.erl
+++ b/apps/emqx/src/emqx_ws_connection.erl
@ -302,7 +302,7 @@ websocket_init([Req, Opts]) ->
                max_size => emqx_config:get_zone_conf(Zone, [mqtt, max_packet_size])
            },
            ParseState = emqx_frame:initial_parse_state(FrameOpts),
-            Serialize = emqx_frame:serialize_opts(),
+            Serialize = emqx_frame:initial_serialize_opts(FrameOpts),
            Channel = emqx_channel:init(ConnInfo, Opts),
            GcState = get_force_gc(Zone),
            StatsTimer = get_stats_enable(Zone),