From 5006dbba6e769b4fa1309b446de893763b0b0cf9 Mon Sep 17 00:00:00 2001 From: HeeeJianBo Date: Fri, 1 Dec 2017 22:12:27 +0800 Subject: [PATCH 1/4] Add ws/wss proxy cofingurations for getting client original ip address --- etc/emq.conf | 8 ++++++++ priv/emq.schema | 32 ++++++++++++++++++++++++++++++-- src/emqttd_ws_client.erl | 33 +++++++++++++++++++++++++++++++-- src/emqttd_ws_client_sup.erl | 19 ++++++++++++++++++- 4 files changed, 87 insertions(+), 5 deletions(-) diff --git a/etc/emq.conf b/etc/emq.conf index 60e9c421f..779cb899b 100644 --- a/etc/emq.conf +++ b/etc/emq.conf @@ -496,6 +496,10 @@ listener.ws.external.max_clients = 64 listener.ws.external.access.1 = allow all +listener.ws.external.proxy_ipaddress_header = x-forwarded-for + +listener.ws.external.proxy_port_header = x-remote-port + ## TCP Options listener.ws.external.backlog = 1024 @@ -518,6 +522,10 @@ listener.wss.external.max_clients = 64 listener.wss.external.access.1 = allow all +listener.wss.external.proxy_ipaddress_header = x-forwarded-for + +listener.wss.external.proxy_port_header = x-remote-port + ## SSL Options listener.wss.external.handshake_timeout = 15s diff --git a/priv/emq.schema b/priv/emq.schema index d05cc79cf..7d756b434 100644 --- a/priv/emq.schema +++ b/priv/emq.schema @@ -987,6 +987,16 @@ end}. {datatype, string} ]}. +{mapping, "listener.ws.$name.proxy_port_header", "emqttd.listeners", [ + {datatype, string}, + hidden +]}. + +{mapping, "listener.ws.$name.proxy_ipaddress_header", "emqttd.listeners", [ + {datatype, string}, + hidden +]}. + {mapping, "listener.ws.$name.access.$id", "emqttd.listeners", [ {datatype, string} ]}. @@ -1050,6 +1060,16 @@ end}. {datatype, string} ]}. +{mapping, "listener.wss.$name.proxy_port_header", "emqttd.listeners", [ + {datatype, string}, + hidden +]}. + +{mapping, "listener.wss.$name.proxy_ipaddress_header", "emqttd.listeners", [ + {datatype, string}, + hidden +]}. + {mapping, "listener.wss.$name.access.$id", "emqttd.listeners", [ {datatype, string} ]}. @@ -1127,6 +1147,13 @@ end}. end end, + WsProxyOpts = fun(Prefix) when Prefix =:= "listener.ws.external" orelse + Prefix =:= "listener.wss.external" -> + Filter([{proxy_port_header, cuttlefish:conf_get(Prefix ++ ".proxy_port_header", Conf, undefined)}, + {proxy_ipaddress_header, cuttlefish:conf_get(Prefix ++ ".proxy_ipaddress_header", Conf, undefined)}]); + (_) -> [] + end, + MountPoint = fun(undefined) -> undefined; (S) -> list_to_binary(S) end, ConnOpts = fun(Prefix) -> @@ -1178,7 +1205,8 @@ end}. undefined -> []; ListenOn -> - [{Atom(Type), ListenOn, [{connopts, ConnOpts(Prefix)}, {sockopts, TcpOpts(Prefix)} | LisOpts(Prefix)]}] + [{Atom(Type), ListenOn, [{connopts, ConnOpts(Prefix)}, + {sockopts, TcpOpts(Prefix)} | LisOpts(Prefix) ++ WsProxyOpts(Prefix)]}] end end, @@ -1190,7 +1218,7 @@ end}. ListenOn -> [{Atom(Type), ListenOn, [{connopts, ConnOpts(Prefix)}, {sockopts, TcpOpts(Prefix)}, - {sslopts, SslOpts(Prefix)} | LisOpts(Prefix)]}] + {sslopts, SslOpts(Prefix)} | LisOpts(Prefix) ++ WsProxyOpts(Prefix)]}] end end, diff --git a/src/emqttd_ws_client.erl b/src/emqttd_ws_client.erl index b9d25ad3e..7a66dff3c 100644 --- a/src/emqttd_ws_client.erl +++ b/src/emqttd_ws_client.erl @@ -28,7 +28,7 @@ -include("emqttd_internal.hrl"). --import(proplists, [get_value/3]). +-import(proplists, [get_value/3, get_value/2]). %% API Exports -export([start_link/4]). @@ -93,7 +93,7 @@ init([Env, WsPid, Req, ReplyChannel]) -> process_flag(trap_exit, true), Conn = Req:get(connection), true = link(WsPid), - case Req:get(peername) of + case peername(Env, Req) of {ok, Peername} -> Headers = mochiweb_headers:to_list( mochiweb_request:get(headers, Req)), @@ -321,3 +321,32 @@ gc(State) -> Cb = fun() -> emit_stats(State) end, emqttd_gc:maybe_force_gc(#wsclient_state.force_gc_count, State, Cb). +peername(Env, Req) -> + Conn = Req:get(connection), + case Conn:peername() of + {ok, Peername} -> + % return original address, if existed + case last_forwarded(get_value(Conn:type(), Env, []), Req) of + undefined -> {ok, Peername}; + Forwarded -> {ok, Forwarded} + end; + {error, Reason} -> {error, Reason} + end. + +last_forwarded([], _) -> undefined; +last_forwarded(Conf, Req) -> + HostHeader = get_value(proxy_ipaddress_header, Conf), + PortHeader = get_value(proxy_port_header, Conf), + case tune_host(Req:get_header_value(HostHeader)) of + undefined -> undefined; + Host -> {Host, tune_port(Req:get_header_value(PortHeader))} + end. + +tune_host(undefined) -> undefined; +tune_host(Hosts) -> + {ok, Last} = inet:parse_address(string:strip(lists:last(string:tokens(Hosts, ",")))), + Last. + +tune_port(undefined) -> undefined; +tune_port(Port) -> list_to_integer(Port). + diff --git a/src/emqttd_ws_client_sup.erl b/src/emqttd_ws_client_sup.erl index 21f683eaa..ec46b8714 100644 --- a/src/emqttd_ws_client_sup.erl +++ b/src/emqttd_ws_client_sup.erl @@ -39,8 +39,25 @@ start_client(WsPid, Req, ReplyChannel) -> %%-------------------------------------------------------------------- init([]) -> - Env = lists:append(emqttd:env(client, []), emqttd:env(protocol, [])), + Env = lists:append(emqttd:env(client, []), + emqttd:env(protocol, []) ++ forwarded_header()), {ok, {{simple_one_for_one, 0, 1}, [{ws_client, {emqttd_ws_client, start_link, [Env]}, temporary, 5000, worker, [emqttd_ws_client]}]}}. +forwarded_header() -> + Env = [{Proto, Opts} || {Proto, _, Opts} <- emqttd:env(listeners, []), Proto == ws orelse Proto == wss], + lists:foldl(fun({Proto, Opts}, Acc) -> + Proto1 = case Proto of + ws -> tcp; + wss -> ssl + end, + case {proplists:get_value(proxy_ipaddress_header, Opts), + proplists:get_value(proxy_port_header, Opts)} of + {undefined, _} -> Acc; + {AddrHeader, undefined} -> [{Proto1, [{proxy_ipaddress_header, AddrHeader}]} | Acc]; + {AddrHeader, PortHeader} -> [{Proto1, [{proxy_ipaddress_header, AddrHeader}, + {proxy_port_header, PortHeader}]} | Acc] + end + end, [], Env). + From 4915195b1e1f1e5a06a0cdd7b450251aaa9d2f90 Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Sat, 2 Dec 2017 17:59:16 +0800 Subject: [PATCH 2/4] Fix issue #1335 - Forward real client IP using a reverse proxy for websocket --- src/emqttd_ws_client.erl | 33 ++------------------------------- src/emqttd_ws_client_sup.erl | 19 +------------------ 2 files changed, 3 insertions(+), 49 deletions(-) diff --git a/src/emqttd_ws_client.erl b/src/emqttd_ws_client.erl index 7a66dff3c..b9d25ad3e 100644 --- a/src/emqttd_ws_client.erl +++ b/src/emqttd_ws_client.erl @@ -28,7 +28,7 @@ -include("emqttd_internal.hrl"). --import(proplists, [get_value/3, get_value/2]). +-import(proplists, [get_value/3]). %% API Exports -export([start_link/4]). @@ -93,7 +93,7 @@ init([Env, WsPid, Req, ReplyChannel]) -> process_flag(trap_exit, true), Conn = Req:get(connection), true = link(WsPid), - case peername(Env, Req) of + case Req:get(peername) of {ok, Peername} -> Headers = mochiweb_headers:to_list( mochiweb_request:get(headers, Req)), @@ -321,32 +321,3 @@ gc(State) -> Cb = fun() -> emit_stats(State) end, emqttd_gc:maybe_force_gc(#wsclient_state.force_gc_count, State, Cb). -peername(Env, Req) -> - Conn = Req:get(connection), - case Conn:peername() of - {ok, Peername} -> - % return original address, if existed - case last_forwarded(get_value(Conn:type(), Env, []), Req) of - undefined -> {ok, Peername}; - Forwarded -> {ok, Forwarded} - end; - {error, Reason} -> {error, Reason} - end. - -last_forwarded([], _) -> undefined; -last_forwarded(Conf, Req) -> - HostHeader = get_value(proxy_ipaddress_header, Conf), - PortHeader = get_value(proxy_port_header, Conf), - case tune_host(Req:get_header_value(HostHeader)) of - undefined -> undefined; - Host -> {Host, tune_port(Req:get_header_value(PortHeader))} - end. - -tune_host(undefined) -> undefined; -tune_host(Hosts) -> - {ok, Last} = inet:parse_address(string:strip(lists:last(string:tokens(Hosts, ",")))), - Last. - -tune_port(undefined) -> undefined; -tune_port(Port) -> list_to_integer(Port). - diff --git a/src/emqttd_ws_client_sup.erl b/src/emqttd_ws_client_sup.erl index ec46b8714..21f683eaa 100644 --- a/src/emqttd_ws_client_sup.erl +++ b/src/emqttd_ws_client_sup.erl @@ -39,25 +39,8 @@ start_client(WsPid, Req, ReplyChannel) -> %%-------------------------------------------------------------------- init([]) -> - Env = lists:append(emqttd:env(client, []), - emqttd:env(protocol, []) ++ forwarded_header()), + Env = lists:append(emqttd:env(client, []), emqttd:env(protocol, [])), {ok, {{simple_one_for_one, 0, 1}, [{ws_client, {emqttd_ws_client, start_link, [Env]}, temporary, 5000, worker, [emqttd_ws_client]}]}}. -forwarded_header() -> - Env = [{Proto, Opts} || {Proto, _, Opts} <- emqttd:env(listeners, []), Proto == ws orelse Proto == wss], - lists:foldl(fun({Proto, Opts}, Acc) -> - Proto1 = case Proto of - ws -> tcp; - wss -> ssl - end, - case {proplists:get_value(proxy_ipaddress_header, Opts), - proplists:get_value(proxy_port_header, Opts)} of - {undefined, _} -> Acc; - {AddrHeader, undefined} -> [{Proto1, [{proxy_ipaddress_header, AddrHeader}]} | Acc]; - {AddrHeader, PortHeader} -> [{Proto1, [{proxy_ipaddress_header, AddrHeader}, - {proxy_port_header, PortHeader}]} | Acc] - end - end, [], Env). - From bceb72853dff279efc14dd5bbc051b5a38264beb Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Sat, 2 Dec 2017 17:59:29 +0800 Subject: [PATCH 3/4] Fix issue #1335 - Forward real client IP using a reverse proxy for websocket --- etc/emq.conf | 8 ++++---- priv/emq.schema | 19 +++++++------------ 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/etc/emq.conf b/etc/emq.conf index 53f070f2d..e1f31e843 100644 --- a/etc/emq.conf +++ b/etc/emq.conf @@ -496,9 +496,9 @@ listener.ws.external.max_clients = 64 listener.ws.external.access.1 = allow all -listener.ws.external.proxy_ipaddress_header = x-forwarded-for +## listener.ws.external.proxy_address_header = x-forwarded-for -listener.ws.external.proxy_port_header = x-remote-port +## listener.ws.external.proxy_port_header = x-remote-port ## TCP Options listener.ws.external.backlog = 1024 @@ -522,9 +522,9 @@ listener.wss.external.max_clients = 64 listener.wss.external.access.1 = allow all -listener.wss.external.proxy_ipaddress_header = x-forwarded-for +## listener.wss.external.proxy_address_header = x-forwarded-for -listener.wss.external.proxy_port_header = x-remote-port +## listener.wss.external.proxy_port_header = x-remote-port ## SSL Options listener.wss.external.handshake_timeout = 15s diff --git a/priv/emq.schema b/priv/emq.schema index 7d756b434..7aba5304c 100644 --- a/priv/emq.schema +++ b/priv/emq.schema @@ -992,7 +992,7 @@ end}. hidden ]}. -{mapping, "listener.ws.$name.proxy_ipaddress_header", "emqttd.listeners", [ +{mapping, "listener.ws.$name.proxy_address_header", "emqttd.listeners", [ {datatype, string}, hidden ]}. @@ -1065,7 +1065,7 @@ end}. hidden ]}. -{mapping, "listener.wss.$name.proxy_ipaddress_header", "emqttd.listeners", [ +{mapping, "listener.wss.$name.proxy_address_header", "emqttd.listeners", [ {datatype, string}, hidden ]}. @@ -1147,13 +1147,6 @@ end}. end end, - WsProxyOpts = fun(Prefix) when Prefix =:= "listener.ws.external" orelse - Prefix =:= "listener.wss.external" -> - Filter([{proxy_port_header, cuttlefish:conf_get(Prefix ++ ".proxy_port_header", Conf, undefined)}, - {proxy_ipaddress_header, cuttlefish:conf_get(Prefix ++ ".proxy_ipaddress_header", Conf, undefined)}]); - (_) -> [] - end, - MountPoint = fun(undefined) -> undefined; (S) -> list_to_binary(S) end, ConnOpts = fun(Prefix) -> @@ -1162,7 +1155,9 @@ end}. {proxy_protocol, cuttlefish:conf_get(Prefix ++ ".proxy_protocol", Conf, undefined)}, {proxy_protocol_timeout, cuttlefish:conf_get(Prefix ++ ".proxy_protocol_timeout", Conf, undefined)}, {mountpoint, MountPoint(cuttlefish:conf_get(Prefix ++ ".mountpoint", Conf, undefined))}, - {peer_cert_as_username, cuttlefish:conf_get(Prefix ++ ".peer_cert_as_username", Conf, undefined)}]) + {peer_cert_as_username, cuttlefish:conf_get(Prefix ++ ".peer_cert_as_username", Conf, undefined)}, + {proxy_port_header, cuttlefish:conf_get(Prefix ++ ".proxy_port_header", Conf, undefined)}, + {proxy_address_header, cuttlefish:conf_get(Prefix ++ ".proxy_address_header", Conf, undefined)}]) end, LisOpts = fun(Prefix) -> @@ -1206,7 +1201,7 @@ end}. []; ListenOn -> [{Atom(Type), ListenOn, [{connopts, ConnOpts(Prefix)}, - {sockopts, TcpOpts(Prefix)} | LisOpts(Prefix) ++ WsProxyOpts(Prefix)]}] + {sockopts, TcpOpts(Prefix)} | LisOpts(Prefix)]}] end end, @@ -1218,7 +1213,7 @@ end}. ListenOn -> [{Atom(Type), ListenOn, [{connopts, ConnOpts(Prefix)}, {sockopts, TcpOpts(Prefix)}, - {sslopts, SslOpts(Prefix)} | LisOpts(Prefix) ++ WsProxyOpts(Prefix)]}] + {sslopts, SslOpts(Prefix)} | LisOpts(Prefix)]}] end end, From 0fe530a50263f9119ef38810b493145d9a4f3388 Mon Sep 17 00:00:00 2001 From: Feng Lee Date: Sat, 2 Dec 2017 18:00:56 +0800 Subject: [PATCH 4/4] Depends on the develop branch of mochiweb --- Makefile | 4 ++-- src/emqttd.app.src | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index de2827e23..6923c8b5d 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ PROJECT = emqttd PROJECT_DESCRIPTION = Erlang MQTT Broker -PROJECT_VERSION = 2.3.0 +PROJECT_VERSION = 2.3.1 DEPS = goldrush gproc lager esockd ekka mochiweb pbkdf2 lager_syslog bcrypt clique jsx @@ -10,7 +10,7 @@ dep_getopt = git https://github.com/jcomellas/getopt v0.8.2 dep_lager = git https://github.com/basho/lager master dep_esockd = git https://github.com/emqtt/esockd master dep_ekka = git https://github.com/emqtt/ekka master -dep_mochiweb = git https://github.com/emqtt/mochiweb master +dep_mochiweb = git https://github.com/emqtt/mochiweb develop dep_pbkdf2 = git https://github.com/emqtt/pbkdf2 2.0.1 dep_lager_syslog = git https://github.com/basho/lager_syslog dep_bcrypt = git https://github.com/smarkets/erlang-bcrypt master diff --git a/src/emqttd.app.src b/src/emqttd.app.src index 269601bb8..67af8854e 100644 --- a/src/emqttd.app.src +++ b/src/emqttd.app.src @@ -1,6 +1,6 @@ {application,emqttd, [{description,"Erlang MQTT Broker"}, - {vsn,"2.3.0"}, + {vsn,"2.3.1"}, {modules,[]}, {registered,[emqttd_sup]}, {applications,[kernel,stdlib,gproc,lager,esockd,mochiweb,