From 5aa4565e84a2f300fc77e92021c82bb59abea3eb Mon Sep 17 00:00:00 2001 From: Turtle Date: Thu, 10 Jun 2021 11:29:31 +0800 Subject: [PATCH 01/78] chore: remove lager schema info --- priv/emqx.schema | 2470 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 2470 insertions(+) create mode 100644 priv/emqx.schema diff --git a/priv/emqx.schema b/priv/emqx.schema new file mode 100644 index 000000000..2d67c9677 --- /dev/null +++ b/priv/emqx.schema @@ -0,0 +1,2470 @@ +%%-*- mode: erlang -*- +%% EMQ X R4.0 config mapping + +%%-------------------------------------------------------------------- +%% Cluster +%%-------------------------------------------------------------------- + +%% @doc Cluster name +{mapping, "cluster.name", "ekka.cluster_name", [ + {default, emqxcl}, + {datatype, atom} +]}. + +%% @doc Cluster discovery +{mapping, "cluster.discovery", "ekka.cluster_discovery", [ + {default, manual}, + {datatype, atom} +]}. + +%% @doc Clean down node from the cluster +{mapping, "cluster.autoclean", "ekka.cluster_autoclean", [ + {datatype, {duration, ms}} +]}. + +%% @doc Cluster autoheal +{mapping, "cluster.autoheal", "ekka.cluster_autoheal", [ + {datatype, flag}, + {default, off} +]}. + +%%-------------------------------------------------------------------- +%% Cluster by static node list + +{mapping, "cluster.static.seeds", "ekka.cluster_discovery", [ + {datatype, string} +]}. + +%%-------------------------------------------------------------------- +%% Cluster by UDP Multicast + +{mapping, "cluster.mcast.addr", "ekka.cluster_discovery", [ + {default, "239.192.0.1"}, + {datatype, string} +]}. + +{mapping, "cluster.mcast.ports", "ekka.cluster_discovery", [ + {default, "4369"}, + {datatype, string} +]}. + +{mapping, "cluster.mcast.iface", "ekka.cluster_discovery", [ + {datatype, string}, + {default, "0.0.0.0"} +]}. + +{mapping, "cluster.mcast.ttl", "ekka.cluster_discovery", [ + {datatype, integer}, + {default, 255} +]}. + +{mapping, "cluster.mcast.loop", "ekka.cluster_discovery", [ + {datatype, flag}, + {default, on} +]}. + +{mapping, "cluster.mcast.sndbuf", "ekka.cluster_discovery", [ + {datatype, bytesize}, + {default, "16KB"} +]}. + +{mapping, "cluster.mcast.recbuf", "ekka.cluster_discovery", [ + {datatype, bytesize}, + {default, "16KB"} +]}. + +{mapping, "cluster.mcast.buffer", "ekka.cluster_discovery", [ + {datatype, bytesize}, + {default, "32KB"} +]}. + +%%-------------------------------------------------------------------- +%% Cluster by DNS A Record + +{mapping, "cluster.dns.name", "ekka.cluster_discovery", [ + {datatype, string} +]}. + +%% @doc The erlang distributed protocol +{mapping, "cluster.proto_dist", "ekka.proto_dist", [ + {default, "inet_tcp"}, + {datatype, {enum, [inet_tcp, inet6_tcp, inet_tls]}}, + hidden +]}. + +{mapping, "cluster.dns.app", "ekka.cluster_discovery", [ + {datatype, string} +]}. + +%%-------------------------------------------------------------------- +%% Cluster using etcd + +{mapping, "cluster.etcd.server", "ekka.cluster_discovery", [ + {datatype, string} +]}. + +{mapping, "cluster.etcd.prefix", "ekka.cluster_discovery", [ + {datatype, string} +]}. + +{mapping, "cluster.etcd.node_ttl", "ekka.cluster_discovery", [ + {datatype, {duration, ms}}, + {default, "1m"} +]}. + +{mapping, "cluster.etcd.ssl.keyfile", "ekka.cluster_discovery", [ + {datatype, string} +]}. + +{mapping, "cluster.etcd.ssl.certfile", "ekka.cluster_discovery", [ + {datatype, string} +]}. + +{mapping, "cluster.etcd.ssl.cacertfile", "ekka.cluster_discovery", [ + {datatype, string} +]}. + +%%-------------------------------------------------------------------- +%% Cluster on K8s + +{mapping, "cluster.k8s.apiserver", "ekka.cluster_discovery", [ + {datatype, string} +]}. + +{mapping, "cluster.k8s.service_name", "ekka.cluster_discovery", [ + {datatype, string} +]}. + +{mapping, "cluster.k8s.address_type", "ekka.cluster_discovery", [ + {datatype, {enum, [ip, dns, hostname]}} +]}. + +{mapping, "cluster.k8s.app_name", "ekka.cluster_discovery", [ + {datatype, string} +]}. + +{mapping, "cluster.k8s.namespace", "ekka.cluster_discovery", [ + {datatype, string} +]}. + +{mapping, "cluster.k8s.suffix", "ekka.cluster_discovery", [ + {datatype, string}, + {default, ""} + ]}. + +{translation, "ekka.cluster_discovery", fun(Conf) -> + Strategy = cuttlefish:conf_get("cluster.discovery", Conf), + Filter = fun(Opts) -> [{K, V} || {K, V} <- Opts, V =/= undefined] end, + IpPort = fun(S) -> + [Addr, Port] = string:tokens(S, ":"), + {ok, Ip} = inet:parse_address(Addr), + {Ip, Port} + end, + Options = fun(static) -> + [{seeds, [list_to_atom(S) || S <- string:tokens(cuttlefish:conf_get("cluster.static.seeds", Conf, ""), ",")]}]; + (mcast) -> + {ok, Addr} = inet:parse_address(cuttlefish:conf_get("cluster.mcast.addr", Conf)), + {ok, Iface} = inet:parse_address(cuttlefish:conf_get("cluster.mcast.iface", Conf)), + Ports = [list_to_integer(S) || S <- string:tokens(cuttlefish:conf_get("cluster.mcast.ports", Conf), ",")], + [{addr, Addr}, {ports, Ports}, {iface, Iface}, + {ttl, cuttlefish:conf_get("cluster.mcast.ttl", Conf, 1)}, + {loop, cuttlefish:conf_get("cluster.mcast.loop", Conf, true)}]; + (dns) -> + [{name, cuttlefish:conf_get("cluster.dns.name", Conf)}, + {app, cuttlefish:conf_get("cluster.dns.app", Conf)}]; + (etcd) -> + SslOpts = fun(Conf) -> + Options = cuttlefish_variable:filter_by_prefix("cluster.etcd.ssl", Conf), + lists:map(fun({["cluster", "etcd", "ssl", Name], Value}) -> + {list_to_atom(Name), Value} + end, Options) + end, + [{server, string:tokens(cuttlefish:conf_get("cluster.etcd.server", Conf), ",")}, + {prefix, cuttlefish:conf_get("cluster.etcd.prefix", Conf, "emqcl")}, + {node_ttl, cuttlefish:conf_get("cluster.etcd.node_ttl", Conf, 60)}, + {ssl_options, SslOpts(Conf)}]; + (k8s) -> + [{apiserver, cuttlefish:conf_get("cluster.k8s.apiserver", Conf)}, + {service_name, cuttlefish:conf_get("cluster.k8s.service_name", Conf)}, + {address_type, cuttlefish:conf_get("cluster.k8s.address_type", Conf, ip)}, + {app_name, cuttlefish:conf_get("cluster.k8s.app_name", Conf)}, + {namespace, cuttlefish:conf_get("cluster.k8s.namespace", Conf)}, + {suffix, cuttlefish:conf_get("cluster.k8s.suffix", Conf, "")}]; + (manual) -> + [ ] + end, + {Strategy, Filter(Options(Strategy))} +end}. + +%%-------------------------------------------------------------------- +%% Node +%%-------------------------------------------------------------------- + +%% @doc Node name +{mapping, "node.name", "vm_args.-name", [ + {default, "emqx@127.0.0.1"}, + {override_env, "NODE_NAME"} +]}. + +%% @doc Specify SSL Options in the file if using SSL for erlang distribution +{mapping, "node.ssl_dist_optfile", "vm_args.-ssl_dist_optfile", [ + {datatype, string}, + hidden +]}. + +%% @doc Secret cookie for distributed erlang node +{mapping, "node.cookie", "vm_args.-setcookie", [ + {default, "emqxsecretcookie"}, + {override_env, "NODE_COOKIE"} +]}. + +{mapping, "node.data_dir", "emqx.data_dir", [ + {datatype, string} +]}. + +%% @doc http://erlang.org/doc/man/heart.html +{mapping, "node.heartbeat", "vm_args.-heart", [ + {datatype, flag}, + hidden +]}. + +{translation, "vm_args.-heart", fun(Conf) -> + case cuttlefish:conf_get("node.heartbeat", Conf) of + true -> ""; + false -> cuttlefish:invalid("should be 'on' or comment the line!") + end +end}. + +%% @doc More information at: http://erlang.org/doc/man/erl.html +{mapping, "node.async_threads", "vm_args.+A", [ + {datatype, integer}, + {validators, ["range:0-1024"]} +]}. + +%% @doc Erlang Process Limit +{mapping, "node.process_limit", "vm_args.+P", [ + {datatype, integer}, + hidden +]}. + +%% @doc The maximum number of concurrent ports/sockets. +%% Valid range is 1024-134217727 +{mapping, "node.max_ports", "vm_args.+Q", [ + {datatype, integer}, + {validators, ["range4ports"]}, + {override_env, "MAX_PORTS"} +]}. + +{validator, "range4ports", "must be 1024 to 134217727", + fun(X) -> X >= 1024 andalso X =< 134217727 end}. + +%% @doc http://www.erlang.org/doc/man/erl.html#%2bzdbbl +{mapping, "node.dist_buffer_size", "vm_args.+zdbbl", [ + {datatype, bytesize}, + {commented, "32MB"}, + hidden, + {validators, ["zdbbl_range"]} +]}. + +{translation, "vm_args.+zdbbl", + fun(Conf) -> + ZDBBL = cuttlefish:conf_get("node.dist_buffer_size", Conf, undefined), + case ZDBBL of + undefined -> undefined; + X when is_integer(X) -> cuttlefish_util:ceiling(X / 1024); %% Bytes to Kilobytes; + _ -> undefined + end + end}. + +{validator, "zdbbl_range", "must be between 1KB and 2097151KB", + fun(ZDBBL) -> + %% 2097151KB = 2147482624 + ZDBBL >= 1024 andalso ZDBBL =< 2147482624 + end +}. + +%% @doc Global GC Interval +{mapping, "node.global_gc_interval", "emqx.global_gc_interval", [ + {datatype, {duration, s}} +]}. + +%% @doc http://www.erlang.org/doc/man/erlang.html#system_flag-2 +{mapping, "node.fullsweep_after", "vm_args.-env ERL_FULLSWEEP_AFTER", [ + {default, 1000}, + {datatype, integer}, + hidden, + {validators, ["positive_integer"]} +]}. + +{validator, "positive_integer", "must be a positive integer", + fun(X) -> X >= 0 end}. + +%% Note: OTP R15 and earlier uses -env ERL_MAX_ETS_TABLES, +%% R16+ uses +e +%% @doc The ETS table limit +{mapping, "node.max_ets_tables", + cuttlefish:otp("R16", "vm_args.+e", "vm_args.-env ERL_MAX_ETS_TABLES"), [ + {default, 256000}, + {datatype, integer}, + hidden +]}. + +%% @doc Set the location of crash dumps +{mapping, "node.crash_dump", "vm_args.-env ERL_CRASH_DUMP", [ + {default, "{{crash_dump}}"}, + {datatype, file}, + hidden +]}. + +%% @doc http://www.erlang.org/doc/man/kernel_app.html#net_ticktime +{mapping, "node.dist_net_ticktime", "vm_args.-kernel net_ticktime", [ + {datatype, integer}, + hidden +]}. + +%% @doc http://www.erlang.org/doc/man/kernel_app.html +{mapping, "node.dist_listen_min", "kernel.inet_dist_listen_min", [ + {commented, 6369}, + {datatype, integer}, + hidden +]}. + +%% @see node.dist_listen_min +{mapping, "node.dist_listen_max", "kernel.inet_dist_listen_max", [ + {commented, 6369}, + {datatype, integer}, + hidden +]}. + +{mapping, "node.backtrace_depth", "emqx.backtrace_depth", [ + {default, 16}, + {datatype, integer} +]}. + +%%-------------------------------------------------------------------- +%% RPC +%%-------------------------------------------------------------------- + +%% RPC Mode. +{mapping, "rpc.mode", "emqx.rpc_mode", [ + {default, async}, + {datatype, {enum, [sync, async]}} +]}. + +{mapping, "rpc.async_batch_size", "gen_rpc.max_batch_size", [ + {default, 256}, + {datatype, integer} +]}. + +{mapping, "rpc.port_discovery", "gen_rpc.port_discovery", [ + {default, stateless}, + {datatype, {enum, [manual, stateless]}} +]}. + +%% RPC server port. +{mapping, "rpc.tcp_server_port", "gen_rpc.tcp_server_port", [ + {default, 5369}, + {datatype, integer} +]}. + +%% Number of tcp connections when connecting to RPC server +{mapping, "rpc.tcp_client_num", "gen_rpc.tcp_client_num", [ + {default, 0}, + {datatype, integer}, + {validators, ["range:gt_0_lt_256"]} +]}. + +{translation, "gen_rpc.tcp_client_num", fun(Conf) -> + case cuttlefish:conf_get("rpc.tcp_client_num", Conf) of + 0 -> max(1, erlang:system_info(schedulers) div 2); + V -> V + end +end}. + +%% Client connect timeout +{mapping, "rpc.connect_timeout", "gen_rpc.connect_timeout", [ + {default, "5s"}, + {datatype, {duration, ms}} +]}. + +%% Client and Server send timeout +{mapping, "rpc.send_timeout", "gen_rpc.send_timeout", [ + {default, 5000}, + {datatype, {duration, ms}} +]}. + +%% Authentication timeout +{mapping, "rpc.authentication_timeout", "gen_rpc.authentication_timeout", [ + {default, 5000}, + {datatype, {duration, ms}} +]}. + +%% Default receive timeout for call() functions +{mapping, "rpc.call_receive_timeout", "gen_rpc.call_receive_timeout", [ + {default, 15000}, + {datatype, {duration, ms}} +]}. + +%% Socket keepalive configuration +{mapping, "rpc.socket_keepalive_idle", "gen_rpc.socket_keepalive_idle", [ + {default, 7200}, + {datatype, {duration, s}} +]}. + +%% Seconds between probes +{mapping, "rpc.socket_keepalive_interval", "gen_rpc.socket_keepalive_interval", [ + {default, 75}, + {datatype, {duration, s}} +]}. + +%% Probes lost to close the connection +{mapping, "rpc.socket_keepalive_count", "gen_rpc.socket_keepalive_count", [ + {default, 9}, + {datatype, integer} +]}. + +%% Size of TCP send buffer +{mapping, "rpc.socket_sndbuf", "gen_rpc.socket_sndbuf", [ + {default, "1MB"}, + {datatype, bytesize} +]}. + +%% Size of TCP receive buffer +{mapping, "rpc.socket_recbuf", "gen_rpc.socket_recbuf", [ + {default, "1MB"}, + {datatype, bytesize} +]}. + +%% Size of TCP receive buffer +{mapping, "rpc.socket_buffer", "gen_rpc.socket_buffer", [ + {default, "1MB"}, + {datatype, bytesize} +]}. + +{validator, "range:gt_0_lt_256", "must greater than 0 and less than 256", + fun(X) -> X >= 0 andalso X < 256 end +}. + +%% Force client to use server listening port, because we do no provide +%% per-node listening port manual mapping from configs. +%% i.e. all nodes in the cluster should agree to the same +%% listening port number. +{translation, "gen_rpc.tcp_client_port", fun(_, _, Conf) -> + cuttlefish:conf_get("rpc.tcp_server_port", Conf) +end}. + +%%-------------------------------------------------------------------- +%% Log +%%-------------------------------------------------------------------- + +{mapping, "log.to", "kernel.logger", [ + {default, file}, + {datatype, {enum, [file, console, both]}} +]}. + +{mapping, "log.level", "kernel.logger", [ + {default, warning}, + {datatype, {enum, [debug, info, notice, warning, error, critical, alert, emergency, all]}} +]}. + +{mapping, "log.primary_log_level", "kernel.logger_level", [ + {default, warning}, + {datatype, {enum, [debug, info, notice, warning, error, critical, alert, emergency, all]}} +]}. + +{mapping, "log.dir", "kernel.logger", [ + {default, "log"}, + {datatype, string} +]}. + +{mapping, "log.file", "kernel.logger", [ + {default, "emqx.log"}, + {datatype, file} +]}. + +{mapping, "log.chars_limit", "kernel.logger", [ + {default, -1}, + {datatype, integer} +]}. + +{mapping, "log.supervisor_reports", "kernel.logger", [ + {default, error}, + {datatype, {enum, [error, progress]}}, + hidden +]}. + +%% @doc Maximum depth in Erlang term log formatting +%% and message queue inspection. +{mapping, "log.max_depth", "kernel.error_logger_format_depth", [ + {default, 20}, + {datatype, [{enum, [unlimited]}, integer]} +]}. + +%% @doc format logs as JSON objects +{mapping, "log.formatter", "kernel.logger", [ + {default, text}, + {datatype, {enum, [text, json]}} +]}. + +%% @doc format logs in a single line. +{mapping, "log.single_line", "kernel.logger", [ + {default, true}, + {datatype, {enum, [true, false]}} +]}. + +{mapping, "log.rotation", "kernel.logger", [ + {default, on}, + {datatype, flag} +]}. + +{mapping, "log.rotation.size", "kernel.logger", [ + {default, "10MB"}, + {datatype, bytesize} +]}. + +{mapping, "log.size", "kernel.logger", [ + {default, infinity}, + {datatype, [bytesize, atom]} +]}. + +{mapping, "log.rotation.count", "kernel.logger", [ + {default, 5}, + {datatype, integer} +]}. + +{mapping, "log.$level.file", "kernel.logger", [ + {datatype, file} +]}. + +{mapping, "log.sync_mode_qlen", "kernel.logger", [ + {default, 100}, + {datatype, integer} +]}. + +{mapping, "log.drop_mode_qlen", "kernel.logger", [ + {default, 3000}, + {datatype, integer} +]}. + +{mapping, "log.flush_qlen", "kernel.logger", [ + {default, 8000}, + {datatype, integer} +]}. + +{mapping, "log.overload_kill", "kernel.logger", [ + {default, on}, + {datatype, flag} +]}. + +{mapping, "log.overload_kill_mem_size", "kernel.logger", [ + {default, "30MB"}, + {datatype, bytesize} +]}. + +{mapping, "log.overload_kill_qlen", "kernel.logger", [ + {default, 20000}, + {datatype, integer} +]}. + +{mapping, "log.overload_kill_restart_after", "kernel.logger", [ + {default, "5s"}, + {datatype, [{duration, ms}, atom]} +]}. + +{mapping, "log.burst_limit", "kernel.logger", [ + {default, "disabled"}, + {datatype, string} +]}. + +{mapping, "log.error_logger", "kernel.error_logger", [ + {default, silent}, + {datatype, {enum, [silent]}}, + hidden +]}. + +{translation, "kernel.logger_level", fun(_, _, Conf) -> + cuttlefish:conf_get("log.level", Conf) +end}. + +{translation, "kernel.logger", fun(Conf) -> + LogTo = cuttlefish:conf_get("log.to", Conf), + LogLevel = cuttlefish:conf_get("log.level", Conf), + LogType = case cuttlefish:conf_get("log.rotation", Conf) of + true -> wrap; + false -> halt + end, + CharsLimit = case cuttlefish:conf_get("log.chars_limit", Conf) of + -1 -> unlimited; + V -> V + end, + SingleLine = cuttlefish:conf_get("log.single_line", Conf), + FmtName = cuttlefish:conf_get("log.formatter", Conf), + Formatter = + case FmtName of + json -> + {emqx_logger_jsonfmt, + #{chars_limit => CharsLimit, + single_line => SingleLine + }}; + text -> + {emqx_logger_textfmt, + #{template => + [time," [",level,"] ", + {clientid, + [{peername, + [clientid,"@",peername," "], + [clientid, " "]}], + [{peername, + [peername," "], + []}]}, + msg,"\n"], + chars_limit => CharsLimit, + single_line => SingleLine + }} + end, + {BustLimitOn, {MaxBurstCount, TimeWindow}} = + case string:tokens(cuttlefish:conf_get("log.burst_limit", Conf), ", ") of + ["disabled"] -> {false, {20000, 1000}}; + [Count, Window] -> + {true, {list_to_integer(Count), + case cuttlefish_duration:parse(Window, ms) of + Secs when is_integer(Secs) -> Secs; + {error, Reason1} -> error(Reason1) + end}} + end, + FileConf = fun(Filename) -> + BasicConf = + #{type => LogType, + file => filename:join(cuttlefish:conf_get("log.dir", Conf), Filename), + max_no_files => cuttlefish:conf_get("log.rotation.count", Conf), + sync_mode_qlen => cuttlefish:conf_get("log.sync_mode_qlen", Conf), + drop_mode_qlen => cuttlefish:conf_get("log.drop_mode_qlen", Conf), + flush_qlen => cuttlefish:conf_get("log.flush_qlen", Conf), + overload_kill_enable => cuttlefish:conf_get("log.overload_kill", Conf), + overload_kill_qlen => cuttlefish:conf_get("log.overload_kill_qlen", Conf), + overload_kill_mem_size => cuttlefish:conf_get("log.overload_kill_mem_size", Conf), + overload_kill_restart_after => cuttlefish:conf_get("log.overload_kill_restart_after", Conf), + burst_limit_enable => BustLimitOn, + burst_limit_max_count => MaxBurstCount, + burst_limit_window_time => TimeWindow + }, + MaxNoBytes = case LogType of + wrap -> cuttlefish:conf_get("log.rotation.size", Conf); + halt -> cuttlefish:conf_get("log.size", Conf) + end, + BasicConf#{max_no_bytes => MaxNoBytes} + end, + + Filters = case cuttlefish:conf_get("log.supervisor_reports", Conf) of + error -> [{drop_progress_reports, {fun logger_filters:progress/2, stop}}]; + progress -> [] + end, + + %% For the default logger that outputs to console + DefaultHandler = + if LogTo =:= console orelse LogTo =:= both -> + [{handler, console, logger_std_h, + #{level => LogLevel, + config => #{type => standard_io}, + formatter => Formatter, + filters => Filters + } + }]; + true -> + [{handler, default, undefined}] + end, + + %% For the file logger + FileHandler = + if LogTo =:= file orelse LogTo =:= both -> + [{handler, file, logger_disk_log_h, + #{level => LogLevel, + config => FileConf(cuttlefish:conf_get("log.file", Conf)), + formatter => Formatter, + filesync_repeat_interval => no_repeat, + filters => Filters + }}]; + true -> [] + end, + + %% For creating additional log files for specific log levels. + AdditionalLogFiles = + lists:foldl( + fun({[_, Level, _] = K, Filename}, Acc) when LogTo =:= file; LogTo =:= both -> + case cuttlefish_variable:is_fuzzy_match(K, ["log", "$level", "file"]) of + true -> [{Level, Filename} | Acc]; + false -> Acc + end; + ({_K, _V}, Acc) -> + Acc + end, [], Conf), + AdditionalHandlers = + [{handler, list_to_atom("file_for_"++Level), logger_disk_log_h, + #{level => list_to_atom(Level), + config => FileConf(Filename), + formatter => Formatter, + filesync_repeat_interval => no_repeat}} + || {Level, Filename} <- AdditionalLogFiles], + + DefaultHandler ++ FileHandler ++ AdditionalHandlers +end}. + +%%-------------------------------------------------------------------- +%% Authentication/ACL +%%-------------------------------------------------------------------- + +%% @doc Allow anonymous authentication. +{mapping, "allow_anonymous", "emqx.allow_anonymous", [ + {default, false}, + {datatype, {enum, [true, false]}} +]}. + +%% @doc ACL nomatch. +{mapping, "acl_nomatch", "emqx.acl_nomatch", [ + {default, deny}, + {datatype, {enum, [allow, deny]}} +]}. + +%% @doc Default ACL file. +{mapping, "acl_file", "emqx.acl_file", [ + {datatype, string}, + hidden +]}. + +%% @doc Enable ACL cache for publish. +{mapping, "enable_acl_cache", "emqx.enable_acl_cache", [ + {default, on}, + {datatype, flag} +]}. + +%% @doc ACL cache time-to-live. +{mapping, "acl_cache_ttl", "emqx.acl_cache_ttl", [ + {default, "1m"}, + {datatype, {duration, ms}} +]}. + +%% @doc ACL cache size. +{mapping, "acl_cache_max_size", "emqx.acl_cache_max_size", [ + {default, 32}, + {datatype, integer}, + {validators, ["range:gt_0"]} +]}. + +%% @doc Action when acl check reject current operation +{mapping, "acl_deny_action", "emqx.acl_deny_action", [ + {default, ignore}, + {datatype, {enum, [ignore, disconnect]}} +]}. + +%% @doc Flapping detect policy +{mapping, "flapping_detect_policy", "emqx.flapping_detect_policy", [ + {datatype, string}, + {default, "30,1m,5m"} +]}. + +{translation, "emqx.flapping_detect_policy", fun(Conf) -> + Policy = cuttlefish:conf_get("flapping_detect_policy", Conf), + [Threshold, Duration, Interval] = string:tokens(Policy, ", "), + ParseDuration = fun(S, Dur) -> + case cuttlefish_duration:parse(S, Dur) of + I when is_integer(I) -> I; + {error, Reason} -> error(Reason) + end + end, + #{threshold => list_to_integer(Threshold), + duration => ParseDuration(Duration, ms), + banned_interval => ParseDuration(Interval, s) + } +end}. + +{validator, "range:gt_0", "must greater than 0", + fun(X) -> X > 0 end +}. + +%%-------------------------------------------------------------------- +%% MQTT Protocol +%%-------------------------------------------------------------------- + +%% @doc Max Packet Size Allowed, 1MB by default. +{mapping, "mqtt.max_packet_size", "emqx.max_packet_size", [ + {default, "1MB"}, + {datatype, bytesize}, + {override_env, "MAX_PACKET_SIZE"} +]}. + +%% @doc Set the Max ClientId Length Allowed. +{mapping, "mqtt.max_clientid_len", "emqx.max_clientid_len", [ + {default, 65535}, + {datatype, integer} +]}. + +%% @doc Set the Maximum topic levels. +{mapping, "mqtt.max_topic_levels", "emqx.max_topic_levels", [ + {default, 0}, + {datatype, integer} +]}. + +%% @doc Set the Maximum QoS allowed. +{mapping, "mqtt.max_qos_allowed", "emqx.max_qos_allowed", [ + {default, 2}, + {datatype, integer}, + {validators, ["range:0-2"]} +]}. + +%% @doc Set the Maximum Topic Alias. +{mapping, "mqtt.max_topic_alias", "emqx.max_topic_alias", [ + {default, 65535}, + {datatype, integer} +]}. + +%% @doc Whether the server supports MQTT retained messages. +{mapping, "mqtt.retain_available", "emqx.retain_available", [ + {default, true}, + {datatype, {enum, [true, false]}} +]}. + +%% @doc Whether the Server supports MQTT Wildcard Subscriptions. +{mapping, "mqtt.wildcard_subscription", "emqx.wildcard_subscription", [ + {default, true}, + {datatype, {enum, [true, false]}} +]}. + +%% @doc Whether the Server supports MQTT Shared Subscriptions. +{mapping, "mqtt.shared_subscription", "emqx.shared_subscription", [ + {default, true}, + {datatype, {enum, [true, false]}} +]}. + +%% @doc Whether to ignore loop delivery of messages.(for mqtt v3.1.1) +{mapping, "mqtt.ignore_loop_deliver", "emqx.ignore_loop_deliver", [ + {default, true}, + {datatype, {enum, [true, false]}} +]}. + +%% @doc Whether to parse the MQTT frame in strict mode +{mapping, "mqtt.strict_mode", "emqx.strict_mode", [ + {default, false}, + {datatype, {enum, [true, false]}} +]}. + +%% @doc Specify the response information returned to the client +{mapping, "mqtt.response_information", "emqx.response_information", [ + {datatype, string} +]}. + +%%-------------------------------------------------------------------- +%% Zones +%%-------------------------------------------------------------------- + +%% @doc Idle timeout of the MQTT connection. +{mapping, "zone.$name.idle_timeout", "emqx.zones", [ + {default, "15s"}, + {datatype, {duration, ms}} +]}. + +{mapping, "zone.$name.allow_anonymous", "emqx.zones", [ + {datatype, {enum, [true, false]}} +]}. + +{mapping, "zone.$name.acl_nomatch", "emqx.zones", [ + {datatype, {enum, [allow, deny]}} +]}. + +%% @doc Enable ACL check. +{mapping, "zone.$name.enable_acl", "emqx.zones", [ + {default, off}, + {datatype, flag} +]}. + +%% @doc Action when acl check reject current operation +{mapping, "zone.$name.acl_deny_action", "emqx.zones", [ + {default, ignore}, + {datatype, {enum, [ignore, disconnect]}} +]}. + +%% @doc Enable Ban. +{mapping, "zone.$name.enable_ban", "emqx.zones", [ + {default, off}, + {datatype, flag} +]}. + +%% @doc Enable per connection statistics. +{mapping, "zone.$name.enable_stats", "emqx.zones", [ + {default, off}, + {datatype, flag} +]}. + +%% @doc Publish limit of the MQTT connections. +{mapping, "zone.$name.publish_limit", "emqx.zones", [ + {datatype, string} +]}. + +%% @doc Max Packet Size Allowed, 64K by default. +{mapping, "zone.$name.max_packet_size", "emqx.zones", [ + {datatype, bytesize} +]}. + +%% @doc Set the Max ClientId Length Allowed. +{mapping, "zone.$name.max_clientid_len", "emqx.zones", [ + {datatype, integer} +]}. + +%% @doc Set the Maximum topic levels. +{mapping, "zone.$name.max_topic_levels", "emqx.zones", [ + {datatype, integer} +]}. + +%% @doc Set the Maximum QoS allowed. +{mapping, "zone.$name.max_qos_allowed", "emqx.zones", [ + {datatype, integer}, + {validators, ["range:0-2"]} +]}. + +%% @doc Set the Maximum topic alias. +{mapping, "zone.$name.max_topic_alias", "emqx.zones", [ + {datatype, integer} +]}. + +%% @doc Whether the server supports retained messages. +{mapping, "zone.$name.retain_available", "emqx.zones", [ + {datatype, {enum, [true, false]}} +]}. + +%% @doc Whether the Server supports Wildcard Subscriptions. +{mapping, "zone.$name.wildcard_subscription", "emqx.zones", [ + {datatype, {enum, [true, false]}} +]}. + +%% @doc Whether the Server supports Shared Subscriptions. +{mapping, "zone.$name.shared_subscription", "emqx.zones", [ + {datatype, {enum, [true, false]}} +]}. + +%% @doc Server Keepalive +{mapping, "zone.$name.server_keepalive", "emqx.zones", [ + {datatype, integer} +]}. + +%% @doc Keepalive backoff +{mapping, "zone.$name.keepalive_backoff", "emqx.zones", [ + {default, 0.75}, + {datatype, float} +]}. + +%% @doc Max Number of Subscriptions Allowed. +{mapping, "zone.$name.max_subscriptions", "emqx.zones", [ + {default, 0}, + {datatype, integer} +]}. + +%% @doc Upgrade QoS according to subscription? +{mapping, "zone.$name.upgrade_qos", "emqx.zones", [ + {default, off}, + {datatype, flag} +]}. + +%% @doc Max number of QoS 1 and 2 messages that can be “inflight” at one time. +%% 0 is equivalent to maximum allowed +{mapping, "zone.$name.max_inflight", "emqx.zones", [ + {default, 0}, + {datatype, integer}, + {validators, ["range:1-65535"]} +]}. + +%% @doc Retry interval for redelivering QoS1/2 messages. +{mapping, "zone.$name.retry_interval", "emqx.zones", [ + {default, "30s"}, + {datatype, {duration, s}} +]}. + +%% @doc Max Packets that Awaiting PUBREL, 0 means no limit +{mapping, "zone.$name.max_awaiting_rel", "emqx.zones", [ + {default, 0}, + {datatype, integer} +]}. + +%% @doc Awaiting PUBREL timeout +{mapping, "zone.$name.await_rel_timeout", "emqx.zones", [ + {default, "300s"}, + {datatype, {duration, s}} +]}. + +%% @doc Ignore loop delivery of messages +{mapping, "zone.$name.ignore_loop_deliver", "emqx.zones", [ + {datatype, {enum, [true, false]}} +]}. + +%% @doc Session Expiry Interval +{mapping, "zone.$name.session_expiry_interval", "emqx.zones", [ + {default, "2h"}, + {datatype, {duration, s}} +]}. + +%% @doc Max queue length. Enqueued messages when persistent client +%% disconnected, or inflight window is full. 0 means no limit. +{mapping, "zone.$name.max_mqueue_len", "emqx.zones", [ + {default, 1000}, + {datatype, integer} +]}. + +%% @doc Topic Priorities, comma separated topic=priority pairs, +%% where priority should be integer in range 1-255 (inclusive) +%% 1 being the lowest and 255 being the highest. +%% default value `none` to indicate no priority table, hence all +%% messages are treated equal, which means either highest ('infinity'), +%% or lowest (0) depending on mqueue_default_priority config. +{mapping, "zone.$name.mqueue_priorities", "emqx.zones", [ + {default, "none"}, + {datatype, string} +]}. + +%% @doc Default priority for topics not in priority table. +{mapping, "zone.$name.mqueue_default_priority", "emqx.zones", [ + {default, lowest}, + {datatype, {enum, [highest, lowest]}} +]}. + +%% @doc Queue Qos0 messages? +{mapping, "zone.$name.mqueue_store_qos0", "emqx.zones", [ + {default, true}, + {datatype, {enum, [true, false]}} +]}. + +{mapping, "zone.$name.enable_flapping_detect", "emqx.zones", [ + {datatype, flag}, + {default, off} +]}. + +{mapping, "zone.$name.rate_limit.conn_messages_in", "emqx.zones", [ + {datatype, string} +]}. + +{mapping, "zone.$name.rate_limit.conn_bytes_in", "emqx.zones", [ + {datatype, string} +]}. + +{mapping, "zone.$name.conn_congestion.alarm", "emqx.zones", [ + {datatype, flag}, + {default, off} +]}. + +{mapping, "zone.$name.conn_congestion.min_alarm_sustain_duration", "emqx.zones", [ + {default, "1m"}, + {datatype, {duration, ms}} +]}. + +{mapping, "zone.$name.quota.conn_messages_routing", "emqx.zones", [ + {datatype, string} +]}. + +{mapping, "zone.$name.quota.overall_messages_routing", "emqx.zones", [ + {datatype, string} +]}. + +%% @doc Force connection/session process GC after this number of +%% messages | bytes passed through. +%% Numbers delimited by `|'. Zero or negative is to disable. +{mapping, "zone.$name.force_gc_policy", "emqx.zones", [ + {datatype, string} + ]}. + +%% @doc Max message queue length and total heap size to force shutdown +%% connection/session process. +%% Message queue here is the Erlang process mailbox, but not the number +%% of queued MQTT messages of QoS 1 and 2. +%% Zero or negative is to disable. +{mapping, "zone.$name.force_shutdown_policy", "emqx.zones", [ + {default, "default"}, + {datatype, string} +]}. + +{mapping, "zone.$name.mountpoint", "emqx.zones", [ + {datatype, string} +]}. + +%% @doc Use username replace client id +{mapping, "zone.$name.use_username_as_clientid", "emqx.zones", [ + {default, false}, + {datatype, {enum, [true, false]}} +]}. + +%% @doc Whether to parse the MQTT frame in strict mode +{mapping, "zone.$name.strict_mode", "emqx.zones", [ + {default, false}, + {datatype, {enum, [true, false]}} +]}. + +%% @doc Specify the response information returned to the client +{mapping, "zone.$name.response_information", "emqx.zones", [ + {datatype, string} +]}. + +%% @doc Whether to bypass the authentication step +{mapping, "zone.$name.bypass_auth_plugins", "emqx.zones", [ + {default, false}, + {datatype, {enum, [true, false]}} +]}. + +{translation, "emqx.zones", fun(Conf) -> + Ratelimit = fun(Val) -> + [L, D] = string:tokens(Val, ", "), + Limit = case cuttlefish_bytesize:parse(L) of + Sz when is_integer(Sz) -> Sz; + {error, Reason1} -> error(Reason1) + end, + Duration = case cuttlefish_duration:parse(D, s) of + Secs when is_integer(Secs) -> Secs; + {error, Reason} -> error(Reason) + end, + {Limit, Duration} + end, + Mapping = fun(["publish_limit"], Val) -> + %% XXX: Deprecated at v4.2 + {publish_limit, Ratelimit(Val)}; + (["force_gc_policy"], Val) -> + [Count, Bytes] = string:tokens(Val, "| "), + GcPolicy = case cuttlefish_bytesize:parse(Bytes) of + {error, Reason} -> + error(Reason); + Bytes1 -> + #{bytes => Bytes1, + count => list_to_integer(Count)} + end, + {force_gc_policy, GcPolicy}; + (["force_shutdown_policy"], "default") -> + {DefaultLen, DefaultSize} = + case WordSize = erlang:system_info(wordsize) of + 8 -> % arch_64 + {10000, cuttlefish_bytesize:parse("64MB")}; + 4 -> % arch_32 + {1000, cuttlefish_bytesize:parse("32MB")} + end, + {force_shutdown_policy, #{message_queue_len => DefaultLen, + max_heap_size => DefaultSize div WordSize + }}; + (["force_shutdown_policy"], Val) -> + [Len, Siz] = string:tokens(Val, "| "), + MaxSiz = case WordSize = erlang:system_info(wordsize) of + 8 -> % arch_64 + (1 bsl 59) - 1; + 4 -> % arch_32 + (1 bsl 27) - 1 + end, + ShutdownPolicy = + case cuttlefish_bytesize:parse(Siz) of + {error, Reason} -> + error(Reason); + Siz1 when Siz1 > MaxSiz -> + cuttlefish:invalid(io_lib:format("force_shutdown_policy: heap-size ~s is too large", [Siz])); + Siz1 -> + #{message_queue_len => list_to_integer(Len), + max_heap_size => Siz1 div WordSize} + end, + {force_shutdown_policy, ShutdownPolicy}; + (["mqueue_priorities"], Val) -> + case Val of + "none" -> {mqueue_priorities, none}; % NO_PRIORITY_TABLE + _ -> + MqueuePriorities = lists:foldl(fun(T, Acc) -> + %% NOTE: space in "= " is intended + [Topic, Prio] = string:tokens(T, "= "), + P = list_to_integer(Prio), + (P < 0 orelse P > 255) andalso error({bad_priority, Topic, Prio}), + maps:put(iolist_to_binary(Topic), P, Acc) + end, #{}, string:tokens(Val, ",")), + {mqueue_priorities, MqueuePriorities} + end; + (["mountpoint"], Val) -> + {mountpoint, iolist_to_binary(Val)}; + (["response_information"], Val) -> + {response_information, iolist_to_binary(Val)}; + (["rate_limit", "conn_messages_in"], Val) -> + {ratelimit, {conn_messages_in, Ratelimit(Val)}}; + (["rate_limit", "conn_bytes_in"], Val) -> + {ratelimit, {conn_bytes_in, Ratelimit(Val)}}; + (["conn_congestion", "alarm"], Val) -> + {conn_congestion_alarm_enabled, Val}; + (["conn_congestion", "min_alarm_sustain_duration"], Val) -> + {conn_congestion_min_alarm_sustain_duration, Val}; + (["quota", "conn_messages_routing"], Val) -> + {quota, {conn_messages_routing, Ratelimit(Val)}}; + (["quota", "overall_messages_routing"], Val) -> + {quota, {overall_messages_routing, Ratelimit(Val)}}; + ([Opt], Val) -> + {list_to_atom(Opt), Val} + end, + maps:to_list( + lists:foldl( + fun({["zone", Name | Opt], Val}, Zones) -> + NVal = Mapping(Opt, Val), + maps:update_with(list_to_atom(Name), + fun(Opts) -> + case NVal of + {Key, Rl} when Key == ratelimit; + Key == quota -> + Rls = proplists:get_value(Key, Opts, []), + lists:keystore(Key, 1, Opts, {Key, [Rl|Rls]}); + _ -> + [NVal|Opts] + end + end, [NVal], Zones) + end, #{}, lists:usort(cuttlefish_variable:filter_by_prefix("zone.", Conf)))) +end}. + +%%-------------------------------------------------------------------- +%% Listeners +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% TCP Listeners + +{mapping, "listener.tcp.$name", "emqx.listeners", [ + {datatype, [integer, ip]} +]}. + +{mapping, "listener.tcp.$name.acceptors", "emqx.listeners", [ + {default, 8}, + {datatype, integer} +]}. + +{mapping, "listener.tcp.$name.max_connections", "emqx.listeners", [ + {default, 1024}, + {datatype, integer} +]}. + +{mapping, "listener.tcp.$name.max_conn_rate", "emqx.listeners", [ + {datatype, integer} +]}. + +{mapping, "listener.tcp.$name.active_n", "emqx.listeners", [ + {default, 100}, + {datatype, integer} +]}. + +{mapping, "listener.tcp.$name.zone", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.tcp.$name.rate_limit", "emqx.listeners", [ + {default, undefined}, + {datatype, string} +]}. + +{mapping, "listener.tcp.$name.access.$id", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.tcp.$name.proxy_protocol", "emqx.listeners", [ + {datatype, flag} +]}. + +{mapping, "listener.tcp.$name.proxy_protocol_timeout", "emqx.listeners", [ + {datatype, {duration, ms}} +]}. + +%% The proxy-protocol protocol can get the certificate CN through tcp +{mapping, "listener.tcp.$name.peer_cert_as_username", "emqx.listeners", [ + {datatype, {enum, [cn]}} +]}. + +%% The proxy-protocol protocol can get the certificate CN through tcp +{mapping, "listener.tcp.$name.peer_cert_as_clientid", "emqx.listeners", [ + {datatype, {enum, [cn]}} +]}. + +{mapping, "listener.tcp.$name.backlog", "emqx.listeners", [ + {datatype, integer}, + {default, 1024} +]}. + +{mapping, "listener.tcp.$name.send_timeout", "emqx.listeners", [ + {datatype, {duration, ms}}, + {default, "15s"} +]}. + +{mapping, "listener.tcp.$name.send_timeout_close", "emqx.listeners", [ + {datatype, flag}, + {default, on} +]}. + +{mapping, "listener.tcp.$name.recbuf", "emqx.listeners", [ + {datatype, bytesize}, + hidden +]}. + +{mapping, "listener.tcp.$name.sndbuf", "emqx.listeners", [ + {datatype, bytesize}, + hidden +]}. + +{mapping, "listener.tcp.$name.buffer", "emqx.listeners", [ + {datatype, bytesize}, + hidden +]}. + +{mapping, "listener.tcp.$name.high_watermark", "emqx.listeners", [ + {datatype, bytesize}, + {default, "1MB"} + ]}. + +{mapping, "listener.tcp.$name.tune_buffer", "emqx.listeners", [ + {datatype, flag}, + hidden +]}. + +{mapping, "listener.tcp.$name.nodelay", "emqx.listeners", [ + {datatype, {enum, [true, false]}}, + hidden +]}. + +{mapping, "listener.tcp.$name.reuseaddr", "emqx.listeners", [ + {datatype, {enum, [true, false]}}, + hidden +]}. + +%%-------------------------------------------------------------------- +%% SSL Listeners + +{mapping, "listener.ssl.$name", "emqx.listeners", [ + {datatype, [integer, ip]} +]}. + +{mapping, "listener.ssl.$name.acceptors", "emqx.listeners", [ + {default, 8}, + {datatype, integer} +]}. + +{mapping, "listener.ssl.$name.max_connections", "emqx.listeners", [ + {default, 1024}, + {datatype, integer} +]}. + +{mapping, "listener.ssl.$name.max_conn_rate", "emqx.listeners", [ + {datatype, integer} +]}. + +{mapping, "listener.ssl.$name.active_n", "emqx.listeners", [ + {default, 100}, + {datatype, integer} +]}. + +{mapping, "listener.ssl.$name.zone", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.ssl.$name.rate_limit", "emqx.listeners", [ + {default, undefined}, + {datatype, string} +]}. + +{mapping, "listener.ssl.$name.access.$id", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.ssl.$name.proxy_protocol", "emqx.listeners", [ + {datatype, flag} +]}. + +{mapping, "listener.ssl.$name.proxy_protocol_timeout", "emqx.listeners", [ + {datatype, {duration, ms}} +]}. + +{mapping, "listener.ssl.$name.backlog", "emqx.listeners", [ + {default, 1024}, + {datatype, integer} +]}. + +{mapping, "listener.ssl.$name.send_timeout", "emqx.listeners", [ + {datatype, {duration, ms}}, + {default, "15s"} +]}. + +{mapping, "listener.ssl.$name.send_timeout_close", "emqx.listeners", [ + {datatype, flag}, + {default, on} +]}. + +{mapping, "listener.ssl.$name.recbuf", "emqx.listeners", [ + {datatype, bytesize}, + hidden +]}. + +{mapping, "listener.ssl.$name.sndbuf", "emqx.listeners", [ + {datatype, bytesize}, + hidden +]}. + +{mapping, "listener.ssl.$name.buffer", "emqx.listeners", [ + {datatype, bytesize}, + hidden +]}. + +{mapping, "listener.ssl.$name.high_watermark", "emqx.listeners", [ + {datatype, bytesize}, + {default, "1MB"} + ]}. + +{mapping, "listener.ssl.$name.tune_buffer", "emqx.listeners", [ + {datatype, flag}, + hidden +]}. + +{mapping, "listener.ssl.$name.nodelay", "emqx.listeners", [ + {datatype, {enum, [true, false]}}, + hidden +]}. + +{mapping, "listener.ssl.$name.reuseaddr", "emqx.listeners", [ + {datatype, {enum, [true, false]}}, + hidden +]}. + +{mapping, "listener.ssl.$name.tls_versions", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.ssl.$name.ciphers", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.ssl.$name.psk_ciphers", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.ssl.$name.handshake_timeout", "emqx.listeners", [ + {default, "15s"}, + {datatype, {duration, ms}} +]}. + +{mapping, "listener.ssl.$name.depth", "emqx.listeners", [ + {default, 10}, + {datatype, integer} +]}. + +{mapping, "listener.ssl.$name.key_password", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.ssl.$name.dhfile", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.ssl.$name.keyfile", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.ssl.$name.certfile", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.ssl.$name.cacertfile", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.ssl.$name.verify", "emqx.listeners", [ + {datatype, atom} +]}. + +{mapping, "listener.ssl.$name.fail_if_no_peer_cert", "emqx.listeners", [ + {datatype, {enum, [true, false]}} +]}. + +{mapping, "listener.ssl.$name.secure_renegotiate", "emqx.listeners", [ + {datatype, flag} +]}. + +{mapping, "listener.ssl.$name.reuse_sessions", "emqx.listeners", [ + {default, on}, + {datatype, flag} +]}. + +{mapping, "listener.ssl.$name.honor_cipher_order", "emqx.listeners", [ + {datatype, flag} +]}. + +{mapping, "listener.ssl.$name.peer_cert_as_username", "emqx.listeners", [ + {datatype, {enum, [cn, dn, crt, pem, md5]}} +]}. + +{mapping, "listener.ssl.$name.peer_cert_as_clientid", "emqx.listeners", [ + {datatype, {enum, [cn, dn, crt, pem, md5]}} +]}. + +%%-------------------------------------------------------------------- +%% MQTT/WebSocket Listeners + +{mapping, "listener.ws.$name", "emqx.listeners", [ + {datatype, [integer, ip]} +]}. + +{mapping, "listener.ws.$name.mqtt_path", "emqx.listeners", [ + {default, "/mqtt"}, + {datatype, string} +]}. + +{mapping, "listener.ws.$name.acceptors", "emqx.listeners", [ + {default, 8}, + {datatype, integer} +]}. + +{mapping, "listener.ws.$name.max_connections", "emqx.listeners", [ + {default, 1024}, + {datatype, integer} +]}. + +{mapping, "listener.ws.$name.max_conn_rate", "emqx.listeners", [ + {datatype, integer} +]}. + +{mapping, "listener.ws.$name.active_n", "emqx.listeners", [ + {default, 100}, + {datatype, integer} +]}. + +{mapping, "listener.ws.$name.zone", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.ws.$name.rate_limit", "emqx.listeners", [ + {default, undefined}, + {datatype, string} +]}. + +{mapping, "listener.ws.$name.access.$id", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.ws.$name.fail_if_no_subprotocol", "emqx.listeners", [ + {default, true}, + {datatype, {enum, [true, false]}} +]}. + +{mapping, "listener.ws.$name.supported_subprotocols", "emqx.listeners", [ + {default, "mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5"}, + {datatype, string} +]}. + +{mapping, "listener.ws.$name.proxy_address_header", "emqx.listeners", [ + {default, "X-Forwarded-For"}, + {datatype, string} +]}. + +{mapping, "listener.ws.$name.proxy_port_header", "emqx.listeners", [ + {default, "X-Forwarded-Port"}, + {datatype, string} +]}. + +{mapping, "listener.ws.$name.proxy_protocol", "emqx.listeners", [ + {datatype, flag} +]}. + +{mapping, "listener.ws.$name.proxy_protocol_timeout", "emqx.listeners", [ + {datatype, {duration, ms}} +]}. + +{mapping, "listener.ws.$name.backlog", "emqx.listeners", [ + {default, 1024}, + {datatype, integer} +]}. + +{mapping, "listener.ws.$name.send_timeout", "emqx.listeners", [ + {datatype, {duration, ms}}, + {default, "15s"} +]}. + +{mapping, "listener.ws.$name.send_timeout_close", "emqx.listeners", [ + {datatype, flag}, + {default, on} +]}. + +{mapping, "listener.ws.$name.recbuf", "emqx.listeners", [ + {datatype, bytesize}, + hidden +]}. + +{mapping, "listener.ws.$name.sndbuf", "emqx.listeners", [ + {datatype, bytesize}, + hidden +]}. + +{mapping, "listener.ws.$name.buffer", "emqx.listeners", [ + {datatype, bytesize}, + hidden +]}. + +{mapping, "listener.ws.$name.tune_buffer", "emqx.listeners", [ + {datatype, flag}, + hidden +]}. + +{mapping, "listener.ws.$name.nodelay", "emqx.listeners", [ + {datatype, {enum, [true, false]}}, + hidden +]}. + +{mapping, "listener.ws.$name.compress", "emqx.listeners", [ + {datatype, {enum, [true, false]}}, + hidden +]}. + +{mapping, "listener.ws.$name.deflate_opts.level", "emqx.listeners", [ + {datatype, {enum, [none, default, best_compression, best_speed]}}, + hidden +]}. + +{mapping, "listener.ws.$name.deflate_opts.mem_level", "emqx.listeners", [ + {datatype, integer}, + {validators, ["range:1-9"]}, + hidden +]}. + +{mapping, "listener.ws.$name.deflate_opts.strategy", "emqx.listeners", [ + {datatype, {enum, [default, filtered, huffman_only, rle]}}, + hidden +]}. + +{mapping, "listener.ws.$name.deflate_opts.server_context_takeover", "emqx.listeners", [ + {datatype, {enum, [takeover, no_takeover]}}, + hidden +]}. + +{mapping, "listener.ws.$name.deflate_opts.client_context_takeover", "emqx.listeners", [ + {datatype, {enum, [takeover, no_takeover]}}, + hidden +]}. + +{mapping, "listener.ws.$name.deflate_opts.server_max_window_bits", "emqx.listeners", [ + {datatype, integer}, + hidden +]}. + +{mapping, "listener.ws.$name.deflate_opts.client_max_window_bits", "emqx.listeners", [ + {datatype, integer}, + hidden +]}. + +{mapping, "listener.ws.$name.idle_timeout", "emqx.listeners", [ + {datatype, {duration, ms}}, + hidden +]}. + +{mapping, "listener.ws.$name.max_frame_size", "emqx.listeners", [ + {datatype, integer}, + hidden +]}. + +{mapping, "listener.ws.$name.mqtt_piggyback", "emqx.listeners", [ + {datatype, {enum, [single, multiple]}}, + {default, multiple}, + hidden +]}. + +{mapping, "listener.ws.$name.peer_cert_as_username", "emqx.listeners", [ + {datatype, {enum, [cn]}} +]}. + +{mapping, "listener.ws.$name.peer_cert_as_clientid", "emqx.listeners", [ + {datatype, {enum, [cn]}} +]}. + +{mapping, "listener.ws.$name.check_origin_enable", "emqx.listeners", [ + {datatype, {enum, [true, false]}}, + {default, false}, + hidden +]}. + +{mapping, "listener.ws.$name.allow_origin_absence", "emqx.listeners", [ + {datatype, {enum, [true, false]}}, + {default, true}, + hidden +]}. + +{mapping, "listener.ws.$name.check_origins", "emqx.listeners", [ + {datatype, string}, + hidden +]}. + +%%-------------------------------------------------------------------- +%% MQTT/WebSocket/SSL Listeners + +{mapping, "listener.wss.$name", "emqx.listeners", [ + {datatype, [integer, ip]} +]}. + +{mapping, "listener.wss.$name.mqtt_path", "emqx.listeners", [ + {default, "/mqtt"}, + {datatype, string} +]}. + +{mapping, "listener.wss.$name.acceptors", "emqx.listeners", [ + {default, 8}, + {datatype, integer} +]}. + +{mapping, "listener.wss.$name.max_connections", "emqx.listeners", [ + {default, 1024}, + {datatype, integer} +]}. + +{mapping, "listener.wss.$name.max_conn_rate", "emqx.listeners", [ + {datatype, integer} +]}. + +{mapping, "listener.wss.$name.active_n", "emqx.listeners", [ + {default, 100}, + {datatype, integer} +]}. + +{mapping, "listener.wss.$name.zone", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.wss.$name.rate_limit", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.wss.$name.fail_if_no_subprotocol", "emqx.listeners", [ + {default, true}, + {datatype, {enum, [true, false]}} +]}. + +{mapping, "listener.wss.$name.supported_subprotocols", "emqx.listeners", [ + {default, "mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5"}, + {datatype, string} +]}. + +{mapping, "listener.wss.$name.access.$id", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.wss.$name.proxy_address_header", "emqx.listeners", [ + {default, "X-Forwarded-For"}, + {datatype, string} +]}. + +{mapping, "listener.wss.$name.proxy_port_header", "emqx.listeners", [ + {default, "X-Forwarded-Port"}, + {datatype, string} +]}. + +{mapping, "listener.wss.$name.proxy_protocol", "emqx.listeners", [ + {datatype, flag} +]}. + +{mapping, "listener.wss.$name.proxy_protocol_timeout", "emqx.listeners", [ + {datatype, {duration, ms}} +]}. + +%%{mapping, "listener.wss.$name.handshake_timeout", "emqx.listeners", [ +%% {default, "15s"}, +%% {datatype, {duration, ms}} +%%]}. + +{mapping, "listener.wss.$name.backlog", "emqx.listeners", [ + {default, 1024}, + {datatype, integer} +]}. + +{mapping, "listener.wss.$name.send_timeout", "emqx.listeners", [ + {datatype, {duration, ms}}, + {default, "15s"} +]}. + +{mapping, "listener.wss.$name.send_timeout_close", "emqx.listeners", [ + {datatype, flag}, + {default, on} +]}. + +{mapping, "listener.wss.$name.recbuf", "emqx.listeners", [ + {datatype, bytesize}, + hidden +]}. + +{mapping, "listener.wss.$name.sndbuf", "emqx.listeners", [ + {datatype, bytesize}, + hidden +]}. + +{mapping, "listener.wss.$name.buffer", "emqx.listeners", [ + {datatype, bytesize}, + hidden +]}. + +{mapping, "listener.wss.$name.tune_buffer", "emqx.listeners", [ + {datatype, flag}, + hidden +]}. + +{mapping, "listener.wss.$name.nodelay", "emqx.listeners", [ + {datatype, {enum, [true, false]}}, + hidden +]}. + +{mapping, "listener.wss.$name.tls_versions", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.wss.$name.ciphers", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.wss.$name.psk_ciphers", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.wss.$name.keyfile", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.wss.$name.certfile", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.wss.$name.cacertfile", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.wss.$name.dhfile", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.wss.$name.depth", "emqx.listeners", [ + {default, 10}, + {datatype, integer} +]}. + +{mapping, "listener.wss.$name.key_password", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.wss.$name.verify", "emqx.listeners", [ + {datatype, atom} +]}. + +{mapping, "listener.wss.$name.fail_if_no_peer_cert", "emqx.listeners", [ + {datatype, {enum, [true, false]}} +]}. + +{mapping, "listener.wss.$name.secure_renegotiate", "emqx.listeners", [ + {datatype, flag} +]}. + +{mapping, "listener.wss.$name.reuse_sessions", "emqx.listeners", [ + {default, on}, + {datatype, flag} +]}. + +{mapping, "listener.wss.$name.honor_cipher_order", "emqx.listeners", [ + {datatype, flag} +]}. + +{mapping, "listener.wss.$name.peer_cert_as_username", "emqx.listeners", [ + {datatype, {enum, [cn, dn, crt, pem, md5]}} +]}. + +{mapping, "listener.wss.$name.peer_cert_as_clientid", "emqx.listeners", [ + {datatype, {enum, [cn, dn, crt, pem, md5]}} +]}. + +{mapping, "listener.wss.$name.compress", "emqx.listeners", [ + {datatype, {enum, [true, false]}}, + hidden +]}. + +{mapping, "listener.wss.$name.deflate_opts.level", "emqx.listeners", [ + {datatype, {enum, [none, default, best_compression, best_speed]}}, + hidden +]}. + +{mapping, "listener.wss.$name.deflate_opts.mem_level", "emqx.listeners", [ + {datatype, integer}, + {validators, ["range:1-9"]}, + hidden +]}. + +{mapping, "listener.wss.$name.deflate_opts.strategy", "emqx.listeners", [ + {datatype, {enum, [default, filtered, huffman_only, rle]}}, + hidden +]}. + +{mapping, "listener.wss.$name.deflate_opts.server_context_takeover", "emqx.listeners", [ + {datatype, {enum, [takeover, no_takeover]}}, + hidden +]}. + +{mapping, "listener.wss.$name.deflate_opts.client_context_takeover", "emqx.listeners", [ + {datatype, {enum, [takeover, no_takeover]}}, + hidden +]}. + +{mapping, "listener.wss.$name.deflate_opts.server_max_window_bits", "emqx.listeners", [ + {datatype, integer}, + {validators, ["range:8-15"]}, + hidden +]}. + +{mapping, "listener.wss.$name.deflate_opts.client_max_window_bits", "emqx.listeners", [ + {datatype, integer}, + {validators, ["range:8-15"]}, + hidden +]}. + +{mapping, "listener.wss.$name.idle_timeout", "emqx.listeners", [ + {datatype, {duration, ms}}, + hidden +]}. + +{mapping, "listener.wss.$name.max_frame_size", "emqx.listeners", [ + {datatype, integer}, + hidden +]}. + +{mapping, "listener.wss.$name.mqtt_piggyback", "emqx.listeners", [ + {datatype, {enum, [single, multiple]}}, + {default, multiple}, + hidden +]}. + +{mapping, "listener.wss.$name.check_origin_enable", "emqx.listeners", [ + {datatype, {enum, [true, false]}}, + {default, false}, + hidden +]}. + +{mapping, "listener.wss.$name.allow_origin_absence", "emqx.listeners", [ + {datatype, {enum, [true, false]}}, + {default, true}, + hidden +]}. + +{mapping, "listener.wss.$name.check_origins", "emqx.listeners", [ + {datatype, string}, + hidden +]}. + +{translation, "emqx.listeners", fun(Conf) -> + + Filter = fun(Opts) -> [{K, V} || {K, V} <- Opts, V =/= undefined] end, + + Atom = fun(undefined) -> undefined; (S) -> list_to_atom(S) end, + + Access = fun(S) -> + [A, CIDR] = string:tokens(S, " "), + {list_to_atom(A), case CIDR of "all" -> all; _ -> CIDR end} + end, + + AccOpts = fun(Prefix) -> + case cuttlefish_variable:filter_by_prefix(Prefix ++ ".access", Conf) of + [] -> []; + Rules -> [{access_rules, [Access(Rule) || {_, Rule} <- Rules]}] + end + end, + + RateLimit = fun(undefined) -> + undefined; + (Val) -> + [L, D] = string:tokens(Val, ", "), + Limit = case cuttlefish_bytesize:parse(L) of + Sz when is_integer(Sz) -> Sz; + {error, Reason} -> error(Reason) + end, + Duration = case cuttlefish_duration:parse(D, s) of + Secs when is_integer(Secs) -> Secs; + {error, Reason1} -> error(Reason1) + end, + {Limit, Duration} + end, + + CheckOrigin = fun(S) -> + Origins = string:tokens(S, ","), + [ list_to_binary(string:trim(O)) || O <- Origins] + end, + + WsOpts = fun(Prefix) -> + case cuttlefish_variable:filter_by_prefix(Prefix ++ ".check_origins", Conf) of + [] -> undefined; + Rules -> + OriginList = [CheckOrigin(Rule) || {_, Rule} <- Rules], + lists:flatten(OriginList) + end + end, + + LisOpts = fun(Prefix) -> + Filter([{acceptors, cuttlefish:conf_get(Prefix ++ ".acceptors", Conf)}, + {mqtt_path, cuttlefish:conf_get(Prefix ++ ".mqtt_path", Conf, undefined)}, + {max_connections, cuttlefish:conf_get(Prefix ++ ".max_connections", Conf)}, + {max_conn_rate, cuttlefish:conf_get(Prefix ++ ".max_conn_rate", Conf, undefined)}, + {active_n, cuttlefish:conf_get(Prefix ++ ".active_n", Conf, undefined)}, + {tune_buffer, cuttlefish:conf_get(Prefix ++ ".tune_buffer", Conf, undefined)}, + {zone, Atom(cuttlefish:conf_get(Prefix ++ ".zone", Conf, undefined))}, + {rate_limit, RateLimit(cuttlefish:conf_get(Prefix ++ ".rate_limit", Conf, undefined))}, + {proxy_protocol, cuttlefish:conf_get(Prefix ++ ".proxy_protocol", Conf, undefined)}, + {proxy_address_header, list_to_binary(string:lowercase(cuttlefish:conf_get(Prefix ++ ".proxy_address_header", Conf, "")))}, + {proxy_port_header, list_to_binary(string:lowercase(cuttlefish:conf_get(Prefix ++ ".proxy_port_header", Conf, "")))}, + {proxy_protocol_timeout, cuttlefish:conf_get(Prefix ++ ".proxy_protocol_timeout", Conf, undefined)}, + {fail_if_no_subprotocol, cuttlefish:conf_get(Prefix ++ ".fail_if_no_subprotocol", Conf, undefined)}, + {supported_subprotocols, string:tokens(cuttlefish:conf_get(Prefix ++ ".supported_subprotocols", Conf, ""), ", ")}, + {peer_cert_as_username, cuttlefish:conf_get(Prefix ++ ".peer_cert_as_username", Conf, undefined)}, + {peer_cert_as_clientid, cuttlefish:conf_get(Prefix ++ ".peer_cert_as_clientid", Conf, undefined)}, + {compress, cuttlefish:conf_get(Prefix ++ ".compress", Conf, undefined)}, + {idle_timeout, cuttlefish:conf_get(Prefix ++ ".idle_timeout", Conf, undefined)}, + {max_frame_size, cuttlefish:conf_get(Prefix ++ ".max_frame_size", Conf, undefined)}, + {mqtt_piggyback, cuttlefish:conf_get(Prefix ++ ".mqtt_piggyback", Conf, undefined)}, + {check_origin_enable, cuttlefish:conf_get(Prefix ++ ".check_origin_enable", Conf, undefined)}, + {allow_origin_absence, cuttlefish:conf_get(Prefix ++ ".allow_origin_absence", Conf, undefined)}, + {check_origins, WsOpts(Prefix)} | AccOpts(Prefix)]) + end, + DeflateOpts = fun(Prefix) -> + Filter([{level, cuttlefish:conf_get(Prefix ++ ".deflate_opts.level", Conf, undefined)}, + {mem_level, cuttlefish:conf_get(Prefix ++ ".deflate_opts.mem_level", Conf, undefined)}, + {strategy, cuttlefish:conf_get(Prefix ++ ".deflate_opts.strategy", Conf, undefined)}, + {server_context_takeover, cuttlefish:conf_get(Prefix ++ ".deflate_opts.server_context_takeover", Conf, undefined)}, + {client_context_takeover, cuttlefish:conf_get(Prefix ++ ".deflate_opts.client_context_takeover", Conf, undefined)}, + {server_max_windows_bits, cuttlefish:conf_get(Prefix ++ ".deflate_opts.server_max_window_bits", Conf, undefined)}, + {client_max_windows_bits, cuttlefish:conf_get(Prefix ++ ".deflate_opts.client_max_window_bits", Conf, undefined)}]) + end, + TcpOpts = fun(Prefix) -> + Filter([{backlog, cuttlefish:conf_get(Prefix ++ ".backlog", Conf, undefined)}, + {send_timeout, cuttlefish:conf_get(Prefix ++ ".send_timeout", Conf, undefined)}, + {send_timeout_close, cuttlefish:conf_get(Prefix ++ ".send_timeout_close", Conf, undefined)}, + {recbuf, cuttlefish:conf_get(Prefix ++ ".recbuf", Conf, undefined)}, + {sndbuf, cuttlefish:conf_get(Prefix ++ ".sndbuf", Conf, undefined)}, + {buffer, cuttlefish:conf_get(Prefix ++ ".buffer", Conf, undefined)}, + {high_watermark, cuttlefish:conf_get(Prefix ++ ".high_watermark", Conf, undefined)}, + {nodelay, cuttlefish:conf_get(Prefix ++ ".nodelay", Conf, true)}, + {reuseaddr, cuttlefish:conf_get(Prefix ++ ".reuseaddr", Conf, undefined)}]) + end, + SplitFun = fun(undefined) -> undefined; (S) -> string:tokens(S, ",") end, + MapPSKCiphers = fun(PSKCiphers) -> + lists:map( + fun("PSK-AES128-CBC-SHA") -> {psk, aes_128_cbc, sha}; + ("PSK-AES256-CBC-SHA") -> {psk, aes_256_cbc, sha}; + ("PSK-3DES-EDE-CBC-SHA") -> {psk, '3des_ede_cbc', sha}; + ("PSK-RC4-SHA") -> {psk, rc4_128, sha} + end, PSKCiphers) + end, + SslOpts = fun(Prefix) -> + Versions = case SplitFun(cuttlefish:conf_get(Prefix ++ ".tls_versions", Conf, undefined)) of + undefined -> undefined; + L -> [list_to_atom(V) || V <- L] + end, + TLSCiphers = cuttlefish:conf_get(Prefix++".ciphers", Conf, undefined), + PSKCiphers = cuttlefish:conf_get(Prefix++".psk_ciphers", Conf, undefined), + Ciphers = + case {TLSCiphers, PSKCiphers} of + {undefined, undefined} -> + cuttlefish:invalid(Prefix++".ciphers or "++Prefix++".psk_ciphers is absent"); + {TLSCiphers, undefined} -> + SplitFun(TLSCiphers); + {undefined, PSKCiphers} -> + MapPSKCiphers(SplitFun(PSKCiphers)); + {_TLSCiphers, _PSKCiphers} -> + cuttlefish:invalid(Prefix++".ciphers and "++Prefix++".psk_ciphers cannot be configured at the same time") + end, + UserLookupFun = + case PSKCiphers of + undefined -> undefined; + _ -> {fun emqx_psk:lookup/3, <<>>} + end, + Filter([{versions, Versions}, + {ciphers, Ciphers}, + {user_lookup_fun, UserLookupFun}, + {handshake_timeout, cuttlefish:conf_get(Prefix ++ ".handshake_timeout", Conf, undefined)}, + {depth, cuttlefish:conf_get(Prefix ++ ".depth", Conf, undefined)}, + {password, cuttlefish:conf_get(Prefix ++ ".key_password", Conf, undefined)}, + {dhfile, cuttlefish:conf_get(Prefix ++ ".dhfile", Conf, undefined)}, + {keyfile, cuttlefish:conf_get(Prefix ++ ".keyfile", Conf, undefined)}, + {certfile, cuttlefish:conf_get(Prefix ++ ".certfile", Conf, undefined)}, + {cacertfile, cuttlefish:conf_get(Prefix ++ ".cacertfile", Conf, undefined)}, + {verify, cuttlefish:conf_get(Prefix ++ ".verify", Conf, undefined)}, + {fail_if_no_peer_cert, cuttlefish:conf_get(Prefix ++ ".fail_if_no_peer_cert", Conf, undefined)}, + {secure_renegotiate, cuttlefish:conf_get(Prefix ++ ".secure_renegotiate", Conf, undefined)}, + {reuse_sessions, cuttlefish:conf_get(Prefix ++ ".reuse_sessions", Conf, undefined)}, + {honor_cipher_order, cuttlefish:conf_get(Prefix ++ ".honor_cipher_order", Conf, undefined)}]) + end, + + Listen_fix = fun({Ip, Port}) -> case inet:parse_address(Ip) of + {ok, R} -> {R, Port}; + _ -> {Ip, Port} + end; + (Other) -> Other + end, + + TcpListeners = fun(Type, Name) -> + Prefix = string:join(["listener", Type, Name], "."), + ListenOnN = case cuttlefish:conf_get(Prefix, Conf, undefined) of + undefined -> []; + ListenOn -> Listen_fix(ListenOn) + end, + [#{ proto => Atom(Type) + , name => Name + , listen_on => ListenOnN + , opts => [ {deflate_options, DeflateOpts(Prefix)} + , {tcp_options, TcpOpts(Prefix)} + | LisOpts(Prefix) + ] + } + ] + end, + SslListeners = fun(Type, Name) -> + Prefix = string:join(["listener", Type, Name], "."), + case cuttlefish:conf_get(Prefix, Conf, undefined) of + undefined -> + []; + ListenOn -> + [#{ proto => Atom(Type) + , name => Name + , listen_on => Listen_fix(ListenOn) + , opts => [ {deflate_options, DeflateOpts(Prefix)} + , {tcp_options, TcpOpts(Prefix)} + , {ssl_options, SslOpts(Prefix)} + | LisOpts(Prefix) + ] + } + ] + end + end, + + lists:flatten([TcpListeners(Type, Name) || {["listener", Type, Name], ListenOn} + <- cuttlefish_variable:filter_by_prefix("listener.tcp", Conf) + ++ cuttlefish_variable:filter_by_prefix("listener.ws", Conf)] + ++ + [SslListeners(Type, Name) || {["listener", Type, Name], ListenOn} + <- cuttlefish_variable:filter_by_prefix("listener.ssl", Conf) + ++ cuttlefish_variable:filter_by_prefix("listener.wss", Conf)]) +end}. + +%%-------------------------------------------------------------------- +%% Modules +%%-------------------------------------------------------------------- + +{mapping, "modules.loaded_file", "emqx.modules_loaded_file", [ + {datatype, string} +]}. + +{mapping, "module.presence.qos", "emqx.modules", [ + {default, 1}, + {datatype, integer}, + {validators, ["range:0-2"]} +]}. + +{mapping, "module.subscription.$id.topic", "emqx.modules", [ + {datatype, string} +]}. + +{mapping, "module.subscription.$id.qos", "emqx.modules", [ + {default, 1}, + {datatype, integer}, + {validators, ["range:0-2"]} +]}. + +{mapping, "module.subscription.$id.nl", "emqx.modules", [ + {default, 0}, + {datatype, integer}, + {validators, ["range:0-1"]} +]}. + +{mapping, "module.subscription.$id.rap", "emqx.modules", [ + {default, 0}, + {datatype, integer}, + {validators, ["range:0-1"]} +]}. + +{mapping, "module.subscription.$id.rh", "emqx.modules", [ + {default, 0}, + {datatype, integer}, + {validators, ["range:0-2"]} +]}. + +{mapping, "module.rewrite.rule.$id", "emqx.modules", [ + {datatype, string} +]}. + +{mapping, "module.rewrite.pub.rule.$id", "emqx.modules", [ + {datatype, string} +]}. + +{mapping, "module.rewrite.sub.rule.$id", "emqx.modules", [ + {datatype, string} +]}. + +{translation, "emqx.modules", fun(Conf, _, Conf1) -> + Subscriptions = fun() -> + List = cuttlefish_variable:filter_by_prefix("module.subscription", Conf), + TopicList = [{N, Topic}|| {[_,"subscription",N,"topic"], Topic} <- List], + [{iolist_to_binary(T), #{ qos => cuttlefish:conf_get("module.subscription." ++ N ++ ".qos", Conf, 0), + nl => cuttlefish:conf_get("module.subscription." ++ N ++ ".nl", Conf, 0), + rap => cuttlefish:conf_get("module.subscription." ++ N ++ ".rap", Conf, 0), + rh => cuttlefish:conf_get("module.subscription." ++ N ++ ".rh", Conf, 0) + }} || {N, T} <- TopicList] + end, + Rewrites = fun() -> + Rules = cuttlefish_variable:filter_by_prefix("module.rewrite.rule", Conf), + PubRules = cuttlefish_variable:filter_by_prefix("module.rewrite.pub.rule", Conf), + SubRules = cuttlefish_variable:filter_by_prefix("module.rewrite.sub.rule", Conf), + TotalRules = lists:append( + [ {["module", "rewrite", "pub", "rule", I], Rule} || {["module", "rewrite", "rule", I], Rule} <- Rules] ++ PubRules, + [ {["module", "rewrite", "sub", "rule", I], Rule} || {["module", "rewrite", "rule", I], Rule} <- Rules] ++ SubRules + ), + lists:map(fun({[_, "rewrite", PubOrSub, "rule", I], Rule}) -> + [Topic, Re, Dest] = string:tokens(Rule, " "), + {rewrite, list_to_atom(PubOrSub), list_to_binary(Topic), list_to_binary(Re), list_to_binary(Dest)} + end, TotalRules) + end, + lists:append([ + [{emqx_mod_presence, [{qos, cuttlefish:conf_get("module.presence.qos", Conf, 1)}]}], + [{emqx_mod_subscription, Subscriptions()}], + [{emqx_mod_rewrite, Rewrites()}], + [{emqx_mod_topic_metrics, []}], + [{emqx_mod_delayed, []}], + [{emqx_mod_acl_internal, [{acl_file, cuttlefish:conf_get("acl_file", Conf1)}]}] + ]) +end}. + +%%------------------------------------------------------------------- +%% Plugins +%%------------------------------------------------------------------- + +{mapping, "plugins.etc_dir", "emqx.plugins_etc_dir", [ + {datatype, string} +]}. + +{mapping, "plugins.loaded_file", "emqx.plugins_loaded_file", [ + {datatype, string} +]}. + +{mapping, "plugins.expand_plugins_dir", "emqx.expand_plugins_dir", [ + {datatype, string} +]}. + +%%-------------------------------------------------------------------- +%% Broker +%%-------------------------------------------------------------------- + +{mapping, "broker.sys_interval", "emqx.broker_sys_interval", [ + {datatype, {duration, ms}}, + {default, "1m"} +]}. + +{mapping, "broker.sys_heartbeat", "emqx.broker_sys_heartbeat", [ + {datatype, {duration, ms}}, + {default, "30s"} +]}. + +{mapping, "broker.enable_session_registry", "emqx.enable_session_registry", [ + {default, on}, + {datatype, flag} +]}. + +{mapping, "broker.session_locking_strategy", "emqx.session_locking_strategy", [ + {default, quorum}, + {datatype, {enum, [local,leader,quorum,all]}} +]}. + +%% @doc Shared Subscription Dispatch Strategy. +{mapping, "broker.shared_subscription_strategy", "emqx.shared_subscription_strategy", [ + {default, round_robin}, + {datatype, + {enum, + [random, %% randomly pick a subscriber + round_robin, %% round robin alive subscribers one message after another + sticky, %% pick a random subscriber and stick to it + hash, %% hash client ID to a group member + hash_clientid, + hash_topic + ]}} +]}. + +%% @doc Enable or disable shared dispatch acknowledgement for QoS1 and QoS2 messages +{mapping, "broker.shared_dispatch_ack_enabled", "emqx.shared_dispatch_ack_enabled", + [ {default, false}, + {datatype, {enum, [true, false]}} + ]}. + +{mapping, "broker.route_batch_clean", "emqx.route_batch_clean", [ + {default, on}, + {datatype, flag} +]}. + +%% @doc Performance toggle for subscribe/unsubscribe wildcard topic. +%% Change this toggle only when there are many wildcard topics. +%% key: mnesia translational updates with per-key locks. recommended for single node setup. +%% tab: mnesia translational updates with table lock. recommended for multi-nodes setup. +%% global: global lock protected updates. recommended for larger cluster. +%% NOTE: when changing from/to 'global' lock, it requires all nodes in the cluster +%% +{mapping, "broker.perf.route_lock_type", "emqx.route_lock_type", [ + {default, key}, + {datatype, {enum, [key, tab, global]}} +]}. + +%% @doc Enable trie path compaction. +%% Enabling it significantly improves wildcard topic subscribe rate, +%% if wildcard topics have unique prefixes like: 'sensor/{{id}}/+/', +%% where ID is unique per subscriber. +%% +%% Topic match performance (when publishing) may degrade if messages +%% are mostly published to topics with large number of levels. +%% +%% NOTE: This is a cluster-wide configuration. +%% It rquires all nodes to be stopped before changing it. +{mapping, "broker.perf.trie_compaction", "emqx.trie_compaction", [ + {default, true}, + {datatype, {enum, [true, false]}} +]}. + +%%-------------------------------------------------------------------- +%% System Monitor +%%-------------------------------------------------------------------- + +%% @doc Long GC, don't monitor in production mode for: +%% https://github.com/erlang/otp/blob/feb45017da36be78d4c5784d758ede619fa7bfd3/erts/emulator/beam/erl_gc.c#L421 +{mapping, "sysmon.long_gc", "emqx.sysmon", [ + {default, 0}, + {datatype, [integer, {duration, ms}]} +]}. + +%% @doc Long Schedule(ms) +{mapping, "sysmon.long_schedule", "emqx.sysmon", [ + {default, 240}, + {datatype, [integer, {duration, ms}]} +]}. + +%% @doc Large Heap +{mapping, "sysmon.large_heap", "emqx.sysmon", [ + {default, "8MB"}, + {datatype, bytesize} +]}. + +%% @doc Monitor Busy Port +{mapping, "sysmon.busy_port", "emqx.sysmon", [ + {default, false}, + {datatype, {enum, [true, false]}} +]}. + +%% @doc Monitor Busy Dist Port +{mapping, "sysmon.busy_dist_port", "emqx.sysmon", [ + {default, true}, + {datatype, {enum, [true, false]}} +]}. + +{translation, "emqx.sysmon", fun(Conf) -> + Configs = cuttlefish_variable:filter_by_prefix("sysmon", Conf), + [{list_to_atom(Name), Value} || {[_, Name], Value} <- Configs] +end}. + +%%-------------------------------------------------------------------- +%% Operating System Monitor +%%-------------------------------------------------------------------- + +{mapping, "os_mon.cpu_check_interval", "emqx.os_mon", [ + {default, 60}, + {datatype, {duration, s}} +]}. + +{mapping, "os_mon.cpu_high_watermark", "emqx.os_mon", [ + {default, "80%"}, + {datatype, {percent, float}} +]}. + +{mapping, "os_mon.cpu_low_watermark", "emqx.os_mon", [ + {default, "60%"}, + {datatype, {percent, float}} +]}. + +{mapping, "os_mon.mem_check_interval", "emqx.os_mon", [ + {default, 60}, + {datatype, {duration, s}} +]}. + +{mapping, "os_mon.sysmem_high_watermark", "emqx.os_mon", [ + {default, "70%"}, + {datatype, {percent, float}} +]}. + +{mapping, "os_mon.procmem_high_watermark", "emqx.os_mon", [ + {default, "5%"}, + {datatype, {percent, float}} +]}. + +{translation, "emqx.os_mon", fun(Conf) -> + [{cpu_check_interval, cuttlefish:conf_get("os_mon.cpu_check_interval", Conf)}, + {cpu_high_watermark, cuttlefish:conf_get("os_mon.cpu_high_watermark", Conf) * 100}, + {cpu_low_watermark, cuttlefish:conf_get("os_mon.cpu_low_watermark", Conf) * 100}, + {mem_check_interval, cuttlefish:conf_get("os_mon.mem_check_interval", Conf)}, + {sysmem_high_watermark, cuttlefish:conf_get("os_mon.sysmem_high_watermark", Conf) * 100}, + {procmem_high_watermark, cuttlefish:conf_get("os_mon.procmem_high_watermark", Conf) * 100}] +end}. + +%%-------------------------------------------------------------------- +%% VM Monitor +%%-------------------------------------------------------------------- +{mapping, "vm_mon.check_interval", "emqx.vm_mon", [ + {default, 30}, + {datatype, {duration, s}} +]}. + +{mapping, "vm_mon.process_high_watermark", "emqx.vm_mon", [ + {default, "80%"}, + {datatype, {percent, float}} +]}. + +{mapping, "vm_mon.process_low_watermark", "emqx.vm_mon", [ + {default, "60%"}, + {datatype, {percent, float}} +]}. + +{translation, "emqx.vm_mon", fun(Conf) -> + [{check_interval, cuttlefish:conf_get("vm_mon.check_interval", Conf)}, + {process_high_watermark, cuttlefish:conf_get("vm_mon.process_high_watermark", Conf) * 100}, + {process_low_watermark, cuttlefish:conf_get("vm_mon.process_low_watermark", Conf) * 100}] +end}. + +%%-------------------------------------------------------------------- +%% Alarm +%%-------------------------------------------------------------------- +{mapping, "alarm.actions", "emqx.alarm", [ + {default, "log,publish"}, + {datatype, string} +]}. + +{mapping, "alarm.size_limit", "emqx.alarm", [ + {default, 1000}, + {datatype, integer} +]}. + +{mapping, "alarm.validity_period", "emqx.alarm", [ + {default, "24h"}, + {datatype, {duration, s}} +]}. + +{translation, "emqx.alarm", fun(Conf) -> + [{actions, [list_to_atom(Action) || Action <- string:tokens(cuttlefish:conf_get("alarm.actions", Conf), ",")]}, + {size_limit, cuttlefish:conf_get("alarm.size_limit", Conf)}, + {validity_period, cuttlefish:conf_get("alarm.validity_period", Conf)}] +end}. + +%%-------------------------------------------------------------------- +%% Telemetry +%%-------------------------------------------------------------------- +{mapping, "telemetry.enabled", "emqx.telemetry", [ + {default, false}, + {datatype, {enum, [true, false]}} +]}. + +{mapping, "telemetry.url", "emqx.telemetry", [ + {default, "https://telemetry-emqx-io.bigpar.vercel.app/api/telemetry"}, + {datatype, string} +]}. + +{mapping, "telemetry.report_interval", "emqx.telemetry", [ + {default, "7d"}, + {datatype, {duration, s}} +]}. + +{translation, "emqx.telemetry", fun(Conf) -> + [ {enabled, cuttlefish:conf_get("telemetry.enabled", Conf)} + , {url, cuttlefish:conf_get("telemetry.url", Conf)} + , {report_interval, cuttlefish:conf_get("telemetry.report_interval", Conf)} + ] +end}. From 7af4b80ef4e7bee89ca15b9cbd1d9f2457a3bd57 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 10 Jun 2021 10:24:38 +0800 Subject: [PATCH 02/78] fix(lwm2m): base64 decode for opaque value --- apps/emqx_lwm2m/src/emqx_lwm2m.app.src | 2 +- apps/emqx_lwm2m/src/emqx_lwm2m.appup.src | 8 ++++++++ apps/emqx_lwm2m/src/emqx_lwm2m_message.erl | 5 ++++- apps/emqx_lwm2m/test/emqx_lwm2m_SUITE.erl | 11 +++++++---- 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m.app.src b/apps/emqx_lwm2m/src/emqx_lwm2m.app.src index 19fabc526..209499fc5 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m.app.src +++ b/apps/emqx_lwm2m/src/emqx_lwm2m.app.src @@ -1,6 +1,6 @@ {application,emqx_lwm2m, [{description,"EMQ X LwM2M Gateway"}, - {vsn, "4.3.1"}, % strict semver, bump manually! + {vsn, "4.3.2"}, % strict semver, bump manually! {modules,[]}, {registered,[emqx_lwm2m_sup]}, {applications,[kernel,stdlib,lwm2m_coap]}, diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m.appup.src b/apps/emqx_lwm2m/src/emqx_lwm2m.appup.src index fb22c843d..db46dd42e 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m.appup.src +++ b/apps/emqx_lwm2m/src/emqx_lwm2m.appup.src @@ -1,13 +1,21 @@ %% -*-: erlang -*- {VSN, [ + {"4.3.1", [ + {load_module, emqx_lwm2m_message, brutal_purge, soft_purge, []} + ]}, {"4.3.0", [ + {load_module, emqx_lwm2m_message, brutal_purge, soft_purge, []}, {load_module, emqx_lwm2m_protocol, brutal_purge, soft_purge, []} ]}, {<<".*">>, []} ], [ + {"4.3.1", [ + {load_module, emqx_lwm2m_message, brutal_purge, soft_purge, []} + ]}, {"4.3.0", [ + {load_module, emqx_lwm2m_message, brutal_purge, soft_purge, []}, {load_module, emqx_lwm2m_protocol, brutal_purge, soft_purge, []} ]}, {<<".*">>, []} diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_message.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_message.erl index 2dc9fa08a..6b8bc8d50 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_message.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_message.erl @@ -197,7 +197,10 @@ value_ex(K, Value) when K =:= <<"Integer">>; K =:= <<"Float">>; K =:= <<"Time">> value_ex(K, Value) when K =:= <<"String">> -> Value; value_ex(K, Value) when K =:= <<"Opaque">> -> - Value; + %% XXX: force to decode it with base64 + %% This may not be a good implementation, but it is + %% consistent with the treatment of Opaque in value/3 + base64:decode(Value); value_ex(K, <<"true">>) when K =:= <<"Boolean">> -> <<1>>; value_ex(K, <<"false">>) when K =:= <<"Boolean">> -> <<0>>; diff --git a/apps/emqx_lwm2m/test/emqx_lwm2m_SUITE.erl b/apps/emqx_lwm2m/test/emqx_lwm2m_SUITE.erl index 257502f2f..7f2f37466 100644 --- a/apps/emqx_lwm2m/test/emqx_lwm2m_SUITE.erl +++ b/apps/emqx_lwm2m/test/emqx_lwm2m_SUITE.erl @@ -40,6 +40,7 @@ all() -> , {group, test_grp_4_discover} , {group, test_grp_5_write_attr} , {group, test_grp_6_observe} + , {group, test_grp_8_object_19} ]. suite() -> [{timetrap, {seconds, 90}}]. @@ -98,9 +99,9 @@ groups() -> ]}, {test_grp_8_object_19, [RepeatOpt], [ case80_specail_object_19_1_0_write, - case80_specail_object_19_0_0_notify, - case80_specail_object_19_0_0_response, - case80_normal_object_19_0_0_read + case80_specail_object_19_0_0_notify + %case80_specail_object_19_0_0_response, + %case80_normal_object_19_0_0_read ]}, {test_grp_9_psm_queue_mode, [RepeatOpt], [ case90_psm_mode, @@ -1655,6 +1656,7 @@ case80_specail_object_19_1_0_write(Config) -> <<"value">> => base64:encode(<<12345:32>>) } }, + CommandJson = emqx_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), @@ -1663,7 +1665,7 @@ case80_specail_object_19_1_0_write(Config) -> Path2 = get_coap_path(Options2), ?assertEqual(put, Method2), ?assertEqual(<<"/19/1/0">>, Path2), - ?assertEqual(<<12345:32>>, Payload2), + ?assertEqual(<<3:2, 0:1, 0:2, 4:3, 0, 12345:32>>, Payload2), timer:sleep(50), test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, changed}, #coap_content{}, Request2, true), @@ -1672,6 +1674,7 @@ case80_specail_object_19_1_0_write(Config) -> ReadResult = emqx_json:encode(#{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, <<"data">> => #{ + <<"reqPath">> => <<"/19/1/0">>, <<"code">> => <<"2.04">>, <<"codeMsg">> => <<"changed">> }, From 501b9c94358d25cf34e4e0962907b07aaa86eb55 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Fri, 11 Jun 2021 09:56:11 +0800 Subject: [PATCH 03/78] fix(minirest): encode response message failed (#4965) --- .ci/fvt_tests/http_server/rebar.config | 2 +- rebar.config | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/fvt_tests/http_server/rebar.config b/.ci/fvt_tests/http_server/rebar.config index 29cc392ea..314679264 100644 --- a/.ci/fvt_tests/http_server/rebar.config +++ b/.ci/fvt_tests/http_server/rebar.config @@ -1,7 +1,7 @@ {erl_opts, [debug_info]}. {deps, [ - {minirest, {git, "https://github.com/emqx/minirest.git", {tag, "0.3.5"}}} + {minirest, {git, "https://github.com/emqx/minirest.git", {tag, "0.3.6"}}} ]}. {shell, [ diff --git a/rebar.config b/rebar.config index 03a36570f..ae1d1e83a 100644 --- a/rebar.config +++ b/rebar.config @@ -42,7 +42,7 @@ , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.10.1"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}} , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v4.0.1"}}} % TODO: delete when all apps moved to hocon - , {minirest, {git, "https://github.com/emqx/minirest", {tag, "0.3.5"}}} + , {minirest, {git, "https://github.com/emqx/minirest", {tag, "0.3.6"}}} , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.1"}}} , {replayq, {git, "https://github.com/emqx/replayq", {tag, "0.3.2"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {branch, "2.0.4"}}} From 8352fa6f311b114659a7e473af1523f93d9546ea Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 11 Jun 2021 11:08:24 +0800 Subject: [PATCH 04/78] fix(mqttsn): fix proto_name to MQTT-SN instead of MQTT (#4961) --- apps/emqx/include/emqx_mqtt.hrl | 2 ++ apps/emqx/src/emqx.appup.src | 27 +++++++++++++++++++-------- apps/emqx_sn/src/emqx_sn.appup.src | 6 ++++++ apps/emqx_sn/src/emqx_sn_gateway.erl | 10 ++++++---- 4 files changed, 33 insertions(+), 12 deletions(-) diff --git a/apps/emqx/include/emqx_mqtt.hrl b/apps/emqx/include/emqx_mqtt.hrl index e6e9bffe5..5dd9a317c 100644 --- a/apps/emqx/include/emqx_mqtt.hrl +++ b/apps/emqx/include/emqx_mqtt.hrl @@ -30,11 +30,13 @@ %% MQTT Protocol Version and Names %%-------------------------------------------------------------------- +-define(MQTT_SN_PROTO_V1, 1). -define(MQTT_PROTO_V3, 3). -define(MQTT_PROTO_V4, 4). -define(MQTT_PROTO_V5, 5). -define(PROTOCOL_NAMES, [ + {?MQTT_SN_PROTO_V1, <<"MQTT-SN">>}, %% XXX:Compatible with emqx-sn plug-in {?MQTT_PROTO_V3, <<"MQIsdp">>}, {?MQTT_PROTO_V4, <<"MQTT">>}, {?MQTT_PROTO_V5, <<"MQTT">>}]). diff --git a/apps/emqx/src/emqx.appup.src b/apps/emqx/src/emqx.appup.src index b51a7f3b7..8c778ef36 100644 --- a/apps/emqx/src/emqx.appup.src +++ b/apps/emqx/src/emqx.appup.src @@ -1,12 +1,17 @@ %% -*- mode: erlang -*- {VSN, - [{"4.3.2", - [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, + [ + {"4.3.3", + [{load_module,emqx_packet,brutal_purge,soft_purge,[]}]}, + {"4.3.2", + [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, + {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}]}, {"4.3.1", - [{load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, @@ -18,7 +23,8 @@ {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}]}, {"4.3.0", - [{load_module,emqx_logger_jsonfmt,brutal_purge,soft_purge,[]}, + [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, + {load_module,emqx_logger_jsonfmt,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_congestion,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}, @@ -34,13 +40,17 @@ {apply,{emqx_metrics,upgrade_retained_delayed_counter_type,[]}}, {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.2", - [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, + [{"4.3.3", + [{load_module,emqx_packet,brutal_purge,soft_purge,[]}]}, + {"4.3.2", + [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, + {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}]}, {"4.3.1", - [{load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, @@ -52,7 +62,8 @@ {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}]}, {"4.3.0", - [{load_module,emqx_logger_jsonfmt,brutal_purge,soft_purge,[]}, + [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, + {load_module,emqx_logger_jsonfmt,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_congestion,brutal_purge,soft_purge,[]}, diff --git a/apps/emqx_sn/src/emqx_sn.appup.src b/apps/emqx_sn/src/emqx_sn.appup.src index a6a1a6c84..2bd6f5646 100644 --- a/apps/emqx_sn/src/emqx_sn.appup.src +++ b/apps/emqx_sn/src/emqx_sn.appup.src @@ -1,11 +1,17 @@ %% -*-: erlang -*- {VSN, [ + {"4.3.2", [ + {load_module, emqx_sn_gateway, brutal_purge, soft_purge, []} + ]}, {<<"4.3.[0-1]">>, [ {restart_application, emqx_sn} ]} ], [ + {"4.3.2", [ + {load_module, emqx_sn_gateway, brutal_purge, soft_purge, []} + ]}, {<<"4.3.[0-1]">>, [ {restart_application, emqx_sn} ]} diff --git a/apps/emqx_sn/src/emqx_sn_gateway.erl b/apps/emqx_sn/src/emqx_sn_gateway.erl index 2339961cf..9a8c68583 100644 --- a/apps/emqx_sn/src/emqx_sn_gateway.erl +++ b/apps/emqx_sn/src/emqx_sn_gateway.erl @@ -826,15 +826,17 @@ do_connect(ClientId, CleanStart, WillFlag, Duration, State) -> clean_start = CleanStart, username = State#state.username, password = State#state.password, + proto_name = <<"MQTT-SN">>, keepalive = Duration, - properties = OnlyOneInflight + properties = OnlyOneInflight, + proto_ver = 1 }, case WillFlag of true -> State0 = send_message(?SN_WILLTOPICREQ_MSG(), State), NState = State0#state{connpkt = ConnPkt, - clientid = ClientId, - keepalive_interval = Duration - }, + clientid = ClientId, + keepalive_interval = Duration + }, {next_state, wait_for_will_topic, NState}; false -> NState = State#state{clientid = ClientId, From 0451f89c2222f96ddc6797a7422e998f49da0d61 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Thu, 10 Jun 2021 19:59:10 +0200 Subject: [PATCH 05/78] chore(conf): change default number of gen_rpc connections to 1 --- apps/emqx/etc/emqx.conf | 4 ++-- priv/emqx.schema | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/etc/emqx.conf b/apps/emqx/etc/emqx.conf index f6627bf1c..b75e12a59 100644 --- a/apps/emqx/etc/emqx.conf +++ b/apps/emqx/etc/emqx.conf @@ -350,8 +350,8 @@ rpc.port_discovery = stateless ## Number of outgoing RPC connections. ## ## Value: Interger [0-256] -## Defaults to NumberOfCPUSchedulers / 2 when set to 0 -#rpc.tcp_client_num = 0 +## Default = 1 +#rpc.tcp_client_num = 1 ## RCP Client connect timeout. ## diff --git a/priv/emqx.schema b/priv/emqx.schema index 2d67c9677..142a9acf4 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -376,7 +376,7 @@ end}. {translation, "gen_rpc.tcp_client_num", fun(Conf) -> case cuttlefish:conf_get("rpc.tcp_client_num", Conf) of - 0 -> max(1, erlang:system_info(schedulers) div 2); + 0 -> 1; %% keep allowing 0 for backward compatibility V -> V end end}. From c1d2bae833f06478e1d25e51d5636f7f67364f48 Mon Sep 17 00:00:00 2001 From: k32 <10274441+k32@users.noreply.github.com> Date: Wed, 9 Jun 2021 11:40:52 +0200 Subject: [PATCH 06/78] fix(auth_ldap): Handle missing attributes Fixes: #4953 --- apps/emqx_auth_ldap/src/emqx_acl_ldap.erl | 11 +++-------- apps/emqx_auth_ldap/src/emqx_auth_ldap.appup.src | 8 ++++++++ 2 files changed, 11 insertions(+), 8 deletions(-) create mode 100644 apps/emqx_auth_ldap/src/emqx_auth_ldap.appup.src diff --git a/apps/emqx_auth_ldap/src/emqx_acl_ldap.erl b/apps/emqx_auth_ldap/src/emqx_acl_ldap.erl index 25287052c..8324f6414 100644 --- a/apps/emqx_auth_ldap/src/emqx_acl_ldap.erl +++ b/apps/emqx_auth_ldap/src/emqx_acl_ldap.erl @@ -27,10 +27,6 @@ , description/0 ]). --import(proplists, [get_value/2]). - --import(emqx_auth_ldap_cli, [search/4]). - -spec(register_metrics() -> ok). register_metrics() -> lists:foreach(fun emqx_metrics:ensure/1, ?ACL_METRICS). @@ -70,14 +66,14 @@ do_check_acl(#{username := Username}, PubSub, Topic, _NoMatchAction, BaseDN = emqx_auth_ldap:replace_vars(CustomBaseDN, ReplaceRules), - case search(Pool, BaseDN, Filter, [Attribute, Attribute1]) of + case emqx_auth_ldap_cli:search(Pool, BaseDN, Filter, [Attribute, Attribute1]) of {error, noSuchObject} -> ok; {ok, #eldap_search_result{entries = []}} -> ok; {ok, #eldap_search_result{entries = [Entry]}} -> - Topics = get_value(Attribute, Entry#eldap_entry.attributes) - ++ get_value(Attribute1, Entry#eldap_entry.attributes), + Topics = proplists:get_value(Attribute, Entry#eldap_entry.attributes, []) + ++ proplists:get_value(Attribute1, Entry#eldap_entry.attributes, []), match(Topic, Topics); Error -> ?LOG(error, "[LDAP] search error:~p", [Error]), @@ -95,4 +91,3 @@ match(Topic, [Filter | Topics]) -> description() -> "ACL with LDAP". - diff --git a/apps/emqx_auth_ldap/src/emqx_auth_ldap.appup.src b/apps/emqx_auth_ldap/src/emqx_auth_ldap.appup.src new file mode 100644 index 000000000..8b70bf484 --- /dev/null +++ b/apps/emqx_auth_ldap/src/emqx_auth_ldap.appup.src @@ -0,0 +1,8 @@ +%% -*- mode: erlang -*- +{VSN, + [{"4.3.0", + [{load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]}]}, + {<<".*">>,[]}], + [{"4.3.0", + [{load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]}]}, + {<<".*">>,[]}]}. From 2bc586b93054c91e97996e67e4ebc5de746bacfa Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 11 Jun 2021 11:48:44 +0800 Subject: [PATCH 07/78] fix(ws): avoid funcation_clause for un-inited websocket --- apps/emqx/src/emqx.appup.src | 10 ++++++++-- apps/emqx/src/emqx_ws_connection.erl | 5 ++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/apps/emqx/src/emqx.appup.src b/apps/emqx/src/emqx.appup.src index 8c778ef36..53e77a103 100644 --- a/apps/emqx/src/emqx.appup.src +++ b/apps/emqx/src/emqx.appup.src @@ -2,9 +2,12 @@ {VSN, [ {"4.3.3", - [{load_module,emqx_packet,brutal_purge,soft_purge,[]}]}, + [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]} + ]}, {"4.3.2", [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, @@ -41,9 +44,12 @@ {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.3.3", - [{load_module,emqx_packet,brutal_purge,soft_purge,[]}]}, + [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]} + ]}, {"4.3.2", [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, diff --git a/apps/emqx/src/emqx_ws_connection.erl b/apps/emqx/src/emqx_ws_connection.erl index 7bc68c271..d686e1611 100644 --- a/apps/emqx/src/emqx_ws_connection.erl +++ b/apps/emqx/src/emqx_ws_connection.erl @@ -403,7 +403,10 @@ websocket_close(Reason, State) -> terminate(Reason, _Req, #state{channel = Channel}) -> ?LOG(debug, "Terminated due to ~p", [Reason]), - emqx_channel:terminate(Reason, Channel). + emqx_channel:terminate(Reason, Channel); + +terminate(_Reason, _Req, _UnExpectedState) -> + ok. %%-------------------------------------------------------------------- %% Handle call From 56ac459b7f876d309cf0e0f7132b9f9773bdbd73 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 15 Jun 2021 20:48:07 +0800 Subject: [PATCH 08/78] fix(modules): fix start/stop exhook module failure --- apps/emqx_exhook/src/emqx_exhook.app.src | 2 +- apps/emqx_exhook/src/emqx_exhook.appup.src | 12 ++++++++++-- apps/emqx_exhook/src/emqx_exhook_server.erl | 9 ++++++++- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/apps/emqx_exhook/src/emqx_exhook.app.src b/apps/emqx_exhook/src/emqx_exhook.app.src index 452d2a742..e703cfe5e 100644 --- a/apps/emqx_exhook/src/emqx_exhook.app.src +++ b/apps/emqx_exhook/src/emqx_exhook.app.src @@ -1,6 +1,6 @@ {application, emqx_exhook, [{description, "EMQ X Extension for Hook"}, - {vsn, "4.3.1"}, + {vsn, "4.3.2"}, {modules, []}, {registered, []}, {mod, {emqx_exhook_app, []}}, diff --git a/apps/emqx_exhook/src/emqx_exhook.appup.src b/apps/emqx_exhook/src/emqx_exhook.appup.src index 2811c1554..26e84d88f 100644 --- a/apps/emqx_exhook/src/emqx_exhook.appup.src +++ b/apps/emqx_exhook/src/emqx_exhook.appup.src @@ -1,14 +1,22 @@ %% -*-: erlang -*- {VSN, [ + {"4.3.1", [ + {load_module, emqx_exhook_server, brutal_purge, soft_purge, []} + ]}, {"4.3.0", [ - {load_module, emqx_exhook_pb, brutal_purge, soft_purge, []} + {load_module, emqx_exhook_pb, brutal_purge, soft_purge, []}, + {load_module, emqx_exhook_server, brutal_purge, soft_purge, []} ]}, {<<".*">>, []} ], [ + {"4.3.1", [ + {load_module, emqx_exhook_server, brutal_purge, soft_purge, []} + ]}, {"4.3.0", [ - {load_module, emqx_exhook_pb, brutal_purge, soft_purge, []} + {load_module, emqx_exhook_pb, brutal_purge, soft_purge, []}, + {load_module, emqx_exhook_server, brutal_purge, soft_purge, []} ]}, {<<".*">>, []} ] diff --git a/apps/emqx_exhook/src/emqx_exhook_server.erl b/apps/emqx_exhook/src/emqx_exhook_server.erl index 848a3f59d..f4965e4ca 100644 --- a/apps/emqx_exhook/src/emqx_exhook_server.erl +++ b/apps/emqx_exhook/src/emqx_exhook_server.erl @@ -122,7 +122,7 @@ channel_opts(Opts) -> Scheme = proplists:get_value(scheme, Opts), Host = proplists:get_value(host, Opts), Port = proplists:get_value(port, Opts), - SvrAddr = lists:flatten(io_lib:format("~s://~s:~w", [Scheme, Host, Port])), + SvrAddr = format_http_uri(Scheme, Host, Port), ClientOpts = case Scheme of https -> SslOpts = lists:keydelete(ssl, 1, proplists:get_value(ssl_options, Opts, [])), @@ -133,6 +133,13 @@ channel_opts(Opts) -> end, {SvrAddr, ClientOpts}. +format_http_uri(Scheme, Host0, Port) -> + Host = case is_tuple(Host0) of + true -> inet:ntoa(Host0); + _ -> Host0 + end, + lists:flatten(io_lib:format("~s://~s:~w", [Scheme, Host, Port])). + -spec unload(server()) -> ok. unload(#server{name = Name, hookspec = HookSpecs}) -> _ = do_deinit(Name), From bcedd7fe9eefe04b95cdbcdb4975c54b9d37523b Mon Sep 17 00:00:00 2001 From: tigercl Date: Thu, 17 Jun 2021 16:12:08 +0800 Subject: [PATCH 09/78] fix(query string): support query string in path (#4981) --- apps/emqx_auth_http/src/emqx_auth_http.app.src | 2 +- apps/emqx_auth_http/src/emqx_auth_http.appup.src | 16 ++++++++++++++++ apps/emqx_auth_http/src/emqx_auth_http_app.erl | 13 ++++++++----- apps/emqx_web_hook/src/emqx_web_hook.app.src | 2 +- apps/emqx_web_hook/src/emqx_web_hook.appup.src | 12 +++++++----- apps/emqx_web_hook/src/emqx_web_hook_actions.erl | 14 +++++++++++--- apps/emqx_web_hook/src/emqx_web_hook_app.erl | 13 ++++++++----- 7 files changed, 52 insertions(+), 20 deletions(-) create mode 100644 apps/emqx_auth_http/src/emqx_auth_http.appup.src diff --git a/apps/emqx_auth_http/src/emqx_auth_http.app.src b/apps/emqx_auth_http/src/emqx_auth_http.app.src index b2c3221e6..305487171 100644 --- a/apps/emqx_auth_http/src/emqx_auth_http.app.src +++ b/apps/emqx_auth_http/src/emqx_auth_http.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_http, [{description, "EMQ X Authentication/ACL with HTTP API"}, - {vsn, "4.3.0"}, % strict semver, bump manually! + {vsn, "4.3.1"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_http_sup]}, {applications, [kernel,stdlib,ehttpc]}, diff --git a/apps/emqx_auth_http/src/emqx_auth_http.appup.src b/apps/emqx_auth_http/src/emqx_auth_http.appup.src new file mode 100644 index 000000000..620194064 --- /dev/null +++ b/apps/emqx_auth_http/src/emqx_auth_http.appup.src @@ -0,0 +1,16 @@ +%% -*-: erlang -*- + +{VSN, + [ + {"4.3.0", [ + {restart_application, emqx_auth_http} + ]}, + {<<".*">>, []} + ], + [ + {"4.3.0", [ + {restart_application, emqx_auth_http} + ]}, + {<<".*">>, []} + ] +}. diff --git a/apps/emqx_auth_http/src/emqx_auth_http_app.erl b/apps/emqx_auth_http/src/emqx_auth_http_app.erl index 7d4781f7c..51e376f6a 100644 --- a/apps/emqx_auth_http/src/emqx_auth_http_app.erl +++ b/apps/emqx_auth_http/src/emqx_auth_http_app.erl @@ -54,10 +54,9 @@ translate_env(EnvName) -> {ok, ConnectTimeout} = application:get_env(?APP, connect_timeout), URL = proplists:get_value(url, Req), {ok, #{host := Host, - path := Path0, port := Port, - scheme := Scheme}} = emqx_http_lib:uri_parse(URL), - Path = path(Path0), + scheme := Scheme} = URIMap} = emqx_http_lib:uri_parse(URL), + Path = path(URIMap), MoreOpts = case Scheme of http -> [{transport_opts, emqx_misc:ipv6_probe([])}]; @@ -151,8 +150,12 @@ ensure_content_type_header(Method, Headers) ensure_content_type_header(_Method, Headers) -> lists:keydelete("content-type", 1, Headers). -path("") -> +path(#{path := "", 'query' := Query}) -> + "?" ++ Query; +path(#{path := Path, 'query' := Query}) -> + Path ++ "?" ++ Query; +path(#{path := ""}) -> "/"; -path(Path) -> +path(#{path := Path}) -> Path. diff --git a/apps/emqx_web_hook/src/emqx_web_hook.app.src b/apps/emqx_web_hook/src/emqx_web_hook.app.src index 97624a900..e1cfda173 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook.app.src +++ b/apps/emqx_web_hook/src/emqx_web_hook.app.src @@ -1,6 +1,6 @@ {application, emqx_web_hook, [{description, "EMQ X WebHook Plugin"}, - {vsn, "4.3.1"}, % strict semver, bump manually! + {vsn, "4.3.2"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_web_hook_sup]}, {applications, [kernel,stdlib,ehttpc]}, diff --git a/apps/emqx_web_hook/src/emqx_web_hook.appup.src b/apps/emqx_web_hook/src/emqx_web_hook.appup.src index b92284b64..ae6e9e1ae 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook.appup.src +++ b/apps/emqx_web_hook/src/emqx_web_hook.appup.src @@ -2,14 +2,16 @@ {VSN, [ - {"4.3.0", [ - {load_module, emqx_web_hook_actions, brutal_purge, soft_purge, []} - ]}, + {<<"4.3.[0-1]">>, [ + {restart_application, emqx_web_hook}, + {apply,{emqx_rule_engine,refresh_resource,[web_hook]}} + ]}, {<<".*">>, []} ], [ - {"4.3.0", [ - {load_module, emqx_web_hook_actions, brutal_purge, soft_purge, []} + {<<"4.3.[0-1]">>, [ + {restart_application, emqx_web_hook}, + {apply,{emqx_rule_engine,refresh_resource,[web_hook]}} ]}, {<<".*">>, []} ] diff --git a/apps/emqx_web_hook/src/emqx_web_hook_actions.erl b/apps/emqx_web_hook/src/emqx_web_hook_actions.erl index f8e4d8f08..8bb1c7de0 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook_actions.erl +++ b/apps/emqx_web_hook/src/emqx_web_hook_actions.erl @@ -292,7 +292,7 @@ parse_action_params(Params = #{<<"url">> := URL}) -> Headers = headers(maps:get(<<"headers">>, Params, undefined)), NHeaders = ensure_content_type_header(Headers, Method), #{method => Method, - path => path(filename:join(CommonPath, maps:get(<<"path">>, Params, <<>>))), + path => merge_path(CommonPath, maps:get(<<"path">>, Params, <<>>)), headers => NHeaders, body => maps:get(<<"body">>, Params, <<>>), request_timeout => cuttlefish_duration:parse(str(maps:get(<<"request_timeout">>, Params, <<"5s">>))), @@ -306,8 +306,16 @@ ensure_content_type_header(Headers, Method) when Method =:= post orelse Method = ensure_content_type_header(Headers, _Method) -> lists:keydelete("content-type", 1, Headers). -path(<<>>) -> <<"/">>; -path(Path) -> Path. +merge_path(CommonPath, <<>>) -> + CommonPath; +merge_path(CommonPath, Path0) -> + case emqx_http_lib:uri_parse(Path0) of + {ok, #{path := Path1, 'query' := Query}} -> + Path2 = filename:join(CommonPath, Path1), + <>; + {ok, #{path := Path1}} -> + filename:join(CommonPath, Path1) + end. method(GET) when GET == <<"GET">>; GET == <<"get">> -> get; method(POST) when POST == <<"POST">>; POST == <<"post">> -> post; diff --git a/apps/emqx_web_hook/src/emqx_web_hook_app.erl b/apps/emqx_web_hook/src/emqx_web_hook_app.erl index 492af628d..580742c47 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook_app.erl +++ b/apps/emqx_web_hook/src/emqx_web_hook_app.erl @@ -42,10 +42,9 @@ stop(_State) -> translate_env() -> {ok, URL} = application:get_env(?APP, url), {ok, #{host := Host, - path := Path0, port := Port, - scheme := Scheme}} = emqx_http_lib:uri_parse(URL), - Path = path(Path0), + scheme := Scheme} = URIMap} = emqx_http_lib:uri_parse(URL), + Path = path(URIMap), PoolSize = application:get_env(?APP, pool_size, 32), MoreOpts = case Scheme of http -> @@ -89,9 +88,13 @@ translate_env() -> NHeaders = set_content_type(Headers), application:set_env(?APP, headers, NHeaders). -path("") -> +path(#{path := "", 'query' := Query}) -> + "?" ++ Query; +path(#{path := Path, 'query' := Query}) -> + Path ++ "?" ++ Query; +path(#{path := ""}) -> "/"; -path(Path) -> +path(#{path := Path}) -> Path. set_content_type(Headers) -> From a610c3d1f300e2c2f037c7f9bb06d5d98f56a997 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Thu, 17 Jun 2021 16:45:44 +0800 Subject: [PATCH 10/78] fix(rule): delete resource failed when searching dependent rules (#4996) --- .../src/emqx_rule_engine.app.src | 2 +- .../src/emqx_rule_engine.appup.src | 20 ++++++++++++++----- .../src/emqx_rule_registry.erl | 7 +++++-- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index 83b2d7632..aebb73150 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -1,6 +1,6 @@ {application, emqx_rule_engine, [{description, "EMQ X Rule Engine"}, - {vsn, "4.3.2"}, % strict semver, bump manually! + {vsn, "4.3.3"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_rule_engine_sup, emqx_rule_registry]}, {applications, [kernel,stdlib,rulesql,getopt]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index 446e082b7..554fe2fd8 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -1,21 +1,31 @@ %% -*-: erlang -*- -{"4.3.2", +{"4.3.3", [ {"4.3.0", - [ {load_module, emqx_rule_funcs, brutal_purge, soft_purge, []}, - {load_module, emqx_rule_engine, brutal_purge, soft_purge, []} + [ {load_module, emqx_rule_funcs, brutal_purge, soft_purge, []} + , {load_module, emqx_rule_engine, brutal_purge, soft_purge, []} + , {load_module, emqx_rule_registry, brutal_purge, soft_purge, []} ]}, {"4.3.1", [ {load_module, emqx_rule_engine, brutal_purge, soft_purge, []} + , {load_module, emqx_rule_registry, brutal_purge, soft_purge, []} + ]}, + {"4.3.2", + [ {load_module, emqx_rule_registry, brutal_purge, soft_purge, []} ]}, {<<".*">>, []} ], [ {"4.3.0", - [ {load_module, emqx_rule_funcs, brutal_purge, soft_purge, []}, - {load_module, emqx_rule_engine, brutal_purge, soft_purge, []} + [ {load_module, emqx_rule_funcs, brutal_purge, soft_purge, []} + , {load_module, emqx_rule_engine, brutal_purge, soft_purge, []} + , {load_module, emqx_rule_registry, brutal_purge, soft_purge, []} ]}, {"4.3.1", [ {load_module, emqx_rule_engine, brutal_purge, soft_purge, []} + , {load_module, emqx_rule_registry, brutal_purge, soft_purge, []} + ]}, + {"4.3.2", + [ {load_module, emqx_rule_registry, brutal_purge, soft_purge, []} ]}, {<<".*">>, []} ] diff --git a/apps/emqx_rule_engine/src/emqx_rule_registry.erl b/apps/emqx_rule_engine/src/emqx_rule_registry.erl index 4ec5d1cb7..2ebca47e4 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_registry.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_registry.erl @@ -392,8 +392,11 @@ find_rules_depends_on_resource(ResId) -> end, [], get_rules()). search_action_despends_on_resource(ResId, Actions) -> - lists:search(fun(#action_instance{args = #{<<"$resource">> := ResId0}}) -> - ResId0 =:= ResId + lists:search(fun + (#action_instance{args = #{<<"$resource">> := ResId0}}) -> + ResId0 =:= ResId; + (_) -> + false end, Actions). %%------------------------------------------------------------------------------ From c1b21633623154aefa2059b334a093fd607b0dbd Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Sat, 19 Jun 2021 17:02:43 +0800 Subject: [PATCH 11/78] feat(rules): remove stats update from rule_engine_registry (#5029) --- .../src/emqx_rule_engine.appup.src | 6 ++++++ .../src/emqx_rule_registry.erl | 18 +----------------- 2 files changed, 7 insertions(+), 17 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index 554fe2fd8..f8411e0b5 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -4,13 +4,16 @@ [ {load_module, emqx_rule_funcs, brutal_purge, soft_purge, []} , {load_module, emqx_rule_engine, brutal_purge, soft_purge, []} , {load_module, emqx_rule_registry, brutal_purge, soft_purge, []} + , {apply, {supervisor, restart_child, [emqx_rule_engine_sup, emqx_rule_registry]}} ]}, {"4.3.1", [ {load_module, emqx_rule_engine, brutal_purge, soft_purge, []} , {load_module, emqx_rule_registry, brutal_purge, soft_purge, []} + , {apply, {supervisor, restart_child, [emqx_rule_engine_sup, emqx_rule_registry]}} ]}, {"4.3.2", [ {load_module, emqx_rule_registry, brutal_purge, soft_purge, []} + , {apply, {supervisor, restart_child, [emqx_rule_engine_sup, emqx_rule_registry]}} ]}, {<<".*">>, []} ], @@ -19,13 +22,16 @@ [ {load_module, emqx_rule_funcs, brutal_purge, soft_purge, []} , {load_module, emqx_rule_engine, brutal_purge, soft_purge, []} , {load_module, emqx_rule_registry, brutal_purge, soft_purge, []} + , {apply, {supervisor, restart_child, [emqx_rule_engine_sup, emqx_rule_registry]}} ]}, {"4.3.1", [ {load_module, emqx_rule_engine, brutal_purge, soft_purge, []} , {load_module, emqx_rule_registry, brutal_purge, soft_purge, []} + , {apply, {supervisor, restart_child, [emqx_rule_engine_sup, emqx_rule_registry]}} ]}, {"4.3.2", [ {load_module, emqx_rule_registry, brutal_purge, soft_purge, []} + , {apply, {supervisor, restart_child, [emqx_rule_engine_sup, emqx_rule_registry]}} ]}, {<<".*">>, []} ] diff --git a/apps/emqx_rule_engine/src/emqx_rule_registry.erl b/apps/emqx_rule_engine/src/emqx_rule_registry.erl index 2ebca47e4..2d029f8e3 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_registry.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_registry.erl @@ -93,13 +93,6 @@ -define(REGISTRY, ?MODULE). -%% Statistics --define(STATS, - [ {?RULE_TAB, 'rules.count', 'rules.max'} - , {?ACTION_TAB, 'actions.count', 'actions.max'} - , {?RES_TAB, 'resources.count', 'resources.max'} - ]). - -define(T_CALL, 10000). %%------------------------------------------------------------------------------ @@ -442,8 +435,6 @@ delete_resource_type(Type) -> %%------------------------------------------------------------------------------ init([]) -> - %% Enable stats timer - ok = emqx_stats:update_interval(rule_registery_stats, fun update_stats/0), _TableId = ets:new(?KV_TAB, [named_table, set, public, {write_concurrency, true}, {read_concurrency, true}]), {ok, #{}}. @@ -469,7 +460,7 @@ handle_info(Info, State) -> {noreply, State}. terminate(_Reason, _State) -> - emqx_stats:cancel_update(rule_registery_stats). + ok. code_change(_OldVsn, State, _Extra) -> {ok, State}. @@ -478,13 +469,6 @@ code_change(_OldVsn, State, _Extra) -> %% Private functions %%------------------------------------------------------------------------------ -update_stats() -> - lists:foreach( - fun({Tab, Stat, MaxStat}) -> - Size = mnesia:table_info(Tab, size), - emqx_stats:setstat(Stat, MaxStat, Size) - end, ?STATS). - get_all_records(Tab) -> %mnesia:dirty_match_object(Tab, mnesia:table_info(Tab, wild_pattern)). ets:tab2list(Tab). From 8e08e83090fa08e8a01676cc477b7b9e21741d31 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Mon, 21 Jun 2021 11:37:46 +0800 Subject: [PATCH 12/78] fix(appup): relup for emqx_rule_registry failed --- apps/emqx_rule_engine/src/emqx_rule_engine.appup.src | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index f8411e0b5..01c07c124 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -4,16 +4,16 @@ [ {load_module, emqx_rule_funcs, brutal_purge, soft_purge, []} , {load_module, emqx_rule_engine, brutal_purge, soft_purge, []} , {load_module, emqx_rule_registry, brutal_purge, soft_purge, []} - , {apply, {supervisor, restart_child, [emqx_rule_engine_sup, emqx_rule_registry]}} + , {apply, {emqx_stats, cancel_update, [rule_registery_stats]}} ]}, {"4.3.1", [ {load_module, emqx_rule_engine, brutal_purge, soft_purge, []} , {load_module, emqx_rule_registry, brutal_purge, soft_purge, []} - , {apply, {supervisor, restart_child, [emqx_rule_engine_sup, emqx_rule_registry]}} + , {apply, {emqx_stats, cancel_update, [rule_registery_stats]}} ]}, {"4.3.2", [ {load_module, emqx_rule_registry, brutal_purge, soft_purge, []} - , {apply, {supervisor, restart_child, [emqx_rule_engine_sup, emqx_rule_registry]}} + , {apply, {emqx_stats, cancel_update, [rule_registery_stats]}} ]}, {<<".*">>, []} ], @@ -22,16 +22,16 @@ [ {load_module, emqx_rule_funcs, brutal_purge, soft_purge, []} , {load_module, emqx_rule_engine, brutal_purge, soft_purge, []} , {load_module, emqx_rule_registry, brutal_purge, soft_purge, []} - , {apply, {supervisor, restart_child, [emqx_rule_engine_sup, emqx_rule_registry]}} + , {apply, {emqx_stats, cancel_update, [rule_registery_stats]}} ]}, {"4.3.1", [ {load_module, emqx_rule_engine, brutal_purge, soft_purge, []} , {load_module, emqx_rule_registry, brutal_purge, soft_purge, []} - , {apply, {supervisor, restart_child, [emqx_rule_engine_sup, emqx_rule_registry]}} + , {apply, {emqx_stats, cancel_update, [rule_registery_stats]}} ]}, {"4.3.2", [ {load_module, emqx_rule_registry, brutal_purge, soft_purge, []} - , {apply, {supervisor, restart_child, [emqx_rule_engine_sup, emqx_rule_registry]}} + , {apply, {emqx_stats, cancel_update, [rule_registery_stats]}} ]}, {<<".*">>, []} ] From c7ebc12ce11eb60a0f685e6cd6d46f0a047b3134 Mon Sep 17 00:00:00 2001 From: Turtle Date: Thu, 17 Jun 2021 19:29:36 +0800 Subject: [PATCH 13/78] feat(lwm2m): add emqx_lwm2m http API --- apps/emqx_lwm2m/src/emqx_lwm2m.appup.src | 22 +-- apps/emqx_lwm2m/src/emqx_lwm2m_api.erl | 162 ++++++++++++++++++ apps/emqx_lwm2m/src/emqx_lwm2m_cm.erl | 153 +++++++++++++++++ apps/emqx_lwm2m/src/emqx_lwm2m_cm_sup.erl | 41 +++++ apps/emqx_lwm2m/src/emqx_lwm2m_sup.erl | 9 +- apps/emqx_lwm2m/src/emqx_lwm2m_xml_object.erl | 11 ++ .../src/emqx_lwm2m_xml_object_db.erl | 8 +- 7 files changed, 386 insertions(+), 20 deletions(-) create mode 100644 apps/emqx_lwm2m/src/emqx_lwm2m_api.erl create mode 100644 apps/emqx_lwm2m/src/emqx_lwm2m_cm.erl create mode 100644 apps/emqx_lwm2m/src/emqx_lwm2m_cm_sup.erl diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m.appup.src b/apps/emqx_lwm2m/src/emqx_lwm2m.appup.src index db46dd42e..b6d49302f 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m.appup.src +++ b/apps/emqx_lwm2m/src/emqx_lwm2m.appup.src @@ -1,23 +1,13 @@ %% -*-: erlang -*- {VSN, [ - {"4.3.1", [ - {load_module, emqx_lwm2m_message, brutal_purge, soft_purge, []} - ]}, - {"4.3.0", [ - {load_module, emqx_lwm2m_message, brutal_purge, soft_purge, []}, - {load_module, emqx_lwm2m_protocol, brutal_purge, soft_purge, []} - ]}, - {<<".*">>, []} + {<<"4.3.[0-1]">>, [ + {restart_application, emqx_lwm2m} + ]} ], [ - {"4.3.1", [ - {load_module, emqx_lwm2m_message, brutal_purge, soft_purge, []} - ]}, - {"4.3.0", [ - {load_module, emqx_lwm2m_message, brutal_purge, soft_purge, []}, - {load_module, emqx_lwm2m_protocol, brutal_purge, soft_purge, []} - ]}, - {<<".*">>, []} + {<<"4.3.[0-1]">>, [ + {restart_application, emqx_lwm2m} + ]} ] }. diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_api.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_api.erl new file mode 100644 index 000000000..6018aa7c7 --- /dev/null +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_api.erl @@ -0,0 +1,162 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_lwm2m_api). + +-import(minirest, [return/1]). + +-rest_api(#{name => list, + method => 'GET', + path => "/lwm2m_channels/", + func => list, + descr => "A list of all lwm2m channel" + }). + +-rest_api(#{name => list, + method => 'GET', + path => "/nodes/:atom:node/lwm2m_channels/", + func => list, + descr => "A list of lwm2m channel of a node" + }). + +-rest_api(#{name => lookup_cmd, + method => 'GET', + path => "/lookup_cmd/:bin:ep/", + func => lookup_cmd, + descr => "Send a lwm2m downlink command" + }). + +-rest_api(#{name => lookup_cmd, + method => 'GET', + path => "/nodes/:atom:node/lookup_cmd/:bin:ep/", + func => lookup_cmd, + descr => "Send a lwm2m downlink command of a node" + }). + +-export([ list/2 + , lookup_cmd/2 + ]). + +list(#{node := Node }, Params) -> + case Node = node() of + true -> list(#{}, Params); + _ -> rpc_call(Node, list, [#{}, Params]) + end; + +list(#{}, _Params) -> + Channels = emqx_lwm2m_cm:all_channels(), + return({ok, format(Channels)}). + +lookup_cmd(#{ep := Ep, node := Node}, Params) -> + case Node = node() of + true -> lookup_cmd(#{ep => Ep}, Params); + _ -> rpc_call(Node, lookup_cmd, [#{ep => Ep}, Params]) + end; + +lookup_cmd(#{ep := Ep}, Params) -> + MsgType = proplists:get_value(<<"msgType">>, Params), + Path0 = proplists:get_value(<<"path">>, Params), + case emqx_lwm2m_cm:lookup_cmd(Ep, Path0, MsgType) of + [] -> return({ok, []}); + [{_, undefined} | _] -> return({ok, []}); + [{{IMEI, Path, MsgType}, undefined}] -> + return({ok, [{imei, IMEI}, + {'msgType', IMEI}, + {'code', <<"6.01">>}, + {'codeMsg', <<"reply_not_received">>}, + {'path', Path}]}); + [{{IMEI, Path, MsgType}, {Code, CodeMsg, Content}}] -> + Payload1 = format_cmd_content(Content, MsgType), + return({ok, [{imei, IMEI}, + {'msgType', IMEI}, + {'code', Code}, + {'codeMsg', CodeMsg}, + {'path', Path}] ++ Payload1}) + end. + +rpc_call(Node, Fun, Args) -> + case rpc:call(Node, ?MODULE, Fun, Args) of + {badrpc, Reason} -> {error, Reason}; + Res -> Res + end. + +format(Channels) -> + lists:map(fun({IMEI, #{lifetime := LifeTime, + peername := Peername, + version := Version, + reg_info := RegInfo}}) -> + ObjectList = lists:map(fun(Path) -> + [ObjId | _] = path_list(Path), + case emqx_lwm2m_xml_object:get_obj_def(binary_to_integer(ObjId), true) of + {error, _} -> + {Path, Path}; + ObjDefinition -> + ObjectName = emqx_lwm2m_xml_object:get_object_name(ObjDefinition), + {Path, list_to_binary(ObjectName)} + end + end, maps:get(<<"objectList">>, RegInfo)), + {IpAddr, Port} = Peername, + [{imei, IMEI}, + {lifetime, LifeTime}, + {ip_address, iolist_to_binary(ntoa(IpAddr))}, + {port, Port}, + {version, Version}, + {'objectList', ObjectList}] + end, Channels). + +format_cmd_content(undefined, _MsgType) -> []; +format_cmd_content(Content, <<"discover">>) -> + [H | Content1] = Content, + {_, [HObjId]} = emqx_lwm2m_coap_resource:parse_object_list(H), + [ObjId | _]= path_list(HObjId), + ObjectList = case Content1 of + [Content2 | _] -> + {_, ObjL} = emqx_lwm2m_coap_resource:parse_object_list(Content2), + ObjL; + [] -> [] + end, + R = case emqx_lwm2m_xml_object:get_obj_def(binary_to_integer(ObjId), true) of + {error, _} -> + lists:map(fun(Object) -> {Object, Object} end, ObjectList); + ObjDefinition -> + lists:map(fun(Object) -> + [_, _, ResId| _] = path_list(Object), + Operations = case emqx_lwm2m_xml_object:get_resource_operations(binary_to_integer(ResId), ObjDefinition) of + "E" -> [{operations, list_to_binary("E")}]; + Oper -> [{'dataType', list_to_binary(emqx_lwm2m_xml_object:get_resource_type(binary_to_integer(ResId), ObjDefinition))}, + {operations, list_to_binary(Oper)}] + end, + [{path, Object}, + {name, list_to_binary(emqx_lwm2m_xml_object:get_resource_name(binary_to_integer(ResId), ObjDefinition))} + ] ++ Operations + end, ObjectList) + end, + [{content, R}]; +format_cmd_content(Content, _) -> + [{content, Content}]. + +ntoa({0,0,0,0,0,16#ffff,AB,CD}) -> + inet_parse:ntoa({AB bsr 8, AB rem 256, CD bsr 8, CD rem 256}); +ntoa(IP) -> + inet_parse:ntoa(IP). + +path_list(Path) -> + case binary:split(binary_util:trim(Path, $/), [<<$/>>], [global]) of + [ObjId, ObjInsId, ResId, ResInstId] -> [ObjId, ObjInsId, ResId, ResInstId]; + [ObjId, ObjInsId, ResId] -> [ObjId, ObjInsId, ResId]; + [ObjId, ObjInsId] -> [ObjId, ObjInsId]; + [ObjId] -> [ObjId] + end. diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_cm.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_cm.erl new file mode 100644 index 000000000..16e938b84 --- /dev/null +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_cm.erl @@ -0,0 +1,153 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_lwm2m_cm). + +-export([start_link/0]). + +-export([ register_channel/5 + , update_reg_info/2 + , unregister_channel/1 + ]). + +-export([ lookup_channel/1 + , all_channels/0 + ]). + +-export([ register_cmd/3 + , register_cmd/4 + , lookup_cmd/3 + , lookup_cmd_by_imei/1 + ]). + +%% gen_server callbacks +-export([ init/1 + , handle_call/3 + , handle_cast/2 + , handle_info/2 + , terminate/2 + , code_change/3 + ]). + +-define(LOG(Level, Format, Args), logger:Level("LWM2M-CM: " ++ Format, Args)). + +%% Server name +-define(CM, ?MODULE). + +-define(LWM2M_CHANNEL_TAB, emqx_lwm2m_channel). +-define(LWM2M_CMD_TAB, emqx_lwm2m_cmd). + +%% Batch drain +-define(BATCH_SIZE, 100000). + +%% @doc Start the channel manager. +start_link() -> + gen_server:start_link({local, ?CM}, ?MODULE, [], []). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- + +register_channel(IMEI, RegInfo, LifeTime, Ver, Peername) -> + Info = #{ + reg_info => RegInfo, + lifetime => LifeTime, + version => Ver, + peername => Peername + }, + true = ets:insert(?LWM2M_CHANNEL_TAB, {IMEI, Info}), + cast({registered, {IMEI, self()}}). + +update_reg_info(IMEI, RegInfo) -> + case lookup_channel(IMEI) of + [{_, RegInfo0}] -> + true = ets:insert(?LWM2M_CHANNEL_TAB, {IMEI, RegInfo0#{reg_info => RegInfo}}), + ok; + [] -> + ok + end. + +unregister_channel(IMEI) when is_binary(IMEI) -> + true = ets:delete(?LWM2M_CHANNEL_TAB, IMEI), + ok. + +lookup_channel(IMEI) -> + ets:lookup(?LWM2M_CHANNEL_TAB, IMEI). + +all_channels() -> + ets:tab2list(?LWM2M_CHANNEL_TAB). + +register_cmd(IMEI, Path, Type) -> + true = ets:insert(?LWM2M_CMD_TAB, {{IMEI, Path, Type}, undefined}). + +register_cmd(_IMEI, undefined, _Type, _Result) -> + ok; +register_cmd(IMEI, Path, Type, Result) -> + true = ets:insert(?LWM2M_CMD_TAB, {{IMEI, Path, Type}, Result}). + +lookup_cmd(IMEI, Path, Type) -> + ets:lookup(?LWM2M_CMD_TAB, {IMEI, Path, Type}). + +lookup_cmd_by_imei(IMEI) -> + ets:select(?LWM2M_CHANNEL_TAB, [{{{IMEI, '_', '_'}, '$1'}, [], ['$_']}]). + +%% @private +cast(Msg) -> gen_server:cast(?CM, Msg). + +%%-------------------------------------------------------------------- +%% gen_server callbacks +%%-------------------------------------------------------------------- + +init([]) -> + TabOpts = [public, {write_concurrency, true}, {read_concurrency, true}], + ok = emqx_tables:new(?LWM2M_CHANNEL_TAB, [set, compressed | TabOpts]), + ok = emqx_tables:new(?LWM2M_CMD_TAB, [set, compressed | TabOpts]), + {ok, #{chan_pmon => emqx_pmon:new()}}. + +handle_call(Req, _From, State) -> + ?LOG(error, "Unexpected call: ~p", [Req]), + {reply, ignored, State}. + +handle_cast({registered, {IMEI, ChanPid}}, State = #{chan_pmon := PMon}) -> + PMon1 = emqx_pmon:monitor(ChanPid, IMEI, PMon), + {noreply, State#{chan_pmon := PMon1}}; + +handle_cast(Msg, State) -> + ?LOG(error, "Unexpected cast: ~p", [Msg]), + {noreply, State}. + +handle_info({'DOWN', _MRef, process, Pid, _Reason}, State = #{chan_pmon := PMon}) -> + ChanPids = [Pid | emqx_misc:drain_down(?BATCH_SIZE)], + {Items, PMon1} = emqx_pmon:erase_all(ChanPids, PMon), + ok = emqx_pool:async_submit(fun lists:foreach/2, [fun clean_down/1, Items]), + {noreply, State#{chan_pmon := PMon1}}; + +handle_info(Info, State) -> + ?LOG(error, "Unexpected info: ~p", [Info]), + {noreply, State}. + +terminate(_Reason, _State) -> + emqx_stats:cancel_update(chan_stats). + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +clean_down({_ChanPid, IMEI}) -> + unregister_channel(IMEI). diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_cm_sup.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_cm_sup.erl new file mode 100644 index 000000000..b55f4f33c --- /dev/null +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_cm_sup.erl @@ -0,0 +1,41 @@ + +%%-------------------------------------------------------------------- +%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_lwm2m_cm_sup). + +-behaviour(supervisor). + +-export([start_link/0]). + +-export([init/1]). + +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +init([]) -> + CM = #{id => emqx_lwm2m_cm, + start => {emqx_lwm2m_cm, start_link, []}, + restart => permanent, + shutdown => 5000, + type => worker, + modules => [emqx_lwm2m_cm]}, + SupFlags = #{strategy => one_for_one, + intensity => 100, + period => 10 + }, + {ok, {SupFlags, [CM]}}. + diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_sup.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_sup.erl index d28e00874..0546e0080 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_sup.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_sup.erl @@ -29,4 +29,11 @@ start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). init(_Args) -> - {ok, { {one_for_all, 10, 3600}, [?CHILD(emqx_lwm2m_xml_object_db)] }}. + CmSup = #{id => emqx_lwm2m_cm_sup, + start => {emqx_lwm2m_cm_sup, start_link, []}, + restart => permanent, + shutdown => infinity, + type => supervisor, + modules => [emqx_lwm2m_cm_sup] + }, + {ok, { {one_for_all, 10, 3600}, [?CHILD(emqx_lwm2m_xml_object_db), CmSup] }}. diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object.erl index 5931ae75e..dd9911407 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object.erl @@ -21,9 +21,11 @@ -export([ get_obj_def/2 , get_object_id/1 + , get_object_name/1 , get_object_and_resource_id/2 , get_resource_type/2 , get_resource_name/2 + , get_resource_operations/2 ]). -define(LOG(Level, Format, Args), @@ -42,6 +44,10 @@ get_object_id(ObjDefinition) -> [#xmlText{value=ObjectId}] = xmerl_xpath:string("ObjectID/text()", ObjDefinition), ObjectId. +get_object_name(ObjDefinition) -> + [#xmlText{value=ObjectName}] = xmerl_xpath:string("Name/text()", ObjDefinition), + ObjectName. + get_object_and_resource_id(ResourceNameBinary, ObjDefinition) -> ResourceNameString = binary_to_list(ResourceNameBinary), @@ -60,3 +66,8 @@ get_resource_name(ResourceIdInt, ObjDefinition) -> ResourceIdString = integer_to_list(ResourceIdInt), [#xmlText{value=Name}] = xmerl_xpath:string("Resources/Item[@ID=\""++ResourceIdString++"\"]/Name/text()", ObjDefinition), Name. + +get_resource_operations(ResourceIdInt, ObjDefinition) -> + ResourceIdString = integer_to_list(ResourceIdInt), + [#xmlText{value=Operations}] = xmerl_xpath:string("Resources/Item[@ID=\""++ResourceIdString++"\"]/Operations/text()", ObjDefinition), + Operations. diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object_db.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object_db.erl index b974c148c..012d0a649 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object_db.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object_db.erl @@ -58,7 +58,7 @@ find_objectid(ObjectId) -> false -> ObjectId end, case ets:lookup(?LWM2M_OBJECT_DEF_TAB, ObjectIdInt) of - [] -> error(no_xml_definition); + [] -> {error, no_xml_definition}; [{ObjectId, Xml}] -> Xml end. @@ -121,8 +121,10 @@ load(BaseDir) -> true -> BaseDir++"*.xml"; false -> BaseDir++"/*.xml" end, - AllXmlFiles = filelib:wildcard(Wild), - load_loop(AllXmlFiles). + case filelib:wildcard(Wild) of + [] -> error(no_xml_files_found, BaseDir); + AllXmlFiles -> load_loop(AllXmlFiles) + end. load_loop([]) -> ok; From f3617aa08299e777fac92909293eb8562cccf19d Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 21 Jun 2021 10:01:06 +0800 Subject: [PATCH 14/78] chore: upgrade lwm2m_coap to 1.1.4 --- apps/emqx_lwm2m/rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_lwm2m/rebar.config b/apps/emqx_lwm2m/rebar.config index fbfcdc02f..92648be28 100644 --- a/apps/emqx_lwm2m/rebar.config +++ b/apps/emqx_lwm2m/rebar.config @@ -1,5 +1,5 @@ {deps, - [{lwm2m_coap, {git, "https://github.com/emqx/lwm2m-coap", {tag, "v1.1.2"}}} + [{lwm2m_coap, {git, "https://github.com/emqx/lwm2m-coap", {tag, "v1.1.4"}}} ]}. {profiles, From 36685cc945a8ee13088c72b2aeeaa74b98440160 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Mon, 21 Jun 2021 09:29:34 +0200 Subject: [PATCH 15/78] chore(relup): add relup dependency injection --- rebar.config.erl | 7 ++ scripts/inject-deps.escript | 141 ++++++++++++++++++++++++++++++++++++ 2 files changed, 148 insertions(+) create mode 100755 scripts/inject-deps.escript diff --git a/rebar.config.erl b/rebar.config.erl index 0248e6dab..543333895 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -126,23 +126,30 @@ prod_compile_opts() -> prod_overrides() -> [{add, [ {erl_opts, [deterministic]}]}]. +relup_deps(Profile) -> + {post_hooks, [{"(linux|darwin|solaris|freebsd|netbsd|openbsd)", compile, "scripts/inject-deps.escript " ++ atom_to_list(Profile)}]}. + profiles() -> Vsn = get_vsn(), [ {'emqx', [ {erl_opts, prod_compile_opts()} , {relx, relx(Vsn, cloud, bin)} , {overrides, prod_overrides()} + , relup_deps('emqx') ]} , {'emqx-pkg', [ {erl_opts, prod_compile_opts()} , {relx, relx(Vsn, cloud, pkg)} , {overrides, prod_overrides()} + , relup_deps('emqx-pkg') ]} , {'emqx-edge', [ {erl_opts, prod_compile_opts()} , {relx, relx(Vsn, edge, bin)} , {overrides, prod_overrides()} + , relup_deps('emqx-edge') ]} , {'emqx-edge-pkg', [ {erl_opts, prod_compile_opts()} , {relx, relx(Vsn, edge, pkg)} , {overrides, prod_overrides()} + , relup_deps('emqx-edge-pkg') ]} , {check, [ {erl_opts, common_compile_opts()} ]} diff --git a/scripts/inject-deps.escript b/scripts/inject-deps.escript new file mode 100755 index 000000000..e0c50d772 --- /dev/null +++ b/scripts/inject-deps.escript @@ -0,0 +1,141 @@ +#!/usr/bin/env escript + +%% This script injects implicit relup dependencies for emqx applications. +%% +%% By 'implicit', it means that it is not feasible to define application +%% dependencies in .app.src files. +%% +%% For instance, during upgrade/downgrade, emqx_dashboard usually requires +%% a restart after (but not before) all plugins are upgraded (and maybe +%% restarted), however, the dependencies are not resolvable at build time +%% when relup is generated. +%% +%% This script is to be executed after compile, with the profile given as the +%% first argument. For each dependency, it modifies the .app file to +%% have the `relup_deps` list extended to application attributes. +%% +%% The `relup_deps` application attribute is then picked up by (EMQ's fork of) +%% `relx` when top-sorting apps to generate relup instructions + +-mode(compile). + +usage() -> + "Usage: " ++ escript:script_name() ++ " emqx|emqx-edge". + +-type app() :: atom(). +-type deps_overlay() :: {re, string()} | app(). + +%% deps/0 returns the dependency overlays. +%% {re, Pattern} to match application names using regexp pattern +-spec deps(string()) -> [{app(), [deps_overlay()]}]. +deps("emqx-edge" ++ _) -> + %% special case for edge + base_deps() ++ [{{re, ".+"}, [{exclude, emqx_reloader}]}]; +deps(_Profile) -> + base_deps(). + +base_deps() -> + [ {emqx_dashboard, [{re, "emqx_.*"}]} + , {emqx_management, [{re, "emqx_.*"}, {exclude, emqx_dashboard}]} + , {{re, "emqx_.*"}, [emqx]} + ]. + +main([Profile | _]) -> + ok = inject(Profile); +main(_Args) -> + io:format(standard_error, "~s", [usage()]), + erlang:halt(1). + +expand_names({Name, Deps}, AppNames) -> + Names = match_pattern(Name, AppNames), + [{N, Deps} || N <- Names]. + +%% merge k-v pairs with v1 ++ v2 +merge([], Acc) -> Acc; +merge([{K, V0} | Rest], Acc) -> + V = case lists:keyfind(K, 1, Acc) of + {K, V1} -> V1 ++ V0; + false -> V0 + end, + NewAcc = lists:keystore(K, 1, Acc, {K, V}), + merge(Rest, NewAcc). + +expand_deps([], _AppNames, Acc) -> Acc; +expand_deps([{exclude, Dep} | Deps], AppNames, Acc) -> + Matches = expand_deps([Dep], AppNames, []), + expand_deps(Deps, AppNames, Acc -- Matches); +expand_deps([Dep | Deps], AppNames, Acc) -> + NewAcc = add_to_list(Acc, match_pattern(Dep, AppNames)), + expand_deps(Deps, AppNames, NewAcc). + +inject(Profile) -> + LibDir = lib_dir(Profile), + AppNames = list_apps(LibDir), + Deps0 = lists:flatmap(fun(Dep) -> expand_names(Dep, AppNames) end, deps(Profile)), + Deps1 = merge(Deps0, []), + Deps2 = lists:map(fun({Name, DepsX}) -> + NewDeps = expand_deps(DepsX, AppNames, []), + {Name, NewDeps} + end, Deps1), + lists:foreach(fun({App, Deps}) -> inject(App, Deps, LibDir) end, Deps2). + +%% list the profile/lib dir to get all apps +list_apps(LibDir) -> + Apps = filelib:wildcard("*", LibDir), + lists:foldl(fun(App, Acc) -> [App || is_app(LibDir, App)] ++ Acc end, [], Apps). + +is_app(_LibDir, "." ++ _) -> false; %% ignore hidden dir +is_app(LibDir, AppName) -> + filelib:is_regular(filename:join([ebin_dir(LibDir, AppName), AppName ++ ".app"])) orelse + error({unknown_app, AppName}). %% wtf + +lib_dir(Profile) -> + filename:join(["_build", Profile, lib]). + +ebin_dir(LibDir, AppName) -> filename:join([LibDir, AppName, "ebin"]). + +inject(App0, DepsToAdd, LibDir) -> + App = str(App0), + AppEbinDir = ebin_dir(LibDir, App), + [AppFile0] = filelib:wildcard("*.app", AppEbinDir), + AppFile = filename:join(AppEbinDir, AppFile0), + {ok, [{application, AppName, Props}]} = file:consult(AppFile), + Deps0 = case lists:keyfind(relup_deps, 1, Props) of + {_, X} -> X; + false -> [] + end, + %% merge extra deps, but do not self-include + Deps = add_to_list(Deps0, DepsToAdd) -- [App0], + case Deps =:= [] of + true -> ok; + _ -> + NewProps = lists:keystore(relup_deps, 1, Props, {relup_deps, Deps}), + AppSpec = {application, AppName, NewProps}, + AppSpecIoData = io_lib:format("~p.", [AppSpec]), + io:format(user, "updated_relup_deps for ~p~n", [App]), + file:write_file(AppFile, AppSpecIoData) + end. + +str(A) when is_atom(A) -> atom_to_list(A). + +match_pattern({re, Re}, AppNames) -> + Match = fun(AppName) -> re:run(AppName, Re) =/= nomatch end, + AppNamesToAdd = lists:filter(Match, AppNames), + AppsToAdd = lists:map(fun(N) -> list_to_atom(N) end, AppNamesToAdd), + case AppsToAdd =:= [] of + true -> error({nomatch, Re}); + false -> AppsToAdd + end; +match_pattern(NameAtom, AppNames) -> + case lists:member(str(NameAtom), AppNames) of + true -> [NameAtom]; + false -> error({notfound, NameAtom}) + end. + +%% Append elements to list without duplication. No reordering. +add_to_list(List, []) -> List; +add_to_list(List, [H | T]) -> + case lists:member(H, List) of + true -> add_to_list(List, T); + false -> add_to_list(List ++ [H], T) + end. From c7fe49c20092ac71784da42be360ff2914bc56ae Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Mon, 21 Jun 2021 09:47:00 +0200 Subject: [PATCH 16/78] test(ci): add plugin list status check after relup new vsn install --- .ci/fvt_tests/relup.lux | 10 +++++++++- apps/emqx_dashboard/src/emqx_dashboard.appup.src | 16 ++++++++++++++++ scripts/inject-deps.escript | 3 +++ 3 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 apps/emqx_dashboard/src/emqx_dashboard.appup.src diff --git a/.ci/fvt_tests/relup.lux b/.ci/fvt_tests/relup.lux index 33c35cb3e..91c11d3ae 100644 --- a/.ci/fvt_tests/relup.lux +++ b/.ci/fvt_tests/relup.lux @@ -43,7 +43,7 @@ !sed -i '/emqx_telemetry/d' data/loaded_plugins !./bin/emqx start - ?EMQ X (.*) is started successfully! + ?EMQ X .* is started successfully! ?SH-PROMPT !./bin/emqx_ctl cluster join emqx@127.0.0.1 @@ -99,6 +99,10 @@ """ ?SH-PROMPT + !./bin/emqx_ctl plugins list | grep emqx_management + ?Plugin\(emqx_management.*active=true\) + ?SH-PROMPT + [shell emqx2] !echo "" > log/emqx.log.1 ?SH-PROMPT @@ -120,6 +124,10 @@ """ ?SH-PROMPT + !./bin/emqx_ctl plugins list | grep emqx_management + ?Plugin\(emqx_management.*active=true\) + ?SH-PROMPT + [shell bench] ???publish complete ??SH-PROMPT: diff --git a/apps/emqx_dashboard/src/emqx_dashboard.appup.src b/apps/emqx_dashboard/src/emqx_dashboard.appup.src new file mode 100644 index 000000000..678bd3b22 --- /dev/null +++ b/apps/emqx_dashboard/src/emqx_dashboard.appup.src @@ -0,0 +1,16 @@ +%% -*- mode: erlang -*- +{VSN, + [ {"4.3.0", + %% load all plugins + %% NOTE: this depends on the fact that emqx_dashboard is always + %% the last application gets upgraded + [ {apply, {emqx_plugins, load, []}} + ]}, + {<<".*">>, []} + ], + [ {"4.3.0", + [ {apply, {emqx_plugins, load, []}} + ]}, + {<<".*">>, []} + ] +}. diff --git a/scripts/inject-deps.escript b/scripts/inject-deps.escript index e0c50d772..1465acc8f 100755 --- a/scripts/inject-deps.escript +++ b/scripts/inject-deps.escript @@ -35,6 +35,9 @@ deps(_Profile) -> base_deps(). base_deps() -> + %% make sure emqx_dashboard depends on all other emqx_xxx apps + %% so the appup instructions for emqx_dashboard is always the last + %% to be executed [ {emqx_dashboard, [{re, "emqx_.*"}]} , {emqx_management, [{re, "emqx_.*"}, {exclude, emqx_dashboard}]} , {{re, "emqx_.*"}, [emqx]} From ab7590106108d5866cc24e747c9cb8b5e827e59f Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 23 Jun 2021 08:50:20 +0800 Subject: [PATCH 17/78] fix(coap): fix coap uri format (#5059) --- apps/emqx_lwm2m/rebar.config | 2 +- apps/emqx_lwm2m/src/emqx_lwm2m_coap_resource.erl | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx_lwm2m/rebar.config b/apps/emqx_lwm2m/rebar.config index 92648be28..23b03eaa4 100644 --- a/apps/emqx_lwm2m/rebar.config +++ b/apps/emqx_lwm2m/rebar.config @@ -1,5 +1,5 @@ {deps, - [{lwm2m_coap, {git, "https://github.com/emqx/lwm2m-coap", {tag, "v1.1.4"}}} + [{lwm2m_coap, {git, "https://github.com/emqx/lwm2m-coap", {tag, "v1.1.5"}}} ]}. {profiles, diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_coap_resource.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_coap_resource.erl index de83643a8..f94c2bc72 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_coap_resource.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_coap_resource.erl @@ -48,11 +48,11 @@ -define(LOG(Level, Format, Args), logger:Level("LWM2M-RESOURCE: " ++ Format, Args)). -dialyzer([{nowarn_function, [coap_discover/2]}]). -% we use {'absolute', string(), [{atom(), binary()}]} as coap_uri() +% we use {'absolute', list(binary()), [{atom(), binary()}]} as coap_uri() % https://github.com/emqx/lwm2m-coap/blob/258e9bd3762124395e83c1e68a1583b84718230f/src/lwm2m_coap_resource.erl#L61 % resource operations coap_discover(_Prefix, _Args) -> - [{absolute, "mqtt", []}]. + [{absolute, [<<"mqtt">>], []}]. coap_get(ChId, [?PREFIX], Query, Content, Lwm2mState) -> ?LOG(debug, "~p ~p GET Query=~p, Content=~p", [self(),ChId, Query, Content]), From 96c07a505583cd7038e3be2e997341655e443cf7 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Tue, 22 Jun 2021 09:30:52 +0800 Subject: [PATCH 18/78] chore(CI): upload rebar3.crashdump file when slim build failure --- .github/workflows/build_slim_packages.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 6c9bbf04a..bf85578c5 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -38,6 +38,11 @@ jobs: run: make ${EMQX_NAME}-zip - name: build deb/rpm packages run: make ${EMQX_NAME}-pkg + - uses: actions/upload-artifact@v1 + if: failure() + with: + name: rebar3.crashdump + path: ./rebar3.crashdump - name: pakcages test run: | export CODE_PATH=$GITHUB_WORKSPACE @@ -94,6 +99,11 @@ jobs: make ensure-rebar3 sudo cp rebar3 /usr/local/bin/rebar3 make ${EMQX_NAME}-zip + - uses: actions/upload-artifact@v1 + if: failure() + with: + name: rebar3.crashdump + path: ./rebar3.crashdump - name: test run: | pkg_name=$(basename _packages/${EMQX_NAME}/emqx-*.zip) From 8da6d5cf1688993663d5e71cf25f5196c9fb5b53 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 23 Jun 2021 09:45:24 +0800 Subject: [PATCH 19/78] fix(emqx_cm): catch noproc exception from rpc_call (#5048) --- apps/emqx/src/emqx.appup.src | 14 ++++++++++---- apps/emqx/src/emqx_cm.erl | 3 +++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/apps/emqx/src/emqx.appup.src b/apps/emqx/src/emqx.appup.src index 53e77a103..a8b6136bb 100644 --- a/apps/emqx/src/emqx.appup.src +++ b/apps/emqx/src/emqx.appup.src @@ -3,7 +3,8 @@ [ {"4.3.3", [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]} + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]} ]}, {"4.3.2", [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, @@ -11,7 +12,9 @@ {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]} + ]}, {"4.3.1", [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, @@ -45,7 +48,8 @@ {<<".*">>,[]}], [{"4.3.3", [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]} + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]} ]}, {"4.3.2", [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, @@ -53,7 +57,9 @@ {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]} + ]}, {"4.3.1", [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, diff --git a/apps/emqx/src/emqx_cm.erl b/apps/emqx/src/emqx_cm.erl index b15a2ff79..6eb375aba 100644 --- a/apps/emqx/src/emqx_cm.erl +++ b/apps/emqx/src/emqx_cm.erl @@ -294,6 +294,9 @@ do_discard_session(ClientId, Pid) -> _ : {noproc, _} -> % emqx_connection: gen_server:call ?tp(debug, "session_already_gone", #{pid => Pid}), ok; + _ : {'EXIT', {noproc, _}} -> % rpc_call/3 + ?tp(debug, "session_already_gone", #{pid => Pid}), + ok; _ : {{shutdown, _}, _} -> ?tp(debug, "session_already_shutdown", #{pid => Pid}), ok; From 8c0c6974d9feb3445eb1c456ed67c7e8760a86f9 Mon Sep 17 00:00:00 2001 From: Turtle Date: Wed, 23 Jun 2021 11:08:14 +0800 Subject: [PATCH 20/78] fix: Ignore repeatedly receiving connection packet in the wait_will_msg/wait_will_topic/connected state --- apps/emqx_sn/src/emqx_sn_gateway.erl | 36 +++++--------------- apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl | 13 ------- 2 files changed, 9 insertions(+), 40 deletions(-) diff --git a/apps/emqx_sn/src/emqx_sn_gateway.erl b/apps/emqx_sn/src/emqx_sn_gateway.erl index 9a8c68583..1bccf0c1a 100644 --- a/apps/emqx_sn/src/emqx_sn_gateway.erl +++ b/apps/emqx_sn/src/emqx_sn_gateway.erl @@ -250,8 +250,9 @@ wait_for_will_topic(cast, {incoming, ?SN_ADVERTISE_MSG(_GwId, _Radius)}, _State) % ignore keep_state_and_data; -wait_for_will_topic(cast, {incoming, ?SN_CONNECT_MSG(Flags, _ProtoId, Duration, ClientId)}, State) -> - do_2nd_connect(Flags, Duration, ClientId, State); +wait_for_will_topic(cast, {incoming, ?SN_CONNECT_MSG(_Flags, _ProtoId, _Duration, _ClientId)}, _State) -> + ?LOG(warning, "Receive connect packet in wait_for_will_topic state", []), + keep_state_and_data; wait_for_will_topic(cast, {outgoing, Packet}, State) -> {keep_state, handle_outgoing(Packet, State)}; @@ -275,9 +276,9 @@ wait_for_will_msg(cast, {incoming, ?SN_ADVERTISE_MSG(_GwId, _Radius)}, _State) - % ignore keep_state_and_data; -%% XXX: ?? Why we will handling the 2nd CONNECT packet ?? -wait_for_will_msg(cast, {incoming, ?SN_CONNECT_MSG(Flags, _ProtoId, Duration, ClientId)}, State) -> - do_2nd_connect(Flags, Duration, ClientId, State); +wait_for_will_msg(cast, {incoming, ?SN_CONNECT_MSG(_Flags, _ProtoId, _Duration, _ClientId)}, _State) -> + ?LOG(warning, "Receive connect packet in wait_for_will_msg state", []), + keep_state_and_data; wait_for_will_msg(cast, {outgoing, Packet}, State) -> {keep_state, handle_outgoing(Packet, State)}; @@ -365,8 +366,9 @@ connected(cast, {incoming, ?SN_ADVERTISE_MSG(_GwId, _Radius)}, State) -> % ignore {keep_state, State}; -connected(cast, {incoming, ?SN_CONNECT_MSG(Flags, _ProtoId, Duration, ClientId)}, State) -> - do_2nd_connect(Flags, Duration, ClientId, State); +connected(cast, {incoming, ?SN_CONNECT_MSG(_Flags, _ProtoId, _Duration, _ClientId)}, _State) -> + ?LOG(warning, "Receive connect packet in wait_for_will_topic state", []), + keep_state_and_data; connected(cast, {outgoing, Packet}, State) -> {keep_state, handle_outgoing(Packet, State)}; @@ -845,26 +847,6 @@ do_connect(ClientId, CleanStart, WillFlag, Duration, State) -> handle_incoming(?CONNECT_PACKET(ConnPkt), NState) end. -do_2nd_connect(Flags, Duration, ClientId, State = #state{sockname = Sockname, - peername = Peername, - channel = Channel}) -> - emqx_logger:set_metadata_clientid(ClientId), - #mqtt_sn_flags{will = Will, clean_start = CleanStart} = Flags, - NChannel = case CleanStart of - true -> - emqx_channel:terminate(normal, Channel), - emqx_sn_registry:unregister_topic(ClientId), - emqx_channel:init(#{socktype => udp, - sockname => Sockname, - peername => Peername, - peercert => ?NO_PEERCERT, - conn_mod => ?MODULE - }, ?DEFAULT_CHAN_OPTIONS); - false -> Channel - end, - NState = State#state{channel = NChannel}, - do_connect(ClientId, CleanStart, Will, Duration, NState). - handle_subscribe(?SN_NORMAL_TOPIC, TopicName, QoS, MsgId, State=#state{channel = Channel}) -> ClientId = emqx_channel:info(clientid, Channel), diff --git a/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl b/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl index a953a45e7..0947bdaca 100644 --- a/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl +++ b/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl @@ -98,19 +98,6 @@ t_connect(_) -> ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), gen_udp:close(Socket). -t_do_2nd_connect(_) -> - {ok, Socket} = gen_udp:open(0, [binary]), - ClientId = ?CLIENTID, - send_connect_msg(Socket, ClientId), - ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), - timer:sleep(100), - send_connect_msg(Socket, <<"client_id_other">>), - ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), - - send_disconnect_msg(Socket, undefined), - ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), - gen_udp:close(Socket). - t_subscribe(_) -> Dup = 0, QoS = 0, From bde9d052a5b2829a99cca89ae9495e63f720c9da Mon Sep 17 00:00:00 2001 From: Turtle Date: Wed, 23 Jun 2021 16:23:31 +0800 Subject: [PATCH 21/78] feat(lwm2m): fix check dialyzer fail --- .../emqx_lwm2m/src/emqx_lwm2m_cmd_handler.erl | 1 + apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl | 74 ++++++++++++------- 2 files changed, 50 insertions(+), 25 deletions(-) diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_cmd_handler.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_cmd_handler.erl index 2d208bdb4..b3251a275 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_cmd_handler.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_cmd_handler.erl @@ -23,6 +23,7 @@ -export([ mqtt2coap/2 , coap2mqtt/4 , ack2mqtt/1 + , extract_path/1 ]). -export([path_list/1]). diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl index 55f992da6..34c72dcca 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl @@ -103,6 +103,7 @@ init(CoapPid, EndpointName, Peername = {_Peerhost, _Port}, RegInfo = #{<<"lt">> emqx_cm:register_channel(EndpointName, CoapPid, conninfo(Lwm2mState1)) end), emqx_cm:insert_channel_info(EndpointName, info(Lwm2mState1), stats(Lwm2mState1)), + emqx_lwm2m_cm:register_channel(EndpointName, RegInfo, LifeTime, Ver, Peername), {ok, Lwm2mState1#lwm2m_state{life_timer = emqx_lwm2m_timer:start_timer(LifeTime, {life_timer, expired})}}; {error, Error} -> @@ -120,10 +121,8 @@ post_init(Lwm2mState = #lwm2m_state{endpoint_name = _EndpointName, _ = send_to_broker(<<"register">>, #{<<"data">> => RegInfo}, Lwm2mState), Lwm2mState#lwm2m_state{mqtt_topic = Topic}. -update_reg_info(NewRegInfo, Lwm2mState = #lwm2m_state{ - life_timer = LifeTimer, register_info = RegInfo, - coap_pid = CoapPid}) -> - +update_reg_info(NewRegInfo, Lwm2mState=#lwm2m_state{life_timer = LifeTimer, register_info = RegInfo, + coap_pid = CoapPid, endpoint_name = Epn}) -> UpdatedRegInfo = maps:merge(RegInfo, NewRegInfo), _ = case proplists:get_value(update_msg_publish_condition, @@ -134,6 +133,7 @@ update_reg_info(NewRegInfo, Lwm2mState = #lwm2m_state{ %% - report the registration info update, but only when objectList is updated. case NewRegInfo of #{<<"objectList">> := _} -> + emqx_lwm2m_cm:update_reg_info(Epn, NewRegInfo), send_to_broker(<<"update">>, #{<<"data">> => UpdatedRegInfo}, Lwm2mState); _ -> ok end @@ -151,7 +151,8 @@ update_reg_info(NewRegInfo, Lwm2mState = #lwm2m_state{ register_info = UpdatedRegInfo}. replace_reg_info(NewRegInfo, Lwm2mState=#lwm2m_state{life_timer = LifeTimer, - coap_pid = CoapPid}) -> + coap_pid = CoapPid, + endpoint_name = EndpointName}) -> _ = send_to_broker(<<"register">>, #{<<"data">> => NewRegInfo}, Lwm2mState), %% - flush cached donwlink commands @@ -161,7 +162,7 @@ replace_reg_info(NewRegInfo, Lwm2mState=#lwm2m_state{life_timer = LifeTimer, UpdatedLifeTimer = emqx_lwm2m_timer:refresh_timer( maps:get(<<"lt">>, NewRegInfo), LifeTimer), - _ = send_auto_observe(CoapPid, NewRegInfo), + _ = send_auto_observe(CoapPid, NewRegInfo, EndpointName), ?LOG(debug, "Replace RegInfo to: ~p", [NewRegInfo]), Lwm2mState#lwm2m_state{life_timer = UpdatedLifeTimer, @@ -174,15 +175,20 @@ send_ul_data(EventType, Payload, Lwm2mState=#lwm2m_state{coap_pid = CoapPid}) -> Lwm2mState. auto_observe(Lwm2mState = #lwm2m_state{register_info = RegInfo, - coap_pid = CoapPid}) -> - _ = send_auto_observe(CoapPid, RegInfo), + coap_pid = CoapPid, + endpoint_name = EndpointName}) -> + _ = send_auto_observe(CoapPid, RegInfo, EndpointName), Lwm2mState. -deliver(#message{topic = Topic, payload = Payload}, Lwm2mState = #lwm2m_state{coap_pid = CoapPid, register_info = RegInfo, started_at = StartedAt}) -> +deliver(#message{topic = Topic, payload = Payload}, + Lwm2mState = #lwm2m_state{coap_pid = CoapPid, + register_info = RegInfo, + started_at = StartedAt, + endpoint_name = EndpointName}) -> IsCacheMode = is_cache_mode(RegInfo, StartedAt), ?LOG(debug, "Get MQTT message from broker, IsCacheModeNow?: ~p, Topic: ~p, Payload: ~p", [IsCacheMode, Topic, Payload]), AlternatePath = maps:get(<<"alternatePath">>, RegInfo, <<"/">>), - deliver_to_coap(AlternatePath, Payload, CoapPid, IsCacheMode), + deliver_to_coap(AlternatePath, Payload, CoapPid, IsCacheMode, EndpointName), Lwm2mState. get_info(Lwm2mState = #lwm2m_state{endpoint_name = EndpointName, peername = {PeerHost, _}, @@ -238,20 +244,21 @@ time_now() -> erlang:system_time(millisecond). %% Deliver downlink message to coap %%-------------------------------------------------------------------- -deliver_to_coap(AlternatePath, JsonData, CoapPid, CacheMode) when is_binary(JsonData)-> +deliver_to_coap(AlternatePath, JsonData, CoapPid, CacheMode, EndpointName) when is_binary(JsonData)-> try TermData = emqx_json:decode(JsonData, [return_maps]), - deliver_to_coap(AlternatePath, TermData, CoapPid, CacheMode) + deliver_to_coap(AlternatePath, TermData, CoapPid, CacheMode, EndpointName) catch C:R:Stack -> ?LOG(error, "deliver_to_coap - Invalid JSON: ~p, Exception: ~p, stacktrace: ~p", [JsonData, {C, R}, Stack]) end; -deliver_to_coap(AlternatePath, TermData, CoapPid, CacheMode) when is_map(TermData) -> +deliver_to_coap(AlternatePath, TermData, CoapPid, CacheMode, EndpointName) when is_map(TermData) -> ?LOG(info, "SEND To CoAP, AlternatePath=~p, Data=~p", [AlternatePath, TermData]), {CoapRequest, Ref} = emqx_lwm2m_cmd_handler:mqtt2coap(AlternatePath, TermData), - + MsgType = maps:get(<<"msgType">>, Ref), + emqx_lwm2m_cm:register_cmd(EndpointName, emqx_lwm2m_cmd_handler:extract_path(Ref), MsgType), case CacheMode of false -> do_deliver_to_coap(CoapPid, CoapRequest, Ref); @@ -266,7 +273,12 @@ deliver_to_coap(AlternatePath, TermData, CoapPid, CacheMode) when is_map(TermDat send_to_broker(EventType, Payload = #{}, Lwm2mState) -> do_send_to_broker(EventType, Payload, Lwm2mState). -do_send_to_broker(EventType, Payload, Lwm2mState) -> +do_send_to_broker(EventType, #{<<"data">> := Data} = Payload, #lwm2m_state{endpoint_name = EndpointName} = Lwm2mState) -> + ReqPath = maps:get(<<"reqPath">>, Data, undefined), + Code = maps:get(<<"code">>, Data, undefined), + CodeMsg = maps:get(<<"codeMsg">>, Data, undefined), + Content = maps:get(<<"content">>, Data, undefined), + emqx_lwm2m_cm:register_cmd(EndpointName, ReqPath, EventType, {Code, CodeMsg, Content}), NewPayload = maps:put(<<"msgType">>, EventType, Payload), Topic = uplink_topic(EventType, Lwm2mState), publish(Topic, emqx_json:encode(NewPayload), _Qos = 0, Lwm2mState#lwm2m_state.endpoint_name). @@ -281,7 +293,7 @@ auto_observe_object_list(Expected, Registered) -> Expected1 = lists:map(fun(S) -> iolist_to_binary(S) end, Expected), lists:filter(fun(S) -> lists:member(S, Expected1) end, Registered). -send_auto_observe(CoapPid, RegInfo) -> +send_auto_observe(CoapPid, RegInfo, EndpointName) -> %% - auto observe the objects case proplists:get_value(auto_observe, lwm2m_coap_responder:options(), false) of false -> @@ -292,25 +304,37 @@ send_auto_observe(CoapPid, RegInfo) -> maps:get(<<"objectList">>, RegInfo, []) ), AlternatePath = maps:get(<<"alternatePath">>, RegInfo, <<"/">>), - auto_observe(AlternatePath, Objectlists, CoapPid) + auto_observe(AlternatePath, Objectlists, CoapPid, EndpointName) end. -auto_observe(AlternatePath, ObjectList, CoapPid) -> +auto_observe(AlternatePath, ObjectList, CoapPid, EndpointName) -> ?LOG(info, "Auto Observe on: ~p", [ObjectList]), erlang:spawn(fun() -> - observe_object_list(AlternatePath, ObjectList, CoapPid) + observe_object_list(AlternatePath, ObjectList, CoapPid, EndpointName) end). -observe_object_list(AlternatePath, ObjectList, CoapPid) -> +observe_object_list(AlternatePath, ObjectList, CoapPid, EndpointName) -> lists:foreach(fun(ObjectPath) -> - observe_object_slowly(AlternatePath, ObjectPath, CoapPid, 100) + [ObjId| LastPath] = emqx_lwm2m_cmd_handler:path_list(ObjectPath), + case ObjId of + <<"19">> -> + [ObjInsId | _LastPath1] = LastPath, + case ObjInsId of + <<"0">> -> + observe_object_slowly(AlternatePath, <<"/19/0/0">>, CoapPid, 100, EndpointName); + _ -> + observe_object_slowly(AlternatePath, ObjectPath, CoapPid, 100, EndpointName) + end; + _ -> + observe_object_slowly(AlternatePath, ObjectPath, CoapPid, 100, EndpointName) + end end, ObjectList). -observe_object_slowly(AlternatePath, ObjectPath, CoapPid, Interval) -> - observe_object(AlternatePath, ObjectPath, CoapPid), +observe_object_slowly(AlternatePath, ObjectPath, CoapPid, Interval, EndpointName) -> + observe_object(AlternatePath, ObjectPath, CoapPid, EndpointName), timer:sleep(Interval). -observe_object(AlternatePath, ObjectPath, CoapPid) -> +observe_object(AlternatePath, ObjectPath, CoapPid, EndpointName) -> Payload = #{ <<"msgType">> => <<"observe">>, <<"data">> => #{ @@ -318,7 +342,7 @@ observe_object(AlternatePath, ObjectPath, CoapPid) -> } }, ?LOG(info, "Observe ObjectPath: ~p", [ObjectPath]), - deliver_to_coap(AlternatePath, Payload, CoapPid, false). + deliver_to_coap(AlternatePath, Payload, CoapPid, false, EndpointName). do_deliver_to_coap_slowly(CoapPid, CoapRequestList, Interval) -> erlang:spawn(fun() -> From 22f1e8d7eda93814bab6f33437572d9fee7493af Mon Sep 17 00:00:00 2001 From: Turtle Date: Wed, 23 Jun 2021 16:50:28 +0800 Subject: [PATCH 22/78] chore(review): review 4.3.4 --- apps/emqx/src/emqx.appup.src | 6 ++++-- apps/emqx_auth_ldap/src/emqx_auth_ldap.appup.src | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/apps/emqx/src/emqx.appup.src b/apps/emqx/src/emqx.appup.src index a8b6136bb..0b2b9b2d5 100644 --- a/apps/emqx/src/emqx.appup.src +++ b/apps/emqx/src/emqx.appup.src @@ -4,7 +4,8 @@ {"4.3.3", [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]} + {load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]} ]}, {"4.3.2", [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, @@ -49,7 +50,8 @@ [{"4.3.3", [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]} + {load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]} ]}, {"4.3.2", [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, diff --git a/apps/emqx_auth_ldap/src/emqx_auth_ldap.appup.src b/apps/emqx_auth_ldap/src/emqx_auth_ldap.appup.src index 8b70bf484..9750f3cf1 100644 --- a/apps/emqx_auth_ldap/src/emqx_auth_ldap.appup.src +++ b/apps/emqx_auth_ldap/src/emqx_auth_ldap.appup.src @@ -1,8 +1,8 @@ %% -*- mode: erlang -*- {VSN, [{"4.3.0", - [{load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]}]}, + [{load_module,emqx_acl_ldap,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.3.0", - [{load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]}]}, + [{load_module,emqx_acl_ldap,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}]}. From 4b1bb915164977d27a9beed296213f7045ac0fae Mon Sep 17 00:00:00 2001 From: Turtle Date: Wed, 23 Jun 2021 19:02:04 +0800 Subject: [PATCH 23/78] chore: fix inject deps notfound emqx_reloader --- scripts/inject-deps.escript | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/scripts/inject-deps.escript b/scripts/inject-deps.escript index 1465acc8f..359462a66 100755 --- a/scripts/inject-deps.escript +++ b/scripts/inject-deps.escript @@ -25,15 +25,6 @@ usage() -> -type app() :: atom(). -type deps_overlay() :: {re, string()} | app(). -%% deps/0 returns the dependency overlays. -%% {re, Pattern} to match application names using regexp pattern --spec deps(string()) -> [{app(), [deps_overlay()]}]. -deps("emqx-edge" ++ _) -> - %% special case for edge - base_deps() ++ [{{re, ".+"}, [{exclude, emqx_reloader}]}]; -deps(_Profile) -> - base_deps(). - base_deps() -> %% make sure emqx_dashboard depends on all other emqx_xxx apps %% so the appup instructions for emqx_dashboard is always the last @@ -74,7 +65,7 @@ expand_deps([Dep | Deps], AppNames, Acc) -> inject(Profile) -> LibDir = lib_dir(Profile), AppNames = list_apps(LibDir), - Deps0 = lists:flatmap(fun(Dep) -> expand_names(Dep, AppNames) end, deps(Profile)), + Deps0 = lists:flatmap(fun(Dep) -> expand_names(Dep, AppNames) end, base_deps()), Deps1 = merge(Deps0, []), Deps2 = lists:map(fun({Name, DepsX}) -> NewDeps = expand_deps(DepsX, AppNames, []), From e4cb11fb43fac6a3a726de90eaebcb23ff349ffb Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Wed, 23 Jun 2021 13:35:49 +0200 Subject: [PATCH 24/78] chore: add more info in error message --- scripts/inject-deps.escript | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/inject-deps.escript b/scripts/inject-deps.escript index 359462a66..e5cd47140 100755 --- a/scripts/inject-deps.escript +++ b/scripts/inject-deps.escript @@ -23,7 +23,6 @@ usage() -> "Usage: " ++ escript:script_name() ++ " emqx|emqx-edge". -type app() :: atom(). --type deps_overlay() :: {re, string()} | app(). base_deps() -> %% make sure emqx_dashboard depends on all other emqx_xxx apps @@ -80,8 +79,8 @@ list_apps(LibDir) -> is_app(_LibDir, "." ++ _) -> false; %% ignore hidden dir is_app(LibDir, AppName) -> - filelib:is_regular(filename:join([ebin_dir(LibDir, AppName), AppName ++ ".app"])) orelse - error({unknown_app, AppName}). %% wtf + Path = filename:join([ebin_dir(LibDir, AppName), AppName ++ ".app"]), + filelib:is_regular(Path) orelse error({unknown_app, AppName, Path}). %% wtf lib_dir(Profile) -> filename:join(["_build", Profile, lib]). From 1ecd0a54b9f1e4b8105cbd13d8211deabeb82d66 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Wed, 23 Jun 2021 20:14:47 +0800 Subject: [PATCH 25/78] chore(CI): add DIAGNOSTIC=1 when build windows --- .github/workflows/build_packages.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 2090f722e..4b23affd8 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -83,6 +83,7 @@ jobs: - name: build env: PYTHON: python + DIAGNOSTIC: 1 run: | $env:PATH = "${{ steps.install_erlang.outputs.erlpath }}\bin;$env:PATH" From 9471f5ae07c1f929731f8a613de47502aa63bef8 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Wed, 23 Jun 2021 15:24:14 +0200 Subject: [PATCH 26/78] fix(script): exclude non-edge apps in relup dependency --- scripts/inject-deps.escript | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/scripts/inject-deps.escript b/scripts/inject-deps.escript index e5cd47140..0b6371342 100755 --- a/scripts/inject-deps.escript +++ b/scripts/inject-deps.escript @@ -23,6 +23,29 @@ usage() -> "Usage: " ++ escript:script_name() ++ " emqx|emqx-edge". -type app() :: atom(). +-type deps_overlay() :: {re, string()} | app(). + +%% deps/0 returns the dependency overlays. +%% {re, Pattern} to match application names using regexp pattern +-spec deps(string()) -> [{app(), [deps_overlay()]}]. +deps("emqx-edge" ++ _) -> + %% special case for edge + base_deps() ++ [{{re, ".+"}, [{exclude, App} || App <- edge_excludes()]}]; +deps(_Profile) -> + base_deps(). + +edge_excludes() -> + [ emqx_lwm2m + , emqx_auth_ldap + , emqx_auth_pgsql + , emqx_auth_redis + , emqx_auth_mongo + , emqx_lua_hook + , emqx_exhook + , emqx_exproto + , emqx_prometheus + , emqx_psk_file + ]. base_deps() -> %% make sure emqx_dashboard depends on all other emqx_xxx apps @@ -64,7 +87,7 @@ expand_deps([Dep | Deps], AppNames, Acc) -> inject(Profile) -> LibDir = lib_dir(Profile), AppNames = list_apps(LibDir), - Deps0 = lists:flatmap(fun(Dep) -> expand_names(Dep, AppNames) end, base_deps()), + Deps0 = lists:flatmap(fun(Dep) -> expand_names(Dep, AppNames) end, deps(Profile)), Deps1 = merge(Deps0, []), Deps2 = lists:map(fun({Name, DepsX}) -> NewDeps = expand_deps(DepsX, AppNames, []), From d7da6b2379c167af9b13a75bb81d851b1f716650 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Wed, 23 Jun 2021 21:08:39 +0800 Subject: [PATCH 27/78] chore(CI): delete needless link when build packages --- .github/workflows/build_packages.yaml | 8 +++++--- deploy/docker/Dockerfile | 4 +++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 4b23affd8..b983eaa67 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -169,9 +169,11 @@ jobs: - name: build run: | . $HOME/.kerl/${{ matrix.erl_otp }}/activate - make -C source ensure-rebar3 - sudo cp source/rebar3 /usr/local/bin/rebar3 - make -C source ${{ matrix.profile }}-zip + cd source + make ensure-rebar3 + sudo cp rebar3 /usr/local/bin/rebar3 + rm -rf _build/${{ matrix.profile }}/lib + make ${{ matrix.profile }}-zip - name: test run: | cd source diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index 4e25a001d..9e0ba9d5b 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -26,7 +26,9 @@ COPY . /emqx ARG PKG_VSN ARG EMQX_NAME=emqx -RUN cd /emqx && make $EMQX_NAME +RUN cd /emqx \ + && rm -rf _build/$EMQX_NAME/lib \ + && make $EMQX_NAME FROM $RUN_FROM From ce324a190bf3fe193d239f05527b36281c4a59f1 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Thu, 24 Jun 2021 11:42:46 +0800 Subject: [PATCH 28/78] docs(docker): fix env name error --- deploy/docker/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/docker/README.md b/deploy/docker/README.md index 5702b622c..c94794513 100644 --- a/deploy/docker/README.md +++ b/deploy/docker/README.md @@ -83,7 +83,7 @@ These environment variables will ignore for configuration file. The list is incomplete and may changed with [etc/emqx.conf](https://github.com/emqx/emqx/blob/master/etc/emqx.conf) and plugin configuration files. But the mapping rule is similar. -If set ``EMQX_NAME`` and ``EMQX_HOST``, and unset ``EMQX_NODE__NAME``, ``EMQX_NODE__NAME=$EMQX_NAME@$EMQX_HOST``. +If set ``EMQX_NAME`` and ``EMQX_HOST``, and unset ``EMQX_NODE_NAME``, ``EMQX_NODE_NAME=$EMQX_NAME@$EMQX_HOST``. For example, set mqtt tcp port to 1883 From 2f7373dd1cce6e0c739384bfc96b3c9c5f519d79 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Thu, 24 Jun 2021 14:54:05 +0800 Subject: [PATCH 29/78] chore(CI): update events that trigger workflows --- .github/workflows/apps_version_check.yaml | 5 ++++- .github/workflows/build_slim_packages.yaml | 2 ++ .github/workflows/check_deps_integrity.yaml | 5 ++++- .github/workflows/elvis_lint.yaml | 5 ++++- .github/workflows/run_cts_tests.yaml | 2 ++ .github/workflows/run_fvt_tests.yaml | 2 ++ .github/workflows/run_gitlint.yaml | 5 ++++- .github/workflows/run_test_cases.yaml | 2 ++ .github/workflows/shellcheck.yaml | 5 ++++- 9 files changed, 28 insertions(+), 5 deletions(-) diff --git a/.github/workflows/apps_version_check.yaml b/.github/workflows/apps_version_check.yaml index 4975e5c11..aac7c91d3 100644 --- a/.github/workflows/apps_version_check.yaml +++ b/.github/workflows/apps_version_check.yaml @@ -1,6 +1,9 @@ name: Check Apps Version -on: [pull_request] +on: + pull_request: + types: + - ready_for_review jobs: check_apps_version: diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index bf85578c5..aa101d056 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -6,6 +6,8 @@ on: - v* - e* pull_request: + types: + - ready_for_review workflow_dispatch: jobs: diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml index 0aa8f7903..3c72dcb0c 100644 --- a/.github/workflows/check_deps_integrity.yaml +++ b/.github/workflows/check_deps_integrity.yaml @@ -1,6 +1,9 @@ name: Check Rebar Dependencies -on: [pull_request] +on: + pull_request: + types: + - ready_for_review jobs: check_deps_integrity: diff --git a/.github/workflows/elvis_lint.yaml b/.github/workflows/elvis_lint.yaml index 1fdbeba87..6e95fe752 100644 --- a/.github/workflows/elvis_lint.yaml +++ b/.github/workflows/elvis_lint.yaml @@ -1,6 +1,9 @@ name: Elvis Linter -on: [pull_request] +on: + pull_request: + types: + - ready_for_review jobs: build: diff --git a/.github/workflows/run_cts_tests.yaml b/.github/workflows/run_cts_tests.yaml index bfa1d1394..3e33376ce 100644 --- a/.github/workflows/run_cts_tests.yaml +++ b/.github/workflows/run_cts_tests.yaml @@ -6,6 +6,8 @@ on: - v* - e* pull_request: + types: + - ready_for_review jobs: ldap: diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 035c0d0e3..44d6aac52 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -6,6 +6,8 @@ on: - v* - e* pull_request: + types: + - ready_for_review jobs: docker_test: diff --git a/.github/workflows/run_gitlint.yaml b/.github/workflows/run_gitlint.yaml index 01b35461f..0cefc4f13 100644 --- a/.github/workflows/run_gitlint.yaml +++ b/.github/workflows/run_gitlint.yaml @@ -1,6 +1,9 @@ name: Run gitlint -on: [pull_request] +on: + pull_request: + types: + - ready_for_review jobs: run_gitlint: diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 3fbe88c40..0ece640dc 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -6,6 +6,8 @@ on: - v* - e* pull_request: + types: + - ready_for_review jobs: run_static_analysis: diff --git a/.github/workflows/shellcheck.yaml b/.github/workflows/shellcheck.yaml index 2084f3863..3905074ce 100644 --- a/.github/workflows/shellcheck.yaml +++ b/.github/workflows/shellcheck.yaml @@ -1,6 +1,9 @@ name: Shellcheck -on: [pull_request] +on: + pull_request: + types: + - ready_for_review jobs: shellcheck: From 05150ce58bcd60b12000250665ee85ad2373056b Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Fri, 25 Jun 2021 19:22:36 +0800 Subject: [PATCH 30/78] revert: chore(CI): update events that trigger workflows This reverts commit 002cbb6d8bce61fbbf1b174284ac78e539598062 --- .github/workflows/apps_version_check.yaml | 5 +---- .github/workflows/build_slim_packages.yaml | 2 -- .github/workflows/check_deps_integrity.yaml | 5 +---- .github/workflows/elvis_lint.yaml | 5 +---- .github/workflows/run_cts_tests.yaml | 2 -- .github/workflows/run_fvt_tests.yaml | 2 -- .github/workflows/run_gitlint.yaml | 5 +---- .github/workflows/run_test_cases.yaml | 2 -- .github/workflows/shellcheck.yaml | 5 +---- 9 files changed, 5 insertions(+), 28 deletions(-) diff --git a/.github/workflows/apps_version_check.yaml b/.github/workflows/apps_version_check.yaml index aac7c91d3..4975e5c11 100644 --- a/.github/workflows/apps_version_check.yaml +++ b/.github/workflows/apps_version_check.yaml @@ -1,9 +1,6 @@ name: Check Apps Version -on: - pull_request: - types: - - ready_for_review +on: [pull_request] jobs: check_apps_version: diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index aa101d056..bf85578c5 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -6,8 +6,6 @@ on: - v* - e* pull_request: - types: - - ready_for_review workflow_dispatch: jobs: diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml index 3c72dcb0c..0aa8f7903 100644 --- a/.github/workflows/check_deps_integrity.yaml +++ b/.github/workflows/check_deps_integrity.yaml @@ -1,9 +1,6 @@ name: Check Rebar Dependencies -on: - pull_request: - types: - - ready_for_review +on: [pull_request] jobs: check_deps_integrity: diff --git a/.github/workflows/elvis_lint.yaml b/.github/workflows/elvis_lint.yaml index 6e95fe752..1fdbeba87 100644 --- a/.github/workflows/elvis_lint.yaml +++ b/.github/workflows/elvis_lint.yaml @@ -1,9 +1,6 @@ name: Elvis Linter -on: - pull_request: - types: - - ready_for_review +on: [pull_request] jobs: build: diff --git a/.github/workflows/run_cts_tests.yaml b/.github/workflows/run_cts_tests.yaml index 3e33376ce..bfa1d1394 100644 --- a/.github/workflows/run_cts_tests.yaml +++ b/.github/workflows/run_cts_tests.yaml @@ -6,8 +6,6 @@ on: - v* - e* pull_request: - types: - - ready_for_review jobs: ldap: diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 44d6aac52..035c0d0e3 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -6,8 +6,6 @@ on: - v* - e* pull_request: - types: - - ready_for_review jobs: docker_test: diff --git a/.github/workflows/run_gitlint.yaml b/.github/workflows/run_gitlint.yaml index 0cefc4f13..01b35461f 100644 --- a/.github/workflows/run_gitlint.yaml +++ b/.github/workflows/run_gitlint.yaml @@ -1,9 +1,6 @@ name: Run gitlint -on: - pull_request: - types: - - ready_for_review +on: [pull_request] jobs: run_gitlint: diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 0ece640dc..3fbe88c40 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -6,8 +6,6 @@ on: - v* - e* pull_request: - types: - - ready_for_review jobs: run_static_analysis: diff --git a/.github/workflows/shellcheck.yaml b/.github/workflows/shellcheck.yaml index 3905074ce..2084f3863 100644 --- a/.github/workflows/shellcheck.yaml +++ b/.github/workflows/shellcheck.yaml @@ -1,9 +1,6 @@ name: Shellcheck -on: - pull_request: - types: - - ready_for_review +on: [pull_request] jobs: shellcheck: From 8f15a41f542fbf3d9c75c4ef56394144ba570e0e Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Fri, 25 Jun 2021 19:00:57 +0800 Subject: [PATCH 31/78] fix(shared_sub): failed to clean the emqx_shared_subscription tab A trick that fixes the issue that we demonitored the shared subscriber too early if it not unsubscribed all of the topics. --- apps/emqx/src/emqx_shared_sub.erl | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/apps/emqx/src/emqx_shared_sub.erl b/apps/emqx/src/emqx_shared_sub.erl index 4707f63db..865aeda18 100644 --- a/apps/emqx/src/emqx_shared_sub.erl +++ b/apps/emqx/src/emqx_shared_sub.erl @@ -336,9 +336,13 @@ handle_info({mnesia_table_event, {write, NewRecord, _}}, State = #state{pmon = P #emqx_shared_subscription{subpid = SubPid} = NewRecord, {noreply, update_stats(State#state{pmon = emqx_pmon:monitor(SubPid, PMon)})}; -handle_info({mnesia_table_event, {delete_object, OldRecord, _}}, State = #state{pmon = PMon}) -> - #emqx_shared_subscription{subpid = SubPid} = OldRecord, - {noreply, update_stats(State#state{pmon = emqx_pmon:demonitor(SubPid, PMon)})}; +%% The subscriber may have subscribed multiple topics, so we need to keep monitoring the PID until +%% it `unsubscribed` the last topic. +%% The trick is we don't demonitor the subscriber here, and (after a long time) it will eventually +%% be disconnected. +% handle_info({mnesia_table_event, {delete_object, OldRecord, _}}, State = #state{pmon = PMon}) -> +% #emqx_shared_subscription{subpid = SubPid} = OldRecord, +% {noreply, update_stats(State#state{pmon = emqx_pmon:demonitor(SubPid, PMon)})}; handle_info({mnesia_table_event, _Event}, State) -> {noreply, State}; From 95e7a4dd7bbe248d7ef3a2895083adbc65a6002a Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Fri, 25 Jun 2021 19:20:06 +0800 Subject: [PATCH 32/78] chore(appup): update the appup for 4.3.5 --- apps/emqx/src/emqx.appup.src | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx.appup.src b/apps/emqx/src/emqx.appup.src index 0b2b9b2d5..4f9f00673 100644 --- a/apps/emqx/src/emqx.appup.src +++ b/apps/emqx/src/emqx.appup.src @@ -1,14 +1,20 @@ %% -*- mode: erlang -*- {VSN, [ + {"4.3.4", + [{load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]} + ]}, {"4.3.3", [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]} ]}, {"4.3.2", [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, @@ -18,6 +24,7 @@ ]}, {"4.3.1", [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, @@ -31,6 +38,7 @@ {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}]}, {"4.3.0", [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_logger_jsonfmt,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_congestion,brutal_purge,soft_purge,[]}, @@ -47,14 +55,21 @@ {apply,{emqx_metrics,upgrade_retained_delayed_counter_type,[]}}, {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.3", + [ + {"4.3.4", + [{load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]} + ]}, + {"4.3.3", [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]} ]}, {"4.3.2", [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, @@ -64,6 +79,7 @@ ]}, {"4.3.1", [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, @@ -77,6 +93,7 @@ {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}]}, {"4.3.0", [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_logger_jsonfmt,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}, From ff54fbd1d120a828825ad6f8b3776fbf618d510d Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Mon, 28 Jun 2021 09:35:39 +0800 Subject: [PATCH 33/78] fix(shared_sub): discard all unexpected msgs --- apps/emqx/src/emqx_shared_sub.erl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/emqx/src/emqx_shared_sub.erl b/apps/emqx/src/emqx_shared_sub.erl index 865aeda18..97aa778f3 100644 --- a/apps/emqx/src/emqx_shared_sub.erl +++ b/apps/emqx/src/emqx_shared_sub.erl @@ -352,8 +352,7 @@ handle_info({'DOWN', _MRef, process, SubPid, _Reason}, State = #state{pmon = PMo cleanup_down(SubPid), {noreply, update_stats(State#state{pmon = emqx_pmon:erase(SubPid, PMon)})}; -handle_info(Info, State) -> - ?LOG(error, "Unexpected info: ~p", [Info]), +handle_info(_Info, State) -> {noreply, State}. terminate(_Reason, _State) -> From 07ade268f98e4886f15ed5891c02b2e8c0764b23 Mon Sep 17 00:00:00 2001 From: Turtle Date: Thu, 17 Jun 2021 15:11:26 +0800 Subject: [PATCH 34/78] chore: mv emqx_connector to emqx_data_bridge --- apps/emqx_connector/.gitignore | 19 ---------- apps/emqx_connector/README.md | 27 -------------- apps/emqx_connector/etc/emqx_connector.conf | 4 --- .../emqx_connector/priv/emqx_connector.schema | 2 -- apps/emqx_connector/rebar.config | 13 ------- .../emqx_connector/src/emqx_connector.app.src | 17 --------- apps/emqx_connector/src/emqx_connector.erl | 16 --------- .../emqx_connector/src/emqx_connector_app.erl | 31 ---------------- .../emqx_connector/src/emqx_connector_sup.erl | 36 ------------------- .../include/emqx_data_bridge.hrl} | 0 .../priv/emqx_data_bridge.schema | 16 --------- apps/emqx_data_bridge/rebar.config | 1 + .../src/connector}/emqx_connector_ldap.erl | 2 +- .../src/connector}/emqx_connector_mongo.erl | 2 +- .../src/connector}/emqx_connector_mysql.erl | 0 .../src/connector}/emqx_connector_pgsql.erl | 0 .../src/connector}/emqx_connector_redis.erl | 2 +- .../connector}/emqx_connector_schema_lib.erl | 2 +- .../src/emqx_data_bridge.app.src | 3 +- .../emqx_data_bridge/src/emqx_data_bridge.erl | 5 +-- apps/emqx_resource/priv/emqx_resource.schema | 2 -- data/loaded_plugins.tmpl | 3 -- rebar.config.erl | 9 ++--- 23 files changed, 13 insertions(+), 199 deletions(-) delete mode 100644 apps/emqx_connector/.gitignore delete mode 100644 apps/emqx_connector/README.md delete mode 100644 apps/emqx_connector/etc/emqx_connector.conf delete mode 100644 apps/emqx_connector/priv/emqx_connector.schema delete mode 100644 apps/emqx_connector/rebar.config delete mode 100644 apps/emqx_connector/src/emqx_connector.app.src delete mode 100644 apps/emqx_connector/src/emqx_connector.erl delete mode 100644 apps/emqx_connector/src/emqx_connector_app.erl delete mode 100644 apps/emqx_connector/src/emqx_connector_sup.erl rename apps/{emqx_connector/include/emqx_connector.hrl => emqx_data_bridge/include/emqx_data_bridge.hrl} (100%) delete mode 100644 apps/emqx_data_bridge/priv/emqx_data_bridge.schema rename apps/{emqx_connector/src => emqx_data_bridge/src/connector}/emqx_connector_ldap.erl (99%) rename apps/{emqx_connector/src => emqx_data_bridge/src/connector}/emqx_connector_mongo.erl (99%) rename apps/{emqx_connector/src => emqx_data_bridge/src/connector}/emqx_connector_mysql.erl (100%) rename apps/{emqx_connector/src => emqx_data_bridge/src/connector}/emqx_connector_pgsql.erl (100%) rename apps/{emqx_connector/src => emqx_data_bridge/src/connector}/emqx_connector_redis.erl (99%) rename apps/{emqx_connector/src => emqx_data_bridge/src/connector}/emqx_connector_schema_lib.erl (99%) delete mode 100644 apps/emqx_resource/priv/emqx_resource.schema diff --git a/apps/emqx_connector/.gitignore b/apps/emqx_connector/.gitignore deleted file mode 100644 index f1c455451..000000000 --- a/apps/emqx_connector/.gitignore +++ /dev/null @@ -1,19 +0,0 @@ -.rebar3 -_* -.eunit -*.o -*.beam -*.plt -*.swp -*.swo -.erlang.cookie -ebin -log -erl_crash.dump -.rebar -logs -_build -.idea -*.iml -rebar3.crashdump -*~ diff --git a/apps/emqx_connector/README.md b/apps/emqx_connector/README.md deleted file mode 100644 index 879669f93..000000000 --- a/apps/emqx_connector/README.md +++ /dev/null @@ -1,27 +0,0 @@ -# emqx_connector - -This application is a collection of `connectors`. - -A `connector` is a callback module of `emqx_resource` that maintains the data related to -external resources. Put all resource related callback modules in a single application is good as -we can put some util functions/modules here for reusing purpose. - -For example, a mysql connector is an emqx resource that maintains all the mysql connection -related parameters (configs) and the TCP connections to the mysql server. - -An mysql connector can be used as following: - -``` -(emqx@127.0.0.1)5> emqx_resource:list_instances_verbose(). -[#{config => - #{auto_reconnect => true,cacertfile => [],certfile => [], - database => "mqtt",keyfile => [],password => "public", - pool_size => 1, - server => {{127,0,0,1},3306}, - ssl => false,user => "root",verify => false}, - id => <<"mysql-abc">>,mod => emqx_connector_mysql, - state => #{poolname => 'mysql-abc'}, - status => started}] -(emqx@127.0.0.1)6> emqx_resource:query(<<"mysql-abc">>, {sql, <<"SELECT count(1)">>}). -{ok,[<<"count(1)">>],[[1]]} -``` diff --git a/apps/emqx_connector/etc/emqx_connector.conf b/apps/emqx_connector/etc/emqx_connector.conf deleted file mode 100644 index db4402d47..000000000 --- a/apps/emqx_connector/etc/emqx_connector.conf +++ /dev/null @@ -1,4 +0,0 @@ -##-------------------------------------------------------------------- -## EMQ X CONNECTOR Plugin -##-------------------------------------------------------------------- - diff --git a/apps/emqx_connector/priv/emqx_connector.schema b/apps/emqx_connector/priv/emqx_connector.schema deleted file mode 100644 index b8476c4d9..000000000 --- a/apps/emqx_connector/priv/emqx_connector.schema +++ /dev/null @@ -1,2 +0,0 @@ -%%-*- mode: erlang -*- -%% emqx_connector config mapping diff --git a/apps/emqx_connector/rebar.config b/apps/emqx_connector/rebar.config deleted file mode 100644 index 53b5c63e8..000000000 --- a/apps/emqx_connector/rebar.config +++ /dev/null @@ -1,13 +0,0 @@ -{erl_opts, [ - nowarn_unused_import, - debug_info -]}. - -{deps, [ - {mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.1"}}} -]}. - -{shell, [ - % {config, "config/sys.config"}, - {apps, [emqx_connector]} -]}. diff --git a/apps/emqx_connector/src/emqx_connector.app.src b/apps/emqx_connector/src/emqx_connector.app.src deleted file mode 100644 index a821b8f13..000000000 --- a/apps/emqx_connector/src/emqx_connector.app.src +++ /dev/null @@ -1,17 +0,0 @@ -{application, emqx_connector, - [{description, "An OTP application"}, - {vsn, "0.1.0"}, - {registered, []}, - {mod, {emqx_connector_app, []}}, - {applications, - [kernel, - stdlib, - emqx_resource, - ecpool - ]}, - {env,[]}, - {modules, []}, - - {licenses, ["Apache 2.0"]}, - {links, []} - ]}. diff --git a/apps/emqx_connector/src/emqx_connector.erl b/apps/emqx_connector/src/emqx_connector.erl deleted file mode 100644 index dd0359348..000000000 --- a/apps/emqx_connector/src/emqx_connector.erl +++ /dev/null @@ -1,16 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- --module(emqx_connector). diff --git a/apps/emqx_connector/src/emqx_connector_app.erl b/apps/emqx_connector/src/emqx_connector_app.erl deleted file mode 100644 index 4bbad75cf..000000000 --- a/apps/emqx_connector/src/emqx_connector_app.erl +++ /dev/null @@ -1,31 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_connector_app). - --behaviour(application). - --emqx_plugin(?MODULE). - --export([start/2, stop/1]). - -start(_StartType, _StartArgs) -> - emqx_connector_sup:start_link(). - -stop(_State) -> - ok. - -%% internal functions diff --git a/apps/emqx_connector/src/emqx_connector_sup.erl b/apps/emqx_connector/src/emqx_connector_sup.erl deleted file mode 100644 index 603b9a8ad..000000000 --- a/apps/emqx_connector/src/emqx_connector_sup.erl +++ /dev/null @@ -1,36 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- --module(emqx_connector_sup). - --behaviour(supervisor). - --export([start_link/0]). - --export([init/1]). - --define(SERVER, ?MODULE). - -start_link() -> - supervisor:start_link({local, ?SERVER}, ?MODULE, []). - -init([]) -> - SupFlags = #{strategy => one_for_all, - intensity => 0, - period => 1}, - ChildSpecs = [], - {ok, {SupFlags, ChildSpecs}}. - -%% internal functions diff --git a/apps/emqx_connector/include/emqx_connector.hrl b/apps/emqx_data_bridge/include/emqx_data_bridge.hrl similarity index 100% rename from apps/emqx_connector/include/emqx_connector.hrl rename to apps/emqx_data_bridge/include/emqx_data_bridge.hrl diff --git a/apps/emqx_data_bridge/priv/emqx_data_bridge.schema b/apps/emqx_data_bridge/priv/emqx_data_bridge.schema deleted file mode 100644 index c9cb3f2c0..000000000 --- a/apps/emqx_data_bridge/priv/emqx_data_bridge.schema +++ /dev/null @@ -1,16 +0,0 @@ -%%-*- mode: erlang -*- -%% emqx_data_bridge config mapping - -{mapping, "emqx_data_bridge.bridges", "emqx_data_bridge.bridges", [ - {default, []}, - {datatype, string} -]}. - -% fields("emqx_data_bridge") -> -% [ -% {bridges, -% [fun(mapping) -> "emqx_data_bridge.bridges"; -% (type) -> list(); -% (_) -> undefined -% end]} -% ] \ No newline at end of file diff --git a/apps/emqx_data_bridge/rebar.config b/apps/emqx_data_bridge/rebar.config index cf4cfcf1b..2778ec974 100644 --- a/apps/emqx_data_bridge/rebar.config +++ b/apps/emqx_data_bridge/rebar.config @@ -1,5 +1,6 @@ {erl_opts, [debug_info]}. {deps, []}. +% {extra_src_dirs, [{"src", [{recursive, true}]}]}. {shell, [ % {config, "config/sys.config"}, diff --git a/apps/emqx_connector/src/emqx_connector_ldap.erl b/apps/emqx_data_bridge/src/connector/emqx_connector_ldap.erl similarity index 99% rename from apps/emqx_connector/src/emqx_connector_ldap.erl rename to apps/emqx_data_bridge/src/connector/emqx_connector_ldap.erl index 3b0af5aa7..322bce1ef 100644 --- a/apps/emqx_connector/src/emqx_connector_ldap.erl +++ b/apps/emqx_data_bridge/src/connector/emqx_connector_ldap.erl @@ -15,7 +15,7 @@ %%-------------------------------------------------------------------- -module(emqx_connector_ldap). --include("emqx_connector.hrl"). +-include("emqx_data_bridge.hrl"). -include_lib("typerefl/include/types.hrl"). -include_lib("emqx_resource/include/emqx_resource_behaviour.hrl"). diff --git a/apps/emqx_connector/src/emqx_connector_mongo.erl b/apps/emqx_data_bridge/src/connector/emqx_connector_mongo.erl similarity index 99% rename from apps/emqx_connector/src/emqx_connector_mongo.erl rename to apps/emqx_data_bridge/src/connector/emqx_connector_mongo.erl index 26a36dd0a..9e22e5ac5 100644 --- a/apps/emqx_connector/src/emqx_connector_mongo.erl +++ b/apps/emqx_data_bridge/src/connector/emqx_connector_mongo.erl @@ -15,7 +15,7 @@ %%-------------------------------------------------------------------- -module(emqx_connector_mongo). --include("emqx_connector.hrl"). +-include("emqx_data_bridge.hrl"). -include_lib("typerefl/include/types.hrl"). -include_lib("emqx_resource/include/emqx_resource_behaviour.hrl"). diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_data_bridge/src/connector/emqx_connector_mysql.erl similarity index 100% rename from apps/emqx_connector/src/emqx_connector_mysql.erl rename to apps/emqx_data_bridge/src/connector/emqx_connector_mysql.erl diff --git a/apps/emqx_connector/src/emqx_connector_pgsql.erl b/apps/emqx_data_bridge/src/connector/emqx_connector_pgsql.erl similarity index 100% rename from apps/emqx_connector/src/emqx_connector_pgsql.erl rename to apps/emqx_data_bridge/src/connector/emqx_connector_pgsql.erl diff --git a/apps/emqx_connector/src/emqx_connector_redis.erl b/apps/emqx_data_bridge/src/connector/emqx_connector_redis.erl similarity index 99% rename from apps/emqx_connector/src/emqx_connector_redis.erl rename to apps/emqx_data_bridge/src/connector/emqx_connector_redis.erl index 3eebbcf87..1037afede 100644 --- a/apps/emqx_connector/src/emqx_connector_redis.erl +++ b/apps/emqx_data_bridge/src/connector/emqx_connector_redis.erl @@ -15,7 +15,7 @@ %%-------------------------------------------------------------------- -module(emqx_connector_redis). --include("emqx_connector.hrl"). +-include("emqx_data_bridge.hrl"). -include_lib("typerefl/include/types.hrl"). -include_lib("emqx_resource/include/emqx_resource_behaviour.hrl"). diff --git a/apps/emqx_connector/src/emqx_connector_schema_lib.erl b/apps/emqx_data_bridge/src/connector/emqx_connector_schema_lib.erl similarity index 99% rename from apps/emqx_connector/src/emqx_connector_schema_lib.erl rename to apps/emqx_data_bridge/src/connector/emqx_connector_schema_lib.erl index 03572d91d..aaacd0e80 100644 --- a/apps/emqx_connector/src/emqx_connector_schema_lib.erl +++ b/apps/emqx_data_bridge/src/connector/emqx_connector_schema_lib.erl @@ -15,7 +15,7 @@ %%-------------------------------------------------------------------- -module(emqx_connector_schema_lib). --include("emqx_connector.hrl"). +-include("emqx_data_bridge.hrl"). -include_lib("typerefl/include/types.hrl"). -export([ relational_db_fields/0 diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge.app.src b/apps/emqx_data_bridge/src/emqx_data_bridge.app.src index 360511d9b..edfcd4a9d 100644 --- a/apps/emqx_data_bridge/src/emqx_data_bridge.app.src +++ b/apps/emqx_data_bridge/src/emqx_data_bridge.app.src @@ -5,7 +5,8 @@ {mod, {emqx_data_bridge_app, []}}, {applications, [kernel, - stdlib + stdlib, + ecpool ]}, {env,[]}, {modules, []}, diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge.erl b/apps/emqx_data_bridge/src/emqx_data_bridge.erl index 5877cd3dd..54a228763 100644 --- a/apps/emqx_data_bridge/src/emqx_data_bridge.erl +++ b/apps/emqx_data_bridge/src/emqx_data_bridge.erl @@ -25,8 +25,9 @@ ]). load_bridges() -> - Bridges = proplists:get_value(bridges, - application:get_all_env(emqx_data_bridge), []), + ConfFile = filename:join([emqx:get_env(plugins_etc_dir), ?MODULE]) ++ ".conf", + {ok, #{<<"emqx_data_bridge">> := RawConfig}} = hocon:load(ConfFile), + Bridges = maps:get(<<"bridges">>, RawConfig, []), emqx_data_bridge_monitor:ensure_all_started(Bridges). resource_type(<<"mysql">>) -> emqx_connector_mysql; diff --git a/apps/emqx_resource/priv/emqx_resource.schema b/apps/emqx_resource/priv/emqx_resource.schema deleted file mode 100644 index 8246dc6a7..000000000 --- a/apps/emqx_resource/priv/emqx_resource.schema +++ /dev/null @@ -1,2 +0,0 @@ -%%-*- mode: erlang -*- -%% emqx-resource config mapping diff --git a/data/loaded_plugins.tmpl b/data/loaded_plugins.tmpl index 5ac46e0e3..d0dac7fe1 100644 --- a/data/loaded_plugins.tmpl +++ b/data/loaded_plugins.tmpl @@ -5,7 +5,4 @@ {emqx_retainer, {{enable_plugin_emqx_retainer}}}. {emqx_telemetry, {{enable_plugin_emqx_telemetry}}}. {emqx_rule_engine, {{enable_plugin_emqx_rule_engine}}}. -{emqx_resource, {{enable_plugin_emqx_resource}}}. -{emqx_connector, {{enable_plugin_emqx_connector}}}. -{emqx_data_bridge, {{enable_plugin_emqx_data_bridge}}}. {emqx_bridge_mqtt, {{enable_plugin_emqx_bridge_mqtt}}}. diff --git a/rebar.config.erl b/rebar.config.erl index 543333895..24020ad45 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -193,9 +193,6 @@ overlay_vars_rel(RelType) -> end, [ {enable_plugin_emqx_rule_engine, RelType =:= cloud} , {enable_plugin_emqx_bridge_mqtt, RelType =:= edge} - , {enable_plugin_emqx_resource, true} - , {enable_plugin_emqx_connector, true} - , {enable_plugin_emqx_data_bridge, true} , {enable_plugin_emqx_modules, false} %% modules is not a plugin in ce , {enable_plugin_emqx_recon, true} , {enable_plugin_emqx_retainer, true} @@ -254,6 +251,8 @@ relx_apps(ReleaseType) -> , {emqx_plugin_libs, load} , observer_cli , emqx_http_lib + , emqx_resource + , emqx_data_bridge ] ++ [emqx_modules || not is_enterprise()] ++ [emqx_license || is_enterprise()] @@ -291,10 +290,8 @@ relx_plugin_apps(ReleaseType) -> , emqx_auth_mnesia , emqx_web_hook , emqx_recon - , emqx_resource - , emqx_connector - , emqx_data_bridge , emqx_rule_engine + , emqx_data_bridge , emqx_sasl ] ++ [emqx_telemetry || not is_enterprise()] From 786fd6fe3e4a7692935d6529a7a20e7db9ba5805 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Thu, 17 Jun 2021 18:05:43 +0800 Subject: [PATCH 35/78] feat(connector): mysql and pgsql query support params --- .../src/connector/emqx_connector_mysql.erl | 4 +++- .../src/connector/emqx_connector_pgsql.erl | 10 ++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/apps/emqx_data_bridge/src/connector/emqx_connector_mysql.erl b/apps/emqx_data_bridge/src/connector/emqx_connector_mysql.erl index e90d5f171..61e13901e 100644 --- a/apps/emqx_data_bridge/src/connector/emqx_connector_mysql.erl +++ b/apps/emqx_data_bridge/src/connector/emqx_connector_mysql.erl @@ -71,8 +71,10 @@ on_stop(InstId, #{poolname := PoolName}) -> emqx_plugin_libs_pool:stop_pool(PoolName). on_query(InstId, {sql, SQL}, AfterQuery, #{poolname := PoolName} = State) -> + on_query(InstId, {sql, SQL, []}, AfterQuery, #{poolname := PoolName} = State); +on_query(InstId, {sql, SQL, Params}, AfterQuery, #{poolname := PoolName} = State) -> logger:debug("mysql connector ~p received sql query: ~p, at state: ~p", [InstId, SQL, State]), - case Result = ecpool:pick_and_do(PoolName, {mysql, query, [SQL]}, no_handover) of + case Result = ecpool:pick_and_do(PoolName, {mysql, query, [SQL, Params]}, no_handover) of {error, Reason} -> logger:debug("mysql connector ~p do sql query failed, sql: ~p, reason: ~p", [InstId, SQL, Reason]), emqx_resource:query_failed(AfterQuery); diff --git a/apps/emqx_data_bridge/src/connector/emqx_connector_pgsql.erl b/apps/emqx_data_bridge/src/connector/emqx_connector_pgsql.erl index 329138ab1..0ff0d6779 100644 --- a/apps/emqx_data_bridge/src/connector/emqx_connector_pgsql.erl +++ b/apps/emqx_data_bridge/src/connector/emqx_connector_pgsql.erl @@ -31,7 +31,7 @@ -export([connect/1]). --export([query/2]). +-export([query/3]). -export([do_health_check/1]). @@ -74,8 +74,10 @@ on_stop(InstId, #{poolname := PoolName}) -> emqx_plugin_libs_pool:stop_pool(PoolName). on_query(InstId, {sql, SQL}, AfterQuery, #{poolname := PoolName} = State) -> + on_query(InstId, {sql, SQL, []}, AfterQuery, #{poolname := PoolName} = State); +on_query(InstId, {sql, SQL, Params}, AfterQuery, #{poolname := PoolName} = State) -> logger:debug("postgresql connector ~p received sql query: ~p, at state: ~p", [InstId, SQL, State]), - case Result = ecpool:pick_and_do(PoolName, {?MODULE, query, [SQL]}, no_handover) of + case Result = ecpool:pick_and_do(PoolName, {?MODULE, query, [SQL, Params]}, no_handover) of {error, Reason} -> logger:debug("postgresql connector ~p do sql query failed, sql: ~p, reason: ~p", [InstId, SQL, Reason]), emqx_resource:query_failed(AfterQuery); @@ -100,8 +102,8 @@ connect(Opts) -> Password = proplists:get_value(password, Opts), epgsql:connect(Host, Username, Password, conn_opts(Opts)). -query(Conn, SQL) -> - epgsql:squery(Conn, SQL). +query(Conn, SQL, Params) -> + epgsql:equery(Conn, SQL, Params). conn_opts(Opts) -> conn_opts(Opts, []). From 271869b817f891eb7755be223f4c3c6b57e0abe7 Mon Sep 17 00:00:00 2001 From: turtleDeng Date: Thu, 17 Jun 2021 19:10:38 +0800 Subject: [PATCH 36/78] Revert "chore: mv emqx_connector to emqx_data_bridge" This reverts commit d640e2ccfa68a1c0529ef302b3eadc627d07f5e5. --- apps/emqx_connector/.gitignore | 19 ++++++++++ apps/emqx_connector/README.md | 27 ++++++++++++++ apps/emqx_connector/etc/emqx_connector.conf | 4 +++ .../include/emqx_connector.hrl} | 0 .../emqx_connector/priv/emqx_connector.schema | 2 ++ apps/emqx_connector/rebar.config | 13 +++++++ .../emqx_connector/src/emqx_connector.app.src | 17 +++++++++ apps/emqx_connector/src/emqx_connector.erl | 16 +++++++++ .../emqx_connector/src/emqx_connector_app.erl | 31 ++++++++++++++++ .../src}/emqx_connector_ldap.erl | 2 +- .../src}/emqx_connector_mongo.erl | 2 +- .../src}/emqx_connector_mysql.erl | 0 .../src}/emqx_connector_pgsql.erl | 0 .../src}/emqx_connector_redis.erl | 2 +- .../src}/emqx_connector_schema_lib.erl | 2 +- .../emqx_connector/src/emqx_connector_sup.erl | 36 +++++++++++++++++++ .../priv/emqx_data_bridge.schema | 16 +++++++++ apps/emqx_data_bridge/rebar.config | 1 - .../src/emqx_data_bridge.app.src | 3 +- .../emqx_data_bridge/src/emqx_data_bridge.erl | 5 ++- apps/emqx_resource/priv/emqx_resource.schema | 2 ++ data/loaded_plugins.tmpl | 3 ++ rebar.config.erl | 9 +++-- 23 files changed, 199 insertions(+), 13 deletions(-) create mode 100644 apps/emqx_connector/.gitignore create mode 100644 apps/emqx_connector/README.md create mode 100644 apps/emqx_connector/etc/emqx_connector.conf rename apps/{emqx_data_bridge/include/emqx_data_bridge.hrl => emqx_connector/include/emqx_connector.hrl} (100%) create mode 100644 apps/emqx_connector/priv/emqx_connector.schema create mode 100644 apps/emqx_connector/rebar.config create mode 100644 apps/emqx_connector/src/emqx_connector.app.src create mode 100644 apps/emqx_connector/src/emqx_connector.erl create mode 100644 apps/emqx_connector/src/emqx_connector_app.erl rename apps/{emqx_data_bridge/src/connector => emqx_connector/src}/emqx_connector_ldap.erl (99%) rename apps/{emqx_data_bridge/src/connector => emqx_connector/src}/emqx_connector_mongo.erl (99%) rename apps/{emqx_data_bridge/src/connector => emqx_connector/src}/emqx_connector_mysql.erl (100%) rename apps/{emqx_data_bridge/src/connector => emqx_connector/src}/emqx_connector_pgsql.erl (100%) rename apps/{emqx_data_bridge/src/connector => emqx_connector/src}/emqx_connector_redis.erl (99%) rename apps/{emqx_data_bridge/src/connector => emqx_connector/src}/emqx_connector_schema_lib.erl (99%) create mode 100644 apps/emqx_connector/src/emqx_connector_sup.erl create mode 100644 apps/emqx_data_bridge/priv/emqx_data_bridge.schema create mode 100644 apps/emqx_resource/priv/emqx_resource.schema diff --git a/apps/emqx_connector/.gitignore b/apps/emqx_connector/.gitignore new file mode 100644 index 000000000..f1c455451 --- /dev/null +++ b/apps/emqx_connector/.gitignore @@ -0,0 +1,19 @@ +.rebar3 +_* +.eunit +*.o +*.beam +*.plt +*.swp +*.swo +.erlang.cookie +ebin +log +erl_crash.dump +.rebar +logs +_build +.idea +*.iml +rebar3.crashdump +*~ diff --git a/apps/emqx_connector/README.md b/apps/emqx_connector/README.md new file mode 100644 index 000000000..879669f93 --- /dev/null +++ b/apps/emqx_connector/README.md @@ -0,0 +1,27 @@ +# emqx_connector + +This application is a collection of `connectors`. + +A `connector` is a callback module of `emqx_resource` that maintains the data related to +external resources. Put all resource related callback modules in a single application is good as +we can put some util functions/modules here for reusing purpose. + +For example, a mysql connector is an emqx resource that maintains all the mysql connection +related parameters (configs) and the TCP connections to the mysql server. + +An mysql connector can be used as following: + +``` +(emqx@127.0.0.1)5> emqx_resource:list_instances_verbose(). +[#{config => + #{auto_reconnect => true,cacertfile => [],certfile => [], + database => "mqtt",keyfile => [],password => "public", + pool_size => 1, + server => {{127,0,0,1},3306}, + ssl => false,user => "root",verify => false}, + id => <<"mysql-abc">>,mod => emqx_connector_mysql, + state => #{poolname => 'mysql-abc'}, + status => started}] +(emqx@127.0.0.1)6> emqx_resource:query(<<"mysql-abc">>, {sql, <<"SELECT count(1)">>}). +{ok,[<<"count(1)">>],[[1]]} +``` diff --git a/apps/emqx_connector/etc/emqx_connector.conf b/apps/emqx_connector/etc/emqx_connector.conf new file mode 100644 index 000000000..db4402d47 --- /dev/null +++ b/apps/emqx_connector/etc/emqx_connector.conf @@ -0,0 +1,4 @@ +##-------------------------------------------------------------------- +## EMQ X CONNECTOR Plugin +##-------------------------------------------------------------------- + diff --git a/apps/emqx_data_bridge/include/emqx_data_bridge.hrl b/apps/emqx_connector/include/emqx_connector.hrl similarity index 100% rename from apps/emqx_data_bridge/include/emqx_data_bridge.hrl rename to apps/emqx_connector/include/emqx_connector.hrl diff --git a/apps/emqx_connector/priv/emqx_connector.schema b/apps/emqx_connector/priv/emqx_connector.schema new file mode 100644 index 000000000..b8476c4d9 --- /dev/null +++ b/apps/emqx_connector/priv/emqx_connector.schema @@ -0,0 +1,2 @@ +%%-*- mode: erlang -*- +%% emqx_connector config mapping diff --git a/apps/emqx_connector/rebar.config b/apps/emqx_connector/rebar.config new file mode 100644 index 000000000..53b5c63e8 --- /dev/null +++ b/apps/emqx_connector/rebar.config @@ -0,0 +1,13 @@ +{erl_opts, [ + nowarn_unused_import, + debug_info +]}. + +{deps, [ + {mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.1"}}} +]}. + +{shell, [ + % {config, "config/sys.config"}, + {apps, [emqx_connector]} +]}. diff --git a/apps/emqx_connector/src/emqx_connector.app.src b/apps/emqx_connector/src/emqx_connector.app.src new file mode 100644 index 000000000..a821b8f13 --- /dev/null +++ b/apps/emqx_connector/src/emqx_connector.app.src @@ -0,0 +1,17 @@ +{application, emqx_connector, + [{description, "An OTP application"}, + {vsn, "0.1.0"}, + {registered, []}, + {mod, {emqx_connector_app, []}}, + {applications, + [kernel, + stdlib, + emqx_resource, + ecpool + ]}, + {env,[]}, + {modules, []}, + + {licenses, ["Apache 2.0"]}, + {links, []} + ]}. diff --git a/apps/emqx_connector/src/emqx_connector.erl b/apps/emqx_connector/src/emqx_connector.erl new file mode 100644 index 000000000..dd0359348 --- /dev/null +++ b/apps/emqx_connector/src/emqx_connector.erl @@ -0,0 +1,16 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_connector). diff --git a/apps/emqx_connector/src/emqx_connector_app.erl b/apps/emqx_connector/src/emqx_connector_app.erl new file mode 100644 index 000000000..4bbad75cf --- /dev/null +++ b/apps/emqx_connector/src/emqx_connector_app.erl @@ -0,0 +1,31 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_connector_app). + +-behaviour(application). + +-emqx_plugin(?MODULE). + +-export([start/2, stop/1]). + +start(_StartType, _StartArgs) -> + emqx_connector_sup:start_link(). + +stop(_State) -> + ok. + +%% internal functions diff --git a/apps/emqx_data_bridge/src/connector/emqx_connector_ldap.erl b/apps/emqx_connector/src/emqx_connector_ldap.erl similarity index 99% rename from apps/emqx_data_bridge/src/connector/emqx_connector_ldap.erl rename to apps/emqx_connector/src/emqx_connector_ldap.erl index 322bce1ef..3b0af5aa7 100644 --- a/apps/emqx_data_bridge/src/connector/emqx_connector_ldap.erl +++ b/apps/emqx_connector/src/emqx_connector_ldap.erl @@ -15,7 +15,7 @@ %%-------------------------------------------------------------------- -module(emqx_connector_ldap). --include("emqx_data_bridge.hrl"). +-include("emqx_connector.hrl"). -include_lib("typerefl/include/types.hrl"). -include_lib("emqx_resource/include/emqx_resource_behaviour.hrl"). diff --git a/apps/emqx_data_bridge/src/connector/emqx_connector_mongo.erl b/apps/emqx_connector/src/emqx_connector_mongo.erl similarity index 99% rename from apps/emqx_data_bridge/src/connector/emqx_connector_mongo.erl rename to apps/emqx_connector/src/emqx_connector_mongo.erl index 9e22e5ac5..26a36dd0a 100644 --- a/apps/emqx_data_bridge/src/connector/emqx_connector_mongo.erl +++ b/apps/emqx_connector/src/emqx_connector_mongo.erl @@ -15,7 +15,7 @@ %%-------------------------------------------------------------------- -module(emqx_connector_mongo). --include("emqx_data_bridge.hrl"). +-include("emqx_connector.hrl"). -include_lib("typerefl/include/types.hrl"). -include_lib("emqx_resource/include/emqx_resource_behaviour.hrl"). diff --git a/apps/emqx_data_bridge/src/connector/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl similarity index 100% rename from apps/emqx_data_bridge/src/connector/emqx_connector_mysql.erl rename to apps/emqx_connector/src/emqx_connector_mysql.erl diff --git a/apps/emqx_data_bridge/src/connector/emqx_connector_pgsql.erl b/apps/emqx_connector/src/emqx_connector_pgsql.erl similarity index 100% rename from apps/emqx_data_bridge/src/connector/emqx_connector_pgsql.erl rename to apps/emqx_connector/src/emqx_connector_pgsql.erl diff --git a/apps/emqx_data_bridge/src/connector/emqx_connector_redis.erl b/apps/emqx_connector/src/emqx_connector_redis.erl similarity index 99% rename from apps/emqx_data_bridge/src/connector/emqx_connector_redis.erl rename to apps/emqx_connector/src/emqx_connector_redis.erl index 1037afede..3eebbcf87 100644 --- a/apps/emqx_data_bridge/src/connector/emqx_connector_redis.erl +++ b/apps/emqx_connector/src/emqx_connector_redis.erl @@ -15,7 +15,7 @@ %%-------------------------------------------------------------------- -module(emqx_connector_redis). --include("emqx_data_bridge.hrl"). +-include("emqx_connector.hrl"). -include_lib("typerefl/include/types.hrl"). -include_lib("emqx_resource/include/emqx_resource_behaviour.hrl"). diff --git a/apps/emqx_data_bridge/src/connector/emqx_connector_schema_lib.erl b/apps/emqx_connector/src/emqx_connector_schema_lib.erl similarity index 99% rename from apps/emqx_data_bridge/src/connector/emqx_connector_schema_lib.erl rename to apps/emqx_connector/src/emqx_connector_schema_lib.erl index aaacd0e80..03572d91d 100644 --- a/apps/emqx_data_bridge/src/connector/emqx_connector_schema_lib.erl +++ b/apps/emqx_connector/src/emqx_connector_schema_lib.erl @@ -15,7 +15,7 @@ %%-------------------------------------------------------------------- -module(emqx_connector_schema_lib). --include("emqx_data_bridge.hrl"). +-include("emqx_connector.hrl"). -include_lib("typerefl/include/types.hrl"). -export([ relational_db_fields/0 diff --git a/apps/emqx_connector/src/emqx_connector_sup.erl b/apps/emqx_connector/src/emqx_connector_sup.erl new file mode 100644 index 000000000..603b9a8ad --- /dev/null +++ b/apps/emqx_connector/src/emqx_connector_sup.erl @@ -0,0 +1,36 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_connector_sup). + +-behaviour(supervisor). + +-export([start_link/0]). + +-export([init/1]). + +-define(SERVER, ?MODULE). + +start_link() -> + supervisor:start_link({local, ?SERVER}, ?MODULE, []). + +init([]) -> + SupFlags = #{strategy => one_for_all, + intensity => 0, + period => 1}, + ChildSpecs = [], + {ok, {SupFlags, ChildSpecs}}. + +%% internal functions diff --git a/apps/emqx_data_bridge/priv/emqx_data_bridge.schema b/apps/emqx_data_bridge/priv/emqx_data_bridge.schema new file mode 100644 index 000000000..c9cb3f2c0 --- /dev/null +++ b/apps/emqx_data_bridge/priv/emqx_data_bridge.schema @@ -0,0 +1,16 @@ +%%-*- mode: erlang -*- +%% emqx_data_bridge config mapping + +{mapping, "emqx_data_bridge.bridges", "emqx_data_bridge.bridges", [ + {default, []}, + {datatype, string} +]}. + +% fields("emqx_data_bridge") -> +% [ +% {bridges, +% [fun(mapping) -> "emqx_data_bridge.bridges"; +% (type) -> list(); +% (_) -> undefined +% end]} +% ] \ No newline at end of file diff --git a/apps/emqx_data_bridge/rebar.config b/apps/emqx_data_bridge/rebar.config index 2778ec974..cf4cfcf1b 100644 --- a/apps/emqx_data_bridge/rebar.config +++ b/apps/emqx_data_bridge/rebar.config @@ -1,6 +1,5 @@ {erl_opts, [debug_info]}. {deps, []}. -% {extra_src_dirs, [{"src", [{recursive, true}]}]}. {shell, [ % {config, "config/sys.config"}, diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge.app.src b/apps/emqx_data_bridge/src/emqx_data_bridge.app.src index edfcd4a9d..360511d9b 100644 --- a/apps/emqx_data_bridge/src/emqx_data_bridge.app.src +++ b/apps/emqx_data_bridge/src/emqx_data_bridge.app.src @@ -5,8 +5,7 @@ {mod, {emqx_data_bridge_app, []}}, {applications, [kernel, - stdlib, - ecpool + stdlib ]}, {env,[]}, {modules, []}, diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge.erl b/apps/emqx_data_bridge/src/emqx_data_bridge.erl index 54a228763..5877cd3dd 100644 --- a/apps/emqx_data_bridge/src/emqx_data_bridge.erl +++ b/apps/emqx_data_bridge/src/emqx_data_bridge.erl @@ -25,9 +25,8 @@ ]). load_bridges() -> - ConfFile = filename:join([emqx:get_env(plugins_etc_dir), ?MODULE]) ++ ".conf", - {ok, #{<<"emqx_data_bridge">> := RawConfig}} = hocon:load(ConfFile), - Bridges = maps:get(<<"bridges">>, RawConfig, []), + Bridges = proplists:get_value(bridges, + application:get_all_env(emqx_data_bridge), []), emqx_data_bridge_monitor:ensure_all_started(Bridges). resource_type(<<"mysql">>) -> emqx_connector_mysql; diff --git a/apps/emqx_resource/priv/emqx_resource.schema b/apps/emqx_resource/priv/emqx_resource.schema new file mode 100644 index 000000000..8246dc6a7 --- /dev/null +++ b/apps/emqx_resource/priv/emqx_resource.schema @@ -0,0 +1,2 @@ +%%-*- mode: erlang -*- +%% emqx-resource config mapping diff --git a/data/loaded_plugins.tmpl b/data/loaded_plugins.tmpl index d0dac7fe1..5ac46e0e3 100644 --- a/data/loaded_plugins.tmpl +++ b/data/loaded_plugins.tmpl @@ -5,4 +5,7 @@ {emqx_retainer, {{enable_plugin_emqx_retainer}}}. {emqx_telemetry, {{enable_plugin_emqx_telemetry}}}. {emqx_rule_engine, {{enable_plugin_emqx_rule_engine}}}. +{emqx_resource, {{enable_plugin_emqx_resource}}}. +{emqx_connector, {{enable_plugin_emqx_connector}}}. +{emqx_data_bridge, {{enable_plugin_emqx_data_bridge}}}. {emqx_bridge_mqtt, {{enable_plugin_emqx_bridge_mqtt}}}. diff --git a/rebar.config.erl b/rebar.config.erl index 24020ad45..543333895 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -193,6 +193,9 @@ overlay_vars_rel(RelType) -> end, [ {enable_plugin_emqx_rule_engine, RelType =:= cloud} , {enable_plugin_emqx_bridge_mqtt, RelType =:= edge} + , {enable_plugin_emqx_resource, true} + , {enable_plugin_emqx_connector, true} + , {enable_plugin_emqx_data_bridge, true} , {enable_plugin_emqx_modules, false} %% modules is not a plugin in ce , {enable_plugin_emqx_recon, true} , {enable_plugin_emqx_retainer, true} @@ -251,8 +254,6 @@ relx_apps(ReleaseType) -> , {emqx_plugin_libs, load} , observer_cli , emqx_http_lib - , emqx_resource - , emqx_data_bridge ] ++ [emqx_modules || not is_enterprise()] ++ [emqx_license || is_enterprise()] @@ -290,8 +291,10 @@ relx_plugin_apps(ReleaseType) -> , emqx_auth_mnesia , emqx_web_hook , emqx_recon - , emqx_rule_engine + , emqx_resource + , emqx_connector , emqx_data_bridge + , emqx_rule_engine , emqx_sasl ] ++ [emqx_telemetry || not is_enterprise()] From 22b0fa0b92ccce22989e6c10bc862c6bc874e012 Mon Sep 17 00:00:00 2001 From: DDDHuang <904897578@qq.com> Date: Thu, 10 Jun 2021 16:59:27 +0800 Subject: [PATCH 37/78] chore: more node info --- .../src/emqx_mgmt_api_nodes.erl | 28 +++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_nodes.erl b/apps/emqx_management/src/emqx_mgmt_api_nodes.erl index 2be151aaa..c24e46de9 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_nodes.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_nodes.erl @@ -41,6 +41,30 @@ get(#{node := Node}, _Params) -> format(Node, {error, Reason}) -> #{node => Node, error => Reason}; format(_Node, Info = #{memory_total := Total, memory_used := Used}) -> - Info#{memory_total := emqx_mgmt_util:kmg(Total), - memory_used := emqx_mgmt_util:kmg(Used)}. + {ok, SysPathBinary} = file:get_cwd(), + SysPath = list_to_binary(SysPathBinary), + ConfigPath = <>, + LogPath = case log_path() of + undefined -> + <<"not found">>; + Path0 -> + Path = list_to_binary(Path0), + <> + end, + Info#{ memory_total := emqx_mgmt_util:kmg(Total) + , memory_used := emqx_mgmt_util:kmg(Used) + , sys_path => SysPath + , config_path => ConfigPath + , log_path => LogPath}. +log_path() -> + Configs = logger:get_handler_config(), + get_log_path(Configs). + +get_log_path([#{id := file} = LoggerConfig | _LoggerConfigs]) -> + Config = maps:get(config, LoggerConfig), + maps:get(file, Config); +get_log_path([_LoggerConfig | LoggerConfigs]) -> + get_log_path(LoggerConfigs); +get_log_path([]) -> + undefined. From 1fca3cd1b5f43cee98722ae2fcffe9e6bf6e9f19 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Thu, 17 Jun 2021 16:35:03 +0200 Subject: [PATCH 38/78] chore(rebar.config): use hocon 0.6.0 hocon 0.6.0 added supports for sensitive config flagging sub-struct type checking, and remote struct referencing --- apps/emqx/rebar.config | 2 +- rebar.config | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index a33aa2e64..693e96c74 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -16,7 +16,7 @@ , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.10.1"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}} , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v4.0.1"}}} %% todo delete when plugins use hocon - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.5.1"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.6.0"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {branch, "2.0.4"}}} , {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.13.0"}}} diff --git a/rebar.config b/rebar.config index ae1d1e83a..db95d94ae 100644 --- a/rebar.config +++ b/rebar.config @@ -52,7 +52,7 @@ , {observer_cli, "1.6.1"} % NOTE: depends on recon 2.5.1 , {getopt, "1.0.1"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.13.0"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.5.1"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.6.0"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.2.1"}}} ]}. From 62df9f03ada00cb58ca6c3eebf9a2948e4d19ecc Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Thu, 17 Jun 2021 16:46:45 +0200 Subject: [PATCH 39/78] fix(emqx_schema): add 'sensitive' flag for passowrd configs --- apps/emqx/src/emqx_schema.erl | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index bee052926..cb631ea22 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -113,7 +113,11 @@ fields("rlog") -> fields("node") -> [ {"name", t(string(), "vm_args.-name", "emqx@127.0.0.1", "EMQX_NODE_NAME")} , {"ssl_dist_optfile", t(string(), "vm_args.-ssl_dist_optfile", undefined)} - , {"cookie", t(string(), "vm_args.-setcookie", "emqxsecretcookie", "EMQX_NODE_COOKIE")} + , {"cookie", hoconsc:t(string(), #{mapping => "vm_args.-setcookie", + default => "emqxsecretcookie", + sensitive => true, + override_env => "EMQX_NODE_COOKIE" + })} , {"data_dir", t(string(), "emqx.data_dir", undefined)} , {"heartbeat", t(flag(), undefined, false)} , {"async_threads", t(range(1, 1024), "vm_args.+A", undefined)} @@ -1098,7 +1102,10 @@ ssl(Mapping, Defaults) -> , {"honor_cipher_order", t(flag(), M("honor_cipher_order"), D("honor_cipher_order"))} , {"handshake_timeout", t(duration(), M("handshake_timeout"), D("handshake_timeout"))} , {"depth", t(integer(), M("depth"), D("depth"))} - , {"password", t(string(), M("key_password"), D("key_password"))} + , {"password", hoconsc:t(string(), #{mapping => M("key_password"), + default => D("key_password"), + sensitive => true + })} , {"dhfile", t(string(), M("dhfile"), D("dhfile"))} , {"server_name_indication", t(union(disable, string()), M("server_name_indication"), D("server_name_indication"))} From 3475967c40afda8a1a91a0b33c55857f9886f8f2 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Thu, 17 Jun 2021 17:14:57 +0200 Subject: [PATCH 40/78] chore(bin/emqx): print env override when generating config --- bin/emqx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/emqx b/bin/emqx index 3e54dc17b..a5f76ac72 100755 --- a/bin/emqx +++ b/bin/emqx @@ -230,7 +230,7 @@ generate_config() { ## ths command populates two files: app.