From 3e9b91e53a694843d9f203786acb96d28ae6fc6a Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Fri, 8 Apr 2022 14:31:18 +0800 Subject: [PATCH 01/42] fix: disable server_name_indication suport --- apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src | 2 +- .../src/emqx_plugin_libs.appup.src | 4 ++-- apps/emqx_plugin_libs/src/emqx_plugin_libs_ssl.erl | 14 ++++++++++++-- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src index f72ffc229..38d0bc324 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src @@ -1,6 +1,6 @@ {application, emqx_plugin_libs, [{description, "EMQ X Plugin utility libs"}, - {vsn, "4.3.2"}, + {vsn, "4.3.3"}, {modules, []}, {applications, [kernel,stdlib]}, {env, []} diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src b/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src index 62d0ce4f0..cb61d66a8 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src @@ -2,13 +2,13 @@ {VSN, [ - {<<"4\\.3\\.[0-1]">>, [ + {<<"4\\.3\\.[0-2]">>, [ {load_module, emqx_plugin_libs_ssl, brutal_purge, soft_purge, []} ]}, {<<".*">>, []} ], [ - {<<"4\\.3\\.[0-1]">>, [ + {<<"4\\.3\\.[0-2]">>, [ {load_module, emqx_plugin_libs_ssl, brutal_purge, soft_purge, []} ]}, {<<".*">>, []} diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs_ssl.erl b/apps/emqx_plugin_libs/src/emqx_plugin_libs_ssl.erl index 8d5e2daa5..b459e0b69 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs_ssl.erl +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs_ssl.erl @@ -66,13 +66,23 @@ save_files_return_opts(Options, Dir) -> _ -> verify_peer end, SNI = case Get(<<"server_name_indication">>) of + <<"disable">> -> disable; + "disable" -> disable; + "" -> undefined; + <<>> -> undefined; undefined -> undefined; SNI0 -> ensure_str(SNI0) end, Versions = emqx_tls_lib:integral_versions(Get(<<"tls_versions">>)), Ciphers = emqx_tls_lib:integral_ciphers(Versions, Get(<<"ciphers">>)), - filter([{keyfile, Key}, {certfile, Cert}, {cacertfile, CA}, - {verify, Verify}, {server_name_indication, SNI}, {versions, Versions}, {ciphers, Ciphers}]). + filter([ {keyfile, Key} + , {certfile, Cert} + , {cacertfile, CA} + , {verify, Verify} + , {server_name_indication, SNI} + , {versions, Versions} + , {ciphers, Ciphers} + ]). %% @doc Save a key or certificate file in data dir, %% and return path of the saved file. From 2464c110af2f3834726b74d1571d0f1f864b0f44 Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Fri, 8 Apr 2022 14:59:05 +0800 Subject: [PATCH 02/42] fix: add fix sni change log --- CHANGES-4.3.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index 623e2d399..7eaf2e586 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -42,8 +42,8 @@ File format: default value of emqx_dashboard.conf after the node left cluster. Now we persist changed password to protect against reset. [#7518] * Silence grep/sed warnings in docker-entrypoint.sh. [#7520] -* Generate `loaded_modules` and `loaded_plugins` files with default - values when no such files exists. [#7520] +* Generate `loaded_modules` and `loaded_plugins` files with default values when no such files exists. [#7520] +* Fix the configuration `server_name_indication` set to disable does not take effect. ## v4.3.13 From af4250dcf457cd8171e98805e65fddccaf8e60f9 Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Fri, 8 Apr 2022 15:18:44 +0800 Subject: [PATCH 03/42] fix: mqtt bridge config schema sni disable spelling --- apps/emqx_bridge_mqtt/priv/emqx_bridge_mqtt.schema | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/apps/emqx_bridge_mqtt/priv/emqx_bridge_mqtt.schema b/apps/emqx_bridge_mqtt/priv/emqx_bridge_mqtt.schema index 30120a7d5..face6277e 100644 --- a/apps/emqx_bridge_mqtt/priv/emqx_bridge_mqtt.schema +++ b/apps/emqx_bridge_mqtt/priv/emqx_bridge_mqtt.schema @@ -183,8 +183,12 @@ [{verify, verify_peer}]; (verify, false) -> [{verify, verify_none}]; - (server_name_indication, "disabled") -> - [{server_name_indication, disabled}]; + (server_name_indication, "disable") -> + [{server_name_indication, disable}]; + (server_name_indication, undefined) -> + []; + (server_name_indication, "") -> + []; (server_name_indication, Hostname) -> [{server_name_indication, Hostname}]; (Opt, Val) -> From 4a692f0c2e4706dde5e0e731ae56a335d25438eb Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Fri, 8 Apr 2022 15:44:08 +0800 Subject: [PATCH 04/42] fix: config schema sni disable --- apps/emqx_auth_http/src/emqx_auth_http.app.src | 2 +- .../emqx_auth_http/src/emqx_auth_http.appup.src | 9 +++++++-- apps/emqx_auth_http/src/emqx_auth_http_app.erl | 1 + apps/emqx_auth_ldap/priv/emqx_auth_ldap.schema | 1 + apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src | 2 +- .../emqx_auth_ldap/src/emqx_auth_ldap.appup.src | 8 ++++++-- .../emqx_auth_mongo/priv/emqx_auth_mongo.schema | 1 + .../emqx_auth_mysql/priv/emqx_auth_mysql.schema | 1 + .../emqx_auth_mysql/src/emqx_auth_mysql.app.src | 2 +- .../src/emqx_auth_mysql.appup.src | 9 +++++++-- .../emqx_auth_pgsql/priv/emqx_auth_pgsql.schema | 1 + .../emqx_auth_pgsql/src/emqx_auth_pgsql.app.src | 2 +- .../src/emqx_auth_pgsql.appup.src | 9 +++++++-- .../emqx_auth_redis/priv/emqx_auth_redis.schema | 1 + .../emqx_auth_redis/src/emqx_auth_redis.app.src | 2 +- .../src/emqx_auth_redis.appup.src | 9 +++++++-- .../src/emqx_bridge_mqtt.appup.src | 4 ++-- .../src/emqx_bridge_mqtt_actions.erl | 17 +++++++++++++++++ .../src/emqx_plugin_libs.app.src | 2 +- .../src/emqx_plugin_libs.appup.src | 4 ++-- apps/emqx_web_hook/src/emqx_web_hook.app.src | 2 +- apps/emqx_web_hook/src/emqx_web_hook.appup.src | 2 ++ apps/emqx_web_hook/src/emqx_web_hook_app.erl | 1 + 23 files changed, 71 insertions(+), 21 deletions(-) diff --git a/apps/emqx_auth_http/src/emqx_auth_http.app.src b/apps/emqx_auth_http/src/emqx_auth_http.app.src index 21d0b569b..fd8d1e046 100644 --- a/apps/emqx_auth_http/src/emqx_auth_http.app.src +++ b/apps/emqx_auth_http/src/emqx_auth_http.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_http, [{description, "EMQ X Authentication/ACL with HTTP API"}, - {vsn, "4.3.4"}, % strict semver, bump manually! + {vsn, "4.3.5"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_http_sup]}, {applications, [kernel,stdlib,ehttpc]}, diff --git a/apps/emqx_auth_http/src/emqx_auth_http.appup.src b/apps/emqx_auth_http/src/emqx_auth_http.appup.src index abad22e47..256bd7566 100644 --- a/apps/emqx_auth_http/src/emqx_auth_http.appup.src +++ b/apps/emqx_auth_http/src/emqx_auth_http.appup.src @@ -1,6 +1,9 @@ %% -*- mode: erlang -*- {VSN, - [{"4.3.3", + [{"4.3.4", + [{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]} + ]}, + {"4.3.3", [{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_http,brutal_purge,soft_purge,[]}]}, {"4.3.2", @@ -11,7 +14,9 @@ {<<"4.3.[0-1]">>, [{restart_application,emqx_auth_http}]}, {<<".*">>,[]}], - [{"4.3.3", + [{"4.3.4", + [{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]}]}, + {"4.3.3", [{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_http,brutal_purge,soft_purge,[]}]}, {"4.3.2", diff --git a/apps/emqx_auth_http/src/emqx_auth_http_app.erl b/apps/emqx_auth_http/src/emqx_auth_http_app.erl index 15fc97283..6b868d601 100644 --- a/apps/emqx_auth_http/src/emqx_auth_http_app.erl +++ b/apps/emqx_auth_http/src/emqx_auth_http_app.erl @@ -71,6 +71,7 @@ translate_env(EnvName) -> end, SNI = case application:get_env(?APP, server_name_indication, undefined) of "disable" -> disable; + "" -> undefined; SNI0 -> SNI0 end, TLSOpts = lists:filter( diff --git a/apps/emqx_auth_ldap/priv/emqx_auth_ldap.schema b/apps/emqx_auth_ldap/priv/emqx_auth_ldap.schema index b3d3de1a2..04e9d40df 100644 --- a/apps/emqx_auth_ldap/priv/emqx_auth_ldap.schema +++ b/apps/emqx_auth_ldap/priv/emqx_auth_ldap.schema @@ -73,6 +73,7 @@ {verify, cuttlefish:conf_get("auth.ldap.ssl.verify", Conf, undefined)}, {server_name_indication, case cuttlefish:conf_get("auth.ldap.ssl.server_name_indication", Conf, undefined) of "disable" -> disable; + "" -> undefined; SNI -> SNI end}] end, diff --git a/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src b/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src index bf4148ed9..a072234d7 100644 --- a/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src +++ b/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_ldap, [{description, "EMQ X Authentication/ACL with LDAP"}, - {vsn, "4.3.3"}, % strict semver, bump manually! + {vsn, "4.3.4"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_ldap_sup]}, {applications, [kernel,stdlib,eldap2,ecpool]}, diff --git a/apps/emqx_auth_ldap/src/emqx_auth_ldap.appup.src b/apps/emqx_auth_ldap/src/emqx_auth_ldap.appup.src index f202cf29d..ea44d7a1a 100644 --- a/apps/emqx_auth_ldap/src/emqx_auth_ldap.appup.src +++ b/apps/emqx_auth_ldap/src/emqx_auth_ldap.appup.src @@ -1,6 +1,10 @@ %% -*-: erlang -*- {VSN, - [ {"4.3.0", + [ {"4.3.3", [ + %% There are only changes to the schema file, so we don't need + %% any commands here. + ]}, + {"4.3.0", [ {load_module, emqx_acl_ldap, brutal_purge, soft_purge, []} , {load_module, emqx_auth_ldap_cli, brutal_purge, soft_purge, []} , {load_module, emqx_auth_ldap_app, brutal_purge, soft_purge, []} @@ -16,7 +20,7 @@ ]}, {<<".*">>, []} ], - [ + [ {"4.3.3", []}, {"4.3.0", [ {load_module, emqx_acl_ldap, brutal_purge, soft_purge, []} , {load_module, emqx_auth_ldap_cli, brutal_purge, soft_purge, []} diff --git a/apps/emqx_auth_mongo/priv/emqx_auth_mongo.schema b/apps/emqx_auth_mongo/priv/emqx_auth_mongo.schema index 8a2ff98b3..0c491e005 100644 --- a/apps/emqx_auth_mongo/priv/emqx_auth_mongo.schema +++ b/apps/emqx_auth_mongo/priv/emqx_auth_mongo.schema @@ -141,6 +141,7 @@ Filter([{verify, Verify}, {server_name_indication, case cuttlefish:conf_get(Prefix ++ ".server_name_indication", Conf, undefined) of "disable" -> disable; + "" -> undefined; SNI -> SNI end}, {keyfile, cuttlefish:conf_get(Prefix ++ ".keyfile", Conf, undefined)}, diff --git a/apps/emqx_auth_mysql/priv/emqx_auth_mysql.schema b/apps/emqx_auth_mysql/priv/emqx_auth_mysql.schema index 5e4619301..cacea41a1 100644 --- a/apps/emqx_auth_mysql/priv/emqx_auth_mysql.schema +++ b/apps/emqx_auth_mysql/priv/emqx_auth_mysql.schema @@ -109,6 +109,7 @@ end, SNI = case cuttlefish:conf_get("auth.mysql.ssl.server_name_indication", Conf, undefined) of "disable" -> disable; + "" -> undefined; SNI0 -> SNI0 end, Options ++ [{ssl, Filter([{server_name_indication, SNI}, diff --git a/apps/emqx_auth_mysql/src/emqx_auth_mysql.app.src b/apps/emqx_auth_mysql/src/emqx_auth_mysql.app.src index e3d3677d7..6df4545fd 100644 --- a/apps/emqx_auth_mysql/src/emqx_auth_mysql.app.src +++ b/apps/emqx_auth_mysql/src/emqx_auth_mysql.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_mysql, [{description, "EMQ X Authentication/ACL with MySQL"}, - {vsn, "4.3.1"}, % strict semver, bump manually! + {vsn, "4.3.2"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_mysql_sup]}, {applications, [kernel,stdlib,mysql,ecpool]}, diff --git a/apps/emqx_auth_mysql/src/emqx_auth_mysql.appup.src b/apps/emqx_auth_mysql/src/emqx_auth_mysql.appup.src index edc09aa2d..88ff1b38e 100644 --- a/apps/emqx_auth_mysql/src/emqx_auth_mysql.appup.src +++ b/apps/emqx_auth_mysql/src/emqx_auth_mysql.appup.src @@ -1,10 +1,15 @@ %% -*- mode: erlang -*- {VSN, - [{"4.3.0", + [{"4.3.1", [ + %% There are only changes to the schema file, so we don't need + %% any commands here. + ]}, + {"4.3.0", [{load_module,emqx_auth_mysql_app,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_mysql,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.0", + [{"4.3.1", []}, + {"4.3.0", [{load_module,emqx_auth_mysql_app,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_mysql,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}] diff --git a/apps/emqx_auth_pgsql/priv/emqx_auth_pgsql.schema b/apps/emqx_auth_pgsql/priv/emqx_auth_pgsql.schema index f78bcc597..5d45354b9 100644 --- a/apps/emqx_auth_pgsql/priv/emqx_auth_pgsql.schema +++ b/apps/emqx_auth_pgsql/priv/emqx_auth_pgsql.schema @@ -109,6 +109,7 @@ {verify, Verify}, {server_name_indication, case cuttlefish:conf_get(Prefix ++ ".server_name_indication", Conf, undefined) of "disable" -> disable; + "" -> undefined; SNI -> SNI end}, {versions, [list_to_existing_atom(Value) diff --git a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.app.src b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.app.src index d18021ff3..75175a814 100644 --- a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.app.src +++ b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_pgsql, [{description, "EMQ X Authentication/ACL with PostgreSQL"}, - {vsn, "4.3.1"}, % strict semver, bump manually! + {vsn, "4.3.2"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_pgsql_sup]}, {applications, [kernel,stdlib,epgsql,ecpool]}, diff --git a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.appup.src b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.appup.src index f7e802fdc..1c9a0c572 100644 --- a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.appup.src +++ b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.appup.src @@ -1,10 +1,15 @@ %% -*- mode: erlang -*- {VSN, - [{"4.3.0", + [{"4.3.1", [ + %% There are only changes to the schema file, so we don't need + %% any commands here. + ]}, + {"4.3.0", [{load_module,emqx_auth_pgsql_app,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_pgsql,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.0", + [{"4.3.1", []}, + {"4.3.0", [{load_module,emqx_auth_pgsql_app,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_pgsql,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}] diff --git a/apps/emqx_auth_redis/priv/emqx_auth_redis.schema b/apps/emqx_auth_redis/priv/emqx_auth_redis.schema index 3ce34d597..f36dd9420 100644 --- a/apps/emqx_auth_redis/priv/emqx_auth_redis.schema +++ b/apps/emqx_auth_redis/priv/emqx_auth_redis.schema @@ -98,6 +98,7 @@ end, SNI = case cuttlefish:conf_get("auth.redis.ssl.server_name_indication", Conf, undefined) of "disable" -> disable; + "" -> undefined; SNI0 -> SNI0 end, [{options, [{ssl_options, diff --git a/apps/emqx_auth_redis/src/emqx_auth_redis.app.src b/apps/emqx_auth_redis/src/emqx_auth_redis.app.src index 31c8e7cb6..ea6aaefcf 100644 --- a/apps/emqx_auth_redis/src/emqx_auth_redis.app.src +++ b/apps/emqx_auth_redis/src/emqx_auth_redis.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_redis, [{description, "EMQ X Authentication/ACL with Redis"}, - {vsn, "4.3.1"}, % strict semver, bump manually! + {vsn, "4.3.2"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_redis_sup]}, {applications, [kernel,stdlib,eredis,eredis_cluster,ecpool]}, diff --git a/apps/emqx_auth_redis/src/emqx_auth_redis.appup.src b/apps/emqx_auth_redis/src/emqx_auth_redis.appup.src index 719f5ef80..83f5d46be 100644 --- a/apps/emqx_auth_redis/src/emqx_auth_redis.appup.src +++ b/apps/emqx_auth_redis/src/emqx_auth_redis.appup.src @@ -1,10 +1,15 @@ %% -*- mode: erlang -*- {VSN, - [{"4.3.0", + [{"4.3.1", [ + %% There are only changes to the schema file, so we don't need + %% any commands here. + ]}, + {"4.3.0", [{load_module,emqx_auth_redis_app,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_redis,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.0", + [{"4.3.1", []}, + {"4.3.0", [{load_module,emqx_auth_redis_app,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_redis,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}] diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.appup.src b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.appup.src index b1fa8b85d..6edf6048c 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.appup.src +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.appup.src @@ -2,8 +2,7 @@ {VSN, [ {"4.3.4", [ - %% There are only changes to the schema file, so we don't need - %% any commands here. + {load_module, emqx_bridge_mqtt_actions, brutal_purge, soft_purge, []} ]}, {"4.3.3", [ {load_module, emqx_bridge_mqtt, brutal_purge, soft_purge, []} @@ -21,6 +20,7 @@ ], [ {"4.3.4", [ + {load_module, emqx_bridge_mqtt_actions, brutal_purge, soft_purge, []} ]}, {"4.3.3", [ {load_module, emqx_bridge_mqtt, brutal_purge, soft_purge, []} diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl index 21cda5b6d..d0db149d1 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl @@ -240,6 +240,23 @@ zh => <<"SSL 加密算法"/utf8>>}, description => #{en => <<"SSL Ciphers">>, zh => <<"SSL 加密算法"/utf8>>} + }, + verify => #{ + order => 19, + type => boolean, + default => false, + title => #{en => <<"Verify Server Certfile">>, + zh => <<"校验服务器证书"/utf8>>}, + description => #{en => <<"Whether to verify the server certificate. By default, the client will not verify the server's certificate. If verification is required, please set it to true.">>, + zh => <<"是否校验服务器证书。 默认客户端不会去校验服务器的证书，如果需要校验，请设置成true。"/utf8>>} + }, + server_name_indication => #{ + order => 20, + type => string, + title => #{en => <<"Server Name Indication">>, + zh => <<"服务器名称指示"/utf8>>}, + description => #{en => <<"Specify the hostname used for peer certificate verification, or set to disable to turn off this verification.">>, + zh => <<"指定用于对端证书验证时使用的主机名，或者设置为 disable 以关闭此项验证。"/utf8>>} } }). diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src index 38d0bc324..f72ffc229 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src @@ -1,6 +1,6 @@ {application, emqx_plugin_libs, [{description, "EMQ X Plugin utility libs"}, - {vsn, "4.3.3"}, + {vsn, "4.3.2"}, {modules, []}, {applications, [kernel,stdlib]}, {env, []} diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src b/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src index cb61d66a8..62d0ce4f0 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs.appup.src @@ -2,13 +2,13 @@ {VSN, [ - {<<"4\\.3\\.[0-2]">>, [ + {<<"4\\.3\\.[0-1]">>, [ {load_module, emqx_plugin_libs_ssl, brutal_purge, soft_purge, []} ]}, {<<".*">>, []} ], [ - {<<"4\\.3\\.[0-2]">>, [ + {<<"4\\.3\\.[0-1]">>, [ {load_module, emqx_plugin_libs_ssl, brutal_purge, soft_purge, []} ]}, {<<".*">>, []} diff --git a/apps/emqx_web_hook/src/emqx_web_hook.app.src b/apps/emqx_web_hook/src/emqx_web_hook.app.src index 64cd28784..93db474d2 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook.app.src +++ b/apps/emqx_web_hook/src/emqx_web_hook.app.src @@ -1,6 +1,6 @@ {application, emqx_web_hook, [{description, "EMQ X WebHook Plugin"}, - {vsn, "4.3.10"}, % strict semver, bump manually! + {vsn, "4.3.11"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_web_hook_sup]}, {applications, [kernel,stdlib,ehttpc]}, diff --git a/apps/emqx_web_hook/src/emqx_web_hook.appup.src b/apps/emqx_web_hook/src/emqx_web_hook.appup.src index 3f0980b3c..a1bd558c2 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook.appup.src +++ b/apps/emqx_web_hook/src/emqx_web_hook.appup.src @@ -16,6 +16,7 @@ [ %% nothing so far %% 4.3.9 is taken by release 4.3.12 ]}, + {"4.3.10", [{load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{<<"4\\.3\\.[0-2]">>, [{apply,{application,stop,[emqx_web_hook]}}, @@ -33,4 +34,5 @@ [ %% nothing so far %% 4.3.9 is taken by release 4.3.12 ]}, + {"4.3.10", [{load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}]}. diff --git a/apps/emqx_web_hook/src/emqx_web_hook_app.erl b/apps/emqx_web_hook/src/emqx_web_hook_app.erl index e728aca9d..1156111b5 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook_app.erl +++ b/apps/emqx_web_hook/src/emqx_web_hook_app.erl @@ -61,6 +61,7 @@ translate_env() -> end, SNI = case application:get_env(?APP, server_name_indication, undefined) of "disable" -> disable; + "" -> undefined; SNI0 -> SNI0 end, TLSOpts = lists:filter(fun({_K, V}) -> From a483f5ffa4eed8d45f6b3d6b988c7d2db2ee91a8 Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Fri, 8 Apr 2022 20:03:57 +0800 Subject: [PATCH 05/42] fix: bad appup --- .../src/emqx_bridge_mqtt.appup.src | 66 ++++++++----------- .../emqx_web_hook/src/emqx_web_hook.appup.src | 21 +++--- 2 files changed, 37 insertions(+), 50 deletions(-) diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.appup.src b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.appup.src index 6edf6048c..9e1eda7f4 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.appup.src +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.appup.src @@ -1,39 +1,29 @@ -%% -*-: erlang -*- +%% -*- mode: erlang -*- +%% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [ - {"4.3.4", [ - {load_module, emqx_bridge_mqtt_actions, brutal_purge, soft_purge, []} - ]}, - {"4.3.3", [ - {load_module, emqx_bridge_mqtt, brutal_purge, soft_purge, []} - ]}, - {<<"4\\.3\\.[1-2]">>, [ - {load_module, emqx_bridge_mqtt, brutal_purge, soft_purge, []}, - {load_module, emqx_bridge_mqtt_actions, brutal_purge, soft_purge, []} - ]}, - {"4.3.0", [ - {load_module, emqx_bridge_mqtt, brutal_purge, soft_purge, []}, - {load_module, emqx_bridge_worker, brutal_purge, soft_purge, []}, - {load_module, emqx_bridge_mqtt_actions, brutal_purge, soft_purge, []} - ]}, - {<<".*">>, []} - ], - [ - {"4.3.4", [ - {load_module, emqx_bridge_mqtt_actions, brutal_purge, soft_purge, []} - ]}, - {"4.3.3", [ - {load_module, emqx_bridge_mqtt, brutal_purge, soft_purge, []} - ]}, - {<<"4\\.3\\.[1-2]">>, [ - {load_module, emqx_bridge_mqtt, brutal_purge, soft_purge, []}, - {load_module, emqx_bridge_mqtt_actions, brutal_purge, soft_purge, []} - ]}, - {"4.3.0", [ - {load_module, emqx_bridge_mqtt, brutal_purge, soft_purge, []}, - {load_module, emqx_bridge_worker, brutal_purge, soft_purge, []}, - {load_module, emqx_bridge_mqtt_actions, brutal_purge, soft_purge, []} - ]}, - {<<".*">>, []} - ] -}. + [{"4.3.4", + [{load_module,emqx_bridge_mqtt_actions,brutal_purge,soft_purge,[]}]}, + {"4.3.3", + [{load_module,emqx_bridge_mqtt_actions,brutal_purge,soft_purge,[]}, + {load_module,emqx_bridge_mqtt,brutal_purge,soft_purge,[]}]}, + {<<"4\\.3\\.[1-2]">>, + [{load_module,emqx_bridge_mqtt,brutal_purge,soft_purge,[]}, + {load_module,emqx_bridge_mqtt_actions,brutal_purge,soft_purge,[]}]}, + {"4.3.0", + [{load_module,emqx_bridge_mqtt,brutal_purge,soft_purge,[]}, + {load_module,emqx_bridge_worker,brutal_purge,soft_purge,[]}, + {load_module,emqx_bridge_mqtt_actions,brutal_purge,soft_purge,[]}]}, + {<<".*">>,[]}], + [{"4.3.4", + [{load_module,emqx_bridge_mqtt_actions,brutal_purge,soft_purge,[]}]}, + {"4.3.3", + [{load_module,emqx_bridge_mqtt_actions,brutal_purge,soft_purge,[]}, + {load_module,emqx_bridge_mqtt,brutal_purge,soft_purge,[]}]}, + {<<"4\\.3\\.[1-2]">>, + [{load_module,emqx_bridge_mqtt,brutal_purge,soft_purge,[]}, + {load_module,emqx_bridge_mqtt_actions,brutal_purge,soft_purge,[]}]}, + {"4.3.0", + [{load_module,emqx_bridge_mqtt,brutal_purge,soft_purge,[]}, + {load_module,emqx_bridge_worker,brutal_purge,soft_purge,[]}, + {load_module,emqx_bridge_mqtt_actions,brutal_purge,soft_purge,[]}]}, + {<<".*">>,[]}]}. diff --git a/apps/emqx_web_hook/src/emqx_web_hook.appup.src b/apps/emqx_web_hook/src/emqx_web_hook.appup.src index a1bd558c2..4e3e28a21 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook.appup.src +++ b/apps/emqx_web_hook/src/emqx_web_hook.appup.src @@ -1,4 +1,5 @@ %% -*- mode: erlang -*- +%% Unless you know what you are doing, DO NOT edit manually!! {VSN, [{<<"4\\.3\\.[0-2]">>, [{apply,{application,stop,[emqx_web_hook]}}, @@ -11,12 +12,10 @@ {load_module,emqx_web_hook,brutal_purge,soft_purge,[]}, {load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, {"4.3.8", - [{load_module,emqx_web_hook,brutal_purge,soft_purge,[]}]}, - {"4.3.9", - [ %% nothing so far - %% 4.3.9 is taken by release 4.3.12 - ]}, - {"4.3.10", [{load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}]}, + [{load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_web_hook,brutal_purge,soft_purge,[]}]}, + {"4.3.9",[{load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}]}, + {"4.3.10",[{load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{<<"4\\.3\\.[0-2]">>, [{apply,{application,stop,[emqx_web_hook]}}, @@ -29,10 +28,8 @@ {load_module,emqx_web_hook,brutal_purge,soft_purge,[]}, {load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, {"4.3.8", - [{load_module,emqx_web_hook,brutal_purge,soft_purge,[]}]}, - {"4.3.9", - [ %% nothing so far - %% 4.3.9 is taken by release 4.3.12 - ]}, - {"4.3.10", [{load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}]}, + [{load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_web_hook,brutal_purge,soft_purge,[]}]}, + {"4.3.9",[{load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}]}, + {"4.3.10",[{load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}]}. From 703441c631bf7684a5401c7be6adb24948af3520 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 11 Apr 2022 18:23:08 +0800 Subject: [PATCH 06/42] chore: update ehttpc to 0.2.0 In ehttpc v0.2.0, we have optimized the implementation for request collection to reduce resource consumption. see: https://github.com/emqx/ehttpc/pull/31 --- rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config b/rebar.config index d7ab16fe3..0a7faf889 100644 --- a/rebar.config +++ b/rebar.config @@ -39,7 +39,7 @@ {deps, [ {gpb, "4.11.2"} %% gpb only used to build, but not for release, pin it here to avoid fetching a wrong version due to rebar plugins scattered in all the deps - , {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.1.15"}}} + , {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.2.0"}}} , {eredis_cluster, {git, "https://github.com/emqx/eredis_cluster", {tag, "0.7.1"}}} , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} From 73ac4c1ff8925198f359ba20bd7465cf6314455f Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Mon, 11 Apr 2022 18:33:20 +0800 Subject: [PATCH 07/42] fix: random id ensure do not delete resource --- apps/emqx_rule_engine/src/emqx_rule_engine.erl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.erl b/apps/emqx_rule_engine/src/emqx_rule_engine.erl index 292e10311..b210d4f8d 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.erl @@ -332,7 +332,9 @@ start_resource(ResId) -> test_resource(#{type := Type} = Params) -> case emqx_rule_registry:find_resource_type(Type) of {ok, #resource_type{}} -> - ResId = maps:get(id, Params, resource_id()), + %% Resource will be deleted after test. + %% Use random resource id, ensure test func will not delete the resource in used. + ResId = resource_id(), try case create_resource(maps:put(id, ResId, Params), no_retry) of {ok, _} -> From 5e652217c82b220af0d1b90b35c47d53591b3dad Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 11 Apr 2022 22:33:41 +0200 Subject: [PATCH 08/42] fix: make possible to have white spaces in root path --- CHANGES-4.3.md | 6 ++ bin/emqx | 153 +++++++++++++++++++++-------------------------- bin/emqx_ctl | 4 +- data/emqx_vars | 1 - rebar.config.erl | 2 - 5 files changed, 76 insertions(+), 90 deletions(-) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index 7eaf2e586..ed1bbd3a5 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -10,6 +10,12 @@ File format: - One list item per change topic Change log ends with a list of github PRs +## v4.3.15 + +### Enhancements + +* Made possible for EMQX to boot from a Linux directory which has white spaces in its path. + ## v4.3.14 ### Enhancements diff --git a/bin/emqx b/bin/emqx index 19c154d0d..1722e96e0 100755 --- a/bin/emqx +++ b/bin/emqx @@ -4,9 +4,9 @@ set -e -ROOT_DIR="$(cd "$(dirname "$(readlink "$0" || echo "$0")")"/..; pwd -P)" +RUNNER_ROOT_DIR="$(cd "$(dirname "$(readlink "$0" || echo "$0")")"/..; pwd -P)" # shellcheck disable=SC1090 -. "$ROOT_DIR"/releases/emqx_vars +. "$RUNNER_ROOT_DIR"/releases/emqx_vars RUNNER_SCRIPT="$RUNNER_BIN_DIR/$REL_NAME" CODE_LOADING_MODE="${CODE_LOADING_MODE:-embedded}" @@ -146,7 +146,8 @@ fi if [ -z "$WITH_EPMD" ]; then EPMD_ARG="-start_epmd false -epmd_module ekka_epmd -proto_dist ekka" else - EPMD_ARG="-start_epmd true $PROTO_DIST_ARG" + PROTO_DIST=$(grep -E '^[ \t]*cluster.proto_dist[ \t]*=[ \t]*' "$RUNNER_ETC_DIR/emqx.conf" 2> /dev/null | tail -1 | awk -F"= " '{print $NF}') + EPMD_ARG="-start_epmd true -proto_dist $PROTO_DIST" fi # Warn the user if ulimit -n is less than 1024 @@ -157,9 +158,6 @@ if [ "$ULIMIT_F" -lt 1024 ]; then echo "!!!!" fi -# By default, use cuttlefish to generate app.config and vm.args -CUTTLEFISH="${USE_CUTTLEFISH:-yes}" - SED_REPLACE="sed -i " case $(sed --help 2>&1) in *GNU*) SED_REPLACE="sed -i ";; @@ -230,73 +228,69 @@ relx_start_command() { "$START_OPTION" } +trim() { + echo -e "${1}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' +} + # Function to generate app.config and vm.args generate_config() { ## Delete the *.siz files first or it cann't start after ## changing the config 'log.rotation.size' rm -rf "${RUNNER_LOG_DIR}"/*.siz - if [ "$CUTTLEFISH" != "yes" ]; then - # Note: we have added a parameter '-vm_args' to this. It - # appears redundant but it is not! the erlang vm allows us to - # access all arguments to the erl command EXCEPT '-args_file', - # so in order to get access to this file location from within - # the vm, we need to pass it in twice. - CONFIG_ARGS=" -config $RUNNER_ETC_DIR/app.config -args_file $RUNNER_ETC_DIR/vm.args -vm_args $RUNNER_ETC_DIR/vm.args " - else - EMQX_LICENSE_CONF_OPTION="" - if [ "${EMQX_LICENSE_CONF:-}" != "" ]; then - EMQX_LICENSE_CONF_OPTION="-i ${EMQX_LICENSE_CONF}" - fi - - set +e - # shellcheck disable=SC2086 - CUTTLEFISH_OUTPUT="$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/cuttlefish -v -i "$REL_DIR"/emqx.schema $EMQX_LICENSE_CONF_OPTION -c "$RUNNER_ETC_DIR"/emqx.conf -d "$RUNNER_DATA_DIR"/configs generate)" - # shellcheck disable=SC2181 - RESULT=$? - set -e - if [ $RESULT -gt 0 ]; then - echo "$CUTTLEFISH_OUTPUT" - exit $RESULT - fi - # print override from environment variables (EMQX_*) - echo "$CUTTLEFISH_OUTPUT" | sed -e '$d' - CONFIG_ARGS=$(echo "$CUTTLEFISH_OUTPUT" | tail -n 1) - - ## Merge cuttlefish generated *.args into the vm.args - CUTTLE_GEN_ARG_FILE=$(echo "$CONFIG_ARGS" | sed -n 's/^.*\(vm_args[[:space:]]\)//p' | awk '{print $1}') - TMP_ARG_FILE="$RUNNER_DATA_DIR/configs/vm.args.tmp" - cp "$RUNNER_ETC_DIR/vm.args" "$TMP_ARG_FILE" - echo "" >> "$TMP_ARG_FILE" - echo "-pa ${REL_DIR}/consolidated" >> "$TMP_ARG_FILE" - sed '/^#/d' "$CUTTLE_GEN_ARG_FILE" | sed '/^$/d' | while IFS='' read -r ARG_LINE || [ -n "$ARG_LINE" ]; do - ARG_KEY=$(echo "$ARG_LINE" | awk '{$NF="";print}') - ARG_VALUE=$(echo "$ARG_LINE" | awk '{print $NF}') - if [ "$ARG_KEY" = '' ]; then - ## for the flags, e.g. -heart -emu_args etc - ARG_KEY=$(echo "$ARG_LINE" | awk '{print $1}') - ARG_VALUE='' - TMP_ARG_KEY=$(grep "^$ARG_KEY" "$TMP_ARG_FILE" | awk '{print $1}') - if [ "$TMP_ARG_KEY" = '' ]; then - echo "$ARG_KEY" >> "$TMP_ARG_FILE" - fi - else - TMP_ARG_VALUE=$(grep "^$ARG_KEY" "$TMP_ARG_FILE" | awk '{print $NF}') - if [ "$ARG_VALUE" != "$TMP_ARG_VALUE" ] ; then - if [ -n "$TMP_ARG_VALUE" ]; then - sh -c "$SED_REPLACE 's/^$ARG_KEY.*$/$ARG_LINE/' $TMP_ARG_FILE" - else - echo "$ARG_LINE" >> "$TMP_ARG_FILE" - fi - fi - fi - done - mv -f "$TMP_ARG_FILE" "$CUTTLE_GEN_ARG_FILE" + EMQX_LICENSE_CONF_OPTION="" + if [ "${EMQX_LICENSE_CONF:-}" != "" ]; then + EMQX_LICENSE_CONF_OPTION="-i ${EMQX_LICENSE_CONF}" fi + set +e # shellcheck disable=SC2086 - if ! relx_nodetool chkconfig $CONFIG_ARGS; then - echoerr "Error reading $CONFIG_ARGS" + CUTTLEFISH_OUTPUT="$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/cuttlefish -v -i "$REL_DIR"/emqx.schema $EMQX_LICENSE_CONF_OPTION -c "$RUNNER_ETC_DIR"/emqx.conf -d "$RUNNER_DATA_DIR"/configs generate)" + # shellcheck disable=SC2181 + RESULT=$? + set -e + if [ $RESULT -gt 0 ]; then + echo "$CUTTLEFISH_OUTPUT" + exit $RESULT + fi + ## transform a single line args list like '-config ... -args_file ... -vm_args ...' to lines and get path for each file respectively + ## NOTE: the -args_file and -vm_args are the same file passed twice because args_file is used by beam, but not possible to get at runtime + ## by calling init:get_arguments/0 + lines="$(echo "$CUTTLEFISH_OUTPUT" | tail -1 | sed 's/-config/\nconfig=/g' | sed 's/-args_file/\nargs_file=/g' | sed 's/-vm_args/\nvm_args=/g')" + CONFIG_FILE="$(trim "$(echo -e "$lines" | grep 'config=' | sed 's/config=//g')")" + CUTTLE_GEN_ARG_FILE="$(trim "$(echo -e "$lines" | grep 'vm_args=' | sed 's/vm_args=//g')")" + + ## Merge cuttlefish generated *.args into the vm.args + TMP_ARG_FILE="$RUNNER_DATA_DIR/configs/vm.args.tmp" + cp "$RUNNER_ETC_DIR/vm.args" "$TMP_ARG_FILE" + echo "" >> "$TMP_ARG_FILE" + echo "-pa \"${REL_DIR}/consolidated\"" >> "$TMP_ARG_FILE" + sed '/^#/d' "$CUTTLE_GEN_ARG_FILE" | sed '/^$/d' | while IFS='' read -r ARG_LINE || [ -n "$ARG_LINE" ]; do + ARG_KEY=$(echo "$ARG_LINE" | awk '{$NF="";print}') + ARG_VALUE=$(echo "$ARG_LINE" | awk '{print $NF}') + if [ "$ARG_KEY" = '' ]; then + ## for the flags, e.g. -heart -emu_args etc + ARG_KEY=$(echo "$ARG_LINE" | awk '{print $1}') + ARG_VALUE='' + TMP_ARG_KEY=$(grep "^$ARG_KEY" "$TMP_ARG_FILE" | awk '{print $1}') + if [ "$TMP_ARG_KEY" = '' ]; then + echo "$ARG_KEY" >> "$TMP_ARG_FILE" + fi + else + TMP_ARG_VALUE=$(grep "^$ARG_KEY" "$TMP_ARG_FILE" | awk '{print $NF}') + if [ "$ARG_VALUE" != "$TMP_ARG_VALUE" ] ; then + if [ -n "$TMP_ARG_VALUE" ]; then + sh -c "$SED_REPLACE 's/^$ARG_KEY.*$/$ARG_LINE/' \"$TMP_ARG_FILE\"" + else + echo "$ARG_LINE" >> "$TMP_ARG_FILE" + fi + fi + fi + done + mv -f "$TMP_ARG_FILE" "$CUTTLE_GEN_ARG_FILE" + + if ! relx_nodetool chkconfig -config "$CONFIG_FILE"; then + echoerr "Error reading $CONFIG_FILE" exit 1 fi } @@ -341,15 +335,6 @@ wait_for() { done } -# Use $CWD/etc/sys.config if exists -if [ -z "$RELX_CONFIG_PATH" ]; then - if [ -f "$RUNNER_ETC_DIR/sys.config" ]; then - RELX_CONFIG_PATH="-config $RUNNER_ETC_DIR/sys.config" - else - RELX_CONFIG_PATH="" - fi -fi - IS_BOOT_COMMAND='no' case "$1" in start|start_boot) @@ -423,14 +408,6 @@ if [ -z "$COOKIE" ]; then exit 1 fi -# Support for IPv6 Dist. See: https://github.com/emqtt/emqttd/issues/1460 -PROTO_DIST=$(grep -E '^[ \t]*cluster.proto_dist[ \t]*=[ \t]*' "$RUNNER_ETC_DIR/emqx.conf" 2> /dev/null | tail -1 | awk -F"= " '{print $NF}') -if [ -z "$PROTO_DIST" ]; then - PROTO_DIST_ARG="" -else - PROTO_DIST_ARG="-proto_dist $PROTO_DIST" -fi - cd "$ROOTDIR" # User can specify an sname without @hostname @@ -667,14 +644,17 @@ case "$1" in # Store passed arguments since they will be erased by `set` ARGS="$*" - # shellcheck disable=SC2086 # $RELX_CONFIG_PATH $CONFIG_ARGS $EPMD_ARG are supposed to be split by whitespace + # shellcheck disable=SC2086 # $EPMD_ARG is supposed to be split by whitespace # Build an array of arguments to pass to exec later on # Build it here because this command will be used for logging. set -- "$BINDIR/erlexec" \ -boot "$BOOTFILE" -mode "$CODE_LOADING_MODE" \ -boot_var ERTS_LIB_DIR "$ERTS_LIB_DIR" \ -mnesia dir "\"${MNESIA_DATA_DIR}\"" \ - $RELX_CONFIG_PATH $CONFIG_ARGS $EPMD_ARG + -config "$CONFIG_FILE" \ + -args_file "$CUTTLE_GEN_ARG_FILE" \ + -vm_args "$CUTTLE_GEN_ARG_FILE" \ + $EPMD_ARG # Log the startup logger -t "${REL_NAME}[$$]" "$* -- ${1+$ARGS}" @@ -708,14 +688,17 @@ case "$1" in # Store passed arguments since they will be erased by `set` ARGS="$*" - # shellcheck disable=SC2086 # $RELX_CONFIG_PATH $CONFIG_ARGS $EPMD_ARG are supposed to be split by whitespace + # shellcheck disable=SC2086 # $EPMD_ARG is supposed to be split by whitespace # Build an array of arguments to pass to exec later on # Build it here because this command will be used for logging. set -- "$BINDIR/erlexec" $FOREGROUNDOPTIONS \ -boot "$REL_DIR/$BOOTFILE" -mode "$CODE_LOADING_MODE" \ -boot_var ERTS_LIB_DIR "$ERTS_LIB_DIR" \ -mnesia dir "\"${MNESIA_DATA_DIR}\"" \ - $RELX_CONFIG_PATH $CONFIG_ARGS $EPMD_ARG + -config "$CONFIG_FILE" \ + -args_file "$CUTTLE_GEN_ARG_FILE" \ + -vm_args "$CUTTLE_GEN_ARG_FILE" \ + $EPMD_ARG # Log the startup logger -t "${REL_NAME}[$$]" "$* -- ${1+$ARGS}" diff --git a/bin/emqx_ctl b/bin/emqx_ctl index 8d729083d..bed351b7b 100755 --- a/bin/emqx_ctl +++ b/bin/emqx_ctl @@ -4,9 +4,9 @@ set -e -ROOT_DIR="$(cd "$(dirname "$(readlink "$0" || echo "$0")")"/..; pwd -P)" +RUNNER_ROOT_DIR="$(cd "$(dirname "$(readlink "$0" || echo "$0")")"/..; pwd -P)" # shellcheck disable=SC1090 -. "$ROOT_DIR"/releases/emqx_vars +. "$RUNNER_ROOT_DIR"/releases/emqx_vars export RUNNER_ROOT_DIR export REL_VSN diff --git a/data/emqx_vars b/data/emqx_vars index 453bd618f..a872da03a 100644 --- a/data/emqx_vars +++ b/data/emqx_vars @@ -6,7 +6,6 @@ REL_VSN="{{ release_version }}" ERTS_VSN="{{ erts_vsn }}" ERL_OPTS="{{ erl_opts }}" -RUNNER_ROOT_DIR="{{ runner_root_dir }}" RUNNER_BIN_DIR="{{ runner_bin_dir }}" RUNNER_LOG_DIR="{{ runner_log_dir }}" RUNNER_LIB_DIR="{{ runner_lib_dir }}" diff --git a/rebar.config.erl b/rebar.config.erl index 1000a2c92..226597967 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -223,7 +223,6 @@ overlay_vars_pkg(bin) -> , {platform_lib_dir, "lib"} , {platform_log_dir, "log"} , {platform_plugins_dir, "etc/plugins"} - , {runner_root_dir, "$(cd $(dirname $(readlink $0 || echo $0))/..; pwd -P)"} , {runner_bin_dir, "$RUNNER_ROOT_DIR/bin"} , {runner_etc_dir, "$RUNNER_ROOT_DIR/etc"} , {runner_lib_dir, "$RUNNER_ROOT_DIR/lib"} @@ -238,7 +237,6 @@ overlay_vars_pkg(pkg) -> , {platform_lib_dir, ""} , {platform_log_dir, "/var/log/emqx"} , {platform_plugins_dir, "/var/lib/emqx/plugins"} - , {runner_root_dir, "/usr/lib/emqx"} , {runner_bin_dir, "/usr/bin"} , {runner_etc_dir, "/etc/emqx"} , {runner_lib_dir, "$RUNNER_ROOT_DIR/lib"} From e9ed65de0642e53dece70911c1110f144e4e0337 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 12 Apr 2022 09:39:41 +0800 Subject: [PATCH 09/42] chore: bump vsn to 4.3.14-rc.2 --- include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index 6526e9a6c..c5d14e53e 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.3.14-rc.1"}). +-define(EMQX_RELEASE, {opensource, "4.3.14-rc.2"}). -else. From 5b9452979d65b90ab9f097b7655101ac9479770b Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Tue, 12 Apr 2022 16:47:39 +0800 Subject: [PATCH 10/42] fix: bridge mqtt config schema, subscriptions with default QoS 0 --- .../emqx_bridge_mqtt/priv/emqx_bridge_mqtt.schema | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/apps/emqx_bridge_mqtt/priv/emqx_bridge_mqtt.schema b/apps/emqx_bridge_mqtt/priv/emqx_bridge_mqtt.schema index face6277e..c8738cee6 100644 --- a/apps/emqx_bridge_mqtt/priv/emqx_bridge_mqtt.schema +++ b/apps/emqx_bridge_mqtt/priv/emqx_bridge_mqtt.schema @@ -214,8 +214,19 @@ end, Subscriptions = fun(Name) -> Configs = cuttlefish_variable:filter_by_prefix("bridge.mqtt." ++ Name ++ ".subscription", Conf), - lists:zip([Topic || {_, Topic} <- lists:sort([{I, Topic} || {[_, _, _, "subscription", I, "topic"], Topic} <- Configs])], - [QoS || {_, QoS} <- lists:sort([{I, QoS} || {[_, _, _, "subscription", I, "qos"], QoS} <- Configs])]) + Configs = cuttlefish_variable:filter_by_prefix("bridge.mqtt." ++ Name ++ ".subscription", Conf), + SubConfId = lists:usort([{T, I} || {[_, _, _, "subscription", I, "topic"], T} <- Configs]), + FindQoS = fun({T, I}) -> + QoSFilter = fun + ({[_, _, _, "subscription", Index, "qos"], QoS}) -> I == Index andalso {true, QoS}; + (_Topic) -> false + end, + case lists:filtermap(QoSFilter, Configs) of + [] -> {T, 0}; + QoSList -> {T, hd(QoSList)} + end + end, + [FindQoS(TopicAndIndex) || TopicAndIndex <- SubConfId] end, IsNodeAddr = fun(Addr) -> case string:tokens(Addr, "@") of From 2c478694262f0fa4e6c5eb20e5cc869c15d6bfb1 Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 12 Apr 2022 18:40:37 +0800 Subject: [PATCH 11/42] test(frame): fix empty topic name case not running --- test/emqx_frame_SUITE.erl | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/test/emqx_frame_SUITE.erl b/test/emqx_frame_SUITE.erl index 8074749a9..c3c6f6c1c 100644 --- a/test/emqx_frame_SUITE.erl +++ b/test/emqx_frame_SUITE.erl @@ -46,6 +46,8 @@ groups() -> t_parse_frame_malformed_variable_byte_integer, t_parse_frame_variable_byte_integer, t_parse_malformed_utf8_string, + t_parse_empty_topic_name, + t_parse_empty_topic_name_with_alias, t_parse_frame_proxy_protocol %% proxy_protocol_config_disabled packet. ]}, {connect, [parallel], @@ -170,8 +172,12 @@ t_parse_empty_topic_name(_) -> t_parse_empty_topic_name_with_alias(_) -> Props = #{'Topic-Alias' => 16#AB}, Packet = ?PUBLISH_PACKET(?QOS_1, <<>>, 1, Props, <<>>), - ?assertEqual(Packet, parse_serialize(Packet, #{strict_mode => false})), - ?assertEqual(Packet, parse_serialize(Packet, #{strict_mode => true})). + ?assertEqual( + Packet, parse_serialize(Packet, #{strict_mode => false, version => ?MQTT_PROTO_V5}) + ), + ?assertEqual( + Packet, parse_serialize(Packet, #{strict_mode => true, version => ?MQTT_PROTO_V5}) + ). t_parse_frame_proxy_protocol(_) -> BinList = [ <<"PROXY TCP4 ">>, <<"PROXY TCP6 ">>, <<"PROXY UNKNOWN">> From 2da27392f7602295cbd0d7772bb7d71c7b832780 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 11 Apr 2022 23:48:00 +0200 Subject: [PATCH 12/42] chore: refine boot script with more readable help info --- bin/emqx | 246 +++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 166 insertions(+), 80 deletions(-) diff --git a/bin/emqx b/bin/emqx index 1722e96e0..ac459b6f7 100755 --- a/bin/emqx +++ b/bin/emqx @@ -31,6 +31,12 @@ ERTS_LIB_DIR="$ERTS_DIR/../lib" # Echo to stderr on errors echoerr() { echo "$*" 1>&2; } +assert_node_alive() { + if ! relx_nodetool "ping" > /dev/null; then + die "node_is_not_running!" 1 + fi +} + check_eralng_start() { "$BINDIR/$PROGNAME" -noshell -boot "$REL_DIR/start_clean" -s crypto start -s init stop } @@ -60,62 +66,157 @@ fi # cuttlefish try to read environment variables starting with "EMQX_" export CUTTLEFISH_ENV_OVERRIDE_PREFIX='EMQX_' -relx_usage() { - command="$1" +usage() { + local command="$1" case "$command" in - unpack) - echo "Usage: $REL_NAME unpack [VERSION]" - echo "Unpacks a release package VERSION, it assumes that this" - echo "release package tarball has already been deployed at one" - echo "of the following locations:" - echo " releases/-.tar.gz" - echo " releases/-.zip" - ;; - install) - echo "Usage: $REL_NAME install [VERSION]" - echo "Installs a release package VERSION, it assumes that this" - echo "release package tarball has already been deployed at one" - echo "of the following locations:" - echo " releases/-.tar.gz" - echo " releases/-.zip" - echo "" - echo " --no-permanent Install release package VERSION but" - echo " don't make it permanent" - ;; - uninstall) - echo "Usage: $REL_NAME uninstall [VERSION]" - echo "Uninstalls a release VERSION, it will only accept" - echo "versions that are not currently in use" - ;; - upgrade) - echo "Usage: $REL_NAME upgrade [VERSION]" - echo "Upgrades the currently running release to VERSION, it assumes" - echo "that a release package tarball has already been deployed at one" - echo "of the following locations:" - echo " releases/-.tar.gz" - echo " releases/-.zip" - echo "" - echo " --no-permanent Install release package VERSION but" - echo " don't make it permanent" - ;; - downgrade) - echo "Usage: $REL_NAME downgrade [VERSION]" - echo "Downgrades the currently running release to VERSION, it assumes" - echo "that a release package tarball has already been deployed at one" - echo "of the following locations:" - echo " releases/-.tar.gz" - echo " releases/-.zip" - echo "" - echo " --no-permanent Install release package VERSION but" - echo " don't make it permanent" - ;; - *) - echo "Usage: $REL_NAME {start|start_boot |ertspath|foreground|stop|restart|reboot|pid|ping|console|console_clean|console_boot |attach|remote_console|upgrade|downgrade|install|uninstall|versions|escript|rpc|rpcterms|eval|root_dir}" - ;; + start) + echo "Start EMQX service in daemon mode" + ;; + stop) + echo "Stop the running EMQX program" + ;; + console) + echo "Boot up EMQX service in an interactive Erlang shell" + echo "This command needs a tty" + ;; + console_clean) + echo "This command does NOT boot up the EMQX service" + echo "It only starts an interactive Erlang shell with all the" + echo "EMQX code available" + ;; + foreground) + echo "Start EMQX in foreground mode without an interactive shell" + ;; + pid) + echo "Print out EMQX process identifier" + ;; + ping) + echo "Check if the EMQX node is up and running" + echo "This command exit with 0 silently if node is running" + ;; + escript) + echo "Execute a escript using the Erlang runtime from EMQX package installation" + echo "For example $REL_NAME escript /path/to/my/escript my_arg1 my_arg2" + ;; + attach) + echo "This command is applicable when EMQX is started in daemon mode." + echo "It attaches the current shell to EMQX's control console" + echo "through a named pipe." + echo "WARNING: try to use the safer alternative, remote_console command." + ;; + remote_console) + echo "Start an interactive shell running an Erlang node which " + echo "hidden-connects to the running EMQX node". + echo "This command is mostly used for troubleshooting." + ;; + ertspath) + echo "Print path to Erlang runtime dir" + ;; + rpc) + echo "Usge $REL_NAME rpc MODULE FUNCTION [ARGS, ...]" + echo "Connect to the EMQX node and make an Erlang RPC" + echo "This command blocks for at most 60 seconds." + echo "It exits with non-zero code in case of any RPC failure" + echo "including connection error and runtime exception" + ;; + rpcterms) + echo "Usge $REL_NAME rpcterms MODULE FUNCTION [ARGS, ...]" + echo "Connect to the EMQX node and make an Erlang RPC" + echo "The result of the RPC call is pretty-printed as an " + echo "Erlang term" + ;; + root_dir) + echo "Print EMQX installation root dir" + ;; + eval) + echo "Evaluate an Erlang expression in the EMQX node" + ;; + versions) + echo "List installed EMQX versions and their status" + ;; + unpack) + echo "Usage: $REL_NAME unpack [VERSION]" + echo "Unpacks a release package VERSION, it assumes that this" + echo "release package tarball has already been deployed at one" + echo "of the following locations:" + echo " releases/-.zip" + ;; + install) + echo "Usage: $REL_NAME install [VERSION]" + echo "Installs a release package VERSION, it assumes that this" + echo "release package tarball has already been deployed at one" + echo "of the following locations:" + echo " releases/-.zip" + echo "" + echo " --no-permanent Install release package VERSION but" + echo " don't make it permanent" + ;; + uninstall) + echo "Usage: $REL_NAME uninstall [VERSION]" + echo "Uninstalls a release VERSION, it will only accept" + echo "versions that are not currently in use" + ;; + upgrade) + echo "Usage: $REL_NAME upgrade [VERSION]" + echo "Upgrades the currently running release to VERSION, it assumes" + echo "that a release package tarball has already been deployed at one" + echo "of the following locations:" + echo " releases/-.zip" + echo "" + echo " --no-permanent Install release package VERSION but" + echo " don't make it permanent" + ;; + downgrade) + echo "Usage: $REL_NAME downgrade [VERSION]" + echo "Downgrades the currently running release to VERSION, it assumes" + echo "that a release package tarball has already been deployed at one" + echo "of the following locations:" + echo " releases/-.zip" + echo "" + echo " --no-permanent Install release package VERSION but" + echo " don't make it permanent" + ;; + *) + echo "Usage: $REL_NAME COMMAND [help]" + echo '' + echo "Commonly used COMMANDs:" + echo " start: Start EMQX in daemon mode" + echo " console: Start EMQX in an interactive Erlang shell" + echo " foreground: Start EMQX in foreground mode without an interactive shell" + echo " stop: Stop the running EMQX node" + echo " ctl: Administration commands, execute '$REL_NAME ctl help' for more details" + echo '' + echo "More:" + echo " Shell attach: remote_console | attach" + echo " Up/Down-grade: upgrade | downgrade | install | uninstall" + echo " Install info: ertspath | root_dir | versions" + echo " Runtime info: pid | ping | versions" + echo " Advanced: console_clean | escript | rpc | rpcterms | eval" + echo '' + echo "Execute '$REL_NAME COMMAND help' for more information" + ;; esac } +COMMAND="${1:-}" + +if [ -z "$COMMAND" ]; then + usage 'help' + exit 1 +elif [ "$COMMAND" = 'help' ]; then + usage 'help' + exit 0 +fi + +if [ "${2:-}" = 'help' ]; then + ## 'ctl' command has its own usage info + if [ "$COMMAND" != 'ctl' ]; then + usage "$COMMAND" + exit 0 + fi +fi + # Simple way to check the correct user and fail early check_user() { # Validate that the user running the script is the owner of the @@ -135,7 +236,6 @@ check_user() { fi } - # Make sure the user running this script is the owner and/or su to that user check_user "$@" ES=$? @@ -357,9 +457,10 @@ if [ -z "$NAME_ARG" ]; then if [ "$IS_BOOT_COMMAND" = 'no' ]; then # for non-boot commands, inspect vm.