yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #10605 from zmstone/0503-no-telemetry-app-for-ee 

0503 refactoring: no telemetry app for ee
This commit is contained in:
Zaiming (Stone) Shi 2023-05-27 10:57:15 +02:00 committed by GitHub
parent fe81e9521a 3595a86724
commit c2450d230d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
34 changed files with 498 additions and 238 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
apps/emqx/test/emqx_common_test_helpers.erl
View File

@ -232,11 +232,12 @@ render_and_load_app_config(App, Opts) ->
try
do_render_app_config(App, Schema, Conf, Opts)
catch
throw:skip ->
ok;
throw:E:St ->
%% turn throw into error
error({Conf, E, St})
end.
do_render_app_config(App, Schema, ConfigFile, Opts) ->
%% copy acl_conf must run before read_schema_configs
copy_acl_conf(),
@ -318,6 +319,7 @@ render_config_file(ConfigFile, Vars0) ->
Temp =
case file:read_file(ConfigFile) of
{ok, T} -> T;
{error, enoent} -> throw(skip);
{error, Reason} -> error({failed_to_read_config_template, ConfigFile, Reason})
end,
Vars = [{atom_to_list(N), iolist_to_binary(V)} || {N, V} <- maps:to_list(Vars0)],

19
apps/emqx_conf/src/emqx_conf_schema.erl
View File

@ -52,6 +52,7 @@
emqx_authn_schema,
emqx_authz_schema,
emqx_auto_subscribe_schema,
{emqx_telemetry_schema, ce},
emqx_modules_schema,
emqx_plugins_schema,
emqx_dashboard_schema,
@ -108,11 +109,25 @@ roots() ->
] ++
emqx_schema:roots(medium) ++
emqx_schema:roots(low) ++
lists:flatmap(fun roots/1, ?MERGED_CONFIGS).
lists:flatmap(fun roots/1, common_apps()).
validations() ->
hocon_schema:validations(emqx_schema) ++
lists:flatmap(fun hocon_schema:validations/1, ?MERGED_CONFIGS).
lists:flatmap(fun hocon_schema:validations/1, common_apps()).
common_apps() ->
Edition = emqx_release:edition(),
lists:filtermap(
fun
({N, E}) ->
case E =:= Edition of
true -> {true, N};
false -> false
end;
(N) when is_atom(N) -> {true, N}
end,
?MERGED_CONFIGS
).
fields("cluster") ->
[

1
apps/emqx_enterprise/etc/emqx-enterprise.conf
View File

@ -1 +0,0 @@
telemetry.enable = false

2
apps/emqx_machine/src/emqx_machine_boot.erl
View File

@ -149,7 +149,7 @@ basic_reboot_apps() ->
],
case emqx_release:edition() of
ce ->
CE;
CE ++ [emqx_telemetry];
ee ->
CE ++
[

10
apps/emqx_modules/README.md
View File

@ -1,7 +1,7 @@
# EMQX Modules
The application provides some minor functional modules that are not included in the MQTT
protocol standard, including "Delayed Publish", "Topic Rewrite", "Topic Metrics" and "Telemetry".
protocol standard, including "Delayed Publish", "Topic Rewrite", "Topic Metrics".
## Delayed Publish
@ -42,12 +42,4 @@ See HTTP API docs to [List all monitored topics](https://www.emqx.io/docs/en/v5.
and [Get the monitored result](https://www.emqx.io/docs/en/v5.0/admin/api-docs.html#tag/MQTT/paths/~1mqtt~1topic_metrics~1%7Btopic%7D/get).
## Telemetry
Telemetry is used for collecting non-sensitive information about the EMQX cluster.
More introduction about [Telemetry](https://www.emqx.io/docs/en/v5.0/telemetry/telemetry.html#telemetry).
See HTTP API docs to [Enable/Disable telemetry](https://www.emqx.io/docs/en/v5.0/admin/api-docs.html#tag/Telemetry/paths/~1telemetry~1status/put),
[Get the enabled status](https://www.emqx.io/docs/en/v5.0/admin/api-docs.html#tag/Telemetry/paths/~1telemetry~1status/get)
and [Get the data of the module collected](https://www.emqx.io/docs/en/v5.0/admin/api-docs.html#tag/Telemetry/paths/~1telemetry~1data/get).

6
apps/emqx_modules/include/emqx_modules.hrl
View File

@ -14,11 +14,5 @@
%% limitations under the License.
%%--------------------------------------------------------------------
%% The destination URL for the telemetry data report
-define(TELEMETRY_URL, "https://telemetry.emqx.io/api/telemetry").
%% Interval for reporting telemetry data, Default: 7d
-define(REPORT_INTERVAL, 604800).
-define(API_TAG_MQTT, [<<"MQTT">>]).
-define(API_SCHEMA_MODULE, emqx_modules_schema).

2
apps/emqx_modules/src/emqx_modules.app.src
View File

@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_modules, [
{description, "EMQX Modules"},
{vsn, "5.0.14"},
{vsn, "5.0.15"},
{modules, []},
{applications, [kernel, stdlib, emqx, emqx_ctl]},
{mod, {emqx_modules_app, []}},

2
apps/emqx_modules/src/emqx_modules_app.erl
View File

@ -34,7 +34,6 @@ stop(_State) ->
maybe_enable_modules() ->
emqx_conf:get([delayed, enable], true) andalso emqx_delayed:load(),
emqx_modules_conf:is_telemetry_enabled() andalso emqx_telemetry:enable(),
emqx_observer_cli:enable(),
emqx_conf_cli:load(),
ok = emqx_rewrite:enable(),
@ -43,7 +42,6 @@ maybe_enable_modules() ->
maybe_disable_modules() ->
emqx_conf:get([delayed, enable], true) andalso emqx_delayed:unload(),
emqx_modules_conf:is_telemetry_enabled() andalso emqx_telemetry:disable(),
emqx_conf:get([observer_cli, enable], true) andalso emqx_observer_cli:disable(),
emqx_rewrite:disable(),
emqx_conf_cli:unload(),

35
apps/emqx_modules/src/emqx_modules_conf.erl
View File

@ -28,9 +28,7 @@
-export([
topic_metrics/0,
add_topic_metrics/1,
remove_topic_metrics/1,
is_telemetry_enabled/0,
set_telemetry_status/1
remove_topic_metrics/1
]).
%% config handlers
@ -45,12 +43,10 @@
-spec load() -> ok.
load() ->
emqx_conf:add_handler([topic_metrics], ?MODULE),
emqx_conf:add_handler([telemetry], ?MODULE).
emqx_conf:add_handler([topic_metrics], ?MODULE).
-spec unload() -> ok.
unload() ->
emqx_conf:remove_handler([telemetry]),
emqx_conf:remove_handler([topic_metrics]).
%%--------------------------------------------------------------------
@ -82,18 +78,6 @@ remove_topic_metrics(Topic) ->
{error, Reason} -> {error, Reason}
end.
-spec is_telemetry_enabled() -> boolean().
is_telemetry_enabled() ->
IsOfficial = emqx_telemetry:official_version(emqx_release:version()),
emqx_conf:get([telemetry, enable], IsOfficial).
-spec set_telemetry_status(boolean()) -> ok | {error, term()}.
set_telemetry_status(Status) ->
case cfg_update([telemetry], set_telemetry_status, Status) of
{ok, _} -> ok;
{error, _} = Error -> Error
end.
%%--------------------------------------------------------------------
%% Config Handler
%%--------------------------------------------------------------------
@ -119,9 +103,7 @@ pre_config_update(_, {remove_topic_metrics, Topic0}, RawConf) ->
{ok, RawConf -- [Topic]};
_ ->
{error, not_found}
end;
pre_config_update(_, {set_telemetry_status, Status}, RawConf) ->
{ok, RawConf#{<<"enable">> => Status}}.
end.
-spec post_config_update(
list(atom()),
@ -153,17 +135,6 @@ post_config_update(
case emqx_topic_metrics:deregister(Topic) of
ok -> ok;
{error, Reason} -> {error, Reason}
end;
post_config_update(
_,
{set_telemetry_status, Status},
_NewConfig,
_OldConfig,
_AppEnvs
) ->
case Status of
true -> emqx_telemetry:enable();
false -> emqx_telemetry:disable()
end.
%%--------------------------------------------------------------------

5
apps/emqx_modules/src/emqx_modules_schema.erl
View File

@ -33,7 +33,6 @@ namespace() -> modules.
roots() ->
[
"delayed",
"telemetry",
array("rewrite", #{
desc => "List of topic rewrite rules.",
importance => ?IMPORTANCE_HIDDEN,
@ -46,8 +45,6 @@ roots() ->
})
].
fields("telemetry") ->
[{enable, ?HOCON(boolean(), #{default => true, desc => "Enable telemetry."})}];
fields("delayed") ->
[
{enable, ?HOCON(boolean(), #{default => true, desc => ?DESC(enable)})},
@ -76,8 +73,6 @@ fields("rewrite") ->
fields("topic_metrics") ->
[{topic, ?HOCON(binary(), #{desc => "Collect metrics for the topic."})}].
desc("telemetry") ->
"Settings for the telemetry module.";
desc("delayed") ->
"Settings for the delayed module.";
desc("rewrite") ->

1
apps/emqx_modules/src/emqx_modules_sup.erl
View File

@ -41,7 +41,6 @@ start_link() ->
init([]) ->
{ok,
{{one_for_one, 10, 3600}, [
?CHILD(emqx_telemetry),
?CHILD(emqx_topic_metrics),
?CHILD(emqx_trace),
?CHILD(emqx_delayed)

55
apps/emqx_telemetry/README.md Normal file
View File

@ -0,0 +1,55 @@
# emqx_telemetry
In order for EMQ to understand how EMQX opensource edition is being used,
this app reports some telemetry data to [EMQ's telemetry server](https://telemetry.emqx.io/api/telemetry)
To turn it off, you can set `telemetry.enable = false` in emqx.conf,
or start EMQX with environment variable `EMQX_TELEMETRY__ENABLE=false`.
## Reported Data
This application is only relesaed in EMQX opensource edition.
There is nothing in the reported data which can back-track the origin.
The report interval is 7 days, so there is no performance impact.
Here is the full catalog of the reported data.
### Installation
- EMQX version string
- License (always stubed with "opensource")
- VM Stats (number of cores, total memory)
- Operating system name
- Operating system version
- Erlang/OTP version
- Number of enabled plugins
- Cluster UUID (auto-generated random ID)
- Node UUID (auto-generated random ID)
### Provisioning
- Advanced MQTT features
- Number of retained messages
- Number of topic-rewrite rules
- Number of delay-publish messages
- Number of auto-subscription rules
- Enabled authentication and authorization mechanisms
- Enabled gateway names
- Enabled bridge types
- Enabled exhooks (the number of endpoints and hookpoints)
### Runtime
- Uptime (how long the server has been running since last start/restart)
- Number of connected clients
- Number of messages sent
- Messaging rates
## More info
More introduction about [Telemetry](https://www.emqx.io/docs/en/v5.0/telemetry/telemetry.html#telemetry).
See HTTP API docs to [Enable/Disable telemetry](https://www.emqx.io/docs/en/v5.0/admin/api-docs.html#tag/Telemetry/paths/~1telemetry~1status/put),
[Get the enabled status](https://www.emqx.io/docs/en/v5.0/admin/api-docs.html#tag/Telemetry/paths/~1telemetry~1status/get)
and [Get the data of the module collected](https://www.emqx.io/docs/en/v5.0/admin/api-docs.html#tag/Telemetry/paths/~1telemetry~1data/get).

8
apps/emqx_telemetry/rebar.config Normal file
View File

@ -0,0 +1,8 @@
%% -*- mode: erlang -*-
{deps, [
{emqx, {path, "../emqx"}},
{emqx_utils, {path, "../emqx_utils"}},
{emqx_conf, {path, "../emqx_conf"}}
]}.
{project_plugins, [erlfmt]}.

15
apps/emqx_telemetry/src/emqx_telemetry.app.src Normal file
View File

@ -0,0 +1,15 @@
{application, emqx_telemetry, [
{description, "Report telemetry data for EMQX Opensource edition"},
{vsn, "0.1.0"},
{registered, [emqx_telemetry_sup, emqx_telemetry]},
{mod, {emqx_telemetry_app, []}},
{applications, [
kernel,
stdlib
]},
{env, []},
{modules, []},
{licenses, ["Apache-2.0"]},
{links, []}
]}.

75
apps/emqx_modules/src/emqx_telemetry.erl → apps/emqx_telemetry/src/emqx_telemetry.erl
View File

@ -18,13 +18,6 @@
-behaviour(gen_server).
-include_lib("emqx/include/emqx.hrl").
-include_lib("emqx/include/logger.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-include("emqx_modules.hrl").
-export([
start_link/0,
stop/0
@ -41,9 +34,10 @@
code_change/3
]).
%% config change hook points
-export([
enable/0,
disable/0
start_reporting/0,
stop_reporting/0
]).
-export([
@ -52,8 +46,6 @@
get_telemetry/0
]).
-export([official_version/1]).
%% Internal exports (RPC)
-export([
do_ensure_uuids/0
@ -72,6 +64,15 @@
get_value/3
]).
-include_lib("emqx/include/emqx.hrl").
-include_lib("emqx/include/logger.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
%% The destination URL for the telemetry data report
-define(TELEMETRY_URL, "https://telemetry.emqx.io/api/telemetry").
-define(REPORT_INTERVAL, 604800).
-record(telemetry, {
id :: atom(),
uuid :: binary()
@ -112,11 +113,17 @@ start_link() ->
stop() ->
gen_server:stop(?MODULE).
enable() ->
gen_server:call(?MODULE, enable, 15_000).
%% @doc Re-start the reporting timer after disabled.
%% This is an async notification which never fails.
%% This is a no-op in enterprise edition.
start_reporting() ->
gen_server:cast(?MODULE, start_reporting).
disable() ->
gen_server:call(?MODULE, disable).
%% @doc Stop the reporting timer.
%% This is an async notification which never fails.
%% This is a no-op in enterprise eidtion.
stop_reporting() ->
gen_server:cast(?MODULE, stop_reporting).
get_node_uuid() ->
gen_server:call(?MODULE, get_node_uuid).
@ -132,7 +139,9 @@ get_telemetry() ->
%%--------------------------------------------------------------------
init(_Opts) ->
{ok, undefined, {continue, init}}.
process_flag(trap_exit, true),
emqx_telemetry_config:on_server_start(),
{ok, "ignored", {continue, init}}.
handle_continue(init, _) ->
ok = mria:create_table(
@ -148,21 +157,12 @@ handle_continue(init, _) ->
ok = mria:wait_for_tables([?TELEMETRY]),
State0 = empty_state(),
{NodeUUID, ClusterUUID} = ensure_uuids(),
{noreply, State0#state{node_uuid = NodeUUID, cluster_uuid = ClusterUUID}};
handle_continue(Continue, State) ->
?SLOG(error, #{msg => "unexpected_continue", continue => Continue}),
{noreply, State}.
case is_enabled() of
true -> ok = start_reporting();
false -> ok
end,
{noreply, State0#state{node_uuid = NodeUUID, cluster_uuid = ClusterUUID}}.
handle_call(enable, _From, State) ->
%% Wait a few moments before reporting the first telemetry, as the
%% apps might still be starting up. Also, this avoids hanging
%% `emqx_modules_app' initialization in case the POST request
%% takes a lot of time.
FirstReportTimeoutMS = timer:seconds(10),
{reply, ok, ensure_report_timer(FirstReportTimeoutMS, State)};
handle_call(disable, _From, State = #state{timer = Timer}) ->
emqx_utils:cancel_timer(Timer),
{reply, ok, State#state{timer = undefined}};
handle_call(get_node_uuid, _From, State = #state{node_uuid = UUID}) ->
{reply, {ok, UUID}, State};
handle_call(get_cluster_uuid, _From, State = #state{cluster_uuid = UUID}) ->
@ -174,6 +174,14 @@ handle_call(Req, _From, State) ->
?SLOG(error, #{msg => "unexpected_call", call => Req}),
{reply, ignored, State}.
handle_cast(start_reporting, State) ->
%% Wait a few moments before reporting the first telemetry, as the
%% apps might still be starting up.
FirstReportTimeoutMS = timer:seconds(10),
{noreply, ensure_report_timer(FirstReportTimeoutMS, State)};
handle_cast(stop_reporting, State = #state{timer = Timer}) ->
emqx_utils:cancel_timer(Timer),
{noreply, State#state{timer = undefined}};
handle_cast(Msg, State) ->
?SLOG(error, #{msg => "unexpected_cast", cast => Msg}),
{noreply, State}.
@ -188,7 +196,7 @@ handle_info(Info, State) ->
{noreply, State}.
terminate(_Reason, _State) ->
ok.
emqx_telemetry_config:on_server_stop().
code_change(_OldVsn, State, _Extra) ->
{ok, State}.
@ -197,9 +205,8 @@ code_change(_OldVsn, State, _Extra) ->
%% Internal functions
%%------------------------------------------------------------------------------
official_version(Version) ->
Pt = "^\\d+\\.\\d+(?:\\.\\d+)?(?:(-(?:alpha|beta|rc)\\.[1-9][0-9]*))?$",
match =:= re:run(Version, Pt, [{capture, none}]).
is_enabled() ->
emqx_telemetry_config:is_enabled().
ensure_report_timer(State = #state{report_interval = ReportInterval}) ->
ensure_report_timer(ReportInterval, State).

11
apps/emqx_modules/src/emqx_telemetry_api.erl → apps/emqx_telemetry/src/emqx_telemetry_api.erl
View File

@ -220,7 +220,7 @@ status(get, _Params) ->
{200, get_telemetry_status()};
status(put, #{body := Body}) ->
Enable = maps:get(<<"enable">>, Body),
case Enable =:= emqx_modules_conf:is_telemetry_enabled() of
case Enable =:= is_enabled() of
true ->
Reason =
case Enable of
@ -241,7 +241,7 @@ status(put, #{body := Body}) ->
end.
data(get, _Request) ->
case emqx_modules_conf:is_telemetry_enabled() of
case is_enabled() of
true ->
{200, emqx_utils_json:encode(get_telemetry_data())};
false ->
@ -256,11 +256,14 @@ data(get, _Request) ->
%%--------------------------------------------------------------------
enable_telemetry(Enable) ->
emqx_modules_conf:set_telemetry_status(Enable).
emqx_telemetry_config:set_telemetry_status(Enable).
get_telemetry_status() ->
#{enable => emqx_modules_conf:is_telemetry_enabled()}.
#{enable => is_enabled()}.
get_telemetry_data() ->
{ok, TelemetryData} = emqx_telemetry:get_telemetry(),
TelemetryData.
is_enabled() ->
emqx_telemetry_config:is_enabled().

29
apps/emqx_telemetry/src/emqx_telemetry_app.erl Normal file
View File

@ -0,0 +1,29 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_telemetry_app).
-behaviour(application).
-export([start/2, stop/1]).
start(_StartType, _StartArgs) ->
emqx_telemetry_sup:start_link().
stop(_State) ->
ok.
%% internal functions

79
apps/emqx_telemetry/src/emqx_telemetry_config.erl Normal file
View File

@ -0,0 +1,79 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_telemetry_config).
%% Public API
-export([
set_telemetry_status/1,
is_enabled/0
]).
%% emqx_config_handler callback
-export([
pre_config_update/3,
post_config_update/5
]).
%% internal use
-export([
on_server_start/0,
on_server_stop/0,
is_official_version/1
]).
is_enabled() ->
IsOfficial = ?MODULE:is_official_version(emqx_release:version()),
emqx_conf:get([telemetry, enable], IsOfficial).
on_server_start() ->
emqx_conf:add_handler([telemetry], ?MODULE).
on_server_stop() ->
emqx_conf:remove_handler([telemetry]).
-spec set_telemetry_status(boolean()) -> ok | {error, term()}.
set_telemetry_status(Status) ->
case cfg_update([telemetry], set_telemetry_status, Status) of
{ok, _} -> ok;
{error, _} = Error -> Error
end.
pre_config_update(_, {set_telemetry_status, Status}, RawConf) ->
{ok, RawConf#{<<"enable">> => Status}}.
post_config_update(
_,
{set_telemetry_status, Status},
_NewConfig,
_OldConfig,
_AppEnvs
) ->
case Status of
true -> emqx_telemetry:start_reporting();
false -> emqx_telemetry:stop_reporting()
end.
cfg_update(Path, Action, Params) ->
emqx_conf:update(
Path,
{Action, Params},
#{override_to => cluster}
).
is_official_version(Version) ->
Pt = "^\\d+\\.\\d+(?:\\.\\d+)?(?:(-(?:alpha|beta|rc)\\.[1-9][0-9]*))?$",
match =:= re:run(Version, Pt, [{capture, none}]).

38
apps/emqx_telemetry/src/emqx_telemetry_schema.erl Normal file
View File

@ -0,0 +1,38 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_telemetry_schema).
-include_lib("hocon/include/hoconsc.hrl").
-include_lib("typerefl/include/types.hrl").
-behaviour(hocon_schema).
-export([
roots/0,
fields/1,
desc/1
]).
roots() -> ["telemetry"].
fields("telemetry") ->
[{enable, ?HOCON(boolean(), #{required => false, desc => ?DESC("enable")})}].
desc("telemetry") ->
?DESC("telemetry_root_doc");
desc(_) ->
undefined.

45
apps/emqx_telemetry/src/emqx_telemetry_sup.erl Normal file
View File

@ -0,0 +1,45 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_telemetry_sup).
-behaviour(supervisor).
-export([start_link/0]).
-export([init/1]).
-define(SERVER, ?MODULE).
-define(CHILD(Mod), #{
id => Mod,
start => {Mod, start_link, []},
restart => transient,
shutdown => 5000,
type => worker,
modules => [Mod]
}).
start_link() ->
supervisor:start_link({local, ?SERVER}, ?MODULE, []).
init([]) ->
SupFlags = #{
strategy => one_for_one,
intensity => 10,
period => 5
},
ChildSpecs = [?CHILD(emqx_telemetry)],
{ok, {SupFlags, ChildSpecs}}.

0
apps/emqx_modules/src/proto/emqx_telemetry_proto_v1.erl → apps/emqx_telemetry/src/proto/emqx_telemetry_proto_v1.erl
View File

156
apps/emqx_modules/test/emqx_telemetry_SUITE.erl → apps/emqx_telemetry/test/emqx_telemetry_SUITE.erl
View File

@ -25,13 +25,21 @@
-import(proplists, [get_value/2]).
-define(BASE_CONF, #{
-define(MODULES_CONF, #{
<<"dealyed">> => <<"true">>,
<<"max_delayed_messages">> => <<"0">>
}).
all() -> emqx_common_test_helpers:all(?MODULE).
suite() ->
[
{timetrap, {minutes, 1}},
{repeat, 1}
].
apps() -> [emqx_conf, emqx_retainer, emqx_authn, emqx_authz, emqx_modules, emqx_telemetry].
init_per_suite(Config) ->
net_kernel:start(['master@127.0.0.1', longnames]),
ok = meck:new(emqx_authz, [non_strict, passthrough, no_history, no_link]),
@ -42,12 +50,9 @@ init_per_suite(Config) ->
emqx_common_test_helpers:deps_path(emqx_authz, "etc/acl.conf")
end
),
ok = emqx_common_test_helpers:load_config(emqx_modules_schema, ?BASE_CONF),
ok = emqx_common_test_helpers:load_config(emqx_modules_schema, ?MODULES_CONF),
emqx_gateway_test_utils:load_all_gateway_apps(),
emqx_common_test_helpers:start_apps(
[emqx_conf, emqx_authn, emqx_authz, emqx_modules],
fun set_special_configs/1
),
start_apps(),
Config.
end_per_suite(_Config) ->
@ -61,7 +66,7 @@ end_per_suite(_Config) ->
),
mnesia:clear_table(cluster_rpc_commit),
mnesia:clear_table(cluster_rpc_mfa),
emqx_common_test_helpers:stop_apps([emqx_conf, emqx_authn, emqx_authz, emqx_modules]),
stop_apps(),
meck:unload(emqx_authz),
ok.
@ -100,13 +105,9 @@ init_per_testcase(t_get_telemetry, Config) ->
Config;
init_per_testcase(t_advanced_mqtt_features, Config) ->
{ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000),
{ok, Retainer} = emqx_retainer:start_link(),
{atomic, ok} = mria:clear_table(emqx_delayed),
mock_advanced_mqtt_features(),
[
{retainer, Retainer}
| Config
];
Config;
init_per_testcase(t_authn_authz_info, Config) ->
mock_httpc(),
{ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000),
@ -116,13 +117,13 @@ init_per_testcase(t_authn_authz_info, Config) ->
create_authz(postgresql),
Config;
init_per_testcase(t_enable, Config) ->
ok = meck:new(emqx_telemetry, [non_strict, passthrough, no_history, no_link]),
ok = meck:expect(emqx_telemetry, official_version, fun(_) -> true end),
ok = meck:new(emqx_telemetry_config, [non_strict, passthrough, no_history, no_link]),
ok = meck:expect(emqx_telemetry_config, is_official_version, fun(_) -> true end),
mock_httpc(),
Config;
init_per_testcase(t_send_after_enable, Config) ->
ok = meck:new(emqx_telemetry, [non_strict, passthrough, no_history, no_link]),
ok = meck:expect(emqx_telemetry, official_version, fun(_) -> true end),
ok = meck:new(emqx_telemetry_config, [non_strict, passthrough, no_history, no_link]),
ok = meck:expect(emqx_telemetry_config, is_official_version, fun(_) -> true end),
mock_httpc(),
Config;
init_per_testcase(t_rule_engine_and_data_bridge_info, Config) ->
@ -172,12 +173,9 @@ init_per_testcase(t_uuid_restored_from_file, Config) ->
%% clear the UUIDs in the DB
{atomic, ok} = mria:clear_table(emqx_telemetry),
emqx_common_test_helpers:stop_apps([emqx_conf, emqx_authn, emqx_authz, emqx_modules]),
ok = emqx_common_test_helpers:load_config(emqx_modules_schema, ?BASE_CONF),
emqx_common_test_helpers:start_apps(
[emqx_conf, emqx_authn, emqx_authz, emqx_modules],
fun set_special_configs/1
),
stop_apps(),
ok = emqx_common_test_helpers:load_config(emqx_modules_schema, ?MODULES_CONF),
start_apps(),
Node = start_slave(n1),
[
{n1, Node},
@ -205,11 +203,9 @@ end_per_testcase(t_get_telemetry, _Config) ->
meck:unload([httpc, emqx_telemetry]),
application:stop(emqx_gateway),
ok;
end_per_testcase(t_advanced_mqtt_features, Config) ->
Retainer = ?config(retainer, Config),
end_per_testcase(t_advanced_mqtt_features, _Config) ->
process_flag(trap_exit, true),
ok = emqx_retainer:clean(),
exit(Retainer, kill),
{ok, _} = emqx_auto_subscribe:update([]),
ok = emqx_rewrite:update([]),
{atomic, ok} = mria:clear_table(emqx_delayed),
@ -228,9 +224,9 @@ end_per_testcase(t_authn_authz_info, _Config) ->
),
ok;
end_per_testcase(t_enable, _Config) ->
meck:unload([httpc, emqx_telemetry]);
meck:unload([httpc, emqx_telemetry_config]);
end_per_testcase(t_send_after_enable, _Config) ->
meck:unload([httpc, emqx_telemetry]);
meck:unload([httpc, emqx_telemetry_config]);
end_per_testcase(t_rule_engine_and_data_bridge_info, _Config) ->
meck:unload(httpc),
lists:foreach(
@ -278,10 +274,10 @@ t_node_uuid(_) ->
Parts = binary:split(UUID, <<"-">>, [global, trim]),
?assertEqual(5, length(Parts)),
{ok, NodeUUID2} = emqx_telemetry:get_node_uuid(),
emqx_telemetry:disable(),
emqx_telemetry:enable(),
emqx_modules_conf:set_telemetry_status(false),
emqx_modules_conf:set_telemetry_status(true),
emqx_telemetry:stop_reporting(),
emqx_telemetry:start_reporting(),
emqx_telemetry_config:set_telemetry_status(false),
emqx_telemetry_config:set_telemetry_status(true),
{ok, NodeUUID3} = emqx_telemetry:get_node_uuid(),
{ok, NodeUUID4} = emqx_telemetry_proto_v1:get_node_uuid(node()),
?assertEqual(NodeUUID2, NodeUUID3),
@ -325,12 +321,9 @@ t_uuid_saved_to_file(_Config) ->
%% clear the UUIDs in the DB
{atomic, ok} = mria:clear_table(emqx_telemetry),
emqx_common_test_helpers:stop_apps([emqx_conf, emqx_authn, emqx_authz, emqx_modules]),
ok = emqx_common_test_helpers:load_config(emqx_modules_schema, ?BASE_CONF),
emqx_common_test_helpers:start_apps(
[emqx_conf, emqx_authn, emqx_authz, emqx_modules],
fun set_special_configs/1
),
stop_apps(),
ok = emqx_common_test_helpers:load_config(emqx_modules_schema, ?MODULES_CONF),
start_apps(),
{ok, NodeUUID} = emqx_telemetry:get_node_uuid(),
{ok, ClusterUUID} = emqx_telemetry:get_cluster_uuid(),
?assertEqual(
@ -343,30 +336,35 @@ t_uuid_saved_to_file(_Config) ->
),
ok.
t_official_version(_) ->
true = emqx_telemetry:official_version("0.0.0"),
true = emqx_telemetry:official_version("1.1.1"),
true = emqx_telemetry:official_version("10.10.10"),
false = emqx_telemetry:official_version("0.0.0.0"),
false = emqx_telemetry:official_version("1.1.a"),
true = emqx_telemetry:official_version("0.0-alpha.1"),
true = emqx_telemetry:official_version("1.1-alpha.1"),
true = emqx_telemetry:official_version("10.10-alpha.10"),
false = emqx_telemetry:official_version("1.1-alpha.0"),
true = emqx_telemetry:official_version("1.1-beta.1"),
true = emqx_telemetry:official_version("1.1-rc.1"),
false = emqx_telemetry:official_version("1.1-alpha.a"),
true = emqx_telemetry:official_version("5.0.0"),
true = emqx_telemetry:official_version("5.0.0-alpha.1"),
true = emqx_telemetry:official_version("5.0.0-beta.4"),
true = emqx_telemetry:official_version("5.0-rc.1"),
true = emqx_telemetry:official_version("5.0.0-rc.1"),
false = emqx_telemetry:official_version("5.0.0-alpha.a"),
false = emqx_telemetry:official_version("5.0.0-beta.a"),
false = emqx_telemetry:official_version("5.0.0-rc.a"),
false = emqx_telemetry:official_version("5.0.0-foo"),
false = emqx_telemetry:official_version("5.0.0-rc.1-ccdf7920"),
ok.
t_is_official_version(_) ->
Is = fun(V) -> is_official_version(V) end,
true = lists:all(Is, [
"0.0.0",
"1.1.1",
"10.10.10",
"0.0-alpha.1",
"1.1-alpha.1",
"10.10-alpha.10",
"1.1-rc.1",
"1.1-beta.1",
"5.0.0",
"5.0.0-alpha.1",
"5.0.0-beta.4",
"5.0-rc.1",
"5.0.0-rc.1"
]),
false = lists:any(Is, [
"1.1-alpha.a",
"1.1-alpha.0",
"0.0.0.0",
"1.1.a",
"5.0.0-alpha.a",
"5.0.0-beta.a",
"5.0.0-rc.a",
"5.0.0-foo",
"5.0.0-rc.1-ccdf7920"
]).
t_get_telemetry(_Config) ->
{ok, TelemetryData} = emqx_telemetry:get_telemetry(),
@ -432,23 +430,25 @@ t_num_clients(_Config) ->
{port, 1883},
{clean_start, false}
]),
?wait_async_action(
{{ok, _}, _} = ?wait_async_action(
{ok, _} = emqtt:connect(Client),
#{
?snk_kind := emqx_stats_setstat,
count_stat := 'live_connections.count',
value := 1
}
},
2000
),
{ok, TelemetryData0} = emqx_telemetry:get_telemetry(),
?assertEqual(1, get_value(num_clients, TelemetryData0)),
?wait_async_action(
{ok, _} = ?wait_async_action(
ok = emqtt:disconnect(Client),
#{
?snk_kind := emqx_stats_setstat,
count_stat := 'live_connections.count',
value := 0
}
},
2000
),
{ok, TelemetryData1} = emqx_telemetry:get_telemetry(),
?assertEqual(0, get_value(num_clients, TelemetryData1)),
@ -485,19 +485,19 @@ t_authn_authz_info(_) ->
).
t_enable(_) ->
ok = emqx_telemetry:enable(),
ok = emqx_telemetry:disable().
ok = emqx_telemetry:start_reporting(),
ok = emqx_telemetry:stop_reporting().
t_send_after_enable(_) ->
ok = emqx_telemetry:disable(),
ok = emqx_telemetry:stop_reporting(),
ok = snabbkaffe:start_trace(),
try
ok = emqx_telemetry:enable(),
ok = emqx_telemetry:start_reporting(),
Timeout = 12_000,
?assertMatch(
{ok, _},
?wait_async_action(
ok = emqx_telemetry:enable(),
ok = emqx_telemetry:start_reporting(),
#{?snk_kind := telemetry_data_reported},
Timeout
)
@ -818,11 +818,10 @@ start_slave(Name) ->
(emqx) ->
application:set_env(emqx, boot_modules, []),
ekka:join(TestNode),
emqx_common_test_helpers:load_config(emqx_modules_schema, ?BASE_CONF),
emqx_common_test_helpers:load_config(emqx_modules_schema, ?MODULES_CONF),
ok;
(_App) ->
emqx_common_test_helpers:load_config(emqx_modules_schema, ?BASE_CONF),
emqx_common_test_helpers:load_config(emqx_modules_schema, ?MODULES_CONF),
ok
end,
Opts = #{
@ -837,7 +836,7 @@ start_slave(Name) ->
env_handler => Handler,
load_apps => [gen_rpc, emqx],
listener_ports => [],
apps => [emqx_conf, emqx_modules]
apps => [emqx, emqx_conf, emqx_retainer, emqx_modules, emqx_telemetry]
},
emqx_common_test_helpers:start_slave(Name, Opts).
@ -861,3 +860,12 @@ leave_cluster() ->
application:set_env(mria, db_backend, mnesia),
ekka:leave()
end.
is_official_version(V) ->
emqx_telemetry_config:is_official_version(V).
start_apps() ->
emqx_common_test_helpers:start_apps(apps(), fun set_special_configs/1).
stop_apps() ->
emqx_common_test_helpers:stop_apps(lists:reverse(apps())).

0
apps/emqx_modules/test/emqx_telemetry_SUITE_data/BUILD_INFO → apps/emqx_telemetry/test/emqx_telemetry_SUITE_data/BUILD_INFO
View File

4
apps/emqx_modules/test/emqx_telemetry_api_SUITE.erl → apps/emqx_telemetry/test/emqx_telemetry_api_SUITE.erl
View File

@ -31,7 +31,7 @@ all() ->
init_per_suite(Config) ->
ok = emqx_common_test_helpers:load_config(emqx_modules_schema, ?BASE_CONF),
ok = emqx_mgmt_api_test_util:init_suite(
[emqx_conf, emqx_authn, emqx_authz, emqx_modules],
[emqx_conf, emqx_authn, emqx_authz, emqx_telemetry],
fun set_special_configs/1
),
@ -47,7 +47,7 @@ end_per_suite(_Config) ->
}
),
emqx_mgmt_api_test_util:end_suite([
emqx_conf, emqx_authn, emqx_authz, emqx_modules
emqx_conf, emqx_authn, emqx_authz, emqx_telemetry
]),
ok.

2
bin/emqx
View File

@ -612,7 +612,7 @@ check_license() {
set +x
logerr "License not found."
logerr "Please specify one via the EMQX_LICENSE__KEY variable"
logerr "or via license.key in emqx-enterprise.conf."
logerr "or via license.key in emqx.conf."
return 1
fi
}

43
build
View File

@ -131,8 +131,25 @@ make_docs() {
halt(0)."
}
assert_no_compile_time_only_deps() {
:
## arg1 is the profile for which the following args (as app names) should be excluded
assert_no_excluded_deps() {
local profile="$1"
shift 1
if [ "$PROFILE" != "$profile" ]; then
# not currently building the profile which has apps to be excluded
return 0
fi
local rel_dir="_build/$PROFILE/rel/emqx/lib"
local excluded_apps=( "$@" )
local found
for app in "${excluded_apps[@]}"; do
found="$($FIND "$rel_dir" -maxdepth 1 -type d -name "$app-*")"
if [ -n "${found}" ]; then
echo "ERROR: ${app} should not be included in ${PROFILE}"
echo "ERROR: found ${app} in ${rel_dir}"
exit 1
fi
done
}
just_compile() {
@ -150,20 +167,20 @@ make_rel() {
just_compile
# now assemble the release tar
./rebar3 as "$PROFILE" "$release_or_tar"
assert_no_compile_time_only_deps
assert_no_excluded_deps emqx-enterprise emqx_telemetry
}
make_elixir_rel() {
./scripts/pre-compile.sh "$PROFILE"
export_elixir_release_vars "$PROFILE"
# for some reason, this has to be run outside "do"...
mix local.rebar --if-missing --force
# shellcheck disable=SC1010
mix do local.hex --if-missing --force, \
local.rebar rebar3 "${PWD}/rebar3" --if-missing --force, \
deps.get
mix release --overwrite
assert_no_compile_time_only_deps
./scripts/pre-compile.sh "$PROFILE"
export_elixir_release_vars "$PROFILE"
# for some reason, this has to be run outside "do"...
mix local.rebar --if-missing --force
# shellcheck disable=SC1010
mix do local.hex --if-missing --force, \
local.rebar rebar3 "${PWD}/rebar3" --if-missing --force, \
deps.get
mix release --overwrite
assert_no_excluded_deps emqx-enterprise emqx_telemetry
}
## extract previous version .tar.gz files to _build/$PROFILE/rel/emqx before making relup

25
mix.exs
View File

@ -102,11 +102,11 @@ defmodule EMQXUmbrella.MixProject do
end
defp emqx_apps(profile_info, version) do
apps = umbrella_apps() ++ enterprise_apps(profile_info)
apps = umbrella_apps(profile_info) ++ enterprise_apps(profile_info)
set_emqx_app_system_env(apps, profile_info, version)
end
defp umbrella_apps() do
defp umbrella_apps(profile_info) do
enterprise_apps = enterprise_umbrella_apps()
"apps/*"
@ -124,6 +124,15 @@ defmodule EMQXUmbrella.MixProject do
|> elem(0)
|> then(&MapSet.member?(enterprise_apps, &1))
end)
|> Enum.reject(fn {app, _} ->
case profile_info do
%{edition_type: :enterprise} ->
app == :emqx_telemetry
_ ->
false
end
end)
end
defp enterprise_apps(_profile_info = %{edition_type: :enterprise}) do
@ -414,7 +423,9 @@ defmodule EMQXUmbrella.MixProject do
emqx_node_rebalance: :permanent,
emqx_ft: :permanent
],
else: []
else: [
emqx_telemetry: :permanent
]
)
end
@ -553,14 +564,6 @@ defmodule EMQXUmbrella.MixProject do
Path.join(etc, "emqx.conf")
)
if edition_type == :enterprise do
render_template(
"apps/emqx_conf/etc/emqx-enterprise.conf.all",
assigns,
Path.join(etc, "emqx-enterprise.conf")
)
end
render_template(
"rel/emqx_vars",
assigns,

22
rebar.config.erl
View File

@ -485,7 +485,7 @@ relx_apps_per_edition(ee) ->
emqx_ft
];
relx_apps_per_edition(ce) ->
[].
[emqx_telemetry].
relx_overlay(ReleaseType, Edition) ->
[
@ -514,8 +514,8 @@ relx_overlay(ReleaseType, Edition) ->
{copy, "bin/nodetool", "bin/nodetool-{{release_version}}"}
] ++ etc_overlay(ReleaseType, Edition).
etc_overlay(ReleaseType, Edition) ->
Templates = emqx_etc_overlay(ReleaseType, Edition),
etc_overlay(ReleaseType, _Edition) ->
Templates = emqx_etc_overlay(ReleaseType),
[
{mkdir, "etc/"},
{copy, "{{base_dir}}/lib/emqx/etc/certs", "etc/"},
@ -529,24 +529,16 @@ etc_overlay(ReleaseType, Edition) ->
Templates
).
emqx_etc_overlay(ReleaseType, Edition) ->
emqx_etc_overlay(ReleaseType) ->
emqx_etc_overlay_per_rel(ReleaseType) ++
emqx_etc_overlay_per_edition(Edition) ++
emqx_etc_overlay_common().
emqx_etc_overlay().
emqx_etc_overlay_per_rel(cloud) ->
[{"{{base_dir}}/lib/emqx/etc/vm.args.cloud", "etc/vm.args"}].
emqx_etc_overlay_common() ->
[{"{{base_dir}}/lib/emqx/etc/ssl_dist.conf", "etc/ssl_dist.conf"}].
emqx_etc_overlay_per_edition(ce) ->
emqx_etc_overlay() ->
[
{"{{base_dir}}/lib/emqx_conf/etc/emqx.conf.all", "etc/emqx.conf"}
];
emqx_etc_overlay_per_edition(ee) ->
[
{"{{base_dir}}/lib/emqx_conf/etc/emqx-enterprise.conf.all", "etc/emqx-enterprise.conf"},
{"{{base_dir}}/lib/emqx/etc/ssl_dist.conf", "etc/ssl_dist.conf"},
{"{{base_dir}}/lib/emqx_conf/etc/emqx.conf.all", "etc/emqx.conf"}
].

8
rel/i18n/emqx_telemetry_schema.hocon Normal file
View File

@ -0,0 +1,8 @@
emqx_telemetry_schema {
telemetry_root_doc.desc:
"""Configure telemetry data report from this EMQX node to EMQ's telemetry data collection server.
See https://www.emqx.io/docs/en/v5.0/telemetry/telemetry.html for more details."""
enable.desc:
"""Set to `false` disable telemetry data report"""
}

8
rel/i18n/zh/emqx_telemetry_schema.hocon Normal file
View File

@ -0,0 +1,8 @@
emqx_telemetry_schema {
telemetry_root_doc.desc:
"""配置 EMQX 节点向EMQ 的遥测服务器发送要测数据。
详情请参考 https://www.emqx.io/docs/zh/v5.0/telemetry/telemetry.html。"""
enable.desc:
"""设置为 `false` 可以关闭数据发送。"""
}

22
scripts/merge-config.escript
View File

@ -14,35 +14,15 @@
main(_) ->
{ok, BaseConf} = file:read_file("apps/emqx_conf/etc/emqx_conf.conf"),
Cfgs = get_all_cfgs("apps/"),
IsEnterprise = is_enterprise(),
Enterprise =
case IsEnterprise of
false -> [];
true -> [io_lib:nl(), "include emqx-enterprise.conf", io_lib:nl()]
end,
Conf = [
merge(BaseConf, Cfgs),
io_lib:nl(),
Enterprise
io_lib:nl()
],
ok = file:write_file("apps/emqx_conf/etc/emqx.conf.all", Conf),
case IsEnterprise of
true ->
EnterpriseCfgs = get_all_cfgs("lib-ee"),
EnterpriseConf = merge(<<"">>, EnterpriseCfgs),
ok = file:write_file("apps/emqx_conf/etc/emqx-enterprise.conf.all", EnterpriseConf);
false ->
ok
end,
merge_desc_files_per_lang("en"),
%% TODO: remove this when we have zh translation moved to dashboard package
merge_desc_files_per_lang("zh").
is_enterprise() ->
Profile = os:getenv("PROFILE", "emqx"),
nomatch =/= string:find(Profile, "enterprise").
merge(BaseConf, Cfgs) ->
Confs = [BaseConf | lists:map(fun read_conf/1, Cfgs)],
infix(lists:filter(fun(I) -> iolist_size(I) > 0 end, Confs), [io_lib:nl(), io_lib:nl()]).

1
scripts/spellcheck/dicts/emqx.txt
View File

@ -23,6 +23,7 @@ DevOps
Dialyzer
Diffie
EIP
EMQ
EMQX
EPMD
ERL

2
scripts/spellcheck/spellcheck.sh
View File

@ -7,7 +7,7 @@ cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/../.."
PROJ_ROOT="$(pwd)"
if [ -z "${1:-}" ]; then
SCHEMA="${PROJ_ROOT}/_build/emqx/lib/emqx_dashboard/priv/www/static/schema.json"
SCHEMA="${PROJ_ROOT}/_build/docgen/emqx/schema-en.json"
else
SCHEMA="$(realpath "$1")"
fi

1
scripts/test/influx/emqx.conf
View File

@ -90,5 +90,4 @@ authorization {
]
}
include emqx-enterprise.conf
include influx-bridge.conf