feat(license): copy license file to cluster when updating it

This commit is contained in:
Thales Macedo Garitezi 2022-07-28 17:27:50 -03:00
parent 80d35feb33
commit f8a1bd0715
6 changed files with 343 additions and 33 deletions

View File

@ -595,6 +595,7 @@ setup_node(Node, Opts) when is_map(Opts) ->
EnvHandler = maps:get(env_handler, Opts, fun(_) -> ok end), EnvHandler = maps:get(env_handler, Opts, fun(_) -> ok end),
ConfigureGenRpc = maps:get(configure_gen_rpc, Opts, true), ConfigureGenRpc = maps:get(configure_gen_rpc, Opts, true),
LoadSchema = maps:get(load_schema, Opts, true), LoadSchema = maps:get(load_schema, Opts, true),
SchemaMod = maps:get(schema_mod, Opts, emqx_schema),
LoadApps = maps:get(load_apps, Opts, [gen_rpc, emqx, ekka, mria] ++ Apps), LoadApps = maps:get(load_apps, Opts, [gen_rpc, emqx, ekka, mria] ++ Apps),
Env = maps:get(env, Opts, []), Env = maps:get(env, Opts, []),
Conf = maps:get(conf, Opts, []), Conf = maps:get(conf, Opts, []),
@ -630,7 +631,7 @@ setup_node(Node, Opts) when is_map(Opts) ->
%% Otherwise, configuration get's loaded and all preset env in envhandler is lost %% Otherwise, configuration get's loaded and all preset env in envhandler is lost
LoadSchema andalso LoadSchema andalso
begin begin
emqx_config:init_load(emqx_schema), emqx_config:init_load(SchemaMod),
application:set_env(emqx, init_config_load_done, true) application:set_env(emqx, init_config_load_done, true)
end, end,

View File

@ -22,7 +22,9 @@
read_license/0, read_license/0,
read_license/1, read_license/1,
update_file/1, update_file/1,
update_key/1 update_key/1,
license_dir/0,
save_and_backup_license/1
]). ]).
-define(CONF_KEY_PATH, [license]). -define(CONF_KEY_PATH, [license]).
@ -54,15 +56,29 @@ unload() ->
emqx_conf:remove_handler(?CONF_KEY_PATH), emqx_conf:remove_handler(?CONF_KEY_PATH),
emqx_license_cli:unload(). emqx_license_cli:unload().
-spec license_dir() -> file:filename().
license_dir() ->
filename:join([emqx:data_dir(), licenses]).
%% Subdirectory relative to data dir.
-spec relative_license_path() -> file:filename().
relative_license_path() ->
filename:join([licenses, "emqx.lic"]).
-spec update_file(binary() | string()) -> -spec update_file(binary() | string()) ->
{ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}.
update_file(Filename) when is_binary(Filename); is_list(Filename) -> update_file(Filename) when is_binary(Filename); is_list(Filename) ->
Result = emqx_conf:update( case file:read_file(Filename) of
?CONF_KEY_PATH, {ok, Contents} ->
{file, Filename}, Result = emqx_conf:update(
#{rawconf_with_defaults => true, override_to => local} ?CONF_KEY_PATH,
), {file, Contents},
handle_config_update_result(Result). #{rawconf_with_defaults => true, override_to => local}
),
handle_config_update_result(Result);
{error, Error} ->
{error, Error}
end.
-spec update_key(binary() | string()) -> -spec update_key(binary() | string()) ->
{ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}.
@ -125,18 +141,14 @@ del_license_hook() ->
_ = emqx_hooks:del('client.connect', {?MODULE, check, []}), _ = emqx_hooks:del('client.connect', {?MODULE, check, []}),
ok. ok.
do_update({file, Filename}, Conf) -> do_update({file, NewContents}, Conf) ->
case file:read_file(Filename) of Res = emqx_license_proto_v2:save_and_backup_license(mria_mnesia:running_nodes(), NewContents),
{ok, Content} -> %% assert
case emqx_license_parser:parse(Content) of true = lists:all(fun(X) -> X =:= {ok, ok} end, Res),
{ok, _License} -> %% Must be relative to the data dir, since different nodes might
maps:remove(<<"key">>, Conf#{<<"type">> => file, <<"file">> => Filename}); %% have different data directories configured...
{error, Reason} -> LicensePath = relative_license_path(),
erlang:throw(Reason) maps:remove(<<"key">>, Conf#{<<"type">> => file, <<"file">> => LicensePath});
end;
{error, Reason} ->
erlang:throw({invalid_license_file, Reason})
end;
do_update({key, Content}, Conf) when is_binary(Content); is_list(Content) -> do_update({key, Content}, Conf) when is_binary(Content); is_list(Content) ->
case emqx_license_parser:parse(Content) of case emqx_license_parser:parse(Content) of
{ok, _License} -> {ok, _License} ->
@ -148,17 +160,61 @@ do_update({key, Content}, Conf) when is_binary(Content); is_list(Content) ->
do_update(_Other, Conf) -> do_update(_Other, Conf) ->
Conf. Conf.
save_and_backup_license(NewLicenseKey) ->
%% Must be relative to the data dir, since different nodes might
%% have different data directories configured...
CurrentLicensePath = filename:join(emqx:data_dir(), relative_license_path()),
LicenseDir = filename:dirname(CurrentLicensePath),
case filelib:ensure_dir(CurrentLicensePath) of
ok -> ok;
{error, EnsureError} -> throw({error_creating_license_dir, EnsureError})
end,
case file:read_file(CurrentLicensePath) of
{ok, NewLicenseKey} ->
%% same contents; nothing to do.
ok;
{ok, _OldContents} ->
Time = calendar:system_time_to_rfc3339(erlang:system_time(second)),
BackupPath = filename:join([
LicenseDir,
"emqx.lic." ++ Time ++ ".backup"
]),
case file:copy(CurrentLicensePath, BackupPath) of
{ok, _} -> ok;
{error, CopyError} -> throw({error_backing_up_license, CopyError})
end,
ok;
{error, enoent} ->
ok;
{error, Error} ->
throw({error_reading_existing_license, Error})
end,
case file:write_file(CurrentLicensePath, NewLicenseKey) of
ok -> ok;
{error, WriteError} -> throw({error_writing_license, WriteError})
end,
ok.
check_max_clients_exceeded(MaxClients) -> check_max_clients_exceeded(MaxClients) ->
emqx_license_resources:connection_count() > MaxClients * 1.1. emqx_license_resources:connection_count() > MaxClients * 1.1.
read_license(#{type := file, file := Filename}) -> read_license(#{type := file, file := Filename}) ->
case file:read_file(Filename) of case file:read_file(Filename) of
{ok, Content} -> emqx_license_parser:parse(Content); {ok, Content} ->
{error, _} = Error -> Error emqx_license_parser:parse(Content);
{error, _} = Error ->
%% Could be a relative path in data folder after update.
FilenameDataDir = filename:join(emqx:data_dir(), Filename),
case file:read_file(FilenameDataDir) of
{ok, Content} -> emqx_license_parser:parse(Content);
_Error -> Error
end
end; end;
read_license(#{type := key, key := Content}) -> read_license(#{type := key, key := Content}) ->
emqx_license_parser:parse(Content). emqx_license_parser:parse(Content).
handle_config_update_result({error, {post_config_update, ?MODULE, Error}}) ->
{error, Error};
handle_config_update_result({error, _} = Error) -> handle_config_update_result({error, _} = Error) ->
Error; Error;
handle_config_update_result({ok, #{post_config_update := #{emqx_license := Result}}}) -> handle_config_update_result({ok, #{post_config_update := #{emqx_license := Result}}}) ->

View File

@ -128,6 +128,6 @@ ensure_timer(#{check_peer_interval := CheckInterval} = State) ->
remote_connection_count() -> remote_connection_count() ->
Nodes = mria_mnesia:running_nodes() -- [node()], Nodes = mria_mnesia:running_nodes() -- [node()],
Results = emqx_license_proto_v1:remote_connection_counts(Nodes), Results = emqx_license_proto_v2:remote_connection_counts(Nodes),
Counts = [Count || {ok, Count} <- Results], Counts = [Count || {ok, Count} <- Results],
lists:sum(Counts). lists:sum(Counts).

View File

@ -0,0 +1,30 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
-module(emqx_license_proto_v2).
-behaviour(emqx_bpapi).
-include_lib("emqx/include/bpapi.hrl").
-export([introduced_in/0]).
-export([
remote_connection_counts/1,
save_and_backup_license/2
]).
-define(TIMEOUT, 500).
-define(BACKUP_TIMEOUT, 15_000).
introduced_in() ->
"5.0.5".
-spec remote_connection_counts(list(node())) -> list({atom(), term()}).
remote_connection_counts(Nodes) ->
erpc:multicall(Nodes, emqx_license_resources, local_connection_count, [], ?TIMEOUT).
-spec save_and_backup_license(list(node()), binary()) -> list({atom(), term()}).
save_and_backup_license(Nodes, NewLicenseKey) ->
erpc:multicall(Nodes, emqx_license, save_and_backup_license, [NewLicenseKey], ?BACKUP_TIMEOUT).

View File

@ -29,11 +29,13 @@ init_per_testcase(Case, Config) ->
{ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000),
set_invalid_license_file(Case), set_invalid_license_file(Case),
Paths = set_override_paths(Case), Paths = set_override_paths(Case),
Paths ++ Config. Config0 = setup_test(Case, Config),
Paths ++ Config0 ++ Config.
end_per_testcase(Case, Config) -> end_per_testcase(Case, Config) ->
restore_valid_license_file(Case), restore_valid_license_file(Case),
clean_overrides(Case, Config), clean_overrides(Case, Config),
teardown_test(Case, Config),
ok. ok.
set_override_paths(TestCase) when set_override_paths(TestCase) when
@ -71,6 +73,114 @@ clean_overrides(TestCase, Config) when
clean_overrides(_TestCase, _Config) -> clean_overrides(_TestCase, _Config) ->
ok. ok.
setup_test(TestCase, Config) when
TestCase =:= t_update_file_cluster_backup
->
DataDir = ?config(data_dir, Config),
{LicenseKey, _License} = mk_license(
[
%% license format version
"220111",
%% license type
"0",
%% customer type
"10",
%% customer name
"Foo",
%% customer email
"contact@foo.com",
%% deplayment name
"bar-deployment",
%% start date
"20220111",
%% days
"100000",
%% max connections
"19"
]
),
Cluster = emqx_common_test_helpers:emqx_cluster(
[core, core],
[
{apps, [emqx_conf, emqx_license]},
{load_schema, false},
{schema_mod, emqx_enterprise_conf_schema},
{env_handler, fun
(emqx) ->
emqx_config:save_schema_mod_and_names(emqx_enterprise_conf_schema),
%% emqx_config:save_schema_mod_and_names(emqx_license_schema),
application:set_env(emqx, boot_modules, []),
application:set_env(
emqx,
data_dir,
filename:join([
DataDir,
TestCase,
node()
])
),
ok;
(emqx_conf) ->
emqx_config:save_schema_mod_and_names(emqx_enterprise_conf_schema),
%% emqx_config:save_schema_mod_and_names(emqx_license_schema),
application:set_env(
emqx,
data_dir,
filename:join([
DataDir,
TestCase,
node()
])
),
ok;
(emqx_license) ->
LicensePath = filename:join(emqx_license:license_dir(), "emqx.lic"),
filelib:ensure_dir(LicensePath),
ok = file:write_file(LicensePath, LicenseKey),
LicConfig = #{type => file, file => LicensePath},
emqx_config:put([license], LicConfig),
RawConfig = #{<<"type">> => file, <<"file">> => LicensePath},
emqx_config:put_raw([<<"license">>], RawConfig),
ok = meck:new(emqx_license, [non_strict, passthrough, no_history, no_link]),
%% meck:expect(emqx_license, read_license, fun() -> {ok, License} end),
meck:expect(
emqx_license_parser,
parse,
fun(X) ->
emqx_license_parser:parse(
X,
emqx_license_test_lib:public_key_pem()
)
end
),
ok;
(_) ->
ok
end}
]
),
Nodes = [emqx_common_test_helpers:start_slave(Name, Opts) || {Name, Opts} <- Cluster],
[{nodes, Nodes}, {cluster, Cluster}, {old_license, LicenseKey}];
setup_test(_TestCase, _Config) ->
[].
teardown_test(TestCase, Config) when
TestCase =:= t_update_file_cluster_backup
->
Nodes = ?config(nodes, Config),
lists:foreach(
fun(N) ->
LicenseDir = erpc:call(N, emqx_license, license_dir, []),
{ok, _} = emqx_common_test_helpers:stop_slave(N),
ok = file:del_dir_r(LicenseDir),
ok
end,
Nodes
),
ok;
teardown_test(_TestCase, _Config) ->
ok.
set_invalid_license_file(t_read_license_from_invalid_file) -> set_invalid_license_file(t_read_license_from_invalid_file) ->
Config = #{type => file, file => "/invalid/file"}, Config = #{type => file, file => "/invalid/file"},
emqx_config:put([license], Config); emqx_config:put([license], Config);
@ -91,13 +201,17 @@ set_special_configs(emqx_license) ->
set_special_configs(_) -> set_special_configs(_) ->
ok. ok.
assert_on_nodes(Nodes, RunFun, CheckFun) ->
Res = [{N, erpc:call(N, RunFun)} || N <- Nodes],
lists:foreach(CheckFun, Res).
%%------------------------------------------------------------------------------ %%------------------------------------------------------------------------------
%% Tests %% Tests
%%------------------------------------------------------------------------------ %%------------------------------------------------------------------------------
t_update_file(_Config) -> t_update_file(_Config) ->
?assertMatch( ?assertMatch(
{error, {invalid_license_file, enoent}}, {error, enoent},
emqx_license:update_file("/unknown/path") emqx_license:update_file("/unknown/path")
), ),
@ -112,6 +226,115 @@ t_update_file(_Config) ->
emqx_license:update_file(emqx_license_test_lib:default_license()) emqx_license:update_file(emqx_license_test_lib:default_license())
). ).
t_update_file_cluster_backup(Config) ->
OldLicenseKey = ?config(old_license, Config),
Nodes = [N1 | _] = ?config(nodes, Config),
%% update the license file for the cluster
{NewLicenseKey, NewDecodedLicense} = mk_license(
[
%% license format version
"220111",
%% license type
"0",
%% customer type
"10",
%% customer name
"Foo",
%% customer email
"contact@foo.com",
%% deplayment name
"bar-deployment",
%% start date
"20220111",
%% days
"100000",
%% max connections
"190"
]
),
NewLicensePath = "tmp_new_license.lic",
ok = file:write_file(NewLicensePath, NewLicenseKey),
{ok, _} = erpc:call(N1, emqx_license, update_file, [NewLicensePath]),
assert_on_nodes(
Nodes,
fun() ->
Conf = emqx_conf:get([license]),
emqx_license:read_license(Conf)
end,
fun({N, Res}) ->
?assertMatch({ok, _}, Res, #{node => N}),
{ok, License} = Res,
?assertEqual(NewDecodedLicense, License, #{node => N})
end
),
assert_on_nodes(
Nodes,
fun() ->
LicenseDir = emqx_license:license_dir(),
file:list_dir(LicenseDir)
end,
fun({N, Res}) ->
?assertMatch({ok, _}, Res, #{node => N}),
{ok, DirContents} = Res,
%% the now current license
?assert(lists:member("emqx.lic", DirContents), #{node => N, dir_contents => DirContents}),
%% the backed up old license
?assert(
lists:any(
fun
("emqx.lic." ++ Suffix) -> lists:suffix(".backup", Suffix);
(_) -> false
end,
DirContents
),
#{node => N, dir_contents => DirContents}
)
end
),
assert_on_nodes(
Nodes,
fun() ->
LicenseDir = emqx_license:license_dir(),
{ok, DirContents} = file:list_dir(LicenseDir),
[BackupLicensePath0] = [
F
|| "emqx.lic." ++ F <- DirContents, lists:suffix(".backup", F)
],
BackupLicensePath = "emqx.lic." ++ BackupLicensePath0,
{ok, BackupLicense} = file:read_file(filename:join(LicenseDir, BackupLicensePath)),
{ok, NewLicense} = file:read_file(filename:join(LicenseDir, "emqx.lic")),
#{
backup => BackupLicense,
new => NewLicense
}
end,
fun({N, #{backup := BackupLicense, new := NewLicense}}) ->
?assertEqual(OldLicenseKey, BackupLicense, #{node => N}),
?assertEqual(NewLicenseKey, NewLicense, #{node => N})
end
),
%% uploading the same license twice should not generate extra backups.
{ok, _} = erpc:call(N1, emqx_license, update_file, [NewLicensePath]),
assert_on_nodes(
Nodes,
fun() ->
LicenseDir = emqx_license:license_dir(),
{ok, DirContents} = file:list_dir(LicenseDir),
[F || "emqx.lic." ++ F <- DirContents, lists:suffix(".backup", F)]
end,
fun({N, Backups}) ->
?assertMatch([_], Backups, #{node => N})
end
),
ok.
t_update_value(_Config) -> t_update_value(_Config) ->
?assertMatch( ?assertMatch(
{error, [_ | _]}, {error, [_ | _]},
@ -132,7 +355,7 @@ t_read_license_from_invalid_file(_Config) ->
). ).
t_check_exceeded(_Config) -> t_check_exceeded(_Config) ->
License = mk_license( {_, License} = mk_license(
[ [
"220111", "220111",
"0", "0",
@ -161,7 +384,7 @@ t_check_exceeded(_Config) ->
). ).
t_check_ok(_Config) -> t_check_ok(_Config) ->
License = mk_license( {_, License} = mk_license(
[ [
"220111", "220111",
"0", "0",
@ -190,7 +413,7 @@ t_check_ok(_Config) ->
). ).
t_check_expired(_Config) -> t_check_expired(_Config) ->
License = mk_license( {_, License} = mk_license(
[ [
"220111", "220111",
%% Official customer %% Official customer
@ -263,4 +486,4 @@ mk_license(Fields) ->
EncodedLicense, EncodedLicense,
emqx_license_test_lib:public_key_pem() emqx_license_test_lib:public_key_pem()
), ),
License. {EncodedLicense, License}.

View File

@ -59,9 +59,9 @@ t_connection_count(_Config) ->
meck:new(emqx_cm, [passthrough]), meck:new(emqx_cm, [passthrough]),
meck:expect(emqx_cm, get_connected_client_count, fun() -> 10 end), meck:expect(emqx_cm, get_connected_client_count, fun() -> 10 end),
meck:new(emqx_license_proto_v1, [passthrough]), meck:new(emqx_license_proto_v2, [passthrough]),
meck:expect( meck:expect(
emqx_license_proto_v1, emqx_license_proto_v2,
remote_connection_counts, remote_connection_counts,
fun(_Nodes) -> fun(_Nodes) ->
[{ok, 5}, {error, some_error}] [{ok, 5}, {error, some_error}]
@ -82,8 +82,8 @@ t_connection_count(_Config) ->
end end
), ),
meck:unload(emqx_license_proto_v1), meck:unload(emqx_license_proto_v2),
meck:unload(emqx_cm). meck:unload(emqx_cm).
t_emqx_license_proto(_Config) -> t_emqx_license_proto(_Config) ->
?assert("5.0.0" =< emqx_license_proto_v1:introduced_in()). ?assert("5.0.0" =< emqx_license_proto_v2:introduced_in()).