diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index d0ee99d6d..b3292ded3 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -595,6 +595,7 @@ setup_node(Node, Opts) when is_map(Opts) -> EnvHandler = maps:get(env_handler, Opts, fun(_) -> ok end), ConfigureGenRpc = maps:get(configure_gen_rpc, Opts, true), LoadSchema = maps:get(load_schema, Opts, true), + SchemaMod = maps:get(schema_mod, Opts, emqx_schema), LoadApps = maps:get(load_apps, Opts, [gen_rpc, emqx, ekka, mria] ++ Apps), Env = maps:get(env, Opts, []), Conf = maps:get(conf, Opts, []), @@ -630,7 +631,7 @@ setup_node(Node, Opts) when is_map(Opts) -> %% Otherwise, configuration get's loaded and all preset env in envhandler is lost LoadSchema andalso begin - emqx_config:init_load(emqx_schema), + emqx_config:init_load(SchemaMod), application:set_env(emqx, init_config_load_done, true) end, diff --git a/lib-ee/emqx_license/src/emqx_license.erl b/lib-ee/emqx_license/src/emqx_license.erl index e7a0cce48..24b2cc709 100644 --- a/lib-ee/emqx_license/src/emqx_license.erl +++ b/lib-ee/emqx_license/src/emqx_license.erl @@ -22,7 +22,9 @@ read_license/0, read_license/1, update_file/1, - update_key/1 + update_key/1, + license_dir/0, + save_and_backup_license/1 ]). -define(CONF_KEY_PATH, [license]). @@ -54,15 +56,29 @@ unload() -> emqx_conf:remove_handler(?CONF_KEY_PATH), emqx_license_cli:unload(). +-spec license_dir() -> file:filename(). +license_dir() -> + filename:join([emqx:data_dir(), licenses]). + +%% Subdirectory relative to data dir. +-spec relative_license_path() -> file:filename(). +relative_license_path() -> + filename:join([licenses, "emqx.lic"]). + -spec update_file(binary() | string()) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. update_file(Filename) when is_binary(Filename); is_list(Filename) -> - Result = emqx_conf:update( - ?CONF_KEY_PATH, - {file, Filename}, - #{rawconf_with_defaults => true, override_to => local} - ), - handle_config_update_result(Result). + case file:read_file(Filename) of + {ok, Contents} -> + Result = emqx_conf:update( + ?CONF_KEY_PATH, + {file, Contents}, + #{rawconf_with_defaults => true, override_to => local} + ), + handle_config_update_result(Result); + {error, Error} -> + {error, Error} + end. -spec update_key(binary() | string()) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. @@ -125,18 +141,14 @@ del_license_hook() -> _ = emqx_hooks:del('client.connect', {?MODULE, check, []}), ok. -do_update({file, Filename}, Conf) -> - case file:read_file(Filename) of - {ok, Content} -> - case emqx_license_parser:parse(Content) of - {ok, _License} -> - maps:remove(<<"key">>, Conf#{<<"type">> => file, <<"file">> => Filename}); - {error, Reason} -> - erlang:throw(Reason) - end; - {error, Reason} -> - erlang:throw({invalid_license_file, Reason}) - end; +do_update({file, NewContents}, Conf) -> + Res = emqx_license_proto_v2:save_and_backup_license(mria_mnesia:running_nodes(), NewContents), + %% assert + true = lists:all(fun(X) -> X =:= {ok, ok} end, Res), + %% Must be relative to the data dir, since different nodes might + %% have different data directories configured... + LicensePath = relative_license_path(), + maps:remove(<<"key">>, Conf#{<<"type">> => file, <<"file">> => LicensePath}); do_update({key, Content}, Conf) when is_binary(Content); is_list(Content) -> case emqx_license_parser:parse(Content) of {ok, _License} -> @@ -148,17 +160,61 @@ do_update({key, Content}, Conf) when is_binary(Content); is_list(Content) -> do_update(_Other, Conf) -> Conf. +save_and_backup_license(NewLicenseKey) -> + %% Must be relative to the data dir, since different nodes might + %% have different data directories configured... + CurrentLicensePath = filename:join(emqx:data_dir(), relative_license_path()), + LicenseDir = filename:dirname(CurrentLicensePath), + case filelib:ensure_dir(CurrentLicensePath) of + ok -> ok; + {error, EnsureError} -> throw({error_creating_license_dir, EnsureError}) + end, + case file:read_file(CurrentLicensePath) of + {ok, NewLicenseKey} -> + %% same contents; nothing to do. + ok; + {ok, _OldContents} -> + Time = calendar:system_time_to_rfc3339(erlang:system_time(second)), + BackupPath = filename:join([ + LicenseDir, + "emqx.lic." ++ Time ++ ".backup" + ]), + case file:copy(CurrentLicensePath, BackupPath) of + {ok, _} -> ok; + {error, CopyError} -> throw({error_backing_up_license, CopyError}) + end, + ok; + {error, enoent} -> + ok; + {error, Error} -> + throw({error_reading_existing_license, Error}) + end, + case file:write_file(CurrentLicensePath, NewLicenseKey) of + ok -> ok; + {error, WriteError} -> throw({error_writing_license, WriteError}) + end, + ok. + check_max_clients_exceeded(MaxClients) -> emqx_license_resources:connection_count() > MaxClients * 1.1. read_license(#{type := file, file := Filename}) -> case file:read_file(Filename) of - {ok, Content} -> emqx_license_parser:parse(Content); - {error, _} = Error -> Error + {ok, Content} -> + emqx_license_parser:parse(Content); + {error, _} = Error -> + %% Could be a relative path in data folder after update. + FilenameDataDir = filename:join(emqx:data_dir(), Filename), + case file:read_file(FilenameDataDir) of + {ok, Content} -> emqx_license_parser:parse(Content); + _Error -> Error + end end; read_license(#{type := key, key := Content}) -> emqx_license_parser:parse(Content). +handle_config_update_result({error, {post_config_update, ?MODULE, Error}}) -> + {error, Error}; handle_config_update_result({error, _} = Error) -> Error; handle_config_update_result({ok, #{post_config_update := #{emqx_license := Result}}}) -> diff --git a/lib-ee/emqx_license/src/emqx_license_resources.erl b/lib-ee/emqx_license/src/emqx_license_resources.erl index 551601923..4aaf853a3 100644 --- a/lib-ee/emqx_license/src/emqx_license_resources.erl +++ b/lib-ee/emqx_license/src/emqx_license_resources.erl @@ -128,6 +128,6 @@ ensure_timer(#{check_peer_interval := CheckInterval} = State) -> remote_connection_count() -> Nodes = mria_mnesia:running_nodes() -- [node()], - Results = emqx_license_proto_v1:remote_connection_counts(Nodes), + Results = emqx_license_proto_v2:remote_connection_counts(Nodes), Counts = [Count || {ok, Count} <- Results], lists:sum(Counts). diff --git a/lib-ee/emqx_license/src/proto/emqx_license_proto_v2.erl b/lib-ee/emqx_license/src/proto/emqx_license_proto_v2.erl new file mode 100644 index 000000000..6af1cea77 --- /dev/null +++ b/lib-ee/emqx_license/src/proto/emqx_license_proto_v2.erl @@ -0,0 +1,30 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_license_proto_v2). + +-behaviour(emqx_bpapi). + +-include_lib("emqx/include/bpapi.hrl"). + +-export([introduced_in/0]). + +-export([ + remote_connection_counts/1, + save_and_backup_license/2 +]). + +-define(TIMEOUT, 500). +-define(BACKUP_TIMEOUT, 15_000). + +introduced_in() -> + "5.0.5". + +-spec remote_connection_counts(list(node())) -> list({atom(), term()}). +remote_connection_counts(Nodes) -> + erpc:multicall(Nodes, emqx_license_resources, local_connection_count, [], ?TIMEOUT). + +-spec save_and_backup_license(list(node()), binary()) -> list({atom(), term()}). +save_and_backup_license(Nodes, NewLicenseKey) -> + erpc:multicall(Nodes, emqx_license, save_and_backup_license, [NewLicenseKey], ?BACKUP_TIMEOUT). diff --git a/lib-ee/emqx_license/test/emqx_license_SUITE.erl b/lib-ee/emqx_license/test/emqx_license_SUITE.erl index ba898b360..a648595d2 100644 --- a/lib-ee/emqx_license/test/emqx_license_SUITE.erl +++ b/lib-ee/emqx_license/test/emqx_license_SUITE.erl @@ -29,11 +29,13 @@ init_per_testcase(Case, Config) -> {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), set_invalid_license_file(Case), Paths = set_override_paths(Case), - Paths ++ Config. + Config0 = setup_test(Case, Config), + Paths ++ Config0 ++ Config. end_per_testcase(Case, Config) -> restore_valid_license_file(Case), clean_overrides(Case, Config), + teardown_test(Case, Config), ok. set_override_paths(TestCase) when @@ -71,6 +73,114 @@ clean_overrides(TestCase, Config) when clean_overrides(_TestCase, _Config) -> ok. +setup_test(TestCase, Config) when + TestCase =:= t_update_file_cluster_backup +-> + DataDir = ?config(data_dir, Config), + {LicenseKey, _License} = mk_license( + [ + %% license format version + "220111", + %% license type + "0", + %% customer type + "10", + %% customer name + "Foo", + %% customer email + "contact@foo.com", + %% deplayment name + "bar-deployment", + %% start date + "20220111", + %% days + "100000", + %% max connections + "19" + ] + ), + Cluster = emqx_common_test_helpers:emqx_cluster( + [core, core], + [ + {apps, [emqx_conf, emqx_license]}, + {load_schema, false}, + {schema_mod, emqx_enterprise_conf_schema}, + {env_handler, fun + (emqx) -> + emqx_config:save_schema_mod_and_names(emqx_enterprise_conf_schema), + %% emqx_config:save_schema_mod_and_names(emqx_license_schema), + application:set_env(emqx, boot_modules, []), + application:set_env( + emqx, + data_dir, + filename:join([ + DataDir, + TestCase, + node() + ]) + ), + ok; + (emqx_conf) -> + emqx_config:save_schema_mod_and_names(emqx_enterprise_conf_schema), + %% emqx_config:save_schema_mod_and_names(emqx_license_schema), + application:set_env( + emqx, + data_dir, + filename:join([ + DataDir, + TestCase, + node() + ]) + ), + ok; + (emqx_license) -> + LicensePath = filename:join(emqx_license:license_dir(), "emqx.lic"), + filelib:ensure_dir(LicensePath), + ok = file:write_file(LicensePath, LicenseKey), + LicConfig = #{type => file, file => LicensePath}, + emqx_config:put([license], LicConfig), + RawConfig = #{<<"type">> => file, <<"file">> => LicensePath}, + emqx_config:put_raw([<<"license">>], RawConfig), + ok = meck:new(emqx_license, [non_strict, passthrough, no_history, no_link]), + %% meck:expect(emqx_license, read_license, fun() -> {ok, License} end), + meck:expect( + emqx_license_parser, + parse, + fun(X) -> + emqx_license_parser:parse( + X, + emqx_license_test_lib:public_key_pem() + ) + end + ), + ok; + (_) -> + ok + end} + ] + ), + Nodes = [emqx_common_test_helpers:start_slave(Name, Opts) || {Name, Opts} <- Cluster], + [{nodes, Nodes}, {cluster, Cluster}, {old_license, LicenseKey}]; +setup_test(_TestCase, _Config) -> + []. + +teardown_test(TestCase, Config) when + TestCase =:= t_update_file_cluster_backup +-> + Nodes = ?config(nodes, Config), + lists:foreach( + fun(N) -> + LicenseDir = erpc:call(N, emqx_license, license_dir, []), + {ok, _} = emqx_common_test_helpers:stop_slave(N), + ok = file:del_dir_r(LicenseDir), + ok + end, + Nodes + ), + ok; +teardown_test(_TestCase, _Config) -> + ok. + set_invalid_license_file(t_read_license_from_invalid_file) -> Config = #{type => file, file => "/invalid/file"}, emqx_config:put([license], Config); @@ -91,13 +201,17 @@ set_special_configs(emqx_license) -> set_special_configs(_) -> ok. +assert_on_nodes(Nodes, RunFun, CheckFun) -> + Res = [{N, erpc:call(N, RunFun)} || N <- Nodes], + lists:foreach(CheckFun, Res). + %%------------------------------------------------------------------------------ %% Tests %%------------------------------------------------------------------------------ t_update_file(_Config) -> ?assertMatch( - {error, {invalid_license_file, enoent}}, + {error, enoent}, emqx_license:update_file("/unknown/path") ), @@ -112,6 +226,115 @@ t_update_file(_Config) -> emqx_license:update_file(emqx_license_test_lib:default_license()) ). +t_update_file_cluster_backup(Config) -> + OldLicenseKey = ?config(old_license, Config), + Nodes = [N1 | _] = ?config(nodes, Config), + + %% update the license file for the cluster + {NewLicenseKey, NewDecodedLicense} = mk_license( + [ + %% license format version + "220111", + %% license type + "0", + %% customer type + "10", + %% customer name + "Foo", + %% customer email + "contact@foo.com", + %% deplayment name + "bar-deployment", + %% start date + "20220111", + %% days + "100000", + %% max connections + "190" + ] + ), + NewLicensePath = "tmp_new_license.lic", + ok = file:write_file(NewLicensePath, NewLicenseKey), + {ok, _} = erpc:call(N1, emqx_license, update_file, [NewLicensePath]), + + assert_on_nodes( + Nodes, + fun() -> + Conf = emqx_conf:get([license]), + emqx_license:read_license(Conf) + end, + fun({N, Res}) -> + ?assertMatch({ok, _}, Res, #{node => N}), + {ok, License} = Res, + ?assertEqual(NewDecodedLicense, License, #{node => N}) + end + ), + + assert_on_nodes( + Nodes, + fun() -> + LicenseDir = emqx_license:license_dir(), + file:list_dir(LicenseDir) + end, + fun({N, Res}) -> + ?assertMatch({ok, _}, Res, #{node => N}), + {ok, DirContents} = Res, + %% the now current license + ?assert(lists:member("emqx.lic", DirContents), #{node => N, dir_contents => DirContents}), + %% the backed up old license + ?assert( + lists:any( + fun + ("emqx.lic." ++ Suffix) -> lists:suffix(".backup", Suffix); + (_) -> false + end, + DirContents + ), + #{node => N, dir_contents => DirContents} + ) + end + ), + + assert_on_nodes( + Nodes, + fun() -> + LicenseDir = emqx_license:license_dir(), + {ok, DirContents} = file:list_dir(LicenseDir), + [BackupLicensePath0] = [ + F + || "emqx.lic." ++ F <- DirContents, lists:suffix(".backup", F) + ], + BackupLicensePath = "emqx.lic." ++ BackupLicensePath0, + {ok, BackupLicense} = file:read_file(filename:join(LicenseDir, BackupLicensePath)), + {ok, NewLicense} = file:read_file(filename:join(LicenseDir, "emqx.lic")), + #{ + backup => BackupLicense, + new => NewLicense + } + end, + fun({N, #{backup := BackupLicense, new := NewLicense}}) -> + ?assertEqual(OldLicenseKey, BackupLicense, #{node => N}), + ?assertEqual(NewLicenseKey, NewLicense, #{node => N}) + end + ), + + %% uploading the same license twice should not generate extra backups. + {ok, _} = erpc:call(N1, emqx_license, update_file, [NewLicensePath]), + + assert_on_nodes( + Nodes, + fun() -> + LicenseDir = emqx_license:license_dir(), + {ok, DirContents} = file:list_dir(LicenseDir), + [F || "emqx.lic." ++ F <- DirContents, lists:suffix(".backup", F)] + end, + fun({N, Backups}) -> + ?assertMatch([_], Backups, #{node => N}) + end + ), + + ok. + t_update_value(_Config) -> ?assertMatch( {error, [_ | _]}, @@ -132,7 +355,7 @@ t_read_license_from_invalid_file(_Config) -> ). t_check_exceeded(_Config) -> - License = mk_license( + {_, License} = mk_license( [ "220111", "0", @@ -161,7 +384,7 @@ t_check_exceeded(_Config) -> ). t_check_ok(_Config) -> - License = mk_license( + {_, License} = mk_license( [ "220111", "0", @@ -190,7 +413,7 @@ t_check_ok(_Config) -> ). t_check_expired(_Config) -> - License = mk_license( + {_, License} = mk_license( [ "220111", %% Official customer @@ -263,4 +486,4 @@ mk_license(Fields) -> EncodedLicense, emqx_license_test_lib:public_key_pem() ), - License. + {EncodedLicense, License}. diff --git a/lib-ee/emqx_license/test/emqx_license_resources_SUITE.erl b/lib-ee/emqx_license/test/emqx_license_resources_SUITE.erl index fc4e19e5b..84a63e611 100644 --- a/lib-ee/emqx_license/test/emqx_license_resources_SUITE.erl +++ b/lib-ee/emqx_license/test/emqx_license_resources_SUITE.erl @@ -59,9 +59,9 @@ t_connection_count(_Config) -> meck:new(emqx_cm, [passthrough]), meck:expect(emqx_cm, get_connected_client_count, fun() -> 10 end), - meck:new(emqx_license_proto_v1, [passthrough]), + meck:new(emqx_license_proto_v2, [passthrough]), meck:expect( - emqx_license_proto_v1, + emqx_license_proto_v2, remote_connection_counts, fun(_Nodes) -> [{ok, 5}, {error, some_error}] @@ -82,8 +82,8 @@ t_connection_count(_Config) -> end ), - meck:unload(emqx_license_proto_v1), + meck:unload(emqx_license_proto_v2), meck:unload(emqx_cm). t_emqx_license_proto(_Config) -> - ?assert("5.0.0" =< emqx_license_proto_v1:introduced_in()). + ?assert("5.0.0" =< emqx_license_proto_v2:introduced_in()).