From 63d3a7b52506d5462275be6fb8f51d41dbca019e Mon Sep 17 00:00:00 2001 From: zhouzb Date: Tue, 14 Sep 2021 13:38:09 +0800 Subject: [PATCH] feat(upload certs): save certs to file --- apps/emqx/src/emqx_authentication.erl | 106 +++++++++++++++++- apps/emqx/test/emqx_authentication_SUITE.erl | 53 ++++++++- apps/emqx_authn/src/emqx_authn_api.erl | 3 + .../src/emqx_connector_schema_lib.erl | 6 +- 4 files changed, 158 insertions(+), 10 deletions(-) diff --git a/apps/emqx/src/emqx_authentication.erl b/apps/emqx/src/emqx_authentication.erl index 2e53d85eb..388f107e7 100644 --- a/apps/emqx/src/emqx_authentication.erl +++ b/apps/emqx/src/emqx_authentication.erl @@ -73,6 +73,11 @@ , code_change/3 ]). +-ifdef(TEST). +-compile(export_all). +-compile(nowarn_export_all). +-endif. + -define(CHAINS_TAB, emqx_authn_chains). -define(VER_1, <<"1">>). @@ -193,20 +198,31 @@ pre_config_update(UpdateReq, OldConfig) -> end. do_pre_config_update({create_authenticator, _ChainName, Config}, OldConfig) -> - {ok, OldConfig ++ [Config]}; + try convert_cert_options(Config) of + NConfig -> + {ok, OldConfig ++ [NConfig]} + catch + error:{convert_cert_option, _} = Reason -> + {error, Reason} + end; do_pre_config_update({delete_authenticator, _ChainName, AuthenticatorID}, OldConfig) -> NewConfig = lists:filter(fun(OldConfig0) -> AuthenticatorID =/= generate_id(OldConfig0) end, OldConfig), {ok, NewConfig}; do_pre_config_update({update_authenticator, _ChainName, AuthenticatorID, Config}, OldConfig) -> - NewConfig = lists:map(fun(OldConfig0) -> + try lists:map(fun(OldConfig0) -> case AuthenticatorID =:= generate_id(OldConfig0) of - true -> maps:merge(OldConfig0, Config); + true -> convert_cert_options(Config, OldConfig0); false -> OldConfig0 end - end, OldConfig), - {ok, NewConfig}; + end, OldConfig) of + NewConfig -> + {ok, NewConfig} + catch + error:{convert_cert_option, _} = Reason -> + {error, Reason} + end; do_pre_config_update({move_authenticator, _ChainName, AuthenticatorID, Position}, OldConfig) -> case split_by_id(AuthenticatorID, OldConfig) of {error, Reason} -> {error, Reason}; @@ -600,6 +616,83 @@ reply(Reply, State) -> %% Internal functions %%------------------------------------------------------------------------------ +convert_cert_options(Config) -> + Keys = maps:keys(filter_empty(maps:with([<<"certfile">>, <<"keyfile">>, <<"cacertfile">>], Config))), + lists:foldl(fun(Key, Acc) -> + convert_cert_option(Key, Acc) + end, Config, Keys). + +convert_cert_options(NewConfig, OldConfig) -> + Keys = [<<"cacertfile">>, <<"certfile">>, <<"keyfile">>], + NewCerts = maps:with(Keys, NewConfig), + OldCerts = maps:fold(fun(K, V, Acc) -> + {ok, Bin} = file:read_file(V), + Acc#{K => Bin} + end, #{}, maps:with(Keys, OldConfig)), + Diff = diff_certs(NewCerts, OldCerts), + lists:foldl(fun({identical, K}, Acc) -> + Acc#{K => maps:get(K, OldConfig)}; + ({T, K}, Acc) when T =:= added orelse T =:= changed -> + convert_cert_option(K, Acc) + end, NewConfig, Diff). + +convert_cert_option(Key, Config) -> + PemBin = maps:get(Key, Config), + case public_key:pem_decode(PemBin) =/= [] of + true -> + Filename = to_bin(filename:join([emqx:get_config([node, data_dir]), "certs/authn", generate_filename(Key)])), + case filelib:ensure_dir(Filename) of + ok -> + case file:write_file(Filename, PemBin) of + ok -> Config#{Key => Filename}; + {error, Reason} -> error({convert_cert_option, {write_file, Reason}}) + end; + {error, Reason} -> + error({convert_cert_option, {ensure_dir, Reason}}) + end; + false -> + error({convert_cert_option, invalid_certificate}) + end. + +generate_filename(Key) -> + Prefix = case Key of + <<"keyfile">> -> "key-"; + <<"certfile">> -> "cert-"; + <<"cacertfile">> -> "cacert-" + end, + Prefix ++ emqx_plugin_libs_id:gen() ++ ".pem". + +filter_empty(L) when is_list(L) -> + [I || I <- L, I =/= "" andalso I =/= undefined]; +filter_empty(M) when is_map(M) -> + maps:from_list(filter_empty(maps:to_list(M))). + +diff_certs(NewCerts0, OldCerts0) -> + NewCerts = filter_empty(NewCerts0), + OldCerts = filter_empty(OldCerts0), + Diff = lists:foldl(fun({OldK, OldPem}, Acc) -> + case maps:find(OldK, NewCerts) of + error -> + Acc; + {ok, NewPem} -> + case diff_cert(NewPem, OldPem) of + true -> + [{changed, OldK} | Acc]; + false -> + [{identical, OldK} | Acc] + end + end + end, + [], maps:to_list(OldCerts)), + Added = [{added, K} || K <- maps:keys(maps:without(maps:keys(OldCerts), NewCerts))], + Diff ++ Added. + +diff_cert(Pem1, Pem2) -> + cal_md5_for_cert(Pem1) =/= cal_md5_for_cert(Pem2). + +cal_md5_for_cert(Pem) -> + crypto:hash(md5, term_to_binary(public_key:pem_decode(Pem))). + split_by_id(ID, AuthenticatorsConfig) -> case lists:foldl( fun(C, {P1, P2, F0}) -> @@ -777,3 +870,6 @@ to_list(M) when is_map(M) -> [M]; to_list(L) when is_list(L) -> L. + +to_bin(B) when is_binary(B) -> B; +to_bin(L) when is_list(L) -> list_to_binary(L). diff --git a/apps/emqx/test/emqx_authentication_SUITE.erl b/apps/emqx/test/emqx_authentication_SUITE.erl index 001a4b40e..4af54f842 100644 --- a/apps/emqx/test/emqx_authentication_SUITE.erl +++ b/apps/emqx/test/emqx_authentication_SUITE.erl @@ -92,6 +92,19 @@ end_per_suite(_) -> emqx_ct_helpers:stop_apps([]), ok. +init_per_testcase(_, Config) -> + meck:new(emqx, [non_strict, passthrough, no_history, no_link]), + meck:expect(emqx, get_config, fun([node, data_dir]) -> + {data_dir, Data} = lists:keyfind(data_dir, 1, Config), + Data; + (C) -> meck:passthrough([C]) + end), + Config. + +end_per_testcase(_, _Config) -> + meck:unload(emqx), + ok. + t_chain(_) -> % CRUD of authentication chain ChainName = 'test', @@ -203,7 +216,7 @@ t_update_config(_) -> ?assertMatch({ok, _}, update_config([authentication], {create_authenticator, Global, AuthenticatorConfig2})), ?assertMatch({ok, #{id := ID2, state := #{mark := 1}}}, ?AUTHN:lookup_authenticator(Global, ID2)), - ?assertMatch({ok, _}, update_config([authentication], {update_authenticator, Global, ID1, #{}})), + ?assertMatch({ok, _}, update_config([authentication], {update_authenticator, Global, ID1, AuthenticatorConfig1#{enable => false}})), ?assertMatch({ok, #{id := ID1, state := #{mark := 2}}}, ?AUTHN:lookup_authenticator(Global, ID1)), ?assertMatch({ok, _}, update_config([authentication], {move_authenticator, Global, ID2, top})), @@ -220,7 +233,7 @@ t_update_config(_) -> ?assertMatch({ok, _}, update_config(ConfKeyPath, {create_authenticator, ListenerID, AuthenticatorConfig2})), ?assertMatch({ok, #{id := ID2, state := #{mark := 1}}}, ?AUTHN:lookup_authenticator(ListenerID, ID2)), - ?assertMatch({ok, _}, update_config(ConfKeyPath, {update_authenticator, ListenerID, ID1, #{}})), + ?assertMatch({ok, _}, update_config(ConfKeyPath, {update_authenticator, ListenerID, ID1, AuthenticatorConfig1#{enable => false}})), ?assertMatch({ok, #{id := ID1, state := #{mark := 2}}}, ?AUTHN:lookup_authenticator(ListenerID, ID1)), ?assertMatch({ok, _}, update_config(ConfKeyPath, {move_authenticator, ListenerID, ID2, top})), @@ -234,5 +247,41 @@ t_update_config(_) -> ?AUTHN:remove_provider(AuthNType2), ok. +t_convert_cert_options(_) -> + Certs = certs([ {<<"keyfile">>, "key.pem"} + , {<<"certfile">>, "cert.pem"} + , {<<"cacertfile">>, "cacert.pem"} + ]), + NCerts = ?AUTHN:convert_cert_options(Certs), + ?assertEqual(false, diff_cert(maps:get(<<"keyfile">>, NCerts), maps:get(<<"keyfile">>, Certs))), + + Certs2 = certs([ {<<"keyfile">>, "key.pem"} + , {<<"certfile">>, "cert.pem"} + ]), + NCerts2 = ?AUTHN:convert_cert_options(Certs2, NCerts), + ?assertEqual(false, diff_cert(maps:get(<<"keyfile">>, NCerts2), maps:get(<<"keyfile">>, Certs2))), + ?assertEqual(maps:get(<<"keyfile">>, NCerts), maps:get(<<"keyfile">>, NCerts2)), + ?assertEqual(maps:get(<<"certfile">>, NCerts), maps:get(<<"certfile">>, NCerts2)), + + Certs3 = certs([ {<<"keyfile">>, "client-key.pem"} + , {<<"certfile">>, "client-cert.pem"} + , {<<"cacertfile">>, "cacert.pem"} + ]), + NCerts3 = ?AUTHN:convert_cert_options(Certs3, NCerts2), + ?assertEqual(false, diff_cert(maps:get(<<"keyfile">>, NCerts3), maps:get(<<"keyfile">>, Certs3))), + ?assertNotEqual(maps:get(<<"keyfile">>, NCerts2), maps:get(<<"keyfile">>, NCerts3)), + ?assertNotEqual(maps:get(<<"certfile">>, NCerts2), maps:get(<<"certfile">>, NCerts3)). + update_config(Path, ConfigRequest) -> emqx:update_config(Path, ConfigRequest, #{rawconf_with_defaults => true}). + +certs(Certs) -> + CertsPath = emqx_ct_helpers:deps_path(emqx, "etc/certs"), + lists:foldl(fun({Key, Filename}, Acc) -> + {ok, Bin} = file:read_file(filename:join([CertsPath, Filename])), + Acc#{Key => Bin} + end, #{}, Certs). + +diff_cert(CertFile, CertPem2) -> + {ok, CertPem1} = file:read_file(CertFile), + ?AUTHN:diff_cert(CertPem1, CertPem2). \ No newline at end of file diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index e492100ee..59897b51a 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -1835,8 +1835,11 @@ find_listener(ListenerID) -> {ok, {Type, Name}} end. +% convert_tls_options(Config)-> + create_authenticator(ConfKeyPath, ChainName0, Config) -> ChainName = to_atom(ChainName0), + % {NConfig, Certs} = convert_tls_options(Config), case update_config(ConfKeyPath, {create_authenticator, ChainName, Config}) of {ok, #{post_config_update := #{?AUTHN := #{id := ID}}, raw_config := AuthenticatorsConfig}} -> diff --git a/apps/emqx_connector/src/emqx_connector_schema_lib.erl b/apps/emqx_connector/src/emqx_connector_schema_lib.erl index 6dcc564af..ecdfb1416 100644 --- a/apps/emqx_connector/src/emqx_connector_schema_lib.erl +++ b/apps/emqx_connector/src/emqx_connector_schema_lib.erl @@ -107,15 +107,15 @@ auto_reconnect(default) -> true; auto_reconnect(_) -> undefined. cacertfile(type) -> string(); -cacertfile(default) -> ""; +cacertfile(nullable) -> true; cacertfile(_) -> undefined. keyfile(type) -> string(); -keyfile(default) -> ""; +keyfile(nullable) -> true; keyfile(_) -> undefined. certfile(type) -> string(); -certfile(default) -> ""; +certfile(nullable) -> true; certfile(_) -> undefined. verify(type) -> boolean();