From 5d91deb83e5d3f48523642418e6d5511acf1a04a Mon Sep 17 00:00:00 2001 From: William Yang Date: Mon, 29 Aug 2022 14:33:48 +0200 Subject: [PATCH 01/61] ci: add escript xref check Check all the released modules --- Makefile | 1 + scripts/xref-check.escript | 41 ++ scripts/xref_check.eterm | 863 +++++++++++++++++++++++++++++++++++++ 3 files changed, 905 insertions(+) create mode 100755 scripts/xref-check.escript create mode 100644 scripts/xref_check.eterm diff --git a/Makefile b/Makefile index f7f46c48f..0f651098c 100644 --- a/Makefile +++ b/Makefile @@ -116,6 +116,7 @@ $(PROFILES:%=deps-%): $(REBAR) get-dashboard .PHONY: xref xref: $(REBAR) @$(REBAR) as check xref + @scripts/xref-check.escript .PHONY: dialyzer dialyzer: $(REBAR) diff --git a/scripts/xref-check.escript b/scripts/xref-check.escript new file mode 100755 index 000000000..65c0fa857 --- /dev/null +++ b/scripts/xref-check.escript @@ -0,0 +1,41 @@ +#!/usr/bin/env escript + +main(_) -> + {ok, [Jobs]} = file:consult("scripts/xref_check.eterm"), + lists:foreach(fun(#{ name := Name + , analysis := Analysis + , excl_apps := ExclApps + , excl_mods := ExclMods + , filters := Filters + }) -> + xref:start(Name), + Tid = ets:new(Name, [ordered_set, named_table]), + xref:set_default(Name, [{verbose,false}, {warnings,false}]), + xref:add_release(Name, "_build/emqx/rel/emqx/lib/"), + xref:add_application(Name, code:lib_dir(erts)), + [ok = xref:remove_application(Name, App) || + App <- ExclApps + ], + + [ok = xref:remove_module(Name, M) || + M <- ExclMods + ], + + ModuleInfos = xref:info(Name, modules), + LibInfos = xref:info(Name, modules), + true = ets:insert(Tid, ModuleInfos ++ LibInfos), + {ok, Res0} = xref:analyse(Name, Analysis), + Res = Res0 -- Filters, + Res =/= [] andalso + begin + put(is_warn_found, true), + io:format("** Warnings for ~p~n : ~p~n", [Name, Res]) + end, + xref:stop(Name) + end, Jobs), + case get(is_warn_found) of + true -> + halt(1); + _ -> + ok + end. diff --git a/scripts/xref_check.eterm b/scripts/xref_check.eterm new file mode 100644 index 000000000..82fc3465f --- /dev/null +++ b/scripts/xref_check.eterm @@ -0,0 +1,863 @@ +[ %% Check undefined_function_calls + #{ name => undefined_function_calls + , analysis => undefined_function_calls + , excl_apps => + [ observer + , redbug + ] + , excl_mods => + [ hipe_unified_loader + , systools_make + , basho_bench_driver_erldis + ] + , filters => + [{{coap_client,channel_apply,3},{coap_dtls_socket,close,1}}, + {{coap_client,channel_apply,3},{coap_dtls_socket,connect,2}}, + {{coap_client,channel_apply,3},{coap_udp_socket,close,1}}, + {{coap_client,channel_apply,3},{coap_udp_socket,get_channel,2}}, + {{coap_client,channel_apply,3},{coap_udp_socket,start_link,0}}, + {{coap_client,test,0},{eunit,test,1}}, + {{coap_core_link,test,0},{eunit,test,1}}, + {{coap_message_parser,test,0},{eunit,test,1}}, + {{coap_observer,init,1},{coap_dtls_socket,connect,2}}, + {{coap_observer,init,1},{coap_udp_socket,get_channel,2}}, + {{coap_observer,init,1},{coap_udp_socket,start_link,0}}, + {{coap_observer,terminate,2},{coap_dtls_socket,close,1}}, + {{coap_observer,terminate,2},{coap_udp_socket,close,1}}, + {{code,beam_file_native_md5,2},{hipe_unified_loader,chunk_name,1}}, + {{code,get_native_fun,0},{hipe_unified_loader,chunk_name,1}}, + {{code,load_native_code_for_all_loaded,1}, + {hipe_unified_loader,chunk_name,1}}, + {{code_server,handle_call,3},{hipe_unified_loader,load,3}}, + {{code_server,handle_call,3},{hipe_unified_loader,load_module,4}}, + {{code_server,try_load_module_2,6}, + {hipe_unified_loader,load_native_code,3}}, + {{compile,embed_native_code,2},{hipe_unified_loader,chunk_name,1}}, + {{compile,native_compile_1,2},{hipe,compile,4}}, + {{core_link,test,0},{eunit,test,1}}, + {{cuttlefish_rebar_plugin,generate,2},{rebar_rel_utils,get_target_dir,2}}, + {{cuttlefish_rebar_plugin,make_default_file,3},{rebar_config,get_local,3}}, + {{cuttlefish_rebar_plugin,should_i_run,2},{rebar_rel_utils,is_rel_dir,0}}, + {{cuttlefish_rebar_plugin,should_i_run,2},{rebar_rel_utils,load_config,2}}, + {{emqx_bridge_worker,common,4},{replayq,append,2}}, + {{emqx_bridge_worker,drop_acked_batches,2},{replayq,ack,2}}, + {{emqx_bridge_worker,open_replayq,1},{replayq,open,1}}, + {{emqx_bridge_worker,pop_and_send_loop,2},{replayq,is_empty,1}}, + {{emqx_bridge_worker,pop_and_send_loop,2},{replayq,pop,2}}, + {{emqx_bridge_worker,terminate,3},{replayq,close,1}}, + {{emqx_misc,ipv6_probe,1},{gen_tcp,ipv6_probe,0}}, + {{eredis_parser,test,0},{eunit,test,1}}, + {{gen_rpc_driver_ssl,authenticate_client,3}, + {ssl_verify_hostname,verify_cert_hostname,2}}, + {{gen_rpc_driver_ssl,merge_ssl_options,2}, + {ssl_verify_hostname,verify_fun,3}}, + {{gproc_dist,from_leader,3},{gen_leader,leader_cast,2}}, + {{gproc_dist,handle_call,4},{gen_leader,leader_node,1}}, + {{gproc_dist,handle_leader_cast,3},{gen_leader,alive,1}}, + {{gproc_dist,handle_leader_cast,3},{gen_leader,broadcast,3}}, + {{gproc_dist,initiate_sync,3},{gen_leader,alive,1}}, + {{gproc_dist,initiate_sync,3},{gen_leader,broadcast,3}}, + {{gproc_dist,leader_call,1},{gen_leader,leader_call,2}}, + {{gproc_dist,leader_cast,1},{gen_leader,leader_cast,2}}, + {{gproc_dist,send_sync_complete,3},{gen_leader,broadcast,3}}, + {{gproc_dist,start_link,1},{gen_leader,start_link,6}}, + {{inets_ftp_wrapper,start_service,1},{ftp,start_service,1}}, + {{inets_ftp_wrapper,start_standalone,1},{ftp,start_standalone,1}}, + {{inets_ftp_wrapper,stop_service,1},{ftp,stop_service,1}}, + {{inets_tftp_wrapper,start_service,1},{tftp,start_service,1}}, + {{inets_tftp_wrapper,start_standalone,1},{tftp,start_standalone,1}}, + {{inets_tftp_wrapper,stop_service,1},{tftp,stop_service,1}}, + {{jose_chacha20_poly1305_libsodium,authenticate,3}, + {libsodium_crypto_onetimeauth_poly1305,crypto_onetimeauth_poly1305,2}}, + {{jose_chacha20_poly1305_libsodium,decrypt,5}, + {libsodium_crypto_aead_chacha20poly1305,ietf_decrypt_detached,5}}, + {{jose_chacha20_poly1305_libsodium,encrypt,4}, + {libsodium_crypto_aead_chacha20poly1305,ietf_encrypt_detached,4}}, + {{jose_chacha20_poly1305_libsodium,one_time_key,2}, + {libsodium_crypto_stream_chacha20,ietf_xor_ic,4}}, + {{jose_chacha20_poly1305_libsodium,verify,4}, + {libsodium_crypto_onetimeauth_poly1305,verify,3}}, + {{jose_curve25519_libdecaf,ed25519_sign,2}, + {libdecaf_curve25519,ed25519_sign,2}}, + {{jose_curve25519_libdecaf,ed25519_verify,3}, + {libdecaf_curve25519,ed25519_verify,3}}, + {{jose_curve25519_libdecaf,ed25519ph_sign,2}, + {libdecaf_curve25519,ed25519ph_sign,2}}, + {{jose_curve25519_libdecaf,ed25519ph_verify,3}, + {libdecaf_curve25519,ed25519ph_verify,3}}, + {{jose_curve25519_libdecaf,eddsa_keypair,0}, + {libdecaf_curve25519,eddsa_keypair,0}}, + {{jose_curve25519_libdecaf,eddsa_keypair,1}, + {libdecaf_curve25519,eddsa_keypair,1}}, + {{jose_curve25519_libdecaf,eddsa_secret_to_public,1}, + {libdecaf_curve25519,eddsa_secret_to_pk,1}}, + {{jose_curve25519_libdecaf,x25519_keypair,0}, + {libdecaf_curve25519,x25519_keypair,0}}, + {{jose_curve25519_libdecaf,x25519_keypair,1}, + {libdecaf_curve25519,x25519_keypair,1}}, + {{jose_curve25519_libdecaf,x25519_secret_to_public,1}, + {libdecaf_curve25519,x25519,1}}, + {{jose_curve25519_libdecaf,x25519_shared_secret,2}, + {libdecaf_curve25519,x25519,2}}, + {{jose_curve25519_libsodium,ed25519_sign,2}, + {libsodium_crypto_sign_ed25519,detached,2}}, + {{jose_curve25519_libsodium,ed25519_verify,3}, + {libsodium_crypto_sign_ed25519,verify_detached,3}}, + {{jose_curve25519_libsodium,ed25519ph_sign,2}, + {libsodium_crypto_hash_sha512,crypto_hash_sha512,1}}, + {{jose_curve25519_libsodium,ed25519ph_verify,3}, + {libsodium_crypto_hash_sha512,crypto_hash_sha512,1}}, + {{jose_curve25519_libsodium,eddsa_keypair,0}, + {libsodium_crypto_sign_ed25519,keypair,0}}, + {{jose_curve25519_libsodium,eddsa_keypair,1}, + {libsodium_crypto_sign_ed25519,seed_keypair,1}}, + {{jose_curve25519_libsodium,eddsa_secret_to_public,1}, + {libsodium_crypto_sign_ed25519,seed_keypair,1}}, + {{jose_curve25519_libsodium,x25519_keypair,0}, + {libsodium_crypto_box_curve25519xsalsa20poly1305,keypair,0}}, + {{jose_curve25519_libsodium,x25519_secret_to_public,1}, + {libsodium_crypto_scalarmult_curve25519,base,1}}, + {{jose_curve25519_libsodium,x25519_shared_secret,2}, + {libsodium_crypto_scalarmult_curve25519,crypto_scalarmult_curve25519,2}}, + {{jose_curve448_libdecaf,ed448_sign,2},{libdecaf_curve448,ed448_sign,2}}, + {{jose_curve448_libdecaf,ed448_sign,3},{libdecaf_curve448,ed448_sign,3}}, + {{jose_curve448_libdecaf,ed448_verify,3}, + {libdecaf_curve448,ed448_verify,3}}, + {{jose_curve448_libdecaf,ed448_verify,4}, + {libdecaf_curve448,ed448_verify,4}}, + {{jose_curve448_libdecaf,ed448ph_sign,2}, + {libdecaf_curve448,ed448ph_sign,2}}, + {{jose_curve448_libdecaf,ed448ph_sign,3}, + {libdecaf_curve448,ed448ph_sign,3}}, + {{jose_curve448_libdecaf,ed448ph_verify,3}, + {libdecaf_curve448,ed448ph_verify,3}}, + {{jose_curve448_libdecaf,ed448ph_verify,4}, + {libdecaf_curve448,ed448ph_verify,4}}, + {{jose_curve448_libdecaf,eddsa_keypair,0}, + {libdecaf_curve448,eddsa_keypair,0}}, + {{jose_curve448_libdecaf,eddsa_keypair,1}, + {libdecaf_curve448,eddsa_keypair,1}}, + {{jose_curve448_libdecaf,eddsa_secret_to_public,1}, + {libdecaf_curve448,eddsa_secret_to_pk,1}}, + {{jose_curve448_libdecaf,x448_keypair,0}, + {libdecaf_curve448,x448_keypair,0}}, + {{jose_curve448_libdecaf,x448_keypair,1}, + {libdecaf_curve448,x448_keypair,1}}, + {{jose_curve448_libdecaf,x448_secret_to_public,1}, + {libdecaf_curve448,x448,1}}, + {{jose_curve448_libdecaf,x448_shared_secret,2},{libdecaf_curve448,x448,2}}, + {{jose_json_jason,decode,1},{'Elixir.Jason','decode!',1}}, + {{jose_json_jason,encode,1},{'Elixir.Jason','encode!',1}}, + {{jose_json_jsone,decode,1},{jsone,decode,1}}, + {{jose_json_jsone,encode,1},{jsone,encode,2}}, + {{jose_json_jsx,decode,1},{jsx,decode,2}}, + {{jose_json_jsx,encode,1},{jsx,encode,1}}, + {{jose_json_ojson,decode,1},{ojson,'decode!',1}}, + {{jose_json_ojson,encode,1},{ojson,'encode!',1}}, + {{jose_json_poison,decode,1},{'Elixir.Poison','decode!',1}}, + {{jose_json_poison,encode,1},{'Elixir.Poison','encode!',1}}, + {{jose_json_poison_compat_encoder,decode,1},{'Elixir.Poison','decode!',1}}, + {{jose_json_poison_compat_encoder,encode,1}, + {'Elixir.IO',iodata_to_binary,1}}, + {{jose_json_poison_compat_encoder,lexical_encode,1}, + {'Elixir.Poison.EncodeError',exception,1}}, + {{jose_json_poison_compat_encoder,lexical_encode,1}, + {'Elixir.Poison.Encoder.Atom',encode,2}}, + {{jose_json_poison_compat_encoder,lexical_encode,1}, + {'Elixir.Poison.Encoder.BitString',encode,2}}, + {{jose_json_poison_compat_encoder,lexical_encode,1}, + {'Elixir.Poison.Encoder.Float',encode,2}}, + {{jose_json_poison_compat_encoder,lexical_encode,1}, + {'Elixir.Poison.Encoder.Integer',encode,2}}, + {{jose_json_poison_compat_encoder,lexical_encode_map,1}, + {'Elixir.Poison.Encoder.BitString',encode,2}}, + {{jose_json_poison_compat_encoder,lexical_encode_name,1}, + {'Elixir.Atom',to_string,1}}, + {{jose_json_poison_compat_encoder,lexical_encode_name,1}, + {'Elixir.Kernel',inspect,1}}, + {{jose_json_poison_compat_encoder,lexical_encode_name,1}, + {'Elixir.Poison.EncodeError',exception,1}}, + {{jose_json_poison_compat_encoder,lexical_encode_struct,2}, + {'Elixir.Enum',flat_map,2}}, + {{jose_json_poison_compat_encoder,lexical_encode_struct,2}, + {'Elixir.HashDict',size,1}}, + {{jose_json_poison_compat_encoder,lexical_encode_struct,2}, + {'Elixir.Map',from_struct,1}}, + {{jose_json_poison_compat_encoder,lexical_encode_struct,2}, + {'Elixir.Poison.Encoder.BitString',encode,2}}, + {{jose_json_poison_lexical_encoder,decode,1}, + {'Elixir.Poison','decode!',1}}, + {{jose_json_poison_lexical_encoder,encode,1}, + {'Elixir.JOSE.Poison','lexical_encode!',1}}, + {{jose_jwk_kty_rsa,try_generate_key,3},{cutkey,rsa,3}}, + {{jose_sha3_keccakf1600_driver,sha3_224,1}, + {keccakf1600_fips202,sha3_224,1}}, + {{jose_sha3_keccakf1600_driver,sha3_256,1}, + {keccakf1600_fips202,sha3_256,1}}, + {{jose_sha3_keccakf1600_driver,sha3_384,1}, + {keccakf1600_fips202,sha3_384,1}}, + {{jose_sha3_keccakf1600_driver,sha3_512,1}, + {keccakf1600_fips202,sha3_512,1}}, + {{jose_sha3_keccakf1600_driver,shake128,2}, + {keccakf1600_fips202,shake128,2}}, + {{jose_sha3_keccakf1600_driver,shake256,2}, + {keccakf1600_fips202,shake256,2}}, + {{jose_sha3_keccakf1600_nif,sha3_224,1},{keccakf1600,hash,2}}, + {{jose_sha3_keccakf1600_nif,sha3_256,1},{keccakf1600,hash,2}}, + {{jose_sha3_keccakf1600_nif,sha3_384,1},{keccakf1600,hash,2}}, + {{jose_sha3_keccakf1600_nif,sha3_512,1},{keccakf1600,hash,2}}, + {{jose_sha3_keccakf1600_nif,shake128,2},{keccakf1600,hash,3}}, + {{jose_sha3_keccakf1600_nif,shake256,2},{keccakf1600,hash,3}}, + {{jose_sha3_libdecaf,sha3_224,1},{libdecaf_sha3,hash,2}}, + {{jose_sha3_libdecaf,sha3_256,1},{libdecaf_sha3,hash,2}}, + {{jose_sha3_libdecaf,sha3_384,1},{libdecaf_sha3,hash,2}}, + {{jose_sha3_libdecaf,sha3_512,1},{libdecaf_sha3,hash,2}}, + {{jose_sha3_libdecaf,shake128,2},{libdecaf_sha3,hash,3}}, + {{jose_sha3_libdecaf,shake256,2},{libdecaf_sha3,hash,3}}, + {{lwm2m_coap_client,channel_apply,3},{lwm2m_coap_dtls_socket,close,1}}, + {{lwm2m_coap_client,channel_apply,3},{lwm2m_coap_dtls_socket,connect,2}}, + {{lwm2m_coap_client,channel_apply,3},{lwm2m_coap_udp_socket,close,1}}, + {{lwm2m_coap_client,channel_apply,3}, + {lwm2m_coap_udp_socket,get_channel,2}}, + {{lwm2m_coap_client,channel_apply,3},{lwm2m_coap_udp_socket,start_link,0}}, + {{lwm2m_coap_client,test,0},{eunit,test,1}}, + {{lwm2m_coap_message_parser,test,0},{eunit,test,1}}, + {{lwm2m_coap_observer,init,1},{lwm2m_coap_dtls_socket,connect,2}}, + {{lwm2m_coap_observer,init,1},{lwm2m_coap_udp_socket,get_channel,2}}, + {{lwm2m_coap_observer,init,1},{lwm2m_coap_udp_socket,start_link,0}}, + {{lwm2m_coap_observer,terminate,2},{lwm2m_coap_dtls_socket,close,1}}, + {{lwm2m_coap_observer,terminate,2},{lwm2m_coap_udp_socket,close,1}}, + {{ranch_app,consider_profiling,0},{eprof,start,0}}, + {{ranch_app,consider_profiling,0},{eprof,start_profiling,1}}, + {{ranch_app,profile_output,0},{eprof,analyze,1}}, + {{ranch_app,profile_output,0},{eprof,log,1}}, + {{ranch_app,profile_output,0},{eprof,stop_profiling,0}}, + {{release_handler,new_emulator_make_hybrid_boot,6}, + {systools_make,make_hybrid_boot,4}}, + {{release_handler,read_app,3},{systools_make,read_application,4}}, + {{systools,compile_rel,3},{systools_make,make_script,3}}, + {{systools,make_script,2},{systools_make,make_script,2}}, + {{systools,make_tar,2},{systools_make,make_tar,2}}, + {{systools,script2boot,1},{systools_make,format_error,1}}, + {{systools,script2boot,3},{systools_make,format_error,1}}, + {{systools_relup,do_mk_relup,5},{systools_make,get_release,2}}, + {{systools_relup,foreach_baserel_dn,7},{systools_make,get_release,2}}, + {{systools_relup,foreach_baserel_up,7},{systools_make,get_release,2}}] + } + + %% Check undefined_functions +, #{ name => undefined_functions + , analysis => undefined_functions + , excl_apps => + [ + + ] + , excl_mods => + [ + ] + , filters => + [{'Elixir.Atom',to_string,1}, + {'Elixir.Enum',flat_map,2}, + {'Elixir.HashDict',size,1}, + {'Elixir.IO',iodata_to_binary,1}, + {'Elixir.JOSE.Poison','lexical_encode!',1}, + {'Elixir.Jason','decode!',1}, + {'Elixir.Jason','encode!',1}, + {'Elixir.Kernel',inspect,1}, + {'Elixir.Map',from_struct,1}, + {'Elixir.Poison','decode!',1}, + {'Elixir.Poison','encode!',1}, + {'Elixir.Poison.EncodeError',exception,1}, + {'Elixir.Poison.Encoder.Atom',encode,2}, + {'Elixir.Poison.Encoder.BitString',encode,2}, + {'Elixir.Poison.Encoder.Float',encode,2}, + {'Elixir.Poison.Encoder.Integer',encode,2}, + {coap_dtls_socket,close,1}, + {coap_dtls_socket,connect,2}, + {coap_udp_socket,close,1}, + {coap_udp_socket,get_channel,2}, + {coap_udp_socket,start_link,0}, + {ct_slave,start,2}, + {ct_slave,stop,1}, + {cutkey,rsa,3}, + {eprof,analyze,1}, + {eprof,log,1}, + {eprof,start,0}, + {eprof,start_profiling,1}, + {eprof,stop_profiling,0}, + {erldis,mget,2}, + {erldis,set,3}, + {erldis_client,connect,0}, + {et_collector,iterate,5}, + {et_collector,report,2}, + {et_selector,parse_event,2}, + {et_viewer,get_collector_pid,1}, + {et_viewer,start_link,1}, + {eunit,test,1}, + {ftp,start_service,1}, + {ftp,start_standalone,1}, + {ftp,stop_service,1}, + {gen_leader,alive,1}, + {gen_leader,broadcast,3}, + {gen_leader,leader_call,2}, + {gen_leader,leader_cast,2}, + {gen_leader,leader_node,1}, + {gen_leader,start_link,6}, + {gen_tcp,ipv6_probe,0}, + {hipe,compile,4}, + {hipe_bifs,add_ref,2}, + {hipe_bifs,alloc_data,3}, + {hipe_bifs,alloc_loader_state,1}, + {hipe_bifs,atom_to_word,1}, + {hipe_bifs,bif_address,3}, + {hipe_bifs,check_crc,1}, + {hipe_bifs,commit_patch_load,1}, + {hipe_bifs,enter_code,3}, + {hipe_bifs,enter_sdesc,2}, + {hipe_bifs,find_na_or_make_stub,1}, + {hipe_bifs,fun_to_address,1}, + {hipe_bifs,get_fe,2}, + {hipe_bifs,merge_term,1}, + {hipe_bifs,patch_call,3}, + {hipe_bifs,patch_insn,3}, + {hipe_bifs,primop_address,1}, + {hipe_bifs,set_funinfo_native_address,3}, + {hipe_bifs,set_native_address,3}, + {hipe_bifs,set_native_address_in_fe,2}, + {hipe_bifs,term_to_word,1}, + {hipe_bifs,write_u32,2}, + {hipe_bifs,write_u64,2}, + {hipe_bifs,write_u8,2}, + {jsone,decode,1}, + {jsone,encode,2}, + {jsx,decode,2}, + {jsx,encode,1}, + {keccakf1600,hash,2}, + {keccakf1600,hash,3}, + {keccakf1600_fips202,sha3_224,1}, + {keccakf1600_fips202,sha3_256,1}, + {keccakf1600_fips202,sha3_384,1}, + {keccakf1600_fips202,sha3_512,1}, + {keccakf1600_fips202,shake128,2}, + {keccakf1600_fips202,shake256,2}, + {libdecaf_curve25519,ed25519_sign,2}, + {libdecaf_curve25519,ed25519_verify,3}, + {libdecaf_curve25519,ed25519ph_sign,2}, + {libdecaf_curve25519,ed25519ph_verify,3}, + {libdecaf_curve25519,eddsa_keypair,0}, + {libdecaf_curve25519,eddsa_keypair,1}, + {libdecaf_curve25519,eddsa_secret_to_pk,1}, + {libdecaf_curve25519,x25519,1}, + {libdecaf_curve25519,x25519,2}, + {libdecaf_curve25519,x25519_keypair,0}, + {libdecaf_curve25519,x25519_keypair,1}, + {libdecaf_curve448,ed448_sign,2}, + {libdecaf_curve448,ed448_sign,3}, + {libdecaf_curve448,ed448_verify,3}, + {libdecaf_curve448,ed448_verify,4}, + {libdecaf_curve448,ed448ph_sign,2}, + {libdecaf_curve448,ed448ph_sign,3}, + {libdecaf_curve448,ed448ph_verify,3}, + {libdecaf_curve448,ed448ph_verify,4}, + {libdecaf_curve448,eddsa_keypair,0}, + {libdecaf_curve448,eddsa_keypair,1}, + {libdecaf_curve448,eddsa_secret_to_pk,1}, + {libdecaf_curve448,x448,1}, + {libdecaf_curve448,x448,2}, + {libdecaf_curve448,x448_keypair,0}, + {libdecaf_curve448,x448_keypair,1}, + {libdecaf_sha3,hash,2}, + {libdecaf_sha3,hash,3}, + {libsodium_crypto_aead_chacha20poly1305,ietf_decrypt_detached,5}, + {libsodium_crypto_aead_chacha20poly1305,ietf_encrypt_detached,4}, + {libsodium_crypto_box_curve25519xsalsa20poly1305,keypair,0}, + {libsodium_crypto_hash_sha512,crypto_hash_sha512,1}, + {libsodium_crypto_onetimeauth_poly1305,crypto_onetimeauth_poly1305,2}, + {libsodium_crypto_onetimeauth_poly1305,verify,3}, + {libsodium_crypto_scalarmult_curve25519,base,1}, + {libsodium_crypto_scalarmult_curve25519,crypto_scalarmult_curve25519,2}, + {libsodium_crypto_sign_ed25519,detached,2}, + {libsodium_crypto_sign_ed25519,keypair,0}, + {libsodium_crypto_sign_ed25519,seed_keypair,1}, + {libsodium_crypto_sign_ed25519,verify_detached,3}, + {libsodium_crypto_stream_chacha20,ietf_xor_ic,4}, + {lwm2m_coap_dtls_socket,close,1}, + {lwm2m_coap_dtls_socket,connect,2}, + {lwm2m_coap_udp_socket,close,1}, + {lwm2m_coap_udp_socket,get_channel,2}, + {lwm2m_coap_udp_socket,start_link,0}, + {ojson,'decode!',1}, + {ojson,'encode!',1}, + {rebar_config,get_local,3}, + {rebar_rel_utils,get_target_dir,2}, + {rebar_rel_utils,is_rel_dir,0}, + {rebar_rel_utils,load_config,2}, + {replayq,ack,2}, + {replayq,append,2}, + {replayq,close,1}, + {replayq,is_empty,1}, + {replayq,open,1}, + {replayq,pop,2}, + {ssl_verify_hostname,verify_cert_hostname,2}, + {ssl_verify_hostname,verify_fun,3}, + {tftp,start_service,1}, + {tftp,start_standalone,1}, + {tftp,stop_service,1}, + {wx,batch,1}, + {wx,destroy,0}, + {wx,foldl,3}, + {wx,foreach,2}, + {wx,getObjectType,1}, + {wx,get_env,0}, + {wx,is_null,1}, + {wx,new,0}, + {wx,null,0}, + {wx,set_env,1}, + {wx,typeCast,2}, + {wxBoxSizer,new,1}, + {wxBrush,new,1}, + {wxBufferedPaintDC,new,1}, + {wxButton,connect,3}, + {wxButton,new,3}, + {wxCheckBox,connect,3}, + {wxCheckBox,getValue,1}, + {wxCheckBox,isChecked,1}, + {wxCheckBox,new,3}, + {wxCheckBox,new,4}, + {wxCheckBox,set3StateValue,2}, + {wxCheckBox,setValue,2}, + {wxCheckListBox,check,3}, + {wxCheckListBox,connect,3}, + {wxCheckListBox,isChecked,2}, + {wxCheckListBox,new,3}, + {wxClientDC,new,1}, + {wxCursor,destroy,1}, + {wxCursor,new,1}, + {wxDC,clear,1}, + {wxDC,drawLine,3}, + {wxDC,drawLines,2}, + {wxDC,drawRoundedRectangle,4}, + {wxDC,drawText,3}, + {wxDC,getCharHeight,1}, + {wxDC,getCharWidth,1}, + {wxDC,getMultiLineTextExtent,2}, + {wxDC,getSize,1}, + {wxDC,getTextExtent,2}, + {wxDC,setBrush,2}, + {wxDC,setFont,2}, + {wxDC,setPen,2}, + {wxDC,setTextForeground,2}, + {wxDialog,createButtonSizer,2}, + {wxDialog,destroy,1}, + {wxDialog,endModal,2}, + {wxDialog,getAffirmativeId,1}, + {wxDialog,new,4}, + {wxDialog,show,1}, + {wxDialog,showModal,1}, + {wxEvent,skip,1}, + {wxFileDialog,destroy,1}, + {wxFileDialog,getDirectory,1}, + {wxFileDialog,getFilename,1}, + {wxFileDialog,getPath,1}, + {wxFileDialog,new,2}, + {wxFileDialog,showModal,1}, + {wxFont,getFamily,1}, + {wxFont,getPointSize,1}, + {wxFont,isFixedWidth,1}, + {wxFont,new,4}, + {wxFont,setPointSize,2}, + {wxFrame,center,1}, + {wxFrame,connect,2}, + {wxFrame,connect,3}, + {wxFrame,createStatusBar,2}, + {wxFrame,destroy,1}, + {wxFrame,new,4}, + {wxFrame,raise,1}, + {wxFrame,setFocus,1}, + {wxFrame,setIcon,2}, + {wxFrame,setMenuBar,2}, + {wxFrame,setStatusBar,2}, + {wxFrame,setTitle,2}, + {wxFrame,show,1}, + {wxGauge,new,4}, + {wxGauge,pulse,1}, + {wxGauge,setValue,2}, + {wxGraphicsContext,create,1}, + {wxGraphicsContext,destroy,1}, + {wxGraphicsContext,drawRoundedRectangle,6}, + {wxGraphicsContext,drawText,4}, + {wxGraphicsContext,getTextExtent,2}, + {wxGraphicsContext,setBrush,2}, + {wxGraphicsContext,setFont,3}, + {wxGraphicsContext,setPen,2}, + {wxGraphicsContext,strokeLine,5}, + {wxGraphicsContext,strokeLines,2}, + {wxGraphicsContext,translate,3}, + {wxGraphicsRenderer,getDefaultRenderer,0}, + {wxHtmlWindow,connect,2}, + {wxHtmlWindow,new,2}, + {wxHtmlWindow,setPage,2}, + {wxIcon,destroy,1}, + {wxIcon,new,2}, + {wxListBox,append,3}, + {wxListBox,appendStrings,2}, + {wxListBox,clear,1}, + {wxListBox,connect,2}, + {wxListBox,connect,3}, + {wxListBox,delete,2}, + {wxListBox,disconnect,1}, + {wxListBox,getClientData,2}, + {wxListBox,getCount,1}, + {wxListBox,getSelection,1}, + {wxListBox,getString,2}, + {wxListBox,getStringSelection,1}, + {wxListBox,new,3}, + {wxListBox,setClientData,3}, + {wxListBox,setSelection,2}, + {wxListBox,setString,3}, + {wxListCtrl,connect,2}, + {wxListCtrl,connect,3}, + {wxListCtrl,deleteAllItems,1}, + {wxListCtrl,ensureVisible,2}, + {wxListCtrl,getColumnCount,1}, + {wxListCtrl,getColumnWidth,2}, + {wxListCtrl,getItemCount,1}, + {wxListCtrl,getItemText,2}, + {wxListCtrl,getNextItem,3}, + {wxListCtrl,getParent,1}, + {wxListCtrl,insertColumn,3}, + {wxListCtrl,insertItem,3}, + {wxListCtrl,new,2}, + {wxListCtrl,refreshItem,2}, + {wxListCtrl,refreshItems,3}, + {wxListCtrl,setColumnWidth,3}, + {wxListCtrl,setItem,4}, + {wxListCtrl,setItemBackgroundColour,3}, + {wxListCtrl,setItemCount,2}, + {wxListCtrl,setItemState,4}, + {wxListCtrl,setToolTip,2}, + {wxListItem,destroy,1}, + {wxListItem,new,0}, + {wxListItem,setAlign,2}, + {wxListItem,setText,2}, + {wxListItemAttr,destroy,1}, + {wxListItemAttr,new,3}, + {wxMenu,'Destroy',2}, + {wxMenu,append,3}, + {wxMenu,append,4}, + {wxMenu,appendSeparator,1}, + {wxMenu,check,3}, + {wxMenu,connect,2}, + {wxMenu,connect,3}, + {wxMenu,delete,2}, + {wxMenu,destroy,1}, + {wxMenu,findItem,2}, + {wxMenu,getMenuItems,1}, + {wxMenu,insert,3}, + {wxMenu,insert,4}, + {wxMenu,insertCheckItem,5}, + {wxMenu,insertRadioItem,5}, + {wxMenu,insertSeparator,2}, + {wxMenu,new,0}, + {wxMenuBar,append,3}, + {wxMenuBar,check,3}, + {wxMenuBar,findMenu,2}, + {wxMenuBar,getMenu,2}, + {wxMenuBar,insert,4}, + {wxMenuBar,new,0}, + {wxMenuBar,remove,2}, + {wxMessageDialog,destroy,1}, + {wxMessageDialog,new,2}, + {wxMessageDialog,new,3}, + {wxMessageDialog,setTitle,2}, + {wxMessageDialog,showModal,1}, + {wxMiniFrame,new,4}, + {wxNotebook,addPage,3}, + {wxNotebook,addPage,4}, + {wxNotebook,connect,2}, + {wxNotebook,connect,3}, + {wxNotebook,getPageText,2}, + {wxNotebook,getSelection,1}, + {wxNotebook,new,3}, + {wxNotebook,setSelection,2}, + {wxPaintDC,new,1}, + {wxPanel,connect,2}, + {wxPanel,connect,3}, + {wxPanel,new,1}, + {wxPanel,new,2}, + {wxPanel,setSizer,2}, + {wxPanel,setSizerAndFit,3}, + {wxPen,new,1}, + {wxPen,new,2}, + {wxRadioBox,getSelection,1}, + {wxRadioBox,new,7}, + {wxRadioButton,getValue,1}, + {wxRadioButton,new,3}, + {wxRadioButton,new,4}, + {wxRadioButton,setValue,2}, + {wxScrolledWindow,calcScrolledPosition,2}, + {wxScrolledWindow,calcUnscrolledPosition,3}, + {wxScrolledWindow,doPrepareDC,2}, + {wxScrolledWindow,enableScrolling,3}, + {wxScrolledWindow,new,1}, + {wxScrolledWindow,new,2}, + {wxScrolledWindow,setScrollbars,5}, + {wxScrolledWindow,setSizer,2}, + {wxSizer,add,2}, + {wxSizer,add,3}, + {wxSizer,add,4}, + {wxSizer,addSpacer,2}, + {wxSizer,detach,2}, + {wxSizer,getChildren,1}, + {wxSizer,getMinSize,1}, + {wxSizer,hide,2}, + {wxSizer,layout,1}, + {wxSizer,recalcSizes,1}, + {wxSizer,remove,2}, + {wxSizer,setItemMinSize,4}, + {wxSizer,setSizeHints,2}, + {wxSizer,setVirtualSizeHints,2}, + {wxSizer,show,2}, + {wxSizerItem,deleteWindows,1}, + {wxSlider,connect,3}, + {wxSlider,getValue,1}, + {wxSlider,new,6}, + {wxSlider,setValue,2}, + {wxSplitterWindow,new,2}, + {wxSplitterWindow,setMinimumPaneSize,2}, + {wxSplitterWindow,setSashGravity,2}, + {wxSplitterWindow,splitHorizontally,4}, + {wxSplitterWindow,splitVertically,4}, + {wxStaticBoxSizer,new,3}, + {wxStaticLine,new,2}, + {wxStaticText,destroy,1}, + {wxStaticText,new,3}, + {wxStaticText,new,4}, + {wxStaticText,setForegroundColour,2}, + {wxStaticText,setLabel,2}, + {wxStatusBar,new,1}, + {wxStatusBar,setStatusText,2}, + {wxStyledTextCtrl,getText,1}, + {wxStyledTextCtrl,new,1}, + {wxStyledTextCtrl,setKeyWords,3}, + {wxStyledTextCtrl,setLexer,2}, + {wxStyledTextCtrl,setMarginType,3}, + {wxStyledTextCtrl,setSelectionMode,2}, + {wxStyledTextCtrl,setText,2}, + {wxStyledTextCtrl,setUseHorizontalScrollBar,2}, + {wxStyledTextCtrl,styleClearAll,1}, + {wxStyledTextCtrl,styleSetFont,3}, + {wxStyledTextCtrl,styleSetForeground,3}, + {wxSystemOptions,setOption,2}, + {wxSystemSettings,getColour,1}, + {wxSystemSettings,getFont,1}, + {wxSystemSettings,getMetric,1}, + {wxTextAttr,destroy,1}, + {wxTextAttr,new,1}, + {wxTextAttr,new,2}, + {wxTextCtrl,appendText,2}, + {wxTextCtrl,clear,1}, + {wxTextCtrl,connect,2}, + {wxTextCtrl,connect,3}, + {wxTextCtrl,destroy,1}, + {wxTextCtrl,getTextExtent,2}, + {wxTextCtrl,getValue,1}, + {wxTextCtrl,new,2}, + {wxTextCtrl,new,3}, + {wxTextCtrl,saveFile,2}, + {wxTextCtrl,setDefaultStyle,2}, + {wxTextCtrl,setForegroundColour,2}, + {wxTextCtrl,setValue,2}, + {wxTextCtrl,writeText,2}, + {wxTextEntryDialog,destroy,1}, + {wxTextEntryDialog,getValue,1}, + {wxTextEntryDialog,new,2}, + {wxTextEntryDialog,new,3}, + {wxTextEntryDialog,showModal,1}, + {wxToggleButton,new,4}, + {wxToggleButton,setLabel,2}, + {wxToggleButton,setValue,2}, + {wxToolTip,new,1}, + {wxWindow,connect,2}, + {wxWindow,connect,3}, + {wxWindow,destroy,1}, + {wxWindow,disable,1}, + {wxWindow,enable,1}, + {wxWindow,enable,2}, + {wxWindow,findWindowById,1}, + {wxWindow,freeze,1}, + {wxWindow,getBackgroundColour,1}, + {wxWindow,getClientSize,1}, + {wxWindow,getParent,1}, + {wxWindow,getSize,1}, + {wxWindow,getTextExtent,2}, + {wxWindow,getTextExtent,3}, + {wxWindow,hasScrollbar,2}, + {wxWindow,layout,1}, + {wxWindow,popupMenu,2}, + {wxWindow,refresh,1}, + {wxWindow,setBackgroundStyle,2}, + {wxWindow,setClientSize,2}, + {wxWindow,setCursor,2}, + {wxWindow,setFocus,1}, + {wxWindow,setForegroundColour,2}, + {wxWindow,setMinSize,2}, + {wxWindow,setSizer,2}, + {wxWindow,setSizerAndFit,2}, + {wxWindow,setToolTip,2}, + {wxWindow,setVirtualSize,3}, + {wxWindow,thaw,1}, + {wxWindowDC,new,1}, + {wx_misc,beginBusyCursor,0}, + {wx_misc,endBusyCursor,0}, + {wx_misc,getKeyState,1}, + {wx_misc,launchDefaultBrowser,1}, + {wx_object,call,2}, + {wx_object,cast,2}, + {wx_object,get_pid,1}, + {wx_object,start,3}, + {wx_object,start_link,3}, + {wx_object,start_link,4}, + {wx_object,stop,1}, + {wxe_util,get_const,1}, + {xref,add_application,3}, + {xref,analyze,2}, + {xref,set_default,3}, + {xref,set_library_path,2}, + {xref,start,2}, + {xref,stop,1}] + } + +, #{ name => locals_not_used + , analysis => locals_not_used + , excl_apps => + [ + + ] + , excl_mods => + [ + ] + , filters => + [{coap_core_link_parser,return_error,2}, + {core_link_parser,return_error,2}, + {emqx_exhook_pb,e_type_double,3}, + {emqx_exhook_pb,e_type_fixed32,3}, + {emqx_exhook_pb,e_type_fixed64,3}, + {emqx_exhook_pb,e_type_float,3}, + {emqx_exhook_pb,e_type_int32,3}, + {emqx_exhook_pb,e_type_int64,3}, + {emqx_exhook_pb,e_type_sfixed32,3}, + {emqx_exhook_pb,e_type_sfixed64,3}, + {emqx_exhook_pb,e_type_sint,3}, + {emqx_exhook_pb,m_overwrite,3}, + {emqx_exhook_pb,v_ok,3}, + {emqx_exproto_pb,e_type_bool,3}, + {emqx_exproto_pb,e_type_double,3}, + {emqx_exproto_pb,e_type_fixed32,3}, + {emqx_exproto_pb,e_type_fixed64,3}, + {emqx_exproto_pb,e_type_float,3}, + {emqx_exproto_pb,e_type_int32,3}, + {emqx_exproto_pb,e_type_int64,3}, + {emqx_exproto_pb,e_type_sfixed32,3}, + {emqx_exproto_pb,e_type_sfixed64,3}, + {emqx_exproto_pb,e_type_sint,3}, + {emqx_exproto_pb,m_overwrite,3}, + {emqx_exproto_pb,v_ok,3}, + {grpc_health_pb,cons,3}, + {grpc_health_pb,e_type_bool,3}, + {grpc_health_pb,e_type_bytes,3}, + {grpc_health_pb,e_type_double,3}, + {grpc_health_pb,e_type_fixed32,3}, + {grpc_health_pb,e_type_fixed64,3}, + {grpc_health_pb,e_type_float,3}, + {grpc_health_pb,e_type_int32,3}, + {grpc_health_pb,e_type_int64,3}, + {grpc_health_pb,e_type_sfixed32,3}, + {grpc_health_pb,e_type_sfixed64,3}, + {grpc_health_pb,e_type_sint,3}, + {grpc_health_pb,e_varint,3}, + {grpc_health_pb,'erlang_++',3}, + {grpc_health_pb,lists_reverse,2}, + {grpc_health_pb,m_overwrite,3}, + {grpc_health_pb,v_ok,3}, + {grpc_reflection_pb,e_type_bool,3}, + {grpc_reflection_pb,e_type_double,3}, + {grpc_reflection_pb,e_type_fixed32,3}, + {grpc_reflection_pb,e_type_fixed64,3}, + {grpc_reflection_pb,e_type_float,3}, + {grpc_reflection_pb,e_type_int64,3}, + {grpc_reflection_pb,e_type_sfixed32,3}, + {grpc_reflection_pb,e_type_sfixed64,3}, + {grpc_reflection_pb,e_type_sint,3}, + {grpc_reflection_pb,e_varint,3}, + {grpc_reflection_pb,m_overwrite,3}, + {grpc_reflection_pb,v_ok,3}, + {prometheus_text_format,escape_metric_help,1}, + {redbug_parser,return_error,2}, + {rulesql,return_error,2}, + {xmerl_b64Bin,return_error,2}, + {xmerl_xpath_parse,return_error,2}] + } + +, #{ name => deprecated_function_calls + , analysis => deprecated_function_calls + , excl_apps => + [ + + ] + , excl_mods => + [ + ] + , filters => + [{{coap_core_link,join_uri,1},{http_uri,encode,1}}, + {{disk_log_server,dist_pids,1},{pg2,get_members,1}}, + {{disk_log_server,do_accessible_logs,0},{pg2,which_groups,0}}, + {{disk_log_server,do_get_log_pids,1},{pg2,get_members,1}}, + {{disk_log_server,do_open,3},{pg2,create,1}}, + {{disk_log_server,erase_log,2},{pg2,leave,2}}, + {{disk_log_server,handle_info,2},{pg2,join,2}}, + {{gen_fsm,error_info,7},{gen_fsm,format_log,1}}, + {{gen_fsm,error_info,7},{gen_fsm,format_log,2}}, + {{gen_fsm,handle_msg,8},{gen_fsm,format_log,1}}, + {{gen_fsm,handle_msg,8},{gen_fsm,format_log,2}}, + {{httpd_util,decode_hex,1},{http_uri,decode,1}}, + {{httpd_util,encode_hex,1},{http_uri,encode,1}}, + {{igor,tidy,2},{erl_tidy,module,2}}, + {{jose_public_key,pseudo_random_function,1},{crypto,hmac,4}}, + {{lwm2m_coap_client,make_segment,1},{http_uri,decode,1}}, + {{lwm2m_coap_client,resolve_uri,1},{http_uri,parse,2}}, + {{lwm2m_coap_responder,cancel_observer,2},{pg2,delete,1}}, + {{lwm2m_coap_responder,cancel_observer,2},{pg2,get_members,1}}, + {{lwm2m_coap_responder,cancel_observer,2},{pg2,leave,2}}, + {{lwm2m_coap_responder,handle_observe,4},{pg2,create,1}}, + {{lwm2m_coap_responder,handle_observe,4},{pg2,join,2}}, + {{lwm2m_coap_responder,notify,2},{pg2,get_members,1}}, + {{ranch_ssl,handshake,3},{ssl,ssl_accept,3}}] + } + +, #{ name => deprecated_functions + , analysis => deprecated_functions + , excl_apps => + [ + + ] + , excl_mods => + [ + ] + , filters => + [{crypto,hmac,4}, + {erl_tidy,module,2}, + {gen_fsm,format_log,1}, + {gen_fsm,format_log,2}, + {http_uri,decode,1}, + {http_uri,encode,1}, + {http_uri,parse,2}, + {pg2,create,1}, + {pg2,delete,1}, + {pg2,get_members,1}, + {pg2,join,2}, + {pg2,leave,2}, + {pg2,which_groups,0}, + {ssl,ssl_accept,3}] + } +]. From eed5a24e408fdf65405726bd919b0e7141c4a247 Mon Sep 17 00:00:00 2001 From: William Yang Date: Thu, 1 Sep 2022 08:47:28 +0200 Subject: [PATCH 02/61] ci(xref): don't check observer --- scripts/xref_check.eterm | 320 +-------------------------------------- 1 file changed, 1 insertion(+), 319 deletions(-) diff --git a/scripts/xref_check.eterm b/scripts/xref_check.eterm index 82fc3465f..f1fa0f2f4 100644 --- a/scripts/xref_check.eterm +++ b/scripts/xref_check.eterm @@ -249,8 +249,7 @@ , #{ name => undefined_functions , analysis => undefined_functions , excl_apps => - [ - + [ observer ] , excl_mods => [ @@ -403,323 +402,6 @@ {tftp,start_service,1}, {tftp,start_standalone,1}, {tftp,stop_service,1}, - {wx,batch,1}, - {wx,destroy,0}, - {wx,foldl,3}, - {wx,foreach,2}, - {wx,getObjectType,1}, - {wx,get_env,0}, - {wx,is_null,1}, - {wx,new,0}, - {wx,null,0}, - {wx,set_env,1}, - {wx,typeCast,2}, - {wxBoxSizer,new,1}, - {wxBrush,new,1}, - {wxBufferedPaintDC,new,1}, - {wxButton,connect,3}, - {wxButton,new,3}, - {wxCheckBox,connect,3}, - {wxCheckBox,getValue,1}, - {wxCheckBox,isChecked,1}, - {wxCheckBox,new,3}, - {wxCheckBox,new,4}, - {wxCheckBox,set3StateValue,2}, - {wxCheckBox,setValue,2}, - {wxCheckListBox,check,3}, - {wxCheckListBox,connect,3}, - {wxCheckListBox,isChecked,2}, - {wxCheckListBox,new,3}, - {wxClientDC,new,1}, - {wxCursor,destroy,1}, - {wxCursor,new,1}, - {wxDC,clear,1}, - {wxDC,drawLine,3}, - {wxDC,drawLines,2}, - {wxDC,drawRoundedRectangle,4}, - {wxDC,drawText,3}, - {wxDC,getCharHeight,1}, - {wxDC,getCharWidth,1}, - {wxDC,getMultiLineTextExtent,2}, - {wxDC,getSize,1}, - {wxDC,getTextExtent,2}, - {wxDC,setBrush,2}, - {wxDC,setFont,2}, - {wxDC,setPen,2}, - {wxDC,setTextForeground,2}, - {wxDialog,createButtonSizer,2}, - {wxDialog,destroy,1}, - {wxDialog,endModal,2}, - {wxDialog,getAffirmativeId,1}, - {wxDialog,new,4}, - {wxDialog,show,1}, - {wxDialog,showModal,1}, - {wxEvent,skip,1}, - {wxFileDialog,destroy,1}, - {wxFileDialog,getDirectory,1}, - {wxFileDialog,getFilename,1}, - {wxFileDialog,getPath,1}, - {wxFileDialog,new,2}, - {wxFileDialog,showModal,1}, - {wxFont,getFamily,1}, - {wxFont,getPointSize,1}, - {wxFont,isFixedWidth,1}, - {wxFont,new,4}, - {wxFont,setPointSize,2}, - {wxFrame,center,1}, - {wxFrame,connect,2}, - {wxFrame,connect,3}, - {wxFrame,createStatusBar,2}, - {wxFrame,destroy,1}, - {wxFrame,new,4}, - {wxFrame,raise,1}, - {wxFrame,setFocus,1}, - {wxFrame,setIcon,2}, - {wxFrame,setMenuBar,2}, - {wxFrame,setStatusBar,2}, - {wxFrame,setTitle,2}, - {wxFrame,show,1}, - {wxGauge,new,4}, - {wxGauge,pulse,1}, - {wxGauge,setValue,2}, - {wxGraphicsContext,create,1}, - {wxGraphicsContext,destroy,1}, - {wxGraphicsContext,drawRoundedRectangle,6}, - {wxGraphicsContext,drawText,4}, - {wxGraphicsContext,getTextExtent,2}, - {wxGraphicsContext,setBrush,2}, - {wxGraphicsContext,setFont,3}, - {wxGraphicsContext,setPen,2}, - {wxGraphicsContext,strokeLine,5}, - {wxGraphicsContext,strokeLines,2}, - {wxGraphicsContext,translate,3}, - {wxGraphicsRenderer,getDefaultRenderer,0}, - {wxHtmlWindow,connect,2}, - {wxHtmlWindow,new,2}, - {wxHtmlWindow,setPage,2}, - {wxIcon,destroy,1}, - {wxIcon,new,2}, - {wxListBox,append,3}, - {wxListBox,appendStrings,2}, - {wxListBox,clear,1}, - {wxListBox,connect,2}, - {wxListBox,connect,3}, - {wxListBox,delete,2}, - {wxListBox,disconnect,1}, - {wxListBox,getClientData,2}, - {wxListBox,getCount,1}, - {wxListBox,getSelection,1}, - {wxListBox,getString,2}, - {wxListBox,getStringSelection,1}, - {wxListBox,new,3}, - {wxListBox,setClientData,3}, - {wxListBox,setSelection,2}, - {wxListBox,setString,3}, - {wxListCtrl,connect,2}, - {wxListCtrl,connect,3}, - {wxListCtrl,deleteAllItems,1}, - {wxListCtrl,ensureVisible,2}, - {wxListCtrl,getColumnCount,1}, - {wxListCtrl,getColumnWidth,2}, - {wxListCtrl,getItemCount,1}, - {wxListCtrl,getItemText,2}, - {wxListCtrl,getNextItem,3}, - {wxListCtrl,getParent,1}, - {wxListCtrl,insertColumn,3}, - {wxListCtrl,insertItem,3}, - {wxListCtrl,new,2}, - {wxListCtrl,refreshItem,2}, - {wxListCtrl,refreshItems,3}, - {wxListCtrl,setColumnWidth,3}, - {wxListCtrl,setItem,4}, - {wxListCtrl,setItemBackgroundColour,3}, - {wxListCtrl,setItemCount,2}, - {wxListCtrl,setItemState,4}, - {wxListCtrl,setToolTip,2}, - {wxListItem,destroy,1}, - {wxListItem,new,0}, - {wxListItem,setAlign,2}, - {wxListItem,setText,2}, - {wxListItemAttr,destroy,1}, - {wxListItemAttr,new,3}, - {wxMenu,'Destroy',2}, - {wxMenu,append,3}, - {wxMenu,append,4}, - {wxMenu,appendSeparator,1}, - {wxMenu,check,3}, - {wxMenu,connect,2}, - {wxMenu,connect,3}, - {wxMenu,delete,2}, - {wxMenu,destroy,1}, - {wxMenu,findItem,2}, - {wxMenu,getMenuItems,1}, - {wxMenu,insert,3}, - {wxMenu,insert,4}, - {wxMenu,insertCheckItem,5}, - {wxMenu,insertRadioItem,5}, - {wxMenu,insertSeparator,2}, - {wxMenu,new,0}, - {wxMenuBar,append,3}, - {wxMenuBar,check,3}, - {wxMenuBar,findMenu,2}, - {wxMenuBar,getMenu,2}, - {wxMenuBar,insert,4}, - {wxMenuBar,new,0}, - {wxMenuBar,remove,2}, - {wxMessageDialog,destroy,1}, - {wxMessageDialog,new,2}, - {wxMessageDialog,new,3}, - {wxMessageDialog,setTitle,2}, - {wxMessageDialog,showModal,1}, - {wxMiniFrame,new,4}, - {wxNotebook,addPage,3}, - {wxNotebook,addPage,4}, - {wxNotebook,connect,2}, - {wxNotebook,connect,3}, - {wxNotebook,getPageText,2}, - {wxNotebook,getSelection,1}, - {wxNotebook,new,3}, - {wxNotebook,setSelection,2}, - {wxPaintDC,new,1}, - {wxPanel,connect,2}, - {wxPanel,connect,3}, - {wxPanel,new,1}, - {wxPanel,new,2}, - {wxPanel,setSizer,2}, - {wxPanel,setSizerAndFit,3}, - {wxPen,new,1}, - {wxPen,new,2}, - {wxRadioBox,getSelection,1}, - {wxRadioBox,new,7}, - {wxRadioButton,getValue,1}, - {wxRadioButton,new,3}, - {wxRadioButton,new,4}, - {wxRadioButton,setValue,2}, - {wxScrolledWindow,calcScrolledPosition,2}, - {wxScrolledWindow,calcUnscrolledPosition,3}, - {wxScrolledWindow,doPrepareDC,2}, - {wxScrolledWindow,enableScrolling,3}, - {wxScrolledWindow,new,1}, - {wxScrolledWindow,new,2}, - {wxScrolledWindow,setScrollbars,5}, - {wxScrolledWindow,setSizer,2}, - {wxSizer,add,2}, - {wxSizer,add,3}, - {wxSizer,add,4}, - {wxSizer,addSpacer,2}, - {wxSizer,detach,2}, - {wxSizer,getChildren,1}, - {wxSizer,getMinSize,1}, - {wxSizer,hide,2}, - {wxSizer,layout,1}, - {wxSizer,recalcSizes,1}, - {wxSizer,remove,2}, - {wxSizer,setItemMinSize,4}, - {wxSizer,setSizeHints,2}, - {wxSizer,setVirtualSizeHints,2}, - {wxSizer,show,2}, - {wxSizerItem,deleteWindows,1}, - {wxSlider,connect,3}, - {wxSlider,getValue,1}, - {wxSlider,new,6}, - {wxSlider,setValue,2}, - {wxSplitterWindow,new,2}, - {wxSplitterWindow,setMinimumPaneSize,2}, - {wxSplitterWindow,setSashGravity,2}, - {wxSplitterWindow,splitHorizontally,4}, - {wxSplitterWindow,splitVertically,4}, - {wxStaticBoxSizer,new,3}, - {wxStaticLine,new,2}, - {wxStaticText,destroy,1}, - {wxStaticText,new,3}, - {wxStaticText,new,4}, - {wxStaticText,setForegroundColour,2}, - {wxStaticText,setLabel,2}, - {wxStatusBar,new,1}, - {wxStatusBar,setStatusText,2}, - {wxStyledTextCtrl,getText,1}, - {wxStyledTextCtrl,new,1}, - {wxStyledTextCtrl,setKeyWords,3}, - {wxStyledTextCtrl,setLexer,2}, - {wxStyledTextCtrl,setMarginType,3}, - {wxStyledTextCtrl,setSelectionMode,2}, - {wxStyledTextCtrl,setText,2}, - {wxStyledTextCtrl,setUseHorizontalScrollBar,2}, - {wxStyledTextCtrl,styleClearAll,1}, - {wxStyledTextCtrl,styleSetFont,3}, - {wxStyledTextCtrl,styleSetForeground,3}, - {wxSystemOptions,setOption,2}, - {wxSystemSettings,getColour,1}, - {wxSystemSettings,getFont,1}, - {wxSystemSettings,getMetric,1}, - {wxTextAttr,destroy,1}, - {wxTextAttr,new,1}, - {wxTextAttr,new,2}, - {wxTextCtrl,appendText,2}, - {wxTextCtrl,clear,1}, - {wxTextCtrl,connect,2}, - {wxTextCtrl,connect,3}, - {wxTextCtrl,destroy,1}, - {wxTextCtrl,getTextExtent,2}, - {wxTextCtrl,getValue,1}, - {wxTextCtrl,new,2}, - {wxTextCtrl,new,3}, - {wxTextCtrl,saveFile,2}, - {wxTextCtrl,setDefaultStyle,2}, - {wxTextCtrl,setForegroundColour,2}, - {wxTextCtrl,setValue,2}, - {wxTextCtrl,writeText,2}, - {wxTextEntryDialog,destroy,1}, - {wxTextEntryDialog,getValue,1}, - {wxTextEntryDialog,new,2}, - {wxTextEntryDialog,new,3}, - {wxTextEntryDialog,showModal,1}, - {wxToggleButton,new,4}, - {wxToggleButton,setLabel,2}, - {wxToggleButton,setValue,2}, - {wxToolTip,new,1}, - {wxWindow,connect,2}, - {wxWindow,connect,3}, - {wxWindow,destroy,1}, - {wxWindow,disable,1}, - {wxWindow,enable,1}, - {wxWindow,enable,2}, - {wxWindow,findWindowById,1}, - {wxWindow,freeze,1}, - {wxWindow,getBackgroundColour,1}, - {wxWindow,getClientSize,1}, - {wxWindow,getParent,1}, - {wxWindow,getSize,1}, - {wxWindow,getTextExtent,2}, - {wxWindow,getTextExtent,3}, - {wxWindow,hasScrollbar,2}, - {wxWindow,layout,1}, - {wxWindow,popupMenu,2}, - {wxWindow,refresh,1}, - {wxWindow,setBackgroundStyle,2}, - {wxWindow,setClientSize,2}, - {wxWindow,setCursor,2}, - {wxWindow,setFocus,1}, - {wxWindow,setForegroundColour,2}, - {wxWindow,setMinSize,2}, - {wxWindow,setSizer,2}, - {wxWindow,setSizerAndFit,2}, - {wxWindow,setToolTip,2}, - {wxWindow,setVirtualSize,3}, - {wxWindow,thaw,1}, - {wxWindowDC,new,1}, - {wx_misc,beginBusyCursor,0}, - {wx_misc,endBusyCursor,0}, - {wx_misc,getKeyState,1}, - {wx_misc,launchDefaultBrowser,1}, - {wx_object,call,2}, - {wx_object,cast,2}, - {wx_object,get_pid,1}, - {wx_object,start,3}, - {wx_object,start_link,3}, - {wx_object,start_link,4}, - {wx_object,stop,1}, - {wxe_util,get_const,1}, {xref,add_application,3}, {xref,analyze,2}, {xref,set_default,3}, From aee1350b1d4f5215260ebeb5450cc23677c12973 Mon Sep 17 00:00:00 2001 From: William Yang Date: Thu, 1 Sep 2022 08:54:55 +0200 Subject: [PATCH 03/61] ci(xref): remove reqplayq --- scripts/xref_check.eterm | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/scripts/xref_check.eterm b/scripts/xref_check.eterm index f1fa0f2f4..14692e6ce 100644 --- a/scripts/xref_check.eterm +++ b/scripts/xref_check.eterm @@ -24,6 +24,8 @@ {{coap_observer,init,1},{coap_udp_socket,start_link,0}}, {{coap_observer,terminate,2},{coap_dtls_socket,close,1}}, {{coap_observer,terminate,2},{coap_udp_socket,close,1}}, + + %% Exclude hipe which is disabled {{code,beam_file_native_md5,2},{hipe_unified_loader,chunk_name,1}}, {{code,get_native_fun,0},{hipe_unified_loader,chunk_name,1}}, {{code,load_native_code_for_all_loaded,1}, @@ -34,17 +36,14 @@ {hipe_unified_loader,load_native_code,3}}, {{compile,embed_native_code,2},{hipe_unified_loader,chunk_name,1}}, {{compile,native_compile_1,2},{hipe,compile,4}}, + + {{core_link,test,0},{eunit,test,1}}, {{cuttlefish_rebar_plugin,generate,2},{rebar_rel_utils,get_target_dir,2}}, {{cuttlefish_rebar_plugin,make_default_file,3},{rebar_config,get_local,3}}, {{cuttlefish_rebar_plugin,should_i_run,2},{rebar_rel_utils,is_rel_dir,0}}, {{cuttlefish_rebar_plugin,should_i_run,2},{rebar_rel_utils,load_config,2}}, - {{emqx_bridge_worker,common,4},{replayq,append,2}}, - {{emqx_bridge_worker,drop_acked_batches,2},{replayq,ack,2}}, - {{emqx_bridge_worker,open_replayq,1},{replayq,open,1}}, - {{emqx_bridge_worker,pop_and_send_loop,2},{replayq,is_empty,1}}, - {{emqx_bridge_worker,pop_and_send_loop,2},{replayq,pop,2}}, - {{emqx_bridge_worker,terminate,3},{replayq,close,1}}, + {{emqx_misc,ipv6_probe,1},{gen_tcp,ipv6_probe,0}}, {{eredis_parser,test,0},{eunit,test,1}}, {{gen_rpc_driver_ssl,authenticate_client,3}, @@ -391,23 +390,20 @@ {rebar_rel_utils,get_target_dir,2}, {rebar_rel_utils,is_rel_dir,0}, {rebar_rel_utils,load_config,2}, - {replayq,ack,2}, - {replayq,append,2}, - {replayq,close,1}, - {replayq,is_empty,1}, - {replayq,open,1}, - {replayq,pop,2}, {ssl_verify_hostname,verify_cert_hostname,2}, {ssl_verify_hostname,verify_fun,3}, {tftp,start_service,1}, {tftp,start_standalone,1}, {tftp,stop_service,1}, + + %% Xref could not check itself {xref,add_application,3}, {xref,analyze,2}, {xref,set_default,3}, {xref,set_library_path,2}, {xref,start,2}, - {xref,stop,1}] + {xref,stop,1} + ] } , #{ name => locals_not_used From 8b6b9a0d202fb8b2dcedba4377b8da2d5cad7b65 Mon Sep 17 00:00:00 2001 From: William Yang Date: Thu, 1 Sep 2022 11:52:09 +0200 Subject: [PATCH 04/61] ci(xref): excl release related modules --- scripts/xref-check.escript | 6 ++++- scripts/xref_check.eterm | 45 +++++++++++++++++--------------------- 2 files changed, 25 insertions(+), 26 deletions(-) diff --git a/scripts/xref-check.escript b/scripts/xref-check.escript index 65c0fa857..e0ff1d3a7 100755 --- a/scripts/xref-check.escript +++ b/scripts/xref-check.escript @@ -1,5 +1,10 @@ #!/usr/bin/env escript +%%% @doc +%%% Unlike rebar3 xref check, this script runs the full xref checks in the EMQX release dir, +%%% meaning all the modules for release are analysed. +%%% For behavior configuration, all rebar3 related modules attributes, filters are not used in this script, +%%% instead all the filters, checks are defined in `xref_check.eterm` main(_) -> {ok, [Jobs]} = file:consult("scripts/xref_check.eterm"), lists:foreach(fun(#{ name := Name @@ -20,7 +25,6 @@ main(_) -> [ok = xref:remove_module(Name, M) || M <- ExclMods ], - ModuleInfos = xref:info(Name, modules), LibInfos = xref:info(Name, modules), true = ets:insert(Tid, ModuleInfos ++ LibInfos), diff --git a/scripts/xref_check.eterm b/scripts/xref_check.eterm index 14692e6ce..9616173ca 100644 --- a/scripts/xref_check.eterm +++ b/scripts/xref_check.eterm @@ -8,7 +8,11 @@ , excl_mods => [ hipe_unified_loader , systools_make + , systools + , systools_relup , basho_bench_driver_erldis + , release_handler + , cuttlefish_rebar_plugin ] , filters => [{{coap_client,channel_apply,3},{coap_dtls_socket,close,1}}, @@ -39,11 +43,6 @@ {{core_link,test,0},{eunit,test,1}}, - {{cuttlefish_rebar_plugin,generate,2},{rebar_rel_utils,get_target_dir,2}}, - {{cuttlefish_rebar_plugin,make_default_file,3},{rebar_config,get_local,3}}, - {{cuttlefish_rebar_plugin,should_i_run,2},{rebar_rel_utils,is_rel_dir,0}}, - {{cuttlefish_rebar_plugin,should_i_run,2},{rebar_rel_utils,load_config,2}}, - {{emqx_misc,ipv6_probe,1},{gen_tcp,ipv6_probe,0}}, {{eredis_parser,test,0},{eunit,test,1}}, {{gen_rpc_driver_ssl,authenticate_client,3}, @@ -230,18 +229,8 @@ {{ranch_app,consider_profiling,0},{eprof,start_profiling,1}}, {{ranch_app,profile_output,0},{eprof,analyze,1}}, {{ranch_app,profile_output,0},{eprof,log,1}}, - {{ranch_app,profile_output,0},{eprof,stop_profiling,0}}, - {{release_handler,new_emulator_make_hybrid_boot,6}, - {systools_make,make_hybrid_boot,4}}, - {{release_handler,read_app,3},{systools_make,read_application,4}}, - {{systools,compile_rel,3},{systools_make,make_script,3}}, - {{systools,make_script,2},{systools_make,make_script,2}}, - {{systools,make_tar,2},{systools_make,make_tar,2}}, - {{systools,script2boot,1},{systools_make,format_error,1}}, - {{systools,script2boot,3},{systools_make,format_error,1}}, - {{systools_relup,do_mk_relup,5},{systools_make,get_release,2}}, - {{systools_relup,foreach_baserel_dn,7},{systools_make,get_release,2}}, - {{systools_relup,foreach_baserel_up,7},{systools_make,get_release,2}}] + {{ranch_app,profile_output,0},{eprof,stop_profiling,0}} + ] } %% Check undefined_functions @@ -251,7 +240,11 @@ [ observer ] , excl_mods => - [ + [ systools + , systools_make + , release_handler + , systools_relup + , cuttlefish_rebar_plugin ] , filters => [{'Elixir.Atom',to_string,1}, @@ -302,6 +295,8 @@ {gen_leader,leader_node,1}, {gen_leader,start_link,6}, {gen_tcp,ipv6_probe,0}, + + %% We dont use hipes {hipe,compile,4}, {hipe_bifs,add_ref,2}, {hipe_bifs,alloc_data,3}, @@ -326,6 +321,8 @@ {hipe_bifs,write_u32,2}, {hipe_bifs,write_u64,2}, {hipe_bifs,write_u8,2}, + + {jsone,decode,1}, {jsone,encode,2}, {jsx,decode,2}, @@ -386,10 +383,6 @@ {lwm2m_coap_udp_socket,start_link,0}, {ojson,'decode!',1}, {ojson,'encode!',1}, - {rebar_config,get_local,3}, - {rebar_rel_utils,get_target_dir,2}, - {rebar_rel_utils,is_rel_dir,0}, - {rebar_rel_utils,load_config,2}, {ssl_verify_hostname,verify_cert_hostname,2}, {ssl_verify_hostname,verify_fun,3}, {tftp,start_service,1}, @@ -410,7 +403,6 @@ , analysis => locals_not_used , excl_apps => [ - ] , excl_mods => [ @@ -471,6 +463,11 @@ {grpc_reflection_pb,m_overwrite,3}, {grpc_reflection_pb,v_ok,3}, {prometheus_text_format,escape_metric_help,1}, + + %% + %% To be used in grammar files to throw an error message to the parser + %% toplevel. Doesn't have to be exported! + %% {redbug_parser,return_error,2}, {rulesql,return_error,2}, {xmerl_b64Bin,return_error,2}, @@ -481,7 +478,6 @@ , analysis => deprecated_function_calls , excl_apps => [ - ] , excl_mods => [ @@ -517,7 +513,6 @@ , analysis => deprecated_functions , excl_apps => [ - ] , excl_mods => [ From fef088220cada23c8b586d4ef49eecfc30e90305 Mon Sep 17 00:00:00 2001 From: William Yang Date: Fri, 2 Sep 2022 10:06:09 +0200 Subject: [PATCH 05/61] ci(xref): handle non-existing apps/modules --- scripts/xref-check.escript | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/scripts/xref-check.escript b/scripts/xref-check.escript index e0ff1d3a7..4bf5e95f1 100755 --- a/scripts/xref-check.escript +++ b/scripts/xref-check.escript @@ -18,12 +18,16 @@ main(_) -> xref:set_default(Name, [{verbose,false}, {warnings,false}]), xref:add_release(Name, "_build/emqx/rel/emqx/lib/"), xref:add_application(Name, code:lib_dir(erts)), - [ok = xref:remove_application(Name, App) || - App <- ExclApps + [ case xref:remove_application(Name, App) of + ok -> ok; + {error, xref_base, {no_such_application, _}} -> ok + end || App <- ExclApps ], - [ok = xref:remove_module(Name, M) || - M <- ExclMods + [case xref:remove_module(Name, M) of + ok -> ok; + {error, M, _R} -> ok + end || M <- ExclMods ], ModuleInfos = xref:info(Name, modules), LibInfos = xref:info(Name, modules), From 9dd5e26ddfa4ad703b6da6f5b214bf4b500abb3c Mon Sep 17 00:00:00 2001 From: William Yang Date: Fri, 2 Sep 2022 10:27:31 +0200 Subject: [PATCH 06/61] ci(xref): handle no_such_module --- scripts/xref-check.escript | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/xref-check.escript b/scripts/xref-check.escript index 4bf5e95f1..36c4c57c2 100755 --- a/scripts/xref-check.escript +++ b/scripts/xref-check.escript @@ -25,8 +25,9 @@ main(_) -> ], [case xref:remove_module(Name, M) of - ok -> ok; - {error, M, _R} -> ok + ok -> ok; + %% but in doc it should return '{error, module(), Reason}` + {error, xref_base, {no_such_module, M}} -> ok end || M <- ExclMods ], ModuleInfos = xref:info(Name, modules), From 820b60f44898fafa3a58754b639c527a068608b9 Mon Sep 17 00:00:00 2001 From: William Yang Date: Mon, 5 Sep 2022 10:56:17 +0200 Subject: [PATCH 07/61] build: add rel to xref deps --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 0f651098c..998eaec56 100644 --- a/Makefile +++ b/Makefile @@ -114,7 +114,7 @@ $(PROFILES:%=deps-%): $(REBAR) get-dashboard @rm -f rebar.lock .PHONY: xref -xref: $(REBAR) +xref: $(REBAR) $(REL_PROFILES:%=%-rel) @$(REBAR) as check xref @scripts/xref-check.escript From 432eae6e55e2aaffe0419210152e5d77568c3214 Mon Sep 17 00:00:00 2001 From: William Yang Date: Tue, 6 Sep 2022 10:24:19 +0200 Subject: [PATCH 08/61] ci(xref): support emqx-ee profile --- scripts/xref-check.escript | 78 +++++++++++++++++++++----------------- scripts/xref_check.eterm | 1 + 2 files changed, 44 insertions(+), 35 deletions(-) diff --git a/scripts/xref-check.escript b/scripts/xref-check.escript index 36c4c57c2..a257734d4 100755 --- a/scripts/xref-check.escript +++ b/scripts/xref-check.escript @@ -7,44 +7,52 @@ %%% instead all the filters, checks are defined in `xref_check.eterm` main(_) -> {ok, [Jobs]} = file:consult("scripts/xref_check.eterm"), - lists:foreach(fun(#{ name := Name - , analysis := Analysis - , excl_apps := ExclApps - , excl_mods := ExclMods - , filters := Filters - }) -> - xref:start(Name), - Tid = ets:new(Name, [ordered_set, named_table]), - xref:set_default(Name, [{verbose,false}, {warnings,false}]), - xref:add_release(Name, "_build/emqx/rel/emqx/lib/"), - xref:add_application(Name, code:lib_dir(erts)), - [ case xref:remove_application(Name, App) of - ok -> ok; - {error, xref_base, {no_such_application, _}} -> ok - end || App <- ExclApps - ], - - [case xref:remove_module(Name, M) of - ok -> ok; - %% but in doc it should return '{error, module(), Reason}` - {error, xref_base, {no_such_module, M}} -> ok - end || M <- ExclMods - ], - ModuleInfos = xref:info(Name, modules), - LibInfos = xref:info(Name, modules), - true = ets:insert(Tid, ModuleInfos ++ LibInfos), - {ok, Res0} = xref:analyse(Name, Analysis), - Res = Res0 -- Filters, - Res =/= [] andalso - begin - put(is_warn_found, true), - io:format("** Warnings for ~p~n : ~p~n", [Name, Res]) - end, - xref:stop(Name) - end, Jobs), + lists:foreach(fun(J) -> do_check(J) end, Jobs), case get(is_warn_found) of true -> halt(1); _ -> ok end. + +do_check(#{ name := Name + , analysis := Analysis + , excl_apps := ExclApps + , excl_mods := ExclMods + , filters := Filters + }) -> + xref:start(Name), + %% Build a table for later printing more informative warnings. + %% The table is currently not in use. + Tid = ets:new(Name, [ordered_set, named_table]), + xref:set_default(Name, [{verbose,false}, {warnings,false}]), + Profile = case filelib:is_file("EMQX_ENTERPRISE") of + true -> 'emqx-ee'; + false -> emqx + end, + Dir = filename:join(["_build/", Profile, "rel/emqx/lib/"]), + xref:add_release(Name, Dir), + xref:add_application(Name, code:lib_dir(erts)), + [ case xref:remove_application(Name, App) of + ok -> ok; + {error, xref_base, {no_such_application, _}} -> ok + end || App <- ExclApps + ], + + [case xref:remove_module(Name, M) of + ok -> ok; + %% but in doc it should return '{error, module(), Reason}` + {error, xref_base, {no_such_module, M}} -> ok + end || M <- ExclMods + ], + ModuleInfos = xref:info(Name, modules), + LibInfos = xref:info(Name, libraries), + true = ets:insert(Tid, ModuleInfos ++ LibInfos), + {ok, Res0} = xref:analyse(Name, Analysis), + Res = Res0 -- Filters, + Res =/= [] andalso + begin + put(is_warn_found, true), + io:format("** Warnings for ~p~n : ~p~n", [Name, Res]) + end, + xref:stop(Name). diff --git a/scripts/xref_check.eterm b/scripts/xref_check.eterm index 9616173ca..d09bee6a8 100644 --- a/scripts/xref_check.eterm +++ b/scripts/xref_check.eterm @@ -1,3 +1,4 @@ +%% -*- mode: erlang; -*- [ %% Check undefined_function_calls #{ name => undefined_function_calls , analysis => undefined_function_calls From 0d5c32a706dac993116d650b669d534ce73d8213 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 7 Sep 2022 10:14:39 +0800 Subject: [PATCH 09/61] fix(time): use erlang:system_time/0-1 consistently Avoid the problem of inaccurate timers caused by mixing erlang:system_time/0-1 and os:system_time/0-1 --- apps/emqx_auth_jwt/src/emqx_auth_jwt_svr.erl | 2 +- .../test/emqx_auth_jwt_SUITE.erl | 28 +++++++++---------- lib-ce/emqx_modules/src/emqx_mod_delayed.erl | 6 ++-- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt_svr.erl b/apps/emqx_auth_jwt/src/emqx_auth_jwt_svr.erl index 3a5c619aa..049f07533 100644 --- a/apps/emqx_auth_jwt/src/emqx_auth_jwt_svr.erl +++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt_svr.erl @@ -200,7 +200,7 @@ do_verify(JwsCompacted, [Jwk|More]) -> end. check_claims(Claims) -> - Now = os:system_time(seconds), + Now = erlang:system_time(seconds), Checker = [{<<"exp">>, with_num_value( fun(ExpireTime) -> Now < ExpireTime end)}, {<<"iat">>, with_num_value( diff --git a/apps/emqx_auth_jwt/test/emqx_auth_jwt_SUITE.erl b/apps/emqx_auth_jwt/test/emqx_auth_jwt_SUITE.erl index 596b829a6..7da708b13 100644 --- a/apps/emqx_auth_jwt/test/emqx_auth_jwt_SUITE.erl +++ b/apps/emqx_auth_jwt/test/emqx_auth_jwt_SUITE.erl @@ -74,7 +74,7 @@ t_check_auth(_Config) -> Plain = #{clientid => <<"client1">>, username => <<"plain">>, zone => external}, Jwt = sign([{clientid, <<"client1">>}, {username, <<"plain">>}, - {exp, os:system_time(seconds) + 2}], <<"HS256">>, <<"emqxsecret">>), + {exp, erlang:system_time(seconds) + 2}], <<"HS256">>, <<"emqxsecret">>), ct:pal("Jwt: ~p~n", [Jwt]), Result0 = emqx_access_control:authenticate(Plain#{password => Jwt}), @@ -100,7 +100,7 @@ t_check_nbf(_Config) -> Plain = #{clientid => <<"client1">>, username => <<"plain">>, zone => external}, Jwt = sign([{clientid, <<"client1">>}, {username, <<"plain">>}, - {nbf, os:system_time(seconds) + 3}], <<"HS256">>, <<"emqxsecret">>), + {nbf, erlang:system_time(seconds) + 3}], <<"HS256">>, <<"emqxsecret">>), ct:pal("Jwt: ~p~n", [Jwt]), Result0 = emqx_access_control:authenticate(Plain#{password => Jwt}), @@ -113,7 +113,7 @@ t_check_iat(_Config) -> Plain = #{clientid => <<"client1">>, username => <<"plain">>, zone => external}, Jwt = sign([{clientid, <<"client1">>}, {username, <<"plain">>}, - {iat, os:system_time(seconds) + 3}], <<"HS256">>, <<"emqxsecret">>), + {iat, erlang:system_time(seconds) + 3}], <<"HS256">>, <<"emqxsecret">>), ct:pal("Jwt: ~p~n", [Jwt]), Result0 = emqx_access_control:authenticate(Plain#{password => Jwt}), @@ -146,7 +146,7 @@ t_check_auth_str_exp(init, _Config) -> application:unset_env(emqx_auth_jwt, verify_claims). t_check_auth_str_exp(_Config) -> Plain = #{clientid => <<"client1">>, username => <<"plain">>, zone => external}, - Exp = integer_to_binary(os:system_time(seconds) + 3), + Exp = integer_to_binary(erlang:system_time(seconds) + 3), Jwt0 = sign([{clientid, <<"client1">>}, {username, <<"plain">>}, @@ -166,7 +166,7 @@ t_check_auth_str_exp(_Config) -> ct:pal("Auth result: ~p~n", [Result1]), ?assertMatch({error, _}, Result1), - Exp2 = float_to_binary(os:system_time(seconds) + 3.5), + Exp2 = float_to_binary(erlang:system_time(seconds) + 3.5), Jwt2 = sign([{clientid, <<"client1">>}, {username, <<"plain">>}, @@ -181,7 +181,7 @@ t_check_auth_float_exp(init, _Config) -> application:unset_env(emqx_auth_jwt, verify_claims). t_check_auth_float_exp(_Config) -> Plain = #{clientid => <<"client1">>, username => <<"plain">>, zone => external}, - Exp = os:system_time(seconds) + 3.5, + Exp = erlang:system_time(seconds) + 3.5, Jwt0 = sign([{clientid, <<"client1">>}, {username, <<"plain">>}, @@ -208,7 +208,7 @@ t_check_claims(_Config) -> Jwt = sign([{client_id, <<"client1">>}, {username, <<"plain">>}, {sub, value}, - {exp, os:system_time(seconds) + 3}], <<"HS256">>, <<"emqxsecret">>), + {exp, erlang:system_time(seconds) + 3}], <<"HS256">>, <<"emqxsecret">>), Result0 = emqx_access_control:authenticate(Plain#{password => Jwt}), ct:pal("Auth result: ~p~n", [Result0]), ?assertMatch({ok, #{auth_result := success, jwt_claims := _}}, Result0), @@ -224,7 +224,7 @@ t_check_claims_clientid(_Config) -> Plain = #{clientid => <<"client23">>, username => <<"plain">>, zone => external}, Jwt = sign([{clientid, <<"client23">>}, {username, <<"plain">>}, - {exp, os:system_time(seconds) + 3}], <<"HS256">>, <<"emqxsecret">>), + {exp, erlang:system_time(seconds) + 3}], <<"HS256">>, <<"emqxsecret">>), Result0 = emqx_access_control:authenticate(Plain#{password => Jwt}), ct:pal("Auth result: ~p~n", [Result0]), ?assertMatch({ok, #{auth_result := success, jwt_claims := _}}, Result0), @@ -240,7 +240,7 @@ t_check_claims_username(_Config) -> Plain = #{clientid => <<"client23">>, username => <<"plain">>, zone => external}, Jwt = sign([{client_id, <<"client23">>}, {username, <<"plain">>}, - {exp, os:system_time(seconds) + 3}], <<"HS256">>, <<"emqxsecret">>), + {exp, erlang:system_time(seconds) + 3}], <<"HS256">>, <<"emqxsecret">>), Result0 = emqx_access_control:authenticate(Plain#{password => Jwt}), ct:pal("Auth result: ~p~n", [Result0]), ?assertMatch({ok, #{auth_result := success, jwt_claims := _}}, Result0), @@ -256,7 +256,7 @@ t_check_claims_kid_in_header(_Config) -> Plain = #{clientid => <<"client23">>, username => <<"plain">>, zone => external}, Jwt = sign([{clientid, <<"client23">>}, {username, <<"plain">>}, - {exp, os:system_time(seconds) + 3}], + {exp, erlang:system_time(seconds) + 3}], #{<<"alg">> => <<"HS256">>, <<"kid">> => <<"a_kid_str">>}, <<"emqxsecret">>), Result0 = emqx_access_control:authenticate(Plain#{password => Jwt}), @@ -297,7 +297,7 @@ t_check_jwt_acl(_Config) -> {sub, value}, {acl, [{sub, [<<"a/b">>]}, {pub, [<<"c/d">>]}]}, - {exp, os:system_time(seconds) + 10}], + {exp, erlang:system_time(seconds) + 10}], <<"HS256">>, <<"emqxsecret">>), @@ -337,7 +337,7 @@ t_check_jwt_acl_no_recs(_Config) -> {username, <<"plain">>}, {sub, value}, {acl, []}, - {exp, os:system_time(seconds) + 10}], + {exp, erlang:system_time(seconds) + 10}], <<"HS256">>, <<"emqxsecret">>), @@ -360,7 +360,7 @@ t_check_jwt_acl_no_acl_claim(_Config) -> Jwt = sign([{client_id, <<"client1">>}, {username, <<"plain">>}, {sub, value}, - {exp, os:system_time(seconds) + 10}], + {exp, erlang:system_time(seconds) + 10}], <<"HS256">>, <<"emqxsecret">>), @@ -384,7 +384,7 @@ t_check_jwt_acl_expire(_Config) -> {username, <<"plain">>}, {sub, value}, {acl, [{sub, [<<"a/b">>]}]}, - {exp, os:system_time(seconds) + 1}], + {exp, erlang:system_time(seconds) + 1}], <<"HS256">>, <<"emqxsecret">>), diff --git a/lib-ce/emqx_modules/src/emqx_mod_delayed.erl b/lib-ce/emqx_modules/src/emqx_mod_delayed.erl index 4fed5e191..3525b8449 100644 --- a/lib-ce/emqx_modules/src/emqx_mod_delayed.erl +++ b/lib-ce/emqx_modules/src/emqx_mod_delayed.erl @@ -160,7 +160,7 @@ handle_cast(Msg, State) -> %% Do Publish... handle_info({timeout, TRef, do_publish}, State = #{timer := TRef}) -> - DeletedKeys = do_publish(mnesia:dirty_first(?TAB), os:system_time(seconds)), + DeletedKeys = do_publish(mnesia:dirty_first(?TAB), erlang:system_time(seconds)), lists:foreach(fun(Key) -> mnesia:dirty_delete(?TAB, Key) end, DeletedKeys), {noreply, ensure_publish_timer(State#{timer := undefined, publish_at := 0})}; @@ -203,11 +203,11 @@ ensure_publish_timer(State) -> ensure_publish_timer('$end_of_table', State) -> State#{timer := undefined, publish_at := 0}; ensure_publish_timer({Ts, _Id}, State = #{timer := undefined}) -> - ensure_publish_timer(Ts, os:system_time(seconds), State); + ensure_publish_timer(Ts, erlang:system_time(seconds), State); ensure_publish_timer({Ts, _Id}, State = #{timer := TRef, publish_at := PubAt}) when Ts < PubAt -> ok = emqx_misc:cancel_timer(TRef), - ensure_publish_timer(Ts, os:system_time(seconds), State); + ensure_publish_timer(Ts, erlang:system_time(seconds), State); ensure_publish_timer(_Key, State) -> State. From 271112ad68e3b34dae9ade73756b47ab8d0008fd Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 7 Sep 2022 10:35:46 +0800 Subject: [PATCH 10/61] chore: update changes --- CHANGES-4.3.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index 26e7075e2..2520478c5 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -16,6 +16,7 @@ File format: - Fix rule-engine update behaviour which may initialize actions for disabled rules. [#8849](https://github.com/emqx/emqx/pull/8849) - Fix JWT plugin don't support non-integer timestamp claims. [#8862](https://github.com/emqx/emqx/pull/8862) +- Fix delayed publish inaccurate caused by os time change. [#8908](https://github.com/emqx/emqx/pull/8908) ## v4.3.19 From 7b4fbfef3b7696ba7a8dcb05f38bada861d3c986 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 7 Sep 2022 11:03:29 +0800 Subject: [PATCH 11/61] chore: update app.src & appup.src --- lib-ce/emqx_modules/src/emqx_modules.app.src | 2 +- lib-ce/emqx_modules/src/emqx_modules.appup.src | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/lib-ce/emqx_modules/src/emqx_modules.app.src b/lib-ce/emqx_modules/src/emqx_modules.app.src index 055e4c780..0c3328f8c 100644 --- a/lib-ce/emqx_modules/src/emqx_modules.app.src +++ b/lib-ce/emqx_modules/src/emqx_modules.app.src @@ -1,6 +1,6 @@ {application, emqx_modules, [{description, "EMQ X Module Management"}, - {vsn, "4.3.9"}, + {vsn, "4.3.10"}, {modules, []}, {applications, [kernel,stdlib]}, {mod, {emqx_modules_app, []}}, diff --git a/lib-ce/emqx_modules/src/emqx_modules.appup.src b/lib-ce/emqx_modules/src/emqx_modules.appup.src index e4a41be8c..461eefa15 100644 --- a/lib-ce/emqx_modules/src/emqx_modules.appup.src +++ b/lib-ce/emqx_modules/src/emqx_modules.appup.src @@ -1,10 +1,14 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.8",[{load_module,emqx_modules,brutal_purge,soft_purge,[]}]}, + [{"4.3.9",[{load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}]}, + {"4.3.8", + [{load_module,emqx_modules,brutal_purge,soft_purge,[]}, + {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[5-7]">>, [{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}, - {load_module,emqx_modules,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_modules,brutal_purge,soft_purge,[]}, + {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}]}, {"4.3.4", [{load_module,emqx_modules,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_subscription,brutal_purge,soft_purge,[]}, @@ -31,10 +35,14 @@ {load_module,emqx_mod_api_topic_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.8",[{load_module,emqx_modules,brutal_purge,soft_purge,[]}]}, + [{"4.3.9",[{load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}]}, + {"4.3.8", + [{load_module,emqx_modules,brutal_purge,soft_purge,[]}, + {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[5-7]">>, [{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}, - {load_module,emqx_modules,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_modules,brutal_purge,soft_purge,[]}, + {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}]}, {"4.3.4", [{load_module,emqx_modules,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_subscription,brutal_purge,soft_purge,[]}, From c65fad30140d6cfed1ceb1cff5d8dad444426363 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 9 Sep 2022 10:51:58 -0300 Subject: [PATCH 12/61] ci: enable dialyzer checks (4.3) --- .github/workflows/build_slim_packages.yaml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 356ba491b..b5a8623ef 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -49,12 +49,7 @@ jobs: - name: make xref run: make xref - name: make dialyzer - run: | - # TODO: once dialyzer warnings are fixed in 4.4+, we may - # enforce this. - if [[ "$(./pkg-vsn.sh)" =~ 4.3.* ]]; then - make dialyzer - fi + run: make dialyzer - name: build zip packages run: make ${EMQX_NAME}-zip - name: build deb/rpm packages From 1f5103b390fe2d6241b462beefc5925ce53e6d89 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 5 Sep 2022 09:46:26 -0300 Subject: [PATCH 13/61] test: add test case for not publishing will msg when not authorized --- CHANGES-4.3.md | 2 + .../test/emqx_auth_mnesia_SUITE.erl | 59 +++++++++++++++++++ src/emqx_channel.erl | 28 ++++++--- 3 files changed, 82 insertions(+), 7 deletions(-) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index 0e320dbb4..10636eb6b 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -23,6 +23,8 @@ File format: `SELECT topic =~ 't' as r FROM "$events/client_connected"`. The topic is a null value as there's no such field in event `$events/client_connected`, so it should return false if match it to a topic. +- Added a test to prevent a last will testament message to be + published when a client is denied connection. [#8894](https://github.com/emqx/emqx/pull/8894) ## v4.3.19 diff --git a/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl b/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl index 0253110aa..f7071bc17 100644 --- a/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl +++ b/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl @@ -21,6 +21,9 @@ -include("emqx_auth_mnesia.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-include_lib("emqx/include/emqx_mqtt.hrl"). -import(emqx_ct_http, [ request_api/3 , request_api/5 @@ -74,6 +77,20 @@ set_default(ClientId, UserName, Pwd, HashType) -> application:set_env(emqx_auth_mnesia, username_list, [{UserName, Pwd}]), application:set_env(emqx_auth_mnesia, password_hash, HashType), ok. + +init_per_testcase(t_will_message_connection_denied, Config) -> + emqx_zone:set_env(external, allow_anonymous, false), + Config; +init_per_testcase(_TestCase, Config) -> + Config. + +end_per_testcase(t_will_message_connection_denied, _Config) -> + emqx_zone:unset_env(external, allow_anonymous), + application:stop(emqx_auth_mnesia), + ok; +end_per_testcase(_TestCase, _Config) -> + ok. + %%------------------------------------------------------------------------------ %% Testcases %%------------------------------------------------------------------------------ @@ -390,6 +407,48 @@ t_password_hash(_) -> application:stop(emqx_auth_mnesia), ok = application:start(emqx_auth_mnesia). +t_will_message_connection_denied(Config) when is_list(Config) -> + ClientId = Username = <<"subscriber">>, + Password = <<"p">>, + application:stop(emqx_auth_mnesia), + ok = emqx_ct_helpers:start_apps([emqx_auth_mnesia]), + ok = emqx_auth_mnesia_cli:add_user({clientid, ClientId}, Password), + + {ok, Subscriber} = emqtt:start_link([ + {clientid, ClientId}, + {password, Password} + ]), + {ok, _} = emqtt:connect(Subscriber), + {ok, _, [?RC_SUCCESS]} = emqtt:subscribe(Subscriber, <<"lwt">>), + + process_flag(trap_exit, true), + + {ok, Publisher} = emqtt:start_link([ + {clientid, <<"publisher">>}, + {will_topic, <<"lwt">>}, + {will_payload, <<"should not be published">>} + ]), + snabbkaffe:start_trace(), + ?wait_async_action( + {error, _} = emqtt:connect(Publisher), + #{?snk_kind := channel_terminated} + ), + snabbkaffe:stop(), + + timer:sleep(1000), + + receive + {publish, #{ + topic := <<"lwt">>, + payload := <<"should not be published">> + }} -> + ct:fail("should not publish will message") + after 0 -> + ok + end, + + ok. + %%------------------------------------------------------------------------------ %% Helpers %%------------------------------------------------------------------------------ diff --git a/src/emqx_channel.erl b/src/emqx_channel.erl index c49299c58..a7565533b 100644 --- a/src/emqx_channel.erl +++ b/src/emqx_channel.erl @@ -1156,20 +1156,34 @@ interval(will_timer, #channel{will_msg = WillMsg}) -> %%-------------------------------------------------------------------- -spec(terminate(any(), channel()) -> ok). -terminate(_, #channel{conn_state = idle}) -> ok; +terminate(_Reason, #channel{conn_state = idle} = _Channel) -> + ?tp(channel_terminated, #{channel => _Channel, reason => _Reason}), + ok; terminate(normal, Channel) -> run_terminate_hook(normal, Channel); -terminate({shutdown, Reason}, Channel) - when Reason =:= kicked; Reason =:= discarded; Reason =:= takeovered -> - run_terminate_hook(Reason, Channel); terminate(Reason, Channel = #channel{will_msg = WillMsg}) -> - (WillMsg =/= undefined) andalso publish_will_msg(WillMsg), + should_publish_will_message(Reason, Channel) + andalso publish_will_msg(WillMsg), run_terminate_hook(Reason, Channel). -run_terminate_hook(_Reason, #channel{session = undefined}) -> ok; -run_terminate_hook(Reason, #channel{clientinfo = ClientInfo, session = Session}) -> +run_terminate_hook(_Reason, #channel{session = undefined} = _Channel) -> + ?tp(channel_terminated, #{channel => _Channel, reason => _Reason}), + ok; +run_terminate_hook(Reason, #channel{clientinfo = ClientInfo, session = Session} = _Channel) -> + ?tp(channel_terminated, #{channel => _Channel, reason => Reason}), emqx_session:terminate(ClientInfo, Reason, Session). +should_publish_will_message(TerminateReason, Channel) -> + not lists:member(TerminateReason, [ {shutdown, kicked} + , {shutdown, discarded} + , {shutdown, takeovered} + , {shutdown, not_authorized} + ]) + andalso not lists:member(info(conn_state, Channel), [ idle + , connecting + ]) + andalso info(will_msg, Channel) =/= undefined. + %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- From d72ca84af088abdce6df42e3cd3109a97df0a0ed Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 9 Sep 2022 16:17:58 -0300 Subject: [PATCH 14/61] chore: fix ignored dialyzer warnings (part 1) (4.3) --- .../src/emqx_auth_pgsql_cli.erl | 1 - apps/emqx_exhook/src/emqx_exhook_server.erl | 8 +--- apps/emqx_exproto/src/emqx_exproto_conn.erl | 9 +---- .../src/emqx_mgmt_data_backup.erl | 2 - .../emqx_rule_engine/src/emqx_rule_engine.erl | 3 +- .../src/emqx_rule_engine_api.erl | 6 --- .../src/emqx_rule_engine_cli.erl | 1 - .../src/emqx_rule_runtime.erl | 1 - .../src/emqx_rule_sqltester.erl | 10 ----- apps/emqx_stomp/src/emqx_stomp_connection.erl | 9 +---- apps/emqx_stomp/src/emqx_stomp_protocol.erl | 38 +++++++++++-------- include/emqx.hrl | 9 +++-- lib-ce/emqx_telemetry/src/emqx_telemetry.erl | 6 --- src/emqx_alarm.erl | 2 - src/emqx_channel.erl | 30 +++++++-------- src/emqx_connection.erl | 10 +---- src/emqx_frame.erl | 4 -- src/emqx_limiter.erl | 2 - src/emqx_plugins.erl | 15 +++----- src/emqx_router.erl | 21 +++++----- src/emqx_router_helper.erl | 3 -- src/emqx_tracer.erl | 2 - src/emqx_types.erl | 2 - src/emqx_ws_connection.erl | 4 +- test/emqx_plugins_SUITE.erl | 10 ++--- 25 files changed, 71 insertions(+), 137 deletions(-) diff --git a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql_cli.erl b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql_cli.erl index f7170e92b..4905b32bf 100644 --- a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql_cli.erl +++ b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql_cli.erl @@ -147,4 +147,3 @@ safe_get(K, ClientInfo) -> bin(A) when is_atom(A) -> atom_to_binary(A, utf8); bin(B) when is_binary(B) -> B; bin(X) -> X. - diff --git a/apps/emqx_exhook/src/emqx_exhook_server.erl b/apps/emqx_exhook/src/emqx_exhook_server.erl index 9a8468700..b77aaa445 100644 --- a/apps/emqx_exhook/src/emqx_exhook_server.erl +++ b/apps/emqx_exhook/src/emqx_exhook_server.erl @@ -46,7 +46,7 @@ channel :: pid(), %% Registered hook names and options hookspec :: #{hookpoint() => map()}, - %% Metrcis name prefix + %% Metrics name prefix prefix :: list() }). @@ -75,8 +75,6 @@ -export_type([server/0]). --dialyzer({nowarn_function, [inc_metrics/2]}). - %%-------------------------------------------------------------------- %% Load/Unload APIs %%-------------------------------------------------------------------- @@ -249,10 +247,6 @@ call(Hookpoint, Req, #server{name = ChannName, options = ReqOpts, end. %% @private -inc_metrics(IncFun, Name) when is_function(IncFun) -> - %% BACKW: e4.2.0-e4.2.2 - {env, [Prefix|_]} = erlang:fun_info(IncFun, env), - inc_metrics(Prefix, Name); inc_metrics(Prefix, Name) when is_list(Prefix) -> emqx_metrics:inc(list_to_atom(Prefix ++ atom_to_list(Name))). diff --git a/apps/emqx_exproto/src/emqx_exproto_conn.erl b/apps/emqx_exproto/src/emqx_exproto_conn.erl index 152c0edf1..7c6eb6cb3 100644 --- a/apps/emqx_exproto/src/emqx_exproto_conn.erl +++ b/apps/emqx_exproto/src/emqx_exproto_conn.erl @@ -91,14 +91,6 @@ -define(ENABLED(X), (X =/= undefined)). --dialyzer({nowarn_function, - [ system_terminate/4 - , handle_call/3 - , handle_msg/2 - , shutdown/3 - , stop/3 - ]}). - %% udp start_link(Socket = {udp, _SockPid, _Sock}, Peername, Options) -> Args = [self(), Socket, Peername, Options], @@ -501,6 +493,7 @@ terminate(Reason, State = #state{channel = Channel}) -> system_continue(Parent, _Debug, State) -> recvloop(Parent, State). +-spec system_terminate(atom(), term(), term(), state()) -> no_return(). system_terminate(Reason, _Parent, _Debug, State) -> terminate(Reason, State). diff --git a/apps/emqx_management/src/emqx_mgmt_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_data_backup.erl index b0d346489..e66a8834e 100644 --- a/apps/emqx_management/src/emqx_mgmt_data_backup.erl +++ b/apps/emqx_management/src/emqx_mgmt_data_backup.erl @@ -185,7 +185,6 @@ confs_to_binary(Confs) -> -endif. --dialyzer([{nowarn_function, [import_rules/1, import_rule/1]}]). import_rule(#{<<"id">> := RuleId, <<"rawsql">> := RawSQL, <<"actions">> := Actions, @@ -537,7 +536,6 @@ do_import_acl_mnesia(Acls) -> end, Acls). -ifdef(EMQX_ENTERPRISE). --dialyzer({nowarn_function, [import_modules/1]}). import_modules(Modules) -> case ets:info(emqx_modules) of undefined -> diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.erl b/apps/emqx_rule_engine/src/emqx_rule_engine.erl index f48b8a02b..ac0ccaf4f 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.erl @@ -207,7 +207,7 @@ create_rule(Params = #{rawsql := Sql, actions := ActArgs}) -> Reason -> {error, Reason} end. --spec(update_rule(#{id := binary(), _=>_}) -> {ok, rule()} | {error, {not_found, rule_id()}}). +-spec(update_rule(#{id := binary(), _=>_}) -> {ok, rule()} | {error, {not_found, rule_id()} | term()}). update_rule(Params = #{id := RuleId}) -> case emqx_rule_registry:get_rule(RuleId) of {ok, Rule0} -> @@ -336,7 +336,6 @@ start_resource(ResId) -> {error, {resource_not_found, ResId}} end. --dialyzer([{nowarn_function, test_resource/1}]). -spec(test_resource(#{type := _, config := _, _ => _}) -> ok | {error, Reason :: term()}). test_resource(#{type := Type} = Params) -> case emqx_rule_registry:find_resource_type(Type) of diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index 223c672e6..50fdcfe22 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -207,12 +207,6 @@ <<"Bad Arguments: ", R0/binary>> end). --dialyzer({nowarn_function, [create_rule/2, - test_rule_sql/1, - do_create_rule/1, - update_rule/2 - ]}). - %%------------------------------------------------------------------------------ %% Rules API %%------------------------------------------------------------------------------ diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_cli.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_cli.erl index 45a847ada..101fa54f6 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_cli.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_cli.erl @@ -96,7 +96,6 @@ unload() -> %%----------------------------------------------------------------------------- %% 'rules' command %%----------------------------------------------------------------------------- --dialyzer([{nowarn_function, [rules/1]}]). rules(["list"]) -> print_all(emqx_rule_registry:get_rules_ordered_by_ts()); diff --git a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl index 71775f6ee..c6c3da533 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl @@ -180,7 +180,6 @@ select_and_collect([Field|More], Input, {Output, LastKV}) -> {nested_put(Key, Val, Output), LastKV}). %% Filter each item got from FOREACH --dialyzer({nowarn_function, filter_collection/4}). filter_collection(Input, InCase, DoEach, {CollKey, CollVal}) -> lists:filtermap( fun(Item) -> diff --git a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl index 79ac0f328..b1df2f01d 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl @@ -20,16 +20,6 @@ -export([ test/1 ]). -%% Dialyzer gives up on the generated code. -%% probably due to stack depth, or inlines. --dialyzer({nowarn_function, [test/1, - test_rule/4, - flatten/1, - sql_test_action/0, - fill_default_values/2, - envs_examp/1 - ]}). - -spec(test(#{}) -> {ok, map() | list()} | {error, term()}). test(#{<<"rawsql">> := Sql, <<"ctx">> := Context}) -> {ok, Select} = emqx_rule_sqlparser:parse_select(Sql), diff --git a/apps/emqx_stomp/src/emqx_stomp_connection.erl b/apps/emqx_stomp/src/emqx_stomp_connection.erl index 767ea3814..e58b242e5 100644 --- a/apps/emqx_stomp/src/emqx_stomp_connection.erl +++ b/apps/emqx_stomp/src/emqx_stomp_connection.erl @@ -91,13 +91,6 @@ -define(ENABLED(X), (X =/= undefined)). --dialyzer({nowarn_function, [ ensure_stats_timer/2 - ]}). - --dialyzer({no_return, [ init/1 - , init_state/3 - ]}). - start_link(Transport, Sock, ProtoEnv) -> {ok, proc_lib:spawn_link(?MODULE, init, [[Transport, Sock, ProtoEnv]])}. @@ -143,6 +136,7 @@ call(Pid, Req) -> call(Pid, Req, Timeout) -> gen_server:call(Pid, Req, Timeout). +-spec init([term()]) -> no_return(). init([Transport, RawSocket, ProtoEnv]) -> case Transport:wait(RawSocket) of {ok, Socket} -> @@ -152,6 +146,7 @@ init([Transport, RawSocket, ProtoEnv]) -> exit_on_sock_error(Reason) end. +-spec init_state(module(), port(), [proplists:property()]) -> no_return(). init_state(Transport, Socket, ProtoEnv) -> {ok, Peername} = Transport:ensure_ok_or_exit(peername, [Socket]), {ok, Sockname} = Transport:ensure_ok_or_exit(sockname, [Socket]), diff --git a/apps/emqx_stomp/src/emqx_stomp_protocol.erl b/apps/emqx_stomp/src/emqx_stomp_protocol.erl index 0a8a7be75..e2cc2888b 100644 --- a/apps/emqx_stomp/src/emqx_stomp_protocol.erl +++ b/apps/emqx_stomp/src/emqx_stomp_protocol.erl @@ -50,9 +50,29 @@ , handle_recv_nack_frame/2 ]). +-type stomp_conninfo() :: #{socktype := emqx_types:socktype(), + sockname := emqx_types:peername(), + peername := emqx_types:peername(), + peercert := nossl | undefined | esockd_peercert:peercert(), + conn_mod := module(), + proto_name => binary(), + proto_ver => emqx_types:ver(), + clean_start => boolean(), + clientid => emqx_types:clientid(), + username => emqx_types:username(), + conn_props => emqx_types:properties(), + connected => boolean(), + connected_at => undefined | non_neg_integer(), + disconnected_at => non_neg_integer(), + keepalive => undefined | 0..16#FFFF, + receive_maximum => non_neg_integer(), + expiry_interval => non_neg_integer(), + atom() => term() + }. + -record(pstate, { %% Stomp ConnInfo - conninfo :: emqx_types:conninfo(), + conninfo :: stomp_conninfo(), %% Stomp ClientInfo clientinfo :: emqx_types:clientinfo(), %% Stomp Heartbeats @@ -104,10 +124,6 @@ awaiting_rel_max ]). --dialyzer({nowarn_function, [ check_acl/3 - , init/2 - ]}). - -type(pstate() :: #pstate{}). %% @doc Init protocol @@ -687,12 +703,8 @@ backoff({Cx, Cy}) -> parse_topic_filters(TopicFilters) -> lists:map(fun emqx_topic:parse/1, TopicFilters). -check_acl(PubSub, Topic, State = #pstate{clientinfo = ClientInfo}) -> - case is_acl_enabled(State) andalso - emqx_access_control:check_acl(ClientInfo, PubSub, Topic) of - false -> allow; - Res -> Res - end. +check_acl(PubSub, Topic, #pstate{clientinfo = ClientInfo}) -> + emqx_access_control:check_acl(ClientInfo, PubSub, Topic). do_subscribe(TopicFilter, SubOpts, State = #pstate{clientinfo = ClientInfo, subscriptions = Subs}) -> @@ -728,10 +740,6 @@ find_sub_by_id(Id, Subs) -> [Sub | _] -> Sub end. -is_acl_enabled(_) -> - %% TODO: configs from somewhere - true. - %% automaticly fill the next sub-id and ack if sub-id is absent enrich_sub_opts(SubOpts0, Subs) -> SubOpts = maps:merge(?DEFAULT_SUBOPTS, SubOpts0), diff --git a/include/emqx.hrl b/include/emqx.hrl index 4a97f3ccb..ddfc5f19e 100644 --- a/include/emqx.hrl +++ b/include/emqx.hrl @@ -85,9 +85,13 @@ %%-------------------------------------------------------------------- -record(route, { - topic :: binary(), - dest :: node() | {binary(), node()} + topic, + dest }). +-type route() :: #route{ + topic :: binary(), + dest :: node() | {binary(), node()} + }. %%-------------------------------------------------------------------- %% Plugin @@ -132,4 +136,3 @@ }). -endif. - diff --git a/lib-ce/emqx_telemetry/src/emqx_telemetry.erl b/lib-ce/emqx_telemetry/src/emqx_telemetry.erl index b87329076..09765f3e6 100644 --- a/lib-ce/emqx_telemetry/src/emqx_telemetry.erl +++ b/lib-ce/emqx_telemetry/src/emqx_telemetry.erl @@ -133,12 +133,6 @@ get_telemetry() -> %% gen_server callbacks %%-------------------------------------------------------------------- -%% This is to suppress dialyzer warnings for mnesia:dirty_write and -%% dirty_read race condition. Given that the init function is not evaluated -%% concurrently in one node, it should be free of race condition. -%% Given the chance of having two nodes bootstraping with the write -%% is very small, it should be safe to ignore. --dialyzer([{nowarn_function, [init/1]}]). init([Opts]) -> State = #state{url = get_value(url, Opts), report_interval = timer:seconds(get_value(report_interval, Opts))}, diff --git a/src/emqx_alarm.erl b/src/emqx_alarm.erl index e3b45e295..d838e139e 100644 --- a/src/emqx_alarm.erl +++ b/src/emqx_alarm.erl @@ -191,10 +191,8 @@ init([Opts]) -> size_limit = SizeLimit, validity_period = ValidityPeriod})}. -%% suppress dialyzer warning due to dirty read/write race condition. %% TODO: change from dirty_read/write to transactional. %% TODO: handle mnesia write errors. --dialyzer([{nowarn_function, [handle_call/3]}]). handle_call({activate_alarm, Name, Details}, _From, State = #state{actions = Actions}) -> case mnesia:dirty_read(?ACTIVATED_ALARM, Name) of [#activated_alarm{name = Name}] -> diff --git a/src/emqx_channel.erl b/src/emqx_channel.erl index c49299c58..b8849b0ae 100644 --- a/src/emqx_channel.erl +++ b/src/emqx_channel.erl @@ -87,7 +87,7 @@ %% Quota checkers quota :: maybe(emqx_limiter:limiter()), %% Timers - timers :: #{atom() => disabled | maybe(reference())}, + timers :: #{channel_timer() => disabled | maybe(reference())}, %% Conn State conn_state :: conn_state(), %% Takeover @@ -109,6 +109,13 @@ -type(replies() :: emqx_types:packet() | reply() | [reply()]). +-type(channel_timer() :: alive_timer + | retry_timer + | await_timer + | expire_timer + | will_timer + | quota_timer). + -define(IS_MQTT_V5, #channel{conninfo = #{proto_ver := ?MQTT_PROTO_V5}}). -define(TIMER_TABLE, #{ @@ -122,8 +129,6 @@ -define(INFO_KEYS, [conninfo, conn_state, clientinfo, session, will_msg]). --dialyzer({no_match, [shutdown/4, ensure_timer/2, interval/2]}). - %%-------------------------------------------------------------------- %% Info, Attrs and Caps %%-------------------------------------------------------------------- @@ -244,7 +249,6 @@ setting_peercert_infos(Peercert, ClientInfo, Options) -> ClientId = peer_cert_as(peer_cert_as_clientid, Options, Peercert, DN, CN), ClientInfo#{username => Username, clientid => ClientId, dn => DN, cn => CN}. --dialyzer([{nowarn_function, [peer_cert_as/5]}]). % esockd_peercert:peercert is opaque % https://github.com/emqx/esockd/blob/master/src/esockd_peercert.erl peer_cert_as(Key, Options, Peercert, DN, CN) -> @@ -1113,12 +1117,8 @@ handle_timeout(_TRef, Msg, Channel) -> %% Ensure timers %%-------------------------------------------------------------------- -ensure_timer([Name], Channel) -> - ensure_timer(Name, Channel); -ensure_timer([Name | Rest], Channel) -> - ensure_timer(Rest, ensure_timer(Name, Channel)); - -ensure_timer(Name, Channel = #channel{timers = Timers}) -> +-spec ensure_timer(channel_timer(), channel()) -> channel(). +ensure_timer(Name, Channel = #channel{timers = Timers}) when is_atom(Name) -> TRef = maps:get(Name, Timers, undefined), Time = interval(Name, Channel), case TRef == undefined andalso Time > 0 of @@ -1126,6 +1126,7 @@ ensure_timer(Name, Channel = #channel{timers = Timers}) -> false -> Channel %% Timer disabled or exists end. +-spec ensure_timer(channel_timer(), timeout(), channel()) -> channel(). ensure_timer(Name, Time, Channel = #channel{timers = Timers}) -> Msg = maps:get(Name, ?TIMER_TABLE), TRef = emqx_misc:start_timer(Time, Msg), @@ -1140,16 +1141,13 @@ reset_timer(Name, Time, Channel) -> clean_timer(Name, Channel = #channel{timers = Timers}) -> Channel#channel{timers = maps:remove(Name, Timers)}. +-spec interval(channel_timer(), channel()) -> timeout(). interval(alive_timer, #channel{keepalive = KeepAlive}) -> emqx_keepalive:info(interval, KeepAlive); interval(retry_timer, #channel{session = Session}) -> timer:seconds(emqx_session:info(retry_interval, Session)); interval(await_timer, #channel{session = Session}) -> - timer:seconds(emqx_session:info(await_rel_timeout, Session)); -interval(expire_timer, #channel{conninfo = ConnInfo}) -> - timer:seconds(maps:get(expiry_interval, ConnInfo)); -interval(will_timer, #channel{will_msg = WillMsg}) -> - timer:seconds(will_delay_interval(WillMsg)). + timer:seconds(emqx_session:info(await_rel_timeout, Session)). %%-------------------------------------------------------------------- %% Terminate @@ -1757,8 +1755,6 @@ shutdown(success, Reply, Channel) -> shutdown(Reason, Reply, Channel) -> {shutdown, Reason, Reply, Channel}. -shutdown(success, Reply, Packet, Channel) -> - shutdown(normal, Reply, Packet, Channel); shutdown(Reason, Reply, Packet, Channel) -> {shutdown, Reason, Reply, Packet, Channel}. diff --git a/src/emqx_connection.erl b/src/emqx_connection.erl index 14dd76211..7c9cfef0a 100644 --- a/src/emqx_connection.erl +++ b/src/emqx_connection.erl @@ -125,12 +125,6 @@ -define(ALARM_SOCK_OPTS_KEYS, [high_watermark, high_msgq_watermark, sndbuf, recbuf, buffer]). -dialyzer({no_match, [info/2]}). --dialyzer({nowarn_function, [ init/4 - , init_state/3 - , run_loop/2 - , system_terminate/4 - , system_code_change/4 - ]}). -spec(start_link(esockd:transport(), esockd:socket(), proplists:proplist()) -> {ok, pid()}). @@ -286,8 +280,8 @@ run_loop(Parent, State = #state{transport = Transport, peername = Peername, channel = Channel}) -> emqx_logger:set_metadata_peername(esockd:format(Peername)), - emqx_misc:tune_heap_size(emqx_zone:oom_policy( - emqx_channel:info(zone, Channel))), + _ = emqx_misc:tune_heap_size(emqx_zone:oom_policy( + emqx_channel:info(zone, Channel))), case activate_socket(State) of {ok, NState} -> hibernate(Parent, NState); {error, Reason} -> diff --git a/src/emqx_frame.erl b/src/emqx_frame.erl index 5eb971f8a..d32bd6888 100644 --- a/src/emqx_frame.erl +++ b/src/emqx_frame.erl @@ -78,8 +78,6 @@ %% 16#0D,16#0A, 16#0D,16#0A,16#00,16#0D,16#0A,16#51,16#55,16#49,16#54,16#0A -define(PPV2_HEADER_SIG, "\r\n\r\n\0\r\nQUIT\n"). --dialyzer({no_match, [serialize_utf8_string/2]}). - -ifdef(TEST). -export([parse_variable_byte_integer/1]). -endif. @@ -788,8 +786,6 @@ serialize_utf8_pair({Name, Value}) -> serialize_binary_data(Bin) -> [<<(byte_size(Bin)):16/big-unsigned-integer>>, Bin]. -serialize_utf8_string(undefined, false) -> - error(utf8_string_undefined); serialize_utf8_string(undefined, true) -> <<>>; serialize_utf8_string(String, _AllowNull) -> diff --git a/src/emqx_limiter.erl b/src/emqx_limiter.erl index 8e194ed39..0275cb0e5 100644 --- a/src/emqx_limiter.erl +++ b/src/emqx_limiter.erl @@ -55,8 +55,6 @@ -type(limiter() :: #limiter{}). --dialyzer({nowarn_function, [consume/3]}). - %%-------------------------------------------------------------------- %% APIs %%-------------------------------------------------------------------- diff --git a/src/emqx_plugins.erl b/src/emqx_plugins.erl index ca103520d..a01dc2a5b 100644 --- a/src/emqx_plugins.erl +++ b/src/emqx_plugins.erl @@ -41,10 +41,6 @@ -compile(nowarn_export_all). -endif. --dialyzer({no_match, [ plugin_loaded/2 - , plugin_unloaded/2 - ]}). - %%-------------------------------------------------------------------- %% APIs %%-------------------------------------------------------------------- @@ -105,7 +101,7 @@ unload(PluginName) when is_atom(PluginName) -> ?LOG(error, "Plugin ~s is not started", [PluginName]), {error, not_started}; {_, _} -> - unload_plugin(PluginName, true) + unload_plugin(PluginName) end. reload(PluginName) when is_atom(PluginName)-> @@ -384,10 +380,11 @@ start_app(App, SuccFun) -> {error, {ErrApp, Reason}} end. -unload_plugin(App, Persistent) -> +unload_plugin(App) -> case stop_app(App) of ok -> - _ = plugin_unloaded(App, Persistent), ok; + _ = plugin_unloaded(App), + ok; {error, Reason} -> {error, Reason} end. @@ -428,9 +425,7 @@ plugin_loaded(Name, true) -> ?LOG(error, "Cannot read loaded plugins: ~p", [Error]) end. -plugin_unloaded(_Name, false) -> - ok; -plugin_unloaded(Name, true) -> +plugin_unloaded(Name) -> case read_loaded() of {ok, Names0} -> Names = filter_plugins(Names0), diff --git a/src/emqx_router.erl b/src/emqx_router.erl index 760c48d43..2c93d759c 100644 --- a/src/emqx_router.erl +++ b/src/emqx_router.erl @@ -266,9 +266,6 @@ maybe_trans(Fun, Args) -> end, []) end. -%% The created fun only terminates with explicit exception --dialyzer({nowarn_function, [trans/2]}). - -spec(trans(function(), list(any())) -> ok | {error, term()}). trans(Fun, Args) -> {WPid, RefMon} = @@ -277,13 +274,7 @@ trans(Fun, Args) -> %% are caught by mnesia:transaction/2. %% Future changes should keep in mind that this process %% always exit with database write result. - fun() -> - Res = case mnesia:transaction(Fun, Args) of - {atomic, Ok} -> Ok; - {aborted, Reason} -> {error, Reason} - end, - exit({shutdown, Res}) - end), + make_trans(Fun, Args)), %% Receive a 'shutdown' exit to pass result from the short-lived process. %% so the receive below can be receive-mark optimized by the compiler. %% @@ -300,6 +291,16 @@ trans(Fun, Args) -> end end. +-spec make_trans(fun((...) -> term()), [term()]) -> fun(() -> no_return()). +make_trans(Fun, Args) -> + fun() -> + Res = case mnesia:transaction(Fun, Args) of + {atomic, Ok} -> Ok; + {aborted, Reason} -> {error, Reason} + end, + exit({shutdown, Res}) + end. + lock_router() -> %% if Retry is not 0, global:set_lock could sleep a random time up to 8s. %% Considering we have a limited number of brokers, it is safe to use sleep 1 ms. diff --git a/src/emqx_router_helper.erl b/src/emqx_router_helper.erl index 67a706975..fb72ad9c6 100644 --- a/src/emqx_router_helper.erl +++ b/src/emqx_router_helper.erl @@ -53,8 +53,6 @@ -define(ROUTING_NODE, emqx_routing_node). -define(LOCK, {?MODULE, cleanup_routes}). --dialyzer({nowarn_function, [cleanup_routes/1]}). - %%-------------------------------------------------------------------- %% Mnesia bootstrap %%-------------------------------------------------------------------- @@ -176,4 +174,3 @@ cleanup_routes(Node) -> #route{_ = '_', dest = {'_', Node}}], [mnesia:delete_object(?ROUTE, Route, write) || Pat <- Patterns, Route <- mnesia:match_object(?ROUTE, Pat, write)]. - diff --git a/src/emqx_tracer.erl b/src/emqx_tracer.erl index 8f789faa7..2c310f67c 100644 --- a/src/emqx_tracer.erl +++ b/src/emqx_tracer.erl @@ -59,8 +59,6 @@ L =:= info orelse L =:= debug). --dialyzer({nowarn_function, [install_trace_handler/3]}). - %%------------------------------------------------------------------------------ %% APIs %%------------------------------------------------------------------------------ diff --git a/src/emqx_types.erl b/src/emqx_types.erl index 4a0db262c..8a738c111 100644 --- a/src/emqx_types.erl +++ b/src/emqx_types.erl @@ -201,7 +201,6 @@ -type(deliver_result() :: ok | {ok, non_neg_integer()} | {error, term()}). -type(publish_result() :: [{node(), topic(), deliver_result()} | {share, topic(), deliver_result()}]). --type(route() :: #route{}). -type(sub_group() :: tuple() | binary()). -type(route_entry() :: {topic(), node()} | {topic, sub_group()}). -type(plugin() :: #plugin{}). @@ -215,4 +214,3 @@ -type(oom_policy() :: #{message_queue_len => non_neg_integer(), max_heap_size => non_neg_integer() }). - diff --git a/src/emqx_ws_connection.erl b/src/emqx_ws_connection.erl index 6cde8eb01..a0c999ea9 100644 --- a/src/emqx_ws_connection.erl +++ b/src/emqx_ws_connection.erl @@ -100,7 +100,6 @@ -define(ENABLED(X), (X =/= undefined)). -dialyzer({no_match, [info/2]}). --dialyzer({nowarn_function, [websocket_init/1]}). %%-------------------------------------------------------------------- %% Info, Stats @@ -304,7 +303,7 @@ websocket_init([Req, Opts]) -> %% MQTT Idle Timeout IdleTimeout = emqx_zone:idle_timeout(Zone), IdleTimer = start_timer(IdleTimeout, idle_timeout), - emqx_misc:tune_heap_size(emqx_zone:oom_policy(Zone)), + _ = emqx_misc:tune_heap_size(emqx_zone:oom_policy(Zone)), emqx_logger:set_metadata_peername(esockd:format(Peername)), {ok, #state{peername = Peername, sockname = Sockname, @@ -777,4 +776,3 @@ get_peer(Req, Opts) -> set_field(Name, Value, State) -> Pos = emqx_misc:index_of(Name, record_info(fields, state)), setelement(Pos+1, State, Value). - diff --git a/test/emqx_plugins_SUITE.erl b/test/emqx_plugins_SUITE.erl index 57f10f2f0..b26e90743 100644 --- a/test/emqx_plugins_SUITE.erl +++ b/test/emqx_plugins_SUITE.erl @@ -160,8 +160,8 @@ t_plugin_loaded(_) -> ?assertEqual(ok, emqx_plugins:plugin_loaded(emqx_mini_plugin, true)). t_plugin_unloaded(_) -> - ?assertEqual(ok, emqx_plugins:plugin_unloaded(emqx_mini_plugin, false)), - ?assertEqual(ok, emqx_plugins:plugin_unloaded(emqx_mini_plugin, true)). + ?assertEqual(ok, emqx_plugins:plugin_unloaded(emqx_mini_plugin)), + ?assertEqual(ok, emqx_plugins:plugin_unloaded(emqx_mini_plugin)). t_plugin(_) -> try @@ -199,8 +199,8 @@ t_unload_plugin(_) -> (error_app) -> {error, error}; (_) -> ok end), - ?assertEqual(ok, emqx_plugins:unload_plugin(not_started_app, true)), - ?assertEqual(ok, emqx_plugins:unload_plugin(normal, true)), - ?assertEqual({error,error}, emqx_plugins:unload_plugin(error_app, true)), + ?assertEqual(ok, emqx_plugins:unload_plugin(not_started_app)), + ?assertEqual(ok, emqx_plugins:unload_plugin(normal)), + ?assertEqual({error,error}, emqx_plugins:unload_plugin(error_app)), ok = meck:unload(application). From fa6c22b3661f5b4bd3c82eb7f29b6d61d8f0650e Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 9 Sep 2022 17:10:49 -0300 Subject: [PATCH 15/61] chore: update appups --- apps/emqx_exhook/src/emqx_exhook.app.src | 2 +- apps/emqx_exhook/src/emqx_exhook.appup.src | 6 +- apps/emqx_exproto/src/emqx_exproto.appup.src | 14 +- .../src/emqx_rule_engine.appup.src | 42 ++- apps/emqx_stomp/src/emqx_stomp.app.src | 2 +- apps/emqx_stomp/src/emqx_stomp.appup.src | 11 +- .../emqx_telemetry/src/emqx_telemetry.app.src | 2 +- .../src/emqx_telemetry.appup.src | 22 +- src/emqx.appup.src | 258 +++++++++++++++--- 9 files changed, 281 insertions(+), 78 deletions(-) diff --git a/apps/emqx_exhook/src/emqx_exhook.app.src b/apps/emqx_exhook/src/emqx_exhook.app.src index d0067742a..1d05b43f4 100644 --- a/apps/emqx_exhook/src/emqx_exhook.app.src +++ b/apps/emqx_exhook/src/emqx_exhook.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_exhook, [{description, "EMQ X Extension for Hook"}, - {vsn, "4.3.7"}, + {vsn, "4.3.8"}, {modules, []}, {registered, []}, {mod, {emqx_exhook_app, []}}, diff --git a/apps/emqx_exhook/src/emqx_exhook.appup.src b/apps/emqx_exhook/src/emqx_exhook.appup.src index 8aec1b9d4..7919e3e67 100644 --- a/apps/emqx_exhook/src/emqx_exhook.appup.src +++ b/apps/emqx_exhook/src/emqx_exhook.appup.src @@ -1,7 +1,8 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{<<"4\\.3\\.[5-6]">>, + [{"4.3.7",[{load_module,emqx_exhook_server,brutal_purge,soft_purge,[]}]}, + {<<"4\\.3\\.[5-6]">>, [{load_module,emqx_exhook_handler,brutal_purge,soft_purge,[]}, {load_module,emqx_exhook,brutal_purge,soft_purge,[]}, {load_module,emqx_exhook_server,brutal_purge,soft_purge,[]}, @@ -15,7 +16,8 @@ {update,emqx_exhook_mngr,{advanced,["4.3.4"]}}]}, {<<"4\\.3\\.[0-3]">>,[{restart_application,emqx_exhook}]}, {<<".*">>,[]}], - [{<<"4\\.3\\.[5-6]">>, + [{"4.3.7",[{load_module,emqx_exhook_server,brutal_purge,soft_purge,[]}]}, + {<<"4\\.3\\.[5-6]">>, [{load_module,emqx_exhook_handler,brutal_purge,soft_purge,[]}, {load_module,emqx_exhook,brutal_purge,soft_purge,[]}, {load_module,emqx_exhook_mngr,brutal_purge,soft_purge,[]}, diff --git a/apps/emqx_exproto/src/emqx_exproto.appup.src b/apps/emqx_exproto/src/emqx_exproto.appup.src index fcf2fe6db..f98eb2d0e 100644 --- a/apps/emqx_exproto/src/emqx_exproto.appup.src +++ b/apps/emqx_exproto/src/emqx_exproto.appup.src @@ -1,9 +1,12 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.10",[{load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, + [{"4.3.10", + [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, + {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, {"4.3.9", - [{load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}, + [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, + {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_gcli,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[2-8]">>, [{load_module,emqx_exproto_gcli,brutal_purge,soft_purge,[]}, @@ -15,9 +18,12 @@ {load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.10",[{load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, + [{"4.3.10", + [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, + {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, {"4.3.9", - [{load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}, + [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, + {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_gcli,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[2-8]">>, [{load_module,emqx_exproto_gcli,brutal_purge,soft_purge,[]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index c24bead6e..aae833bdf 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -2,23 +2,30 @@ %% Unless you know what you are doing, DO NOT edit manually!! {VSN, [{"4.3.14", - [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, {"4.3.13", - [{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {"4.3.12", - [{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {"4.3.11", - [{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, @@ -26,7 +33,8 @@ {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.3.10", - [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, @@ -34,7 +42,8 @@ {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.3.9", - [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, @@ -185,23 +194,30 @@ {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.3.14", - [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, {"4.3.13", - [{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {"4.3.12", - [{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, {"4.3.11", - [{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, @@ -209,7 +225,8 @@ {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.3.10", - [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, @@ -217,7 +234,8 @@ {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {"4.3.9", - [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, + [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, diff --git a/apps/emqx_stomp/src/emqx_stomp.app.src b/apps/emqx_stomp/src/emqx_stomp.app.src index 5883cbca5..e35e570d3 100644 --- a/apps/emqx_stomp/src/emqx_stomp.app.src +++ b/apps/emqx_stomp/src/emqx_stomp.app.src @@ -1,6 +1,6 @@ {application, emqx_stomp, [{description, "EMQ X Stomp Protocol Plugin"}, - {vsn, "4.3.5"}, % strict semver, bump manually! + {vsn, "4.3.6"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_stomp_sup]}, {applications, [kernel,stdlib]}, diff --git a/apps/emqx_stomp/src/emqx_stomp.appup.src b/apps/emqx_stomp/src/emqx_stomp.appup.src index 3493ade03..69e571934 100644 --- a/apps/emqx_stomp/src/emqx_stomp.appup.src +++ b/apps/emqx_stomp/src/emqx_stomp.appup.src @@ -1,6 +1,10 @@ %% -*- mode: erlang -*- +%% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.4", + [{"4.3.5", + [{load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]}, + {load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]}]}, + {"4.3.4", [{load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]}, {load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]}]}, {"4.3.3", @@ -19,7 +23,10 @@ [{restart_application,emqx_stomp}, {apply,{emqx_stomp,force_clear_after_app_stoped,[]}}]}, {<<".*">>,[]}], - [{"4.3.4", + [{"4.3.5", + [{load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]}, + {load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]}]}, + {"4.3.4", [{load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]}, {load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]}]}, {"4.3.3", diff --git a/lib-ce/emqx_telemetry/src/emqx_telemetry.app.src b/lib-ce/emqx_telemetry/src/emqx_telemetry.app.src index 74f8fa63e..52bfe7b10 100644 --- a/lib-ce/emqx_telemetry/src/emqx_telemetry.app.src +++ b/lib-ce/emqx_telemetry/src/emqx_telemetry.app.src @@ -1,6 +1,6 @@ {application, emqx_telemetry, [{description, "EMQ X Telemetry"}, - {vsn, "4.3.3"}, % strict semver, bump manually! + {vsn, "4.3.4"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_telemetry_sup]}, {applications, [kernel,stdlib]}, diff --git a/lib-ce/emqx_telemetry/src/emqx_telemetry.appup.src b/lib-ce/emqx_telemetry/src/emqx_telemetry.appup.src index 79e6d3d41..3d079e839 100644 --- a/lib-ce/emqx_telemetry/src/emqx_telemetry.appup.src +++ b/lib-ce/emqx_telemetry/src/emqx_telemetry.appup.src @@ -1,15 +1,11 @@ %% -*- mode: erlang -*- +%% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [ - {<<"4\\.3\\.[0-2]">>, [ - {load_module, emqx_telemetry, brutal_purge, soft_purge, []} - ]}, - {<<".*">>, []} - ], - [ - {<<"4\\.3\\.[0-2]">>, [ - {load_module, emqx_telemetry, brutal_purge, soft_purge, []} - ]}, - {<<".*">>, []} - ] -}. + [{"4.3.3",[{load_module,emqx_telemetry,brutal_purge,soft_purge,[]}]}, + {<<"4\\.3\\.[0-2]">>, + [{load_module,emqx_telemetry,brutal_purge,soft_purge,[]}]}, + {<<".*">>,[]}], + [{"4.3.3",[{load_module,emqx_telemetry,brutal_purge,soft_purge,[]}]}, + {<<"4\\.3\\.[0-2]">>, + [{load_module,emqx_telemetry,brutal_purge,soft_purge,[]}]}, + {<<".*">>,[]}]}. diff --git a/src/emqx.appup.src b/src/emqx.appup.src index de5cf7fe0..95186208e 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -2,13 +2,26 @@ %% Unless you know what you are doing, DO NOT edit manually!! {VSN, [{"4.3.20", - [{load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + {load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + {load_module,emqx_router,brutal_purge,soft_purge,[]}, + {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}]}, {"4.3.19", - [{load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + {load_module,emqx_router,brutal_purge,soft_purge,[]}, + {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, @@ -16,7 +29,13 @@ {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}]}, {"4.3.18", - [{load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + {load_module,emqx_router,brutal_purge,soft_purge,[]}, + {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, @@ -24,7 +43,13 @@ {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}]}, {"4.3.17", - [{load_module,emqx_message,brutal_purge,soft_purge,[]}, + [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + {load_module,emqx_router,brutal_purge,soft_purge,[]}, + {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, + {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, @@ -37,7 +62,13 @@ {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}]}, {"4.3.16", - [{load_module,emqx_message,brutal_purge,soft_purge,[]}, + [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + {load_module,emqx_router,brutal_purge,soft_purge,[]}, + {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, + {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}, @@ -57,7 +88,12 @@ {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}]}, {"4.3.15", - [{load_module,emqx_message,brutal_purge,soft_purge,[]}, + [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + {load_module,emqx_router,brutal_purge,soft_purge,[]}, + {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}, {add_module,emqx_calendar}, @@ -85,7 +121,12 @@ {update,emqx_os_mon,{advanced,[]}}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.3.14", - [{load_module,emqx_message,brutal_purge,soft_purge,[]}, + [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + {load_module,emqx_router,brutal_purge,soft_purge,[]}, + {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {add_module,emqx_calendar}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, @@ -115,7 +156,11 @@ {update,emqx_os_mon,{advanced,[]}}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}]}, {"4.3.13", - [{load_module,emqx_message,brutal_purge,soft_purge,[]}, + [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + {load_module,emqx_router,brutal_purge,soft_purge,[]}, + {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_message,brutal_purge,soft_purge,[]}, {add_module,emqx_calendar}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {add_module,emqx_exclusive_subscription}, @@ -150,7 +195,11 @@ {update,emqx_os_mon,{advanced,[]}}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}]}, {"4.3.12", - [{add_module,emqx_calendar}, + [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + {load_module,emqx_router,brutal_purge,soft_purge,[]}, + {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + {add_module,emqx_calendar}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {add_module,emqx_exclusive_subscription}, {apply,{emqx_exclusive_subscription,on_add_module,[]}}, @@ -188,7 +237,11 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.11", - [{add_module,emqx_calendar}, + [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + {load_module,emqx_router,brutal_purge,soft_purge,[]}, + {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + {add_module,emqx_calendar}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {add_module,emqx_exclusive_subscription}, {apply,{emqx_exclusive_subscription,on_add_module,[]}}, @@ -228,7 +281,11 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.10", - [{add_module,emqx_calendar}, + [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + {load_module,emqx_router,brutal_purge,soft_purge,[]}, + {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + {add_module,emqx_calendar}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {add_module,emqx_exclusive_subscription}, {apply,{emqx_exclusive_subscription,on_add_module,[]}}, @@ -268,7 +325,10 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.9", - [{add_module,emqx_calendar}, + [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + {load_module,emqx_router,brutal_purge,soft_purge,[]}, + {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, + {add_module,emqx_calendar}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {add_module,emqx_exclusive_subscription}, {apply,{emqx_exclusive_subscription,on_add_module,[]}}, @@ -312,7 +372,10 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.8", - [{add_module,emqx_calendar}, + [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + {load_module,emqx_router,brutal_purge,soft_purge,[]}, + {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, + {add_module,emqx_calendar}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {add_module,emqx_exclusive_subscription}, {apply,{emqx_exclusive_subscription,on_add_module,[]}}, @@ -356,7 +419,10 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.7", - [{add_module,emqx_calendar}, + [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + {load_module,emqx_router,brutal_purge,soft_purge,[]}, + {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, + {add_module,emqx_calendar}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {add_module,emqx_exclusive_subscription}, {apply,{emqx_exclusive_subscription,on_add_module,[]}}, @@ -400,7 +466,10 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.6", - [{add_module,emqx_calendar}, + [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + {load_module,emqx_router,brutal_purge,soft_purge,[]}, + {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, + {add_module,emqx_calendar}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {add_module,emqx_exclusive_subscription}, {apply,{emqx_exclusive_subscription,on_add_module,[]}}, @@ -444,7 +513,10 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.5", - [{add_module,emqx_calendar}, + [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + {load_module,emqx_router,brutal_purge,soft_purge,[]}, + {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, + {add_module,emqx_calendar}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {add_module,emqx_exclusive_subscription}, {apply,{emqx_exclusive_subscription,on_add_module,[]}}, @@ -488,7 +560,10 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.4", - [{add_module,emqx_calendar}, + [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + {load_module,emqx_router,brutal_purge,soft_purge,[]}, + {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, + {add_module,emqx_calendar}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {add_module,emqx_exclusive_subscription}, {apply,{emqx_exclusive_subscription,on_add_module,[]}}, @@ -532,7 +607,10 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.3", - [{add_module,emqx_calendar}, + [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + {load_module,emqx_router,brutal_purge,soft_purge,[]}, + {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, + {add_module,emqx_calendar}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {add_module,emqx_exclusive_subscription}, {apply,{emqx_exclusive_subscription,on_add_module,[]}}, @@ -576,7 +654,10 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.2", - [{add_module,emqx_calendar}, + [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + {load_module,emqx_router,brutal_purge,soft_purge,[]}, + {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, + {add_module,emqx_calendar}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {add_module,emqx_exclusive_subscription}, {apply,{emqx_exclusive_subscription,on_add_module,[]}}, @@ -620,7 +701,10 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.1", - [{add_module,emqx_calendar}, + [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + {load_module,emqx_router,brutal_purge,soft_purge,[]}, + {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, + {add_module,emqx_calendar}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {add_module,emqx_exclusive_subscription}, {apply,{emqx_exclusive_subscription,on_add_module,[]}}, @@ -666,7 +750,10 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.0", - [{add_module,emqx_calendar}, + [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + {load_module,emqx_router,brutal_purge,soft_purge,[]}, + {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, + {add_module,emqx_calendar}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {add_module,emqx_exclusive_subscription}, {apply,{emqx_exclusive_subscription,on_add_module,[]}}, @@ -716,13 +803,26 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.3.20", - [{load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + {load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + {load_module,emqx_router,brutal_purge,soft_purge,[]}, + {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}]}, {"4.3.19", - [{load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + {load_module,emqx_router,brutal_purge,soft_purge,[]}, + {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, @@ -730,7 +830,13 @@ {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}]}, {"4.3.18", - [{load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + {load_module,emqx_router,brutal_purge,soft_purge,[]}, + {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, @@ -738,7 +844,13 @@ {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}]}, {"4.3.17", - [{load_module,emqx_message,brutal_purge,soft_purge,[]}, + [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + {load_module,emqx_router,brutal_purge,soft_purge,[]}, + {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, + {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, @@ -751,7 +863,13 @@ {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}]}, {"4.3.16", - [{load_module,emqx_message,brutal_purge,soft_purge,[]}, + [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + {load_module,emqx_router,brutal_purge,soft_purge,[]}, + {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, + {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}, @@ -771,7 +889,12 @@ {apply,{emqx_exclusive_subscription,on_delete_module,[]}}, {delete_module,emqx_exclusive_subscription}]}, {"4.3.15", - [{load_module,emqx_message,brutal_purge,soft_purge,[]}, + [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + {load_module,emqx_router,brutal_purge,soft_purge,[]}, + {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}, {delete_module,emqx_calendar}, @@ -798,7 +921,12 @@ {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.3.14", - [{load_module,emqx_message,brutal_purge,soft_purge,[]}, + [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + {load_module,emqx_router,brutal_purge,soft_purge,[]}, + {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {delete_module,emqx_calendar}, {apply,{emqx_exclusive_subscription,on_delete_module,[]}}, @@ -827,7 +955,11 @@ {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}]}, {"4.3.13", - [{load_module,emqx_message,brutal_purge,soft_purge,[]}, + [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + {load_module,emqx_router,brutal_purge,soft_purge,[]}, + {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_message,brutal_purge,soft_purge,[]}, {delete_module,emqx_calendar}, {apply,{emqx_exclusive_subscription,on_delete_module,[]}}, {delete_module,emqx_exclusive_subscription}, @@ -861,7 +993,11 @@ {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}]}, {"4.3.12", - [{delete_module,emqx_calendar}, + [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + {load_module,emqx_router,brutal_purge,soft_purge,[]}, + {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + {delete_module,emqx_calendar}, {apply,{emqx_exclusive_subscription,on_delete_module,[]}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, @@ -897,7 +1033,11 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.11", - [{delete_module,emqx_calendar}, + [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + {load_module,emqx_router,brutal_purge,soft_purge,[]}, + {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + {delete_module,emqx_calendar}, {apply,{emqx_exclusive_subscription,on_delete_module,[]}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, @@ -935,7 +1075,11 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.10", - [{delete_module,emqx_calendar}, + [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + {load_module,emqx_router,brutal_purge,soft_purge,[]}, + {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + {delete_module,emqx_calendar}, {apply,{emqx_exclusive_subscription,on_delete_module,[]}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, @@ -973,7 +1117,10 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.9", - [{delete_module,emqx_calendar}, + [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + {load_module,emqx_router,brutal_purge,soft_purge,[]}, + {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, + {delete_module,emqx_calendar}, {apply,{emqx_exclusive_subscription,on_delete_module,[]}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, @@ -1015,7 +1162,10 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.8", - [{delete_module,emqx_calendar}, + [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + {load_module,emqx_router,brutal_purge,soft_purge,[]}, + {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, + {delete_module,emqx_calendar}, {apply,{emqx_exclusive_subscription,on_delete_module,[]}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, @@ -1057,7 +1207,10 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.7", - [{delete_module,emqx_calendar}, + [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + {load_module,emqx_router,brutal_purge,soft_purge,[]}, + {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, + {delete_module,emqx_calendar}, {apply,{emqx_exclusive_subscription,on_delete_module,[]}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, @@ -1099,7 +1252,10 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.6", - [{delete_module,emqx_calendar}, + [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + {load_module,emqx_router,brutal_purge,soft_purge,[]}, + {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, + {delete_module,emqx_calendar}, {apply,{emqx_exclusive_subscription,on_delete_module,[]}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, @@ -1141,7 +1297,10 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.5", - [{delete_module,emqx_calendar}, + [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + {load_module,emqx_router,brutal_purge,soft_purge,[]}, + {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, + {delete_module,emqx_calendar}, {apply,{emqx_exclusive_subscription,on_delete_module,[]}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, @@ -1183,7 +1342,10 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.4", - [{delete_module,emqx_calendar}, + [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + {load_module,emqx_router,brutal_purge,soft_purge,[]}, + {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, + {delete_module,emqx_calendar}, {apply,{emqx_exclusive_subscription,on_delete_module,[]}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, @@ -1225,7 +1387,10 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.3", - [{delete_module,emqx_calendar}, + [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + {load_module,emqx_router,brutal_purge,soft_purge,[]}, + {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, + {delete_module,emqx_calendar}, {apply,{emqx_exclusive_subscription,on_delete_module,[]}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, @@ -1267,7 +1432,10 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.2", - [{delete_module,emqx_calendar}, + [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + {load_module,emqx_router,brutal_purge,soft_purge,[]}, + {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, + {delete_module,emqx_calendar}, {apply,{emqx_exclusive_subscription,on_delete_module,[]}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, @@ -1309,7 +1477,10 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.1", - [{delete_module,emqx_calendar}, + [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + {load_module,emqx_router,brutal_purge,soft_purge,[]}, + {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, + {delete_module,emqx_calendar}, {apply,{emqx_exclusive_subscription,on_delete_module,[]}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, @@ -1353,7 +1524,10 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.0", - [{delete_module,emqx_calendar}, + [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + {load_module,emqx_router,brutal_purge,soft_purge,[]}, + {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, + {delete_module,emqx_calendar}, {apply,{emqx_exclusive_subscription,on_delete_module,[]}}, {delete_module,emqx_exclusive_subscription}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, From eee18699816dd926fa9bdb00432fc4a3488737a6 Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Tue, 13 Sep 2022 19:06:53 +0200 Subject: [PATCH 16/61] fix(prometheus): Disable auth for prometheus endpoint --- CHANGES-4.3.md | 1 + lib-ce/emqx_dashboard/src/emqx_dashboard.erl | 2 ++ 2 files changed, 3 insertions(+) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index c6a6e9465..247e15dd7 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -26,6 +26,7 @@ File format: should return false if match it to a topic. - Added a test to prevent a last will testament message to be published when a client is denied connection. [#8894](https://github.com/emqx/emqx/pull/8894) +- Disable authorization for `api/v4/emqx_prometheus` endpoint. [8955](https://github.com/emqx/emqx/pull/8955) ## v4.3.19 diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard.erl b/lib-ce/emqx_dashboard/src/emqx_dashboard.erl index d88bd0da9..517211607 100644 --- a/lib-ce/emqx_dashboard/src/emqx_dashboard.erl +++ b/lib-ce/emqx_dashboard/src/emqx_dashboard.erl @@ -109,6 +109,8 @@ http_handlers() -> is_authorized(Req) -> is_authorized(binary_to_list(cowboy_req:path(Req)), Req). +is_authorized("/api/v4/emqx_prometheus", _Req) -> + true; is_authorized("/api/v4/auth", _Req) -> true; is_authorized(_Path, Req) -> From 4ff34e31884c91641f79666551b40d3f5e06e763 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 9 Sep 2022 17:33:33 +0200 Subject: [PATCH 17/61] ci: ensure chart version is in sync with the release version --- .github/workflows/apps_version_check.yaml | 2 ++ scripts/check-chart-vsn.sh | 34 +++++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100755 scripts/check-chart-vsn.sh diff --git a/.github/workflows/apps_version_check.yaml b/.github/workflows/apps_version_check.yaml index 86fe60204..39385b1d0 100644 --- a/.github/workflows/apps_version_check.yaml +++ b/.github/workflows/apps_version_check.yaml @@ -29,6 +29,8 @@ jobs: run: ./scripts/update-appup.sh emqx-ee --check - name: Check apps version run: ./scripts/apps-version-check.sh + - name: Check chart versions + run: ./scripts/check-chart-vsn.sh - uses: actions/upload-artifact@v3.1.0 if: failure() with: diff --git a/scripts/check-chart-vsn.sh b/scripts/check-chart-vsn.sh new file mode 100755 index 000000000..633a51dc2 --- /dev/null +++ b/scripts/check-chart-vsn.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail + +# ensure dir +cd -P -- "$(dirname -- "$0")/.." + +if [ -f "EMQX_ENTERPRISE" ]; then + CHART_FILE='deploy/charts/emqx-ee/Chart.yaml' +else + CHART_FILE='deploy/charts/emqx/Chart.yaml' +fi + +if [ ! -f "$CHART_FILE" ]; then + echo "Chart file $CHART_FILE is not found" + pwd + exit 1 +fi + +CHART_VSN="$(grep -oE '^version:.*' "$CHART_FILE" | cut -d ':' -f 2 | tr -d ' ')" +APP_VSN="$(grep -oE '^appVersion:.*' "$CHART_FILE" | cut -d ':' -f 2 | tr -d ' ')" + +if [ "$CHART_VSN" != "$APP_VSN" ]; then + echo "Chart version and app version mismatch in $CHART_FILE" + exit 2 +fi + +PKG_VSN="$(./pkg-vsn.sh | cut -d '-' -f 1)" + +if [ "$CHART_VSN" != "$PKG_VSN" ]; then + echo "Chart version in $CHART_FILE is not in sync with release version." + echo "Chart version: $CHART_VSN" + echo "Release version: $PKG_VSN" + exit 3 +fi From 05129bd919dede8f1fa8053a7f516f9b1e093f84 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 9 Sep 2022 17:35:03 +0200 Subject: [PATCH 18/61] refactor: rename check scripts to start with 'check-' --- .github/workflows/apps_version_check.yaml | 2 +- scripts/{apps-version-check.sh => check-apps-vsn.sh} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename scripts/{apps-version-check.sh => check-apps-vsn.sh} (100%) diff --git a/.github/workflows/apps_version_check.yaml b/.github/workflows/apps_version_check.yaml index 39385b1d0..257eba4f7 100644 --- a/.github/workflows/apps_version_check.yaml +++ b/.github/workflows/apps_version_check.yaml @@ -28,7 +28,7 @@ jobs: if: endsWith(github.repository, 'enterprise') run: ./scripts/update-appup.sh emqx-ee --check - name: Check apps version - run: ./scripts/apps-version-check.sh + run: ./scripts/check-apps-vsn.sh - name: Check chart versions run: ./scripts/check-chart-vsn.sh - uses: actions/upload-artifact@v3.1.0 diff --git a/scripts/apps-version-check.sh b/scripts/check-apps-vsn.sh similarity index 100% rename from scripts/apps-version-check.sh rename to scripts/check-apps-vsn.sh From 6204481d715fc889625cb85cb6e9c00945072296 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 15 Sep 2022 18:08:20 -0300 Subject: [PATCH 19/61] test: avoid inter-suite flakiness This test runs most of the time fine in CI. But, if run alone locally, will fail consistently because the default `acl.conf` has a catch-all `{allow, all}` clause. Probably another suite that runs before this in CI unloads that and everything seems fine. --- apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl b/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl index 9d1fadd34..66f5253d0 100644 --- a/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl +++ b/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl @@ -52,10 +52,14 @@ all() -> init_per_suite(Cfg) -> emqx_ct_helpers:start_apps([emqx_auth_mongo], fun set_special_confs/1), init_mongo_data(), + %% avoid inter-suite flakiness + ok = emqx_mod_acl_internal:unload([]), Cfg. end_per_suite(_Cfg) -> deinit_mongo_data(), + %% avoid inter-suite flakiness + ok = emqx_mod_acl_internal:load([]), emqx_ct_helpers:stop_apps([emqx_auth_mongo]). set_special_confs(emqx) -> From ed3410864461092e9a19a3e4cd5f278bfa00b034 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Fri, 16 Sep 2022 10:21:56 +0200 Subject: [PATCH 20/61] build: sign macos binaries --- .github/workflows/build_slim_packages.yaml | 11 ++++- build | 36 ++++++++++++++- scripts/macos-sign-binaries.sh | 52 ++++++++++++++++++++++ 3 files changed, 97 insertions(+), 2 deletions(-) create mode 100755 scripts/macos-sign-binaries.sh diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index b5a8623ef..7e281110c 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -97,18 +97,27 @@ jobs: id: cache with: path: ~/.kerl/${{ matrix.erl_otp }} - key: otp-install-${{ matrix.erl_otp }}-${{ matrix.macos }} + key: otp-install-${{ matrix.erl_otp }}-${{ matrix.macos }}-static-ssl-disable-hipe-disable-jit - name: build erlang if: steps.cache.outputs.cache-hit != 'true' timeout-minutes: 60 env: KERL_BUILD_BACKEND: git OTP_GITHUB_URL: https://github.com/emqx/otp + KERL_CONFIGURE_OPTIONS: --disable-dynamic-ssl-lib --with-ssl=/usr/local/opt/openssl@1.1 --disable-hipe --disable-jit run: | kerl update releases kerl build ${{ matrix.erl_otp }} kerl install ${{ matrix.erl_otp }} $HOME/.kerl/${{ matrix.erl_otp }} - name: build + env: + APPLE_SIGN_BINARIES: 1 + APPLE_ID: developers@emqx.io + APPLE_TEAM_ID: 26N6HYJLZA + APPLE_ID_PASSWORD: ${{ secrets.APPLE_ID_PASSWORD }} + APPLE_DEVELOPER_IDENTITY: ${{ secrets.APPLE_DEVELOPER_IDENTITY }} + APPLE_DEVELOPER_ID_BUNDLE: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }} + APPLE_DEVELOPER_ID_BUNDLE_PASSWORD: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }} run: | . $HOME/.kerl/${{ matrix.erl_otp }}/activate make ensure-rebar3 diff --git a/build b/build index ace072b79..0ffb810eb 100755 --- a/build +++ b/build @@ -165,7 +165,41 @@ make_zip() { ## try to be portable for zip packages. ## for DEB and RPM packages the dependencies are resoved by yum and apt cp_dyn_libs "${tard}/emqx" - (cd "${tard}" && zip -qr - emqx) > "${zipball}" + case "$SYSTEM" in + macos*) + # if the flag to sign macos binaries is set, but developer certificate + # or certificate password is not configured, reset the flag + # could happen, for example, when people submit PR from a fork, in this + # case they cannot access secrets + if [[ "${APPLE_SIGN_BINARIES:-0}" == 1 && \ + ( "${APPLE_DEVELOPER_ID_BUNDLE:-0}" == 0 || \ + "${APPLE_DEVELOPER_ID_BUNDLE_PASSWORD:-0}" == 0 ) ]]; then + echo "Apple developer certificate is not configured, skip signing" + APPLE_SIGN_BINARIES=0 + fi + if [ "${APPLE_SIGN_BINARIES:-0}" = 1 ]; then + ./scripts/macos-sign-binaries.sh "${tard}/emqx" + fi + ## create zip after change dir + ## to avoid creating an extra level of 'emqx' dir in the .zip file + (cd "${tard}" && zip -qr - emqx) > "${zipball}" + if [ "${APPLE_SIGN_BINARIES:-0}" = 1 ]; then + # notarize the package + # if fails, you can check what went wrong with this command: + # xcrun notarytool log --apple-id \ + # --apple-id \ + # --password + # --team-id + xcrun notarytool submit \ + --apple-id "${APPLE_ID}" \ + --password "${APPLE_ID_PASSWORD}" \ + --team-id "${APPLE_TEAM_ID}" "${zipball}" --wait + fi + ;; + *) + (cd "${tard}" && zip -qr - emqx) > "${zipball}" + ;; + esac } ## This function builds the default docker image based on alpine:3.14 (by default) diff --git a/scripts/macos-sign-binaries.sh b/scripts/macos-sign-binaries.sh new file mode 100755 index 000000000..7be40f621 --- /dev/null +++ b/scripts/macos-sign-binaries.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +# intended to run on MacOS only +# signs all executable files in a given folder (as $1) with developer certificate + +# required variables: +# APPLE_DEVELOPER_IDENTITY: "Developer ID Application: ()" +# APPLE_DEVELOPER_ID_BUNDLE: base64-encoded content of apple developer id certificate bundle in pksc12 format +# APPLE_DEVELOPER_ID_BUNDLE_PASSWORD: password used when exporting the bundle + +# note: 'bundle' in apple terminology is 'identity' + +set -euo pipefail + +if [[ "${APPLE_DEVELOPER_ID_BUNDLE:-0}" == 0 || "${APPLE_DEVELOPER_ID_BUNDLE_PASSWORD:-0}" == 0 ]]; then + echo "Apple developer certificate is not configured, skip signing" + exit 0 +fi + +REL_DIR="${1}" +PKSC12_FILE="$HOME/developer-id-application.p12" +base64 --decode > "${PKSC12_FILE}" <<<"${APPLE_DEVELOPER_ID_BUNDLE}" + +KEYCHAIN='emqx.keychain-db' +KEYCHAIN_PASSWORD="$(openssl rand -base64 32)" + +security create-keychain -p "${KEYCHAIN_PASSWORD}" "${KEYCHAIN}" +security set-keychain-settings -lut 21600 "${KEYCHAIN}" +security unlock-keychain -p "${KEYCHAIN_PASSWORD}" "${KEYCHAIN}" +security import "${PKSC12_FILE}" -P "${APPLE_DEVELOPER_ID_BUNDLE_PASSWORD}" -t cert -f pkcs12 -k "${KEYCHAIN}" -T /usr/bin/codesign +security set-key-partition-list -S "apple-tool:,apple:,codesign:" -s -k "${KEYCHAIN_PASSWORD}" "${KEYCHAIN}" +security verify-cert -k "${KEYCHAIN}" -c "${PKSC12_FILE}" +security find-identity -p codesigning "${KEYCHAIN}" + +# add new keychain into the search path for codesign, otherwise the stuff does not work +keychains=$(security list-keychains -d user) +keychain_names=(); +for keychain in ${keychains}; do + basename=$(basename "${keychain}") + keychain_name=${basename::${#basename}-4} + keychain_names+=("${keychain_name}") +done +security -v list-keychains -s "${keychain_names[@]}" "${KEYCHAIN}" + +# sign +codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/erts-*/bin/{beam.smp,dyn_erl,epmd,erl,erl_call,erl_child_setup,erlexec,escript,heart,inet_gethost,run_erl,to_erl} +codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/asn1-*/priv/lib/asn1rt_nif.so +codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/bcrypt-*/priv/bcrypt_nif.so +codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/crypto-*/priv/lib/{crypto.so,otp_test_engine.so} +codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/jiffy-*/priv/jiffy.so +codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/os_mon-*/priv/bin/{cpu_sup,memsup} +codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/runtime_tools-*/priv/lib/{dyntrace.so,trace_ip_drv.so,trace_file_drv.so} From d02f483035d30cffdfbf23fccb75e82de5449ac0 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Fri, 16 Sep 2022 10:22:08 +0200 Subject: [PATCH 21/61] build: fix make clean --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 998eaec56..aaa0b683a 100644 --- a/Makefile +++ b/Makefile @@ -90,7 +90,7 @@ $(REL_PROFILES:%=%): $(REBAR) get-dashboard clean: $(PROFILES:%=clean-%) $(PROFILES:%=clean-%): @if [ -d _build/$(@:clean-%=%) ]; then \ - rm rebar.lock \ + rm -f rebar.lock; \ rm -rf _build/$(@:clean-%=%)/rel; \ $(FIND) _build/$(@:clean-%=%) -name '*.beam' -o -name '*.so' -o -name '*.app' -o -name '*.appup' -o -name '*.o' -o -name '*.d' -type f | xargs rm -f; \ $(FIND) _build/$(@:clean-%=%) -type l -delete; \ From bfb53b7f24f7227d505500f3e19e4a340f1132c2 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 16 Sep 2022 12:56:51 +0200 Subject: [PATCH 22/61] chore: bump app versions after merged tag v4.3.20 --- apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src | 2 +- apps/emqx_exproto/src/emqx_exproto.app.src | 2 +- apps/emqx_management/src/emqx_management.app.src | 2 +- apps/emqx_rule_engine/src/emqx_rule_engine.app.src | 2 +- lib-ce/emqx_dashboard/src/emqx_dashboard.app.src | 2 +- src/emqx.app.src | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src b/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src index 44882dc1b..05d4d3754 100644 --- a/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src +++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_jwt, [{description, "EMQ X Authentication with JWT"}, - {vsn, "4.3.6"}, % strict semver, bump manually! + {vsn, "4.3.7"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_jwt_sup]}, {applications, [kernel,stdlib,jose]}, diff --git a/apps/emqx_exproto/src/emqx_exproto.app.src b/apps/emqx_exproto/src/emqx_exproto.app.src index 98610a40c..40be93b33 100644 --- a/apps/emqx_exproto/src/emqx_exproto.app.src +++ b/apps/emqx_exproto/src/emqx_exproto.app.src @@ -1,6 +1,6 @@ {application, emqx_exproto, [{description, "EMQ X Extension for Protocol"}, - {vsn, "4.3.11"}, %% 4.3.3 is used by ee + {vsn, "4.3.12"}, %% 4.3.3 is used by ee {modules, []}, {registered, []}, {mod, {emqx_exproto_app, []}}, diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index fbec68057..1003a1140 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -1,6 +1,6 @@ {application, emqx_management, [{description, "EMQ X Management API and CLI"}, - {vsn, "4.3.17"}, % strict semver, bump manually! + {vsn, "4.3.18"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel,stdlib,minirest]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index cbdb2e7f7..dffcc7024 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -1,6 +1,6 @@ {application, emqx_rule_engine, [{description, "EMQ X Rule Engine"}, - {vsn, "4.3.15"}, % strict semver, bump manually! + {vsn, "4.3.16"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_rule_engine_sup, emqx_rule_registry]}, {applications, [kernel,stdlib,rulesql,getopt]}, diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src index 1cf6184cd..092e5d3b6 100644 --- a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src +++ b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src @@ -1,6 +1,6 @@ {application, emqx_dashboard, [{description, "EMQ X Web Dashboard"}, - {vsn, "4.3.16"}, % strict semver, bump manually! + {vsn, "4.3.17"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [kernel,stdlib,mnesia,minirest]}, diff --git a/src/emqx.app.src b/src/emqx.app.src index 869b13392..433c75326 100644 --- a/src/emqx.app.src +++ b/src/emqx.app.src @@ -6,7 +6,7 @@ %% the emqx `release' version, which in turn is comprised of several %% apps, one of which is this. See `emqx_release.hrl' for more %% info. - {vsn, "4.3.21"}, % strict semver, bump manually! + {vsn, "4.3.22"}, % strict semver, bump manually! {modules, []}, {registered, []}, {applications, [ kernel From a01d01ac198a9b90b7e07d0a71e93c451fff84a6 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 16 Sep 2022 12:57:20 +0200 Subject: [PATCH 23/61] chore: bump release version to 4.3.21 --- include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index b51281a54..f4be73a77 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.3.20-alpha.2"}). +-define(EMQX_RELEASE, {opensource, "4.3.21-alpha.1"}). -else. From ccc47719cdeba4546cc188b6937be65346d5fc6d Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 16 Sep 2022 12:58:02 +0200 Subject: [PATCH 24/61] chore: bump Chart versions to 4.3.21 --- deploy/charts/emqx/Chart.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/charts/emqx/Chart.yaml b/deploy/charts/emqx/Chart.yaml index 2ad425fc0..4569834df 100644 --- a/deploy/charts/emqx/Chart.yaml +++ b/deploy/charts/emqx/Chart.yaml @@ -13,8 +13,8 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 4.3.20 +version: 4.3.21 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 4.3.20 +appVersion: 4.3.21 From b7fe87d76108c35a75da98d94a9aee7c424cf9ef Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 14 Sep 2022 16:17:16 -0300 Subject: [PATCH 25/61] ci: enable running test cases and cover for main-v4.x An attempt to make Coveralls bot to identify and report coverage changes in `main-v4.{3,4}`-based PRs. Currently, `master`-based PRs get their reports correctly. When one checks such a report, e.g.: https://coveralls.io/builds/52535695 , it identifies if coverage increased or decreased relative to the base branch. `main-v4.x`-based PRs, however, always seem to report as "first builds", so it doesn't report if coverage increased or decreased. E.g.: https://coveralls.io/builds/52541065 . The reason _could be_ that, currently, `main-v4.x` branches do not build themselves. That is, PRs run the test + cover analysis, as usual, but once they're merged, tests (and thus coverage analysis) do not run. And that _could be_ why the tool can't see if coverage increased or decreased with such PR. --- .github/workflows/build_slim_packages.yaml | 2 +- .github/workflows/run_test_cases.yaml | 4 ++++ rebar.config.erl | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index b5a8623ef..18d2d5a54 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -59,7 +59,7 @@ jobs: with: name: rebar3.crashdump path: ./rebar3.crashdump - - name: pakcages test + - name: packages test run: | export CODE_PATH=$GITHUB_WORKSPACE .ci/build_packages/tests.sh diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 96cce885a..04391f06c 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -5,6 +5,8 @@ on: tags: - v* - e* + branches: + - 'main-v4.?' pull_request: jobs: @@ -26,6 +28,7 @@ jobs: run_common_test: runs-on: ${{ matrix.runs-on }} strategy: + fail-fast: false matrix: runs-on: - aws-amd64 @@ -110,6 +113,7 @@ jobs: - name: run cover run: | printenv > .env + docker exec -i erlang bash -c "git config --global --add safe.directory /emqx" docker exec -i erlang bash -c "make cover" docker exec --env-file .env -i erlang bash -c "make coveralls" - name: cat rebar.crashdump diff --git a/rebar.config.erl b/rebar.config.erl index 841399a73..1f7731462 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -103,7 +103,7 @@ plugins(HasElixir) -> test_plugins() -> [ rebar3_proper, - {coveralls, {git, "https://github.com/emqx/coveralls-erl", {branch, "fix-git-info"}}} + {coveralls, {git, "https://github.com/emqx/coveralls-erl", {tag, "v2.2.0-emqx-1"}}} ]. test_deps() -> From e6603548d7b14731e5f0a321c7d3e563ff3d60f1 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sat, 17 Sep 2022 18:50:28 +0200 Subject: [PATCH 26/61] chore: generate appups after bumped new version --- .../emqx_auth_jwt/src/emqx_auth_jwt.appup.src | 6 +++-- apps/emqx_exproto/src/emqx_exproto.appup.src | 6 +++-- .../src/emqx_rule_engine.appup.src | 14 ++++++++-- src/emqx.appup.src | 26 +++++++++++++++++-- 4 files changed, 44 insertions(+), 8 deletions(-) diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src b/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src index 5868efc9e..6f07c4449 100644 --- a/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src +++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src @@ -1,7 +1,8 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.5", + [{"4.3.6",[{load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]}, + {"4.3.5", [{load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]}, {"4.3.4", @@ -12,7 +13,8 @@ {load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[0-2]">>,[{restart_application,emqx_auth_jwt}]}, {<<".*">>,[]}], - [{"4.3.5", + [{"4.3.6",[{load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]}, + {"4.3.5", [{load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]}, {"4.3.4", diff --git a/apps/emqx_exproto/src/emqx_exproto.appup.src b/apps/emqx_exproto/src/emqx_exproto.appup.src index f98eb2d0e..f31e4a969 100644 --- a/apps/emqx_exproto/src/emqx_exproto.appup.src +++ b/apps/emqx_exproto/src/emqx_exproto.appup.src @@ -1,7 +1,8 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.10", + [{"4.3.11",[{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}]}, + {"4.3.10", [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, {"4.3.9", @@ -18,7 +19,8 @@ {load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.10", + [{"4.3.11",[{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}]}, + {"4.3.10", [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, {"4.3.9", diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index aae833bdf..6aa35d2d6 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -1,7 +1,12 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.14", + [{"4.3.15", + [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, + {"4.3.14", [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, @@ -193,7 +198,12 @@ {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.14", + [{"4.3.15", + [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}]}, + {"4.3.14", [{load_module,emqx_rule_sqltester,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 95186208e..ecad06bf9 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -1,7 +1,18 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.20", + [{"4.3.21", + [{load_module,emqx_alarm,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + {load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + {load_module,emqx_router,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, + {load_module,emqx_channel,brutal_purge,soft_purge,[]}]}, + {"4.3.20", [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, {load_module,emqx_router,brutal_purge,soft_purge,[]}, @@ -802,7 +813,18 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.20", + [{"4.3.21", + [{load_module,emqx_alarm,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + {load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + {load_module,emqx_router,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, + {load_module,emqx_channel,brutal_purge,soft_purge,[]}]}, + {"4.3.20", [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, {load_module,emqx_router,brutal_purge,soft_purge,[]}, From cb607f760aca92fba653b143557db119924b7b89 Mon Sep 17 00:00:00 2001 From: William Yang Date: Fri, 16 Sep 2022 21:33:07 +0200 Subject: [PATCH 27/61] perf(TLS): hibernate TLS process after 5s --- etc/emqx.conf | 6 ++++++ priv/emqx.schema | 8 +++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/etc/emqx.conf b/etc/emqx.conf index f7dccfd27..19f85a877 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -1651,6 +1651,12 @@ listener.ssl.external.reuseaddr = true ## Value: true | false ## listener.ssl.external.gc_after_handshake = false +## hibernate the SSL process after idling for amount of time +## Default: undefined (off) +## +## Value: Duration +## listener.ssl.external.hibernate_after = 5s + ##-------------------------------------------------------------------- ## External WebSocket listener for MQTT protocol diff --git a/priv/emqx.schema b/priv/emqx.schema index b6b267abf..1b5232b51 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -1604,6 +1604,11 @@ end}. {datatype, {enum, [true, false]}} ]}. +{mapping, "listener.ssl.$name.hibernate_after", "emqx.listeners", [ + {default, undefined}, + {datatype, {duration, ms}} +]}. + %%-------------------------------------------------------------------- %% MQTT/WebSocket Listeners @@ -2228,7 +2233,8 @@ end}. {reuse_sessions, cuttlefish:conf_get(Prefix ++ ".reuse_sessions", Conf, undefined)}, {honor_cipher_order, cuttlefish:conf_get(Prefix ++ ".honor_cipher_order", Conf, undefined)}, {log_level, cuttlefish:conf_get(Prefix ++ ".log_level", Conf, undefined)}, - {gc_after_handshake, cuttlefish:conf_get(Prefix ++ ".gc_after_handshake", Conf, undefined)} + {gc_after_handshake, cuttlefish:conf_get(Prefix ++ ".gc_after_handshake", Conf, undefined)}, + {hibernate_after, cuttlefish:conf_get(Prefix ++ ".hibernate_after", Conf, undefined)} ]) end, From fae590b95771017994efc7a6b63e84f463ba3f46 Mon Sep 17 00:00:00 2001 From: William Yang Date: Fri, 16 Sep 2022 21:42:03 +0200 Subject: [PATCH 28/61] docs(TLS): update change log --- CHANGES-4.3.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index 247e15dd7..e14b7371e 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -9,6 +9,12 @@ File format: - Use weight-2 heading for releases - One list item per change topic Change log ends with a list of GitHub PRs +## v4.3.21 + +### Enhancements + +- TLS listener memory usage optimization + new option 'hibernate_after' to hibernate TLS process after idling ## v4.3.20 From cdadc8ad448169b1dc9614e57aa5db10bed378f3 Mon Sep 17 00:00:00 2001 From: William Yang Date: Mon, 19 Sep 2022 09:30:35 +0200 Subject: [PATCH 29/61] chore: update appup src --- src/emqx.appup.src | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index ecad06bf9..600c87d8d 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -26,7 +26,8 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}]}, {"4.3.19", - [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + [{load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + {load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, {load_module,emqx_router,brutal_purge,soft_purge,[]}, {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, @@ -40,7 +41,8 @@ {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}]}, {"4.3.18", - [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + [{load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + {load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, {load_module,emqx_router,brutal_purge,soft_purge,[]}, {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, @@ -54,7 +56,8 @@ {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}]}, {"4.3.17", - [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + [{load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + {load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, {load_module,emqx_router,brutal_purge,soft_purge,[]}, {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, @@ -73,7 +76,8 @@ {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}]}, {"4.3.16", - [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + [{load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + {load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, {load_module,emqx_router,brutal_purge,soft_purge,[]}, {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, @@ -99,7 +103,8 @@ {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}]}, {"4.3.15", - [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + [{load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + {load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, {load_module,emqx_router,brutal_purge,soft_purge,[]}, {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, @@ -132,7 +137,8 @@ {update,emqx_os_mon,{advanced,[]}}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.3.14", - [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + [{load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + {load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, {load_module,emqx_router,brutal_purge,soft_purge,[]}, {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, @@ -838,7 +844,8 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}]}, {"4.3.19", - [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + [{load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + {load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, {load_module,emqx_router,brutal_purge,soft_purge,[]}, {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, @@ -852,7 +859,8 @@ {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}]}, {"4.3.18", - [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + [{load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + {load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, {load_module,emqx_router,brutal_purge,soft_purge,[]}, {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, @@ -866,7 +874,8 @@ {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}]}, {"4.3.17", - [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + [{load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + {load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, {load_module,emqx_router,brutal_purge,soft_purge,[]}, {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, @@ -885,7 +894,8 @@ {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}]}, {"4.3.16", - [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + [{load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + {load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, {load_module,emqx_router,brutal_purge,soft_purge,[]}, {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, @@ -911,7 +921,8 @@ {apply,{emqx_exclusive_subscription,on_delete_module,[]}}, {delete_module,emqx_exclusive_subscription}]}, {"4.3.15", - [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + [{load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + {load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, {load_module,emqx_router,brutal_purge,soft_purge,[]}, {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, @@ -943,7 +954,8 @@ {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.3.14", - [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + [{load_module,emqx_listeners,brutal_purge,soft_purge,[]}, + {load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, {load_module,emqx_router,brutal_purge,soft_purge,[]}, {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, From 97fb8c1133b2512245a30626c44b629d8a7f9afb Mon Sep 17 00:00:00 2001 From: William Yang Date: Fri, 16 Sep 2022 20:18:45 +0200 Subject: [PATCH 30/61] perf(TLS): set default sndbuf and recbuf to 4K Without using default values, sockets will be opened with OS default buffer sizes. OS default is set by linux aligning to the host memory size that large memory has larger OS default. Now more and more user run EMQX in container which has large OS memory but the container max memory is limited by cgroup that could be very small amount compared to the OS total memory size. To eliminate uncertainty, it is better to have the default value set. --- CHANGES-4.3.md | 4 +++- etc/emqx.conf | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index e14b7371e..38f4cbb48 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -15,7 +15,9 @@ File format: - TLS listener memory usage optimization new option 'hibernate_after' to hibernate TLS process after idling - +- TLS listener default buffer size to 4KB + Eliminate uncertainty that the buffer size is set by OS default + ## v4.3.20 ### Bug fixes diff --git a/etc/emqx.conf b/etc/emqx.conf index 19f85a877..7b1bf1ef0 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -1609,14 +1609,14 @@ listener.ssl.external.ciphers = TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TL ## See: listener.tcp.$name.recbuf ## ## Value: Bytes -## listener.ssl.external.recbuf = 4KB +listener.ssl.external.recbuf = 4KB ## The TCP send buffer(os kernel) for internal MQTT connections. ## ## See: listener.tcp.$name.sndbuf ## ## Value: Bytes -## listener.ssl.external.sndbuf = 4KB +listener.ssl.external.sndbuf = 4KB ## The size of the user-level software buffer used by the driver. ## From f63d53e6f51f6a0078cd518ed336bd41ced9d387 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 20 Sep 2022 15:05:27 +0200 Subject: [PATCH 31/61] docs: update 4.3 changelog --- CHANGES-4.3.md | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index 38f4cbb48..120d0a67d 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -9,15 +9,20 @@ File format: - Use weight-2 heading for releases - One list item per change topic Change log ends with a list of GitHub PRs + ## v4.3.21 ### Enhancements -- TLS listener memory usage optimization - new option 'hibernate_after' to hibernate TLS process after idling -- TLS listener default buffer size to 4KB - Eliminate uncertainty that the buffer size is set by OS default - +- TLS listener memory usage optimization [#9005](https://github.com/emqx/emqx/pull/9005). + New config `listener.ssl.$NAME.hibernate_after` to hibernate TLS connection process after idling. + Hibernation can reduce RAM usage significantly, but may cost more CPU. + This configuration is by default disabled. + Our preliminary test shows a 50% of RAM usage decline when configured to '5s'. + +- TLS listener default buffer size to 4KB [#9007](https://github.com/emqx/emqx/pull/9007) + Eliminate uncertainty that the buffer size is set by OS default. + ## v4.3.20 ### Bug fixes From 86028dae5ac937666945ba42b959a5b3cce907bf Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 20 Sep 2022 14:24:53 -0300 Subject: [PATCH 32/61] ci(coveralls): fix glob pattern to run coveralls after merge --- .github/workflows/run_test_cases.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 04391f06c..def12cb16 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -6,7 +6,7 @@ on: - v* - e* branches: - - 'main-v4.?' + - 'main-v4.[0-9]?' pull_request: jobs: From 9d99bf8b917aa21d5906b0a33c67b8fc80293425 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sat, 17 Sep 2022 15:47:54 +0200 Subject: [PATCH 33/61] build: add release cut script --- scripts/rel/cut4x.sh | 238 ++++++++++++++++++++++++++++++++++++ scripts/rel/sync-remotes.sh | 201 ++++++++++++++++++++++++++++++ 2 files changed, 439 insertions(+) create mode 100755 scripts/rel/cut4x.sh create mode 100755 scripts/rel/sync-remotes.sh diff --git a/scripts/rel/cut4x.sh b/scripts/rel/cut4x.sh new file mode 100755 index 000000000..58131c757 --- /dev/null +++ b/scripts/rel/cut4x.sh @@ -0,0 +1,238 @@ +#!/usr/bin/env bash + +## cut a new 4.x release for EMQX (opensource or enterprise). + +set -euo pipefail +# ensure dir +cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." + +usage() { + cat < Date: Tue, 20 Sep 2022 20:28:30 +0200 Subject: [PATCH 34/61] build(update_appup): red color for error logs --- scripts/update_appup.escript | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/scripts/update_appup.escript b/scripts/update_appup.escript index 9282aa902..f5fb15898 100755 --- a/scripts/update_appup.escript +++ b/scripts/update_appup.escript @@ -3,6 +3,9 @@ -mode(compile). +-define(RED, "\e[31m"). +-define(RESET, "\e[39m"). + usage() -> "A script that fills in boilerplate for appup files. @@ -109,7 +112,7 @@ changes, supervisor changes, process restarts and so on. Also the load order of the beam files might need updating.~n"), halt(0); warn_and_exit(false) -> - log("~nERROR: Incomplete appups found. Please inspect the output for more details.~n"), + logerr("Incomplete appups found. Please inspect the output for more details.~n", []), halt(1). prepare(Baseline, Options = #{make_command := MakeCommand, beams_dir := BeamDir}) -> @@ -474,8 +477,8 @@ check_appup(App, Upgrade, Downgrade, OldUpgrade, OldDowngrade) -> ok; {diffs, Diffs} -> set_invalid(), - log("ERROR: Appup file for '~p' is not complete.~n" - "Missing:~100p~n", [App, Diffs]), + logerr("Appup file for '~p' is not complete.~n" + "Missing:~100p~n", [App, Diffs]), notok end. @@ -494,7 +497,7 @@ render_appup(App, File, Up, Down) -> do_render_appup(File, Up, Down); {error, enoent} when IsCheck -> %% failed to read old file, exit - log("ERROR: ~s is missing", [File]), + logerr("~s is missing", [File]), set_invalid() end. @@ -580,8 +583,8 @@ diff_app(UpOrDown, App, case UpOrDown =:= up of true -> %% only log for the upgrade case because it would be the same result - log("ERROR: Application '~p' contains changes, but its version is not updated. ~s", - [App, format_changes(Changes)]); + logerr("Application '~p' contains changes, but its version is not updated. ~s", + [App, format_changes(Changes)]); false -> ok end; @@ -723,5 +726,8 @@ log(Msg) -> log(Msg, Args) -> io:format(standard_error, Msg, Args). +logerr(Msg, Args) -> + io:format(standard_error, ?RED ++ "ERROR: "++ Msg ++ ?RESET, Args). + otp_standard_apps() -> [ssl, mnesia, kernel, asn1, stdlib]. From 66197f26d42072db77914a51146aa420093f48fe Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 21 Sep 2022 12:35:12 +0200 Subject: [PATCH 35/61] docs: update release note for v4.3.21 --- CHANGES-4.3.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index 120d0a67d..a17f3dcc7 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -23,13 +23,14 @@ File format: - TLS listener default buffer size to 4KB [#9007](https://github.com/emqx/emqx/pull/9007) Eliminate uncertainty that the buffer size is set by OS default. +- Fix delayed publish inaccurate caused by os time change. [#8908](https://github.com/emqx/emqx/pull/8908) + ## v4.3.20 ### Bug fixes - Fix rule-engine update behaviour which may initialize actions for disabled rules. [#8849](https://github.com/emqx/emqx/pull/8849) - Fix JWT plugin don't support non-integer timestamp claims. [#8862](https://github.com/emqx/emqx/pull/8862) -- Fix delayed publish inaccurate caused by os time change. [#8908](https://github.com/emqx/emqx/pull/8908) - Fix a possible dead loop caused by shared subscriptions with `shared_dispatch_ack_enabled=true`. [#8918](https://github.com/emqx/emqx/pull/8918) - Fix dashboard binding IP address not working. [#8916](https://github.com/emqx/emqx/pull/8916) - Fix rule SQL topic matching to null values failed. [#8927](https://github.com/emqx/emqx/pull/8927) From f0f50b3d3f45bcb2651f3eedefc41b701a771624 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 21 Sep 2022 12:50:49 +0200 Subject: [PATCH 36/61] chore: fix CHANGES-4.3.md --- CHANGES-4.3.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index a17f3dcc7..55882077e 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -25,6 +25,11 @@ File format: - Fix delayed publish inaccurate caused by os time change. [#8908](https://github.com/emqx/emqx/pull/8908) +- Disable authorization for `api/v4/emqx_prometheus` endpoint. [8955](https://github.com/emqx/emqx/pull/8955) + +- Added a test to prevent a last will testament message to be + published when a client is denied connection. [#8894](https://github.com/emqx/emqx/pull/8894) + ## v4.3.20 ### Bug fixes @@ -38,9 +43,6 @@ File format: `SELECT topic =~ 't' as r FROM "$events/client_connected"`. The topic is a null value as there's no such field in event `$events/client_connected`, so it should return false if match it to a topic. -- Added a test to prevent a last will testament message to be - published when a client is denied connection. [#8894](https://github.com/emqx/emqx/pull/8894) -- Disable authorization for `api/v4/emqx_prometheus` endpoint. [8955](https://github.com/emqx/emqx/pull/8955) ## v4.3.19 From 514519da33a7eb34e68f964d659ce40289571ee0 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 21 Sep 2022 13:23:26 +0200 Subject: [PATCH 37/61] docs: Add v4.3.22 in CHANGES-4.3.md --- CHANGES-4.3.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index 55882077e..2e2f5157c 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -10,6 +10,8 @@ File format: - One list item per change topic Change log ends with a list of GitHub PRs +## v4.3.22 + ## v4.3.21 ### Enhancements From 4635d522737f4947204a8433259ad9cdb7ed6a35 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 2 Sep 2022 15:04:39 +0800 Subject: [PATCH 38/61] chore: add reboot_hook/shutdown_hook for enterprise --- src/emqx.app.src | 2 +- src/emqx.appup.src | 26 ++++++++++++++++++++------ src/emqx.erl | 33 +++++++++++++++++++++++++++++++-- 3 files changed, 52 insertions(+), 9 deletions(-) diff --git a/src/emqx.app.src b/src/emqx.app.src index 433c75326..99715d1a2 100644 --- a/src/emqx.app.src +++ b/src/emqx.app.src @@ -6,7 +6,7 @@ %% the emqx `release' version, which in turn is comprised of several %% apps, one of which is this. See `emqx_release.hrl' for more %% info. - {vsn, "4.3.22"}, % strict semver, bump manually! + {vsn, "4.3.23"}, % strict semver, bump manually! {modules, []}, {registered, []}, {applications, [ kernel diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 600c87d8d..8a0a2d4ff 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -1,7 +1,10 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.21", + [{"4.3.22", + [{load_module,emqx,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, + {"4.3.21", [{load_module,emqx_alarm,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}, @@ -11,7 +14,8 @@ {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_channel,brutal_purge,soft_purge,[]}, + {load_module,emqx,brutal_purge,soft_purge,[]}]}, {"4.3.20", [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, @@ -24,7 +28,8 @@ {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx,brutal_purge,soft_purge,[]}]}, {"4.3.19", [{load_module,emqx_listeners,brutal_purge,soft_purge,[]}, {load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, @@ -37,6 +42,7 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}]}, @@ -54,6 +60,7 @@ {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}]}, {"4.3.17", [{load_module,emqx_listeners,brutal_purge,soft_purge,[]}, @@ -819,7 +826,10 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.21", + [{"4.3.22", + [{load_module,emqx,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, + {"4.3.21", [{load_module,emqx_alarm,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}, @@ -829,7 +839,8 @@ {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_channel,brutal_purge,soft_purge,[]}, + {load_module,emqx,brutal_purge,soft_purge,[]}]}, {"4.3.20", [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, @@ -842,7 +853,8 @@ {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx,brutal_purge,soft_purge,[]}]}, {"4.3.19", [{load_module,emqx_listeners,brutal_purge,soft_purge,[]}, {load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, @@ -855,6 +867,7 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}]}, @@ -872,6 +885,7 @@ {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}]}, {"4.3.17", [{load_module,emqx_listeners,brutal_purge,soft_purge,[]}, diff --git a/src/emqx.erl b/src/emqx.erl index 469a4672b..4892fa6cc 100644 --- a/src/emqx.erl +++ b/src/emqx.erl @@ -228,6 +228,7 @@ shutdown() -> shutdown(Reason) -> ?LOG(critical, "emqx shutdown for ~s", [Reason]), + on_shutdown_hook(Reason), _ = emqx_plugins:unload(), lists:foreach(fun application:stop/1 , lists:reverse(default_started_applications()) @@ -238,10 +239,12 @@ reboot() -> true -> _ = application:stop(emqx_dashboard), %% dashboard must be started after mnesia lists:foreach(fun application:start/1 , default_started_applications()), - application:start(emqx_dashboard); + _ = application:start(emqx_dashboard), + on_reboot_hooks(); false -> - lists:foreach(fun application:start/1 , default_started_applications()) + lists:foreach(fun application:start/1 , default_started_applications()), + on_reboot_hooks() end. is_application_running(App) -> @@ -256,6 +259,32 @@ default_started_applications() -> [gproc, esockd, ranch, cowboy, ekka, emqx, emqx_modules]. -endif. +-ifdef(EMQX_ENTERPRISE). +on_reboot_hooks() -> + try + _ = emqx_license_api:bootstrap_license(), + ok + catch + Kind:Reason:Stack -> + ?LOG(critical, "~p while rebooting: ~p, ~p", [Kind, Reason, Stack]), + ok + end, + ok. + +on_shutdown_hook(join) -> + emqx_modules:sync_load_modules_file(), + ok; +on_shutdown_hook(_) -> + ok. + +-else. +on_reboot_hooks() -> + ok. + +on_shutdown_hook(_) -> + ok. +-endif. + %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- From 362c176d24842d096108bed0a8eeda6d14b10aef Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 9 Sep 2022 09:52:33 +0800 Subject: [PATCH 39/61] chore: update appup.src --- src/emqx.appup.src | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 8a0a2d4ff..f5fb5b0ba 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -28,8 +28,13 @@ {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, +<<<<<<< HEAD {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}]}, +======= + {load_module,emqx,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}]}, +>>>>>>> 8e5674f14 (chore: update appup.src) {"4.3.19", [{load_module,emqx_listeners,brutal_purge,soft_purge,[]}, {load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, From c2f1f1aab85fc2cd24d919df8885af31d09f5b49 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 14 Sep 2022 10:22:03 +0800 Subject: [PATCH 40/61] chore: rename on_xxx_hooks to on_xxx --- src/emqx.appup.src | 5 ----- src/emqx.erl | 16 ++++++++-------- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index f5fb5b0ba..8a0a2d4ff 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -28,13 +28,8 @@ {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, -<<<<<<< HEAD {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}]}, -======= - {load_module,emqx,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}]}, ->>>>>>> 8e5674f14 (chore: update appup.src) {"4.3.19", [{load_module,emqx_listeners,brutal_purge,soft_purge,[]}, {load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, diff --git a/src/emqx.erl b/src/emqx.erl index 4892fa6cc..ae78e5795 100644 --- a/src/emqx.erl +++ b/src/emqx.erl @@ -228,7 +228,7 @@ shutdown() -> shutdown(Reason) -> ?LOG(critical, "emqx shutdown for ~s", [Reason]), - on_shutdown_hook(Reason), + on_shutdown(Reason), _ = emqx_plugins:unload(), lists:foreach(fun application:stop/1 , lists:reverse(default_started_applications()) @@ -240,11 +240,11 @@ reboot() -> _ = application:stop(emqx_dashboard), %% dashboard must be started after mnesia lists:foreach(fun application:start/1 , default_started_applications()), _ = application:start(emqx_dashboard), - on_reboot_hooks(); + on_reboot(); false -> lists:foreach(fun application:start/1 , default_started_applications()), - on_reboot_hooks() + on_reboot() end. is_application_running(App) -> @@ -260,7 +260,7 @@ default_started_applications() -> -endif. -ifdef(EMQX_ENTERPRISE). -on_reboot_hooks() -> +on_reboot() -> try _ = emqx_license_api:bootstrap_license(), ok @@ -271,17 +271,17 @@ on_reboot_hooks() -> end, ok. -on_shutdown_hook(join) -> +on_shutdown(join) -> emqx_modules:sync_load_modules_file(), ok; -on_shutdown_hook(_) -> +on_shutdown(_) -> ok. -else. -on_reboot_hooks() -> +on_reboot() -> ok. -on_shutdown_hook(_) -> +on_shutdown(_) -> ok. -endif. From 61d745a230218ee59c1083085d6f48f54dda1067 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 22 Sep 2022 11:00:16 +0800 Subject: [PATCH 41/61] feat: add emqx_misc:ipv6_probe/2 function --- src/emqx.appup.src | 6 ++++++ src/emqx_misc.erl | 10 +++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 8a0a2d4ff..c7643fdfc 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -3,6 +3,7 @@ {VSN, [{"4.3.22", [{load_module,emqx,brutal_purge,soft_purge,[]}, + {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.3.21", [{load_module,emqx_alarm,brutal_purge,soft_purge,[]}, @@ -15,6 +16,7 @@ {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, + {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}]}, {"4.3.20", [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, @@ -29,6 +31,7 @@ {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}]}, {"4.3.19", [{load_module,emqx_listeners,brutal_purge,soft_purge,[]}, @@ -828,6 +831,7 @@ {<<".*">>,[]}], [{"4.3.22", [{load_module,emqx,brutal_purge,soft_purge,[]}, + {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.3.21", [{load_module,emqx_alarm,brutal_purge,soft_purge,[]}, @@ -840,6 +844,7 @@ {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, + {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}]}, {"4.3.20", [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, @@ -854,6 +859,7 @@ {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}]}, {"4.3.19", [{load_module,emqx_listeners,brutal_purge,soft_purge,[]}, diff --git a/src/emqx_misc.erl b/src/emqx_misc.erl index 813ab84e8..bfda5bb38 100644 --- a/src/emqx_misc.erl +++ b/src/emqx_misc.erl @@ -45,6 +45,7 @@ , index_of/2 , maybe_parse_ip/1 , ipv6_probe/1 + , ipv6_probe/2 ]). -export([ bin2hexstr_A_F/1 @@ -84,12 +85,15 @@ maybe_parse_ip(Host) -> %% @doc Add `ipv6_probe' socket option if it's supported. ipv6_probe(Opts) -> + ipv6_probe(Opts, true). + +ipv6_probe(Opts, Ipv6Probe) when is_boolean(Ipv6Probe) orelse is_integer(Ipv6Probe) -> Bool = try gen_tcp:ipv6_probe() catch _ : _ -> false end, - ipv6_probe(Bool, Opts). + ipv6_probe(Bool, Opts, Ipv6Probe). -ipv6_probe(false, Opts) -> Opts; -ipv6_probe(true, Opts) -> [{ipv6_probe, true} | Opts]. +ipv6_probe(false, Opts, _) -> Opts; +ipv6_probe(true, Opts, Ipv6Probe) -> [{ipv6_probe, Ipv6Probe} | Opts]. %% @doc Merge options -spec(merge_opts(Opts, Opts) -> Opts when Opts :: proplists:proplist()). From 27983e7df40b1deaaecaf4485eb23ae9c8ec3fb9 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 22 Sep 2022 14:24:59 +0800 Subject: [PATCH 42/61] chore: update emqx.appup.src --- src/emqx.appup.src | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index c7643fdfc..16dfd1464 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -2,7 +2,16 @@ %% Unless you know what you are doing, DO NOT edit manually!! {VSN, [{"4.3.22", - [{load_module,emqx,brutal_purge,soft_purge,[]}, + [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, + {load_module,emqx_channel,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + {load_module,emqx_router,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.3.21", @@ -830,7 +839,16 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.3.22", - [{load_module,emqx,brutal_purge,soft_purge,[]}, + [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, + {load_module,emqx_channel,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, + {load_module,emqx_router,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.3.21", From 9642f25ea0ec99dbe9afd27514785c605bcf398b Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 22 Sep 2022 14:36:30 +0800 Subject: [PATCH 43/61] chore: update changelog --- CHANGES-4.3.md | 6 ++++-- src/emqx.app.src | 2 +- src/emqx.appup.src | 29 ++--------------------------- 3 files changed, 7 insertions(+), 30 deletions(-) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index 2e2f5157c..737303c27 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -12,6 +12,10 @@ File format: ## v4.3.22 +### Minor changes +- Fix delayed publish inaccurate caused by os time change. [#8908](https://github.com/emqx/emqx/pull/8908) + + ## v4.3.21 ### Enhancements @@ -25,8 +29,6 @@ File format: - TLS listener default buffer size to 4KB [#9007](https://github.com/emqx/emqx/pull/9007) Eliminate uncertainty that the buffer size is set by OS default. -- Fix delayed publish inaccurate caused by os time change. [#8908](https://github.com/emqx/emqx/pull/8908) - - Disable authorization for `api/v4/emqx_prometheus` endpoint. [8955](https://github.com/emqx/emqx/pull/8955) - Added a test to prevent a last will testament message to be diff --git a/src/emqx.app.src b/src/emqx.app.src index 99715d1a2..433c75326 100644 --- a/src/emqx.app.src +++ b/src/emqx.app.src @@ -6,7 +6,7 @@ %% the emqx `release' version, which in turn is comprised of several %% apps, one of which is this. See `emqx_release.hrl' for more %% info. - {vsn, "4.3.23"}, % strict semver, bump manually! + {vsn, "4.3.22"}, % strict semver, bump manually! {modules, []}, {registered, []}, {applications, [ kernel diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 16dfd1464..4bca299a1 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -1,20 +1,7 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.22", - [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, - {load_module,emqx_router,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, - {"4.3.21", + [{"4.3.21", [{load_module,emqx_alarm,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}, @@ -838,19 +825,7 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.22", - [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, - {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_cm,brutal_purge,soft_purge,[]}, - {load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, - {load_module,emqx_router,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx,brutal_purge,soft_purge,[]}, - {load_module,emqx_misc,brutal_purge,soft_purge,[]}, - {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, + [ {"4.3.21", [{load_module,emqx_alarm,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, From deca9cc39505efb4276696b1ff2392b1cd3f5414 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 22 Sep 2022 16:09:42 +0800 Subject: [PATCH 44/61] chore: update changelog for delayed publish --- CHANGES-4.3.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index 737303c27..eb5587d78 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -13,8 +13,6 @@ File format: ## v4.3.22 ### Minor changes -- Fix delayed publish inaccurate caused by os time change. [#8908](https://github.com/emqx/emqx/pull/8908) - ## v4.3.21 @@ -34,6 +32,10 @@ File format: - Added a test to prevent a last will testament message to be published when a client is denied connection. [#8894](https://github.com/emqx/emqx/pull/8894) +### Bug fixes + +- Fix delayed publish inaccurate caused by os time change. [#8908](https://github.com/emqx/emqx/pull/8908) + ## v4.3.20 ### Bug fixes From 7a26aae27b88537fc22638d82793f881bf3bd0db Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 22 Sep 2022 12:20:17 +0300 Subject: [PATCH 45/61] chore(retainer): sync retainer from release-e43 --- apps/emqx_retainer/src/emqx_retainer.app.src | 2 +- apps/emqx_retainer/src/emqx_retainer_sup.erl | 17 +++++++++++++++++ .../test/emqx_retainer_ct_helper.erl | 3 +++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/apps/emqx_retainer/src/emqx_retainer.app.src b/apps/emqx_retainer/src/emqx_retainer.app.src index 80ec02c73..dcc48fbae 100644 --- a/apps/emqx_retainer/src/emqx_retainer.app.src +++ b/apps/emqx_retainer/src/emqx_retainer.app.src @@ -1,6 +1,6 @@ {application, emqx_retainer, [{description, "EMQ X Retainer"}, - {vsn, "4.3.4"}, % strict semver, bump manually! + {vsn, "4.3.5"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_retainer_sup]}, {applications, [kernel,stdlib]}, diff --git a/apps/emqx_retainer/src/emqx_retainer_sup.erl b/apps/emqx_retainer/src/emqx_retainer_sup.erl index 2028affb6..ca16a98bb 100644 --- a/apps/emqx_retainer/src/emqx_retainer_sup.erl +++ b/apps/emqx_retainer/src/emqx_retainer_sup.erl @@ -34,6 +34,23 @@ init([Env]) -> type => worker, modules => [emqx_retainer]} || not is_managed_by_modules()]}}. +-ifdef(EMQX_ENTERPRISE). + +is_managed_by_modules() -> + try + case supervisor:get_childspec(emqx_modules_sup, emqx_retainer) of + {ok, _} -> true; + _ -> false + end + catch + exit : {noproc, _} -> + false + end. + +-else. + is_managed_by_modules() -> %% always false for opensource edition false. + +-endif. diff --git a/apps/emqx_retainer/test/emqx_retainer_ct_helper.erl b/apps/emqx_retainer/test/emqx_retainer_ct_helper.erl index fd12a4e01..67f58ec26 100644 --- a/apps/emqx_retainer/test/emqx_retainer_ct_helper.erl +++ b/apps/emqx_retainer/test/emqx_retainer_ct_helper.erl @@ -20,7 +20,9 @@ -export([ensure_start/0, ensure_stop/0]). -ifdef(EMQX_ENTERPRISE). ensure_start() -> + %% for enterprise edition, retainer is started by modules application:stop(emqx_modules), + ensure_stop(), init_conf(), emqx_ct_helpers:start_apps([emqx_retainer]), ok. @@ -29,6 +31,7 @@ ensure_start() -> ensure_start() -> init_conf(), + ensure_stop(), emqx_ct_helpers:start_apps([emqx_retainer]), ok. From 1379f39f26583ca33e4a2652732364f0d3ad28ac Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 22 Sep 2022 18:02:39 +0800 Subject: [PATCH 46/61] chore: sync from release-e4.3 --- src/emqx.appup.src | 20 ++++++++++++++++++++ src/emqx_http_lib.erl | 4 ++-- src/emqx_plugins.erl | 19 +++++++++++++++---- 3 files changed, 37 insertions(+), 6 deletions(-) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 4bca299a1..dc255a221 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -13,6 +13,7 @@ {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, + {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}]}, {"4.3.20", [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, @@ -28,6 +29,7 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, + {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}]}, {"4.3.19", [{load_module,emqx_listeners,brutal_purge,soft_purge,[]}, @@ -44,6 +46,7 @@ {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}]}, {"4.3.18", [{load_module,emqx_listeners,brutal_purge,soft_purge,[]}, @@ -60,6 +63,7 @@ {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}, + {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}]}, {"4.3.17", [{load_module,emqx_listeners,brutal_purge,soft_purge,[]}, @@ -80,6 +84,7 @@ {update,emqx_broker_sup,supervisor}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}]}, {"4.3.16", [{load_module,emqx_listeners,brutal_purge,soft_purge,[]}, @@ -107,6 +112,7 @@ {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, + {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}]}, {"4.3.15", [{load_module,emqx_listeners,brutal_purge,soft_purge,[]}, @@ -141,6 +147,7 @@ {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, {update,emqx_os_mon,{advanced,[]}}, + {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.3.14", [{load_module,emqx_listeners,brutal_purge,soft_purge,[]}, @@ -177,6 +184,7 @@ {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, {update,emqx_os_mon,{advanced,[]}}, + {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}]}, {"4.3.13", [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, @@ -216,6 +224,7 @@ {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, {update,emqx_os_mon,{advanced,[]}}, + {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}]}, {"4.3.12", [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, @@ -258,6 +267,7 @@ {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, + {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.11", [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, @@ -838,6 +848,7 @@ {load_module,emqx_tracer,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, + {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}]}, {"4.3.20", [{load_module,emqx_plugins,brutal_purge,soft_purge,[]}, @@ -853,6 +864,7 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, + {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}]}, {"4.3.19", [{load_module,emqx_listeners,brutal_purge,soft_purge,[]}, @@ -869,6 +881,7 @@ {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}]}, {"4.3.18", [{load_module,emqx_listeners,brutal_purge,soft_purge,[]}, @@ -885,6 +898,7 @@ {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}, + {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}]}, {"4.3.17", [{load_module,emqx_listeners,brutal_purge,soft_purge,[]}, @@ -905,6 +919,7 @@ {update,emqx_broker_sup,supervisor}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, + {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}]}, {"4.3.16", [{load_module,emqx_listeners,brutal_purge,soft_purge,[]}, @@ -932,6 +947,7 @@ {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {apply,{emqx_exclusive_subscription,on_delete_module,[]}}, + {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, {delete_module,emqx_exclusive_subscription}]}, {"4.3.15", [{load_module,emqx_listeners,brutal_purge,soft_purge,[]}, @@ -965,6 +981,7 @@ {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.3.14", [{load_module,emqx_listeners,brutal_purge,soft_purge,[]}, @@ -1000,6 +1017,7 @@ {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}]}, {"4.3.13", [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, @@ -1038,6 +1056,7 @@ {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}]}, {"4.3.12", [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, @@ -1078,6 +1097,7 @@ {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, + {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.11", [{load_module,emqx_router_helper,brutal_purge,soft_purge,[]}, diff --git a/src/emqx_http_lib.erl b/src/emqx_http_lib.erl index 58f908041..8b73a572a 100644 --- a/src/emqx_http_lib.erl +++ b/src/emqx_http_lib.erl @@ -96,8 +96,8 @@ do_parse(URI) -> %% underscores replaced with hyphens %% NOTE: assuming the input Headers list is a proplists, %% that is, when a key is duplicated, list header overrides tail -%% e.g. [{"Content_Type", "applicaiton/binary"}, {<<"content-type">>, "applicaiton/json"}] -%% results in: [{"content-type", "applicaiton/binary"}] +%% e.g. [{"Content_Type", "applicaiton/binary"}, {"content-type", "applicaiton/json"}] +%% results in: [{<<"content-type">>, "applicaiton/binary"}] normalise_headers(Headers0) -> F = fun({K0, V}) -> K = re:replace(K0, "_", "-", [{return,binary}]), diff --git a/src/emqx_plugins.erl b/src/emqx_plugins.erl index a01dc2a5b..0a25e0102 100644 --- a/src/emqx_plugins.erl +++ b/src/emqx_plugins.erl @@ -24,6 +24,7 @@ -export([init/0]). -export([ load/0 + , force_load/0 , load/1 , unload/0 , unload/1 @@ -59,12 +60,17 @@ init() -> %% @doc Load all plugins when the broker started. -spec(load() -> ok | ignore | {error, term()}). load() -> + do_load(#{force_load => false}). +force_load() -> + do_load(#{force_load => true}). + +do_load(Options) -> ok = load_ext_plugins(emqx:get_env(expand_plugins_dir)), case emqx:get_env(plugins_loaded_file) of undefined -> ignore; %% No plugins available File -> _ = ensure_file(File), - with_loaded_file(File, fun(Names) -> load_plugins(Names, false) end) + with_loaded_file(File, fun(Names) -> load_plugins(Names, Options, false) end) end. %% @doc Load a Plugin @@ -282,18 +288,23 @@ filter_plugins([{Name, Load} | Names], Plugins) -> filter_plugins([Name | Names], Plugins) when is_atom(Name) -> filter_plugins([{Name, true} | Names], Plugins). -load_plugins(Names, Persistent) -> +load_plugins(Names, Options, Persistent) -> Plugins = list(), NotFound = Names -- names(Plugins), case NotFound of [] -> ok; NotFound -> ?LOG(alert, "cannot_find_plugins: ~p", [NotFound]) end, - NeedToLoad = (Names -- NotFound) -- names(started_app), + NeedToLoad0 = Names -- NotFound, + NeedToLoad1 = + case Options of + #{force_load := true} -> NeedToLoad0; + _ -> NeedToLoad0 -- names(started_app) + end, lists:foreach(fun(Name) -> Plugin = find_plugin(Name, Plugins), load_plugin(Plugin#plugin.name, Persistent) - end, NeedToLoad). + end, NeedToLoad1). generate_configs(App) -> ConfigFile = filename:join([emqx:get_env(plugins_etc_dir), App]) ++ ".config", From c05ce82933f2667a390927a637acdba9b28fc1b7 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 29 Mar 2022 12:09:19 +0800 Subject: [PATCH 47/61] refactor(psk): create the ets table in supervisor process --- apps/emqx_psk_file/include/emqx_psk_file.hrl | 25 +++++++++++++++++++ apps/emqx_psk_file/src/emqx_psk_file.app.src | 2 +- .../emqx_psk_file/src/emqx_psk_file.appup.src | 10 ++++++++ apps/emqx_psk_file/src/emqx_psk_file.erl | 12 ++++----- apps/emqx_psk_file/src/emqx_psk_file_sup.erl | 7 +++++- 5 files changed, 47 insertions(+), 9 deletions(-) create mode 100644 apps/emqx_psk_file/include/emqx_psk_file.hrl create mode 100644 apps/emqx_psk_file/src/emqx_psk_file.appup.src diff --git a/apps/emqx_psk_file/include/emqx_psk_file.hrl b/apps/emqx_psk_file/include/emqx_psk_file.hrl new file mode 100644 index 000000000..fe8ed1f94 --- /dev/null +++ b/apps/emqx_psk_file/include/emqx_psk_file.hrl @@ -0,0 +1,25 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-ifndef(EMQX_PSK_FILE). +-define(EMQX_PSK_FILE, true). + +-define(PSK_FILE_TAB, emqx_psk_file). + +-record(psk_entry, {psk_id :: binary(), + psk_str :: binary()}). + +-endif. diff --git a/apps/emqx_psk_file/src/emqx_psk_file.app.src b/apps/emqx_psk_file/src/emqx_psk_file.app.src index b8a6f08a0..ef18c8b69 100644 --- a/apps/emqx_psk_file/src/emqx_psk_file.app.src +++ b/apps/emqx_psk_file/src/emqx_psk_file.app.src @@ -1,6 +1,6 @@ {application, emqx_psk_file, [{description,"EMQX PSK Plugin from File"}, - {vsn, "4.3.0"}, % strict semver, bump manually! + {vsn, "4.3.1"}, % strict semver, bump manually! {modules,[]}, {registered,[emqx_psk_file_sup]}, {applications,[kernel,stdlib]}, diff --git a/apps/emqx_psk_file/src/emqx_psk_file.appup.src b/apps/emqx_psk_file/src/emqx_psk_file.appup.src new file mode 100644 index 000000000..c782000b5 --- /dev/null +++ b/apps/emqx_psk_file/src/emqx_psk_file.appup.src @@ -0,0 +1,10 @@ +%% -*- mode: erlang -*- +{VSN, + [{"4.3.0", + [{load_module,emqx_psk_file,brutal_purge,soft_purge,[]}, + {load_module,emqx_psk_file_sup,brutal_purge,soft_purge,[]}]} + ], + [{"4.3.0", + [{load_module,emqx_psk_file,brutal_purge,soft_purge,[]}, + {load_module,emqx_psk_file_sup,brutal_purge,soft_purge,[]}]} + ]}. diff --git a/apps/emqx_psk_file/src/emqx_psk_file.erl b/apps/emqx_psk_file/src/emqx_psk_file.erl index b4daee370..e252393c4 100644 --- a/apps/emqx_psk_file/src/emqx_psk_file.erl +++ b/apps/emqx_psk_file/src/emqx_psk_file.erl @@ -16,6 +16,7 @@ -module(emqx_psk_file). +-include("emqx_psk_file.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/logger.hrl"). @@ -26,15 +27,10 @@ %% Hooks functions -export([on_psk_lookup/2]). --define(TAB, ?MODULE). -define(LF, 10). --record(psk_entry, {psk_id :: binary(), - psk_str :: binary()}). - %% Called when the plugin application start load(Env) -> - _ = ets:new(?TAB, [set, named_table, {keypos, #psk_entry.psk_id}]), {ok, PskFile} = file:open(get_value(path, Env), [read, raw, binary, read_ahead]), preload_psks(PskFile, bin(get_value(delimiter, Env))), _ = file:close(PskFile), @@ -45,7 +41,7 @@ unload() -> emqx:unhook('tls_handshake.psk_lookup', fun ?MODULE:on_psk_lookup/2). on_psk_lookup(ClientPSKID, UserState) -> - case ets:lookup(?TAB, ClientPSKID) of + case ets:lookup(?PSK_FILE_TAB, ClientPSKID) of [#psk_entry{psk_str = PskStr}] -> {stop, PskStr}; [] -> @@ -57,7 +53,9 @@ preload_psks(FileHandler, Delimiter) -> {ok, Line} -> case binary:split(Line, Delimiter) of [Key, Rem] -> - ets:insert(?TAB, #psk_entry{psk_id = Key, psk_str = trim_lf(Rem)}), + ets:insert( + ?PSK_FILE_TAB, + #psk_entry{psk_id = Key, psk_str = trim_lf(Rem)}), preload_psks(FileHandler, Delimiter); [Line] -> ?LOG(warning, "[~p] - Invalid line: ~p, delimiter: ~p", [?MODULE, Line, Delimiter]) diff --git a/apps/emqx_psk_file/src/emqx_psk_file_sup.erl b/apps/emqx_psk_file/src/emqx_psk_file_sup.erl index 2e739519c..e643c000a 100644 --- a/apps/emqx_psk_file/src/emqx_psk_file_sup.erl +++ b/apps/emqx_psk_file/src/emqx_psk_file_sup.erl @@ -16,6 +16,8 @@ -module(emqx_psk_file_sup). +-include("emqx_psk_file.hrl"). + -behaviour(supervisor). %% API @@ -25,8 +27,11 @@ -export([init/1]). start_link() -> + _ = ets:new( + ?PSK_FILE_TAB, + [set, named_table, public, {keypos, #psk_entry.psk_id}] + ), supervisor:start_link({local, ?MODULE}, ?MODULE, []). init([]) -> {ok, { {one_for_one, 0, 1}, []} }. - From 0ac78734bdfa62b6be7959770f567628679e8d19 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sun, 25 Sep 2022 14:17:49 +0200 Subject: [PATCH 48/61] build(update_appup.escript): allow external app non-semver --- scripts/update_appup.escript | 50 +++++++++++++++++++++++++++--------- 1 file changed, 38 insertions(+), 12 deletions(-) diff --git a/scripts/update_appup.escript b/scripts/update_appup.escript index f5fb15898..183888817 100755 --- a/scripts/update_appup.escript +++ b/scripts/update_appup.escript @@ -62,7 +62,8 @@ app_specific_actions(_) -> ignored_apps() -> [gpb, %% only a build tool emqx_dashboard, %% generic appup file for all versions - emqx_management %% generic appup file for all versions + emqx_management, %% generic appup file for all versions + emqx_modules_spec %% generic appup file for all versions ] ++ otp_standard_apps(). main(Args) -> @@ -284,9 +285,9 @@ merge_update_actions(App, Changes, Vsns, PrevVersion) -> %% but there is a 1.1.2 in appup we may skip merging instructions for %% 1.1.2 because it's not used and no way to know what has been changed is_skipped_version(App, Vsn, PrevVersion) when is_list(Vsn) andalso is_list(PrevVersion) -> - case is_app_external(App) andalso parse_version_number(Vsn) of + case is_app_external(App) andalso parse_version(Vsn, non_strict_semver) of {ok, VsnTuple} -> - case parse_version_number(PrevVersion) of + case parse_version(PrevVersion, non_strict_semver) of {ok, PrevVsnTuple} -> VsnTuple > PrevVsnTuple; _ -> @@ -397,7 +398,7 @@ contains_version(Needle, Haystack) when is_list(Needle) -> %% past versions that should be covered by regexes in .appup file %% instructions. enumerate_past_versions(Vsn) when is_list(Vsn) -> - case parse_version_number(Vsn) of + case parse_version(Vsn) of {ok, ParsedVsn} -> {ok, enumerate_past_versions(ParsedVsn)}; Error -> @@ -406,14 +407,39 @@ enumerate_past_versions(Vsn) when is_list(Vsn) -> enumerate_past_versions({Major, Minor, Patch}) -> [{Major, Minor, P} || P <- lists:seq(Patch - 1, 0, -1)]. -parse_version_number(Vsn) when is_list(Vsn) -> - Nums = string:split(Vsn, ".", all), - Results = lists:map(fun string:to_integer/1, Nums), - case Results of - [{Major, []}, {Minor, []}, {Patch, []}] -> - {ok, {Major, Minor, Patch}}; - _ -> - {error, bad_version} +parse_version(Vsn) -> + parse_version(Vsn, strict_semver). + +parse_version(Vsn, MaybeSemver) when is_list(Vsn) -> + case parse_dot_separated_numbers(Vsn) of + {ok, {_Major, _Minor, _Patch}} = Res -> + Res; + {ok, Nums} -> + case MaybeSemver of + strict_semver -> + {error, {bad_semver, Vsn}}; + non_strict_semver -> + {ok, Nums} + end; + {error, Reason} -> + {error, {Reason, Vsn}} + end. + +parse_dot_separated_numbers(Str) when is_list(Str) -> + try + Split = string:split(Str, ".", all), + IntL = lists:map(fun(SubStr) -> + case string:to_integer(SubStr) of + {Int, []} when is_integer(Int) -> + Int; + _ -> + throw(no_integer) + end + end, Split), + {ok, list_to_tuple(IntL)} + catch + _ : _ -> + {error, bad_version_string} end. vsn_number_to_string({Major, Minor, Patch}) -> From 2e0eae54f827d9ab8281c686e1518d6f9f85c7d6 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 26 Sep 2022 10:06:27 -0300 Subject: [PATCH 49/61] fix(acl): check ACL before publishing last will testament (lwt) message (4.3) --- .../test/emqx_acl_mnesia_SUITE.erl | 56 ++++++++++++++++++- src/emqx_channel.erl | 34 +++++++---- 2 files changed, 78 insertions(+), 12 deletions(-) diff --git a/apps/emqx_auth_mnesia/test/emqx_acl_mnesia_SUITE.erl b/apps/emqx_auth_mnesia/test/emqx_acl_mnesia_SUITE.erl index 941ebedb9..213f9c278 100644 --- a/apps/emqx_auth_mnesia/test/emqx_acl_mnesia_SUITE.erl +++ b/apps/emqx_auth_mnesia/test/emqx_acl_mnesia_SUITE.erl @@ -20,6 +20,7 @@ -compile(export_all). -include("emqx_auth_mnesia.hrl"). +-include_lib("emqx/include/emqx.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). @@ -77,15 +78,37 @@ init_per_testcase_migration(_, Config) -> emqx_acl_mnesia_migrator:migrate_records(), Config. +init_per_testcase_other(t_last_will_testament_message_check_acl, Config) -> + OriginalACLNoMatch = application:get_env(emqx, acl_nomatch), + application:set_env(emqx, acl_nomatch, deny), + emqx_mod_acl_internal:unload([]), + %% deny all for this client + ClientID = <<"lwt_client">>, + ok = emqx_acl_mnesia_db:add_acl({clientid, ClientID}, <<"#">>, pubsub, deny), + [ {original_acl_nomatch, OriginalACLNoMatch} + , {clientid, ClientID} + | Config]; +init_per_testcase_other(_TestCase, Config) -> + Config. + init_per_testcase(Case, Config) -> PerTestInitializers = [ fun init_per_testcase_clean/2, fun init_per_testcase_migration/2, - fun init_per_testcase_emqx_hook/2 + fun init_per_testcase_emqx_hook/2, + fun init_per_testcase_other/2 ], lists:foldl(fun(Init, Conf) -> Init(Case, Conf) end, Config, PerTestInitializers). -end_per_testcase(_, Config) -> +end_per_testcase(t_last_will_testament_message_check_acl, Config) -> + emqx:unhook('client.check_acl', fun emqx_acl_mnesia:check_acl/5), + case ?config(original_acl_nomatch, Config) of + {ok, Original} -> application:set_env(emqx, acl_nomatch, Original); + _ -> ok + end, + emqx_mod_acl_internal:load([]), + ok; +end_per_testcase(_TestCase, Config) -> emqx:unhook('client.check_acl', fun emqx_acl_mnesia:check_acl/5), Config. @@ -464,6 +487,35 @@ t_rest_api(_Config) -> {ok, Res3} = request_http_rest_list(["$all"]), ?assertMatch([], get_http_data(Res3)). +%% asserts that we check ACL for the LWT topic before publishing the +%% LWT. +t_last_will_testament_message_check_acl(Config) -> + ClientID = ?config(clientid, Config), + {ok, C} = emqtt:start_link([ + {clientid, ClientID}, + {will_topic, <<"$SYS/lwt">>}, + {will_payload, <<"should not be published">>} + ]), + {ok, _} = emqtt:connect(C), + ok = emqx:subscribe(<<"$SYS/lwt">>), + unlink(C), + ok = snabbkaffe:start_trace(), + {true, {ok, _}} = + ?wait_async_action( + exit(C, kill), + #{?snk_kind := last_will_testament_publish_denied}, + 1_000 + ), + ok = snabbkaffe:stop(), + + receive + {deliver, <<"$SYS/lwt">>, #message{payload = <<"should not be published">>}} -> + error(lwt_should_not_be_published_to_forbidden_topic) + after 1_000 -> + ok + end, + + ok. create_conflicting_records() -> Records = [ diff --git a/src/emqx_channel.erl b/src/emqx_channel.erl index 5ac212678..b8f3c5b2c 100644 --- a/src/emqx_channel.erl +++ b/src/emqx_channel.erl @@ -951,9 +951,10 @@ return_sub_unsub_ack(Packet, Channel) -> handle_call(kick, Channel = #channel{ conn_state = ConnState, will_msg = WillMsg, + clientinfo = ClientInfo, conninfo = #{proto_ver := ProtoVer} }) -> - (WillMsg =/= undefined) andalso publish_will_msg(WillMsg), + (WillMsg =/= undefined) andalso publish_will_msg(ClientInfo, WillMsg), Channel1 = case ConnState of connected -> ensure_disconnected(kicked, Channel); _ -> Channel @@ -1102,8 +1103,9 @@ handle_timeout(_TRef, expire_awaiting_rel, handle_timeout(_TRef, expire_session, Channel) -> shutdown(expired, Channel); -handle_timeout(_TRef, will_message, Channel = #channel{will_msg = WillMsg}) -> - (WillMsg =/= undefined) andalso publish_will_msg(WillMsg), +handle_timeout(_TRef, will_message, Channel = #channel{will_msg = WillMsg, + clientinfo = ClientInfo}) -> + (WillMsg =/= undefined) andalso publish_will_msg(ClientInfo, WillMsg), {ok, clean_timer(will_timer, Channel#channel{will_msg = undefined})}; handle_timeout(_TRef, expire_quota_limit, Channel) -> @@ -1159,9 +1161,10 @@ terminate(_Reason, #channel{conn_state = idle} = _Channel) -> ok; terminate(normal, Channel) -> run_terminate_hook(normal, Channel); -terminate(Reason, Channel = #channel{will_msg = WillMsg}) -> +terminate(Reason, Channel = #channel{will_msg = WillMsg, + clientinfo = ClientInfo}) -> should_publish_will_message(Reason, Channel) - andalso publish_will_msg(WillMsg), + andalso publish_will_msg(ClientInfo, WillMsg), run_terminate_hook(Reason, Channel). run_terminate_hook(_Reason, #channel{session = undefined} = _Channel) -> @@ -1701,10 +1704,11 @@ ensure_disconnected(Reason, Channel = #channel{conninfo = ConnInfo, maybe_publish_will_msg(Channel = #channel{will_msg = undefined}) -> Channel; -maybe_publish_will_msg(Channel = #channel{will_msg = WillMsg}) -> +maybe_publish_will_msg(Channel = #channel{will_msg = WillMsg, + clientinfo = ClientInfo}) -> case will_delay_interval(WillMsg) of 0 -> - ok = publish_will_msg(WillMsg), + ok = publish_will_msg(ClientInfo, WillMsg), Channel#channel{will_msg = undefined}; I -> ensure_timer(will_timer, timer:seconds(I), Channel) @@ -1714,9 +1718,19 @@ will_delay_interval(WillMsg) -> maps:get('Will-Delay-Interval', emqx_message:get_header(properties, WillMsg, #{}), 0). -publish_will_msg(Msg) -> - _ = emqx_broker:publish(Msg), - ok. +publish_will_msg(ClientInfo, Msg = #message{topic = Topic}) -> + case emqx_access_control:check_acl(ClientInfo, publish, Topic) of + allow -> + _ = emqx_broker:publish(Msg), + ok; + deny -> + ?tp( + warning, + last_will_testament_publish_denied, + #{topic => Topic} + ), + ok + end. %%-------------------------------------------------------------------- %% Disconnect Reason From a7709838824bc5650de2e808d2d282476f282fa0 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 27 Sep 2022 08:36:25 +0200 Subject: [PATCH 50/61] fix(bin/emqx): allow space in root path Prior to this fix, space was already allowed in root path for 'start' 'console' etc. (the boot commands). However the non-boot commands such as 'ping' still had trouble. --- bin/emqx | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/bin/emqx b/bin/emqx index 306efe527..82eb90e16 100755 --- a/bin/emqx +++ b/bin/emqx @@ -481,6 +481,16 @@ case "$1" in ;; esac +if [ "$IS_BOOT_COMMAND" = 'no' ]; then + # for non-boot commands, inspect vm.