yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #13392 from thalesmg/20240702-m-sync-r57-mix-umbrella 

sync release-57 to master
This commit is contained in:
zhongwencool 2024-07-03 16:24:24 +08:00 committed by GitHub
parent 532f04da9d cfa29eaa6f
commit 7f17981a12
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
261 changed files with 6410 additions and 1052 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
apps/emqx/include/emqx.hrl
View File

@ -95,4 +95,10 @@
until :: integer()
}).
%%--------------------------------------------------------------------
%% Configurations
%%--------------------------------------------------------------------
-define(KIND_REPLICATE, replicate).
-define(KIND_INITIATE, initiate).
-endif.

37
apps/emqx/lib/mix/tasks/compile.copy_srcs.ex Normal file
View File

@ -0,0 +1,37 @@
defmodule Mix.Tasks.Compile.CopySrcs do
use Mix.Task.Compiler
@recursive true
@impl true
def run(_args) do
Mix.Project.get!()
config = Mix.Project.config()
extra_dirs = config[:extra_dirs]
unless extra_dirs && is_list(extra_dirs) do
Mix.raise("application option :extra_dirs in #{Mix.Project.project_file()} must be a list of directories under the application")
end
app_root = File.cwd!()
app_build_path = Mix.Project.app_path(config)
for extra_dir <- extra_dirs do
src = Path.join([app_root, extra_dir])
dest = Path.join([app_build_path, extra_dir])
File.rm(dest)
case File.ln_s(src, dest) do
:ok ->
:ok
{:error, :eexist} ->
Mix.shell().info(IO.ANSI.format([:yellow, "#{dest} still exists after attempted removal"]))
:ok
{:error, error} ->
Mix.raise("error trying to link #{src} to #{dest}: #{error}")
end
end
{:noop, []}
end
end

59
apps/emqx/mix.exs Normal file
View File

@ -0,0 +1,59 @@
defmodule EMQX.MixProject do
use Mix.Project
alias EMQXUmbrella.MixProject, as: UMP
def project do
[
app: :emqx,
version: "0.1.0",
build_path: "../../_build",
erlc_paths: UMP.erlc_paths(),
erlc_options: [
{:i, "src"}
| UMP.erlc_options()
],
compilers: Mix.compilers() ++ [:copy_srcs],
# used by our `Mix.Tasks.Compile.CopySrcs` compiler
extra_dirs: extra_dirs(),
deps_path: "../../deps",
lockfile: "../../mix.lock",
elixir: "~> 1.14",
start_permanent: Mix.env() == :prod,
deps: deps()
]
end
# Run "mix help compile.app" to learn about applications
def application do
[
## FIXME!!! go though emqx.app.src and add missing stuff...
extra_applications: [:public_key, :ssl, :os_mon, :logger, :mnesia] ++ UMP.extra_applications(),
mod: {:emqx_app, []}
]
end
def deps() do
## FIXME!!! go though emqx.app.src and add missing stuff...
[
{:emqx_utils, in_umbrella: true},
{:emqx_ds_backends, in_umbrella: true},
{:ekka, github: "emqx/ekka", tag: "0.19.3", override: true},
{:esockd, github: "emqx/esockd", tag: "5.11.2"},
{:gproc, github: "emqx/gproc", tag: "0.9.0.1", override: true},
{:hocon, github: "emqx/hocon", tag: "0.42.2", override: true},
{:lc, github: "emqx/lc", tag: "0.3.2", override: true},
{:ranch, github: "emqx/ranch", tag: "1.8.1-emqx", override: true},
] ++ UMP.quicer_dep()
end
defp extra_dirs() do
dirs = ["src", "etc"]
if UMP.test_env?() do
["test", "integration_test" | dirs]
else
dirs
end
end
end

13
apps/emqx/src/config/emqx_config_logger.erl
View File

@ -239,8 +239,9 @@ log_formatter(HandlerName, Conf) ->
end,
TsFormat = timestamp_format(Conf),
WithMfa = conf_get("with_mfa", Conf),
PayloadEncode = conf_get("payload_encode", Conf, text),
do_formatter(
Format, CharsLimit, SingleLine, TimeOffSet, Depth, TsFormat, WithMfa
Format, CharsLimit, SingleLine, TimeOffSet, Depth, TsFormat, WithMfa, PayloadEncode
).
%% auto | epoch | rfc3339
@ -248,16 +249,17 @@ timestamp_format(Conf) ->
conf_get("timestamp_format", Conf).
%% helpers
do_formatter(json, CharsLimit, SingleLine, TimeOffSet, Depth, TsFormat, WithMfa) ->
do_formatter(json, CharsLimit, SingleLine, TimeOffSet, Depth, TsFormat, WithMfa, PayloadEncode) ->
{emqx_logger_jsonfmt, #{
chars_limit => CharsLimit,
single_line => SingleLine,
time_offset => TimeOffSet,
depth => Depth,
timestamp_format => TsFormat,
with_mfa => WithMfa
with_mfa => WithMfa,
payload_encode => PayloadEncode
}};
do_formatter(text, CharsLimit, SingleLine, TimeOffSet, Depth, TsFormat, WithMfa) ->
do_formatter(text, CharsLimit, SingleLine, TimeOffSet, Depth, TsFormat, WithMfa, PayloadEncode) ->
{emqx_logger_textfmt, #{
template => ["[", level, "] ", msg, "\n"],
chars_limit => CharsLimit,
@ -265,7 +267,8 @@ do_formatter(text, CharsLimit, SingleLine, TimeOffSet, Depth, TsFormat, WithMfa)
time_offset => TimeOffSet,
depth => Depth,
timestamp_format => TsFormat,
with_mfa => WithMfa
with_mfa => WithMfa,
payload_encode => PayloadEncode
}}.
%% Don't record all logger message

47
apps/emqx/src/emqx.erl
View File

@ -61,9 +61,12 @@
get_raw_config/2,
update_config/2,
update_config/3,
update_config/4,
remove_config/1,
remove_config/2,
remove_config/3,
reset_config/2,
reset_config/3,
data_dir/0,
etc_file/1,
cert_file/1,
@ -195,7 +198,7 @@ get_raw_config(KeyPath, Default) ->
-spec update_config(emqx_utils_maps:config_key_path(), emqx_config:update_request()) ->
{ok, emqx_config:update_result()} | {error, emqx_config:update_error()}.
update_config(KeyPath, UpdateReq) ->
update_config(KeyPath, UpdateReq, #{}).
update_config(KeyPath, UpdateReq, #{}, #{}).
-spec update_config(
emqx_utils_maps:config_key_path(),
@ -203,30 +206,56 @@ update_config(KeyPath, UpdateReq) ->
emqx_config:update_opts()
) ->
{ok, emqx_config:update_result()} | {error, emqx_config:update_error()}.
update_config([RootName | _] = KeyPath, UpdateReq, Opts) ->
update_config(KeyPath, UpdateReq, Opts) ->
update_config(KeyPath, UpdateReq, Opts, #{}).
-spec update_config(
emqx_utils_maps:config_key_path(),
emqx_config:update_request(),
emqx_config:update_opts(),
emqx_config:cluster_rpc_opts()
) ->
{ok, emqx_config:update_result()} | {error, emqx_config:update_error()}.
update_config([RootName | _] = KeyPath, UpdateReq, Opts, ClusterRpcOpts) ->
emqx_config_handler:update_config(
emqx_config:get_schema_mod(RootName),
KeyPath,
{{update, UpdateReq}, Opts}
{{update, UpdateReq}, Opts},
ClusterRpcOpts
).
-spec remove_config(emqx_utils_maps:config_key_path()) ->
{ok, emqx_config:update_result()} | {error, emqx_config:update_error()}.
remove_config(KeyPath) ->
remove_config(KeyPath, #{}).
remove_config(KeyPath, #{}, #{}).
-spec remove_config(emqx_utils_maps:config_key_path(), emqx_config:update_opts()) ->
{ok, emqx_config:update_result()} | {error, emqx_config:update_error()}.
remove_config([RootName | _] = KeyPath, Opts) ->
remove_config([_RootName | _] = KeyPath, Opts) ->
remove_config(KeyPath, Opts, #{}).
-spec remove_config(
emqx_utils_maps:config_key_path(), emqx_config:update_opts(), emqx_config:cluster_rpc_opts()
) ->
{ok, emqx_config:update_result()} | {error, emqx_config:update_error()}.
remove_config([RootName | _] = KeyPath, Opts, ClusterRpcOpts) ->
emqx_config_handler:update_config(
emqx_config:get_schema_mod(RootName),
KeyPath,
{remove, Opts}
{remove, Opts},
ClusterRpcOpts
).
-spec reset_config(emqx_utils_maps:config_key_path(), emqx_config:update_opts()) ->
{ok, emqx_config:update_result()} | {error, emqx_config:update_error()}.
reset_config([RootName | SubKeys] = KeyPath, Opts) ->
reset_config([RootName | SubKeys] = KeyPath, Opts, #{}).
-spec reset_config(
emqx_utils_maps:config_key_path(), emqx_config:update_opts(), emqx_config:cluster_rpc_opts()
) ->
{ok, emqx_config:update_result()} | {error, emqx_config:update_error()}.
reset_config([RootName | SubKeys] = KeyPath, Opts, ClusterRpcOpts) ->
case emqx_config:get_default_value(KeyPath) of
{ok, Default} ->
Mod = emqx_config:get_schema_mod(RootName),
@ -235,7 +264,8 @@ reset_config([RootName | SubKeys] = KeyPath, Opts) ->
emqx_config_handler:update_config(
Mod,
KeyPath,
{{update, Default}, Opts}
{{update, Default}, Opts},
ClusterRpcOpts
);
false ->
NewConf =
@ -247,7 +277,8 @@ reset_config([RootName | SubKeys] = KeyPath, Opts) ->
emqx_config_handler:update_config(
Mod,
[RootName],
{{update, NewConf}, Opts}
{{update, NewConf}, Opts},
ClusterRpcOpts
)
end;
{error, _} = Error ->

20
apps/emqx/src/emqx_channel.erl
View File

@ -270,7 +270,7 @@ init(
},
Zone
),
{NClientInfo, NConnInfo} = take_ws_cookie(ClientInfo, ConnInfo),
{NClientInfo, NConnInfo} = take_conn_info_fields([ws_cookie, peersni], ClientInfo, ConnInfo),
#channel{
conninfo = NConnInfo,
clientinfo = NClientInfo,
@ -310,13 +310,19 @@ set_peercert_infos(Peercert, ClientInfo, Zone) ->
ClientId = PeercetAs(peer_cert_as_clientid),
ClientInfo#{username => Username, clientid => ClientId, dn => DN, cn => CN}.
take_ws_cookie(ClientInfo, ConnInfo) ->
case maps:take(ws_cookie, ConnInfo) of
{WsCookie, NConnInfo} ->
{ClientInfo#{ws_cookie => WsCookie}, NConnInfo};
take_conn_info_fields(Fields, ClientInfo, ConnInfo) ->
lists:foldl(
fun(Field, {ClientInfo0, ConnInfo0}) ->
case maps:take(Field, ConnInfo0) of
{Value, NConnInfo} ->
{ClientInfo0#{Field => Value}, NConnInfo};
_ ->
{ClientInfo, ConnInfo}
end.
{ClientInfo0, ConnInfo0}
end
end,
{ClientInfo, ConnInfo},
Fields
).
%%--------------------------------------------------------------------
%% Handle incoming packet

2
apps/emqx/src/emqx_config.erl
View File

@ -118,6 +118,7 @@
config/0,
app_envs/0,
update_opts/0,
cluster_rpc_opts/0,
update_cmd/0,
update_args/0,
update_error/0,
@ -147,6 +148,7 @@
raw_config => emqx_config:raw_config(),
post_config_update => #{module() => any()}
}.
-type cluster_rpc_opts() :: #{kind => ?KIND_INITIATE | ?KIND_REPLICATE}.
%% raw_config() is the config that is NOT parsed and translated by hocon schema
-type raw_config() :: #{binary() => term()} | list() | undefined.

200
apps/emqx/src/emqx_config_handler.erl
View File

@ -18,6 +18,7 @@
-module(emqx_config_handler).
-include("logger.hrl").
-include("emqx.hrl").
-include("emqx_schema.hrl").
-include_lib("hocon/include/hocon_types.hrl").
@ -30,6 +31,7 @@
add_handler/2,
remove_handler/1,
update_config/3,
update_config/4,
get_raw_cluster_override_conf/0,
info/0
]).
@ -53,9 +55,13 @@
-optional_callbacks([
pre_config_update/3,
pre_config_update/4,
propagated_pre_config_update/3,
propagated_pre_config_update/4,
post_config_update/5,
propagated_post_config_update/5
post_config_update/6,
propagated_post_config_update/5,
propagated_post_config_update/6
]).
-callback pre_config_update([atom()], emqx_config:update_request(), emqx_config:raw_config()) ->
@ -83,6 +89,38 @@
) ->
ok | {ok, Result :: any()} | {error, Reason :: term()}.
-callback pre_config_update(
[atom()], emqx_config:update_request(), emqx_config:raw_config(), emqx_config:cluster_rpc_opts()
) ->
ok | {ok, emqx_config:update_request()} | {error, term()}.
-callback propagated_pre_config_update(
[binary()],
emqx_config:update_request(),
emqx_config:raw_config(),
emqx_config:cluster_rpc_opts()
) ->
ok | {ok, emqx_config:update_request()} | {error, term()}.
-callback post_config_update(
[atom()],
emqx_config:update_request(),
emqx_config:config(),
emqx_config:config(),
emqx_config:app_envs(),
emqx_config:cluster_rpc_opts()
) ->
ok | {ok, Result :: any()} | {error, Reason :: term()}.
-callback propagated_post_config_update(
[atom()],
emqx_config:update_request(),
emqx_config:config(),
emqx_config:config(),
emqx_config:app_envs(),
emqx_config:cluster_rpc_opts()
) ->
ok | {ok, Result :: any()} | {error, Reason :: term()}.
-type state() :: #{handlers := any()}.
-type config_key_path() :: emqx_utils_maps:config_key_path().
@ -92,12 +130,17 @@ start_link() ->
stop() ->
gen_server:stop(?MODULE).
-spec update_config(module(), config_key_path(), emqx_config:update_args()) ->
{ok, emqx_config:update_result()} | {error, emqx_config:update_error()}.
update_config(SchemaModule, ConfKeyPath, UpdateArgs) ->
update_config(SchemaModule, ConfKeyPath, UpdateArgs, #{}).
-spec update_config(module(), config_key_path(), emqx_config:update_args(), map()) ->
{ok, emqx_config:update_result()} | {error, emqx_config:update_error()}.
update_config(SchemaModule, ConfKeyPath, UpdateArgs, ClusterOpts) ->
%% force convert the path to a list of atoms, as there maybe some wildcard names/ids in the path
AtomKeyPath = [atom(Key) || Key <- ConfKeyPath],
gen_server:call(?MODULE, {change_config, SchemaModule, AtomKeyPath, UpdateArgs}, infinity).
gen_server:call(
?MODULE, {change_config, SchemaModule, AtomKeyPath, UpdateArgs, ClusterOpts}, infinity
).
-spec add_handler(config_key_path(), handler_name()) ->
ok | {error, {conflict, list()}}.
@ -130,11 +173,11 @@ handle_call({add_handler, ConfKeyPath, HandlerName}, _From, State = #{handlers :
{error, _Reason} = Error -> {reply, Error, State}
end;
handle_call(
{change_config, SchemaModule, ConfKeyPath, UpdateArgs},
{change_config, SchemaModule, ConfKeyPath, UpdateArgs, ClusterRpcOpts},
_From,
#{handlers := Handlers} = State
) ->
Result = handle_update_request(SchemaModule, ConfKeyPath, Handlers, UpdateArgs),
Result = handle_update_request(SchemaModule, ConfKeyPath, Handlers, UpdateArgs, ClusterRpcOpts),
{reply, Result, State};
handle_call(get_raw_cluster_override_conf, _From, State) ->
Reply = emqx_config:read_override_conf(#{override_to => cluster}),
@ -203,9 +246,9 @@ filter_top_level_handlers(Handlers) ->
Handlers
).
handle_update_request(SchemaModule, ConfKeyPath, Handlers, UpdateArgs) ->
handle_update_request(SchemaModule, ConfKeyPath, Handlers, UpdateArgs, ClusterRpcOpts) ->
try
do_handle_update_request(SchemaModule, ConfKeyPath, Handlers, UpdateArgs)
do_handle_update_request(SchemaModule, ConfKeyPath, Handlers, UpdateArgs, ClusterRpcOpts)
catch
throw:Reason ->
{error, Reason};
@ -217,13 +260,14 @@ handle_update_request(SchemaModule, ConfKeyPath, Handlers, UpdateArgs) ->
update_req => UpdateArgs,
module => SchemaModule,
key_path => ConfKeyPath,
cluster_rpc_opts => ClusterRpcOpts,
stacktrace => ST
}),
{error, {config_update_crashed, Reason}}
end.
do_handle_update_request(SchemaModule, ConfKeyPath, Handlers, UpdateArgs) ->
case process_update_request(ConfKeyPath, Handlers, UpdateArgs) of
do_handle_update_request(SchemaModule, ConfKeyPath, Handlers, UpdateArgs, ClusterOpts) ->
case process_update_request(ConfKeyPath, Handlers, UpdateArgs, ClusterOpts) of
{ok, NewRawConf, OverrideConf, Opts} ->
check_and_save_configs(
SchemaModule,
@ -232,23 +276,24 @@ do_handle_update_request(SchemaModule, ConfKeyPath, Handlers, UpdateArgs) ->
NewRawConf,
OverrideConf,
UpdateArgs,
Opts
Opts,
ClusterOpts
);
{error, Result} ->
{error, Result}
end.
process_update_request([_], _Handlers, {remove, _Opts}) ->
process_update_request([_], _Handlers, {remove, _Opts}, _ClusterRpcOpts) ->
{error, "remove_root_is_forbidden"};
process_update_request(ConfKeyPath, _Handlers, {remove, Opts}) ->
process_update_request(ConfKeyPath, _Handlers, {remove, Opts}, _ClusterRpcOpts) ->
OldRawConf = emqx_config:get_root_raw(ConfKeyPath),
BinKeyPath = bin_path(ConfKeyPath),
NewRawConf = emqx_utils_maps:deep_remove(BinKeyPath, OldRawConf),
OverrideConf = remove_from_override_config(BinKeyPath, Opts),
{ok, NewRawConf, OverrideConf, Opts};
process_update_request(ConfKeyPath, Handlers, {{update, UpdateReq}, Opts}) ->
process_update_request(ConfKeyPath, Handlers, {{update, UpdateReq}, Opts}, ClusterRpcOpts) ->
OldRawConf = emqx_config:get_root_raw(ConfKeyPath),
case do_update_config(ConfKeyPath, Handlers, OldRawConf, UpdateReq) of
case do_update_config(ConfKeyPath, Handlers, OldRawConf, UpdateReq, ClusterRpcOpts) of
{ok, NewRawConf} ->
OverrideConf = merge_to_override_config(NewRawConf, Opts),
{ok, NewRawConf, OverrideConf, Opts};
@ -256,15 +301,16 @@ process_update_request(ConfKeyPath, Handlers, {{update, UpdateReq}, Opts}) ->
Error
end.
do_update_config(ConfKeyPath, Handlers, OldRawConf, UpdateReq) ->
do_update_config(ConfKeyPath, Handlers, OldRawConf, UpdateReq, []).
do_update_config(ConfKeyPath, Handlers, OldRawConf, UpdateReq, ClusterRpcOpts) ->
do_update_config(ConfKeyPath, Handlers, OldRawConf, UpdateReq, ClusterRpcOpts, []).
do_update_config([], Handlers, OldRawConf, UpdateReq, ConfKeyPath) ->
do_update_config([], Handlers, OldRawConf, UpdateReq, ClusterRpcOpts, ConfKeyPath) ->
call_pre_config_update(#{
handlers => Handlers,
old_raw_conf => OldRawConf,
update_req => UpdateReq,
conf_key_path => ConfKeyPath,
cluster_rpc_opts => ClusterRpcOpts,
callback => pre_config_update,
is_propagated => false
});
@ -273,13 +319,18 @@ do_update_config(
Handlers,
OldRawConf,
UpdateReq,
ClusterRpcOpts,
ConfKeyPath0
) ->
ConfKeyPath = ConfKeyPath0 ++ [ConfKey],
ConfKeyBin = bin(ConfKey),
SubOldRawConf = get_sub_config(ConfKeyBin, OldRawConf),
SubHandlers = get_sub_handlers(ConfKey, Handlers),
case do_update_config(SubConfKeyPath, SubHandlers, SubOldRawConf, UpdateReq, ConfKeyPath) of
case
do_update_config(
SubConfKeyPath, SubHandlers, SubOldRawConf, UpdateReq, ClusterRpcOpts, ConfKeyPath
)
of
{ok, NewUpdateReq} ->
merge_to_old_config(#{ConfKeyBin => NewUpdateReq}, OldRawConf);
Error ->
@ -293,12 +344,18 @@ check_and_save_configs(
NewRawConf,
OverrideConf,
UpdateArgs,
Opts
Opts,
ClusterOpts
) ->
Schema = schema(SchemaModule, ConfKeyPath),
Kind = maps:get(kind, ClusterOpts, ?KIND_INITIATE),
{AppEnvs, NewConf} = emqx_config:check_config(Schema, NewRawConf),
OldConf = emqx_config:get_root(ConfKeyPath),
case do_post_config_update(ConfKeyPath, Handlers, OldConf, NewConf, AppEnvs, UpdateArgs, #{}) of
case
do_post_config_update(
ConfKeyPath, Handlers, OldConf, NewConf, AppEnvs, UpdateArgs, ClusterOpts, #{}
)
of
{ok, Result0} ->
post_update_ok(
AppEnvs,
@ -310,9 +367,12 @@ check_and_save_configs(
UpdateArgs,
Result0
);
{error, {post_config_update, HandlerName, Reason}} ->
HandlePostFailureFun =
fun() ->
{error, {post_config_update, HandlerName, Reason}} when Kind =/= ?KIND_INITIATE ->
?SLOG(critical, #{
msg => "post_config_update_failed_but_save_the_config_anyway",
handler => HandlerName,
reason => Reason
}),
post_update_ok(
AppEnvs,
NewConf,
@ -322,9 +382,9 @@ check_and_save_configs(
ConfKeyPath,
UpdateArgs,
#{}
)
end,
{error, {post_config_update, HandlerName, {Reason, HandlePostFailureFun}}}
);
{error, _} = Error ->
Error
end.
post_update_ok(AppEnvs, NewConf, NewRawConf, OverrideConf, Opts, ConfKeyPath, UpdateArgs, Result0) ->
@ -332,7 +392,9 @@ post_update_ok(AppEnvs, NewConf, NewRawConf, OverrideConf, Opts, ConfKeyPath, Up
Result1 = return_change_result(ConfKeyPath, UpdateArgs),
{ok, Result1#{post_config_update => Result0}}.
do_post_config_update(ConfKeyPath, Handlers, OldConf, NewConf, AppEnvs, UpdateArgs, Result) ->
do_post_config_update(
ConfKeyPath, Handlers, OldConf, NewConf, AppEnvs, UpdateArgs, ClusterOpts, Result
) ->
do_post_config_update(
ConfKeyPath,
Handlers,
@ -340,6 +402,7 @@ do_post_config_update(ConfKeyPath, Handlers, OldConf, NewConf, AppEnvs, UpdateAr
NewConf,
AppEnvs,
UpdateArgs,
ClusterOpts,
Result,
[]
).
@ -352,6 +415,7 @@ do_post_config_update(
AppEnvs,
UpdateArgs,
Result,
ClusterOpts,
ConfKeyPath
) ->
call_post_config_update(#{
@ -362,6 +426,7 @@ do_post_config_update(
update_req => up_req(UpdateArgs),
result => Result,
conf_key_path => ConfKeyPath,
cluster_rpc_opts => ClusterOpts,
callback => post_config_update
});
do_post_config_update(
@ -371,6 +436,7 @@ do_post_config_update(
NewConf,
AppEnvs,
UpdateArgs,
ClusterOpts,
Result,
ConfKeyPath0
) ->
@ -385,6 +451,7 @@ do_post_config_update(
SubNewConf,
AppEnvs,
UpdateArgs,
ClusterOpts,
Result,
ConfKeyPath
).
@ -428,13 +495,11 @@ call_proper_pre_config_update(
#{
handlers := #{?MOD := Module},
callback := Callback,
update_req := UpdateReq,
old_raw_conf := OldRawConf
update_req := UpdateReq
} = Ctx
) ->
case erlang:function_exported(Module, Callback, 3) of
true ->
case apply_pre_config_update(Module, Ctx) of
Arity = get_function_arity(Module, Callback, [3, 4]),
case apply_pre_config_update(Module, Callback, Arity, Ctx) of
{ok, NewUpdateReq} ->
{ok, NewUpdateReq};
ok ->
@ -442,23 +507,29 @@ call_proper_pre_config_update(
{error, Reason} ->
{error, {pre_config_update, Module, Reason}}
end;
false ->
merge_to_old_config(UpdateReq, OldRawConf)
end;
call_proper_pre_config_update(
#{update_req := UpdateReq}
) ->
{ok, UpdateReq}.
apply_pre_config_update(Module, #{
apply_pre_config_update(Module, Callback, 3, #{
conf_key_path := ConfKeyPath,
update_req := UpdateReq,
old_raw_conf := OldRawConf
}) ->
Module:Callback(ConfKeyPath, UpdateReq, OldRawConf);
apply_pre_config_update(Module, Callback, 4, #{
conf_key_path := ConfKeyPath,
update_req := UpdateReq,
old_raw_conf := OldRawConf,
callback := Callback
cluster_rpc_opts := ClusterRpcOpts
}) ->
Module:Callback(
ConfKeyPath, UpdateReq, OldRawConf
).
Module:Callback(ConfKeyPath, UpdateReq, OldRawConf, ClusterRpcOpts);
apply_pre_config_update(_Module, _Callback, false, #{
update_req := UpdateReq,
old_raw_conf := OldRawConf
}) ->
merge_to_old_config(UpdateReq, OldRawConf).
propagate_pre_config_updates_to_subconf(
#{handlers := #{?WKEY := _}} = Ctx
@ -560,28 +631,23 @@ call_proper_post_config_update(
result := Result
} = Ctx
) ->
case erlang:function_exported(Module, Callback, 5) of
true ->
case apply_post_config_update(Module, Ctx) of
Arity = get_function_arity(Module, Callback, [5, 6]),
case apply_post_config_update(Module, Callback, Arity, Ctx) of
ok -> {ok, Result};
{ok, Result1} -> {ok, Result#{Module => Result1}};
{error, Reason} -> {error, {post_config_update, Module, Reason}}
end;
false ->
{ok, Result}
end;
call_proper_post_config_update(
#{result := Result} = _Ctx
) ->
{ok, Result}.
apply_post_config_update(Module, #{
apply_post_config_update(Module, Callback, 5, #{
conf_key_path := ConfKeyPath,
update_req := UpdateReq,
new_conf := NewConf,
old_conf := OldConf,
app_envs := AppEnvs,
callback := Callback
app_envs := AppEnvs
}) ->
Module:Callback(
ConfKeyPath,
@ -589,7 +655,25 @@ apply_post_config_update(Module, #{
NewConf,
OldConf,
AppEnvs
).
);
apply_post_config_update(Module, Callback, 6, #{
conf_key_path := ConfKeyPath,
update_req := UpdateReq,
cluster_rpc_opts := ClusterRpcOpts,
new_conf := NewConf,
old_conf := OldConf,
app_envs := AppEnvs
}) ->
Module:Callback(
ConfKeyPath,
UpdateReq,
NewConf,
OldConf,
AppEnvs,
ClusterRpcOpts
);
apply_post_config_update(_Module, _Callback, false, _Ctx) ->
ok.
propagate_post_config_updates_to_subconf(
#{handlers := #{?WKEY := _}} = Ctx
@ -768,7 +852,9 @@ assert_callback_function(Mod) ->
_ = apply(Mod, module_info, []),
case
erlang:function_exported(Mod, pre_config_update, 3) orelse
erlang:function_exported(Mod, post_config_update, 5)
erlang:function_exported(Mod, post_config_update, 5) orelse
erlang:function_exported(Mod, pre_config_update, 4) orelse
erlang:function_exported(Mod, post_config_update, 6)
of
true -> ok;
false -> error(#{msg => "bad_emqx_config_handler_callback", module => Mod})
@ -811,3 +897,13 @@ load_prev_handlers() ->
save_handlers(Handlers) ->
application:set_env(emqx, ?MODULE, Handlers).
get_function_arity(_Module, _Callback, []) ->
false;
get_function_arity(Module, Callback, [Arity | Opts]) ->
%% ensure module is loaded
Module = Module:module_info(module),
case erlang:function_exported(Module, Callback, Arity) of
true -> Arity;
false -> get_function_arity(Module, Callback, Opts)
end.

2
apps/emqx/src/emqx_connection.erl
View File

@ -305,11 +305,13 @@ init_state(
{ok, Peername} = Transport:ensure_ok_or_exit(peername, [Socket]),
{ok, Sockname} = Transport:ensure_ok_or_exit(sockname, [Socket]),
Peercert = Transport:ensure_ok_or_exit(peercert, [Socket]),
PeerSNI = Transport:ensure_ok_or_exit(peersni, [Socket]),
ConnInfo = #{
socktype => Transport:type(Socket),
peername => Peername,
sockname => Sockname,
peercert => Peercert,
peersni => PeerSNI,
conn_mod => ?MODULE
},

6
apps/emqx/src/emqx_logger_jsonfmt.erl
View File

@ -55,7 +55,8 @@
depth => pos_integer() | unlimited,
report_cb => logger:report_cb(),
single_line => boolean(),
chars_limit => unlimited | pos_integer()
chars_limit => unlimited | pos_integer(),
payload_encode => text | hidden | hex
}.
-define(IS_STRING(String), (is_list(String) orelse is_binary(String))).
@ -103,7 +104,8 @@ format(Msg, Meta, Config) ->
maybe_format_msg(undefined, _Meta, _Config) ->
#{};
maybe_format_msg({report, Report} = Msg, #{report_cb := Cb} = Meta, Config) ->
maybe_format_msg({report, Report0} = Msg, #{report_cb := Cb} = Meta, Config) ->
Report = emqx_logger_textfmt:try_encode_payload(Report0, Config),
case is_map(Report) andalso Cb =:= ?DEFAULT_FORMATTER of
true ->
%% reporting a map without a customised format function

20
apps/emqx/src/emqx_logger_textfmt.erl
View File

@ -20,11 +20,12 @@
-export([format/2]).
-export([check_config/1]).
-export([try_format_unicode/1]).
-export([try_format_unicode/1, try_encode_payload/2]).
%% Used in the other log formatters
-export([evaluate_lazy_values_if_dbg_level/1, evaluate_lazy_values/1]).
check_config(X) -> logger_formatter:check_config(maps:without([timestamp_format, with_mfa], X)).
check_config(X) ->
logger_formatter:check_config(maps:without([timestamp_format, with_mfa, payload_encode], X)).
%% Principle here is to delegate the formatting to logger_formatter:format/2
%% as much as possible, and only enrich the report with clientid, peername, topic, username
@ -107,9 +108,10 @@ is_list_report_acceptable(#{report_cb := Cb}) ->
is_list_report_acceptable(_) ->
false.
enrich_report(ReportRaw, Meta, Config) ->
enrich_report(ReportRaw0, Meta, Config) ->
%% clientid and peername always in emqx_conn's process metadata.
%% topic and username can be put in meta using ?SLOG/3, or put in msg's report by ?SLOG/2
ReportRaw = try_encode_payload(ReportRaw0, Config),
Topic =
case maps:get(topic, Meta, undefined) of
undefined -> maps:get(topic, ReportRaw, undefined);
@ -177,3 +179,15 @@ enrich_topic({Fmt, Args}, #{topic := Topic}) when is_list(Fmt) ->
{" topic: ~ts" ++ Fmt, [Topic | Args]};
enrich_topic(Msg, _) ->
Msg.
try_encode_payload(#{payload := Payload} = Report, #{payload_encode := Encode}) ->
Report#{payload := encode_payload(Payload, Encode)};
try_encode_payload(Report, _Config) ->
Report.
encode_payload(Payload, text) ->
Payload;
encode_payload(_Payload, hidden) ->
"******";
encode_payload(Payload, hex) ->
binary:encode_hex(Payload).

7
apps/emqx/src/emqx_quic_stream.erl
View File

@ -39,7 +39,8 @@
getopts/2,
peername/1,
sockname/1,
peercert/1
peercert/1,
peersni/1
]).
-include_lib("quicer/include/quicer.hrl").
-include_lib("emqx/include/emqx_quic.hrl").
@ -106,6 +107,10 @@ peercert(_S) ->
%% @todo but unsupported by msquic
nossl.
peersni(_S) ->
%% @todo
undefined.
getstat({quic, Conn, _Stream, _Info}, Stats) ->
case quicer:getstat(Conn, Stats) of
{error, _} -> {error, closed};

16
apps/emqx/src/emqx_ws_connection.erl
View File

@ -280,7 +280,7 @@ websocket_init([Req, Opts]) ->
#{zone := Zone, limiter := LimiterCfg, listener := {Type, Listener} = ListenerCfg} = Opts,
case check_max_connection(Type, Listener) of
allow ->
{Peername, PeerCert} = get_peer_info(Type, Listener, Req, Opts),
{Peername, PeerCert, PeerSNI} = get_peer_info(Type, Listener, Req, Opts),
Sockname = cowboy_req:sock(Req),
WsCookie = get_ws_cookie(Req),
ConnInfo = #{
@ -288,6 +288,7 @@ websocket_init([Req, Opts]) ->
peername => Peername,
sockname => Sockname,
peercert => PeerCert,
peersni => PeerSNI,
ws_cookie => WsCookie,
conn_mod => ?MODULE
},
@ -376,11 +377,12 @@ get_ws_cookie(Req) ->
end.
get_peer_info(Type, Listener, Req, Opts) ->
Host = maps:get(host, Req, undefined),
case
emqx_config:get_listener_conf(Type, Listener, [proxy_protocol]) andalso
maps:get(proxy_header, Req)
of
#{src_address := SrcAddr, src_port := SrcPort, ssl := SSL} ->
#{src_address := SrcAddr, src_port := SrcPort, ssl := SSL} = ProxyInfo ->
SourceName = {SrcAddr, SrcPort},
%% Notice: CN is only available in Proxy Protocol V2 additional info.
%% `CN` is unsupported in Proxy Protocol V1
@ -392,12 +394,14 @@ get_peer_info(Type, Listener, Req, Opts) ->
undefined -> undefined;
CN -> [{pp2_ssl_cn, CN}]
end,
{SourceName, SourceSSL};
#{src_address := SrcAddr, src_port := SrcPort} ->
PeerSNI = maps:get(authority, ProxyInfo, Host),
{SourceName, SourceSSL, PeerSNI};
#{src_address := SrcAddr, src_port := SrcPort} = ProxyInfo ->
PeerSNI = maps:get(authority, ProxyInfo, Host),
SourceName = {SrcAddr, SrcPort},
{SourceName, nossl};
{SourceName, nossl, PeerSNI};
_ ->
{get_peer(Req, Opts), cowboy_req:cert(Req)}
{get_peer(Req, Opts), cowboy_req:cert(Req), Host}
end.
websocket_handle({binary, Data}, State) when is_list(Data) ->

12
apps/emqx/test/emqx_authz_cache_SUITE.erl
View File

@ -37,7 +37,7 @@ end_per_suite(Config) ->
%%--------------------------------------------------------------------
t_cache_exclude(_) ->
ClientId = <<"test-id1">>,
ClientId = atom_to_binary(?FUNCTION_NAME),
{ok, Client} = emqtt:start_link([{clientid, ClientId}]),
{ok, _} = emqtt:connect(Client),
{ok, _, _} = emqtt:subscribe(Client, <<"nocache/+/#">>, 0),
@ -47,11 +47,12 @@ t_cache_exclude(_) ->
emqtt:stop(Client).
t_clean_authz_cache(_) ->
{ok, Client} = emqtt:start_link([{clientid, <<"emqx_c">>}]),
ClientId = atom_to_binary(?FUNCTION_NAME),
{ok, Client} = emqtt:start_link([{clientid, ClientId}]),
{ok, _} = emqtt:connect(Client),
{ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0),
emqtt:publish(Client, <<"t1">>, <<"{\"x\":1}">>, 0),
ClientPid = find_client_pid(<<"emqx_c">>),
ClientPid = find_client_pid(ClientId),
Caches = list_cache(ClientPid),
ct:log("authz caches: ~p", [Caches]),
?assert(length(Caches) > 0),
@ -60,11 +61,12 @@ t_clean_authz_cache(_) ->
emqtt:stop(Client).
t_drain_authz_cache(_) ->
{ok, Client} = emqtt:start_link([{clientid, <<"emqx_c">>}]),
ClientId = atom_to_binary(?FUNCTION_NAME),
{ok, Client} = emqtt:start_link([{clientid, ClientId}]),
{ok, _} = emqtt:connect(Client),
{ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0),
emqtt:publish(Client, <<"t1">>, <<"{\"x\":1}">>, 0),
ClientPid = find_client_pid(<<"emqx_c">>),
ClientPid = find_client_pid(ClientId),
Caches = list_cache(ClientPid),
ct:log("authz caches: ~p", [Caches]),
?assert(length(Caches) > 0),

2
apps/emqx/test/emqx_bpapi_SUITE.erl
View File

@ -21,7 +21,7 @@
-include_lib("common_test/include/ct.hrl").
-include_lib("stdlib/include/assert.hrl").
-include_lib("emqx/src/bpapi/emqx_bpapi.hrl").
-include("../src/bpapi/emqx_bpapi.hrl").
all() -> emqx_common_test_helpers:all(?MODULE).

1
apps/emqx/test/emqx_common_test_helpers.erl
View File

@ -248,6 +248,7 @@ render_and_load_app_config(App, Opts) ->
%% turn throw into error
error({Conf, E, St})
end.
do_render_app_config(App, Schema, ConfigFile, Opts) ->
%% copy acl_conf must run before read_schema_configs
copy_acl_conf(),

9
apps/emqx/test/emqx_config_handler_SUITE.erl
View File

@ -415,14 +415,7 @@ assert_update_result(FailedPath, Update, Expect) ->
assert_update_result(Paths, UpdatePath, Update, Expect) ->
with_update_result(Paths, UpdatePath, Update, fun(Old, Result) ->
case Expect of
{error, {post_config_update, ?MODULE, post_config_update_error}} ->
?assertMatch(
{error, {post_config_update, ?MODULE, {post_config_update_error, _}}}, Result
);
_ ->
?assertEqual(Expect, Result)
end,
?assertEqual(Expect, Result),
New = emqx:get_raw_config(UpdatePath, undefined),
?assertEqual(Old, New)
end).

3
apps/emqx/test/emqx_connection_SUITE.erl
View File

@ -84,7 +84,8 @@ init_per_testcase(TestCase, Config) when
fun
(peername, [sock]) -> {ok, {{127, 0, 0, 1}, 3456}};
(sockname, [sock]) -> {ok, {{127, 0, 0, 1}, 1883}};
(peercert, [sock]) -> undefined
(peercert, [sock]) -> undefined;
(peersni, [sock]) -> undefined
end
),
ok = meck:expect(emqx_transport, setopts, fun(_Sock, _Opts) -> ok end),

11
apps/emqx/test/emqx_cth_cluster.erl
View File

@ -391,7 +391,14 @@ node_init(#{name := Node, work_dir := WorkDir}) ->
_ = share_load_module(Node, cthr),
%% Enable snabbkaffe trace forwarding
ok = snabbkaffe:forward_trace(Node),
when_cover_enabled(fun() -> {ok, _} = cover:start([Node]) end),
when_cover_enabled(fun() ->
case cover:start([Node]) of
{ok, _} ->
ok;
{error, {already_started, _}} ->
ok
end
end),
ok.
%% Returns 'true' if this node should appear in running nodes list.
@ -456,7 +463,7 @@ stop(Nodes) ->
stop_node(Name) ->
Node = node_name(Name),
when_cover_enabled(fun() -> cover:flush([Node]) end),
when_cover_enabled(fun() -> ok = cover:flush([Node]) end),
ok = emqx_cth_peer:stop(Node).
%% Ports

145
apps/emqx/test/emqx_cth_listener.erl Normal file
View File

@ -0,0 +1,145 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_cth_listener).
-include_lib("esockd/include/esockd.hrl").
-export([
reload_listener_with_ppv2/1,
reload_listener_with_ppv2/2,
reload_listener_without_ppv2/1
]).
-export([meck_recv_ppv2/1, clear_meck_recv_ppv2/1]).
-define(DEFAULT_OPTS, #{
host => "127.0.0.1",
proto_ver => v5,
connect_timeout => 5,
ssl => false
}).
%%--------------------------------------------------------------------
%% APIs
%%--------------------------------------------------------------------
reload_listener_with_ppv2(Path = [listeners, _Type, _Name]) ->
reload_listener_with_ppv2(Path, <<>>).
reload_listener_with_ppv2(Path = [listeners, Type, Name], DefaultSni) when
Type == tcp; Type == ws
->
Cfg = emqx_config:get(Path),
ok = emqx_config:put(Path, Cfg#{proxy_protocol => true}),
ok = emqx_listeners:restart_listener(
emqx_listeners:listener_id(Type, Name)
),
ok = meck_recv_ppv2(Type),
client_conn_fn(Type, maps:get(bind, Cfg), DefaultSni).
client_conn_fn(tcp, Bind, Sni) ->
client_conn_fn_gen(connect, ?DEFAULT_OPTS#{port => bind2port(Bind), sni => Sni});
client_conn_fn(ws, Bind, Sni) ->
client_conn_fn_gen(ws_connect, ?DEFAULT_OPTS#{port => bind2port(Bind), sni => Sni}).
bind2port({_, Port}) -> Port;
bind2port(Port) when is_integer(Port) -> Port.
client_conn_fn_gen(Connect, Opts0) ->
fun(ClientId, Opts1) ->
Opts2 = maps:merge(Opts0, Opts1#{clientid => ClientId}),
Sni = maps:get(sni, Opts2, undefined),
NOpts = prepare_sni_for_meck(Sni, Opts2),
{ok, C} = emqtt:start_link(NOpts),
case emqtt:Connect(C) of
{ok, _} -> {ok, C};
{error, _} = Err -> Err
end
end.
prepare_sni_for_meck(ClientSni, Opts) when is_binary(ClientSni) ->
ServerSni =
case ClientSni of
disable -> undefined;
_ -> ClientSni
end,
persistent_term:put(current_client_sni, ServerSni),
case maps:get(ssl, Opts, false) of
false ->
Opts;
true ->
SslOpts = maps:get(ssl_opts, Opts, #{}),
Opts#{ssl_opts => [{server_name_indication, ClientSni} | SslOpts]}
end.
reload_listener_without_ppv2(Path = [listeners, Type, Name]) when
Type == tcp; Type == ws
->
Cfg = emqx_config:get(Path),
ok = emqx_config:put(Path, Cfg#{proxy_protocol => false}),
ok = emqx_listeners:restart_listener(
emqx_listeners:listener_id(Type, Name)
),
ok = clear_meck_recv_ppv2(Type).
meck_recv_ppv2(tcp) ->
ok = meck:new(esockd_proxy_protocol, [passthrough, no_history, no_link]),
ok = meck:expect(
esockd_proxy_protocol,
recv,
fun(_Transport, Socket, _Timeout) ->
SNI = persistent_term:get(current_client_sni, undefined),
{ok, {SrcAddr, SrcPort}} = esockd_transport:peername(Socket),
{ok, {DstAddr, DstPort}} = esockd_transport:sockname(Socket),
{ok, #proxy_socket{
inet = inet4,
socket = Socket,
src_addr = SrcAddr,
dst_addr = DstAddr,
src_port = SrcPort,
dst_port = DstPort,
pp2_additional_info = [{pp2_authority, SNI}]
}}
end
);
meck_recv_ppv2(ws) ->
ok = meck:new(ranch_tcp, [passthrough, no_history, no_link]),
ok = meck:expect(
ranch_tcp,
recv_proxy_header,
fun(Socket, _Timeout) ->
SNI = persistent_term:get(current_client_sni, undefined),
{ok, {SrcAddr, SrcPort}} = esockd_transport:peername(Socket),
{ok, {DstAddr, DstPort}} = esockd_transport:sockname(Socket),
{ok, #{
authority => SNI,
command => proxy,
dest_address => DstAddr,
dest_port => DstPort,
src_address => SrcAddr,
src_port => SrcPort,
transport_family => ipv4,
transport_protocol => stream,
version => 2
}}
end
).
clear_meck_recv_ppv2(tcp) ->
ok = meck:unload(esockd_proxy_protocol);
clear_meck_recv_ppv2(ws) ->
ok = meck:unload(ranch_tcp).

8
apps/emqx/test/emqx_cth_suite.erl
View File

@ -71,6 +71,7 @@
-export([start_app/3]).
-export([stop_apps/1]).
-export([default_config/2]).
-export([merge_appspec/2]).
-export([merge_config/2]).
@ -243,6 +244,7 @@ log_appspec(App, #{}) ->
spec_fmt(fc, config) -> "~n~ts";
spec_fmt(fc, _) -> "~p";
spec_fmt(ffun, {config, false}) -> "false (don't inhibit config loader)";
spec_fmt(ffun, {config, C}) -> render_config(C);
spec_fmt(ffun, {_, X}) -> X.
@ -349,6 +351,7 @@ default_appspec(emqx_conf, SuiteOpts) ->
data_dir => unicode:characters_to_binary(maps:get(work_dir, SuiteOpts, "data"))
}
},
SharedApps = maps:get(emqx_conf_shared_apps, SuiteOpts, [emqx, emqx_auth]),
% NOTE
% Since `emqx_conf_schema` manages config for a lot of applications, it's good to include
% their defaults as well.
@ -357,10 +360,7 @@ default_appspec(emqx_conf, SuiteOpts) ->
emqx_utils_maps:deep_merge(Acc, default_config(App, SuiteOpts))
end,
Config,
[
emqx,
emqx_auth
]
SharedApps
),
#{
config => SharedConfig,

2
apps/emqx/test/emqx_listeners_SUITE.erl
View File

@ -593,7 +593,7 @@ t_quic_update_opts_fail(Config) ->
%% THEN: Reload failed but old listener is rollbacked.
?assertMatch(
{error, {post_config_update, emqx_listeners, {{rollbacked, {error, tls_error}}, _}}},
{error, {post_config_update, emqx_listeners, {rollbacked, {error, tls_error}}}},
UpdateResult1
),

21
apps/emqx/test/emqx_logger_fmt_SUITE.erl
View File

@ -36,12 +36,18 @@ t_text_fmt_lazy_values(_) ->
t_text_fmt_lazy_values_only_in_debug_level_events(_) ->
check_fmt_lazy_values_only_in_debug_level_events(emqx_logger_textfmt).
t_text_payload(_) ->
check_fmt_payload(emqx_logger_textfmt).
t_json_fmt_lazy_values(_) ->
check_fmt_lazy_values(emqx_logger_jsonfmt).
t_json_fmt_lazy_values_only_in_debug_level_events(_) ->
check_fmt_lazy_values_only_in_debug_level_events(emqx_logger_jsonfmt).
t_json_payload(_) ->
check_fmt_payload(emqx_logger_jsonfmt).
check_fmt_lazy_values(FormatModule) ->
LogEntryIOData = FormatModule:format(event_with_lazy_value(), conf()),
LogEntryBin = unicode:characters_to_binary(LogEntryIOData),
@ -62,6 +68,18 @@ check_fmt_lazy_values_only_in_debug_level_events(FormatModule) ->
?assertNotEqual(nomatch, binary:match(LogEntryBin, [<<"emqx_trace_format_func_data">>])),
ok.
check_fmt_payload(FormatModule) ->
%% For performace reason we only search for lazy values to evaluate if log level is debug
WarningEvent = (event_with_lazy_value())#{level => info},
Conf = conf(),
LogEntryIOData = FormatModule:format(WarningEvent, Conf#{payload_encode => hidden}),
LogEntryBin = unicode:characters_to_binary(LogEntryIOData),
%% The input data for the formatting should exist
?assertEqual(nomatch, binary:match(LogEntryBin, [<<"content">>])),
%% The lazy value should not have been evaluated
?assertNotEqual(nomatch, binary:match(LogEntryBin, [<<"******">>])),
ok.
conf() ->
#{
time_offset => [],
@ -84,7 +102,8 @@ event_with_lazy_value() ->
{report, #{
reason =>
#emqx_trace_format_func_data{function = fun(Data) -> Data end, data = hej},
msg => hej
msg => hej,
payload => <<"content">>
}},
level => debug
}.

169
apps/emqx/test/emqx_peersni_SUITE.erl Normal file
View File

@ -0,0 +1,169 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_peersni_SUITE).
-compile(export_all).
-compile(nowarn_export_all).
-include_lib("emqx/include/emqx.hrl").
-include_lib("emqx/include/emqx_mqtt.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
-include_lib("esockd/include/esockd.hrl").
-define(SERVER_NAME, <<"localhost">>).
%%--------------------------------------------------------------------
%% setups
%%--------------------------------------------------------------------
all() ->
[
{group, tcp_ppv2},
{group, ws_ppv2},
{group, ssl},
{group, wss}
].
groups() ->
TCs = emqx_common_test_helpers:all(?MODULE),
[
{tcp_ppv2, [], TCs},
{ws_ppv2, [], TCs},
{ssl, [], TCs},
{wss, [], TCs}
].
init_per_suite(Config) ->
Apps = emqx_cth_suite:start(
[{emqx, #{}}],
#{work_dir => emqx_cth_suite:work_dir(Config)}
),
[{apps, Apps} | Config].
end_per_suite(Config) ->
emqx_cth_suite:stop(proplists:get_value(apps, Config)).
init_per_group(tcp_ppv2, Config) ->
ClientFn = emqx_cth_listener:reload_listener_with_ppv2(
[listeners, tcp, default],
?SERVER_NAME
),
[{client_fn, ClientFn} | Config];
init_per_group(ws_ppv2, Config) ->
ClientFn = emqx_cth_listener:reload_listener_with_ppv2(
[listeners, ws, default],
?SERVER_NAME
),
[{client_fn, ClientFn} | Config];
init_per_group(ssl, Config) ->
ClientFn = fun(ClientId, Opts) ->
Opts1 = Opts#{
host => ?SERVER_NAME,
port => 8883,
ssl => true,
ssl_opts => [
{verify, verify_none},
{server_name_indication, binary_to_list(?SERVER_NAME)}
]
},
{ok, C} = emqtt:start_link(Opts1#{clientid => ClientId}),
case emqtt:connect(C) of
{ok, _} -> {ok, C};
{error, _} = Err -> Err
end
end,
[{client_fn, ClientFn} | Config];
init_per_group(wss, Config) ->
ClientFn = fun(ClientId, Opts) ->
Opts1 = Opts#{
host => ?SERVER_NAME,
port => 8084,
ws_transport_options => [
{transport, tls},
{protocols, [http]},
{transport_opts, [
{verify, verify_none},
{server_name_indication, binary_to_list(?SERVER_NAME)},
{customize_hostname_check, []}
]}
]
},
{ok, C} = emqtt:start_link(Opts1#{clientid => ClientId}),
case emqtt:ws_connect(C) of
{ok, _} -> {ok, C};
{error, _} = Err -> Err
end
end,
[{client_fn, ClientFn} | Config];
init_per_group(_, Config) ->
Config.
end_per_group(tcp_ppv2, _Config) ->
emqx_cth_listener:reload_listener_without_ppv2([listeners, tcp, default]);
end_per_group(ws_ppv2, _Config) ->
emqx_cth_listener:reload_listener_without_ppv2([listeners, ws, default]);
end_per_group(_, _Config) ->
ok.
init_per_testcase(TestCase, Config) ->
case erlang:function_exported(?MODULE, TestCase, 2) of
true -> ?MODULE:TestCase(init, Config);
_ -> Config
end.
end_per_testcase(TestCase, Config) ->
case erlang:function_exported(?MODULE, TestCase, 2) of
true -> ?MODULE:TestCase('end', Config);
false -> ok
end,
Config.
%%--------------------------------------------------------------------
%% cases
%%--------------------------------------------------------------------
t_peersni_saved_into_conninfo(Config) ->
process_flag(trap_exit, true),
ClientId = <<"test-clientid1">>,
ClientFn = proplists:get_value(client_fn, Config),
{ok, Client} = ClientFn(ClientId, _Opts = #{}),
?assertMatch(#{clientinfo := #{peersni := ?SERVER_NAME}}, emqx_cm:get_chan_info(ClientId)),
ok = emqtt:disconnect(Client).
t_parse_peersni_to_client_attr(Config) ->
process_flag(trap_exit, true),
%% set the peersni to the client attribute
{ok, Variform} = emqx_variform:compile("nth(1, tokens(peersni, 'h'))"),
emqx_config:put([mqtt, client_attrs_init], [
#{expression => Variform, set_as_attr => mnts}
]),
ClientId = <<"test-clientid2">>,
ClientFn = proplists:get_value(client_fn, Config),
{ok, Client} = ClientFn(ClientId, _Opts = #{}),
?assertMatch(
#{clientinfo := #{client_attrs := #{mnts := <<"local">>}}}, emqx_cm:get_chan_info(ClientId)
),
ok = emqtt:disconnect(Client).

2
apps/emqx/test/emqx_persistent_session_ds_router_SUITE.erl
View File

@ -19,7 +19,7 @@
-compile(export_all).
-compile(nowarn_export_all).
-include_lib("emqx/src/emqx_persistent_session_ds/emqx_ps_ds_int.hrl").
-include("../src/emqx_persistent_session_ds/emqx_ps_ds_int.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").

4
apps/emqx/test/emqx_ws_connection_SUITE.erl
View File

@ -169,7 +169,7 @@ t_header(_) ->
set_ws_opts(proxy_address_header, <<"x-forwarded-for">>),
set_ws_opts(proxy_port_header, <<"x-forwarded-port">>),
{ok, St, _} = ?ws_conn:websocket_init([
req,
#{},
#{
zone => default,
limiter => limiter_cfg(),
@ -573,7 +573,7 @@ t_shutdown(_) ->
st() -> st(#{}).
st(InitFields) when is_map(InitFields) ->
{ok, St, _} = ?ws_conn:websocket_init([
req,
#{},
#{
zone => default,
listener => {ws, default},

27
apps/emqx_audit/mix.exs Normal file
View File

@ -0,0 +1,27 @@
defmodule EMQXAudit.MixProject do
use Mix.Project
alias EMQXUmbrella.MixProject, as: UMP
def project do
[
app: :emqx_audit,
version: "0.1.0",
build_path: "../../_build",
erlc_options: UMP.erlc_options(),
erlc_paths: UMP.erlc_paths(),
deps_path: "../../deps",
lockfile: "../../mix.lock",
elixir: "~> 1.14",
start_permanent: Mix.env() == :prod,
deps: deps()
]
end
def application do
[extra_applications: UMP.extra_applications(), mod: {:emqx_audit_app, []}]
end
def deps() do
[{:emqx, in_umbrella: true}, {:emqx_utils, in_umbrella: true}]
end
end

38
apps/emqx_audit/test/emqx_audit_api_SUITE.erl
View File

@ -18,6 +18,7 @@
-compile(nowarn_export_all).
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
all() ->
[
@ -54,18 +55,27 @@ common_tests() ->
}).
init_per_suite(Config) ->
_ = application:load(emqx_conf),
emqx_config:erase_all(),
emqx_mgmt_api_test_util:init_suite([emqx_ctl, emqx_conf, emqx_audit]),
ok = emqx_common_test_helpers:load_config(emqx_enterprise_schema, ?CONF_DEFAULT),
emqx_config:save_schema_mod_and_names(emqx_enterprise_schema),
ok = emqx_config_logger:refresh_config(),
application:set_env(emqx, boot_modules, []),
emqx_conf_cli:load(),
Config.
Apps = emqx_cth_suite:start(
[
emqx_ctl,
emqx,
{emqx_conf, #{
config => ?CONF_DEFAULT,
schema_mod => emqx_enterprise_schema
}},
emqx_modules,
emqx_audit,
emqx_management,
emqx_mgmt_api_test_util:emqx_dashboard()
],
#{work_dir => emqx_cth_suite:work_dir(Config)}
),
[{apps, Apps} | Config].
end_per_suite(_) ->
emqx_mgmt_api_test_util:end_suite([emqx_audit, emqx_conf, emqx_ctl]).
end_per_suite(Config) ->
Apps = ?config(apps, Config),
ok = emqx_cth_suite:stop(Apps),
ok.
t_http_api(_) ->
process_flag(trap_exit, true),
@ -164,6 +174,7 @@ t_cli(_Config) ->
],
Data
),
[ShowLogEntry] = Data,
%% check create at is valid
[#{<<"created_at">> := CreateAtRaw}] = Data,
CreateAt = calendar:rfc3339_to_system_time(binary_to_list(CreateAtRaw), [{unit, microsecond}]),
@ -172,7 +183,10 @@ t_cli(_Config) ->
%% check cli filter
{ok, Res1} = emqx_mgmt_api_test_util:request_api(get, AuditPath, "from=cli", AuthHeader),
#{<<"data">> := Data1} = emqx_utils_json:decode(Res1, [return_maps]),
?assertEqual(Data, Data1),
?assertMatch(
[ShowLogEntry, #{<<"operation_type">> := <<"emqx">>, <<"args">> := [<<"start">>]}],
Data1
),
{ok, Res2} = emqx_mgmt_api_test_util:request_api(
get, AuditPath, "from=erlang_console", AuthHeader
),

1
apps/emqx_auth/etc/auth-built-in-db-bootstrap.csv Normal file
View File

@ -0,0 +1 @@
user_id,password,is_superuser
1 user_id password is_superuser

44
apps/emqx_auth/mix.exs Normal file
View File

@ -0,0 +1,44 @@
defmodule EMQXAuth.MixProject do
use Mix.Project
alias EMQXUmbrella.MixProject, as: UMP
def project do
[
app: :emqx_auth,
version: "0.1.0",
build_path: "../../_build",
compilers: Mix.compilers() ++ [:copy_srcs],
# used by our `Mix.Tasks.Compile.CopySrcs` compiler
extra_dirs: extra_dirs(),
# config_path: "../../config/config.exs",
erlc_options: UMP.erlc_options(),
erlc_paths: UMP.erlc_paths(),
deps_path: "../../deps",
lockfile: "../../mix.lock",
elixir: "~> 1.14",
start_permanent: Mix.env() == :prod,
deps: deps()
]
end
# Run "mix help compile.app" to learn about applications
def application do
[extra_applications: UMP.extra_applications(), mod: {:emqx_auth_app, []}]
end
def deps() do
[
{:emqx, in_umbrella: true},
{:emqx_utils, in_umbrella: true}
]
end
defp extra_dirs() do
dirs = ["etc"]
if UMP.test_env?() do
["test" | dirs]
else
dirs
end
end
end

2
apps/emqx_auth/src/emqx_authn/emqx_authn_password_hashing.erl
View File

@ -142,7 +142,7 @@ salt_position(desc) -> "Salt position for PLAIN, MD5, SHA, SHA256 and SHA512 alg
salt_position(_) -> undefined.
dk_length(type) ->
integer();
pos_integer();
dk_length(required) ->
false;
dk_length(desc) ->

100
apps/emqx_auth/src/emqx_authz/emqx_authz.erl
View File

@ -51,7 +51,7 @@
set_feature_available/2
]).
-export([post_config_update/5, pre_config_update/3]).
-export([pre_config_update/4, post_config_update/5]).
-export([
maybe_read_source_files/1,
@ -194,8 +194,8 @@ update({?CMD_DELETE, Type}, Sources) ->
update(Cmd, Sources) ->
emqx_authz_utils:update_config(?CONF_KEY_PATH, {Cmd, Sources}).
pre_config_update(Path, Cmd, Sources) ->
try do_pre_config_update(Path, Cmd, Sources) of
pre_config_update(Path, Cmd, Sources, ClusterRpcOpts) ->
try do_pre_config_update(Path, Cmd, Sources, ClusterRpcOpts) of
{error, Reason} -> {error, Reason};
NSources -> {ok, NSources}
catch
@ -215,58 +215,64 @@ pre_config_update(Path, Cmd, Sources) ->
{error, Reason}
end.
do_pre_config_update(?CONF_KEY_PATH, Cmd, Sources) ->
do_pre_config_update(Cmd, Sources);
do_pre_config_update(?ROOT_KEY, {?CMD_MERGE, NewConf}, OldConf) ->
do_pre_config_merge(NewConf, OldConf);
do_pre_config_update(?ROOT_KEY, NewConf, OldConf) ->
do_pre_config_replace(NewConf, OldConf).
do_pre_config_update(?CONF_KEY_PATH, Cmd, Sources, Opts) ->
do_pre_config_update(Cmd, Sources, Opts);
do_pre_config_update(?ROOT_KEY, {?CMD_MERGE, NewConf}, OldConf, Opts) ->
do_pre_config_merge(NewConf, OldConf, Opts);
do_pre_config_update(?ROOT_KEY, NewConf, OldConf, Opts) ->
do_pre_config_replace(NewConf, OldConf, Opts).
do_pre_config_merge(NewConf, OldConf) ->
do_pre_config_merge(NewConf, OldConf, Opts) ->
MergeConf = emqx_utils_maps:deep_merge(OldConf, NewConf),
NewSources = merge_sources(OldConf, NewConf),
do_pre_config_replace(MergeConf#{<<"sources">> => NewSources}, OldConf).
do_pre_config_replace(MergeConf#{<<"sources">> => NewSources}, OldConf, Opts).
%% override the entire config when updating the root key
%% emqx_conf:update(?ROOT_KEY, Conf);
do_pre_config_replace(Conf, Conf) ->
do_pre_config_replace(Conf, Conf, _Opts) ->
Conf;
do_pre_config_replace(NewConf, OldConf) ->
do_pre_config_replace(NewConf, OldConf, Opts) ->
NewSources = get_sources(NewConf),
OldSources = get_sources(OldConf),
ReplaceSources = do_pre_config_update({?CMD_REPLACE, NewSources}, OldSources),
ReplaceSources = do_pre_config_update({?CMD_REPLACE, NewSources}, OldSources, Opts),
NewConf#{<<"sources">> => ReplaceSources}.
do_pre_config_update({?CMD_MOVE, _, _} = Cmd, Sources) ->
do_pre_config_update({?CMD_MOVE, _, _} = Cmd, Sources, _Opts) ->
do_move(Cmd, Sources);
do_pre_config_update({?CMD_PREPEND, Source}, Sources) ->
do_pre_config_update({?CMD_PREPEND, Source}, Sources, _Opts) ->
NSource = maybe_write_source_files(Source),
NSources = [NSource] ++ Sources,
ok = check_dup_types(NSources),
NSources;
do_pre_config_update({?CMD_APPEND, Source}, Sources) ->
do_pre_config_update({?CMD_APPEND, Source}, Sources, _Opts) ->
NSource = maybe_write_source_files(Source),
NSources = Sources ++ [NSource],
ok = check_dup_types(NSources),
NSources;
do_pre_config_update({{?CMD_REPLACE, Type}, Source}, Sources) ->
do_pre_config_update({{?CMD_REPLACE, Type}, Source}, Sources, _Opts) ->
NSource = maybe_write_source_files(Source),
{_Old, Front, Rear} = take(Type, Sources),
NSources = Front ++ [NSource | Rear],
ok = check_dup_types(NSources),
NSources;
do_pre_config_update({{?CMD_DELETE, Type}, _Source}, Sources) ->
{_Old, Front, Rear} = take(Type, Sources),
NSources = Front ++ Rear,
NSources;
do_pre_config_update({?CMD_REPLACE, Sources}, _OldSources) ->
do_pre_config_update({{?CMD_DELETE, Type}, _Source}, Sources, Opts) ->
case type_take(Type, Sources) of
not_found ->
case emqx_cluster_rpc:is_initiator(Opts) of
true -> throw({not_found_source, Type});
false -> Sources
end;
{_Found, NSources} ->
NSources
end;
do_pre_config_update({?CMD_REPLACE, Sources}, _OldSources, _Opts) ->
%% overwrite the entire config!
NSources = lists:map(fun maybe_write_source_files/1, Sources),
ok = check_dup_types(NSources),
NSources;
do_pre_config_update({?CMD_REORDER, NewSourcesOrder}, OldSources) ->
do_pre_config_update({?CMD_REORDER, NewSourcesOrder}, OldSources, _Opts) ->
reorder_sources(NewSourcesOrder, OldSources);
do_pre_config_update({Op, Source}, Sources) ->
do_pre_config_update({Op, Source}, Sources, _Opts) ->
throw({bad_request, #{op => Op, source => Source, sources => Sources}}).
post_config_update(_, _, undefined, _OldSource, _AppEnvs) ->
@ -293,9 +299,13 @@ do_post_config_update(?CONF_KEY_PATH, {{?CMD_REPLACE, Type}, RawNewSource}, Sour
Front ++ [InitedSources] ++ Rear;
do_post_config_update(?CONF_KEY_PATH, {{?CMD_DELETE, Type}, _RawNewSource}, _Sources) ->
OldInitedSources = lookup(),
{OldSource, Front, Rear} = take(Type, OldInitedSources),
ok = ensure_deleted(OldSource, #{clear_metric => true}),
Front ++ Rear;
case type_take(Type, OldInitedSources) of
not_found ->
OldInitedSources;
{Found, NSources} ->
ok = ensure_deleted(Found, #{clear_metric => true}),
NSources
end;
do_post_config_update(?CONF_KEY_PATH, {?CMD_REPLACE, _RawNewSources}, Sources) ->
overwrite_entire_sources(Sources);
do_post_config_update(?CONF_KEY_PATH, {?CMD_REORDER, NewSourcesOrder}, _Sources) ->
@ -317,18 +327,32 @@ do_post_config_update(?ROOT_KEY, _Conf, NewConf) ->
overwrite_entire_sources(Sources) ->
PrevSources = lookup(),
NewSourcesTypes = lists:map(fun type/1, Sources),
EnsureDelete = fun(S) ->
TypeName = type(S),
Opts =
case lists:member(TypeName, NewSourcesTypes) of
true -> #{clear_metric => false};
false -> #{clear_metric => true}
#{
removed := Removed,
added := Added,
identical := Identical,
changed := Changed
} = emqx_utils:diff_lists(Sources, PrevSources, fun type/1),
lists:foreach(
fun(S) -> ensure_deleted(S, #{clear_metric => true}) end,
Removed
),
AddedSources = create_sources(Added),
ChangedSources = lists:map(
fun({Old, New}) ->
update_source(type(New), Old, New)
end,
ensure_deleted(S, Opts)
Changed
),
New = Identical ++ AddedSources ++ ChangedSources,
lists:map(
fun(Type) ->
SearchFun = fun(S) -> type(S) =:= type(Type) end,
{value, Val} = lists:search(SearchFun, New),
Val
end,
lists:foreach(EnsureDelete, PrevSources),
create_sources(Sources).
Sources
).
%% @doc do source move
do_move({?CMD_MOVE, Type, ?CMD_MOVE_FRONT}, Sources) ->

2
apps/emqx_auth/src/emqx_authz/emqx_authz_app.erl
View File

@ -19,6 +19,8 @@
%% @end
%%%-------------------------------------------------------------------
%% TODO: delete this module
-module(emqx_authz_app).
-behaviour(application).

2
apps/emqx_auth/src/emqx_authz/emqx_authz_rule_raw.erl
View File

@ -122,7 +122,7 @@ validate_rule_topics(RuleRaw) ->
throw({missing_topic_or_topics, RuleRaw}).
validate_rule_topic(<<"eq ", TopicRaw/binary>>) ->
{eq, validate_rule_topic(TopicRaw)};
{eq, TopicRaw};
validate_rule_topic(TopicRaw) when is_binary(TopicRaw) -> TopicRaw.
validate_rule_permission(<<"allow">>) -> allow;

3
apps/emqx_auth/test/data/user-credentials-plain_v2.csv Normal file
View File

@ -0,0 +1,3 @@
user_id,password,is_superuser
myuser3,Password4,true
myuser4,Password3,false
1 user_id password is_superuser
2 myuser3 Password4 true
3 myuser4 Password3 false

42
apps/emqx_auth/test/emqx_authz/emqx_authz_api_cache_SUITE.erl
View File

@ -32,32 +32,30 @@ groups() ->
[].
init_per_suite(Config) ->
ok = emqx_mgmt_api_test_util:init_suite(
[emqx_conf, emqx_auth],
fun set_special_configs/1
),
Config.
end_per_suite(_Config) ->
{ok, _} = emqx:update_config(
[authorization],
Apps = emqx_cth_suite:start(
[
emqx_conf,
{emqx_auth, #{
config =>
#{
<<"no_match">> => <<"allow">>,
<<"cache">> => #{<<"enable">> => <<"true">>},
<<"sources">> => []
authorization =>
#{
cache => #{enabled => true},
no_match => deny,
sources => []
}
}
}},
emqx_management,
emqx_mgmt_api_test_util:emqx_dashboard()
],
#{work_dir => emqx_cth_suite:work_dir(Config)}
),
emqx_mgmt_api_test_util:end_suite([emqx_auth, emqx_conf]),
ok.
[{apps, Apps} | Config].
set_special_configs(emqx_dashboard) ->
emqx_dashboard_api_test_helpers:set_default_config();
set_special_configs(emqx_auth) ->
{ok, _} = emqx:update_config([authorization, cache, enable], true),
{ok, _} = emqx:update_config([authorization, no_match], deny),
{ok, _} = emqx:update_config([authorization, sources], []),
ok;
set_special_configs(_App) ->
end_per_suite(Config) ->
Apps = ?config(apps, Config),
emqx_cth_suite:stop(Apps),
ok.
t_clean_cache(_) ->

128
apps/emqx_auth/test/emqx_authz/emqx_authz_api_cluster_SUITE.erl Normal file
View File

@ -0,0 +1,128 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2020-2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_authz_api_cluster_SUITE).
-compile(nowarn_export_all).
-compile(export_all).
-import(emqx_mgmt_api_test_util, [uri/1]).
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
-include_lib("emqx/include/emqx_placeholder.hrl").
-define(SOURCE_HTTP, #{
<<"type">> => <<"http">>,
<<"enable">> => true,
<<"url">> => <<"https://fake.com:443/acl?username=", ?PH_USERNAME/binary>>,
<<"ssl">> => #{<<"enable">> => true},
<<"headers">> => #{},
<<"method">> => <<"get">>,
<<"request_timeout">> => <<"5s">>
}).
-define(ON(NODE, BODY), erpc:call(NODE, fun() -> BODY end)).
all() ->
emqx_common_test_helpers:all(?MODULE).
init_per_suite(Config) ->
WorkDir = ?config(priv_dir, Config),
Cluster = mk_cluster_spec(#{}),
Nodes = [NodePrimary | _] = emqx_cth_cluster:start(Cluster, #{work_dir => WorkDir}),
lists:foreach(fun(N) -> ?ON(N, emqx_authz_test_lib:register_fake_sources([http])) end, Nodes),
{ok, App} = ?ON(NodePrimary, emqx_common_test_http:create_default_app()),
[{api, App}, {cluster_nodes, Nodes}, {node, NodePrimary} | Config].
end_per_suite(Config) ->
ok = emqx_cth_cluster:stop(?config(cluster_nodes, Config)).
t_api(Config) ->
APINode = ?config(node, Config),
{ok, 200, Result1} = request(get, uri(["authorization", "sources"]), [], Config),
?assertEqual([], get_sources(Result1)),
{ok, 204, _} = request(post, uri(["authorization", "sources"]), ?SOURCE_HTTP, Config),
AllNodes = ?config(cluster_nodes, Config),
[OtherNode] = AllNodes -- [APINode],
lists:foreach(
fun(N) ->
?ON(
N,
?assertMatch(
[#{<<"type">> := <<"http">>}],
emqx:get_raw_config([authorization, sources])
)
)
end,
AllNodes
),
%% delete the source from the second node.
?ON(OtherNode, begin
{ok, _} = emqx:update_config([authorization], #{<<"sources">> => []}),
?assertMatch([], emqx:get_raw_config([authorization, sources]))
end),
?assertMatch(
{ok, 204, _},
request(
delete,
uri(["authorization", "sources", "http"]),
[],
Config
)
),
lists:foreach(
fun(N) ->
?ON(
N,
?assertMatch(
[],
emqx:get_raw_config([authorization, sources])
)
)
end,
AllNodes
),
?assertMatch(
{ok, 404, _},
request(
delete,
uri(["authorization", "sources", "http"]),
[],
Config
)
),
ok.
get_sources(Result) ->
maps:get(<<"sources">>, emqx_utils_json:decode(Result, [return_maps])).
mk_cluster_spec(Opts) ->
Apps = [
emqx,
{emqx_conf, "authorization {cache{enable=false},no_match=deny,sources=[]}"},
emqx_auth,
emqx_management
],
Node1Apps = Apps ++ [{emqx_dashboard, "dashboard.listeners.http {enable=true,bind=18083}"}],
Node2Apps = Apps,
[
{emqx_authz_api_cluster_SUITE1, Opts#{role => core, apps => Node1Apps}},
{emqx_authz_api_cluster_SUITE2, Opts#{role => core, apps => Node2Apps}}
].
request(Method, URL, Body, Config) ->
AuthHeader = emqx_common_test_http:auth_header(?config(api, Config)),
Opts = #{compatible_mode => true, httpc_req_opts => [{body_format, binary}]},
emqx_mgmt_api_test_util:request_api(Method, URL, [], AuthHeader, Body, Opts).

43
apps/emqx_auth/test/emqx_authz/emqx_authz_api_settings_SUITE.erl
View File

@ -30,33 +30,30 @@ groups() ->
[].
init_per_suite(Config) ->
ok = emqx_mgmt_api_test_util:init_suite(
[emqx_conf, emqx_auth, emqx_dashboard],
fun set_special_configs/1
),
Config.
end_per_suite(_Config) ->
{ok, _} = emqx:update_config(
[authorization],
Apps = emqx_cth_suite:start(
[
emqx_conf,
{emqx_auth, #{
config =>
#{
<<"no_match">> => <<"allow">>,
<<"cache">> => #{<<"enable">> => <<"true">>},
<<"sources">> => []
authorization =>
#{
cache => #{enabled => true},
no_match => allow,
sources => []
}
}
}},
emqx_management,
emqx_mgmt_api_test_util:emqx_dashboard()
],
#{work_dir => emqx_cth_suite:work_dir(Config)}
),
ok = stop_apps([emqx_resource]),
emqx_mgmt_api_test_util:end_suite([emqx_auth, emqx_conf]),
ok.
[{apps, Apps} | Config].
set_special_configs(emqx_dashboard) ->
emqx_dashboard_api_test_helpers:set_default_config();
set_special_configs(emqx_auth) ->
{ok, _} = emqx:update_config([authorization, cache, enable], false),
{ok, _} = emqx:update_config([authorization, no_match], deny),
{ok, _} = emqx:update_config([authorization, sources], []),
ok;
set_special_configs(_App) ->
end_per_suite(Config) ->
Apps = ?config(apps, Config),
emqx_cth_suite:stop(Apps),
ok.
%%------------------------------------------------------------------------------

42
apps/emqx_auth/test/emqx_authz/emqx_authz_rule_SUITE.erl
View File

@ -34,22 +34,30 @@ all() ->
emqx_common_test_helpers:all(?MODULE).
init_per_suite(Config) ->
ok = emqx_common_test_helpers:start_apps(
[emqx_conf, emqx_auth],
fun set_special_configs/1
),
Config.
end_per_suite(_Config) ->
{ok, _} = emqx:update_config(
[authorization],
Apps = emqx_cth_suite:start(
[
emqx_conf,
{emqx_auth, #{
config =>
#{
<<"no_match">> => <<"allow">>,
<<"cache">> => #{<<"enable">> => <<"true">>},
<<"sources">> => []
authorization =>
#{
cache => #{enabled => false},
no_match => deny,
sources => []
}
}
}},
emqx_management,
emqx_mgmt_api_test_util:emqx_dashboard()
],
#{work_dir => emqx_cth_suite:work_dir(Config)}
),
emqx_common_test_helpers:stop_apps([emqx_auth, emqx_conf]),
[{apps, Apps} | Config].
end_per_suite(Config) ->
Apps = ?config(apps, Config),
emqx_cth_suite:stop(Apps),
ok.
init_per_testcase(_TestCase, Config) ->
@ -58,14 +66,6 @@ end_per_testcase(_TestCase, _Config) ->
_ = emqx_authz:set_feature_available(rich_actions, true),
ok.
set_special_configs(emqx_auth) ->
{ok, _} = emqx:update_config([authorization, cache, enable], false),
{ok, _} = emqx:update_config([authorization, no_match], deny),
{ok, _} = emqx:update_config([authorization, sources], []),
ok;
set_special_configs(_App) ->
ok.
t_compile(_) ->
% NOTE
% Some of the following testcase are relying on the internal representation of

27
apps/emqx_auth_ext/mix.exs Normal file
View File

@ -0,0 +1,27 @@
defmodule EMQXAuthExt.MixProject do
use Mix.Project
alias EMQXUmbrella.MixProject, as: UMP
def project do
[
app: :emqx_auth_ext,
version: "0.1.0",
build_path: "../../_build",
erlc_options: UMP.erlc_options(),
erlc_paths: UMP.erlc_paths(),
deps_path: "../../deps",
lockfile: "../../mix.lock",
elixir: "~> 1.14",
start_permanent: Mix.env() == :prod,
deps: deps()
]
end
def application do
[extra_applications: UMP.extra_applications()]
end
def deps() do
[{:emqx, in_umbrella: true}]
end
end

35
apps/emqx_auth_http/mix.exs Normal file
View File

@ -0,0 +1,35 @@
defmodule EMQXAuthHTTP.MixProject do
use Mix.Project
alias EMQXUmbrella.MixProject, as: UMP
def project do
[
app: :emqx_auth_http,
version: "0.1.0",
build_path: "../../_build",
# config_path: "../../config/config.exs",
erlc_options: UMP.erlc_options(),
erlc_paths: UMP.erlc_paths(),
deps_path: "../../deps",
lockfile: "../../mix.lock",
elixir: "~> 1.14",
start_permanent: Mix.env() == :prod,
deps: deps()
]
end
# Run "mix help compile.app" to learn about applications
def application do
[extra_applications: UMP.extra_applications(), mod: {:emqx_auth_http_app, []}]
end
def deps() do
[
{:emqx, in_umbrella: true},
{:emqx_auth, in_umbrella: true},
{:emqx_resource, in_umbrella: true},
{:emqx_connector, in_umbrella: true},
{:hocon, github: "emqx/hocon", tag: "0.42.2", override: true}
]
end
end

34
apps/emqx_auth_jwt/mix.exs Normal file
View File

@ -0,0 +1,34 @@
defmodule EMQXAuthJWT.MixProject do
use Mix.Project
alias EMQXUmbrella.MixProject, as: UMP
def project do
[
app: :emqx_auth_jwt,
version: "0.1.0",
build_path: "../../_build",
# config_path: "../../config/config.exs",
erlc_options: UMP.erlc_options(),
erlc_paths: UMP.erlc_paths(),
deps_path: "../../deps",
lockfile: "../../mix.lock",
elixir: "~> 1.14",
start_permanent: Mix.env() == :prod,
deps: deps()
]
end
# Run "mix help compile.app" to learn about applications
def application do
[extra_applications: UMP.extra_applications(), mod: {:emqx_auth_jwt_app, []}]
end
def deps() do
[
{:emqx, in_umbrella: true},
{:emqx_auth, in_umbrella: true},
{:emqx_resource, in_umbrella: true},
{:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"}
]
end
end

33
apps/emqx_auth_ldap/mix.exs Normal file
View File

@ -0,0 +1,33 @@
defmodule EMQXAuthLDAP.MixProject do
use Mix.Project
alias EMQXUmbrella.MixProject, as: UMP
def project do
[
app: :emqx_auth_ldap,
version: "0.1.0",
build_path: "../../_build",
# config_path: "../../config/config.exs",
erlc_options: UMP.erlc_options(),
erlc_paths: UMP.erlc_paths(),
deps_path: "../../deps",
lockfile: "../../mix.lock",
elixir: "~> 1.14",
start_permanent: Mix.env() == :prod,
deps: deps()
]
end
# Run "mix help compile.app" to learn about applications
def application do
[extra_applications: [:eldap], mod: {:emqx_auth_ldap_app, []}]
end
def deps() do
[
{:emqx, in_umbrella: true},
{:emqx_auth, in_umbrella: true},
{:emqx_ldap, in_umbrella: true},
]
end
end

24
apps/emqx_auth_ldap/test/emqx_authz_ldap_SUITE.erl
View File

@ -33,25 +33,27 @@ groups() ->
emqx_authz_test_lib:table_groups(t_run_case, cases()).
init_per_suite(Config) ->
ok = stop_apps([emqx_resource]),
case emqx_common_test_helpers:is_tcp_server_available(?LDAP_HOST, ?LDAP_DEFAULT_PORT) of
true ->
ok = emqx_common_test_helpers:start_apps(
[emqx_conf, emqx_auth, emqx_auth_ldap],
fun set_special_configs/1
Apps = emqx_cth_suite:start(
[
emqx,
emqx_conf,
emqx_auth,
emqx_auth_ldap
],
#{work_dir => emqx_cth_suite:work_dir(Config)}
),
ok = start_apps([emqx_resource]),
ok = create_ldap_resource(),
Config;
[{apps, Apps} | Config];
false ->
{skip, no_ldap}
end.
end_per_suite(_Config) ->
ok = emqx_authz_test_lib:restore_authorizers(),
ok = emqx_resource:remove_local(?LDAP_RESOURCE),
ok = stop_apps([emqx_resource]),
ok = emqx_common_test_helpers:stop_apps([emqx_conf, emqx_auth, emqx_auth_ldap]).
end_per_suite(Config) ->
Apps = ?config(apps, Config),
emqx_cth_suite:stop(Apps),
ok.
init_per_group(Group, Config) ->
[{test_case, emqx_authz_test_lib:get_case(Group, cases())} | Config].

29
apps/emqx_auth_mnesia/mix.exs Normal file
View File

@ -0,0 +1,29 @@
defmodule EMQXAuthMnesia.MixProject do
use Mix.Project
alias EMQXUmbrella.MixProject, as: UMP
def project do
[
app: :emqx_auth_mnesia,
version: "0.1.0",
build_path: "../../_build",
# config_path: "../../config/config.exs",
erlc_options: UMP.erlc_options(),
erlc_paths: UMP.erlc_paths(),
deps_path: "../../deps",
lockfile: "../../mix.lock",
elixir: "~> 1.14",
start_permanent: Mix.env() == :prod,
deps: deps()
]
end
# Run "mix help compile.app" to learn about applications
def application do
[extra_applications: UMP.extra_applications(), mod: {:emqx_auth_mnesia_app, []}]
end
def deps() do
[{:emqx, in_umbrella: true}, {:emqx_auth, in_umbrella: true}]
end
end

104
apps/emqx_auth_mnesia/src/emqx_authn_mnesia.erl
View File

@ -171,67 +171,61 @@ do_destroy(UserGroup) ->
mnesia:select(?TAB, group_match_spec(UserGroup), write)
).
import_users({PasswordType, Filename, FileData}, State) ->
import_users(ImportSource, State) ->
import_users(ImportSource, State, #{override => true}).
import_users({PasswordType, Filename, FileData}, State, Opts) ->
Convertor = convertor(PasswordType, State),
try
{_NewUsersCnt, Users} = parse_import_users(Filename, FileData, Convertor),
case length(Users) > 0 andalso do_import_users(Users) of
false ->
error(empty_users);
try parse_import_users(Filename, FileData, Convertor) of
{_NewUsersCnt, Users} ->
case do_import_users(Users, Opts#{filename => Filename}) of
ok ->
ok;
%% Do not log empty user entries.
%% The default etc/auth-built-in-db.csv file contains an empty user entry.
{error, empty_users} ->
{error, empty_users};
{error, Reason} ->
_ = do_clean_imported_users(Users),
error(Reason)
end
catch
error:Reason1:Stk ->
?SLOG(
warning,
#{
msg => "import_users_failed",
reason => Reason1,
msg => "import_authn_users_failed",
reason => Reason,
type => PasswordType,
filename => Filename
}
),
{error, Reason}
end
catch
error:Reason:Stk ->
?SLOG(
warning,
#{
msg => "parse_authn_users_failed",
reason => Reason,
type => PasswordType,
filename => Filename,
stacktrace => Stk
}
),
{error, Reason1}
{error, Reason}
end.
do_import_users(Users) ->
do_import_users([], _Opts) ->
{error, empty_users};
do_import_users(Users, Opts) ->
trans(
fun() ->
lists:foreach(
fun(
#{
<<"user_group">> := UserGroup,
<<"user_id">> := UserID,
<<"password_hash">> := PasswordHash,
<<"salt">> := Salt,
<<"is_superuser">> := IsSuperuser
}
) ->
insert_user(UserGroup, UserID, PasswordHash, Salt, IsSuperuser)
fun(User) ->
insert_user(User, Opts)
end,
Users
)
end
).
do_clean_imported_users(Users) ->
lists:foreach(
fun(
#{
<<"user_group">> := UserGroup,
<<"user_id">> := UserID
}
) ->
mria:dirty_delete(?TAB, {UserGroup, UserID})
end,
Users
).
add_user(
UserInfo,
State
@ -338,7 +332,14 @@ run_fuzzy_filter(
%% Internal functions
%%------------------------------------------------------------------------------
insert_user(UserGroup, UserID, PasswordHash, Salt, IsSuperuser) ->
insert_user(User, Opts) ->
#{
<<"user_group">> := UserGroup,
<<"user_id">> := UserID,
<<"password_hash">> := PasswordHash,
<<"salt">> := Salt,
<<"is_superuser">> := IsSuperuser
} = User,
UserInfoRecord =
#user_info{user_id = DBUserID} =
user_info_record(UserGroup, UserID, PasswordHash, Salt, IsSuperuser),
@ -348,14 +349,22 @@ insert_user(UserGroup, UserID, PasswordHash, Salt, IsSuperuser) ->
[UserInfoRecord] ->
ok;
[_] ->
Msg =
case maps:get(override, Opts, false) of
true ->
insert_user(UserInfoRecord),
"override_an_exists_userid_into_authentication_database_ok";
false ->
"import_an_exists_userid_into_authentication_database_failed"
end,
?SLOG(warning, #{
msg => "bootstrap_authentication_overridden_in_the_built_in_database",
msg => Msg,
user_id => UserID,
group_id => UserGroup,
bootstrap_file => maps:get(filename, Opts),
suggestion =>
"If you have made changes in other way, remove the user_id from the bootstrap file."
}),
insert_user(UserInfoRecord)
"If you've altered it differently, delete the user_id from the bootstrap file."
})
end.
insert_user(#user_info{} = UserInfoRecord) ->
@ -505,7 +514,7 @@ reader_fn(Filename0, Data) ->
error(Reason)
end;
<<".csv">> ->
%% Example: data/user-credentials.csv
%% Example: etc/auth-built-in-db-bootstrap.csv
emqx_utils_stream:csv(Data);
<<>> ->
error(unknown_file_format);
@ -550,16 +559,15 @@ is_superuser(#{<<"is_superuser">> := true}) -> true;
is_superuser(_) -> false.
boostrap_user_from_file(Config, State) ->
case maps:get(boostrap_file, Config, <<>>) of
case maps:get(bootstrap_file, Config, <<>>) of
<<>> ->
ok;
FileName0 ->
#{boostrap_type := Type} = Config,
#{bootstrap_type := Type} = Config,
FileName = emqx_schema:naive_env_interpolation(FileName0),
case file:read_file(FileName) of
{ok, FileData} ->
%% if there is a key conflict, override with the key which from the bootstrap file
_ = import_users({Type, FileName, FileData}, State),
_ = import_users({Type, FileName, FileData}, State, #{override => false}),
ok;
{error, Reason} ->
?SLOG(warning, #{

7
apps/emqx_auth_mnesia/src/emqx_authn_mnesia_schema.erl
View File

@ -46,7 +46,7 @@ select_union_member(_Kind, _Value) ->
fields(builtin_db) ->
[
{password_hash_algorithm, fun emqx_authn_password_hashing:type_rw/1}
] ++ common_fields() ++ bootstrap_fields();
] ++ common_fields();
fields(builtin_db_api) ->
[
{password_hash_algorithm, fun emqx_authn_password_hashing:type_rw_api/1}
@ -68,7 +68,8 @@ common_fields() ->
{mechanism, emqx_authn_schema:mechanism(?AUTHN_MECHANISM_SIMPLE)},
{backend, emqx_authn_schema:backend(?AUTHN_BACKEND)},
{user_id_type, fun user_id_type/1}
] ++ emqx_authn_schema:common_fields().
] ++ bootstrap_fields() ++
emqx_authn_schema:common_fields().
bootstrap_fields() ->
[
@ -78,7 +79,7 @@ bootstrap_fields() ->
#{
desc => ?DESC(bootstrap_file),
required => false,
default => <<>>
default => <<"${EMQX_ETC_DIR}/auth-built-in-db-bootstrap.csv">>
}
)},
{bootstrap_type,

1
apps/emqx_auth_mnesia/test/emqx_authn_api_mnesia_SUITE.erl
View File

@ -51,6 +51,7 @@ end_per_testcase(_, Config) ->
init_per_suite(Config) ->
Apps = emqx_cth_suite:start(
[
emqx_ctl,
emqx,
emqx_conf,
emqx_auth,

23
apps/emqx_auth_mnesia/test/emqx_authn_mnesia_SUITE.erl
View File

@ -56,6 +56,7 @@ t_create(_) ->
Config1 = Config0#{password_hash_algorithm => #{name => sha256}},
{ok, _} = emqx_authn_mnesia:create(?AUTHN_ID, Config1),
ok.
t_bootstrap_file(_) ->
Config = config(),
%% hash to hash
@ -102,25 +103,39 @@ t_bootstrap_file(_) ->
],
test_bootstrap_file(HashConfig, plain, <<"user-credentials-plain.json">>)
),
Opts = #{clean => false},
Result = test_bootstrap_file(HashConfig, plain, <<"user-credentials-plain.csv">>, Opts),
?assertMatch(
[
{user_info, {_, <<"myuser3">>}, _, _, true},
{user_info, {_, <<"myuser4">>}, _, _, false}
],
test_bootstrap_file(HashConfig, plain, <<"user-credentials-plain.csv">>)
Result
),
%% Don't override the exist user id.
?assertMatch(
Result, test_bootstrap_file(HashConfig, plain, <<"user-credentials-plain_v2.csv">>)
),
ok.
test_bootstrap_file(Config0, Type, File) ->
test_bootstrap_file(Config0, Type, File, #{clean => true}).
test_bootstrap_file(Config0, Type, File, Opts) ->
{Type, Filename, _FileData} = sample_filename_and_data(Type, File),
Config2 = Config0#{
boostrap_file => Filename,
boostrap_type => Type
bootstrap_file => Filename,
bootstrap_type => Type
},
{ok, State0} = emqx_authn_mnesia:create(?AUTHN_ID, Config2),
Result = ets:tab2list(emqx_authn_mnesia),
case maps:get(clean, Opts) of
true ->
ok = emqx_authn_mnesia:destroy(State0),
?assertMatch([], ets:tab2list(emqx_authn_mnesia)),
?assertMatch([], ets:tab2list(emqx_authn_mnesia));
_ ->
ok
end,
Result.
t_update(_) ->

26
apps/emqx_auth_mnesia/test/emqx_authz_mnesia_SUITE.erl
View File

@ -18,7 +18,7 @@
-compile(nowarn_export_all).
-compile(export_all).
-include_lib("emqx_authz.hrl").
-include_lib("emqx_auth/include/emqx_authz.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
@ -256,6 +256,30 @@ t_destroy(_Config) ->
emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>)
).
t_conf_cli_load(_Config) ->
ClientInfo = emqx_authz_test_lib:base_client_info(),
ok = emqx_authz_mnesia:store_rules(
{username, <<"username">>},
[#{<<"permission">> => <<"allow">>, <<"action">> => <<"publish">>, <<"topic">> => <<"t">>}]
),
?assertEqual(
allow,
emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>)
),
PrevRules = ets:tab2list(emqx_acl),
Hocon = emqx_conf_cli:get_config("authorization"),
Bin = iolist_to_binary(hocon_pp:do(Hocon, #{})),
ok = emqx_conf_cli:load_config(Bin, #{mode => merge}),
%% ensure emqx_acl table not clear
?assertEqual(PrevRules, ets:tab2list(emqx_acl)),
%% still working
?assertEqual(
allow,
emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>)
).
%%------------------------------------------------------------------------------
%% Helpers
%%------------------------------------------------------------------------------

33
apps/emqx_auth_mongodb/mix.exs Normal file
View File

@ -0,0 +1,33 @@
defmodule EMQXAuthMongoDB.MixProject do
use Mix.Project
alias EMQXUmbrella.MixProject, as: UMP
def project do
[
app: :emqx_auth_mongodb,
version: "0.1.0",
build_path: "../../_build",
# config_path: "../../config/config.exs",
erlc_options: UMP.erlc_options(),
erlc_paths: UMP.erlc_paths(),
deps_path: "../../deps",
lockfile: "../../mix.lock",
elixir: "~> 1.14",
start_permanent: Mix.env() == :prod,
deps: deps()
]
end
# Run "mix help compile.app" to learn about applications
def application do
[extra_applications: UMP.extra_applications(), mod: {:emqx_auth_mongodb_app, []}]
end
def deps() do
[
{:emqx, in_umbrella: true},
{:emqx_auth, in_umbrella: true},
{:emqx_mongodb, in_umbrella: true},
]
end
end

2
apps/emqx_auth_mongodb/test/emqx_authn_mongodb_SUITE.erl
View File

@ -19,7 +19,7 @@
-compile(nowarn_export_all).
-compile(export_all).
-include_lib("emqx_connector/include/emqx_connector.hrl").
-include("../../emqx_connector/include/emqx_connector.hrl").
-include_lib("emqx_auth/include/emqx_authn.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").

2
apps/emqx_auth_mongodb/test/emqx_authn_mongodb_tls_SUITE.erl
View File

@ -19,7 +19,7 @@
-compile(nowarn_export_all).
-compile(export_all).
-include_lib("emqx_connector/include/emqx_connector.hrl").
-include("../../emqx_connector/include/emqx_connector.hrl").
-include_lib("emqx_auth/include/emqx_authn.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").

2
apps/emqx_auth_mongodb/test/emqx_authz_mongodb_SUITE.erl
View File

@ -19,7 +19,7 @@
-compile(export_all).
-include_lib("emqx_auth/include/emqx_authz.hrl").
-include_lib("emqx_connector/include/emqx_connector.hrl").
-include("../../emqx_connector/include/emqx_connector.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
-include_lib("emqx/include/emqx_placeholder.hrl").

33
apps/emqx_auth_mysql/mix.exs Normal file
View File

@ -0,0 +1,33 @@
defmodule EMQXAuthMySQL.MixProject do
use Mix.Project
alias EMQXUmbrella.MixProject, as: UMP
def project do
[
app: :emqx_auth_mysql,
version: "0.1.0",
build_path: "../../_build",
# config_path: "../../config/config.exs",
erlc_options: UMP.erlc_options(),
erlc_paths: UMP.erlc_paths(),
deps_path: "../../deps",
lockfile: "../../mix.lock",
elixir: "~> 1.14",
start_permanent: Mix.env() == :prod,
deps: deps()
]
end
# Run "mix help compile.app" to learn about applications
def application do
[extra_applications: UMP.extra_applications(), mod: {:emqx_auth_mysql_app, []}]
end
def deps() do
[
{:emqx, in_umbrella: true},
{:emqx_auth, in_umbrella: true},
{:emqx_mysql, in_umbrella: true}
]
end
end

2
apps/emqx_auth_mysql/test/emqx_authn_mysql_SUITE.erl
View File

@ -19,7 +19,7 @@
-compile(nowarn_export_all).
-compile(export_all).
-include_lib("emqx_connector/include/emqx_connector.hrl").
-include("../../emqx_connector/include/emqx_connector.hrl").
-include_lib("emqx_auth/include/emqx_authn.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").

2
apps/emqx_auth_mysql/test/emqx_authn_mysql_tls_SUITE.erl
View File

@ -19,7 +19,7 @@
-compile(nowarn_export_all).
-compile(export_all).
-include_lib("emqx_connector/include/emqx_connector.hrl").
-include("../../emqx_connector/include/emqx_connector.hrl").
-include_lib("emqx_auth/include/emqx_authn.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").

2
apps/emqx_auth_mysql/test/emqx_authz_mysql_SUITE.erl
View File

@ -18,7 +18,7 @@
-compile(nowarn_export_all).
-compile(export_all).
-include("emqx_connector.hrl").
-include("../../emqx_connector/include/emqx_connector.hrl").
-include_lib("emqx_auth/include/emqx_authz.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").

33
apps/emqx_auth_postgresql/mix.exs Normal file
View File

@ -0,0 +1,33 @@
defmodule EMQXAuthPostgreSQL.MixProject do
use Mix.Project
alias EMQXUmbrella.MixProject, as: UMP
def project do
[
app: :emqx_auth_postgresql,
version: "0.1.0",
build_path: "../../_build",
# config_path: "../../config/config.exs",
erlc_options: UMP.erlc_options(),
erlc_paths: UMP.erlc_paths(),
deps_path: "../../deps",
lockfile: "../../mix.lock",
elixir: "~> 1.14",
start_permanent: Mix.env() == :prod,
deps: deps()
]
end
# Run "mix help compile.app" to learn about applications
def application do
[extra_applications: UMP.extra_applications(), mod: {:emqx_auth_postgresql_app, []}]
end
def deps() do
[
{:emqx, in_umbrella: true},
{:emqx_auth, in_umbrella: true},
{:emqx_postgresql, in_umbrella: true},
]
end
end

4
apps/emqx_auth_postgresql/test/emqx_authn_postgresql_SUITE.erl
View File

@ -19,7 +19,7 @@
-compile(nowarn_export_all).
-compile(export_all).
-include_lib("emqx_postgresql/include/emqx_postgresql.hrl").
-include_lib("../../emqx_postgresql/include/emqx_postgresql.hrl").
-include_lib("emqx_auth/include/emqx_authn.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
@ -196,7 +196,7 @@ test_user_auth(#{
?GLOBAL
).
t_authenticate_disabled_prepared_statements(Config) ->
t_authenticate_disabled_prepared_statements(_Config) ->
ResConfig = maps:merge(pgsql_config(), #{disable_prepared_statements => true}),
{ok, _} = emqx_resource:recreate_local(?PGSQL_RESOURCE, emqx_postgresql, ResConfig),
on_exit(fun() ->

2
apps/emqx_auth_postgresql/test/emqx_authn_postgresql_tls_SUITE.erl
View File

@ -19,7 +19,7 @@
-compile(nowarn_export_all).
-compile(export_all).
-include_lib("emqx_postgresql/include/emqx_postgresql.hrl").
-include_lib("../../emqx_postgresql/include/emqx_postgresql.hrl").
-include_lib("emqx_auth/include/emqx_authn.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").

2
apps/emqx_auth_postgresql/test/emqx_authz_postgresql_SUITE.erl
View File

@ -18,7 +18,7 @@
-compile(nowarn_export_all).
-compile(export_all).
-include_lib("emqx_postgresql/include/emqx_postgresql.hrl").
-include_lib("../../emqx_postgresql/include/emqx_postgresql.hrl").
-include_lib("emqx_auth/include/emqx_authz.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").

33
apps/emqx_auth_redis/mix.exs Normal file
View File

@ -0,0 +1,33 @@
defmodule EMQXAuthRedis.MixProject do
use Mix.Project
alias EMQXUmbrella.MixProject, as: UMP
def project do
[
app: :emqx_auth_redis,
version: "0.1.0",
build_path: "../../_build",
# config_path: "../../config/config.exs",
erlc_options: UMP.erlc_options(),
erlc_paths: UMP.erlc_paths(),
deps_path: "../../deps",
lockfile: "../../mix.lock",
elixir: "~> 1.14",
start_permanent: Mix.env() == :prod,
deps: deps()
]
end
# Run "mix help compile.app" to learn about applications
def application do
[extra_applications: UMP.extra_applications(), mod: {:emqx_auth_redis_app, []}]
end
def deps() do
[
{:emqx, in_umbrella: true},
{:emqx_auth, in_umbrella: true},
{:emqx_redis, in_umbrella: true}
]
end
end

2
apps/emqx_auth_redis/test/emqx_authn_redis_SUITE.erl
View File

@ -19,7 +19,7 @@
-compile(nowarn_export_all).
-compile(export_all).
-include_lib("emqx_connector/include/emqx_connector.hrl").
-include("../../emqx_connector/include/emqx_connector.hrl").
-include_lib("emqx_auth/include/emqx_authn.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").

2
apps/emqx_auth_redis/test/emqx_authz_redis_SUITE.erl
View File

@ -19,7 +19,7 @@
-compile(nowarn_export_all).
-compile(export_all).
-include("emqx_connector.hrl").
-include("../../emqx_connector/include/emqx_connector.hrl").
-include_lib("emqx_auth/include/emqx_authz.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").

27
apps/emqx_auto_subscribe/mix.exs Normal file
View File

@ -0,0 +1,27 @@
defmodule EMQXAutoSubscribe.MixProject do
use Mix.Project
alias EMQXUmbrella.MixProject, as: UMP
def project do
[
app: :emqx_auto_subscribe,
version: "0.1.0",
build_path: "../../_build",
erlc_options: UMP.erlc_options(),
erlc_paths: UMP.erlc_paths(),
deps_path: "../../deps",
lockfile: "../../mix.lock",
elixir: "~> 1.14",
start_permanent: Mix.env() == :prod,
deps: deps()
]
end
def application do
[extra_applications: UMP.extra_applications(), mod: {:emqx_auto_subscribe_app, []}]
end
def deps() do
[{:emqx, in_umbrella: true}, {:emqx_utils, in_umbrella: true}]
end
end

38
apps/emqx_auto_subscribe/test/emqx_auto_subscribe_SUITE.erl
View File

@ -19,8 +19,7 @@
-compile(nowarn_export_all).
-include_lib("eunit/include/eunit.hrl").
-define(APP, emqx_auto_subscribe).
-include_lib("common_test/include/ct.hrl").
-define(TOPIC_C, <<"/c/${clientid}">>).
-define(TOPIC_U, <<"/u/${username}">>).
@ -44,8 +43,6 @@ all() ->
emqx_common_test_helpers:all(?MODULE).
init_per_suite(Config) ->
mria:start(),
application:stop(?APP),
meck:new(emqx_schema, [non_strict, passthrough, no_history, no_link]),
meck:expect(emqx_schema, fields, fun
("auto_subscribe") ->
@ -60,11 +57,7 @@ init_per_suite(Config) ->
meck:expect(emqx_resource, update, fun(_, _, _, _) -> {ok, meck_data} end),
meck:expect(emqx_resource, remove, fun(_) -> ok end),
application:load(emqx_dashboard),
application:load(?APP),
ok = emqx_common_test_helpers:load_config(
emqx_auto_subscribe_schema,
<<
ASCfg = <<
"auto_subscribe {\n"
" topics = [\n"
" {\n"
@ -91,12 +84,18 @@ init_per_suite(Config) ->
" }\n"
" ]\n"
" }"
>>
>>,
Apps = emqx_cth_suite:start(
[
emqx,
emqx_conf,
{emqx_auto_subscribe, ASCfg},
emqx_management,
emqx_mgmt_api_test_util:emqx_dashboard()
],
#{work_dir => emqx_cth_suite:work_dir(Config)}
),
emqx_mgmt_api_test_util:init_suite(
[emqx_conf, ?APP]
),
Config.
[{apps, Apps} | Config].
init_per_testcase(t_get_basic_usage_info, Config) ->
{ok, _} = emqx_auto_subscribe:update([]),
@ -119,13 +118,10 @@ topic_config(T) ->
nl => 0
}.
end_per_suite(_) ->
application:unload(emqx_management),
application:unload(emqx_conf),
application:unload(?APP),
meck:unload(emqx_resource),
meck:unload(emqx_schema),
emqx_mgmt_api_test_util:end_suite([emqx_conf, ?APP]).
end_per_suite(Config) ->
Apps = ?config(apps, Config),
emqx_cth_suite:stop(Apps),
ok.
t_auto_subscribe(_) ->
emqx_auto_subscribe:update([#{<<"topic">> => Topic} || Topic <- ?TOPICS]),

45
apps/emqx_bridge/mix.exs Normal file
View File

@ -0,0 +1,45 @@
defmodule EMQXBridge.MixProject do
use Mix.Project
alias EMQXUmbrella.MixProject, as: UMP
def project do
[
app: :emqx_bridge,
version: "0.1.0",
build_path: "../../_build",
compilers: Mix.compilers() ++ [:copy_srcs],
# used by our `Mix.Tasks.Compile.CopySrcs` compiler
extra_dirs: extra_dirs(),
# config_path: "../../config/config.exs",
erlc_options: UMP.erlc_options(),
erlc_paths: UMP.erlc_paths(),
deps_path: "../../deps",
lockfile: "../../mix.lock",
elixir: "~> 1.14",
start_permanent: Mix.env() == :prod,
deps: deps()
]
end
# Run "mix help compile.app" to learn about applications
def application do
[extra_applications: UMP.extra_applications(), mod: {:emqx_bridge_app, []}]
end
def deps() do
[
{:emqx, in_umbrella: true},
{:emqx_resource, in_umbrella: true},
{:emqx_connector, in_umbrella: true},
]
end
defp extra_dirs() do
dirs = []
if UMP.test_env?() do
["test" | dirs]
else
dirs
end
end
end

15
apps/emqx_bridge/src/emqx_bridge_api.erl
View File

@ -503,7 +503,18 @@ schema("/bridges_probe") ->
end.
'/bridges/:id'(get, #{bindings := #{id := Id}}) ->
?TRY_PARSE_ID(Id, lookup_from_all_nodes(BridgeType, BridgeName, 200));
?TRY_PARSE_ID(
Id,
begin
CompatErrorMsg = non_compat_bridge_msg(),
case lookup_from_all_nodes(BridgeType, BridgeName, 200) of
{400, #{code := 'BAD_REQUEST', message := CompatErrorMsg}} ->
?BRIDGE_NOT_FOUND(BridgeType, BridgeName);
Res ->
Res
end
end
);
'/bridges/:id'(put, #{bindings := #{id := Id}, body := Conf0}) ->
Conf1 = filter_out_request_body(Conf0),
?TRY_PARSE_ID(
@ -636,7 +647,7 @@ lookup_from_all_nodes(BridgeType, BridgeName, SuccCode) ->
{ok, [{error, not_found} | _]} ->
?BRIDGE_NOT_FOUND(BridgeType, BridgeName);
{ok, [{error, not_bridge_v1_compatible} | _]} ->
?NOT_FOUND(non_compat_bridge_msg());
?BAD_REQUEST(non_compat_bridge_msg());
{error, Reason} ->
?INTERNAL_ERROR(Reason)
end.

20
apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl
View File

@ -22,25 +22,22 @@ init_per_suite(Config, Apps) ->
[{start_apps, Apps} | Config].
end_per_suite(Config) ->
delete_all_bridges_and_connectors(),
emqx_mgmt_api_test_util:end_suite(),
ok = emqx_common_test_helpers:stop_apps([emqx_conf]),
ok = emqx_connector_test_helpers:stop_apps(lists:reverse(?config(start_apps, Config))),
_ = application:stop(emqx_connector),
Apps = ?config(apps, Config),
emqx_cth_suite:stop(Apps),
ok.
init_per_group(TestGroup, BridgeType, Config) ->
ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"),
ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")),
emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
application:load(emqx_bridge),
ok = emqx_common_test_helpers:start_apps([emqx_conf]),
ok = emqx_connector_test_helpers:start_apps(?config(start_apps, Config)),
{ok, _} = application:ensure_all_started(emqx_connector),
emqx_mgmt_api_test_util:init_suite(),
Apps = emqx_cth_suite:start(
?config(start_apps, Config),
#{work_dir => emqx_cth_suite:work_dir(Config)}
),
UniqueNum = integer_to_binary(erlang:unique_integer([positive])),
MQTTTopic = <<"mqtt/topic/abc", UniqueNum/binary>>,
[
{apps, Apps},
{proxy_host, ProxyHost},
{proxy_port, ProxyPort},
{mqtt_topic, MQTTTopic},
@ -50,10 +47,11 @@ init_per_group(TestGroup, BridgeType, Config) ->
].
end_per_group(Config) ->
Apps = ?config(apps, Config),
ProxyHost = ?config(proxy_host, Config),
ProxyPort = ?config(proxy_port, Config),
emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
% delete_all_bridges(),
emqx_cth_suite:stop(Apps),
ok.
init_per_testcase(TestCase, Config0, BridgeConfigCb) ->

38
apps/emqx_bridge_azure_event_hub/mix.exs Normal file
View File

@ -0,0 +1,38 @@
defmodule EMQXBridgeAzureEventHub.MixProject do
use Mix.Project
alias EMQXUmbrella.MixProject, as: UMP
def project do
[
app: :emqx_bridge_azure_event_hub,
version: "0.1.0",
build_path: "../../_build",
erlc_options: UMP.erlc_options(),
erlc_paths: UMP.erlc_paths(),
deps_path: "../../deps",
lockfile: "../../mix.lock",
elixir: "~> 1.14",
start_permanent: Mix.env() == :prod,
deps: deps()
]
end
def application do
[extra_applications: UMP.extra_applications()]
end
def deps() do
[
{:wolff, github: "kafka4beam/wolff", tag: "2.0.0"},
{:kafka_protocol, github: "kafka4beam/kafka_protocol", tag: "4.1.5", override: true},
{:brod_gssapi, github: "kafka4beam/brod_gssapi", tag: "v0.1.1"},
{:brod, github: "kafka4beam/brod", tag: "3.18.0"},
## TODO: remove `mix.exs` from `wolff` and remove this override
## TODO: remove `mix.exs` from `pulsar` and remove this override
{:snappyer, "1.2.9", override: true},
{:emqx_connector, in_umbrella: true, runtime: false},
{:emqx_resource, in_umbrella: true},
{:emqx_bridge, in_umbrella: true, runtime: false}
]
end
end

32
apps/emqx_bridge_cassandra/mix.exs Normal file
View File

@ -0,0 +1,32 @@
defmodule EMQXBridgeCassandra.MixProject do
use Mix.Project
alias EMQXUmbrella.MixProject, as: UMP
def project do
[
app: :emqx_bridge_cassandra,
version: "0.1.0",
build_path: "../../_build",
erlc_options: UMP.erlc_options(),
erlc_paths: UMP.erlc_paths(),
deps_path: "../../deps",
lockfile: "../../mix.lock",
elixir: "~> 1.14",
start_permanent: Mix.env() == :prod,
deps: deps()
]
end
def application do
[extra_applications: UMP.extra_applications()]
end
def deps() do
[
{:ecql, github: "emqx/ecql", tag: "v0.7.0"},
{:emqx_connector, in_umbrella: true, runtime: false},
{:emqx_resource, in_umbrella: true},
{:emqx_bridge, in_umbrella: true, runtime: false}
]
end
end

5
apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE.erl
View File

@ -148,8 +148,6 @@ init_per_suite(Config) ->
Config.
end_per_suite(_Config) ->
emqx_mgmt_api_test_util:end_suite(),
ok = emqx_common_test_helpers:stop_apps([emqx_bridge, emqx_conf]),
ok.
init_per_testcase(_Testcase, Config) ->
@ -191,11 +189,10 @@ common_init(Config0) ->
emqx_bridge,
emqx_rule_engine,
emqx_management,
{emqx_dashboard, "dashboard.listeners.http { enable = true, bind = 18083 }"}
emqx_mgmt_api_test_util:emqx_dashboard()
],
#{work_dir => emqx_cth_suite:work_dir(Config0)}
),
{ok, _Api} = emqx_common_test_http:create_default_app(),
% Connect to cassnadra directly and create the table
catch connect_and_drop_table(Config0),
connect_and_create_table(Config0),

29
apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl
View File

@ -9,7 +9,7 @@
-include_lib("common_test/include/ct.hrl").
-include("emqx_bridge_cassandra.hrl").
-include("emqx_connector/include/emqx_connector.hrl").
-include("../../emqx_connector/include/emqx_connector.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("emqx/include/emqx.hrl").
-include_lib("stdlib/include/assert.hrl").
@ -53,10 +53,24 @@ cassandra_servers(CassandraHost, CassandraPort) ->
).
init_per_suite(Config) ->
ok = emqx_common_test_helpers:start_apps([emqx_conf]),
ok = emqx_connector_test_helpers:start_apps([emqx_resource]),
{ok, _} = application:ensure_all_started(emqx_connector),
Config.
Apps = emqx_cth_suite:start(
[
emqx,
emqx_conf,
emqx_bridge_cassandra,
emqx_bridge,
emqx_rule_engine,
emqx_management,
emqx_mgmt_api_test_util:emqx_dashboard()
],
#{work_dir => emqx_cth_suite:work_dir(Config)}
),
[{apps, Apps} | Config].
end_per_suite(Config) ->
Apps = ?config(apps, Config),
emqx_cth_suite:stop(Apps),
ok.
init_per_group(Group, Config) ->
{CassandraHost, CassandraPort, AuthOpts} =
@ -98,11 +112,6 @@ init_per_group(Group, Config) ->
end_per_group(_Group, _Config) ->
ok.
end_per_suite(_Config) ->
ok = emqx_common_test_helpers:stop_apps([emqx_conf]),
ok = emqx_connector_test_helpers:stop_apps([emqx_resource]),
_ = application:stop(emqx_connector).
init_per_testcase(_, Config) ->
Config.

32
apps/emqx_bridge_clickhouse/mix.exs Normal file
View File

@ -0,0 +1,32 @@
defmodule EMQXBridgeClickhouse.MixProject do
use Mix.Project
alias EMQXUmbrella.MixProject, as: UMP
def project do
[
app: :emqx_bridge_clickhouse,
version: "0.1.0",
build_path: "../../_build",
erlc_options: UMP.erlc_options(),
erlc_paths: UMP.erlc_paths(),
deps_path: "../../deps",
lockfile: "../../mix.lock",
elixir: "~> 1.14",
start_permanent: Mix.env() == :prod,
deps: deps()
]
end
def application do
[extra_applications: UMP.extra_applications()]
end
def deps() do
[
{:clickhouse, github: "emqx/clickhouse-client-erl", tag: "0.3.1"},
{:emqx_connector, in_umbrella: true, runtime: false},
{:emqx_resource, in_umbrella: true},
{:emqx_bridge, in_umbrella: true, runtime: false}
]
end
end

26
apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_SUITE.erl
View File

@ -7,9 +7,9 @@
-compile(nowarn_export_all).
-compile(export_all).
-define(APP, emqx_bridge_clickhouse).
-define(CLICKHOUSE_HOST, "clickhouse").
-define(CLICKHOUSE_PORT, "8123").
-include_lib("common_test/include/ct.hrl").
-include_lib("emqx_connector/include/emqx_connector.hrl").
%% See comment in
@ -25,17 +25,26 @@ init_per_suite(Config) ->
Port = list_to_integer(clickhouse_port()),
case emqx_common_test_helpers:is_tcp_server_available(Host, Port) of
true ->
emqx_common_test_helpers:render_and_load_app_config(emqx_conf),
ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]),
ok = emqx_connector_test_helpers:start_apps([emqx_resource, ?APP]),
snabbkaffe:fix_ct_logging(),
Apps = emqx_cth_suite:start(
[
emqx,
emqx_conf,
emqx_bridge_clickhouse,
emqx_connector,
emqx_bridge,
emqx_rule_engine,
emqx_management,
emqx_mgmt_api_test_util:emqx_dashboard()
],
#{work_dir => emqx_cth_suite:work_dir(Config)}
),
%% Create the db table
Conn = start_clickhouse_connection(),
% erlang:monitor,sb
{ok, _, _} = clickhouse:query(Conn, sql_create_database(), #{}),
{ok, _, _} = clickhouse:query(Conn, sql_create_table(), []),
clickhouse:query(Conn, sql_find_key(42), []),
[{clickhouse_connection, Conn} | Config];
[{apps, Apps}, {clickhouse_connection, Conn} | Config];
false ->
case os:getenv("IS_CI") of
"yes" ->
@ -74,8 +83,9 @@ start_clickhouse_connection() ->
end_per_suite(Config) ->
ClickhouseConnection = proplists:get_value(clickhouse_connection, Config),
clickhouse:stop(ClickhouseConnection),
ok = emqx_connector_test_helpers:stop_apps([?APP, emqx_resource]),
ok = emqx_common_test_helpers:stop_apps([emqx_bridge, emqx_conf]).
Apps = ?config(apps, Config),
emqx_cth_suite:stop(Apps),
ok.
init_per_testcase(_, Config) ->
reset_table(Config),

26
apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_connector_SUITE.erl
View File

@ -7,7 +7,7 @@
-compile(nowarn_export_all).
-compile(export_all).
-include("emqx_connector.hrl").
-include("../../emqx_connector/include/emqx_connector.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("stdlib/include/assert.hrl").
-include_lib("common_test/include/ct.hrl").
@ -43,8 +43,19 @@ init_per_suite(Config) ->
Port = list_to_integer(emqx_bridge_clickhouse_SUITE:clickhouse_port()),
case emqx_common_test_helpers:is_tcp_server_available(Host, Port) of
true ->
ok = emqx_common_test_helpers:start_apps([emqx_conf]),
ok = emqx_connector_test_helpers:start_apps([emqx_resource, ?APP]),
Apps = emqx_cth_suite:start(
[
emqx,
emqx_conf,
emqx_bridge_clickhouse,
emqx_connector,
emqx_bridge,
emqx_rule_engine,
emqx_management,
emqx_mgmt_api_test_util:emqx_dashboard()
],
#{work_dir => emqx_cth_suite:work_dir(Config)}
),
%% Create the db table
{ok, Conn} =
clickhouse:start_link([
@ -55,7 +66,7 @@ init_per_suite(Config) ->
]),
{ok, _, _} = clickhouse:query(Conn, <<"CREATE DATABASE IF NOT EXISTS mqtt">>, #{}),
clickhouse:stop(Conn),
Config;
[{apps, Apps} | Config];
false ->
case os:getenv("IS_CI") of
"yes" ->
@ -65,9 +76,10 @@ init_per_suite(Config) ->
end
end.
end_per_suite(_Config) ->
ok = emqx_common_test_helpers:stop_apps([emqx_conf]),
ok = emqx_connector_test_helpers:stop_apps([?APP, emqx_resource]).
end_per_suite(Config) ->
Apps = ?config(apps, Config),
emqx_cth_suite:stop(Apps),
ok.
init_per_testcase(_, Config) ->
Config.

38
apps/emqx_bridge_confluent/mix.exs Normal file
View File

@ -0,0 +1,38 @@
defmodule EMQXBridgeConfluent.MixProject do
use Mix.Project
alias EMQXUmbrella.MixProject, as: UMP
def project do
[
app: :emqx_bridge_confluent,
version: "0.1.0",
build_path: "../../_build",
erlc_options: UMP.erlc_options(),
erlc_paths: UMP.erlc_paths(),
deps_path: "../../deps",
lockfile: "../../mix.lock",
elixir: "~> 1.14",
start_permanent: Mix.env() == :prod,
deps: deps()
]
end
def application do
[extra_applications: UMP.extra_applications()]
end
def deps() do
[
{:wolff, github: "kafka4beam/wolff", tag: "2.0.0"},
{:kafka_protocol, github: "kafka4beam/kafka_protocol", tag: "4.1.5", override: true},
{:brod_gssapi, github: "kafka4beam/brod_gssapi", tag: "v0.1.1"},
{:brod, github: "kafka4beam/brod", tag: "3.18.0"},
## TODO: remove `mix.exs` from `wolff` and remove this override
## TODO: remove `mix.exs` from `pulsar` and remove this override
{:snappyer, "1.2.9", override: true},
{:emqx_connector, in_umbrella: true, runtime: false},
{:emqx_resource, in_umbrella: true},
{:emqx_bridge, in_umbrella: true, runtime: false}
]
end
end

32
apps/emqx_bridge_dynamo/mix.exs Normal file
View File

@ -0,0 +1,32 @@
defmodule EMQXBridgeDynamo.MixProject do
use Mix.Project
alias EMQXUmbrella.MixProject, as: UMP
def project do
[
app: :emqx_bridge_dynamo,
version: "0.1.0",
build_path: "../../_build",
erlc_options: UMP.erlc_options(),
erlc_paths: UMP.erlc_paths(),
deps_path: "../../deps",
lockfile: "../../mix.lock",
elixir: "~> 1.14",
start_permanent: Mix.env() == :prod,
deps: deps()
]
end
def application do
[extra_applications: UMP.extra_applications()]
end
def deps() do
[
{:erlcloud, github: "emqx/erlcloud", tag: "3.7.0.3"},
{:emqx_connector, in_umbrella: true, runtime: false},
{:emqx_resource, in_umbrella: true},
{:emqx_bridge, in_umbrella: true, runtime: false}
]
end
end

23
apps/emqx_bridge_dynamo/test/emqx_bridge_dynamo_SUITE.erl
View File

@ -69,14 +69,18 @@ init_per_group(_Group, Config) ->
Config.
end_per_group(Group, Config) when Group =:= with_batch; Group =:= without_batch ->
Apps = ?config(apps, Config),
ProxyHost = ?config(proxy_host, Config),
ProxyPort = ?config(proxy_port, Config),
emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
emqx_cth_suite:stop(Apps),
ok;
end_per_group(Group, Config) when Group =:= flaky ->
Apps = ?config(apps, Config),
ProxyHost = ?config(proxy_host, Config),
ProxyPort = ?config(proxy_port, Config),
emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
emqx_cth_suite:stop(Apps),
timer:sleep(1000),
ok;
end_per_group(_Group, _Config) ->
@ -135,18 +139,23 @@ common_init(ConfigT) ->
ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"),
ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")),
emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
% Ensure enterprise bridge module is loaded
ok = emqx_common_test_helpers:start_apps([
emqx_conf, emqx_resource, emqx_bridge, emqx_rule_engine
]),
_ = application:ensure_all_started(erlcloud),
_ = emqx_bridge_enterprise:module_info(),
emqx_mgmt_api_test_util:init_suite(),
Apps = emqx_cth_suite:start(
[
emqx_conf,
emqx_bridge_dynamo,
emqx_bridge,
emqx_rule_engine,
emqx_management,
emqx_mgmt_api_test_util:emqx_dashboard()
],
#{work_dir => emqx_cth_suite:work_dir(Config0)}
),
% setup dynamo
setup_dynamo(Config0),
{Name, TDConf} = dynamo_config(BridgeType, Config0),
Config =
[
{apps, Apps},
{dynamo_config, TDConf},
{dynamo_bridge_type, BridgeType},
{dynamo_name, Name},

33
apps/emqx_bridge_es/mix.exs Normal file
View File

@ -0,0 +1,33 @@
defmodule EMQXBridgeEs.MixProject do
use Mix.Project
alias EMQXUmbrella.MixProject, as: UMP
def project do
[
app: :emqx_bridge_es,
version: "0.1.0",
build_path: "../../_build",
erlc_options: UMP.erlc_options(),
erlc_paths: UMP.erlc_paths(),
deps_path: "../../deps",
lockfile: "../../mix.lock",
elixir: "~> 1.14",
start_permanent: Mix.env() == :prod,
deps: deps()
]
end
def application do
[extra_applications: UMP.extra_applications()]
end
def deps() do
[
{:emqx, in_umbrella: true},
{:emqx_connector, in_umbrella: true, runtime: false},
{:emqx_resource, in_umbrella: true},
{:emqx_bridge, in_umbrella: true, runtime: false},
{:emqx_bridge_http, in_umbrella: true}
]
end
end

32
apps/emqx_bridge_gcp_pubsub/mix.exs Normal file
View File

@ -0,0 +1,32 @@
defmodule EMQXBridgeGcpPubsub.MixProject do
use Mix.Project
alias EMQXUmbrella.MixProject, as: UMP
def project do
[
app: :emqx_bridge_gcp_pubsub,
version: "0.1.0",
build_path: "../../_build",
erlc_options: UMP.erlc_options(),
erlc_paths: UMP.erlc_paths(),
deps_path: "../../deps",
lockfile: "../../mix.lock",
elixir: "~> 1.14",
start_permanent: Mix.env() == :prod,
deps: deps()
]
end
def application do
[extra_applications: UMP.extra_applications()]
end
def deps() do
[
{:emqx_connector, in_umbrella: true, runtime: false},
{:emqx_resource, in_umbrella: true},
{:emqx_bridge, in_umbrella: true, runtime: false},
{:emqx_bridge_http, in_umbrella: true}
]
end
end

22
apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_consumer_SUITE.erl
View File

@ -519,6 +519,7 @@ wait_acked(Opts) ->
%% no need to check return value; we check the property in
%% the check phase. this is just to give it a chance to do
%% so and avoid flakiness. should be fast.
ct:pal("waiting ~b ms until acked...", [Timeout]),
Res = snabbkaffe:block_until(
?match_n_events(N, #{?snk_kind := gcp_pubsub_consumer_worker_acknowledged}),
Timeout
@ -1265,11 +1266,12 @@ t_multiple_topic_mappings(Config) ->
%% 2+ pull workers do not duplicate delivered messages
t_multiple_pull_workers(Config) ->
ct:timetrap({seconds, 120}),
ct:timetrap({seconds, 121}),
BridgeName = ?config(consumer_name, Config),
TopicMapping = ?config(topic_mapping, Config),
ResourceId = resource_id(Config),
?check_trace(
#{timetrap => 120_000},
begin
NConsumers = 3,
start_and_subscribe_mqtt(Config),
@ -1277,7 +1279,7 @@ t_multiple_pull_workers(Config) ->
snabbkaffe:subscribe(
?match_event(#{?snk_kind := "gcp_pubsub_consumer_worker_subscription_ready"}),
NConsumers,
40_000
infinity
),
{ok, _} = create_bridge(
Config,
@ -1287,6 +1289,14 @@ t_multiple_pull_workers(Config) ->
<<"ack_deadline">> => <<"10m">>,
<<"ack_retry_interval">> => <<"1s">>,
<<"consumer_workers_per_topic">> => NConsumers
},
<<"resource_opts">> => #{
%% Used by worker when patching subscritpion; we increase it a bit
%% here because (at least) the gcp emulator tends to time out /
%% throttle (?) workers targeting the same subscription, making
%% the test flakier.
<<"request_ttl">> => <<"5s">>,
<<"resume_interval">> => <<"1s">>
}
}
),
@ -1294,6 +1304,14 @@ t_multiple_pull_workers(Config) ->
[#{pubsub_topic := Topic}] = TopicMapping,
Payload = emqx_guid:to_hexstr(emqx_guid:gen()),
Messages = [#{<<"data">> => Payload}],
?retry(
500,
20,
?assertMatch(
{ok, connected},
health_check(Config)
)
),
pubsub_publish(Config, Topic, Messages),
{ok, Published} = receive_published(),
?assertMatch(

32
apps/emqx_bridge_greptimedb/mix.exs Normal file
View File

@ -0,0 +1,32 @@
defmodule EMQXBridgeGreptimedb.MixProject do
use Mix.Project
alias EMQXUmbrella.MixProject, as: UMP
def project do
[
app: :emqx_bridge_greptimedb,
version: "0.1.0",
build_path: "../../_build",
erlc_options: UMP.erlc_options(),
erlc_paths: UMP.erlc_paths(),
deps_path: "../../deps",
lockfile: "../../mix.lock",
elixir: "~> 1.14",
start_permanent: Mix.env() == :prod,
deps: deps()
]
end
def application do
[extra_applications: UMP.extra_applications()]
end
def deps() do
[
{:emqx_connector, in_umbrella: true, runtime: false},
{:emqx_resource, in_umbrella: true},
{:emqx_bridge, in_umbrella: true, runtime: false},
{:greptimedb, github: "GreptimeTeam/greptimedb-ingester-erl", tag: "v0.1.8"}
]
end
end

22
apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_connector_SUITE.erl
View File

@ -25,11 +25,17 @@ init_per_suite(Config) ->
Servers = [{GreptimedbTCPHost, GreptimedbTCPPort}],
case emqx_common_test_helpers:is_all_tcp_servers_available(Servers) of
true ->
ok = emqx_common_test_helpers:start_apps([emqx_conf]),
ok = emqx_connector_test_helpers:start_apps([emqx_resource]),
{ok, _} = application:ensure_all_started(emqx_connector),
{ok, _} = application:ensure_all_started(greptimedb),
Apps = emqx_cth_suite:start(
[
emqx,
emqx_conf,
emqx_bridge_greptimedb,
emqx_bridge
],
#{work_dir => emqx_cth_suite:work_dir(Config)}
),
[
{apps, Apps},
{greptimedb_tcp_host, GreptimedbTCPHost},
{greptimedb_tcp_port, GreptimedbTCPPort}
| Config
@ -43,11 +49,9 @@ init_per_suite(Config) ->
end
end.
end_per_suite(_Config) ->
ok = emqx_common_test_helpers:stop_apps([emqx_conf]),
ok = emqx_connector_test_helpers:stop_apps([emqx_resource]),
_ = application:stop(emqx_connector),
_ = application:stop(greptimedb),
end_per_suite(Config) ->
Apps = ?config(apps, Config),
emqx_cth_suite:stop(Apps),
ok.
init_per_testcase(_, Config) ->

34
apps/emqx_bridge_hstreamdb/mix.exs Normal file
View File

@ -0,0 +1,34 @@
defmodule EMQXBridgeHstreamdb.MixProject do
use Mix.Project
alias EMQXUmbrella.MixProject, as: UMP
def project do
[
app: :emqx_bridge_hstreamdb,
version: "0.1.0",
build_path: "../../_build",
erlc_options: UMP.erlc_options(),
erlc_paths: UMP.erlc_paths(),
deps_path: "../../deps",
lockfile: "../../mix.lock",
elixir: "~> 1.14",
start_permanent: Mix.env() == :prod,
deps: deps()
]
end
def application do
[extra_applications: UMP.extra_applications()]
end
def deps() do
[
{:hstreamdb_erl,
github: "hstreamdb/hstreamdb_erl", tag: "0.5.18+v0.18.1+ezstd-v1.0.5-emqx1"},
{:emqx, in_umbrella: true},
{:emqx_utils, in_umbrella: true},
{:emqx_connector, in_umbrella: true, runtime: false},
{:emqx_resource, in_umbrella: true}
]
end
end

18
apps/emqx_bridge_hstreamdb/test/emqx_bridge_hstreamdb_SUITE.erl
View File

@ -100,10 +100,12 @@ init_per_group(_Group, Config) ->
Config.
end_per_group(Group, Config) when Group =:= with_batch; Group =:= without_batch ->
Apps = ?config(apps, Config),
connect_and_delete_stream(Config),
ProxyHost = ?config(proxy_host, Config),
ProxyPort = ?config(proxy_port, Config),
emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
emqx_cth_suite:stop(Apps),
ok;
end_per_group(_Group, _Config) ->
ok.
@ -408,11 +410,16 @@ common_init(ConfigT) ->
ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"),
ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")),
emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
% Ensure EE bridge module is loaded
ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_resource, emqx_bridge]),
_ = application:ensure_all_started(hstreamdb_erl),
_ = emqx_bridge_enterprise:module_info(),
emqx_mgmt_api_test_util:init_suite(),
Apps = emqx_cth_suite:start(
[
emqx_conf,
emqx_bridge_hstreamdb,
emqx_bridge,
emqx_management,
emqx_mgmt_api_test_util:emqx_dashboard()
],
#{work_dir => emqx_cth_suite:work_dir(Config0)}
),
% Connect to hstreamdb directly
% drop old stream and then create new one
connect_and_delete_stream(Config0),
@ -420,6 +427,7 @@ common_init(ConfigT) ->
{Name, HStreamDBConf} = hstreamdb_config(BridgeType, Config0),
Config =
[
{apps, Apps},
{hstreamdb_config, HStreamDBConf},
{hstreamdb_bridge_type, BridgeType},
{hstreamdb_name, Name},

33
apps/emqx_bridge_http/mix.exs Normal file
View File

@ -0,0 +1,33 @@
defmodule EMQXBridgeHTTP.MixProject do
use Mix.Project
alias EMQXUmbrella.MixProject, as: UMP
def project do
[
app: :emqx_bridge_http,
version: "0.1.0",
build_path: "../../_build",
# config_path: "../../config/config.exs",
erlc_options: UMP.erlc_options(),
erlc_paths: UMP.erlc_paths(),
deps_path: "../../deps",
lockfile: "../../mix.lock",
elixir: "~> 1.14",
start_permanent: Mix.env() == :prod,
deps: deps()
]
end
# Run "mix help compile.app" to learn about applications
def application do
[extra_applications: UMP.extra_applications()]
end
def deps() do
[
{:emqx, in_umbrella: true},
{:emqx_resource, in_umbrella: true},
{:ehttpc, github: "emqx/ehttpc", tag: "0.4.13"}
]
end
end

10
apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl
View File

@ -640,8 +640,14 @@ on_get_channel_status(
_ChannelId,
State
) ->
%% XXX: Reuse the connector status
on_get_status(InstId, State).
%% N.B.: `on_get_channel_status' expects a different return value than
%% `on_get_status'.
case on_get_status(InstId, State, fun default_health_checker/2) of
{Status, _State, Reason} ->
{Status, Reason};
Res ->
Res
end.
on_format_query_result({ok, Status, Headers, Body}) ->
#{

32
apps/emqx_bridge_influxdb/mix.exs Normal file
View File

@ -0,0 +1,32 @@
defmodule EMQXBridgeInfluxdb.MixProject do
use Mix.Project
alias EMQXUmbrella.MixProject, as: UMP
def project do
[
app: :emqx_bridge_influxdb,
version: "0.1.0",
build_path: "../../_build",
erlc_options: UMP.erlc_options(),
erlc_paths: UMP.erlc_paths(),
deps_path: "../../deps",
lockfile: "../../mix.lock",
elixir: "~> 1.14",
start_permanent: Mix.env() == :prod,
deps: deps()
]
end
def application do
[extra_applications: UMP.extra_applications()]
end
def deps() do
[
{:influxdb, github: "emqx/influxdb-client-erl", tag: "1.1.13"},
{:emqx_connector, in_umbrella: true, runtime: false},
{:emqx_resource, in_umbrella: true},
{:emqx_bridge, in_umbrella: true, runtime: false}
]
end
end

49
apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_SUITE.erl
View File

@ -53,12 +53,6 @@ init_per_suite(Config) ->
Config.
end_per_suite(_Config) ->
delete_all_bridges(),
emqx_mgmt_api_test_util:end_suite(),
ok = emqx_connector_test_helpers:stop_apps([
emqx_conf, emqx_bridge, emqx_resource, emqx_rule_engine
]),
_ = application:stop(emqx_connector),
ok.
init_per_group(InfluxDBType, Config0) when
@ -92,10 +86,18 @@ init_per_group(InfluxDBType, Config0) when
ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"),
ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")),
emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
ok = start_apps(),
{ok, _} = application:ensure_all_started(emqx_connector),
emqx_mgmt_api_test_util:init_suite(),
Config = [{use_tls, UseTLS} | Config0],
Apps = emqx_cth_suite:start(
[
emqx_conf,
emqx_bridge_influxdb,
emqx_bridge,
emqx_rule_engine,
emqx_management,
emqx_mgmt_api_test_util:emqx_dashboard()
],
#{work_dir => emqx_cth_suite:work_dir(Config0)}
),
Config = [{apps, Apps}, {use_tls, UseTLS} | Config0],
{Name, ConfigString, InfluxDBConfig} = influxdb_config(
apiv1, InfluxDBHost, InfluxDBPort, Config
),
@ -164,10 +166,18 @@ init_per_group(InfluxDBType, Config0) when
ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"),
ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")),
emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
ok = start_apps(),
{ok, _} = application:ensure_all_started(emqx_connector),
emqx_mgmt_api_test_util:init_suite(),
Config = [{use_tls, UseTLS} | Config0],
Apps = emqx_cth_suite:start(
[
emqx_conf,
emqx_bridge_influxdb,
emqx_bridge,
emqx_rule_engine,
emqx_management,
emqx_mgmt_api_test_util:emqx_dashboard()
],
#{work_dir => emqx_cth_suite:work_dir(Config0)}
),
Config = [{apps, Apps}, {use_tls, UseTLS} | Config0],
{Name, ConfigString, InfluxDBConfig} = influxdb_config(
apiv2, InfluxDBHost, InfluxDBPort, Config
),
@ -222,12 +232,13 @@ end_per_group(Group, Config) when
Group =:= apiv2_tcp;
Group =:= apiv2_tls
->
Apps = ?config(apps, Config),
ProxyHost = ?config(proxy_host, Config),
ProxyPort = ?config(proxy_port, Config),
EHttpcPoolName = ?config(ehttpc_pool_name, Config),
emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
ehttpc_sup:stop_pool(EHttpcPoolName),
delete_bridge(Config),
emqx_cth_suite:stop(Apps),
ok;
end_per_group(_Group, _Config) ->
ok.
@ -250,14 +261,6 @@ end_per_testcase(_Testcase, Config) ->
%% Helper fns
%%------------------------------------------------------------------------------
start_apps() ->
%% some configs in emqx_conf app are mandatory
%% we want to make sure they are loaded before
%% ekka start in emqx_common_test_helpers:start_apps/1
emqx_common_test_helpers:render_and_load_app_config(emqx_conf),
ok = emqx_common_test_helpers:start_apps([emqx_conf]),
ok = emqx_connector_test_helpers:start_apps([emqx_resource, emqx_bridge, emqx_rule_engine]).
example_write_syntax() ->
%% N.B.: this single space character is relevant
<<"${topic},clientid=${clientid}", " ", "payload=${payload},",

20
apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_connector_SUITE.erl
View File

@ -27,10 +27,16 @@ init_per_suite(Config) ->
Servers = [{InfluxDBTCPHost, InfluxDBTCPPort}, {InfluxDBTLSHost, InfluxDBTLSPort}],
case emqx_common_test_helpers:is_all_tcp_servers_available(Servers) of
true ->
ok = emqx_common_test_helpers:start_apps([emqx_conf]),
ok = emqx_connector_test_helpers:start_apps([emqx_resource]),
{ok, _} = application:ensure_all_started(emqx_connector),
Apps = emqx_cth_suite:start(
[
emqx_conf,
emqx_bridge_influxdb,
emqx_bridge
],
#{work_dir => emqx_cth_suite:work_dir(Config)}
),
[
{apps, Apps},
{influxdb_tcp_host, InfluxDBTCPHost},
{influxdb_tcp_port, InfluxDBTCPPort},
{influxdb_tls_host, InfluxDBTLSHost},
@ -46,10 +52,10 @@ init_per_suite(Config) ->
end
end.
end_per_suite(_Config) ->
ok = emqx_common_test_helpers:stop_apps([emqx_conf]),
ok = emqx_connector_test_helpers:stop_apps([emqx_resource]),
_ = application:stop(emqx_connector).
end_per_suite(Config) ->
Apps = ?config(apps, Config),
emqx_cth_suite:stop(Apps),
ok.
init_per_testcase(_, Config) ->
Config.

33
apps/emqx_bridge_iotdb/mix.exs Normal file
View File

@ -0,0 +1,33 @@
defmodule EMQXBridgeIotdb.MixProject do
use Mix.Project
alias EMQXUmbrella.MixProject, as: UMP
def project do
[
app: :emqx_bridge_iotdb,
version: "0.1.0",
build_path: "../../_build",
erlc_options: UMP.erlc_options(),
erlc_paths: UMP.erlc_paths(),
deps_path: "../../deps",
lockfile: "../../mix.lock",
elixir: "~> 1.14",
start_permanent: Mix.env() == :prod,
deps: deps()
]
end
def application do
[extra_applications: UMP.extra_applications()]
end
def deps() do
[
{:emqx, in_umbrella: true},
{:emqx_connector, in_umbrella: true, runtime: false},
{:emqx_resource, in_umbrella: true},
{:emqx_bridge, in_umbrella: true, runtime: false},
{:emqx_bridge_http, in_umbrella: true}
]
end
end

15
apps/emqx_bridge_iotdb/test/emqx_bridge_iotdb_impl_SUITE.erl
View File

@ -12,7 +12,6 @@
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-define(BRIDGE_TYPE_BIN, <<"iotdb">>).
-define(APPS, [emqx_bridge, emqx_resource, emqx_rule_engine, emqx_bridge_iotdb]).
%%------------------------------------------------------------------------------
%% CT boilerplate
@ -34,7 +33,19 @@ groups() ->
].
init_per_suite(Config) ->
emqx_bridge_v2_testlib:init_per_suite(Config, ?APPS).
emqx_bridge_v2_testlib:init_per_suite(
Config,
[
emqx,
emqx_conf,
emqx_bridge_iotdb,
emqx_connector,
emqx_bridge,
emqx_rule_engine,
emqx_management,
emqx_mgmt_api_test_util:emqx_dashboard()
]
).
end_per_suite(Config) ->
emqx_bridge_v2_testlib:end_per_suite(Config).

38
apps/emqx_bridge_kafka/mix.exs Normal file
View File

@ -0,0 +1,38 @@
defmodule EMQXBridgeKafka.MixProject do
use Mix.Project
alias EMQXUmbrella.MixProject, as: UMP
def project do
[
app: :emqx_bridge_kafka,
version: "0.1.0",
build_path: "../../_build",
erlc_options: UMP.erlc_options(),
erlc_paths: UMP.erlc_paths(),
deps_path: "../../deps",
lockfile: "../../mix.lock",
elixir: "~> 1.14",
start_permanent: Mix.env() == :prod,
deps: deps()
]
end
def application do
[extra_applications: UMP.extra_applications()]
end
def deps() do
[
{:wolff, github: "kafka4beam/wolff", tag: "2.0.0"},
{:kafka_protocol, github: "kafka4beam/kafka_protocol", tag: "4.1.5", override: true},
{:brod_gssapi, github: "kafka4beam/brod_gssapi", tag: "v0.1.1"},
{:brod, github: "kafka4beam/brod", tag: "3.18.0"},
## TODO: remove `mix.exs` from `wolff` and remove this override
## TODO: remove `mix.exs` from `pulsar` and remove this override
{:snappyer, "1.2.9", override: true},
{:emqx_connector, in_umbrella: true, runtime: false},
{:emqx_resource, in_umbrella: true},
{:emqx_bridge, in_umbrella: true, runtime: false}
]
end
end

2
apps/emqx_bridge_kafka/test/emqx_bridge_v2_kafka_producer_SUITE.erl
View File

@ -94,7 +94,7 @@ end_per_testcase(TestCase, Config) when
TestCase =:= t_ancient_v1_config_migration_without_local_topic
->
Cluster = ?config(cluster, Config),
emqx_cth_cluster:stop(Cluster),
ok = emqx_cth_cluster:stop(Cluster),
ok;
end_per_testcase(_TestCase, Config) ->
ProxyHost = ?config(proxy_host, Config),

32
apps/emqx_bridge_kinesis/mix.exs Normal file
View File

@ -0,0 +1,32 @@
defmodule EMQXBridgeKinesis.MixProject do
use Mix.Project
alias EMQXUmbrella.MixProject, as: UMP
def project do
[
app: :emqx_bridge_kinesis,
version: "0.1.0",
build_path: "../../_build",
erlc_options: UMP.erlc_options(),
erlc_paths: UMP.erlc_paths(),
deps_path: "../../deps",
lockfile: "../../mix.lock",
elixir: "~> 1.14",
start_permanent: Mix.env() == :prod,
deps: deps()
]
end
def application do
[extra_applications: UMP.extra_applications()]
end
def deps() do
[
{:erlcloud, github: "emqx/erlcloud", tag: "3.7.0.3"},
{:emqx_connector, in_umbrella: true, runtime: false},
{:emqx_resource, in_umbrella: true},
{:emqx_bridge, in_umbrella: true, runtime: false}
]
end
end

26
apps/emqx_bridge_kinesis/test/emqx_bridge_kinesis_impl_producer_SUITE.erl
View File

@ -42,11 +42,21 @@ init_per_suite(Config) ->
ProxyName = "kinesis",
SecretFile = filename:join(?config(priv_dir, Config), "secret"),
ok = file:write_file(SecretFile, <<?KINESIS_SECRET_KEY>>),
ok = emqx_common_test_helpers:start_apps([emqx_conf]),
ok = emqx_connector_test_helpers:start_apps([emqx_resource, emqx_bridge, emqx_rule_engine]),
{ok, _} = application:ensure_all_started(emqx_connector),
emqx_mgmt_api_test_util:init_suite(),
Apps = emqx_cth_suite:start(
[
emqx,
emqx_conf,
emqx_bridge_kinesis,
emqx_connector,
emqx_bridge,
emqx_rule_engine,
emqx_management,
emqx_mgmt_api_test_util:emqx_dashboard()
],
#{work_dir => emqx_cth_suite:work_dir(Config)}
),
[
{apps, Apps},
{proxy_host, ProxyHost},
{proxy_port, ProxyPort},
{kinesis_port, list_to_integer(os:getenv("KINESIS_PORT", integer_to_list(?KINESIS_PORT)))},
@ -55,11 +65,9 @@ init_per_suite(Config) ->
| Config
].
end_per_suite(_Config) ->
emqx_mgmt_api_test_util:end_suite(),
ok = emqx_common_test_helpers:stop_apps([emqx_conf]),
ok = emqx_connector_test_helpers:stop_apps([emqx_bridge, emqx_resource, emqx_rule_engine]),
_ = application:stop(emqx_connector),
end_per_suite(Config) ->
Apps = ?config(apps, Config),
emqx_cth_suite:stop(Apps),
ok.
init_per_group(with_batch, Config) ->

31
apps/emqx_bridge_matrix/mix.exs Normal file
View File

@ -0,0 +1,31 @@
defmodule EMQXBridgeMatrix.MixProject do
use Mix.Project
alias EMQXUmbrella.MixProject, as: UMP
def project do
[
app: :emqx_bridge_matrix,
version: "0.1.0",
build_path: "../../_build",
erlc_options: UMP.erlc_options(),
erlc_paths: UMP.erlc_paths(),
deps_path: "../../deps",
lockfile: "../../mix.lock",
elixir: "~> 1.14",
start_permanent: Mix.env() == :prod,
deps: deps()
]
end
def application do
[extra_applications: UMP.extra_applications()]
end
def deps() do
[
{:emqx_connector, in_umbrella: true, runtime: false},
{:emqx_resource, in_umbrella: true},
{:emqx_bridge, in_umbrella: true, runtime: false}
]
end
end

32
apps/emqx_bridge_mongodb/mix.exs Normal file
View File

@ -0,0 +1,32 @@
defmodule EMQXBridgeMongodb.MixProject do
use Mix.Project
alias EMQXUmbrella.MixProject, as: UMP
def project do
[
app: :emqx_bridge_mongodb,
version: "0.1.0",
build_path: "../../_build",
erlc_options: UMP.erlc_options(),
erlc_paths: UMP.erlc_paths(),
deps_path: "../../deps",
lockfile: "../../mix.lock",
elixir: "~> 1.14",
start_permanent: Mix.env() == :prod,
deps: deps()
]
end
def application do
[extra_applications: UMP.extra_applications()]
end
def deps() do
[
{:emqx_connector, in_umbrella: true, runtime: false},
{:emqx_resource, in_umbrella: true},
{:emqx_bridge, in_umbrella: true, runtime: false},
{:emqx_mongodb, in_umbrella: true}
]
end
end

Some files were not shown because too many files have changed in this diff Show More