Merge pull request #10320 from zmstone/0403-sync-release-50-back-to-master
0403 sync release 50 back to master
This commit is contained in:
Zaiming (Stone) Shi
GitHub
commit
a9bf633e03
|
|
@ -35,7 +35,7 @@
|
|||
-define(EMQX_RELEASE_CE, "5.0.21").
|
||||
|
||||
%% Enterprise edition
|
||||
-define(EMQX_RELEASE_EE, "5.0.2-rc.2").
|
||||
-define(EMQX_RELEASE_EE, "5.0.2-rc.4").
|
||||
|
||||
%% the HTTP API version
|
||||
-define(EMQX_API_VERSION, "5.0").
|
||||
|
|
|
|||
|
|
@ -270,9 +270,6 @@ fast_forward_to_commit(Node, ToTnxId) ->
|
|||
|
||||
%% @private
|
||||
init([Node, RetryMs]) ->
|
||||
%% Workaround for https://github.com/emqx/mria/issues/94:
|
||||
_ = mria_rlog:wait_for_shards([?CLUSTER_RPC_SHARD], 1000),
|
||||
_ = mria:wait_for_tables([?CLUSTER_MFA, ?CLUSTER_COMMIT]),
|
||||
{ok, _} = mnesia:subscribe({table, ?CLUSTER_MFA, simple}),
|
||||
State = #{node => Node, retry_interval => RetryMs},
|
||||
%% The init transaction ID is set in emqx_conf_app after
|
||||
|
|
@ -286,6 +283,9 @@ init([Node, RetryMs]) ->
|
|||
|
||||
%% @private
|
||||
handle_continue(?CATCH_UP, State) ->
|
||||
%% emqx app must be started before
|
||||
%% trying to catch up the rpc commit logs
|
||||
ok = wait_for_emqx_ready(),
|
||||
{noreply, State, catch_up(State)}.
|
||||
|
||||
handle_call(reset, _From, State) ->
|
||||
|
|
@ -572,3 +572,37 @@ maybe_init_tnx_id(_Node, TnxId) when TnxId < 0 -> ok;
|
|||
maybe_init_tnx_id(Node, TnxId) ->
|
||||
{atomic, _} = transaction(fun ?MODULE:commit/2, [Node, TnxId]),
|
||||
ok.
|
||||
|
||||
%% @priv Cannot proceed until emqx app is ready.
|
||||
%% Otherwise the committed transaction catch up may fail.
|
||||
wait_for_emqx_ready() ->
|
||||
%% wait 10 seconds for emqx to start
|
||||
ok = do_wait_for_emqx_ready(10).
|
||||
|
||||
%% Wait for emqx app to be ready,
|
||||
%% write a log message every 1 second
|
||||
do_wait_for_emqx_ready(0) ->
|
||||
timeout;
|
||||
do_wait_for_emqx_ready(N) ->
|
||||
%% check interval is 100ms
|
||||
%% makes the total wait time 1 second
|
||||
case do_wait_for_emqx_ready2(10) of
|
||||
ok ->
|
||||
ok;
|
||||
timeout ->
|
||||
?SLOG(warning, #{msg => "stil_waiting_for_emqx_app_to_be_ready"}),
|
||||
do_wait_for_emqx_ready(N - 1)
|
||||
end.
|
||||
|
||||
%% Wait for emqx app to be ready,
|
||||
%% check interval is 100ms
|
||||
do_wait_for_emqx_ready2(0) ->
|
||||
timeout;
|
||||
do_wait_for_emqx_ready2(N) ->
|
||||
case emqx:is_running() of
|
||||
true ->
|
||||
ok;
|
||||
false ->
|
||||
timer:sleep(100),
|
||||
do_wait_for_emqx_ready2(N - 1)
|
||||
end.
|
||||
|
|
|
|||
|
|
@ -13,7 +13,9 @@
|
|||
%% See the License for the specific language governing permissions and
|
||||
%% limitations under the License.
|
||||
%%--------------------------------------------------------------------
|
||||
-module(emqx_cluster_rpc_handler).
|
||||
|
||||
%% @doc This module is responsible for cleaning up the cluster RPC MFA.
|
||||
-module(emqx_cluster_rpc_cleaner).
|
||||
|
||||
-behaviour(gen_server).
|
||||
|
||||
|
|
@ -95,19 +95,22 @@ init_load() ->
|
|||
-endif.
|
||||
|
||||
init_conf() ->
|
||||
%% Workaround for https://github.com/emqx/mria/issues/94:
|
||||
_ = mria_rlog:wait_for_shards([?CLUSTER_RPC_SHARD], 1000),
|
||||
_ = mria:wait_for_tables([?CLUSTER_MFA, ?CLUSTER_COMMIT]),
|
||||
{ok, TnxId} = copy_override_conf_from_core_node(),
|
||||
_ = emqx_app:set_init_tnx_id(TnxId),
|
||||
ok = init_load(),
|
||||
ok = emqx_app:set_init_config_load_done().
|
||||
|
||||
cluster_nodes() ->
|
||||
maps:get(running_nodes, ekka_cluster:info()) -- [node()].
|
||||
mria:cluster_nodes(cores) -- [node()].
|
||||
|
||||
copy_override_conf_from_core_node() ->
|
||||
case cluster_nodes() of
|
||||
%% The first core nodes is self.
|
||||
[] ->
|
||||
?SLOG(debug, #{msg => "skip_copy_overide_conf_from_core_node"}),
|
||||
?SLOG(debug, #{msg => "skip_copy_override_conf_from_core_node"}),
|
||||
{ok, ?DEFAULT_INIT_TXN_ID};
|
||||
Nodes ->
|
||||
{Results, Failed} = emqx_conf_proto_v2:get_override_config_file(Nodes),
|
||||
|
|
@ -141,7 +144,7 @@ copy_override_conf_from_core_node() ->
|
|||
%% finish the boot sequence and load the
|
||||
%% config for other nodes to copy it.
|
||||
?SLOG(info, #{
|
||||
msg => "skip_copy_overide_conf_from_core_node",
|
||||
msg => "skip_copy_override_conf_from_core_node",
|
||||
loading_from_disk => true,
|
||||
nodes => Nodes,
|
||||
failed => Failed,
|
||||
|
|
@ -153,7 +156,7 @@ copy_override_conf_from_core_node() ->
|
|||
Jitter = rand:uniform(2_000),
|
||||
Timeout = 10_000 + Jitter,
|
||||
?SLOG(info, #{
|
||||
msg => "copy_overide_conf_from_core_node_retry",
|
||||
msg => "copy_override_conf_from_core_node_retry",
|
||||
timeout => Timeout,
|
||||
nodes => Nodes,
|
||||
failed => Failed,
|
||||
|
|
@ -166,7 +169,7 @@ copy_override_conf_from_core_node() ->
|
|||
[{ok, Info} | _] = lists:sort(fun conf_sort/2, Ready),
|
||||
#{node := Node, conf := RawOverrideConf, tnx_id := TnxId} = Info,
|
||||
?SLOG(debug, #{
|
||||
msg => "copy_overide_conf_from_core_node_success",
|
||||
msg => "copy_override_conf_from_core_node_success",
|
||||
node => Node,
|
||||
cluster_override_conf_file => application:get_env(
|
||||
emqx, cluster_override_conf_file
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ init([]) ->
|
|||
ChildSpecs =
|
||||
[
|
||||
child_spec(emqx_cluster_rpc, []),
|
||||
child_spec(emqx_cluster_rpc_handler, [])
|
||||
child_spec(emqx_cluster_rpc_cleaner, [])
|
||||
],
|
||||
{ok, {SupFlags, ChildSpecs}}.
|
||||
|
||||
|
|
|
|||
|
|
@ -43,6 +43,7 @@ groups() -> [].
|
|||
init_per_suite(Config) ->
|
||||
application:load(emqx_conf),
|
||||
ok = ekka:start(),
|
||||
ok = emqx_common_test_helpers:start_apps([]),
|
||||
ok = mria_rlog:wait_for_shards([?CLUSTER_RPC_SHARD], infinity),
|
||||
ok = emqx_config:put([node, cluster_call, retry_interval], 1000),
|
||||
meck:new(emqx_alarm, [non_strict, passthrough, no_link]),
|
||||
|
|
@ -53,6 +54,7 @@ init_per_suite(Config) ->
|
|||
Config.
|
||||
|
||||
end_per_suite(_Config) ->
|
||||
ok = emqx_common_test_helpers:stop_apps([]),
|
||||
ekka:stop(),
|
||||
mria:stop(),
|
||||
meck:unload(mria),
|
||||
|
|
@ -255,13 +257,13 @@ t_fast_forward_commit(_Config) ->
|
|||
),
|
||||
ok.
|
||||
|
||||
t_handler_unexpected_msg(_Config) ->
|
||||
Handler = emqx_cluster_rpc_handler,
|
||||
OldPid = erlang:whereis(Handler),
|
||||
ok = gen_server:cast(Handler, unexpected_cast_msg),
|
||||
ignore = gen_server:call(Handler, unexpected_cast_msg),
|
||||
erlang:send(Handler, unexpected_info_msg),
|
||||
NewPid = erlang:whereis(Handler),
|
||||
t_cleaner_unexpected_msg(_Config) ->
|
||||
Cleaner = emqx_cluster_cleaner,
|
||||
OldPid = erlang:whereis(Cleaner),
|
||||
ok = gen_server:cast(Cleaner, unexpected_cast_msg),
|
||||
ignore = gen_server:call(Cleaner, unexpected_cast_msg),
|
||||
erlang:send(Cleaner, unexpected_info_msg),
|
||||
NewPid = erlang:whereis(Cleaner),
|
||||
?assertEqual(OldPid, NewPid),
|
||||
ok.
|
||||
|
||||
|
|
@ -279,8 +281,8 @@ start() ->
|
|||
{ok, Pid1} = emqx_cluster_rpc:start_link(),
|
||||
{ok, Pid2} = emqx_cluster_rpc:start_link({node(), ?NODE2}, ?NODE2, 500),
|
||||
{ok, Pid3} = emqx_cluster_rpc:start_link({node(), ?NODE3}, ?NODE3, 500),
|
||||
{ok, Pid4} = emqx_cluster_rpc_handler:start_link(100, 500),
|
||||
true = erlang:register(emqx_cluster_rpc_handler, Pid4),
|
||||
{ok, Pid4} = emqx_cluster_rpc_cleaner:start_link(100, 500),
|
||||
true = erlang:register(emqx_cluster_rpc_cleaner, Pid4),
|
||||
{ok, [Pid1, Pid2, Pid3, Pid4]}.
|
||||
|
||||
stop() ->
|
||||
|
|
@ -296,7 +298,7 @@ stop() ->
|
|||
end
|
||||
|| N <- [?NODE1, ?NODE2, ?NODE3]
|
||||
],
|
||||
gen_server:stop(emqx_cluster_rpc_handler, normal, 5000).
|
||||
gen_server:stop(emqx_cluster_rpc_cleaner, normal, 5000).
|
||||
|
||||
receive_msg(0, _Msg) ->
|
||||
ok;
|
||||
|
|
|
|||
|
|
@ -70,7 +70,7 @@ worker_pool_size(required) -> false;
|
|||
worker_pool_size(_) -> undefined.
|
||||
|
||||
resume_interval(type) -> emqx_schema:duration_ms();
|
||||
resume_interval(importance) -> hidden;
|
||||
resume_interval(importance) -> ?IMPORTANCE_HIDDEN;
|
||||
resume_interval(desc) -> ?DESC("resume_interval");
|
||||
resume_interval(required) -> false;
|
||||
resume_interval(_) -> undefined.
|
||||
|
|
|
|||
|
|
@ -514,6 +514,8 @@ inc_action_metrics({error, {recoverable_error, _}}, RuleId) ->
|
|||
emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.failed.out_of_service');
|
||||
inc_action_metrics(?RESOURCE_ERROR_M(R, _), RuleId) when ?IS_RES_DOWN(R) ->
|
||||
emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.failed.out_of_service');
|
||||
inc_action_metrics({error, {unrecoverable_error, _}}, RuleId) ->
|
||||
emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.failed');
|
||||
inc_action_metrics(R, RuleId) ->
|
||||
case is_ok_result(R) of
|
||||
false ->
|
||||
|
|
@ -523,9 +525,7 @@ inc_action_metrics(R, RuleId) ->
|
|||
emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.success')
|
||||
end.
|
||||
|
||||
is_ok_result(ok) ->
|
||||
true;
|
||||
is_ok_result(R) when is_tuple(R) ->
|
||||
ok == erlang:element(1, R);
|
||||
is_ok_result(ok) ->
|
||||
is_ok_result(_) ->
|
||||
false.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,2 @@
|
|||
Ensure that when the core or replicant node starting, the `cluster-override.conf` file is only copied from the core node.
|
||||
Previously, when sorting nodes by startup time, the core node may have copied this file from the replicant node.
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
确保当 core 或 replicant 节点启动时，仅从 core 节点复制 `cluster-override.conf` 文件。
|
||||
此前按照节点启动时间排序时，core 节点可能从 replicant 节点复制该文件。
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
Don't increment 'actions.failed.unknown' rule metrics counter upon receiving unrecoverable bridge errors.
|
||||
This counter is displayed on the dashboard's rule overview tab ('Action statistics' - 'Unknown').
|
||||
The fix is only applicable for synchronous bridges, as all rule actions for asynchronous bridges
|
||||
are counted as successful (they increment 'actions.success' which is displayed as 'Action statistics' - 'Success').
|
||||
|
|
@ -1 +0,0 @@
|
|||
Clickhouse has got a fix that makes the error message better when users click the test button in the settings dialog.
|
||||
|
|
@ -1 +0,0 @@
|
|||
Clickhouse 已经修复了一个问题，当用户在设置对话框中点击测试按钮时，错误信息会更清晰。
|
||||
|
|
@ -17,17 +17,8 @@ will be forwarded."""
|
|||
}
|
||||
sql_template {
|
||||
desc {
|
||||
en: """SQL Template. The template string can contain placeholders
|
||||
for message metadata and payload field. The placeholders are inserted
|
||||
without any checking and special formatting, so it is important to
|
||||
ensure that the inserted values are formatted and escaped correctly."""
|
||||
zh:
|
||||
"""SQL模板。模板字符串可以包含消息元数据和有效载荷字段的占位符。占位符
|
||||
的插入不需要任何检查和特殊格式化，因此必须确保插入的数值格式化和转义正确。模板字符串可以包含占位符
|
||||
模板字符串可以包含消息元数据和有效载荷字段的占位符。这些占位符被插入
|
||||
所以必须确保插入的值的格式正确。因此，确保插入的值格式化和转义正确是非常重要的。模板字符串可以包含占位符
|
||||
模板字符串可以包含消息元数据和有效载荷字段的占位符。这些占位符被插入
|
||||
所以必须确保插入的值的格式正确。确保插入的值被正确地格式化和转义。"""
|
||||
en: """The template string can contain ${field} placeholders for message metadata and payload field. Make sure that the inserted values are formatted and escaped correctly. [Prepared Statement](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridges.html#Prepared-Statement) is not supported."""
|
||||
zh: """可以使用 ${field} 占位符来引用消息与客户端上下文中的变量，请确保对应字段存在且数据格式符合预期。此处不支持 [SQL 预处理](https://docs.emqx.com/zh/enterprise/v5.0/data-integration/data-bridges.html#sql-预处理)。"""
|
||||
}
|
||||
label {
|
||||
en: "SQL Template"
|
||||
|
|
@ -36,25 +27,8 @@ ensure that the inserted values are formatted and escaped correctly."""
|
|||
}
|
||||
batch_value_separator {
|
||||
desc {
|
||||
en: """The bridge repeats what comes after the VALUES or FORMAT FormatType in the
|
||||
SQL template to form a batch request. The value specified with
|
||||
this parameter will be inserted between the values. The default
|
||||
value ',' works for the VALUES format, but other values
|
||||
might be needed if you specify some other format with the
|
||||
clickhouse FORMAT syntax.
|
||||
|
||||
See https://clickhouse.com/docs/en/sql-reference/statements/insert-into/ and
|
||||
https://clickhouse.com/docs/en/interfaces/formats#formats for more information about
|
||||
the format syntax and the available formats."""
|
||||
zh: """桥接会重复 VALUES 或 FORMAT 格式类型之后的内容。中 VALUES 或
|
||||
FORMAT FormatType 后面的内容，以形成一个批处理请求。用这个参数指定的值
|
||||
这个参数指定的值将被插入到这些值之间。默认的
|
||||
默认值','适用于VALUES格式，但是如果你指定了其他的格式，可能需要其他的值。可能需要其他值，如果你用
|
||||
"clickhouse FORMAT "语法指定其他格式。语法指定其他格式。
|
||||
|
||||
参见https://clickhouse.com/docs/en/sql-reference/statements/insert-into/ 和
|
||||
https://clickhouse.com/docs/en/interfaces/formats#formats 了解更多关于
|
||||
格式语法和可用的格式。"""
|
||||
en: """The default value ',' works for the VALUES format. You can also use other separator if other format is specified. See [INSERT INTO Statement](https://clickhouse.com/docs/en/sql-reference/statements/insert-into)."""
|
||||
zh: """默认为逗号 ','，适用于 VALUE 格式。您也可以使用其他分隔符， 请参考 [INSERT INTO 语句](https://clickhouse.com/docs/en/sql-reference/statements/insert-into)。"""
|
||||
}
|
||||
label {
|
||||
en: "Batch Value Separator"
|
||||
|
|
|
|||
|
|
@ -27,6 +27,13 @@ help() {
|
|||
echo " E.g. ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2-debian11"
|
||||
}
|
||||
|
||||
die() {
|
||||
msg="$1"
|
||||
echo "$msg" >&2
|
||||
help
|
||||
exit 1
|
||||
}
|
||||
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case $1 in
|
||||
-h|--help)
|
||||
|
|
@ -81,13 +88,23 @@ while [ "$#" -gt 0 ]; do
|
|||
esac
|
||||
done
|
||||
|
||||
if [ -z "${PROFILE:-}" ] ||
|
||||
[ -z "${PKGTYPE:-}" ] ||
|
||||
[ -z "${BUILDER:-}" ] ||
|
||||
[ -z "${ARCH:-}" ]; then
|
||||
help
|
||||
exit 1
|
||||
## we have a different naming for them
|
||||
if [[ $(uname -m) == "x86_64" ]]; then
|
||||
NATIVE_ARCH='amd64'
|
||||
elif [[ $(uname -m) == "aarch64" ]]; then
|
||||
NATIVE_ARCH='arm64'
|
||||
elif [[ $(uname -m) == "arm64" ]]; then
|
||||
NATIVE_ARCH='arm64'
|
||||
elif [[ $(uname -m) == "armv7l" ]]; then
|
||||
# CHECKME: really ?
|
||||
NATIVE_ARCH='arm64'
|
||||
fi
|
||||
ARCH="${ARCH:-${NATIVE_ARCH:-}}"
|
||||
|
||||
[ -z "${PROFILE:-}" ] && die "missing --prifile"
|
||||
[ -z "${PKGTYPE:-}" ] && die "missing --pkgtyp"
|
||||
[ -z "${BUILDER:-}" ] && die "missing --builder"
|
||||
[ -z "${ARCH:-}" ] && die "missing --arch"
|
||||
|
||||
# ensure dir
|
||||
cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.."
|
||||
|
|
@ -128,13 +145,7 @@ if [[ "$HOST_SYSTEM" = "$BUILDER_SYSTEM" ]]; then
|
|||
fi
|
||||
|
||||
IS_NATIVE_ARCH='no'
|
||||
if [[ $(uname -m) == "x86_64" && "$ARCH" == "amd64" ]]; then
|
||||
IS_NATIVE_ARCH='yes'
|
||||
elif [[ $(uname -m) == "aarch64" && "$ARCH" == "arm64" ]]; then
|
||||
IS_NATIVE_ARCH='yes'
|
||||
elif [[ $(uname -m) == "arm64" && "$ARCH" == "arm64" ]]; then
|
||||
IS_NATIVE_ARCH='yes'
|
||||
elif [[ $(uname -m) == "armv7l" && "$ARCH" == "arm64" ]]; then
|
||||
if [[ "$NATIVE_ARCH" == "$ARCH" ]]; then
|
||||
IS_NATIVE_ARCH='yes'
|
||||
fi
|
||||
|
||||
|
|
@ -151,7 +162,7 @@ elif docker info; then
|
|||
--platform="linux/$ARCH" \
|
||||
--env ACLOCAL_PATH="/usr/share/aclocal:/usr/local/share/aclocal" \
|
||||
"$BUILDER" \
|
||||
bash -euc "$CMD_RUN"
|
||||
bash -euc "git config --global --add safe.directory /emqx && $CMD_RUN"
|
||||
else
|
||||
echo "Error: Docker not available on unsupported platform"
|
||||
exit 1;
|
||||
|
|
|
|||
Loading…
Reference in New Issue