yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/release-57' into sync-r57-m-20240611

This commit is contained in:
Thales Macedo Garitezi 2024-06-11 15:34:54 -03:00
parent b2d716909f 7cb57606bf
commit db572d35a7
88 changed files with 5792 additions and 527 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.ci/docker-compose-file/docker-compose-kafka.yaml
View File

@ -18,7 +18,7 @@ services:
- /tmp/emqx-ci/emqx-shared-secret:/var/lib/secret
kdc:
hostname: kdc.emqx.net
image: ghcr.io/emqx/emqx-builder/5.3-7:1.15.7-26.2.5-1-ubuntu22.04
image: ghcr.io/emqx/emqx-builder/5.3-8:1.15.7-26.2.5-2-ubuntu22.04
container_name: kdc.emqx.net
expose:
- 88 # kdc

2
.ci/docker-compose-file/docker-compose.yaml
View File

@ -3,7 +3,7 @@ version: '3.9'
services:
erlang:
container_name: erlang
image: ${DOCKER_CT_RUNNER_IMAGE:-ghcr.io/emqx/emqx-builder/5.3-7:1.15.7-26.2.5-1-ubuntu22.04}
image: ${DOCKER_CT_RUNNER_IMAGE:-ghcr.io/emqx/emqx-builder/5.3-8:1.15.7-26.2.5-2-ubuntu22.04}
env_file:
- credentials.env
- conf.env

16
.github/workflows/_pr_entrypoint.yaml vendored
View File

@ -17,16 +17,16 @@ env:
jobs:
sanity-checks:
runs-on: ubuntu-22.04
container: "ghcr.io/emqx/emqx-builder/5.3-7:1.15.7-26.2.5-1-ubuntu22.04"
container: "ghcr.io/emqx/emqx-builder/5.3-8:1.15.7-26.2.5-2-ubuntu22.04"
outputs:
ct-matrix: ${{ steps.matrix.outputs.ct-matrix }}
ct-host: ${{ steps.matrix.outputs.ct-host }}
ct-docker: ${{ steps.matrix.outputs.ct-docker }}
version-emqx: ${{ steps.matrix.outputs.version-emqx }}
version-emqx-enterprise: ${{ steps.matrix.outputs.version-emqx-enterprise }}
builder: "ghcr.io/emqx/emqx-builder/5.3-7:1.15.7-26.2.5-1-ubuntu22.04"
builder_vsn: "5.3-7"
otp_vsn: "26.2.5-1"
builder: "ghcr.io/emqx/emqx-builder/5.3-8:1.15.7-26.2.5-2-ubuntu22.04"
builder_vsn: "5.3-8"
otp_vsn: "26.2.5-2"
elixir_vsn: "1.15.7"
permissions:
@ -96,13 +96,13 @@ jobs:
MATRIX="$(echo "${APPS}" | jq -c '
[
(.[] | select(.profile == "emqx") | . + {
builder: "5.3-7",
otp: "26.2.5-1",
builder: "5.3-8",
otp: "26.2.5-2",
elixir: "1.15.7"
}),
(.[] | select(.profile == "emqx-enterprise") | . + {
builder: "5.3-7",
otp: ["26.2.5-1"][],
builder: "5.3-8",
otp: ["26.2.5-2"][],
elixir: "1.15.7"
})
]

16
.github/workflows/_push-entrypoint.yaml vendored
View File

@ -24,7 +24,7 @@ env:
jobs:
prepare:
runs-on: ubuntu-22.04
container: 'ghcr.io/emqx/emqx-builder/5.3-7:1.15.7-26.2.5-1-ubuntu22.04'
container: 'ghcr.io/emqx/emqx-builder/5.3-8:1.15.7-26.2.5-2-ubuntu22.04'
outputs:
profile: ${{ steps.parse-git-ref.outputs.profile }}
release: ${{ steps.parse-git-ref.outputs.release }}
@ -32,9 +32,9 @@ jobs:
ct-matrix: ${{ steps.matrix.outputs.ct-matrix }}
ct-host: ${{ steps.matrix.outputs.ct-host }}
ct-docker: ${{ steps.matrix.outputs.ct-docker }}
builder: 'ghcr.io/emqx/emqx-builder/5.3-7:1.15.7-26.2.5-1-ubuntu22.04'
builder_vsn: '5.3-7'
otp_vsn: '26.2.5-1'
builder: 'ghcr.io/emqx/emqx-builder/5.3-8:1.15.7-26.2.5-2-ubuntu22.04'
builder_vsn: '5.3-8'
otp_vsn: '26.2.5-2'
elixir_vsn: '1.15.7'
permissions:
@ -66,13 +66,13 @@ jobs:
MATRIX="$(echo "${APPS}" | jq -c '
[
(.[] | select(.profile == "emqx") | . + {
builder: "5.3-7",
otp: "26.2.5-1",
builder: "5.3-8",
otp: "26.2.5-2",
elixir: "1.15.7"
}),
(.[] | select(.profile == "emqx-enterprise") | . + {
builder: "5.3-7",
otp: ["26.2.5-1"][],
builder: "5.3-8",
otp: ["26.2.5-2"][],
elixir: "1.15.7"
})
]

4
.github/workflows/build_and_push_docker_images.yaml vendored
View File

@ -53,7 +53,7 @@ on:
otp_vsn:
required: false
type: string
default: '26.2.5-1'
default: '26.2.5-2'
elixir_vsn:
required: false
type: string
@ -61,7 +61,7 @@ on:
builder_vsn:
required: false
type: string
default: '5.3-7'
default: '5.3-8'
permissions:
contents: read

4
.github/workflows/build_packages.yaml vendored
View File

@ -55,7 +55,7 @@ on:
otp_vsn:
required: false
type: string
default: '26.2.5-1'
default: '26.2.5-2'
elixir_vsn:
required: false
type: string
@ -63,7 +63,7 @@ on:
builder_vsn:
required: false
type: string
default: '5.3-7'
default: '5.3-8'
permissions:
contents: read

6
.github/workflows/build_packages_cron.yaml vendored
View File

@ -23,8 +23,8 @@ jobs:
fail-fast: false
matrix:
profile:
- ['emqx', 'master', '5.3-7:1.15.7-26.2.5-1']
- ['emqx', 'release-57', '5.3-7:1.15.7-26.2.5-1']
- ['emqx', 'master', '5.3-8:1.15.7-26.2.5-2']
- ['emqx', 'release-57', '5.3-8:1.15.7-26.2.5-2']
os:
- ubuntu22.04
- amzn2023
@ -92,7 +92,7 @@ jobs:
branch:
- master
otp:
- 26.2.5-1
- 26.2.5-2
os:
- macos-12-arm64

12
.github/workflows/build_slim_packages.yaml vendored
View File

@ -27,15 +27,15 @@ on:
builder:
required: false
type: string
default: 'ghcr.io/emqx/emqx-builder/5.3-7:1.15.7-26.2.5-1-ubuntu22.04'
default: 'ghcr.io/emqx/emqx-builder/5.3-8:1.15.7-26.2.5-2-ubuntu22.04'
builder_vsn:
required: false
type: string
default: '5.3-7'
default: '5.3-8'
otp_vsn:
required: false
type: string
default: '26.2.5-1'
default: '26.2.5-2'
elixir_vsn:
required: false
type: string
@ -54,9 +54,9 @@ jobs:
fail-fast: false
matrix:
profile:
- ["emqx", "26.2.5-1", "ubuntu22.04", "elixir", "x64"]
- ["emqx", "26.2.5-1", "ubuntu22.04", "elixir", "arm64"]
- ["emqx-enterprise", "26.2.5-1", "ubuntu22.04", "erlang", "x64"]
- ["emqx", "26.2.5-2", "ubuntu22.04", "elixir", "x64"]
- ["emqx", "26.2.5-2", "ubuntu22.04", "elixir", "arm64"]
- ["emqx-enterprise", "26.2.5-2", "ubuntu22.04", "erlang", "x64"]
container: "ghcr.io/emqx/emqx-builder/${{ inputs.builder_vsn }}:${{ inputs.elixir_vsn }}-${{ matrix.profile[1] }}-${{ matrix.profile[2] }}"

2
.github/workflows/codeql.yaml vendored
View File

@ -18,7 +18,7 @@ jobs:
actions: read
security-events: write
container:
image: ghcr.io/emqx/emqx-builder/5.3-7:1.15.7-26.2.5-1-ubuntu22.04
image: ghcr.io/emqx/emqx-builder/5.3-8:1.15.7-26.2.5-2-ubuntu22.04
strategy:
fail-fast: false

2
.github/workflows/performance_test.yaml vendored
View File

@ -26,7 +26,7 @@ jobs:
prepare:
runs-on: ubuntu-latest
if: github.repository_owner == 'emqx'
container: ghcr.io/emqx/emqx-builder/5.3-7:1.15.7-26.2.5-1-ubuntu20.04
container: ghcr.io/emqx/emqx-builder/5.3-8:1.15.7-26.2.5-2-ubuntu20.04
outputs:
BENCH_ID: ${{ steps.prepare.outputs.BENCH_ID }}
PACKAGE_FILE: ${{ steps.package_file.outputs.PACKAGE_FILE }}

2
.tool-versions
View File

@ -1,2 +1,2 @@
erlang 26.2.5-1
erlang 26.2.5-2
elixir 1.15.7-otp-26

2
Makefile
View File

@ -7,7 +7,7 @@ REBAR = $(CURDIR)/rebar3
BUILD = $(CURDIR)/build
SCRIPTS = $(CURDIR)/scripts
export EMQX_RELUP ?= true
export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.3-7:1.15.7-26.2.5-1-debian12
export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.3-8:1.15.7-26.2.5-2-debian12
export EMQX_DEFAULT_RUNNER = public.ecr.aws/debian/debian:12-slim
export EMQX_REL_FORM ?= tgz
export QUICER_DOWNLOAD_FROM_RELEASE = 1

3
apps/emqx/include/emqx_hooks.hrl
View File

@ -25,7 +25,8 @@
-define(HP_AUTHN, 970).
-define(HP_AUTHZ, 960).
-define(HP_SYS_MSGS, 950).
-define(HP_MSG_VALIDATION, 945).
-define(HP_SCHEMA_VALIDATION, 945).
-define(HP_MESSAGE_TRANSFORMATION, 943).
-define(HP_TOPIC_METRICS, 940).
-define(HP_RETAINER, 930).
-define(HP_AUTO_SUB, 920).

13
apps/emqx/src/emqx_channel.erl
View File

@ -685,21 +685,28 @@ process_publish(Packet = ?PUBLISH_PACKET(QoS, Topic, PacketId), Channel) ->
end.
packet_to_message(Packet, #channel{
conninfo = #{proto_ver := ProtoVer},
clientinfo = #{
conninfo = #{
peername := PeerName,
proto_ver := ProtoVer
},
clientinfo =
#{
protocol := Protocol,
clientid := ClientId,
username := Username,
peerhost := PeerHost,
mountpoint := MountPoint
}
} = ClientInfo
}) ->
ClientAttrs = maps:get(client_attrs, ClientInfo, #{}),
emqx_mountpoint:mount(
MountPoint,
emqx_packet:to_message(
Packet,
ClientId,
#{
client_attrs => ClientAttrs,
peername => PeerName,
proto_ver => ProtoVer,
protocol => Protocol,
username => Username,

1
apps/emqx/src/emqx_hookpoints.erl
View File

@ -60,6 +60,7 @@
'message.publish',
'message.puback',
'message.dropped',
'message.transformation_failed',
'schema.validation_failed',
'message.delivered',
'message.acked',

6
apps/emqx/src/emqx_metrics.erl
View File

@ -211,6 +211,10 @@
{counter, 'messages.validation_failed'},
%% % Messages that passed validations
{counter, 'messages.validation_succeeded'},
%% % Messages that failed transformations
{counter, 'messages.transformation_failed'},
%% % Messages that passed transformations
{counter, 'messages.transformation_succeeded'},
% QoS2 Messages expired
{counter, 'messages.dropped.await_pubrel_timeout'},
% Messages dropped
@ -721,4 +725,6 @@ reserved_idx('overload_protection.new_conn') -> 404;
reserved_idx('messages.validation_succeeded') -> 405;
reserved_idx('messages.validation_failed') -> 406;
reserved_idx('messages.persisted') -> 407;
reserved_idx('messages.transformation_succeeded') -> 408;
reserved_idx('messages.transformation_failed') -> 409;
reserved_idx(_) -> undefined.

2
apps/emqx/test/emqx_cth_suite.erl
View File

@ -385,6 +385,8 @@ default_appspec(emqx_schema_registry, _SuiteOpts) ->
#{schema_mod => emqx_schema_registry_schema, config => #{}};
default_appspec(emqx_schema_validation, _SuiteOpts) ->
#{schema_mod => emqx_schema_validation_schema, config => #{}};
default_appspec(emqx_message_transformation, _SuiteOpts) ->
#{schema_mod => emqx_message_transformation_schema, config => #{}};
default_appspec(_, _) ->
#{}.

8
apps/emqx_bridge/src/emqx_bridge_v2.erl
View File

@ -456,7 +456,7 @@ install_bridge_v2_helper(
ConnectorId = emqx_connector_resource:resource_id(
connector_type(BridgeV2Type), ConnectorName
),
emqx_resource_manager:add_channel(
_ = emqx_resource_manager:add_channel(
ConnectorId,
BridgeV2Id,
augment_channel_config(
@ -786,7 +786,11 @@ create_dry_run_helper(ConfRootKey, BridgeV2Type, ConnectorRawConf, BridgeV2RawCo
BridgeName,
BridgeV2Conf
),
case emqx_resource_manager:add_channel(ConnectorId, ChannelTestId, AugmentedConf) of
%% We'll perform it ourselves to get the resulting status afterwards.
Opts = #{perform_health_check => false},
case
emqx_resource_manager:add_channel(ConnectorId, ChannelTestId, AugmentedConf, Opts)
of
{error, Reason} ->
{error, Reason};
ok ->

2
apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src
View File

@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_bridge_mqtt, [
{description, "EMQX MQTT Broker Bridge"},
{vsn, "0.2.0"},
{vsn, "0.2.1"},
{registered, []},
{applications, [
kernel,

19
apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl
View File

@ -57,6 +57,9 @@
-define(HEALTH_CHECK_TIMEOUT, 1000).
-define(INGRESS, "I").
-define(EGRESS, "E").
-define(NO_PREFIX, <<>>).
-define(IS_NO_PREFIX(P), (P =:= undefined orelse P =:= ?NO_PREFIX)).
-define(MAX_PREFIX_BYTES, 19).
%% ===================================================================
%% When use this bridge as a data source, ?MODULE:on_message_received will be called
@ -441,9 +444,9 @@ ms_to_s(Ms) ->
clientid(Name, _Conf = #{clientid_prefix := Prefix}) when
is_binary(Prefix) andalso Prefix =/= <<>>
->
emqx_bridge_mqtt_lib:clientid_base([Prefix, $:, Name]);
{Prefix, emqx_bridge_mqtt_lib:clientid_base(Name)};
clientid(Name, _Conf) ->
emqx_bridge_mqtt_lib:clientid_base([Name]).
{?NO_PREFIX, emqx_bridge_mqtt_lib:clientid_base(Name)}.
%% @doc Start an ingress bridge worker.
-spec connect([option() | {ecpool_worker_id, pos_integer()}]) ->
@ -481,8 +484,16 @@ mk_client_opts(
msg_handler => mk_client_event_handler(Name, TopicToHandlerIndex)
}.
mk_clientid(WorkerId, ClientId) ->
emqx_bridge_mqtt_lib:bytes23([ClientId], WorkerId).
mk_clientid(WorkerId, {Prefix, ClientId}) when ?IS_NO_PREFIX(Prefix) ->
%% When there is no prefix, try to keep the client ID length within 23 bytes
emqx_bridge_mqtt_lib:bytes23(ClientId, WorkerId);
mk_clientid(WorkerId, {Prefix, ClientId}) when size(Prefix) =< ?MAX_PREFIX_BYTES ->
%% Try to respect client ID prefix when it's no more than 19 bytes,
%% meaning there are at least 4 bytes as hash space.
emqx_bridge_mqtt_lib:bytes23_with_prefix(Prefix, ClientId, WorkerId);
mk_clientid(WorkerId, {Prefix, ClientId}) ->
%% There is no other option but to use a long client ID
iolist_to_binary([Prefix, ClientId, $:, integer_to_binary(WorkerId)]).
mk_client_event_handler(Name, TopicToHandlerIndex) ->
#{

23
apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_lib.erl
View File

@ -16,7 +16,7 @@
-module(emqx_bridge_mqtt_lib).
-export([clientid_base/1, bytes23/2]).
-export([clientid_base/1, bytes23/2, bytes23_with_prefix/3]).
%% @doc Make the base ID of client IDs.
%% A base ID is used to concatenate with pool worker ID to build a
@ -28,18 +28,29 @@ clientid_base(Name) ->
bin([Name, shortener(atom_to_list(node()), 8)]).
%% @doc Limit the number of bytes for client ID under 23 bytes.
%% If Prefix and suffix concatenated is longer than 23 bytes
%% If ClientID base and suffix concatenated is longer than 23 bytes
%% it hashes the concatenation and replace the non-random suffix.
bytes23(Prefix, SeqNo) ->
bytes23(ClientId, SeqNo) ->
bytes_n(ClientId, SeqNo, 23).
bytes_n(ClientId, SeqNo, N) ->
Suffix = integer_to_binary(SeqNo),
Concat = bin([Prefix, $:, Suffix]),
case size(Concat) =< 23 of
Concat = bin([ClientId, $:, Suffix]),
case size(Concat) =< N of
true ->
Concat;
false ->
shortener(Concat, 23)
shortener(Concat, N)
end.
%% @doc Limit the number of bytes for client ID under 23 bytes.
%% If Prefix, ClientID base and suffix concatenated is longer than 23 bytes
%% it hashes the ClientID and SeqNo before appended to the Prefix
bytes23_with_prefix(Prefix, ClientId, SeqNo) when Prefix =/= <<>> ->
SuffixLen = 23 - size(Prefix),
true = (SuffixLen > 0),
bin([Prefix, bytes_n(ClientId, SeqNo, SuffixLen)]).
%% @private SHA hash a string and return the prefix of
%% the given length as hex string in binary format.
shortener(Str, Length) when is_list(Str) ->

26
apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl
View File

@ -568,6 +568,7 @@ t_egress_short_clientid(_Config) ->
Name = <<"abc01234">>,
BaseId = emqx_bridge_mqtt_lib:clientid_base([Name]),
ExpectedClientId = iolist_to_binary([BaseId, $:, "1"]),
?assertMatch(<<"abc01234", _/binary>>, ExpectedClientId),
test_egress_clientid(Name, ExpectedClientId).
t_egress_long_clientid(_Config) ->
@ -578,11 +579,34 @@ t_egress_long_clientid(_Config) ->
ExpectedClientId = emqx_bridge_mqtt_lib:bytes23(BaseId, 1),
test_egress_clientid(Name, ExpectedClientId).
t_egress_with_short_prefix(_Config) ->
%% Expect the actual client ID in use is hashed from
%% <prefix>head(sha1(<name><nodename-hash>:<pool_worker_id>), 16)
Prefix = <<"012-">>,
Name = <<"345">>,
BaseId = emqx_bridge_mqtt_lib:clientid_base([Name]),
ExpectedClientId = emqx_bridge_mqtt_lib:bytes23_with_prefix(Prefix, BaseId, 1),
?assertMatch(<<"012-", _/binary>>, ExpectedClientId),
test_egress_clientid(Name, Prefix, ExpectedClientId).
t_egress_with_long_prefix(_Config) ->
%% Expect the actual client ID in use is hashed from
%% <prefix><name><nodename-hash>:<pool_worker_id>
Prefix = <<"0123456789abcdef01234-">>,
Name = <<"345">>,
BaseId = emqx_bridge_mqtt_lib:clientid_base([Name]),
ExpectedClientId = iolist_to_binary([Prefix, BaseId, <<":1">>]),
test_egress_clientid(Name, Prefix, ExpectedClientId).
test_egress_clientid(Name, ExpectedClientId) ->
test_egress_clientid(Name, <<>>, ExpectedClientId).
test_egress_clientid(Name, ClientIdPrefix, ExpectedClientId) ->
BridgeIDEgress = create_bridge(
?SERVER_CONF#{
<<"name">> => Name,
<<"egress">> => (?EGRESS_CONF)#{<<"pool_size">> => 1}
<<"egress">> => (?EGRESS_CONF)#{<<"pool_size">> => 1},
<<"clientid_prefix">> => ClientIdPrefix
}
),
LocalTopic = <<?EGRESS_LOCAL_TOPIC, "/1">>,

82
apps/emqx_bridge_pgsql/test/emqx_bridge_pgsql_SUITE.erl
View File

@ -135,6 +135,7 @@ end_per_testcase(_Testcase, Config) ->
connect_and_clear_table(Config),
ok = snabbkaffe:stop(),
delete_bridge(Config),
emqx_common_test_helpers:call_janitor(),
ok.
%%------------------------------------------------------------------------------
@ -715,6 +716,87 @@ t_missing_table(Config) ->
connect_and_create_table(Config),
ok.
%% We test that we can handle when the prepared statement with the channel
%% name already exists in the connection instance when we try to make a new
%% prepared statement. It is unknown in which scenario this can happen but it
%% has been observed in a production log file.
%% See:
%% https://emqx.atlassian.net/browse/EEC-1036
t_prepared_statement_exists(Config) ->
Name = ?config(pgsql_name, Config),
BridgeType = ?config(pgsql_bridge_type, Config),
emqx_common_test_helpers:on_exit(fun() ->
meck:unload()
end),
MeckOpts = [passthrough, no_link, no_history, non_strict],
meck:new(emqx_postgresql, MeckOpts),
InsertPrepStatementDupAndThenRemoveMeck =
fun(Conn, Key, SQL, List) ->
meck:passthrough([Conn, Key, SQL, List]),
meck:delete(
epgsql,
parse2,
4
),
meck:passthrough([Conn, Key, SQL, List])
end,
meck:expect(
epgsql,
parse2,
InsertPrepStatementDupAndThenRemoveMeck
),
%% We should recover if the prepared statement name already exists in the
%% driver
?check_trace(
begin
?assertMatch({ok, _}, create_bridge(Config)),
?retry(
_Sleep = 1_000,
_Attempts = 20,
?assertMatch(
#{status := Status} when Status == connected,
emqx_bridge_v2:health_check(BridgeType, Name)
)
),
ok
end,
fun(Trace) ->
?assertMatch([_ | _], ?of_kind(pgsql_prepared_statement_exists, Trace)),
ok
end
),
InsertPrepStatementDup =
fun(Conn, Key, SQL, List) ->
meck:passthrough([Conn, Key, SQL, List]),
meck:passthrough([Conn, Key, SQL, List])
end,
meck:expect(
epgsql,
parse2,
InsertPrepStatementDup
),
%% We should get status disconnected if removing already existing statment don't help
?check_trace(
begin
?assertMatch({ok, _}, create_bridge(Config)),
?retry(
_Sleep = 1_000,
_Attempts = 20,
?assertMatch(
#{status := Status} when Status == disconnected,
emqx_bridge_v2:health_check(BridgeType, Name)
)
),
snabbkaffe_nemesis:cleanup(),
ok
end,
fun(Trace) ->
?assertMatch([_ | _], ?of_kind(pgsql_prepared_statement_exists, Trace)),
ok
end
),
ok.
t_table_removed(Config) ->
Name = ?config(pgsql_name, Config),
BridgeType = ?config(pgsql_bridge_type, Config),

15
apps/emqx_bridge_s3/src/emqx_bridge_s3_connector.erl
View File

@ -162,8 +162,13 @@ on_get_status(_InstId, State = #{client_config := Config}) ->
-spec on_add_channel(_InstanceId :: resource_id(), state(), channel_id(), channel_config()) ->
{ok, state()} | {error, _Reason}.
on_add_channel(_InstId, State = #{channels := Channels}, ChannelId, Config) ->
try
ChannelState = start_channel(State, Config),
{ok, State#{channels => Channels#{ChannelId => ChannelState}}}.
{ok, State#{channels => Channels#{ChannelId => ChannelState}}}
catch
throw:Reason ->
{error, Reason}
end.
-spec on_remove_channel(_InstanceId :: resource_id(), state(), channel_id()) ->
{ok, state()}.
@ -221,9 +226,10 @@ start_channel(State, #{
max_records => MaxRecords,
work_dir => work_dir(Type, Name)
},
Template = ensure_ok(emqx_bridge_s3_upload:mk_key_template(Parameters)),
DeliveryOpts = #{
bucket => Bucket,
key => emqx_bridge_s3_upload:mk_key_template(Parameters),
key => Template,
container => Container,
upload_options => emqx_bridge_s3_upload:mk_upload_options(Parameters),
callback_module => ?MODULE,
@ -247,6 +253,11 @@ start_channel(State, #{
on_stop => fun() -> ?AGGREG_SUP:delete_child(AggregId) end
}.
ensure_ok({ok, V}) ->
V;
ensure_ok({error, Reason}) ->
throw(Reason).
upload_options(Parameters) ->
#{acl => maps:get(acl, Parameters, undefined)}.

28
apps/emqx_bridge_s3/src/emqx_bridge_s3_upload.erl
View File

@ -248,17 +248,35 @@ convert_action(Conf = #{<<"parameters">> := Params, <<"resource_opts">> := Resou
%% Interpreting options
-spec mk_key_template(_Parameters :: map()) -> emqx_template:str().
-spec mk_key_template(_Parameters :: map()) ->
{ok, emqx_template:str()} | {error, _Reason}.
mk_key_template(#{key := Key}) ->
Template = emqx_template:parse(Key),
{_, BindingErrors} = emqx_template:render(Template, #{}),
{UsedBindings, _} = lists:unzip(BindingErrors),
case validate_bindings(emqx_template:placeholders(Template)) of
UsedBindings when is_list(UsedBindings) ->
SuffixTemplate = mk_suffix_template(UsedBindings),
case emqx_template:is_const(SuffixTemplate) of
true ->
Template;
{ok, Template};
false ->
Template ++ SuffixTemplate
{ok, Template ++ SuffixTemplate}
end;
Error = {error, _} ->
Error
end.
validate_bindings(Bindings) ->
Formats = ["rfc3339", "rfc3339utc", "unix"],
AllowedBindings = lists:append([
["action", "node", "sequence"],
["datetime." ++ F || F <- Formats],
["datetime_until." ++ F || F <- Formats]
]),
case Bindings -- AllowedBindings of
[] ->
Bindings;
Disallowed ->
{error, {invalid_key_template, {disallowed_placeholders, Disallowed}}}
end.
mk_suffix_template(UsedBindings) ->

9
apps/emqx_conf/src/emqx_conf_cli.erl
View File

@ -37,6 +37,7 @@
-define(AUDIT_MOD, audit).
-define(UPDATE_READONLY_KEYS_PROHIBITED, <<"Cannot update read-only key '~s'.">>).
-define(SCHEMA_VALIDATION_CONF_ROOT_BIN, <<"schema_validation">>).
-define(MESSAGE_TRANSFORMATION_CONF_ROOT_BIN, <<"message_transformation">>).
-dialyzer({no_match, [load/0]}).
@ -335,6 +336,14 @@ update_config_cluster(?SCHEMA_VALIDATION_CONF_ROOT_BIN = Key, NewConf, #{mode :=
check_res(Key, emqx_conf:update([Key], {merge, NewConf}, ?OPTIONS), NewConf, Opts);
update_config_cluster(?SCHEMA_VALIDATION_CONF_ROOT_BIN = Key, NewConf, #{mode := replace} = Opts) ->
check_res(Key, emqx_conf:update([Key], {replace, NewConf}, ?OPTIONS), NewConf, Opts);
update_config_cluster(
?MESSAGE_TRANSFORMATION_CONF_ROOT_BIN = Key, NewConf, #{mode := merge} = Opts
) ->
check_res(Key, emqx_conf:update([Key], {merge, NewConf}, ?OPTIONS), NewConf, Opts);
update_config_cluster(
?MESSAGE_TRANSFORMATION_CONF_ROOT_BIN = Key, NewConf, #{mode := replace} = Opts
) ->
check_res(Key, emqx_conf:update([Key], {replace, NewConf}, ?OPTIONS), NewConf, Opts);
update_config_cluster(Key, NewConf, #{mode := merge} = Opts) ->
Merged = merge_conf(Key, NewConf),
check_res(Key, emqx_conf:update([Key], Merged, ?OPTIONS), NewConf, Opts);

8
apps/emqx_connector_aggregator/src/emqx_connector_aggregator.erl
View File

@ -372,9 +372,13 @@ lookup_current_buffer(Name) ->
%%
enqueue_delivery(Buffer, St = #st{name = Name, deliveries = Ds}) ->
{ok, Pid} = emqx_connector_aggreg_upload_sup:start_delivery(Name, Buffer),
case emqx_connector_aggreg_upload_sup:start_delivery(Name, Buffer) of
{ok, Pid} ->
MRef = erlang:monitor(process, Pid),
St#st{deliveries = Ds#{MRef => Buffer}}.
St#st{deliveries = Ds#{MRef => Buffer}};
{error, _} = Error ->
handle_delivery_exit(Buffer, Error, St)
end.
handle_delivery_exit(Buffer, Normal, St = #st{name = Name}) when
Normal == normal; Normal == noproc

4
apps/emqx_dashboard/include/emqx_dashboard.hrl
View File

@ -67,6 +67,8 @@
%, sent_bytes
validation_succeeded,
validation_failed,
transformation_succeeded,
transformation_failed,
dropped,
persisted
]).
@ -90,6 +92,8 @@
sent => sent_msg_rate,
validation_succeeded => validation_succeeded_rate,
validation_failed => validation_failed_rate,
transformation_succeeded => transformation_succeeded_rate,
transformation_failed => transformation_failed_rate,
dropped => dropped_msg_rate,
persisted => persisted_rate
}).

8
apps/emqx_dashboard/src/emqx_dashboard_monitor.erl
View File

@ -209,7 +209,7 @@ do_call(Request) ->
gen_server:call(?MODULE, Request, 5000).
do_sample(all, Time) ->
do_sample(mria:cluster_nodes(running), Time, #{});
do_sample(emqx:running_nodes(), Time, #{});
do_sample(Node, Time) when Node == node() ->
MS = match_spec(Time),
internal_format(ets:select(?TAB, MS));
@ -259,7 +259,7 @@ merge_cluster_sampler_map(M1, M2) ->
Key =:= subscriptions_durable;
Key =:= disconnected_durable_sessions
->
Map#{Key => maps:get(Key, M1)};
Map#{Key => maps:get(Key, M1, maps:get(Key, M2, 0))};
(Key, Map) ->
Map#{Key => maps:get(Key, M1, 0) + maps:get(Key, M2, 0)}
end,
@ -474,6 +474,10 @@ stats(validation_succeeded) ->
emqx_metrics:val('messages.validation_succeeded');
stats(validation_failed) ->
emqx_metrics:val('messages.validation_failed');
stats(transformation_succeeded) ->
emqx_metrics:val('messages.transformation_succeeded');
stats(transformation_failed) ->
emqx_metrics:val('messages.transformation_failed');
stats(dropped) ->
emqx_metrics:val('messages.dropped');
stats(persisted) ->

8
apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl
View File

@ -198,6 +198,10 @@ swagger_desc(validation_succeeded) ->
swagger_desc_format("Schema validations succeeded ");
swagger_desc(validation_failed) ->
swagger_desc_format("Schema validations failed ");
swagger_desc(transformation_succeeded) ->
swagger_desc_format("Message transformations succeeded ");
swagger_desc(transformation_failed) ->
swagger_desc_format("Message transformations failed ");
swagger_desc(persisted) ->
swagger_desc_format("Messages saved to the durable storage ");
swagger_desc(disconnected_durable_sessions) ->
@ -230,6 +234,10 @@ swagger_desc(validation_succeeded_rate) ->
swagger_desc_format("Schema validations succeeded ", per);
swagger_desc(validation_failed_rate) ->
swagger_desc_format("Schema validations failed ", per);
swagger_desc(transformation_succeeded_rate) ->
swagger_desc_format("Message transformations succeeded ", per);
swagger_desc(transformation_failed_rate) ->
swagger_desc_format("Message transformations failed ", per);
swagger_desc(persisted_rate) ->
swagger_desc_format("Messages saved to the durable storage ", per);
swagger_desc(retained_msg_count) ->

36
apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl
View File

@ -27,6 +27,7 @@
-include_lib("common_test/include/ct.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-include_lib("emqx/include/emqx_mqtt.hrl").
-include_lib("emqx/include/asserts.hrl").
-define(SERVER, "http://127.0.0.1:18083").
-define(BASE_PATH, "/api/v5").
@ -191,6 +192,41 @@ t_monitor_sampler_format(_Config) ->
[?assert(lists:member(SamplerName, SamplerKeys)) || SamplerName <- ?SAMPLER_LIST],
ok.
t_handle_old_monitor_data(_Config) ->
Now = erlang:system_time(second),
FakeOldData = maps:from_list(
lists:map(
fun(N) ->
Time = (Now - N) * 1000,
{Time, #{foo => 123}}
end,
lists:seq(0, 9)
)
),
Self = self(),
ok = meck:new(emqx, [passthrough, no_history]),
ok = meck:expect(emqx, running_nodes, fun() -> [node(), 'other@node'] end),
ok = meck:new(emqx_dashboard_proto_v1, [passthrough, no_history]),
ok = meck:expect(emqx_dashboard_proto_v1, do_sample, fun('other@node', _Time) ->
Self ! sample_called,
FakeOldData
end),
{ok, _} =
snabbkaffe:block_until(
?match_event(#{?snk_kind := dashboard_monitor_flushed}),
infinity
),
?assertMatch(
#{},
hd(emqx_dashboard_monitor:samplers())
),
?assertReceive(sample_called, 1_000),
ok = meck:unload([emqx, emqx_dashboard_proto_v1]),
ok.
t_monitor_api(_) ->
{ok, _} =
snabbkaffe:block_until(

2
apps/emqx_enterprise/src/emqx_enterprise.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_enterprise, [
{description, "EMQX Enterprise Edition"},
{vsn, "0.2.0"},
{vsn, "0.2.1"},
{registered, []},
{applications, [
kernel,

4
apps/emqx_enterprise/src/emqx_enterprise_schema.erl
View File

@ -16,6 +16,7 @@
emqx_license_schema,
emqx_schema_registry_schema,
emqx_schema_validation_schema,
emqx_message_transformation_schema,
emqx_ft_schema
]).
@ -196,6 +197,7 @@ audit_log_conf() ->
tr_prometheus_collectors(Conf) ->
[
{'/prometheus/schema_validation', emqx_prometheus_schema_validation}
{'/prometheus/schema_validation', emqx_prometheus_schema_validation},
{'/prometheus/message_transformation', emqx_prometheus_message_transformation}
| emqx_conf_schema:tr_prometheus_collectors(Conf)
].

1
apps/emqx_machine/priv/reboot_lists.eterm
View File

@ -89,6 +89,7 @@
emqx_license,
emqx_enterprise,
emqx_schema_validation,
emqx_message_transformation,
emqx_connector_aggregator,
emqx_bridge_kafka,
emqx_bridge_pulsar,

94
apps/emqx_message_transformation/BSL.txt Normal file
View File

@ -0,0 +1,94 @@
Business Source License 1.1
Licensor: Hangzhou EMQ Technologies Co., Ltd.
Licensed Work: EMQX Enterprise Edition
The Licensed Work is (c) 2024
Hangzhou EMQ Technologies Co., Ltd.
Additional Use Grant: Students and educators are granted right to copy,
modify, and create derivative work for research
or education.
Change Date: 2028-06-05
Change License: Apache License, Version 2.0
For information about alternative licensing arrangements for the Software,
please contact Licensor: https://www.emqx.com/en/contact
Notice
The Business Source License (this document, or the “License”) is not an Open
Source license. However, the Licensed Work will eventually be made available
under an Open Source License, as stated in this License.
License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved.
“Business Source License” is a trademark of MariaDB Corporation Ab.
-----------------------------------------------------------------------------
Business Source License 1.1
Terms
The Licensor hereby grants you the right to copy, modify, create derivative
works, redistribute, and make non-production use of the Licensed Work. The
Licensor may make an Additional Use Grant, above, permitting limited
production use.
Effective on the Change Date, or the fourth anniversary of the first publicly
available distribution of a specific version of the Licensed Work under this
License, whichever comes first, the Licensor hereby grants you rights under
the terms of the Change License, and the rights granted in the paragraph
above terminate.
If your use of the Licensed Work does not comply with the requirements
currently in effect as described in this License, you must purchase a
commercial license from the Licensor, its affiliated entities, or authorized
resellers, or you must refrain from using the Licensed Work.
All copies of the original and modified Licensed Work, and derivative works
of the Licensed Work, are subject to this License. This License applies
separately for each version of the Licensed Work and the Change Date may vary
for each version of the Licensed Work released by Licensor.
You must conspicuously display this License on each original or modified copy
of the Licensed Work. If you receive the Licensed Work in original or
modified form from a third party, the terms and conditions set forth in this
License apply to your use of that work.
Any use of the Licensed Work in violation of this License will automatically
terminate your rights under this License for the current and all other
versions of the Licensed Work.
This License does not grant you any right in any trademark or logo of
Licensor or its affiliates (provided that you may use a trademark or logo of
Licensor as expressly required by this License).
TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON
AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS,
EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND
TITLE.
MariaDB hereby grants you permission to use this License’s text to license
your works, and to refer to it using the trademark “Business Source License”,
as long as you comply with the Covenants of Licensor below.
Covenants of Licensor
In consideration of the right to use this License’s text and the “Business
Source License” name and trademark, Licensor covenants to MariaDB, and to all
other recipients of the licensed work to be provided by Licensor:
1. To specify as the Change License the GPL Version 2.0 or any later version,
or a license that is compatible with GPL Version 2.0 or a later version,
where “compatible” means that software provided under the Change License can
be included in a program with software provided under GPL Version 2.0 or a
later version. Licensor may specify additional Change Licenses without
limitation.
2. To either: (a) specify an additional grant of rights to use that does not
impose any additional restriction on the right granted in this License, as
the Additional Use Grant; or (b) insert the text “None”.
3. To specify a Change Date.
4. Not to modify this License in any other way.

29
apps/emqx_message_transformation/README.md Normal file
View File

@ -0,0 +1,29 @@
# EMQX Message Transformation
This application encapsulates the functionality to transform incoming or internally
triggered published payloads and take an action upon failure, which can be to just drop
the message without further processing, or to disconnect the offending client as well.
# Documentation
Refer to [Message
Transformation](https://docs.emqx.com/en/enterprise/latest/data-integration/message-transformation.html)
for more information about the semantics.
# HTTP APIs
APIs are provided for transformation management, which includes creating,
updating, looking up, deleting, listing transformations.
Refer to [API Docs -
Bridges](https://docs.emqx.com/en/enterprise/latest/admin/api-docs.html#tag/Message-Transformation)
for more detailed information.
# Contributing
Please see our [contributing.md](../../CONTRIBUTING.md).
# License
EMQ Business Source License 1.1, refer to [LICENSE](BSL.txt).

15
apps/emqx_message_transformation/rebar.config Normal file
View File

@ -0,0 +1,15 @@
%% -*- mode: erlang -*-
{erl_opts, [
warn_unused_vars,
warn_shadow_vars,
warn_unused_import,
warn_obsolete_guard,
warnings_as_errors,
debug_info
]}.
{deps, [
{emqx, {path, "../emqx"}},
{emqx_utils, {path, "../emqx_utils"}},
{emqx_schema_registry, {path, "../emqx_schema_registry"}}
]}.

15
apps/emqx_message_transformation/src/emqx_message_transformation.app.src Normal file
View File

@ -0,0 +1,15 @@
{application, emqx_message_transformation, [
{description, "EMQX Message Transformation"},
{vsn, "0.1.0"},
{registered, [emqx_message_transformation_sup, emqx_message_transformation_registry]},
{mod, {emqx_message_transformation_app, []}},
{applications, [
kernel,
stdlib,
emqx
]},
{env, []},
{modules, []},
{links, []}
]}.

504
apps/emqx_message_transformation/src/emqx_message_transformation.erl Normal file
View File

@ -0,0 +1,504 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
-module(emqx_message_transformation).
-include_lib("snabbkaffe/include/trace.hrl").
-include_lib("emqx_utils/include/emqx_message.hrl").
-include_lib("emqx/include/emqx_hooks.hrl").
-include_lib("emqx/include/logger.hrl").
%% API
-export([
list/0,
reorder/1,
lookup/1,
insert/1,
update/1,
delete/1
]).
%% `emqx_hooks' API
-export([
register_hooks/0,
unregister_hooks/0,
on_message_publish/1
]).
%%------------------------------------------------------------------------------
%% Type declarations
%%------------------------------------------------------------------------------
-define(TRACE_TAG, "MESSAGE_TRANSFORMATION").
-define(CONF_ROOT, message_transformation).
-define(CONF_ROOT_BIN, <<"message_transformation">>).
-define(TRANSFORMATIONS_CONF_PATH, [?CONF_ROOT, transformations]).
-type transformation_name() :: binary().
%% TODO: make more specific typespec
-type transformation() :: #{atom() => term()}.
%% TODO: make more specific typespec
-type variform() :: any().
-type operation() :: #{key := [binary(), ...], value := variform()}.
-type qos() :: 0..2.
-type rendered_value() :: qos() | boolean() | binary().
-type eval_context() :: #{
client_attrs := map(),
payload := _,
qos := _,
retain := _,
topic := _,
user_property := _,
dirty := #{
payload => true,
qos => true,
retain => true,
topic => true,
user_property => true
}
}.
-export_type([
transformation/0,
transformation_name/0
]).
%%------------------------------------------------------------------------------
%% API
%%------------------------------------------------------------------------------
-spec list() -> [transformation()].
list() ->
emqx_message_transformation_config:list().
-spec reorder([transformation_name()]) ->
{ok, _} | {error, _}.
reorder(Order) ->
emqx_message_transformation_config:reorder(Order).
-spec lookup(transformation_name()) -> {ok, transformation()} | {error, not_found}.
lookup(Name) ->
emqx_message_transformation_config:lookup(Name).
-spec insert(transformation()) ->
{ok, _} | {error, _}.
insert(Transformation) ->
emqx_message_transformation_config:insert(Transformation).
-spec update(transformation()) ->
{ok, _} | {error, _}.
update(Transformation) ->
emqx_message_transformation_config:update(Transformation).
-spec delete(transformation_name()) ->
{ok, _} | {error, _}.
delete(Name) ->
emqx_message_transformation_config:delete(Name).
%%------------------------------------------------------------------------------
%% Hooks
%%------------------------------------------------------------------------------
-spec register_hooks() -> ok.
register_hooks() ->
emqx_hooks:put(
'message.publish', {?MODULE, on_message_publish, []}, ?HP_MESSAGE_TRANSFORMATION
).
-spec unregister_hooks() -> ok.
unregister_hooks() ->
emqx_hooks:del('message.publish', {?MODULE, on_message_publish}).
-spec on_message_publish(emqx_types:message()) ->
{ok, emqx_types:message()} | {stop, emqx_types:message()}.
on_message_publish(Message = #message{topic = Topic}) ->
case emqx_message_transformation_registry:matching_transformations(Topic) of
[] ->
ok;
Transformations ->
run_transformations(Transformations, Message)
end.
%%------------------------------------------------------------------------------
%% Internal exports
%%------------------------------------------------------------------------------
%%------------------------------------------------------------------------------
%% Internal functions
%%------------------------------------------------------------------------------
-spec eval_operation(operation(), transformation(), eval_context()) -> {ok, eval_context()} | error.
eval_operation(Operation, Transformation, Context) ->
#{key := K, value := V} = Operation,
case eval_variform(K, V, Context) of
{error, Reason} ->
trace_failure(Transformation, "transformation_eval_operation_failure", #{
reason => Reason
}),
error;
{ok, Rendered} ->
NewContext = put_value(K, Rendered, Context),
{ok, NewContext}
end.
-spec eval_variform([binary(), ...], _, eval_context()) ->
{ok, rendered_value()} | {error, term()}.
eval_variform(K, V, Context) ->
Opts =
case K of
[<<"payload">> | _] ->
#{eval_as_string => false};
_ ->
#{}
end,
case emqx_variform:render(V, Context, Opts) of
{error, Reason} ->
{error, Reason};
{ok, Rendered} ->
map_result(Rendered, K)
end.
-spec put_value([binary(), ...], rendered_value(), eval_context()) -> eval_context().
put_value([<<"payload">> | Rest], Rendered, Context0) ->
Context = maps:update_with(dirty, fun(D) -> D#{payload => true} end, Context0),
maps:update_with(
payload,
fun(P) ->
case Rest of
[] ->
Rendered;
_ ->
emqx_utils_maps:deep_put(Rest, P, Rendered)
end
end,
Context
);
put_value([<<"user_property">>, Key], Rendered, Context0) ->
Context = maps:update_with(dirty, fun(D) -> D#{user_property => true} end, Context0),
maps:update_with(
user_property,
fun(Ps) -> lists:keystore(Key, 1, Ps, {Key, Rendered}) end,
Context
);
put_value([<<"qos">>], Rendered, Context0) ->
Context = maps:update_with(dirty, fun(D) -> D#{qos => true} end, Context0),
Context#{qos := Rendered};
put_value([<<"retain">>], Rendered, Context0) ->
Context = maps:update_with(dirty, fun(D) -> D#{retain => true} end, Context0),
Context#{retain := Rendered};
put_value([<<"topic">>], Rendered, Context0) ->
Context = maps:update_with(dirty, fun(D) -> D#{topic => true} end, Context0),
Context#{topic := Rendered}.
-spec map_result(binary(), [binary(), ...]) ->
{ok, 0..2 | boolean() | binary()} | {error, map()}.
map_result(QoSBin, [<<"qos">>]) ->
case QoSBin of
<<"0">> -> {ok, 0};
<<"1">> -> {ok, 1};
<<"2">> -> {ok, 2};
_ -> {error, #{reason => bad_qos_value, input => QoSBin}}
end;
map_result(RetainBin, [<<"retain">>]) ->
case RetainBin of
<<"true">> -> {ok, true};
<<"false">> -> {ok, false};
_ -> {error, #{reason => bad_retain_value, input => RetainBin}}
end;
map_result(Rendered, _Key) ->
{ok, Rendered}.
run_transformations(Transformations, Message = #message{headers = Headers}) ->
case do_run_transformations(Transformations, Message) of
#message{} = FinalMessage ->
emqx_metrics:inc('messages.transformation_succeeded'),
{ok, FinalMessage};
drop ->
emqx_metrics:inc('messages.transformation_failed'),
{stop, Message#message{headers = Headers#{allow_publish => false}}};
disconnect ->
emqx_metrics:inc('messages.transformation_failed'),
{stop, Message#message{
headers = Headers#{
allow_publish => false,
should_disconnect => true
}
}}
end.
do_run_transformations(Transformations, Message) ->
Fun = fun(Transformation, MessageAcc) ->
#{name := Name} = Transformation,
emqx_message_transformation_registry:inc_matched(Name),
case run_transformation(Transformation, MessageAcc) of
#message{} = NewAcc ->
emqx_message_transformation_registry:inc_succeeded(Name),
{cont, NewAcc};
ignore ->
emqx_message_transformation_registry:inc_failed(Name),
run_message_transformation_failed_hook(Message, Transformation),
{cont, MessageAcc};
FailureAction ->
trace_failure(Transformation, "transformation_failed", #{
transformation => Name,
action => FailureAction
}),
emqx_message_transformation_registry:inc_failed(Name),
run_message_transformation_failed_hook(Message, Transformation),
{halt, FailureAction}
end
end,
case emqx_utils:foldl_while(Fun, Message, Transformations) of
#message{} = FinalMessage ->
case is_payload_properly_encoded(FinalMessage) of
true ->
FinalMessage;
false ->
%% Take the last validation's failure action, as it's the one
%% responsible for getting the right encoding.
LastTransformation = lists:last(Transformations),
#{failure_action := FailureAction} = LastTransformation,
trace_failure(LastTransformation, "transformation_bad_encoding", #{
action => FailureAction,
explain => <<"final payload must be encoded as a binary">>
}),
FailureAction
end;
FailureAction ->
FailureAction
end.
run_transformation(Transformation, MessageIn) ->
#{
operations := Operations,
failure_action := FailureAction,
payload_decoder := PayloadDecoder
} = Transformation,
Fun = fun(Operation, Acc) ->
case eval_operation(Operation, Transformation, Acc) of
{ok, NewAcc} -> {cont, NewAcc};
error -> {halt, FailureAction}
end
end,
PayloadIn = MessageIn#message.payload,
case decode(PayloadIn, PayloadDecoder, Transformation) of
{ok, InitPayload} ->
InitAcc = message_to_context(MessageIn, InitPayload, Transformation),
case emqx_utils:foldl_while(Fun, InitAcc, Operations) of
#{} = ContextOut ->
context_to_message(MessageIn, ContextOut, Transformation);
_ ->
FailureAction
end;
error ->
%% Error already logged
FailureAction
end.
-spec message_to_context(emqx_types:message(), _Payload, transformation()) -> eval_context().
message_to_context(#message{} = Message, Payload, Transformation) ->
#{
payload_decoder := PayloadDecoder,
payload_encoder := PayloadEncoder
} = Transformation,
Dirty =
case PayloadEncoder =:= PayloadDecoder of
true -> #{};
false -> #{payload => true}
end,
#{
dirty => Dirty,
client_attrs => emqx_message:get_header(client_attrs, Message, #{}),
payload => Payload,
qos => Message#message.qos,
retain => emqx_message:get_flag(retain, Message, false),
topic => Message#message.topic,
user_property => maps:get(
'User-Property', emqx_message:get_header(properties, Message, #{}), []
)
}.
-spec context_to_message(emqx_types:message(), eval_context(), transformation()) ->
{ok, emqx_types:message()} | _TODO.
context_to_message(Message, Context, Transformation) ->
#{
failure_action := FailureAction,
payload_encoder := PayloadEncoder
} = Transformation,
#{payload := PayloadOut} = Context,
case encode(PayloadOut, PayloadEncoder, Transformation) of
{ok, Payload} ->
take_from_context(Context#{payload := Payload}, Message);
error ->
FailureAction
end.
take_from_context(Context, Message) ->
maps:fold(
fun
(payload, _, Acc) ->
Acc#message{payload = maps:get(payload, Context)};
(qos, _, Acc) ->
Acc#message{qos = maps:get(qos, Context)};
(topic, _, Acc) ->
Acc#message{topic = maps:get(topic, Context)};
(retain, _, Acc) ->
emqx_message:set_flag(retain, maps:get(retain, Context), Acc);
(user_property, _, Acc) ->
Props0 = emqx_message:get_header(properties, Acc, #{}),
Props = maps:merge(Props0, #{'User-Property' => maps:get(user_property, Context)}),
emqx_message:set_header(properties, Props, Acc)
end,
Message,
maps:get(dirty, Context)
).
decode(Payload, #{type := none}, _Transformation) ->
{ok, Payload};
decode(Payload, #{type := json}, Transformation) ->
case emqx_utils_json:safe_decode(Payload, [return_maps]) of
{ok, JSON} ->
{ok, JSON};
{error, Reason} ->
trace_failure(Transformation, "payload_decode_failed", #{
decoder => json,
reason => Reason
}),
error
end;
decode(Payload, #{type := avro, schema := SerdeName}, Transformation) ->
try
{ok, emqx_schema_registry_serde:decode(SerdeName, Payload)}
catch
error:{serde_not_found, _} ->
trace_failure(Transformation, "payload_decode_schema_not_found", #{
decoder => avro,
schema_name => SerdeName
}),
error;
Class:Error:Stacktrace ->
trace_failure(Transformation, "payload_decode_schema_failure", #{
decoder => avro,
schema_name => SerdeName,
kind => Class,
reason => Error,
stacktrace => Stacktrace
}),
error
end;
decode(
Payload, #{type := protobuf, schema := SerdeName, message_type := MessageType}, Transformation
) ->
try
{ok, emqx_schema_registry_serde:decode(SerdeName, Payload, [MessageType])}
catch
error:{serde_not_found, _} ->
trace_failure(Transformation, "payload_decode_schema_not_found", #{
decoder => protobuf,
schema_name => SerdeName,
message_type => MessageType
}),
error;
Class:Error:Stacktrace ->
trace_failure(Transformation, "payload_decode_schema_failure", #{
decoder => protobuf,
schema_name => SerdeName,
message_type => MessageType,
kind => Class,
reason => Error,
stacktrace => Stacktrace
}),
error
end.
encode(Payload, #{type := none}, _Transformation) ->
{ok, Payload};
encode(Payload, #{type := json}, Transformation) ->
case emqx_utils_json:safe_encode(Payload) of
{ok, Bin} ->
{ok, Bin};
{error, Reason} ->
trace_failure(Transformation, "payload_encode_failed", #{
encoder => json,
reason => Reason
}),
error
end;
encode(Payload, #{type := avro, schema := SerdeName}, Transformation) ->
try
{ok, emqx_schema_registry_serde:encode(SerdeName, Payload)}
catch
error:{serde_not_found, _} ->
trace_failure(Transformation, "payload_encode_schema_not_found", #{
encoder => avro,
schema_name => SerdeName
}),
error;
Class:Error:Stacktrace ->
trace_failure(Transformation, "payload_encode_schema_failure", #{
encoder => avro,
schema_name => SerdeName,
kind => Class,
reason => Error,
stacktrace => Stacktrace
}),
error
end;
encode(
Payload, #{type := protobuf, schema := SerdeName, message_type := MessageType}, Transformation
) ->
try
{ok, emqx_schema_registry_serde:encode(SerdeName, Payload, [MessageType])}
catch
error:{serde_not_found, _} ->
trace_failure(Transformation, "payload_encode_schema_not_found", #{
encoder => protobuf,
schema_name => SerdeName,
message_type => MessageType
}),
error;
Class:Error:Stacktrace ->
trace_failure(Transformation, "payload_encode_schema_failure", #{
encoder => protobuf,
schema_name => SerdeName,
message_type => MessageType,
kind => Class,
reason => Error,
stacktrace => Stacktrace
}),
error
end.
trace_failure(#{log_failure := #{level := none}} = Transformation, _Msg, _Meta) ->
#{
name := _Name,
failure_action := _Action
} = Transformation,
?tp(message_transformation_failed, _Meta#{log_level => none, name => _Name, message => _Msg}),
ok;
trace_failure(#{log_failure := #{level := Level}} = Transformation, Msg, Meta0) ->
#{
name := Name,
failure_action := _Action
} = Transformation,
Meta = maps:merge(#{name => Name}, Meta0),
?tp(message_transformation_failed, Meta#{
log_level => Level, name => Name, action => _Action, message => Msg
}),
?TRACE(Level, ?TRACE_TAG, Msg, Meta).
run_message_transformation_failed_hook(Message, Transformation) ->
#{name := Name} = Transformation,
TransformationContext = #{name => Name},
emqx_hooks:run('message.transformation_failed', [Message, TransformationContext]).
is_payload_properly_encoded(#message{payload = Payload}) ->
try iolist_size(Payload) of
_ ->
true
catch
error:badarg ->
false
end.

34
apps/emqx_message_transformation/src/emqx_message_transformation_app.erl Normal file
View File

@ -0,0 +1,34 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
-module(emqx_message_transformation_app).
-behaviour(application).
%% `application' API
-export([start/2, stop/1]).
%%------------------------------------------------------------------------------
%% Type declarations
%%------------------------------------------------------------------------------
%%------------------------------------------------------------------------------
%% `application' API
%%------------------------------------------------------------------------------
-spec start(application:start_type(), term()) -> {ok, pid()}.
start(_Type, _Args) ->
{ok, Sup} = emqx_message_transformation_sup:start_link(),
ok = emqx_variform:inject_allowed_module(emqx_message_transformation_bif),
ok = emqx_message_transformation_config:add_handler(),
ok = emqx_message_transformation:register_hooks(),
ok = emqx_message_transformation_config:load(),
{ok, Sup}.
-spec stop(term()) -> ok.
stop(_State) ->
ok = emqx_message_transformation_config:unload(),
ok = emqx_message_transformation:unregister_hooks(),
ok = emqx_message_transformation_config:remove_handler(),
ok = emqx_variform:erase_allowed_module(emqx_message_transformation_bif),
ok.

38
apps/emqx_message_transformation/src/emqx_message_transformation_bif.erl Normal file
View File

@ -0,0 +1,38 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
-module(emqx_message_transformation_bif).
%% API
-export([
json_decode/1,
json_encode/1
]).
%%------------------------------------------------------------------------------
%% Type declarations
%%------------------------------------------------------------------------------
%%------------------------------------------------------------------------------
%% API
%%------------------------------------------------------------------------------
json_encode(X) ->
case emqx_utils_json:safe_encode(X) of
{ok, JSON} ->
JSON;
{error, Reason} ->
throw(#{reason => json_encode_failure, detail => Reason})
end.
json_decode(JSON) ->
case emqx_utils_json:safe_decode(JSON, [return_maps]) of
{ok, X} ->
X;
{error, Reason} ->
throw(#{reason => json_decode_failure, detail => Reason})
end.
%%------------------------------------------------------------------------------
%% Internal fns
%%------------------------------------------------------------------------------

395
apps/emqx_message_transformation/src/emqx_message_transformation_config.erl Normal file
View File

@ -0,0 +1,395 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
-module(emqx_message_transformation_config).
%% API
-export([
add_handler/0,
remove_handler/0,
load/0,
unload/0,
list/0,
reorder/1,
lookup/1,
insert/1,
update/1,
delete/1
]).
%% `emqx_config_handler' API
-export([pre_config_update/3, post_config_update/5]).
%% `emqx_config_backup' API
-behaviour(emqx_config_backup).
-export([import_config/1]).
%%------------------------------------------------------------------------------
%% Type declarations
%%------------------------------------------------------------------------------
-define(CONF_ROOT, message_transformation).
-define(CONF_ROOT_BIN, <<"message_transformation">>).
-define(TRANSFORMATIONS_CONF_PATH, [?CONF_ROOT, transformations]).
-type transformation_name() :: emqx_message_transformation:transformation_name().
-type transformation() :: emqx_message_transformation:transformation().
%%------------------------------------------------------------------------------
%% API
%%------------------------------------------------------------------------------
-spec add_handler() -> ok.
add_handler() ->
ok = emqx_config_handler:add_handler([?CONF_ROOT], ?MODULE),
ok = emqx_config_handler:add_handler(?TRANSFORMATIONS_CONF_PATH, ?MODULE),
ok.
-spec remove_handler() -> ok.
remove_handler() ->
ok = emqx_config_handler:remove_handler(?TRANSFORMATIONS_CONF_PATH),
ok = emqx_config_handler:remove_handler([?CONF_ROOT]),
ok.
load() ->
Transformations = emqx:get_config(?TRANSFORMATIONS_CONF_PATH, []),
lists:foreach(
fun({Pos, Transformation}) ->
ok = emqx_message_transformation_registry:insert(Pos, Transformation)
end,
lists:enumerate(Transformations)
).
unload() ->
Transformations = emqx:get_config(?TRANSFORMATIONS_CONF_PATH, []),
lists:foreach(
fun({Pos, Transformation}) ->
ok = emqx_message_transformation_registry:delete(Transformation, Pos)
end,
lists:enumerate(Transformations)
).
-spec list() -> [transformation()].
list() ->
emqx:get_config(?TRANSFORMATIONS_CONF_PATH, []).
-spec reorder([transformation_name()]) ->
{ok, _} | {error, _}.
reorder(Order) ->
emqx_conf:update(
?TRANSFORMATIONS_CONF_PATH,
{reorder, Order},
#{override_to => cluster}
).
-spec lookup(transformation_name()) -> {ok, transformation()} | {error, not_found}.
lookup(Name) ->
Transformations = emqx:get_config(?TRANSFORMATIONS_CONF_PATH, []),
do_lookup(Name, Transformations).
-spec insert(transformation()) ->
{ok, _} | {error, _}.
insert(Transformation) ->
emqx_conf:update(
?TRANSFORMATIONS_CONF_PATH,
{append, Transformation},
#{override_to => cluster}
).
-spec update(transformation()) ->
{ok, _} | {error, _}.
update(Transformation) ->
emqx_conf:update(
?TRANSFORMATIONS_CONF_PATH,
{update, Transformation},
#{override_to => cluster}
).
-spec delete(transformation_name()) ->
{ok, _} | {error, _}.
delete(Name) ->
emqx_conf:update(
?TRANSFORMATIONS_CONF_PATH,
{delete, Name},
#{override_to => cluster}
).
%%------------------------------------------------------------------------------
%% `emqx_config_handler' API
%%------------------------------------------------------------------------------
pre_config_update(?TRANSFORMATIONS_CONF_PATH, {append, Transformation}, OldTransformations) ->
Transformations = OldTransformations ++ [Transformation],
{ok, Transformations};
pre_config_update(?TRANSFORMATIONS_CONF_PATH, {update, Transformation}, OldTransformations) ->
replace(OldTransformations, Transformation);
pre_config_update(?TRANSFORMATIONS_CONF_PATH, {delete, Transformation}, OldTransformations) ->
delete(OldTransformations, Transformation);
pre_config_update(?TRANSFORMATIONS_CONF_PATH, {reorder, Order}, OldTransformations) ->
reorder(OldTransformations, Order);
pre_config_update([?CONF_ROOT], {merge, NewConfig}, OldConfig) ->
#{resulting_config := Config} = prepare_config_merge(NewConfig, OldConfig),
{ok, Config};
pre_config_update([?CONF_ROOT], {replace, NewConfig}, _OldConfig) ->
{ok, NewConfig}.
post_config_update(
?TRANSFORMATIONS_CONF_PATH, {append, #{<<"name">> := Name}}, New, _Old, _AppEnvs
) ->
{Pos, Transformation} = fetch_with_index(New, Name),
ok = emqx_message_transformation_registry:insert(Pos, Transformation),
ok;
post_config_update(?TRANSFORMATIONS_CONF_PATH, {update, #{<<"name">> := Name}}, New, Old, _AppEnvs) ->
{_Pos, OldTransformation} = fetch_with_index(Old, Name),
{Pos, NewTransformation} = fetch_with_index(New, Name),
ok = emqx_message_transformation_registry:update(OldTransformation, Pos, NewTransformation),
ok;
post_config_update(?TRANSFORMATIONS_CONF_PATH, {delete, Name}, _New, Old, _AppEnvs) ->
{Pos, Transformation} = fetch_with_index(Old, Name),
ok = emqx_message_transformation_registry:delete(Transformation, Pos),
ok;
post_config_update(?TRANSFORMATIONS_CONF_PATH, {reorder, _Order}, New, Old, _AppEnvs) ->
ok = emqx_message_transformation_registry:reindex_positions(New, Old),
ok;
post_config_update([?CONF_ROOT], {merge, _}, ResultingConfig, Old, _AppEnvs) ->
#{transformations := ResultingTransformations} = ResultingConfig,
#{transformations := OldTransformations} = Old,
#{added := NewTransformations0} =
emqx_utils:diff_lists(
ResultingTransformations,
OldTransformations,
fun(#{name := N}) -> N end
),
NewTransformations =
lists:map(
fun(#{name := Name}) ->
{Pos, Transformation} = fetch_with_index(ResultingTransformations, Name),
ok = emqx_message_transformation_registry:insert(Pos, Transformation),
#{name => Name, pos => Pos}
end,
NewTransformations0
),
{ok, #{new_transformations => NewTransformations}};
post_config_update([?CONF_ROOT], {replace, Input}, ResultingConfig, Old, _AppEnvs) ->
#{
new_transformations := NewTransformations,
changed_transformations := ChangedTransformations0,
deleted_transformations := DeletedTransformations
} = prepare_config_replace(Input, Old),
#{transformations := ResultingTransformations} = ResultingConfig,
#{transformations := OldTransformations} = Old,
lists:foreach(
fun(Name) ->
{Pos, Transformation} = fetch_with_index(OldTransformations, Name),
ok = emqx_message_transformation_registry:delete(Transformation, Pos)
end,
DeletedTransformations
),
lists:foreach(
fun(Name) ->
{Pos, Transformation} = fetch_with_index(ResultingTransformations, Name),
ok = emqx_message_transformation_registry:insert(Pos, Transformation)
end,
NewTransformations
),
ChangedTransformations =
lists:map(
fun(Name) ->
{_Pos, OldTransformation} = fetch_with_index(OldTransformations, Name),
{Pos, NewTransformation} = fetch_with_index(ResultingTransformations, Name),
ok = emqx_message_transformation_registry:update(
OldTransformation, Pos, NewTransformation
),
#{name => Name, pos => Pos}
end,
ChangedTransformations0
),
ok = emqx_message_transformation_registry:reindex_positions(
ResultingTransformations, OldTransformations
),
{ok, #{changed_transformations => ChangedTransformations}}.
%%------------------------------------------------------------------------------
%% `emqx_config_backup' API
%%------------------------------------------------------------------------------
import_config(#{?CONF_ROOT_BIN := RawConf0}) ->
Result = emqx_conf:update(
[?CONF_ROOT],
{merge, RawConf0},
#{override_to => cluster, rawconf_with_defaults => true}
),
case Result of
{error, Reason} ->
{error, #{root_key => ?CONF_ROOT, reason => Reason}};
{ok, _} ->
Keys0 = maps:keys(RawConf0),
ChangedPaths = Keys0 -- [<<"transformations">>],
{ok, #{root_key => ?CONF_ROOT, changed => ChangedPaths}}
end;
import_config(_RawConf) ->
{ok, #{root_key => ?CONF_ROOT, changed => []}}.
%%------------------------------------------------------------------------------
%% Internal fns
%%------------------------------------------------------------------------------
replace(OldTransformations, Transformation = #{<<"name">> := Name}) ->
{Found, RevNewTransformations} =
lists:foldl(
fun
(#{<<"name">> := NameIn}, {_FoundIn, Acc}) when NameIn =:= Name ->
{true, [Transformation | Acc]};
(Val, {FoundIn, Acc}) ->
{FoundIn, [Val | Acc]}
end,
{false, []},
OldTransformations
),
case Found of
true ->
{ok, lists:reverse(RevNewTransformations)};
false ->
{error, not_found}
end.
delete(OldTransformations, Name) ->
{Found, RevNewTransformations} =
lists:foldl(
fun
(#{<<"name">> := NameIn}, {_FoundIn, Acc}) when NameIn =:= Name ->
{true, Acc};
(Val, {FoundIn, Acc}) ->
{FoundIn, [Val | Acc]}
end,
{false, []},
OldTransformations
),
case Found of
true ->
{ok, lists:reverse(RevNewTransformations)};
false ->
{error, not_found}
end.
reorder(Transformations, Order) ->
Context = #{
not_found => sets:new([{version, 2}]),
duplicated => sets:new([{version, 2}]),
res => [],
seen => sets:new([{version, 2}])
},
reorder(Transformations, Order, Context).
reorder(NotReordered, _Order = [], #{not_found := NotFound0, duplicated := Duplicated0, res := Res}) ->
NotFound = sets:to_list(NotFound0),
Duplicated = sets:to_list(Duplicated0),
case {NotReordered, NotFound, Duplicated} of
{[], [], []} ->
{ok, lists:reverse(Res)};
{_, _, _} ->
Error = #{
not_found => NotFound,
duplicated => Duplicated,
not_reordered => [N || #{<<"name">> := N} <- NotReordered]
},
{error, Error}
end;
reorder(RemainingTransformations, [Name | Rest], Context0 = #{seen := Seen0}) ->
case sets:is_element(Name, Seen0) of
true ->
Context = maps:update_with(
duplicated, fun(S) -> sets:add_element(Name, S) end, Context0
),
reorder(RemainingTransformations, Rest, Context);
false ->
case safe_take(Name, RemainingTransformations) of
error ->
Context = maps:update_with(
not_found, fun(S) -> sets:add_element(Name, S) end, Context0
),
reorder(RemainingTransformations, Rest, Context);
{ok, {Transformation, Front, Rear}} ->
Context1 = maps:update_with(
seen, fun(S) -> sets:add_element(Name, S) end, Context0
),
Context = maps:update_with(res, fun(Vs) -> [Transformation | Vs] end, Context1),
reorder(Front ++ Rear, Rest, Context)
end
end.
fetch_with_index([{Pos, #{name := Name} = Transformation} | _Rest], Name) ->
{Pos, Transformation};
fetch_with_index([{_, _} | Rest], Name) ->
fetch_with_index(Rest, Name);
fetch_with_index(Transformations, Name) ->
fetch_with_index(lists:enumerate(Transformations), Name).
safe_take(Name, Transformations) ->
case lists:splitwith(fun(#{<<"name">> := N}) -> N =/= Name end, Transformations) of
{_Front, []} ->
error;
{Front, [Found | Rear]} ->
{ok, {Found, Front, Rear}}
end.
do_lookup(_Name, _Transformations = []) ->
{error, not_found};
do_lookup(Name, [#{name := Name} = Transformation | _Rest]) ->
{ok, Transformation};
do_lookup(Name, [_ | Rest]) ->
do_lookup(Name, Rest).
%% "Merging" in the context of the transformation array means:
%% * Existing transformations (identified by `name') are left untouched.
%% * No transformations are removed.
%% * New transformations are appended to the existing list.
%% * Existing transformations are not reordered.
prepare_config_merge(NewConfig0, OldConfig) ->
{ImportedRawTransformations, NewConfigNoTransformations} =
case maps:take(<<"transformations">>, NewConfig0) of
error ->
{[], NewConfig0};
{V, R} ->
{V, R}
end,
OldRawTransformations = maps:get(<<"transformations">>, OldConfig, []),
#{added := NewRawTransformations} = emqx_utils:diff_lists(
ImportedRawTransformations,
OldRawTransformations,
fun(#{<<"name">> := N}) -> N end
),
Config0 = emqx_utils_maps:deep_merge(OldConfig, NewConfigNoTransformations),
Config = maps:update_with(
<<"transformations">>,
fun(OldVs) -> OldVs ++ NewRawTransformations end,
NewRawTransformations,
Config0
),
#{
new_transformations => NewRawTransformations,
resulting_config => Config
}.
prepare_config_replace(NewConfig, OldConfig) ->
ImportedRawTransformations = maps:get(<<"transformations">>, NewConfig, []),
OldTransformations = maps:get(transformations, OldConfig, []),
%% Since, at this point, we have an input raw config but a parsed old config, we
%% project both to the to have only their names, and consider common names as changed.
#{
added := NewTransformations,
removed := DeletedTransformations,
changed := ChangedTransformations0,
identical := ChangedTransformations1
} = emqx_utils:diff_lists(
lists:map(fun(#{<<"name">> := N}) -> N end, ImportedRawTransformations),
lists:map(fun(#{name := N}) -> N end, OldTransformations),
fun(N) -> N end
),
#{
new_transformations => NewTransformations,
changed_transformations => ChangedTransformations0 ++ ChangedTransformations1,
deleted_transformations => DeletedTransformations
}.

656
apps/emqx_message_transformation/src/emqx_message_transformation_http_api.erl Normal file
View File

@ -0,0 +1,656 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
-module(emqx_message_transformation_http_api).
-behaviour(minirest_api).
-include_lib("typerefl/include/types.hrl").
-include_lib("hocon/include/hoconsc.hrl").
-include_lib("emqx/include/logger.hrl").
-include_lib("emqx_utils/include/emqx_utils_api.hrl").
%% `minirest' and `minirest_trails' API
-export([
namespace/0,
api_spec/0,
fields/1,
paths/0,
schema/1
]).
%% `minirest' handlers
-export([
'/message_transformations'/2,
'/message_transformations/reorder'/2,
'/message_transformations/transformation/:name'/2,
'/message_transformations/transformation/:name/metrics'/2,
'/message_transformations/transformation/:name/metrics/reset'/2,
'/message_transformations/transformation/:name/enable/:enable'/2
]).
%%-------------------------------------------------------------------------------------------------
%% Type definitions
%%-------------------------------------------------------------------------------------------------
-define(TAGS, [<<"Message Transformation">>]).
-define(METRIC_NAME, message_transformation).
%%-------------------------------------------------------------------------------------------------
%% `minirest' and `minirest_trails' API
%%-------------------------------------------------------------------------------------------------
namespace() -> "message_transformation_http_api".
api_spec() ->
emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}).
paths() ->
[
"/message_transformations",
"/message_transformations/reorder",
"/message_transformations/transformation/:name",
"/message_transformations/transformation/:name/metrics",
"/message_transformations/transformation/:name/metrics/reset",
"/message_transformations/transformation/:name/enable/:enable"
].
schema("/message_transformations") ->
#{
'operationId' => '/message_transformations',
get => #{
tags => ?TAGS,
summary => <<"List transformations">>,
description => ?DESC("list_transformations"),
responses =>
#{
200 =>
emqx_dashboard_swagger:schema_with_examples(
array(
emqx_message_transformation_schema:api_schema(list)
),
example_return_list()
)
}
},
post => #{
tags => ?TAGS,
summary => <<"Append a new transformation">>,
description => ?DESC("append_transformation"),
'requestBody' => emqx_dashboard_swagger:schema_with_examples(
emqx_message_transformation_schema:api_schema(post),
example_input_create()
),
responses =>
#{
201 =>
emqx_dashboard_swagger:schema_with_examples(
emqx_message_transformation_schema:api_schema(post),
example_return_create()
),
400 => error_schema('ALREADY_EXISTS', "Transformation already exists")
}
},
put => #{
tags => ?TAGS,
summary => <<"Update a transformation">>,
description => ?DESC("update_transformation"),
'requestBody' => emqx_dashboard_swagger:schema_with_examples(
emqx_message_transformation_schema:api_schema(put),
example_input_update()
),
responses =>
#{
200 =>
emqx_dashboard_swagger:schema_with_examples(
emqx_message_transformation_schema:api_schema(put),
example_return_update()
),
404 => error_schema('NOT_FOUND', "Transformation not found"),
400 => error_schema('BAD_REQUEST', "Bad params")
}
}
};
schema("/message_transformations/reorder") ->
#{
'operationId' => '/message_transformations/reorder',
post => #{
tags => ?TAGS,
summary => <<"Reorder all transformations">>,
description => ?DESC("reorder_transformations"),
'requestBody' =>
emqx_dashboard_swagger:schema_with_examples(
ref(reorder),
example_input_reorder()
),
responses =>
#{
204 => <<"No Content">>,
400 => error_schema(
'BAD_REQUEST',
<<"Bad request">>,
[
{not_found,
mk(array(binary()), #{desc => "Transformations not found"})},
{not_reordered,
mk(array(binary()), #{
desc => "Transformations not referenced in input"
})},
{duplicated,
mk(array(binary()), #{desc => "Duplicated transformations in input"})}
]
)
}
}
};
schema("/message_transformations/transformation/:name") ->
#{
'operationId' => '/message_transformations/transformation/:name',
get => #{
tags => ?TAGS,
summary => <<"Lookup a transformation">>,
description => ?DESC("lookup_transformation"),
parameters => [param_path_name()],
responses =>
#{
200 =>
emqx_dashboard_swagger:schema_with_examples(
array(
emqx_message_transformation_schema:api_schema(lookup)
),
example_return_lookup()
),
404 => error_schema('NOT_FOUND', "Transformation not found")
}
},
delete => #{
tags => ?TAGS,
summary => <<"Delete a transformation">>,
description => ?DESC("delete_transformation"),
parameters => [param_path_name()],
responses =>
#{
204 => <<"Transformation deleted">>,
404 => error_schema('NOT_FOUND', "Transformation not found")
}
}
};
schema("/message_transformations/transformation/:name/metrics") ->
#{
'operationId' => '/message_transformations/transformation/:name/metrics',
get => #{
tags => ?TAGS,
summary => <<"Get transformation metrics">>,
description => ?DESC("get_transformation_metrics"),
parameters => [param_path_name()],
responses =>
#{
200 =>
emqx_dashboard_swagger:schema_with_examples(
ref(get_metrics),
example_return_metrics()
),
404 => error_schema('NOT_FOUND', "Transformation not found")
}
}
};
schema("/message_transformations/transformation/:name/metrics/reset") ->
#{
'operationId' => '/message_transformations/transformation/:name/metrics/reset',
post => #{
tags => ?TAGS,
summary => <<"Reset transformation metrics">>,
description => ?DESC("reset_transformation_metrics"),
parameters => [param_path_name()],
responses =>
#{
204 => <<"No content">>,
404 => error_schema('NOT_FOUND', "Transformation not found")
}
}
};
schema("/message_transformations/transformation/:name/enable/:enable") ->
#{
'operationId' => '/message_transformations/transformation/:name/enable/:enable',
post => #{
tags => ?TAGS,
summary => <<"Enable or disable transformation">>,
description => ?DESC("enable_disable_transformation"),
parameters => [param_path_name(), param_path_enable()],
responses =>
#{
204 => <<"No content">>,
404 => error_schema('NOT_FOUND', "Transformation not found"),
400 => error_schema('BAD_REQUEST', "Bad params")
}
}
}.
param_path_name() ->
{name,
mk(
binary(),
#{
in => path,
required => true,
example => <<"my_transformation">>,
desc => ?DESC("param_path_name")
}
)}.
param_path_enable() ->
{enable,
mk(
boolean(),
#{
in => path,
required => true,
desc => ?DESC("param_path_enable")
}
)}.
fields(front) ->
[{position, mk(front, #{default => front, required => true, in => body})}];
fields(rear) ->
[{position, mk(rear, #{default => rear, required => true, in => body})}];
fields('after') ->
[
{position, mk('after', #{default => 'after', required => true, in => body})},
{transformation, mk(binary(), #{required => true, in => body})}
];
fields(before) ->
[
{position, mk(before, #{default => before, required => true, in => body})},
{transformation, mk(binary(), #{required => true, in => body})}
];
fields(reorder) ->
[
{order, mk(array(binary()), #{required => true, in => body})}
];
fields(get_metrics) ->
[
{metrics, mk(ref(metrics), #{})},
{node_metrics, mk(ref(node_metrics), #{})}
];
fields(metrics) ->
[
{matched, mk(non_neg_integer(), #{})},
{succeeded, mk(non_neg_integer(), #{})},
{failed, mk(non_neg_integer(), #{})}
];
fields(node_metrics) ->
[
{node, mk(binary(), #{})}
| fields(metrics)
].
%%-------------------------------------------------------------------------------------------------
%% `minirest' handlers
%%-------------------------------------------------------------------------------------------------
'/message_transformations'(get, _Params) ->
Transformations = emqx_message_transformation:list(),
?OK(lists:map(fun transformation_out/1, Transformations));
'/message_transformations'(post, #{body := Params = #{<<"name">> := Name}}) ->
with_transformation(
Name,
return(?BAD_REQUEST('ALREADY_EXISTS', <<"Transformation already exists">>)),
fun() ->
case emqx_message_transformation:insert(Params) of
{ok, _} ->
{ok, Res} = emqx_message_transformation:lookup(Name),
{201, transformation_out(Res)};
{error, Error} ->
?BAD_REQUEST(Error)
end
end
);
'/message_transformations'(put, #{body := Params = #{<<"name">> := Name}}) ->
with_transformation(
Name,
fun() ->
case emqx_message_transformation:update(Params) of
{ok, _} ->
{ok, Res} = emqx_message_transformation:lookup(Name),
{200, transformation_out(Res)};
{error, Error} ->
?BAD_REQUEST(Error)
end
end,
not_found()
).
'/message_transformations/transformation/:name'(get, #{bindings := #{name := Name}}) ->
with_transformation(
Name,
fun(Transformation) -> ?OK(transformation_out(Transformation)) end,
not_found()
);
'/message_transformations/transformation/:name'(delete, #{bindings := #{name := Name}}) ->
with_transformation(
Name,
fun() ->
case emqx_message_transformation:delete(Name) of
{ok, _} ->
?NO_CONTENT;
{error, Error} ->
?BAD_REQUEST(Error)
end
end,
not_found()
).
'/message_transformations/reorder'(post, #{body := #{<<"order">> := Order}}) ->
do_reorder(Order).
'/message_transformations/transformation/:name/enable/:enable'(post, #{
bindings := #{name := Name, enable := Enable}
}) ->
with_transformation(
Name,
fun(Transformation) -> do_enable_disable(Transformation, Enable) end,
not_found()
).
'/message_transformations/transformation/:name/metrics'(get, #{bindings := #{name := Name}}) ->
with_transformation(
Name,
fun() ->
Nodes = emqx:running_nodes(),
Results = emqx_metrics_proto_v2:get_metrics(Nodes, ?METRIC_NAME, Name, 5_000),
NodeResults = lists:zip(Nodes, Results),
NodeErrors = [Result || Result = {_Node, {NOk, _}} <- NodeResults, NOk =/= ok],
NodeErrors == [] orelse
?SLOG(warning, #{
msg => "rpc_get_transformation_metrics_errors",
errors => NodeErrors
}),
NodeMetrics = [format_metrics(Node, Metrics) || {Node, {ok, Metrics}} <- NodeResults],
Response = #{
metrics => aggregate_metrics(NodeMetrics),
node_metrics => NodeMetrics
},
?OK(Response)
end,
not_found()
).
'/message_transformations/transformation/:name/metrics/reset'(post, #{bindings := #{name := Name}}) ->
with_transformation(
Name,
fun() ->
Nodes = emqx:running_nodes(),
Results = emqx_metrics_proto_v2:reset_metrics(Nodes, ?METRIC_NAME, Name, 5_000),
NodeResults = lists:zip(Nodes, Results),
NodeErrors = [Result || Result = {_Node, {NOk, _}} <- NodeResults, NOk =/= ok],
NodeErrors == [] orelse
?SLOG(warning, #{
msg => "rpc_reset_transformation_metrics_errors",
errors => NodeErrors
}),
?NO_CONTENT
end,
not_found()
).
%%-------------------------------------------------------------------------------------------------
%% Internal fns
%%-------------------------------------------------------------------------------------------------
ref(Struct) -> hoconsc:ref(?MODULE, Struct).
mk(Type, Opts) -> hoconsc:mk(Type, Opts).
array(Type) -> hoconsc:array(Type).
%% FIXME: all examples
example_input_create() ->
#{
<<"sql_check">> =>
#{
summary => <<"Using a SQL check">>,
value => example_transformation([example_sql_check()])
},
<<"avro_check">> =>
#{
summary => <<"Using an Avro schema check">>,
value => example_transformation([example_avro_check()])
}
}.
example_input_update() ->
#{
<<"update">> =>
#{
summary => <<"Update">>,
value => example_transformation([example_sql_check()])
}
}.
example_input_reorder() ->
#{
<<"reorder">> =>
#{
summary => <<"Update">>,
value => #{
order => [<<"bar">>, <<"foo">>, <<"baz">>]
}
}
}.
example_return_list() ->
OtherVal0 = example_transformation([example_avro_check()]),
OtherVal = OtherVal0#{name => <<"other_transformation">>},
#{
<<"list">> =>
#{
summary => <<"List">>,
value => [
example_transformation([example_sql_check()]),
OtherVal
]
}
}.
example_return_create() ->
example_input_create().
example_return_update() ->
example_input_update().
example_return_lookup() ->
example_input_create().
example_return_metrics() ->
Metrics = #{
matched => 2,
succeeded => 1,
failed => 1,
rate => 1.23,
rate_last5m => 0.88,
rate_max => 1.87
},
#{
<<"metrics">> =>
#{
summary => <<"Metrics">>,
value => #{
metrics => Metrics,
node_metrics =>
[
#{
node => <<"emqx@127.0.0.1">>,
metrics => Metrics
}
]
}
}
}.
example_transformation(Checks) ->
#{
name => <<"my_transformation">>,
enable => true,
description => <<"my transformation">>,
tags => [<<"transformation">>],
topics => [<<"t/+">>],
strategy => <<"all_pass">>,
failure_action => <<"drop">>,
log_failure => #{<<"level">> => <<"info">>},
checks => Checks
}.
example_sql_check() ->
#{
type => <<"sql">>,
sql => <<"select payload.temp as t where t > 10">>
}.
example_avro_check() ->
#{
type => <<"avro">>,
schema => <<"my_avro_schema">>
}.
error_schema(Code, Message) ->
error_schema(Code, Message, _ExtraFields = []).
error_schema(Code, Message, ExtraFields) when is_atom(Code) ->
error_schema([Code], Message, ExtraFields);
error_schema(Codes, Message, ExtraFields) when is_list(Message) ->
error_schema(Codes, list_to_binary(Message), ExtraFields);
error_schema(Codes, Message, ExtraFields) when is_list(Codes) andalso is_binary(Message) ->
ExtraFields ++ emqx_dashboard_swagger:error_codes(Codes, Message).
do_reorder(Order) ->
case emqx_message_transformation:reorder(Order) of
{ok, _} ->
?NO_CONTENT;
{error,
{pre_config_update, _HandlerMod, #{
not_found := NotFound,
duplicated := Duplicated,
not_reordered := NotReordered
}}} ->
Msg0 = ?ERROR_MSG('BAD_REQUEST', <<"Bad request">>),
Msg = Msg0#{
not_found => NotFound,
duplicated => Duplicated,
not_reordered => NotReordered
},
{400, Msg};
{error, Error} ->
?BAD_REQUEST(Error)
end.
do_enable_disable(Transformation, Enable) ->
RawTransformation = make_serializable(Transformation),
case emqx_message_transformation:update(RawTransformation#{<<"enable">> => Enable}) of
{ok, _} ->
?NO_CONTENT;
{error, Reason} ->
?BAD_REQUEST(Reason)
end.
with_transformation(Name, FoundFn, NotFoundFn) ->
case emqx_message_transformation:lookup(Name) of
{ok, Transformation} ->
{arity, Arity} = erlang:fun_info(FoundFn, arity),
case Arity of
1 -> FoundFn(Transformation);
0 -> FoundFn()
end;
{error, not_found} ->
NotFoundFn()
end.
return(Response) ->
fun() -> Response end.
not_found() ->
return(?NOT_FOUND(<<"Transformation not found">>)).
make_serializable(Transformation0) ->
Schema = emqx_message_transformation_schema,
Transformation1 = transformation_out(Transformation0),
Transformation = emqx_utils_maps:binary_key_map(Transformation1),
RawConfig = #{
<<"message_transformation">> => #{
<<"transformations">> =>
[Transformation]
}
},
#{
<<"message_transformation">> := #{
<<"transformations">> :=
[Serialized]
}
} =
hocon_tconf:make_serializable(Schema, RawConfig, #{}),
Serialized.
format_metrics(Node, #{
counters := #{
'matched' := Matched,
'succeeded' := Succeeded,
'failed' := Failed
},
rate := #{
'matched' := #{
current := MatchedRate,
last5m := Matched5mRate,
max := MatchedMaxRate
}
}
}) ->
#{
metrics => #{
'matched' => Matched,
'succeeded' => Succeeded,
'failed' => Failed,
rate => MatchedRate,
rate_last5m => Matched5mRate,
rate_max => MatchedMaxRate
},
node => Node
};
format_metrics(Node, _) ->
#{
metrics => #{
'matched' => 0,
'succeeded' => 0,
'failed' => 0,
rate => 0,
rate_last5m => 0,
rate_max => 0
},
node => Node
}.
aggregate_metrics(NodeMetrics) ->
ErrorLogger = fun(_) -> ok end,
lists:foldl(
fun(#{metrics := Metrics}, Acc) ->
emqx_utils_maps:best_effort_recursive_sum(Metrics, Acc, ErrorLogger)
end,
#{},
NodeMetrics
).
transformation_out(Transformation) ->
maps:update_with(
operations,
fun(Os) -> lists:map(fun operation_out/1, Os) end,
Transformation
).
operation_out(Operation0) ->
%% TODO: remove injected bif module
Operation = maps:update_with(
value,
fun(V) -> iolist_to_binary(emqx_variform:decompile(V)) end,
Operation0
),
maps:update_with(
key,
fun(Path) -> iolist_to_binary(lists:join(".", Path)) end,
Operation
).

280
apps/emqx_message_transformation/src/emqx_message_transformation_registry.erl Normal file
View File

@ -0,0 +1,280 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
-module(emqx_message_transformation_registry).
-behaviour(gen_server).
%% API
-export([
lookup/1,
insert/2,
update/3,
delete/2,
reindex_positions/2,
matching_transformations/1,
%% metrics
get_metrics/1,
inc_matched/1,
inc_succeeded/1,
inc_failed/1,
start_link/0,
metrics_worker_spec/0
]).
%% `gen_server' API
-export([
init/1,
handle_call/3,
handle_cast/2
]).
%%------------------------------------------------------------------------------
%% Type declarations
%%------------------------------------------------------------------------------
-define(TRANSFORMATION_TOPIC_INDEX, emqx_message_transformation_index).
-define(TRANSFORMATION_TAB, emqx_message_transformation_tab).
-define(METRIC_NAME, message_transformation).
-define(METRICS, [
'matched',
'succeeded',
'failed'
]).
-define(RATE_METRICS, ['matched']).
-type transformation_name() :: binary().
%% TODO
-type transformation() :: #{atom() => term()}.
-type position_index() :: pos_integer().
-record(reindex_positions, {
new_transformations :: [transformation()],
old_transformations :: [transformation()]
}).
-record(insert, {pos :: position_index(), transformation :: transformation()}).
-record(update, {old :: transformation(), pos :: position_index(), new :: transformation()}).
-record(delete, {transformation :: transformation(), pos :: position_index()}).
%%------------------------------------------------------------------------------
%% API
%%------------------------------------------------------------------------------
-spec start_link() -> gen_server:start_ret().
start_link() ->
gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
-spec lookup(transformation_name()) ->
{ok, transformation()} | {error, not_found}.
lookup(Name) ->
case emqx_utils_ets:lookup_value(?TRANSFORMATION_TAB, Name, undefined) of
undefined ->
{error, not_found};
Transformation ->
{ok, Transformation}
end.
-spec reindex_positions([transformation()], [transformation()]) -> ok.
reindex_positions(NewTransformations, OldTransformations) ->
gen_server:call(
?MODULE,
#reindex_positions{
new_transformations = NewTransformations,
old_transformations = OldTransformations
},
infinity
).
-spec insert(position_index(), transformation()) -> ok.
insert(Pos, Transformation) ->
gen_server:call(?MODULE, #insert{pos = Pos, transformation = Transformation}, infinity).
-spec update(transformation(), position_index(), transformation()) -> ok.
update(Old, Pos, New) ->
gen_server:call(?MODULE, #update{old = Old, pos = Pos, new = New}, infinity).
-spec delete(transformation(), position_index()) -> ok.
delete(Transformation, Pos) ->
gen_server:call(?MODULE, #delete{transformation = Transformation, pos = Pos}, infinity).
%% @doc Returns a list of matching transformation names, sorted by their configuration order.
-spec matching_transformations(emqx_types:topic()) -> [transformation()].
matching_transformations(Topic) ->
Transformations0 =
lists:flatmap(
fun(M) ->
case emqx_topic_index:get_record(M, ?TRANSFORMATION_TOPIC_INDEX) of
[Name] ->
[Name];
_ ->
[]
end
end,
emqx_topic_index:matches(Topic, ?TRANSFORMATION_TOPIC_INDEX, [unique])
),
lists:flatmap(
fun(Name) ->
case lookup(Name) of
{ok, Transformation} ->
[Transformation];
_ ->
[]
end
end,
Transformations0
).
-spec metrics_worker_spec() -> supervisor:child_spec().
metrics_worker_spec() ->
emqx_metrics_worker:child_spec(message_transformation_metrics, ?METRIC_NAME).
-spec get_metrics(transformation_name()) -> emqx_metrics_worker:metrics().
get_metrics(Name) ->
emqx_metrics_worker:get_metrics(?METRIC_NAME, Name).
-spec inc_matched(transformation_name()) -> ok.
inc_matched(Name) ->
emqx_metrics_worker:inc(?METRIC_NAME, Name, 'matched').
-spec inc_succeeded(transformation_name()) -> ok.
inc_succeeded(Name) ->
emqx_metrics_worker:inc(?METRIC_NAME, Name, 'succeeded').
-spec inc_failed(transformation_name()) -> ok.
inc_failed(Name) ->
emqx_metrics_worker:inc(?METRIC_NAME, Name, 'failed').
%%------------------------------------------------------------------------------
%% `gen_server' API
%%------------------------------------------------------------------------------
init(_) ->
create_tables(),
State = #{},
{ok, State}.
handle_call(
#reindex_positions{
new_transformations = NewTransformations,
old_transformations = OldTransformations
},
_From,
State
) ->
do_reindex_positions(NewTransformations, OldTransformations),
{reply, ok, State};
handle_call(#insert{pos = Pos, transformation = Transformation}, _From, State) ->
do_insert(Pos, Transformation),
{reply, ok, State};
handle_call(#update{old = OldTransformation, pos = Pos, new = NewTransformation}, _From, State) ->
ok = do_update(OldTransformation, Pos, NewTransformation),
{reply, ok, State};
handle_call(#delete{transformation = Transformation, pos = Pos}, _From, State) ->
do_delete(Transformation, Pos),
{reply, ok, State};
handle_call(_Call, _From, State) ->
{reply, ignored, State}.
handle_cast(_Cast, State) ->
{noreply, State}.
%%------------------------------------------------------------------------------
%% Internal fns
%%------------------------------------------------------------------------------
create_tables() ->
_ = emqx_utils_ets:new(?TRANSFORMATION_TOPIC_INDEX, [
public, ordered_set, {read_concurrency, true}
]),
_ = emqx_utils_ets:new(?TRANSFORMATION_TAB, [public, ordered_set, {read_concurrency, true}]),
ok.
do_reindex_positions(NewTransformations, OldTransformations) ->
lists:foreach(
fun({Pos, Transformation}) ->
#{topics := Topics} = Transformation,
delete_topic_index(Pos, Topics)
end,
lists:enumerate(OldTransformations)
),
lists:foreach(
fun({Pos, Transformation}) ->
#{
name := Name,
topics := Topics
} = Transformation,
do_insert_into_tab(Name, Transformation, Pos),
upsert_topic_index(Name, Pos, Topics)
end,
lists:enumerate(NewTransformations)
).
do_insert(Pos, Transformation) ->
#{
enable := Enabled,
name := Name,
topics := Topics
} = Transformation,
maybe_create_metrics(Name),
do_insert_into_tab(Name, Transformation, Pos),
Enabled andalso upsert_topic_index(Name, Pos, Topics),
ok.
do_update(OldTransformation, Pos, NewTransformation) ->
#{topics := OldTopics} = OldTransformation,
#{
enable := Enabled,
name := Name,
topics := NewTopics
} = NewTransformation,
maybe_create_metrics(Name),
do_insert_into_tab(Name, NewTransformation, Pos),
delete_topic_index(Pos, OldTopics),
Enabled andalso upsert_topic_index(Name, Pos, NewTopics),
ok.
do_delete(Transformation, Pos) ->
#{
name := Name,
topics := Topics
} = Transformation,
ets:delete(?TRANSFORMATION_TAB, Name),
delete_topic_index(Pos, Topics),
drop_metrics(Name),
ok.
do_insert_into_tab(Name, Transformation0, Pos) ->
Transformation = Transformation0#{pos => Pos},
ets:insert(?TRANSFORMATION_TAB, {Name, Transformation}),
ok.
maybe_create_metrics(Name) ->
case emqx_metrics_worker:has_metrics(?METRIC_NAME, Name) of
true ->
ok = emqx_metrics_worker:reset_metrics(?METRIC_NAME, Name);
false ->
ok = emqx_metrics_worker:create_metrics(?METRIC_NAME, Name, ?METRICS, ?RATE_METRICS)
end.
drop_metrics(Name) ->
ok = emqx_metrics_worker:clear_metrics(?METRIC_NAME, Name).
upsert_topic_index(Name, Pos, Topics) ->
lists:foreach(
fun(Topic) ->
true = emqx_topic_index:insert(Topic, Pos, Name, ?TRANSFORMATION_TOPIC_INDEX)
end,
Topics
).
delete_topic_index(Pos, Topics) ->
lists:foreach(
fun(Topic) ->
true = emqx_topic_index:delete(Topic, Pos, ?TRANSFORMATION_TOPIC_INDEX)
end,
Topics
).

331
apps/emqx_message_transformation/src/emqx_message_transformation_schema.erl Normal file
View File

@ -0,0 +1,331 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
-module(emqx_message_transformation_schema).
-include_lib("typerefl/include/types.hrl").
-include_lib("hocon/include/hoconsc.hrl").
%% `hocon_schema' API
-export([
namespace/0,
roots/0,
fields/1
]).
%% `minirest_trails' API
-export([
api_schema/1
]).
%%------------------------------------------------------------------------------
%% Type declarations
%%------------------------------------------------------------------------------
-define(BIF_MOD_STR, "emqx_message_transformation_bif").
-define(ALLOWED_ROOT_KEYS, [
<<"payload">>,
<<"qos">>,
<<"retain">>,
<<"topic">>,
<<"user_property">>
]).
-type key() :: list(binary()) | binary().
-reflect_type([key/0]).
%%------------------------------------------------------------------------------
%% `hocon_schema' API
%%------------------------------------------------------------------------------
namespace() -> message_transformation.
roots() ->
[
{message_transformation,
mk(ref(message_transformation), #{importance => ?IMPORTANCE_HIDDEN})}
].
fields(message_transformation) ->
[
{transformations,
mk(
hoconsc:array(ref(transformation)),
#{
default => [],
desc => ?DESC("transformations"),
validator => fun validate_unique_names/1
}
)}
];
fields(transformation) ->
[
{tags, emqx_schema:tags_schema()},
{description, emqx_schema:description_schema()},
{enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})},
{name,
mk(
binary(),
#{
required => true,
validator => fun emqx_resource:validate_name/1,
desc => ?DESC("name")
}
)},
{topics,
mk(
hoconsc:union([binary(), hoconsc:array(binary())]),
#{
desc => ?DESC("topics"),
converter => fun ensure_array/2,
validator => fun validate_unique_topics/1,
required => true
}
)},
{failure_action,
mk(
hoconsc:enum([drop, disconnect, ignore]),
#{desc => ?DESC("failure_action"), required => true}
)},
{log_failure,
mk(
ref(log_failure),
#{desc => ?DESC("log_failure_at"), default => #{}}
)},
{payload_decoder,
mk(
hoconsc:union(fun payload_serde_member_selector/1),
#{desc => ?DESC("payload_decoder"), default => #{<<"type">> => <<"none">>}}
)},
{payload_encoder,
mk(
hoconsc:union(fun payload_serde_member_selector/1),
#{desc => ?DESC("payload_encoder"), default => #{<<"type">> => <<"none">>}}
)},
{operations,
mk(
hoconsc:array(ref(operation)),
#{
desc => ?DESC("operation"),
required => true,
validator => fun validate_operations/1
}
)}
];
fields(log_failure) ->
[
{level,
mk(
hoconsc:enum([error, warning, notice, info, debug, none]),
#{desc => ?DESC("log_failure_at"), default => info}
)}
];
fields(operation) ->
[
%% TODO: more strict type check??
{key,
mk(
typerefl:alias("string", key()), #{
desc => ?DESC("operation_key"),
required => true,
converter => fun parse_key_path/2
}
)},
{value,
mk(typerefl:alias("string", any()), #{
desc => ?DESC("operation_value"),
required => true,
converter => fun compile_variform/2
})}
];
fields(payload_serde_none) ->
[{type, mk(none, #{default => none, desc => ?DESC("payload_serde_none_type")})}];
fields(payload_serde_json) ->
[{type, mk(json, #{default => json, desc => ?DESC("payload_serde_json_type")})}];
fields(payload_serde_avro) ->
[
{type, mk(avro, #{default => avro, desc => ?DESC("payload_serde_avro_type")})},
{schema, mk(binary(), #{required => true, desc => ?DESC("payload_serde_avro_schema")})}
];
fields(payload_serde_protobuf) ->
[
{type, mk(protobuf, #{default => protobuf, desc => ?DESC("payload_serde_protobuf_type")})},
{schema, mk(binary(), #{required => true, desc => ?DESC("payload_serde_protobuf_schema")})},
{message_type,
mk(binary(), #{required => true, desc => ?DESC("payload_serde_protobuf_message_type")})}
].
%%------------------------------------------------------------------------------
%% `minirest_trails' API
%%------------------------------------------------------------------------------
api_schema(list) ->
hoconsc:array(ref(transformation));
api_schema(lookup) ->
ref(transformation);
api_schema(post) ->
ref(transformation);
api_schema(put) ->
ref(transformation).
%%------------------------------------------------------------------------------
%% Internal exports
%%------------------------------------------------------------------------------
%%------------------------------------------------------------------------------
%% Internal fns
%%------------------------------------------------------------------------------
mk(Type, Meta) -> hoconsc:mk(Type, Meta).
ref(Name) -> hoconsc:ref(?MODULE, Name).
payload_serde_member_selector(all_union_members) ->
payload_serde_refs();
payload_serde_member_selector({value, V}) ->
payload_serde_refs(V).
payload_serde_refs() ->
[
payload_serde_none,
payload_serde_json,
payload_serde_avro,
payload_serde_protobuf
].
payload_serde_refs(#{<<"type">> := Type} = V) when is_atom(Type) ->
payload_serde_refs(V#{<<"type">> := atom_to_binary(Type)});
payload_serde_refs(#{<<"type">> := <<"none">>}) ->
[ref(payload_serde_none)];
payload_serde_refs(#{<<"type">> := <<"json">>}) ->
[ref(payload_serde_json)];
payload_serde_refs(#{<<"type">> := <<"avro">>}) ->
[ref(payload_serde_avro)];
payload_serde_refs(#{<<"type">> := <<"protobuf">>}) ->
[ref(payload_serde_protobuf)];
payload_serde_refs(_Value) ->
Expected = lists:join(
" | ",
[
Name
|| T <- payload_serde_refs(),
"payload_serde_" ++ Name <- [atom_to_list(T)]
]
),
throw(#{
field_name => type,
expected => iolist_to_binary(Expected)
}).
ensure_array(undefined, _) -> undefined;
ensure_array(L, _) when is_list(L) -> L;
ensure_array(B, _) -> [B].
validate_unique_names(Transformations0) ->
Transformations = emqx_utils_maps:binary_key_map(Transformations0),
do_validate_unique_names(Transformations, #{}).
do_validate_unique_names(_Transformations = [], _Acc) ->
ok;
do_validate_unique_names([#{<<"name">> := Name} | _Rest], Acc) when is_map_key(Name, Acc) ->
{error, <<"duplicated name: ", Name/binary>>};
do_validate_unique_names([#{<<"name">> := Name} | Rest], Acc) ->
do_validate_unique_names(Rest, Acc#{Name => true}).
validate_unique_topics(Topics) ->
Grouped = maps:groups_from_list(
fun(T) -> T end,
Topics
),
DuplicatedMap = maps:filter(
fun(_T, Ts) -> length(Ts) > 1 end,
Grouped
),
case maps:keys(DuplicatedMap) of
[] ->
ok;
Duplicated ->
Msg = iolist_to_binary([
<<"duplicated topics: ">>,
lists:join(", ", Duplicated)
]),
{error, Msg}
end.
validate_operations([]) ->
{error, <<"at least one operation must be defined">>};
validate_operations([_ | _]) ->
ok.
compile_variform(Expression, #{make_serializable := true}) ->
case is_binary(Expression) of
true ->
Expression;
false ->
emqx_variform:decompile(Expression)
end;
compile_variform(Expression, _Opts) ->
case emqx_variform:compile(Expression) of
{ok, Compiled} ->
transform_bifs(Compiled);
{error, Reason} ->
throw(#{expression => Expression, reason => Reason})
end.
transform_bifs(#{form := Form} = Compiled) ->
Compiled#{form := traverse_transform_bifs(Form)}.
traverse_transform_bifs({call, FnName, Args}) ->
FQFnName = fully_qualify_local_bif(FnName),
{call, FQFnName, lists:map(fun traverse_transform_bifs/1, Args)};
traverse_transform_bifs({array, Elems}) ->
{array, lists:map(fun traverse_transform_bifs/1, Elems)};
traverse_transform_bifs(Node) ->
Node.
fully_qualify_local_bif("json_encode") ->
?BIF_MOD_STR ++ ".json_encode";
fully_qualify_local_bif("json_decode") ->
?BIF_MOD_STR ++ ".json_decode";
fully_qualify_local_bif(FnName) ->
FnName.
parse_key_path(<<"">>, _Opts) ->
throw(#{reason => <<"key must be non-empty">>});
parse_key_path(Key, #{make_serializable := true}) ->
case is_binary(Key) of
true ->
Key;
false ->
iolist_to_binary(lists:join(".", Key))
end;
parse_key_path(Key, _Opts) when is_binary(Key) ->
Parts = binary:split(Key, <<".">>, [global]),
case lists:any(fun(P) -> P =:= <<"">> end, Parts) of
true ->
throw(#{invalid_key => Key});
false ->
ok
end,
case Parts of
[<<"payload">> | _] ->
ok;
[<<"qos">>] ->
ok;
[<<"retain">>] ->
ok;
[<<"topic">>] ->
ok;
[<<"user_property">>, _] ->
ok;
[<<"user_property">>] ->
throw(#{
invalid_key => Key, reason => <<"must define exactly one key inside user property">>
});
[<<"user_property">> | _] ->
throw(#{
invalid_key => Key, reason => <<"must define exactly one key inside user property">>
});
_ ->
throw(#{invalid_key => Key, allowed_root_keys => ?ALLOWED_ROOT_KEYS})
end,
Parts.

47
apps/emqx_message_transformation/src/emqx_message_transformation_sup.erl Normal file
View File

@ -0,0 +1,47 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
-module(emqx_message_transformation_sup).
-behaviour(supervisor).
%% API
-export([start_link/0]).
%% `supervisor' API
-export([init/1]).
%%------------------------------------------------------------------------------
%% API
%%------------------------------------------------------------------------------
start_link() ->
supervisor:start_link({local, ?MODULE}, ?MODULE, []).
%%------------------------------------------------------------------------------
%% `supervisor' API
%%------------------------------------------------------------------------------
init([]) ->
Registry = worker_spec(emqx_message_transformation_registry),
Metrics = emqx_message_transformation_registry:metrics_worker_spec(),
SupFlags = #{
strategy => one_for_one,
intensity => 10,
period => 10
},
ChildSpecs = [Metrics, Registry],
{ok, {SupFlags, ChildSpecs}}.
%%------------------------------------------------------------------------------
%% Internal fns
%%------------------------------------------------------------------------------
worker_spec(Mod) ->
#{
id => Mod,
start => {Mod, start_link, []},
restart => permanent,
shutdown => 5_000,
type => worker
}.

1493
apps/emqx_message_transformation/test/emqx_message_transformation_http_api_SUITE.erl Normal file
View File

File diff suppressed because it is too large Load Diff

174
apps/emqx_message_transformation/test/emqx_message_transformation_tests.erl Normal file
View File

@ -0,0 +1,174 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
-module(emqx_message_transformation_tests).
-include_lib("eunit/include/eunit.hrl").
-define(TRANSFORMATIONS_PATH, "message_transformation.transformations").
%%------------------------------------------------------------------------------
%% Helper fns
%%------------------------------------------------------------------------------
bin(X) -> emqx_utils_conv:bin(X).
parse_and_check(InnerConfigs) ->
RootBin = <<"message_transformation">>,
InnerBin = <<"transformations">>,
RawConf = #{RootBin => #{InnerBin => InnerConfigs}},
#{RootBin := #{InnerBin := Checked}} = hocon_tconf:check_plain(
emqx_message_transformation_schema,
RawConf,
#{
required => false,
atom_key => false,
make_serializable => false
}
),
Checked.
transformation(Name, Operations) ->
transformation(Name, Operations, _Overrides = #{}).
transformation(Name, Operations0, Overrides) ->
Operations = lists:map(fun normalize_operation/1, Operations0),
Default = #{
<<"tags">> => [<<"some">>, <<"tags">>],
<<"description">> => <<"my transformation">>,
<<"enable">> => true,
<<"name">> => Name,
<<"topics">> => [<<"t/+">>],
<<"failure_action">> => <<"drop">>,
<<"log_failure">> => #{<<"level">> => <<"warning">>},
<<"payload_decoder">> => #{<<"type">> => <<"json">>},
<<"payload_encoder">> => #{<<"type">> => <<"json">>},
<<"operations">> => Operations
},
emqx_utils_maps:deep_merge(Default, Overrides).
normalize_operation({K, V}) ->
#{<<"key">> => bin(K), <<"value">> => bin(V)}.
dummy_operation() ->
topic_operation(<<"concat([topic, '/', payload.t])">>).
topic_operation(VariformExpr) ->
operation(topic, VariformExpr).
operation(Key, VariformExpr) ->
{Key, VariformExpr}.
%%------------------------------------------------------------------------------
%% Test cases
%%------------------------------------------------------------------------------
schema_test_() ->
[
{"topics is always a list 1",
?_assertMatch(
[#{<<"topics">> := [<<"t/1">>]}],
parse_and_check([
transformation(
<<"foo">>,
[dummy_operation()],
#{<<"topics">> => <<"t/1">>}
)
])
)},
{"topics is always a list 2",
?_assertMatch(
[#{<<"topics">> := [<<"t/1">>]}],
parse_and_check([
transformation(
<<"foo">>,
[dummy_operation()],
#{<<"topics">> => [<<"t/1">>]}
)
])
)},
{"names are unique",
?_assertThrow(
{_Schema, [
#{
reason := <<"duplicated name:", _/binary>>,
path := ?TRANSFORMATIONS_PATH,
kind := validation_error
}
]},
parse_and_check([
transformation(<<"foo">>, [dummy_operation()]),
transformation(<<"foo">>, [dummy_operation()])
])
)},
{"operations must be non-empty",
?_assertThrow(
{_Schema, [
#{
reason := <<"at least one operation must be defined">>,
kind := validation_error
}
]},
parse_and_check([
transformation(
<<"foo">>,
[]
)
])
)},
{"bogus check type: decoder",
?_assertThrow(
{_Schema, [
#{
expected := <<"none", _/binary>>,
kind := validation_error,
field_name := type
}
]},
parse_and_check([
transformation(<<"foo">>, [dummy_operation()], #{
<<"payload_decoder">> => #{<<"type">> => <<"foo">>}
})
])
)},
{"bogus check type: encoder",
?_assertThrow(
{_Schema, [
#{
expected := <<"none", _/binary>>,
kind := validation_error,
field_name := type
}
]},
parse_and_check([
transformation(<<"foo">>, [dummy_operation()], #{
<<"payload_encoder">> => #{<<"type">> => <<"foo">>}
})
])
)}
].
invalid_names_test_() ->
[
{InvalidName,
?_assertThrow(
{_Schema, [
#{
kind := validation_error,
path := "message_transformation.transformations.1.name"
}
]},
parse_and_check([transformation(InvalidName, [dummy_operation()])])
)}
|| InvalidName <- [
<<"">>,
<<"_name">>,
<<"name$">>,
<<"name!">>,
<<"some name">>,
<<"nãme"/utf8>>,
<<"test_哈哈"/utf8>>,
%% long name
binary:copy(<<"a">>, 256)
]
].

49
apps/emqx_postgresql/src/emqx_postgresql.erl
View File

@ -268,7 +268,9 @@ close_prepared_statement(ChannelId, #{pool_name := PoolName} = State) ->
close_prepared_statement([WorkerPid | Rest], ChannelId, State) ->
%% We ignore errors since any error probably means that the
%% prepared statement doesn't exist.
%% prepared statement doesn't exist. If it exists when we try
%% to insert one with the same name, we will try to remove it
%% again anyway.
try ecpool_worker:client(WorkerPid) of
{ok, Conn} ->
Statement = get_templated_statement(ChannelId, State),
@ -689,17 +691,21 @@ do_prepare_sql([], _Prepares, LastSts) ->
{ok, LastSts}.
prepare_sql_to_conn(Conn, Prepares) ->
prepare_sql_to_conn(Conn, Prepares, #{}).
prepare_sql_to_conn(Conn, Prepares, #{}, 0).
prepare_sql_to_conn(Conn, [], Statements) when is_pid(Conn) ->
prepare_sql_to_conn(Conn, [], Statements, _Attempts) when is_pid(Conn) ->
{ok, Statements};
prepare_sql_to_conn(Conn, [{Key, {SQL, _RowTemplate}} | Rest], Statements) when is_pid(Conn) ->
prepare_sql_to_conn(Conn, [{_Key, _} | _Rest], _Statements, _MaxAttempts = 2) when is_pid(Conn) ->
failed_to_remove_prev_prepared_statement_error();
prepare_sql_to_conn(
Conn, [{Key, {SQL, _RowTemplate}} | Rest] = ToPrepare, Statements, Attempts
) when is_pid(Conn) ->
LogMeta = #{msg => "postgresql_prepare_statement", name => Key, sql => SQL},
?SLOG(info, LogMeta),
case epgsql:parse2(Conn, Key, SQL, []) of
{ok, Statement} ->
prepare_sql_to_conn(Conn, Rest, Statements#{Key => Statement});
{error, {error, error, _, undefined_table, _, _} = Error} ->
prepare_sql_to_conn(Conn, Rest, Statements#{Key => Statement}, 0);
{error, #error{severity = error, codename = undefined_table} = Error} ->
%% Target table is not created
?tp(pgsql_undefined_table, #{}),
LogMsg =
@ -709,6 +715,30 @@ prepare_sql_to_conn(Conn, [{Key, {SQL, _RowTemplate}} | Rest], Statements) when
),
?SLOG(error, LogMsg),
{error, undefined_table};
{error, #error{severity = error, codename = duplicate_prepared_statement}} = Error ->
?tp(pgsql_prepared_statement_exists, #{}),
LogMsg =
maps:merge(
LogMeta#{
msg => "postgresql_prepared_statment_with_same_name_already_exists",
explain => <<
"A prepared statement with the same name already "
"exists in the driver. Will attempt to remove the "
"previous prepared statement with the name and then "
"try again."
>>
},
translate_to_log_context(Error)
),
?SLOG(warning, LogMsg),
case epgsql:close(Conn, statement, Key) of
ok ->
?SLOG(info, #{msg => "pqsql_closed_statement_successfully"}),
prepare_sql_to_conn(Conn, ToPrepare, Statements, Attempts + 1);
{error, CloseError} ->
?SLOG(error, #{msg => "pqsql_close_statement_failed", cause => CloseError}),
failed_to_remove_prev_prepared_statement_error()
end;
{error, Error} ->
TranslatedError = translate_to_log_context(Error),
LogMsg =
@ -720,6 +750,13 @@ prepare_sql_to_conn(Conn, [{Key, {SQL, _RowTemplate}} | Rest], Statements) when
{error, export_error(TranslatedError)}
end.
failed_to_remove_prev_prepared_statement_error() ->
Msg =
("A previous prepared statement for the action already exists "
"but cannot be closed. Please, try to disable and then enable "
"the connector to resolve this issue."),
{error, unicode:characters_to_binary(Msg)}.
to_bin(Bin) when is_binary(Bin) ->
Bin;
to_bin(Atom) when is_atom(Atom) ->

5
apps/emqx_prometheus/include/emqx_prometheus.hrl
View File

@ -24,10 +24,13 @@
-define(PROMETHEUS_DATA_INTEGRATION_COLLECTOR, emqx_prometheus_data_integration).
-define(PROMETHEUS_SCHEMA_VALIDATION_REGISTRY, '/prometheus/schema_validation').
-define(PROMETHEUS_SCHEMA_VALIDATION_COLLECTOR, emqx_prometheus_schema_validation).
-define(PROMETHEUS_MESSAGE_TRANSFORMATION_REGISTRY, '/prometheus/message_transformation').
-define(PROMETHEUS_MESSAGE_TRANSFORMATION_COLLECTOR, emqx_prometheus_message_transformation).
-if(?EMQX_RELEASE_EDITION == ee).
-define(PROMETHEUS_EE_REGISTRIES, [
?PROMETHEUS_SCHEMA_VALIDATION_REGISTRY
?PROMETHEUS_SCHEMA_VALIDATION_REGISTRY,
?PROMETHEUS_MESSAGE_TRANSFORMATION_REGISTRY
]).
%% ELSE if(?EMQX_RELEASE_EDITION == ee).
-else.

2
apps/emqx_prometheus/src/emqx_prometheus.app.src
View File

@ -2,7 +2,7 @@
{application, emqx_prometheus, [
{description, "Prometheus for EMQX"},
% strict semver, bump manually!
{vsn, "5.2.0"},
{vsn, "5.2.1"},
{modules, []},
{registered, [emqx_prometheus_sup]},
{applications, [kernel, stdlib, prometheus, emqx, emqx_auth, emqx_resource, emqx_management]},

24
apps/emqx_prometheus/src/emqx_prometheus_api.erl
View File

@ -49,7 +49,8 @@
stats/2,
auth/2,
data_integration/2,
schema_validation/2
schema_validation/2,
message_transformation/2
]).
-export([lookup_from_local_nodes/3]).
@ -73,7 +74,10 @@ paths() ->
-if(?EMQX_RELEASE_EDITION == ee).
paths_ee() ->
["/prometheus/schema_validation"].
[
"/prometheus/schema_validation",
"/prometheus/message_transformation"
].
%% ELSE if(?EMQX_RELEASE_EDITION == ee).
-else.
paths_ee() ->
@ -151,6 +155,19 @@ schema("/prometheus/schema_validation") ->
responses =>
#{200 => prometheus_data_schema()}
}
};
schema("/prometheus/message_transformation") ->
#{
'operationId' => message_transformation,
get =>
#{
description => ?DESC(get_prom_message_transformation),
tags => ?TAGS,
parameters => [ref(mode)],
security => security(),
responses =>
#{200 => prometheus_data_schema()}
}
}.
security() ->
@ -226,6 +243,9 @@ data_integration(get, #{headers := Headers, query_string := Qs}) ->
schema_validation(get, #{headers := Headers, query_string := Qs}) ->
collect(emqx_prometheus_schema_validation, collect_opts(Headers, Qs)).
message_transformation(get, #{headers := Headers, query_string := Qs}) ->
collect(emqx_prometheus_message_transformation, collect_opts(Headers, Qs)).
%%--------------------------------------------------------------------
%% Internal funcs
%%--------------------------------------------------------------------

222
apps/emqx_prometheus/src/emqx_prometheus_message_transformation.erl Normal file
View File

@ -0,0 +1,222 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
-module(emqx_prometheus_message_transformation).
-if(?EMQX_RELEASE_EDITION == ee).
%% for bpapi
-behaviour(emqx_prometheus_cluster).
%% Please don't remove this attribute, prometheus uses it to
%% automatically register collectors.
-behaviour(prometheus_collector).
-include("emqx_prometheus.hrl").
-include_lib("prometheus/include/prometheus.hrl").
-import(
prometheus_model_helpers,
[
create_mf/5,
gauge_metrics/1,
counter_metrics/1
]
).
-export([
deregister_cleanup/1,
collect_mf/2,
collect_metrics/2
]).
%% `emqx_prometheus' API
-export([collect/1]).
%% `emqx_prometheus_cluster' API
-export([
fetch_from_local_node/1,
fetch_cluster_consistented_data/0,
aggre_or_zip_init_acc/0,
logic_sum_metrics/0
]).
%%--------------------------------------------------------------------
%% Type definitions
%%--------------------------------------------------------------------
-define(MG(K, MAP), maps:get(K, MAP)).
-define(MG0(K, MAP), maps:get(K, MAP, 0)).
-define(metrics_data_key, message_transformation_metrics_data).
-define(key_enabled, emqx_message_transformation_enable).
-define(key_matched, emqx_message_transformation_matched).
-define(key_failed, emqx_message_transformation_failed).
-define(key_succeeded, emqx_message_transformation_succeeded).
%%--------------------------------------------------------------------
%% `emqx_prometheus_cluster' API
%%--------------------------------------------------------------------
fetch_from_local_node(Mode) ->
Validations = emqx_message_transformation:list(),
{node(), #{
?metrics_data_key => to_validation_data(Mode, Validations)
}}.
fetch_cluster_consistented_data() ->
#{}.
aggre_or_zip_init_acc() ->
#{
?metrics_data_key => maps:from_keys(message_transformation_metric(names), [])
}.
logic_sum_metrics() ->
[
?key_enabled
].
%%--------------------------------------------------------------------
%% Collector API
%%--------------------------------------------------------------------
%% @private
deregister_cleanup(_) -> ok.
%% @private
-spec collect_mf(_Registry, Callback) -> ok when
_Registry :: prometheus_registry:registry(),
Callback :: prometheus_collector:collect_mf_callback().
collect_mf(?PROMETHEUS_MESSAGE_TRANSFORMATION_REGISTRY, Callback) ->
RawData = emqx_prometheus_cluster:raw_data(?MODULE, ?GET_PROM_DATA_MODE()),
%% Schema Validation Metrics
RuleMetricDs = ?MG(?metrics_data_key, RawData),
ok = add_collect_family(Callback, message_transformation_metrics_meta(), RuleMetricDs),
ok;
collect_mf(_, _) ->
ok.
%% @private
collect(<<"json">>) ->
RawData = emqx_prometheus_cluster:raw_data(?MODULE, ?GET_PROM_DATA_MODE()),
#{
message_transformations => collect_json_data(?MG(?metrics_data_key, RawData))
};
collect(<<"prometheus">>) ->
prometheus_text_format:format(?PROMETHEUS_MESSAGE_TRANSFORMATION_REGISTRY).
%%====================
%% API Helpers
add_collect_family(Callback, MetricWithType, Data) ->
_ = [add_collect_family(Name, Data, Callback, Type) || {Name, Type} <- MetricWithType],
ok.
add_collect_family(Name, Data, Callback, Type) ->
%% TODO: help document from Name
Callback(create_mf(Name, _Help = <<"">>, Type, ?MODULE, Data)).
collect_metrics(Name, Metrics) ->
collect_mv(Name, Metrics).
%%--------------------------------------------------------------------
%% Collector
%%--------------------------------------------------------------------
%%========================================
%% Schema Validation Metrics
%%========================================
collect_mv(K = ?key_enabled, Data) -> gauge_metrics(?MG(K, Data));
collect_mv(K = ?key_matched, Data) -> counter_metrics(?MG(K, Data));
collect_mv(K = ?key_failed, Data) -> counter_metrics(?MG(K, Data));
collect_mv(K = ?key_succeeded, Data) -> counter_metrics(?MG(K, Data)).
%%--------------------------------------------------------------------
%% Internal functions
%%--------------------------------------------------------------------
%%========================================
%% Schema Validation Metrics
%%========================================
message_transformation_metrics_meta() ->
[
{?key_enabled, gauge},
{?key_matched, counter},
{?key_failed, counter},
{?key_succeeded, counter}
].
message_transformation_metric(names) ->
emqx_prometheus_cluster:metric_names(message_transformation_metrics_meta()).
to_validation_data(Mode, Validations) ->
lists:foldl(
fun(#{name := Name} = Validation, Acc) ->
merge_acc_with_validations(Mode, Name, get_validation_metrics(Validation), Acc)
end,
maps:from_keys(message_transformation_metric(names), []),
Validations
).
merge_acc_with_validations(Mode, Id, ValidationMetrics, PointsAcc) ->
maps:fold(
fun(K, V, AccIn) ->
AccIn#{K => [validation_point(Mode, Id, V) | ?MG(K, AccIn)]}
end,
PointsAcc,
ValidationMetrics
).
validation_point(Mode, Name, V) ->
{with_node_label(Mode, [{validation_name, Name}]), V}.
get_validation_metrics(#{name := Name, enable := Enabled} = _Rule) ->
#{counters := Counters} = emqx_message_transformation_registry:get_metrics(Name),
#{
?key_enabled => emqx_prometheus_cluster:boolean_to_number(Enabled),
?key_matched => ?MG0('matched', Counters),
?key_failed => ?MG0('failed', Counters),
?key_succeeded => ?MG0('succeeded', Counters)
}.
%%--------------------------------------------------------------------
%% Collect functions
%%--------------------------------------------------------------------
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% merge / zip formatting funcs for type `application/json`
collect_json_data(Data) ->
emqx_prometheus_cluster:collect_json_data(Data, fun zip_json_message_transformation_metrics/3).
zip_json_message_transformation_metrics(Key, Points, [] = _AccIn) ->
lists:foldl(
fun({Labels, Metric}, AccIn2) ->
LabelsKVMap = maps:from_list(Labels),
Point = LabelsKVMap#{Key => Metric},
[Point | AccIn2]
end,
[],
Points
);
zip_json_message_transformation_metrics(Key, Points, AllResultsAcc) ->
ThisKeyResult = lists:foldl(emqx_prometheus_cluster:point_to_map_fun(Key), [], Points),
lists:zipwith(fun maps:merge/2, AllResultsAcc, ThisKeyResult).
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Helper funcs
with_node_label(?PROM_DATA_MODE__NODE, Labels) ->
Labels;
with_node_label(?PROM_DATA_MODE__ALL_NODES_AGGREGATED, Labels) ->
Labels;
with_node_label(?PROM_DATA_MODE__ALL_NODES_UNAGGREGATED, Labels) ->
[{node, node()} | Labels].
%% END if(?EMQX_RELEASE_EDITION == ee).
-endif.

48
apps/emqx_prometheus/test/emqx_prometheus_data_SUITE.erl
View File

@ -82,7 +82,8 @@ all() ->
{group, '/prometheus/stats'},
{group, '/prometheus/auth'},
{group, '/prometheus/data_integration'},
[{group, '/prometheus/schema_validation'} || emqx_release:edition() == ee]
[{group, '/prometheus/schema_validation'} || emqx_release:edition() == ee],
[{group, '/prometheus/message_transformation'} || emqx_release:edition() == ee]
]).
groups() ->
@ -101,6 +102,7 @@ groups() ->
{'/prometheus/auth', ModeGroups},
{'/prometheus/data_integration', ModeGroups},
{'/prometheus/schema_validation', ModeGroups},
{'/prometheus/message_transformation', ModeGroups},
{?PROM_DATA_MODE__NODE, AcceptGroups},
{?PROM_DATA_MODE__ALL_NODES_AGGREGATED, AcceptGroups},
{?PROM_DATA_MODE__ALL_NODES_UNAGGREGATED, AcceptGroups},
@ -136,6 +138,10 @@ init_per_suite(Config) ->
{emqx_schema_validation, #{config => schema_validation_config()}}
|| emqx_release:edition() == ee
],
[
{emqx_message_transformation, #{config => message_transformation_config()}}
|| emqx_release:edition() == ee
],
{emqx_prometheus, emqx_prometheus_SUITE:legacy_conf_default()}
]),
#{
@ -168,6 +174,8 @@ init_per_group('/prometheus/data_integration', Config) ->
[{module, emqx_prometheus_data_integration} | Config];
init_per_group('/prometheus/schema_validation', Config) ->
[{module, emqx_prometheus_schema_validation} | Config];
init_per_group('/prometheus/message_transformation', Config) ->
[{module, emqx_prometheus_message_transformation} | Config];
init_per_group(?PROM_DATA_MODE__NODE, Config) ->
[{mode, ?PROM_DATA_MODE__NODE} | Config];
init_per_group(?PROM_DATA_MODE__ALL_NODES_AGGREGATED, Config) ->
@ -357,6 +365,8 @@ metric_meta(<<"emqx_action_", _Tail/binary>>) -> ?meta(1, 1, 2);
metric_meta(<<"emqx_connector_", _Tail/binary>>) -> ?meta(1, 1, 2);
%% `/prometheus/schema_validation`
metric_meta(<<"emqx_schema_validation_", _Tail/binary>>) -> ?meta(1, 1, 2);
%% `/prometheus/message_transformation`
metric_meta(<<"emqx_message_transformation_", _Tail/binary>>) -> ?meta(1, 1, 2);
%% normal emqx metrics
metric_meta(<<"emqx_", _Tail/binary>>) -> ?meta(0, 0, 1);
metric_meta(_) -> #{}.
@ -840,6 +850,23 @@ assert_json_data__schema_validations(Ms, _) ->
Ms
).
assert_json_data__message_transformations(Ms, _) ->
lists:foreach(
fun(M) ->
?assertMatch(
#{
validation_name := _,
emqx_message_transformation_enable := _,
emqx_message_transformation_matched := _,
emqx_message_transformation_failed := _,
emqx_message_transformation_succeeded := _
},
M
)
end,
Ms
).
schema_validation_config() ->
Validation = #{
<<"enable">> => true,
@ -860,5 +887,24 @@ schema_validation_config() ->
}
}.
message_transformation_config() ->
Transformation = #{
<<"enable">> => true,
<<"name">> => <<"my_transformation">>,
<<"topics">> => [<<"t/#">>],
<<"failure_action">> => <<"drop">>,
<<"operations">> => [
#{
<<"key">> => <<"topic">>,
<<"value">> => <<"concat([topic, '/', payload.t])">>
}
]
},
#{
<<"message_transformation">> => #{
<<"transformations">> => [Transformation]
}
}.
stop_apps(Apps) ->
lists:foreach(fun application:stop/1, Apps).

31
apps/emqx_resource/src/emqx_resource_manager.erl
View File

@ -14,6 +14,9 @@
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_resource_manager).
-feature(maybe_expr, enable).
-behaviour(gen_statem).
-include("emqx_resource.hrl").
@ -34,6 +37,7 @@
health_check/1,
channel_health_check/2,
add_channel/3,
add_channel/4,
remove_channel/2,
get_channels/1
]).
@ -133,6 +137,12 @@
ST =:= ?status_connecting; ST =:= ?status_connected; ST =:= ?status_disconnected
).
-type add_channel_opts() :: #{
%% Whether to immediately perform a health check after adding the channel.
%% Default: `true'
perform_health_check => boolean()
}.
%%------------------------------------------------------------------------------
%% API
%%------------------------------------------------------------------------------
@ -378,11 +388,30 @@ channel_health_check(ResId, ChannelId) ->
_ = health_check(ResId),
safe_call(ResId, {channel_health_check, ChannelId}, ?T_OPERATION).
-spec add_channel(
connector_resource_id(),
action_resource_id() | source_resource_id(),
_Config
) ->
ok | {error, term()}.
add_channel(ResId, ChannelId, Config) ->
add_channel(ResId, ChannelId, Config, _Opts = #{}).
-spec add_channel(
connector_resource_id(),
action_resource_id() | source_resource_id(),
_Config,
add_channel_opts()
) ->
ok | {error, term()}.
add_channel(ResId, ChannelId, Config, Opts) ->
Result = safe_call(ResId, {add_channel, ChannelId, Config}, ?T_OPERATION),
maybe
true ?= maps:get(perform_health_check, Opts, true),
%% Wait for health_check to finish
_ = health_check(ResId),
_ = channel_health_check(ResId, ChannelId),
ok
end,
Result.
remove_channel(ResId, ChannelId) ->

81
apps/emqx_rule_engine/src/emqx_rule_events.erl
View File

@ -45,6 +45,7 @@
on_session_unsubscribed/4,
on_message_publish/2,
on_message_dropped/4,
on_message_transformation_failed/3,
on_schema_validation_failed/3,
on_message_delivered/3,
on_message_acked/3,
@ -80,6 +81,7 @@ event_names() ->
'message.delivered',
'message.acked',
'message.dropped',
'message.transformation_failed',
'schema.validation_failed',
'delivery.dropped'
].
@ -96,6 +98,7 @@ event_topics_enum() ->
'$events/message_delivered',
'$events/message_acked',
'$events/message_dropped',
'$events/message_transformation_failed',
'$events/schema_validation_failed',
'$events/delivery_dropped'
% '$events/message_publish' % not possible to use in SELECT FROM
@ -237,6 +240,19 @@ on_message_dropped(Message, _, Reason, Conf) ->
end,
{ok, Message}.
on_message_transformation_failed(Message, TransformationContext, Conf) ->
case ignore_sys_message(Message) of
true ->
ok;
false ->
apply_event(
'message.transformation_failed',
fun() -> eventmsg_transformation_failed(Message, TransformationContext) end,
Conf
)
end,
{ok, Message}.
on_schema_validation_failed(Message, ValidationContext, Conf) ->
case ignore_sys_message(Message) of
true ->
@ -535,6 +551,38 @@ eventmsg_dropped(
#{headers => Headers}
).
eventmsg_transformation_failed(
Message = #message{
id = Id,
from = ClientId,
qos = QoS,
flags = Flags,
topic = Topic,
headers = Headers,
payload = Payload,
timestamp = Timestamp
},
TransformationContext
) ->
#{name := TransformationName} = TransformationContext,
with_basic_columns(
'message.transformation_failed',
#{
id => emqx_guid:to_hexstr(Id),
transformation => TransformationName,
clientid => ClientId,
username => emqx_message:get_header(username, Message, undefined),
payload => Payload,
peername => ntoa(emqx_message:get_header(peername, Message, undefined)),
topic => Topic,
qos => QoS,
flags => Flags,
pub_props => printable_maps(emqx_message:get_header(properties, Message, #{})),
publish_received_at => Timestamp
},
#{headers => Headers}
).
eventmsg_validation_failed(
Message = #message{
id = Id,
@ -737,9 +785,17 @@ event_info_schema_validation_failed() ->
{<<"messages that do not pass configured validations">>, <<"未通过验证的消息"/utf8>>},
<<"SELECT * FROM \"$events/schema_validation_failed\" WHERE topic =~ 't/#'">>
).
event_info_message_transformation_failed() ->
event_info_common(
'message.transformation_failed',
{<<"message transformation failed">>, <<"message 验证失败"/utf8>>},
{<<"messages that do not pass configured transformation">>, <<"未通过验证的消息"/utf8>>},
<<"SELECT * FROM \"$events/message_transformation_failed\" WHERE topic =~ 't/#'">>
).
ee_event_info() ->
[
event_info_schema_validation_failed()
event_info_schema_validation_failed(),
event_info_message_transformation_failed()
].
-else.
%% END (?EMQX_RELEASE_EDITION == ee).
@ -933,6 +989,9 @@ test_columns(Event) ->
-if(?EMQX_RELEASE_EDITION == ee).
ee_test_columns('schema.validation_failed') ->
[{<<"validation">>, <<"myvalidation">>}] ++
test_columns('message.publish');
ee_test_columns('message.transformation_failed') ->
[{<<"transformation">>, <<"mytransformation">>}] ++
test_columns('message.publish').
%% ELSE (?EMQX_RELEASE_EDITION == ee).
-else.
@ -997,6 +1056,23 @@ columns_with_exam('schema.validation_failed') ->
{<<"timestamp">>, erlang:system_time(millisecond)},
{<<"node">>, node()}
];
columns_with_exam('message.transformation_failed') ->
[
{<<"event">>, 'message.transformation_failed'},
{<<"validation">>, <<"my_transformation">>},
{<<"id">>, emqx_guid:to_hexstr(emqx_guid:gen())},
{<<"clientid">>, <<"c_emqx">>},
{<<"username">>, <<"u_emqx">>},
{<<"payload">>, <<"{\"msg\": \"hello\"}">>},
{<<"peername">>, <<"192.168.0.10:56431">>},
{<<"topic">>, <<"t/a">>},
{<<"qos">>, 1},
{<<"flags">>, #{}},
{<<"publish_received_at">>, erlang:system_time(millisecond)},
columns_example_props(pub_props),
{<<"timestamp">>, erlang:system_time(millisecond)},
{<<"node">>, node()}
];
columns_with_exam('delivery.dropped') ->
[
{<<"event">>, 'delivery.dropped'},
@ -1200,6 +1276,7 @@ hook_fun('session.unsubscribed') -> fun ?MODULE:on_session_unsubscribed/4;
hook_fun('message.delivered') -> fun ?MODULE:on_message_delivered/3;
hook_fun('message.acked') -> fun ?MODULE:on_message_acked/3;
hook_fun('message.dropped') -> fun ?MODULE:on_message_dropped/4;
hook_fun('message.transformation_failed') -> fun ?MODULE:on_message_transformation_failed/3;
hook_fun('schema.validation_failed') -> fun ?MODULE:on_schema_validation_failed/3;
hook_fun('delivery.dropped') -> fun ?MODULE:on_delivery_dropped/4;
hook_fun('message.publish') -> fun ?MODULE:on_message_publish/2;
@ -1231,6 +1308,7 @@ event_name(<<"$events/session_unsubscribed">>) -> 'session.unsubscribed';
event_name(<<"$events/message_delivered">>) -> 'message.delivered';
event_name(<<"$events/message_acked">>) -> 'message.acked';
event_name(<<"$events/message_dropped">>) -> 'message.dropped';
event_name(<<"$events/message_transformation_failed">>) -> 'message.transformation_failed';
event_name(<<"$events/schema_validation_failed">>) -> 'schema.validation_failed';
event_name(<<"$events/delivery_dropped">>) -> 'delivery.dropped';
event_name(_) -> 'message.publish'.
@ -1246,6 +1324,7 @@ event_topic('session.unsubscribed') -> <<"$events/session_unsubscribed">>;
event_topic('message.delivered') -> <<"$events/message_delivered">>;
event_topic('message.acked') -> <<"$events/message_acked">>;
event_topic('message.dropped') -> <<"$events/message_dropped">>;
event_topic('message.transformation_failed') -> <<"$events/message_transformation_failed">>;
event_topic('schema.validation_failed') -> <<"$events/schema_validation_failed">>;
event_topic('delivery.dropped') -> <<"$events/delivery_dropped">>;
event_topic('message.publish') -> <<"$events/message_publish">>.

2
apps/emqx_rule_engine/src/emqx_rule_runtime.erl
View File

@ -27,7 +27,7 @@
inc_action_metrics/2
]).
%% Internal exports used by schema validation
%% Internal exports used by schema validation and message transformation.
-export([evaluate_select/3, clear_rule_payload/0]).
-import(

3
apps/emqx_schema_registry/src/emqx_schema_registry.app.src
View File

@ -11,7 +11,8 @@
stdlib,
erlavro,
gpb,
jesse
jesse,
emqx
]},
{env, []},
{modules, []},

11
apps/emqx_schema_registry/src/emqx_schema_registry.erl
View File

@ -16,6 +16,8 @@
start_link/0,
add_schema/2,
get_schema/1,
is_existing_type/1,
is_existing_type/2,
delete_schema/1,
list_schemas/0
]).
@ -52,6 +54,7 @@
%% API
%%-------------------------------------------------------------------------------------------------
-spec start_link() -> gen_server:start_ret().
start_link() ->
gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
@ -64,6 +67,14 @@ get_serde(SchemaName) ->
{ok, Serde}
end.
-spec is_existing_type(schema_name()) -> boolean().
is_existing_type(SchemaName) ->
is_existing_type(SchemaName, []).
-spec is_existing_type(schema_name(), [binary()]) -> boolean().
is_existing_type(SchemaName, Path) ->
emqx_schema_registry_serde:is_existing_type(SchemaName, Path).
-spec get_schema(schema_name()) -> {ok, map()} | {error, not_found}.
get_schema(SchemaName) ->
case

45
apps/emqx_schema_registry/src/emqx_schema_registry_serde.erl
View File

@ -3,6 +3,8 @@
%%--------------------------------------------------------------------
-module(emqx_schema_registry_serde).
-feature(maybe_expr, enable).
-behaviour(emqx_rule_funcs).
-include("emqx_schema_registry.hrl").
@ -14,6 +16,8 @@
make_serde/3,
handle_rule_function/2,
schema_check/3,
is_existing_type/1,
is_existing_type/2,
destroy/1
]).
@ -27,6 +31,10 @@
eval_encode/2
]).
%%------------------------------------------------------------------------------
%% Type definitions
%%------------------------------------------------------------------------------
-define(BOOL(SerdeName, EXPR),
try
_ = EXPR,
@ -38,10 +46,28 @@
end
).
-type eval_context() :: term().
-export_type([serde_type/0]).
%%------------------------------------------------------------------------------
%% API
%%------------------------------------------------------------------------------
-spec is_existing_type(schema_name()) -> boolean().
is_existing_type(SchemaName) ->
is_existing_type(SchemaName, []).
-spec is_existing_type(schema_name(), [binary()]) -> boolean().
is_existing_type(SchemaName, Path) ->
maybe
{ok, #serde{type = SerdeType, eval_context = EvalContext}} ?=
emqx_schema_registry:get_serde(SchemaName),
has_inner_type(SerdeType, EvalContext, Path)
else
_ -> false
end.
-spec handle_rule_function(atom(), list()) -> any() | {error, no_match_for_function}.
handle_rule_function(sparkplug_decode, [Data]) ->
handle_rule_function(
@ -338,3 +364,22 @@ unload_code(SerdeMod) ->
_ = code:purge(SerdeMod),
_ = code:delete(SerdeMod),
ok.
-spec has_inner_type(serde_type(), eval_context(), [binary()]) ->
boolean().
has_inner_type(protobuf, _SerdeMod, [_, _ | _]) ->
%% Protobuf only has one level of message types.
false;
has_inner_type(protobuf, SerdeMod, [MessageTypeBin]) ->
try apply(SerdeMod, get_msg_names, []) of
Names ->
lists:member(MessageTypeBin, [atom_to_binary(N, utf8) || N <- Names])
catch
_:_ ->
false
end;
has_inner_type(_SerdeType, _EvalContext, []) ->
%% This function is only called if we already found a serde, so the root does exist.
true;
has_inner_type(_SerdeType, _EvalContext, _Path) ->
false.

43
apps/emqx_schema_registry/test/emqx_schema_registry_serde_SUITE.erl
View File

@ -14,7 +14,6 @@
-import(emqx_common_test_helpers, [on_exit/1]).
-define(APPS, [emqx_conf, emqx_rule_engine, emqx_schema_registry]).
-define(INVALID_JSON, #{
reason := #{expected := "emqx_schema:json_binary()"},
kind := validation_error
@ -28,12 +27,20 @@ all() ->
emqx_common_test_helpers:all(?MODULE).
init_per_suite(Config) ->
emqx_config:save_schema_mod_and_names(emqx_schema_registry_schema),
emqx_mgmt_api_test_util:init_suite(?APPS),
Config.
Apps = emqx_cth_suite:start(
[
emqx,
emqx_conf,
emqx_schema_registry,
emqx_rule_engine
],
#{work_dir => emqx_cth_suite:work_dir(Config)}
),
[{apps, Apps} | Config].
end_per_suite(_Config) ->
emqx_mgmt_api_test_util:end_suite(lists:reverse(?APPS)),
end_per_suite(Config) ->
Apps = ?config(apps, Config),
emqx_cth_suite:stop(Apps),
ok.
init_per_testcase(_TestCase, Config) ->
Config.
@ -240,3 +247,27 @@ t_json_validation(_Config) ->
?assertNot(F(schema_check, <<"{\"bar\": 2}">>)),
?assertNot(F(schema_check, <<"{\"foo\": \"notinteger\", \"bar\": 2}">>)),
ok.
t_is_existing_type(_Config) ->
JsonName = <<"myjson">>,
?assertNot(emqx_schema_registry:is_existing_type(JsonName)),
ok = emqx_schema_registry:add_schema(JsonName, schema_params(json)),
AvroName = <<"myavro">>,
?assertNot(emqx_schema_registry:is_existing_type(AvroName)),
ok = emqx_schema_registry:add_schema(AvroName, schema_params(avro)),
ProtobufName = <<"myprotobuf">>,
MessageType = <<"Person">>,
?assertNot(emqx_schema_registry:is_existing_type(ProtobufName)),
ok = emqx_schema_registry:add_schema(ProtobufName, schema_params(protobuf)),
%% JSON Schema: no inner names
?assert(emqx_schema_registry:is_existing_type(JsonName)),
?assertNot(emqx_schema_registry:is_existing_type(JsonName, [JsonName])),
%% Avro: no inner names
?assert(emqx_schema_registry:is_existing_type(AvroName)),
?assertNot(emqx_schema_registry:is_existing_type(AvroName, [AvroName])),
%% Protobuf: one level of message types
?assert(emqx_schema_registry:is_existing_type(ProtobufName)),
?assertNot(emqx_schema_registry:is_existing_type(ProtobufName, [ProtobufName])),
?assert(emqx_schema_registry:is_existing_type(ProtobufName, [MessageType])),
?assertNot(emqx_schema_registry:is_existing_type(ProtobufName, [MessageType, MessageType])),
ok.

2
apps/emqx_schema_validation/BSL.txt
View File

@ -2,7 +2,7 @@ Business Source License 1.1
Licensor: Hangzhou EMQ Technologies Co., Ltd.
Licensed Work: EMQX Enterprise Edition
The Licensed Work is (c) 2023
The Licensed Work is (c) 2024
Hangzhou EMQ Technologies Co., Ltd.
Additional Use Grant: Students and educators are granted right to copy,
modify, and create derivative work for research

2
apps/emqx_schema_validation/README.md
View File

@ -6,7 +6,7 @@ the message without further processing, or to disconnect the offending client as
# Documentation
Refer to [Message
Refer to [Schema
Validation](https://docs.emqx.com/en/enterprise/latest/data-integration/schema-validation.html)
for more information about the semantics and checks available.

5
apps/emqx_schema_validation/src/emqx_schema_validation.app.src
View File

@ -1,11 +1,12 @@
{application, emqx_schema_validation, [
{description, "EMQX Schema Validation"},
{vsn, "0.1.0"},
{vsn, "0.1.1"},
{registered, [emqx_schema_validation_sup, emqx_schema_validation_registry]},
{mod, {emqx_schema_validation_app, []}},
{applications, [
kernel,
stdlib
stdlib,
emqx_schema_registry
]},
{env, []},
{modules, []},

364
apps/emqx_schema_validation/src/emqx_schema_validation.erl
View File

@ -3,6 +3,8 @@
%%--------------------------------------------------------------------
-module(emqx_schema_validation).
-feature(maybe_expr, enable).
-include_lib("snabbkaffe/include/trace.hrl").
-include_lib("emqx_utils/include/emqx_message.hrl").
-include_lib("emqx/include/emqx_hooks.hrl").
@ -10,12 +12,6 @@
%% API
-export([
add_handler/0,
remove_handler/0,
load/0,
unload/0,
list/0,
reorder/1,
lookup/1,
@ -32,13 +28,6 @@
on_message_publish/1
]).
%% `emqx_config_handler' API
-export([pre_config_update/3, post_config_update/5]).
%% `emqx_config_backup' API
-behaviour(emqx_config_backup).
-export([import_config/1]).
%% Internal exports
-export([parse_sql_check/1]).
@ -52,91 +41,52 @@
%%------------------------------------------------------------------------------
-define(TRACE_TAG, "SCHEMA_VALIDATION").
-define(CONF_ROOT, schema_validation).
-define(CONF_ROOT_BIN, <<"schema_validation">>).
-define(VALIDATIONS_CONF_PATH, [?CONF_ROOT, validations]).
-type validation_name() :: binary().
-type validation() :: _TODO.
-type raw_validation() :: #{binary() => _}.
-type validation() :: #{
name := validation_name(),
strategy := all_pass | any_pass,
failure_action := drop | disconnect | ignore,
log_failure := #{level := error | warning | notice | info | debug | none}
}.
-export_type([
validation/0,
validation_name/0
]).
%%------------------------------------------------------------------------------
%% API
%%------------------------------------------------------------------------------
-spec add_handler() -> ok.
add_handler() ->
ok = emqx_config_handler:add_handler([?CONF_ROOT], ?MODULE),
ok = emqx_config_handler:add_handler(?VALIDATIONS_CONF_PATH, ?MODULE),
ok.
-spec remove_handler() -> ok.
remove_handler() ->
ok = emqx_config_handler:remove_handler(?VALIDATIONS_CONF_PATH),
ok = emqx_config_handler:remove_handler([?CONF_ROOT]),
ok.
load() ->
Validations = emqx:get_config(?VALIDATIONS_CONF_PATH, []),
lists:foreach(
fun({Pos, Validation}) ->
ok = emqx_schema_validation_registry:insert(Pos, Validation)
end,
lists:enumerate(Validations)
).
unload() ->
Validations = emqx:get_config(?VALIDATIONS_CONF_PATH, []),
lists:foreach(
fun(Validation) ->
ok = emqx_schema_validation_registry:delete(Validation)
end,
Validations
).
-spec list() -> [validation()].
list() ->
emqx:get_config(?VALIDATIONS_CONF_PATH, []).
emqx_schema_validation_config:list().
-spec reorder([validation_name()]) ->
{ok, _} | {error, _}.
reorder(Order) ->
emqx_conf:update(
?VALIDATIONS_CONF_PATH,
{reorder, Order},
#{override_to => cluster}
).
emqx_schema_validation_config:reorder(Order).
-spec lookup(validation_name()) -> {ok, validation()} | {error, not_found}.
lookup(Name) ->
Validations = emqx:get_config(?VALIDATIONS_CONF_PATH, []),
do_lookup(Name, Validations).
emqx_schema_validation_config:lookup(Name).
-spec insert(validation()) ->
-spec insert(raw_validation()) ->
{ok, _} | {error, _}.
insert(Validation) ->
emqx_conf:update(
?VALIDATIONS_CONF_PATH,
{append, Validation},
#{override_to => cluster}
).
emqx_schema_validation_config:insert(Validation).
-spec update(validation()) ->
-spec update(raw_validation()) ->
{ok, _} | {error, _}.
update(Validation) ->
emqx_conf:update(
?VALIDATIONS_CONF_PATH,
{update, Validation},
#{override_to => cluster}
).
emqx_schema_validation_config:update(Validation).
-spec delete(validation_name()) ->
{ok, _} | {error, _}.
delete(Name) ->
emqx_conf:update(
?VALIDATIONS_CONF_PATH,
{delete, Name},
#{override_to => cluster}
).
emqx_schema_validation_config:delete(Name).
%%------------------------------------------------------------------------------
%% Hooks
@ -144,7 +94,7 @@ delete(Name) ->
-spec register_hooks() -> ok.
register_hooks() ->
emqx_hooks:put('message.publish', {?MODULE, on_message_publish, []}, ?HP_MSG_VALIDATION).
emqx_hooks:put('message.publish', {?MODULE, on_message_publish, []}, ?HP_SCHEMA_VALIDATION).
-spec unregister_hooks() -> ok.
unregister_hooks() ->
@ -175,116 +125,6 @@ on_message_publish(Message = #message{topic = Topic, headers = Headers}) ->
end
end.
%%------------------------------------------------------------------------------
%% `emqx_config_handler' API
%%------------------------------------------------------------------------------
pre_config_update(?VALIDATIONS_CONF_PATH, {append, Validation}, OldValidations) ->
Validations = OldValidations ++ [Validation],
{ok, Validations};
pre_config_update(?VALIDATIONS_CONF_PATH, {update, Validation}, OldValidations) ->
replace(OldValidations, Validation);
pre_config_update(?VALIDATIONS_CONF_PATH, {delete, Validation}, OldValidations) ->
delete(OldValidations, Validation);
pre_config_update(?VALIDATIONS_CONF_PATH, {reorder, Order}, OldValidations) ->
reorder(OldValidations, Order);
pre_config_update([?CONF_ROOT], {merge, NewConfig}, OldConfig) ->
#{resulting_config := Config} = prepare_config_merge(NewConfig, OldConfig),
{ok, Config};
pre_config_update([?CONF_ROOT], {replace, NewConfig}, _OldConfig) ->
{ok, NewConfig}.
post_config_update(?VALIDATIONS_CONF_PATH, {append, #{<<"name">> := Name}}, New, _Old, _AppEnvs) ->
{Pos, Validation} = fetch_with_index(New, Name),
ok = emqx_schema_validation_registry:insert(Pos, Validation),
ok;
post_config_update(?VALIDATIONS_CONF_PATH, {update, #{<<"name">> := Name}}, New, Old, _AppEnvs) ->
{_Pos, OldValidation} = fetch_with_index(Old, Name),
{Pos, NewValidation} = fetch_with_index(New, Name),
ok = emqx_schema_validation_registry:update(OldValidation, Pos, NewValidation),
ok;
post_config_update(?VALIDATIONS_CONF_PATH, {delete, Name}, _New, Old, _AppEnvs) ->
{_Pos, Validation} = fetch_with_index(Old, Name),
ok = emqx_schema_validation_registry:delete(Validation),
ok;
post_config_update(?VALIDATIONS_CONF_PATH, {reorder, _Order}, New, _Old, _AppEnvs) ->
ok = emqx_schema_validation_registry:reindex_positions(New),
ok;
post_config_update([?CONF_ROOT], {merge, _}, ResultingConfig, Old, _AppEnvs) ->
#{validations := ResultingValidations} = ResultingConfig,
#{validations := OldValidations} = Old,
#{added := NewValidations0} =
emqx_utils:diff_lists(
ResultingValidations,
OldValidations,
fun(#{name := N}) -> N end
),
NewValidations =
lists:map(
fun(#{name := Name}) ->
{Pos, Validation} = fetch_with_index(ResultingValidations, Name),
ok = emqx_schema_validation_registry:insert(Pos, Validation),
#{name => Name, pos => Pos}
end,
NewValidations0
),
{ok, #{new_validations => NewValidations}};
post_config_update([?CONF_ROOT], {replace, Input}, ResultingConfig, Old, _AppEnvs) ->
#{
new_validations := NewValidations,
changed_validations := ChangedValidations0,
deleted_validations := DeletedValidations
} = prepare_config_replace(Input, Old),
#{validations := ResultingValidations} = ResultingConfig,
#{validations := OldValidations} = Old,
lists:foreach(
fun(Name) ->
{_Pos, Validation} = fetch_with_index(OldValidations, Name),
ok = emqx_schema_validation_registry:delete(Validation)
end,
DeletedValidations
),
lists:foreach(
fun(Name) ->
{Pos, Validation} = fetch_with_index(ResultingValidations, Name),
ok = emqx_schema_validation_registry:insert(Pos, Validation)
end,
NewValidations
),
ChangedValidations =
lists:map(
fun(Name) ->
{_Pos, OldValidation} = fetch_with_index(OldValidations, Name),
{Pos, NewValidation} = fetch_with_index(ResultingValidations, Name),
ok = emqx_schema_validation_registry:update(OldValidation, Pos, NewValidation),
#{name => Name, pos => Pos}
end,
ChangedValidations0
),
ok = emqx_schema_validation_registry:reindex_positions(ResultingValidations),
{ok, #{changed_validations => ChangedValidations}}.
%%------------------------------------------------------------------------------
%% `emqx_config_backup' API
%%------------------------------------------------------------------------------
import_config(#{?CONF_ROOT_BIN := RawConf0}) ->
Result = emqx_conf:update(
[?CONF_ROOT],
{merge, RawConf0},
#{override_to => cluster, rawconf_with_defaults => true}
),
case Result of
{error, Reason} ->
{error, #{root_key => ?CONF_ROOT, reason => Reason}};
{ok, _} ->
Keys0 = maps:keys(RawConf0),
ChangedPaths = Keys0 -- [<<"validations">>],
{ok, #{root_key => ?CONF_ROOT, changed => ChangedPaths}}
end;
import_config(_RawConf) ->
{ok, #{root_key => ?CONF_ROOT, changed => []}}.
%%------------------------------------------------------------------------------
%% Internal exports
%%------------------------------------------------------------------------------
@ -370,112 +210,6 @@ evaluate_schema_check(Check, Validation, #message{payload = Data}) ->
false
end.
replace(OldValidations, Validation = #{<<"name">> := Name}) ->
{Found, RevNewValidations} =
lists:foldl(
fun
(#{<<"name">> := NameIn}, {_FoundIn, Acc}) when NameIn =:= Name ->
{true, [Validation | Acc]};
(Val, {FoundIn, Acc}) ->
{FoundIn, [Val | Acc]}
end,
{false, []},
OldValidations
),
case Found of
true ->
{ok, lists:reverse(RevNewValidations)};
false ->
{error, not_found}
end.
delete(OldValidations, Name) ->
{Found, RevNewValidations} =
lists:foldl(
fun
(#{<<"name">> := NameIn}, {_FoundIn, Acc}) when NameIn =:= Name ->
{true, Acc};
(Val, {FoundIn, Acc}) ->
{FoundIn, [Val | Acc]}
end,
{false, []},
OldValidations
),
case Found of
true ->
{ok, lists:reverse(RevNewValidations)};
false ->
{error, not_found}
end.
reorder(Validations, Order) ->
Context = #{
not_found => sets:new([{version, 2}]),
duplicated => sets:new([{version, 2}]),
res => [],
seen => sets:new([{version, 2}])
},
reorder(Validations, Order, Context).
reorder(NotReordered, _Order = [], #{not_found := NotFound0, duplicated := Duplicated0, res := Res}) ->
NotFound = sets:to_list(NotFound0),
Duplicated = sets:to_list(Duplicated0),
case {NotReordered, NotFound, Duplicated} of
{[], [], []} ->
{ok, lists:reverse(Res)};
{_, _, _} ->
Error = #{
not_found => NotFound,
duplicated => Duplicated,
not_reordered => [N || #{<<"name">> := N} <- NotReordered]
},
{error, Error}
end;
reorder(RemainingValidations, [Name | Rest], Context0 = #{seen := Seen0}) ->
case sets:is_element(Name, Seen0) of
true ->
Context = maps:update_with(
duplicated, fun(S) -> sets:add_element(Name, S) end, Context0
),
reorder(RemainingValidations, Rest, Context);
false ->
case safe_take(Name, RemainingValidations) of
error ->
Context = maps:update_with(
not_found, fun(S) -> sets:add_element(Name, S) end, Context0
),
reorder(RemainingValidations, Rest, Context);
{ok, {Validation, Front, Rear}} ->
Context1 = maps:update_with(
seen, fun(S) -> sets:add_element(Name, S) end, Context0
),
Context = maps:update_with(res, fun(Vs) -> [Validation | Vs] end, Context1),
reorder(Front ++ Rear, Rest, Context)
end
end.
fetch_with_index([{Pos, #{name := Name} = Validation} | _Rest], Name) ->
{Pos, Validation};
fetch_with_index([{_, _} | Rest], Name) ->
fetch_with_index(Rest, Name);
fetch_with_index(Validations, Name) ->
fetch_with_index(lists:enumerate(Validations), Name).
safe_take(Name, Validations) ->
case lists:splitwith(fun(#{<<"name">> := N}) -> N =/= Name end, Validations) of
{_Front, []} ->
error;
{Front, [Found | Rear]} ->
{ok, {Found, Front, Rear}}
end.
do_lookup(_Name, _Validations = []) ->
{error, not_found};
do_lookup(Name, [#{name := Name} = Validation | _Rest]) ->
{ok, Validation};
do_lookup(Name, [_ | Rest]) ->
do_lookup(Name, Rest).
run_validations(Validations, Message) ->
try
emqx_rule_runtime:clear_rule_payload(),
@ -557,55 +291,3 @@ run_schema_validation_failed_hook(Message, Validation) ->
#{name := Name} = Validation,
ValidationContext = #{name => Name},
emqx_hooks:run('schema.validation_failed', [Message, ValidationContext]).
%% "Merging" in the context of the validation array means:
%% * Existing validations (identified by `name') are left untouched.
%% * No validations are removed.
%% * New validations are appended to the existing list.
%% * Existing validations are not reordered.
prepare_config_merge(NewConfig0, OldConfig) ->
{ImportedRawValidations, NewConfigNoValidations} =
case maps:take(<<"validations">>, NewConfig0) of
error ->
{[], NewConfig0};
{V, R} ->
{V, R}
end,
OldRawValidations = maps:get(<<"validations">>, OldConfig, []),
#{added := NewRawValidations} = emqx_utils:diff_lists(
ImportedRawValidations,
OldRawValidations,
fun(#{<<"name">> := N}) -> N end
),
Config0 = emqx_utils_maps:deep_merge(OldConfig, NewConfigNoValidations),
Config = maps:update_with(
<<"validations">>,
fun(OldVs) -> OldVs ++ NewRawValidations end,
NewRawValidations,
Config0
),
#{
new_validations => NewRawValidations,
resulting_config => Config
}.
prepare_config_replace(NewConfig, OldConfig) ->
ImportedRawValidations = maps:get(<<"validations">>, NewConfig, []),
OldValidations = maps:get(validations, OldConfig, []),
%% Since, at this point, we have an input raw config but a parsed old config, we
%% project both to the to have only their names, and consider common names as changed.
#{
added := NewValidations,
removed := DeletedValidations,
changed := ChangedValidations0,
identical := ChangedValidations1
} = emqx_utils:diff_lists(
lists:map(fun(#{<<"name">> := N}) -> N end, ImportedRawValidations),
lists:map(fun(#{name := N}) -> N end, OldValidations),
fun(N) -> N end
),
#{
new_validations => NewValidations,
changed_validations => ChangedValidations0 ++ ChangedValidations1,
deleted_validations => DeletedValidations
}.

8
apps/emqx_schema_validation/src/emqx_schema_validation_app.erl
View File

@ -19,14 +19,14 @@
-spec start(application:start_type(), term()) -> {ok, pid()}.
start(_Type, _Args) ->
{ok, Sup} = emqx_schema_validation_sup:start_link(),
ok = emqx_schema_validation:add_handler(),
ok = emqx_schema_validation_config:add_handler(),
ok = emqx_schema_validation:register_hooks(),
ok = emqx_schema_validation:load(),
ok = emqx_schema_validation_config:load(),
{ok, Sup}.
-spec stop(term()) -> ok.
stop(_State) ->
ok = emqx_schema_validation:unload(),
ok = emqx_schema_validation_config:unload(),
ok = emqx_schema_validation:unregister_hooks(),
ok = emqx_schema_validation:remove_handler(),
ok = emqx_schema_validation_config:remove_handler(),
ok.

481
apps/emqx_schema_validation/src/emqx_schema_validation_config.erl Normal file
View File

@ -0,0 +1,481 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
-module(emqx_schema_validation_config).
-feature(maybe_expr, enable).
%% API
-export([
add_handler/0,
remove_handler/0,
load/0,
unload/0,
list/0,
reorder/1,
lookup/1,
insert/1,
update/1,
delete/1
]).
%% `emqx_config_handler' API
-export([pre_config_update/3, post_config_update/5]).
%% `emqx_config_backup' API
-behaviour(emqx_config_backup).
-export([import_config/1]).
%%------------------------------------------------------------------------------
%% Type declarations
%%------------------------------------------------------------------------------
-define(CONF_ROOT, schema_validation).
-define(CONF_ROOT_BIN, <<"schema_validation">>).
-define(VALIDATIONS_CONF_PATH, [?CONF_ROOT, validations]).
-type validation_name() :: emqx_schema_validation:validation_name().
-type validation() :: emqx_schema_validation:validation().
-type raw_validation() :: #{binary() => _}.
%%------------------------------------------------------------------------------
%% API
%%------------------------------------------------------------------------------
-spec add_handler() -> ok.
add_handler() ->
ok = emqx_config_handler:add_handler([?CONF_ROOT], ?MODULE),
ok = emqx_config_handler:add_handler(?VALIDATIONS_CONF_PATH, ?MODULE),
ok.
-spec remove_handler() -> ok.
remove_handler() ->
ok = emqx_config_handler:remove_handler(?VALIDATIONS_CONF_PATH),
ok = emqx_config_handler:remove_handler([?CONF_ROOT]),
ok.
load() ->
Validations = emqx:get_config(?VALIDATIONS_CONF_PATH, []),
lists:foreach(
fun({Pos, Validation}) ->
ok = emqx_schema_validation_registry:insert(Pos, Validation)
end,
lists:enumerate(Validations)
).
unload() ->
Validations = emqx:get_config(?VALIDATIONS_CONF_PATH, []),
lists:foreach(
fun({Pos, Validation}) ->
ok = emqx_schema_validation_registry:delete(Validation, Pos)
end,
lists:enumerate(Validations)
).
-spec list() -> [validation()].
list() ->
emqx:get_config(?VALIDATIONS_CONF_PATH, []).
-spec reorder([validation_name()]) ->
{ok, _} | {error, _}.
reorder(Order) ->
emqx_conf:update(
?VALIDATIONS_CONF_PATH,
{reorder, Order},
#{override_to => cluster}
).
-spec lookup(validation_name()) -> {ok, validation()} | {error, not_found}.
lookup(Name) ->
Validations = emqx:get_config(?VALIDATIONS_CONF_PATH, []),
do_lookup(Name, Validations).
-spec insert(raw_validation()) ->
{ok, _} | {error, _}.
insert(Validation) ->
emqx_conf:update(
?VALIDATIONS_CONF_PATH,
{append, Validation},
#{override_to => cluster}
).
-spec update(raw_validation()) ->
{ok, _} | {error, _}.
update(Validation) ->
emqx_conf:update(
?VALIDATIONS_CONF_PATH,
{update, Validation},
#{override_to => cluster}
).
-spec delete(validation_name()) ->
{ok, _} | {error, _}.
delete(Name) ->
emqx_conf:update(
?VALIDATIONS_CONF_PATH,
{delete, Name},
#{override_to => cluster}
).
%%------------------------------------------------------------------------------
%% `emqx_config_handler' API
%%------------------------------------------------------------------------------
pre_config_update(?VALIDATIONS_CONF_PATH, {append, Validation}, OldValidations) ->
Validations = OldValidations ++ [Validation],
{ok, Validations};
pre_config_update(?VALIDATIONS_CONF_PATH, {update, Validation}, OldValidations) ->
replace(OldValidations, Validation);
pre_config_update(?VALIDATIONS_CONF_PATH, {delete, Validation}, OldValidations) ->
delete(OldValidations, Validation);
pre_config_update(?VALIDATIONS_CONF_PATH, {reorder, Order}, OldValidations) ->
reorder(OldValidations, Order);
pre_config_update([?CONF_ROOT], {merge, NewConfig}, OldConfig) ->
#{resulting_config := Config} = prepare_config_merge(NewConfig, OldConfig),
{ok, Config};
pre_config_update([?CONF_ROOT], {replace, NewConfig}, _OldConfig) ->
{ok, NewConfig}.
post_config_update(
?VALIDATIONS_CONF_PATH, {append, #{<<"name">> := Name} = RawValidation}, New, _Old, _AppEnvs
) ->
maybe
ok ?= assert_referenced_schemas_exist(RawValidation),
{Pos, Validation} = fetch_with_index(New, Name),
ok = emqx_schema_validation_registry:insert(Pos, Validation),
ok
end;
post_config_update(
?VALIDATIONS_CONF_PATH, {update, #{<<"name">> := Name} = RawValidation}, New, Old, _AppEnvs
) ->
maybe
ok ?= assert_referenced_schemas_exist(RawValidation),
{_Pos, OldValidation} = fetch_with_index(Old, Name),
{Pos, NewValidation} = fetch_with_index(New, Name),
ok = emqx_schema_validation_registry:update(OldValidation, Pos, NewValidation),
ok
end;
post_config_update(?VALIDATIONS_CONF_PATH, {delete, Name}, _New, Old, _AppEnvs) ->
{Pos, Validation} = fetch_with_index(Old, Name),
ok = emqx_schema_validation_registry:delete(Validation, Pos),
ok;
post_config_update(?VALIDATIONS_CONF_PATH, {reorder, _Order}, New, Old, _AppEnvs) ->
ok = emqx_schema_validation_registry:reindex_positions(New, Old),
ok;
post_config_update([?CONF_ROOT], {merge, _}, ResultingConfig, Old, _AppEnvs) ->
#{validations := ResultingValidations} = ResultingConfig,
#{validations := OldValidations} = Old,
#{added := NewValidations0} =
emqx_utils:diff_lists(
ResultingValidations,
OldValidations,
fun(#{name := N}) -> N end
),
maybe
ok ?= multi_assert_referenced_schemas_exist(NewValidations0),
NewValidations =
lists:map(
fun(#{name := Name}) ->
{Pos, Validation} = fetch_with_index(ResultingValidations, Name),
ok = emqx_schema_validation_registry:insert(Pos, Validation),
#{name => Name, pos => Pos}
end,
NewValidations0
),
{ok, #{new_validations => NewValidations}}
end;
post_config_update([?CONF_ROOT], {replace, Input}, ResultingConfig, Old, _AppEnvs) ->
#{
new_validations := NewValidations,
changed_validations := ChangedValidations0,
deleted_validations := DeletedValidations
} = prepare_config_replace(Input, Old),
#{validations := ResultingValidations} = ResultingConfig,
#{validations := OldValidations} = Old,
NewOrChangedValidationNames = NewValidations ++ ChangedValidations0,
maybe
ok ?=
multi_assert_referenced_schemas_exist(
lists:filter(
fun(#{name := N}) ->
lists:member(N, NewOrChangedValidationNames)
end,
ResultingValidations
)
),
lists:foreach(
fun(Name) ->
{Pos, Validation} = fetch_with_index(OldValidations, Name),
ok = emqx_schema_validation_registry:delete(Validation, Pos)
end,
DeletedValidations
),
lists:foreach(
fun(Name) ->
{Pos, Validation} = fetch_with_index(ResultingValidations, Name),
ok = emqx_schema_validation_registry:insert(Pos, Validation)
end,
NewValidations
),
ChangedValidations =
lists:map(
fun(Name) ->
{_Pos, OldValidation} = fetch_with_index(OldValidations, Name),
{Pos, NewValidation} = fetch_with_index(ResultingValidations, Name),
ok = emqx_schema_validation_registry:update(OldValidation, Pos, NewValidation),
#{name => Name, pos => Pos}
end,
ChangedValidations0
),
ok = emqx_schema_validation_registry:reindex_positions(
ResultingValidations, OldValidations
),
{ok, #{changed_validations => ChangedValidations}}
end.
%%------------------------------------------------------------------------------
%% `emqx_config_backup' API
%%------------------------------------------------------------------------------
import_config(#{?CONF_ROOT_BIN := RawConf0}) ->
Result = emqx_conf:update(
[?CONF_ROOT],
{merge, RawConf0},
#{override_to => cluster, rawconf_with_defaults => true}
),
case Result of
{error, Reason} ->
{error, #{root_key => ?CONF_ROOT, reason => Reason}};
{ok, _} ->
Keys0 = maps:keys(RawConf0),
ChangedPaths = Keys0 -- [<<"validations">>],
{ok, #{root_key => ?CONF_ROOT, changed => ChangedPaths}}
end;
import_config(_RawConf) ->
{ok, #{root_key => ?CONF_ROOT, changed => []}}.
%%------------------------------------------------------------------------------
%% Internal fns
%%------------------------------------------------------------------------------
replace(OldValidations, Validation = #{<<"name">> := Name}) ->
{Found, RevNewValidations} =
lists:foldl(
fun
(#{<<"name">> := NameIn}, {_FoundIn, Acc}) when NameIn =:= Name ->
{true, [Validation | Acc]};
(Val, {FoundIn, Acc}) ->
{FoundIn, [Val | Acc]}
end,
{false, []},
OldValidations
),
case Found of
true ->
{ok, lists:reverse(RevNewValidations)};
false ->
{error, not_found}
end.
delete(OldValidations, Name) ->
{Found, RevNewValidations} =
lists:foldl(
fun
(#{<<"name">> := NameIn}, {_FoundIn, Acc}) when NameIn =:= Name ->
{true, Acc};
(Val, {FoundIn, Acc}) ->
{FoundIn, [Val | Acc]}
end,
{false, []},
OldValidations
),
case Found of
true ->
{ok, lists:reverse(RevNewValidations)};
false ->
{error, not_found}
end.
reorder(Validations, Order) ->
Context = #{
not_found => sets:new([{version, 2}]),
duplicated => sets:new([{version, 2}]),
res => [],
seen => sets:new([{version, 2}])
},
reorder(Validations, Order, Context).
reorder(NotReordered, _Order = [], #{not_found := NotFound0, duplicated := Duplicated0, res := Res}) ->
NotFound = sets:to_list(NotFound0),
Duplicated = sets:to_list(Duplicated0),
case {NotReordered, NotFound, Duplicated} of
{[], [], []} ->
{ok, lists:reverse(Res)};
{_, _, _} ->
Error = #{
not_found => NotFound,
duplicated => Duplicated,
not_reordered => [N || #{<<"name">> := N} <- NotReordered]
},
{error, Error}
end;
reorder(RemainingValidations, [Name | Rest], Context0 = #{seen := Seen0}) ->
case sets:is_element(Name, Seen0) of
true ->
Context = maps:update_with(
duplicated, fun(S) -> sets:add_element(Name, S) end, Context0
),
reorder(RemainingValidations, Rest, Context);
false ->
case safe_take(Name, RemainingValidations) of
error ->
Context = maps:update_with(
not_found, fun(S) -> sets:add_element(Name, S) end, Context0
),
reorder(RemainingValidations, Rest, Context);
{ok, {Validation, Front, Rear}} ->
Context1 = maps:update_with(
seen, fun(S) -> sets:add_element(Name, S) end, Context0
),
Context = maps:update_with(res, fun(Vs) -> [Validation | Vs] end, Context1),
reorder(Front ++ Rear, Rest, Context)
end
end.
fetch_with_index([{Pos, #{name := Name} = Validation} | _Rest], Name) ->
{Pos, Validation};
fetch_with_index([{_, _} | Rest], Name) ->
fetch_with_index(Rest, Name);
fetch_with_index(Validations, Name) ->
fetch_with_index(lists:enumerate(Validations), Name).
safe_take(Name, Validations) ->
case lists:splitwith(fun(#{<<"name">> := N}) -> N =/= Name end, Validations) of
{_Front, []} ->
error;
{Front, [Found | Rear]} ->
{ok, {Found, Front, Rear}}
end.
do_lookup(_Name, _Validations = []) ->
{error, not_found};
do_lookup(Name, [#{name := Name} = Validation | _Rest]) ->
{ok, Validation};
do_lookup(Name, [_ | Rest]) ->
do_lookup(Name, Rest).
%% "Merging" in the context of the validation array means:
%% * Existing validations (identified by `name') are left untouched.
%% * No validations are removed.
%% * New validations are appended to the existing list.
%% * Existing validations are not reordered.
prepare_config_merge(NewConfig0, OldConfig) ->
{ImportedRawValidations, NewConfigNoValidations} =
case maps:take(<<"validations">>, NewConfig0) of
error ->
{[], NewConfig0};
{V, R} ->
{V, R}
end,
OldRawValidations = maps:get(<<"validations">>, OldConfig, []),
#{added := NewRawValidations} = emqx_utils:diff_lists(
ImportedRawValidations,
OldRawValidations,
fun(#{<<"name">> := N}) -> N end
),
Config0 = emqx_utils_maps:deep_merge(OldConfig, NewConfigNoValidations),
Config = maps:update_with(
<<"validations">>,
fun(OldVs) -> OldVs ++ NewRawValidations end,
NewRawValidations,
Config0
),
#{
new_validations => NewRawValidations,
resulting_config => Config
}.
prepare_config_replace(NewConfig, OldConfig) ->
ImportedRawValidations = maps:get(<<"validations">>, NewConfig, []),
OldValidations = maps:get(validations, OldConfig, []),
%% Since, at this point, we have an input raw config but a parsed old config, we
%% project both to the to have only their names, and consider common names as changed.
#{
added := NewValidations,
removed := DeletedValidations,
changed := ChangedValidations0,
identical := ChangedValidations1
} = emqx_utils:diff_lists(
lists:map(fun(#{<<"name">> := N}) -> N end, ImportedRawValidations),
lists:map(fun(#{name := N}) -> N end, OldValidations),
fun(N) -> N end
),
#{
new_validations => NewValidations,
changed_validations => ChangedValidations0 ++ ChangedValidations1,
deleted_validations => DeletedValidations
}.
-spec assert_referenced_schemas_exist(raw_validation()) -> ok | {error, map()}.
assert_referenced_schemas_exist(RawValidation) ->
#{<<"checks">> := RawChecks} = RawValidation,
SchemasToCheck =
lists:filtermap(
fun
(#{<<"schema">> := SchemaName} = Check) ->
%% so far, only protobuf has inner types
InnerPath =
case maps:find(<<"message_type">>, Check) of
{ok, MessageType} -> [MessageType];
error -> []
end,
{true, {SchemaName, InnerPath}};
(_Check) ->
false
end,
RawChecks
),
do_assert_referenced_schemas_exist(SchemasToCheck).
do_assert_referenced_schemas_exist(SchemasToCheck) ->
MissingSchemas =
lists:foldl(
fun({SchemaName, InnerPath}, Acc) ->
case emqx_schema_registry:is_existing_type(SchemaName, InnerPath) of
true ->
Acc;
false ->
[[SchemaName | InnerPath] | Acc]
end
end,
[],
SchemasToCheck
),
case MissingSchemas of
[] ->
ok;
[_ | _] ->
{error, #{missing_schemas => MissingSchemas}}
end.
-spec multi_assert_referenced_schemas_exist([validation()]) -> ok | {error, map()}.
multi_assert_referenced_schemas_exist(Validations) ->
SchemasToCheck =
lists:filtermap(
fun
(#{schema := SchemaName} = Check) ->
%% so far, only protobuf has inner types
InnerPath =
case maps:find(message_type, Check) of
{ok, MessageType} -> [MessageType];
error -> []
end,
{true, {SchemaName, InnerPath}};
(_Check) ->
false
end,
[Check || #{checks := Checks} <- Validations, Check <- Checks]
),
do_assert_referenced_schemas_exist(SchemasToCheck).

98
apps/emqx_schema_validation/src/emqx_schema_validation_registry.erl
View File

@ -10,8 +10,8 @@
lookup/1,
insert/2,
update/3,
delete/1,
reindex_positions/1,
delete/2,
reindex_positions/2,
matching_validations/1,
@ -51,10 +51,10 @@
-type validation() :: _TODO.
-type position_index() :: pos_integer().
-record(reindex_positions, {validations :: [validation()]}).
-record(reindex_positions, {new_validations :: [validation()], old_validations :: [validation()]}).
-record(insert, {pos :: position_index(), validation :: validation()}).
-record(update, {old :: validation(), pos :: position_index(), new :: validation()}).
-record(delete, {validation :: validation()}).
-record(delete, {validation :: validation(), pos :: position_index()}).
%%------------------------------------------------------------------------------
%% API
@ -74,9 +74,16 @@ lookup(Name) ->
{ok, Validation}
end.
-spec reindex_positions([validation()]) -> ok.
reindex_positions(Validations) ->
gen_server:call(?MODULE, #reindex_positions{validations = Validations}, infinity).
-spec reindex_positions([validation()], [validation()]) -> ok.
reindex_positions(NewValidations, OldValidations) ->
gen_server:call(
?MODULE,
#reindex_positions{
new_validations = NewValidations,
old_validations = OldValidations
},
infinity
).
-spec insert(position_index(), validation()) -> ok.
insert(Pos, Validation) ->
@ -86,23 +93,36 @@ insert(Pos, Validation) ->
update(Old, Pos, New) ->
gen_server:call(?MODULE, #update{old = Old, pos = Pos, new = New}, infinity).
-spec delete(validation()) -> ok.
delete(Validation) ->
gen_server:call(?MODULE, #delete{validation = Validation}, infinity).
-spec delete(validation(), position_index()) -> ok.
delete(Validation, Pos) ->
gen_server:call(?MODULE, #delete{validation = Validation, pos = Pos}, infinity).
%% @doc Returns a list of matching validation names, sorted by their configuration order.
-spec matching_validations(emqx_types:topic()) -> [validation()].
matching_validations(Topic) ->
Validations0 = [
{Pos, Validation}
|| M <- emqx_topic_index:matches(Topic, ?VALIDATION_TOPIC_INDEX, [unique]),
[Pos] <- [emqx_topic_index:get_record(M, ?VALIDATION_TOPIC_INDEX)],
{ok, Validation} <- [
lookup(emqx_topic_index:get_id(M))
]
],
Validations1 = lists:sort(fun({Pos1, _V1}, {Pos2, _V2}) -> Pos1 =< Pos2 end, Validations0),
lists:map(fun({_Pos, V}) -> V end, Validations1).
Validations0 =
lists:flatmap(
fun(M) ->
case emqx_topic_index:get_record(M, ?VALIDATION_TOPIC_INDEX) of
[Name] ->
[Name];
_ ->
[]
end
end,
emqx_topic_index:matches(Topic, ?VALIDATION_TOPIC_INDEX, [unique])
),
lists:flatmap(
fun(Name) ->
case lookup(Name) of
{ok, Validation} ->
[Validation];
_ ->
[]
end
end,
Validations0
).
-spec metrics_worker_spec() -> supervisor:child_spec().
metrics_worker_spec() ->
@ -133,8 +153,15 @@ init(_) ->
State = #{},
{ok, State}.
handle_call(#reindex_positions{validations = Validations}, _From, State) ->
do_reindex_positions(Validations),
handle_call(
#reindex_positions{
new_validations = NewValidations,
old_validations = OldValidations
},
_From,
State
) ->
do_reindex_positions(NewValidations, OldValidations),
{reply, ok, State};
handle_call(#insert{pos = Pos, validation = Validation}, _From, State) ->
do_insert(Pos, Validation),
@ -142,8 +169,8 @@ handle_call(#insert{pos = Pos, validation = Validation}, _From, State) ->
handle_call(#update{old = OldValidation, pos = Pos, new = NewValidation}, _From, State) ->
ok = do_update(OldValidation, Pos, NewValidation),
{reply, ok, State};
handle_call(#delete{validation = Validation}, _From, State) ->
do_delete(Validation),
handle_call(#delete{validation = Validation, pos = Pos}, _From, State) ->
do_delete(Validation, Pos),
{reply, ok, State};
handle_call(_Call, _From, State) ->
{reply, ignored, State}.
@ -160,7 +187,14 @@ create_tables() ->
_ = emqx_utils_ets:new(?VALIDATION_TAB, [public, ordered_set, {read_concurrency, true}]),
ok.
do_reindex_positions(Validations) ->
do_reindex_positions(NewValidations, OldValidations) ->
lists:foreach(
fun({Pos, Validation}) ->
#{topics := Topics} = Validation,
delete_topic_index(Pos, Topics)
end,
lists:enumerate(OldValidations)
),
lists:foreach(
fun({Pos, Validation}) ->
#{
@ -170,7 +204,7 @@ do_reindex_positions(Validations) ->
do_insert_into_tab(Name, Validation, Pos),
update_topic_index(Name, Pos, Topics)
end,
lists:enumerate(Validations)
lists:enumerate(NewValidations)
).
do_insert(Pos, Validation) ->
@ -193,17 +227,17 @@ do_update(OldValidation, Pos, NewValidation) ->
} = NewValidation,
maybe_create_metrics(Name),
do_insert_into_tab(Name, NewValidation, Pos),
delete_topic_index(Name, OldTopics),
delete_topic_index(Pos, OldTopics),
Enabled andalso update_topic_index(Name, Pos, NewTopics),
ok.
do_delete(Validation) ->
do_delete(Validation, Pos) ->
#{
name := Name,
topics := Topics
} = Validation,
ets:delete(?VALIDATION_TAB, Name),
delete_topic_index(Name, Topics),
delete_topic_index(Pos, Topics),
drop_metrics(Name),
ok.
@ -226,15 +260,15 @@ drop_metrics(Name) ->
update_topic_index(Name, Pos, Topics) ->
lists:foreach(
fun(Topic) ->
true = emqx_topic_index:insert(Topic, Name, Pos, ?VALIDATION_TOPIC_INDEX)
true = emqx_topic_index:insert(Topic, Pos, Name, ?VALIDATION_TOPIC_INDEX)
end,
Topics
).
delete_topic_index(Name, Topics) ->
delete_topic_index(Pos, Topics) ->
lists:foreach(
fun(Topic) ->
true = emqx_topic_index:delete(Topic, Name, ?VALIDATION_TOPIC_INDEX)
true = emqx_topic_index:delete(Topic, Pos, ?VALIDATION_TOPIC_INDEX)
end,
Topics
).

113
apps/emqx_schema_validation/test/emqx_schema_validation_http_api_SUITE.erl
View File

@ -356,9 +356,22 @@ protobuf_invalid_payloads() ->
].
protobuf_create_serde(SerdeName) ->
Source =
protobuf_upsert_serde(SerdeName, <<"Person">>).
protobuf_upsert_serde(SerdeName, MessageType) ->
Source = protobuf_source(MessageType),
Schema = #{type => protobuf, source => Source},
ok = emqx_schema_registry:add_schema(SerdeName, Schema),
on_exit(fun() -> ok = emqx_schema_registry:delete_schema(SerdeName) end),
ok.
protobuf_source(MessageType) ->
iolist_to_binary(
[
<<"message ">>,
MessageType,
<<" {\n">>,
<<
"message Person {\n"
" required string name = 1;\n"
" required int32 id = 2;\n"
" optional string email = 3;\n"
@ -369,11 +382,9 @@ protobuf_create_serde(SerdeName) ->
" string b = 2;\n"
" }\n"
"}"
>>,
Schema = #{type => protobuf, source => Source},
ok = emqx_schema_registry:add_schema(SerdeName, Schema),
on_exit(fun() -> ok = emqx_schema_registry:delete_schema(SerdeName) end),
ok.
>>
]
).
%% Checks that the internal order in the registry/index matches expectation.
assert_index_order(ExpectedOrder, Topic, Comment) ->
@ -1044,6 +1055,7 @@ t_duplicated_schema_checks(_Config) ->
Name1 = <<"foo">>,
SerdeName = <<"myserde">>,
Check = schema_check(json, SerdeName),
json_create_serde(SerdeName),
Validation1 = validation(Name1, [Check, sql_check(), Check]),
?assertMatch({400, _}, insert(Validation1)),
@ -1133,18 +1145,87 @@ t_multiple_validations(_Config) ->
ok.
%% Test that we validate schema registry serde existency when using the HTTP API.
t_schema_check_non_existent_serde(_Config) ->
SerdeName = <<"idontexist">>,
Name1 = <<"foo">>,
Check1 = schema_check(json, SerdeName),
Validation1 = validation(Name1, [Check1]),
{201, _} = insert(Validation1),
?assertMatch({400, _}, insert(Validation1)),
C = connect(<<"c1">>),
{ok, _, [_]} = emqtt:subscribe(C, <<"t/#">>),
Check2 = schema_check(avro, SerdeName),
Validation2 = validation(Name1, [Check2]),
?assertMatch({400, _}, insert(Validation2)),
ok = publish(C, <<"t/1">>, #{i => 10, s => <<"s">>}),
?assertNotReceive({publish, _}),
MessageType = <<"idontexisteither">>,
Check3 = schema_check(protobuf, SerdeName, #{<<"message_type">> => MessageType}),
Validation3 = validation(Name1, [Check3]),
?assertMatch({400, _}, insert(Validation3)),
protobuf_create_serde(SerdeName),
%% Still fails because reference message type doesn't exist.
?assertMatch({400, _}, insert(Validation3)),
ok.
%% Test that we validate schema registry serde existency when loading configs.
t_schema_check_non_existent_serde_load_config(_Config) ->
Name1 = <<"1">>,
SerdeName1 = <<"serde1">>,
MessageType1 = <<"mt">>,
Check1A = schema_check(protobuf, SerdeName1, #{<<"message_type">> => MessageType1}),
Validation1A = validation(Name1, [Check1A]),
protobuf_upsert_serde(SerdeName1, MessageType1),
{201, _} = insert(Validation1A),
Name2 = <<"2">>,
SerdeName2 = <<"serde2">>,
Check2A = schema_check(json, SerdeName2),
Validation2A = validation(Name2, [Check2A]),
json_create_serde(SerdeName2),
{201, _} = insert(Validation2A),
%% Config to load
%% Will replace existing config
MissingMessageType = <<"missing_mt">>,
Check1B = schema_check(protobuf, SerdeName1, #{<<"message_type">> => MissingMessageType}),
Validation1B = validation(Name1, [Check1B]),
%% Will replace existing config
MissingSerdeName1 = <<"missing1">>,
Check2B = schema_check(json, MissingSerdeName1),
Validation2B = validation(Name2, [Check2B]),
%% New validation; should be appended
Name3 = <<"3">>,
MissingSerdeName2 = <<"missing2">>,
Check3 = schema_check(avro, MissingSerdeName2),
Validation3 = validation(Name3, [Check3]),
ConfRootBin = <<"schema_validation">>,
ConfigToLoad1 = #{
ConfRootBin => #{
<<"validations">> => [Validation1B, Validation2B, Validation3]
}
},
ConfigToLoadBin1 = iolist_to_binary(hocon_pp:do(ConfigToLoad1, #{})),
%% Merge
ResMerge = emqx_conf_cli:load_config(ConfigToLoadBin1, #{mode => merge}),
?assertMatch({error, _}, ResMerge),
{error, ErrorMessage1} = ResMerge,
?assertEqual(match, re:run(ErrorMessage1, <<"missing_schemas">>, [{capture, none}])),
?assertEqual(match, re:run(ErrorMessage1, MissingSerdeName1, [{capture, none}])),
?assertEqual(match, re:run(ErrorMessage1, MissingSerdeName2, [{capture, none}])),
?assertEqual(match, re:run(ErrorMessage1, MissingMessageType, [{capture, none}])),
%% Replace
ResReplace = emqx_conf_cli:load_config(ConfigToLoadBin1, #{mode => replace}),
?assertMatch({error, _}, ResReplace),
{error, ErrorMessage2} = ResReplace,
?assertEqual(match, re:run(ErrorMessage2, <<"missing_schemas">>, [{capture, none}])),
?assertEqual(match, re:run(ErrorMessage2, MissingSerdeName1, [{capture, none}])),
?assertEqual(match, re:run(ErrorMessage2, MissingSerdeName2, [{capture, none}])),
?assertEqual(match, re:run(ErrorMessage2, MissingMessageType, [{capture, none}])),
ok.
@ -1235,16 +1316,16 @@ t_schema_check_protobuf(_Config) ->
),
%% Bad config: unknown message name
Check2 = schema_check(protobuf, SerdeName, #{<<"message_type">> => <<"idontexist">>}),
Validation2 = validation(Name1, [Check2]),
{200, _} = update(Validation2),
%% Schema updated to use another message type after validation was created
OtherMessageType = <<"NewPersonType">>,
protobuf_upsert_serde(SerdeName, OtherMessageType),
lists:foreach(
fun(Payload) ->
ok = publish(C, <<"t/1">>, {raw, Payload}),
?assertNotReceive({publish, _})
end,
protobuf_valid_payloads(SerdeName, MessageType)
protobuf_valid_payloads(SerdeName, OtherMessageType)
),
ok.

35
apps/emqx_utils/src/emqx_template.erl
View File

@ -20,6 +20,7 @@
-export([parse/2]).
-export([parse_deep/1]).
-export([parse_deep/2]).
-export([placeholders/1]).
-export([validate/2]).
-export([is_const/1]).
-export([unparse/1]).
@ -143,14 +144,19 @@ parse_accessor(Var) ->
Name
end.
-spec placeholders(t()) -> [varname()].
placeholders(Template) when is_list(Template) ->
[Name || {var, Name, _} <- Template];
placeholders({'$tpl', Template}) ->
placeholders_deep(Template).
%% @doc Validate a template against a set of allowed variables.
%% If the given template contains any variable not in the allowed set, an error
%% is returned.
-spec validate([varname() | {var_namespace, varname()}], t()) ->
ok | {error, [_Error :: {varname(), disallowed}]}.
validate(Allowed, Template) ->
{_, Errors} = render(Template, #{}),
{Used, _} = lists:unzip(Errors),
Used = placeholders(Template),
case find_disallowed(lists:usort(Used), Allowed) of
[] ->
ok;
@ -192,10 +198,13 @@ is_allowed(Var, [{var_namespace, VarPrefix} | Allowed]) ->
false ->
is_allowed(Var, Allowed)
end;
is_allowed(Var, [Var | _Allowed]) ->
is_allowed(Var, [VarAllowed | Rest]) ->
is_same_varname(Var, VarAllowed) orelse is_allowed(Var, Rest).
is_same_varname("", ".") ->
true;
is_allowed(Var, [_ | Allowed]) ->
is_allowed(Var, Allowed).
is_same_varname(V1, V2) ->
V1 =:= V2.
%% @doc Check if a template is constant with respect to rendering, i.e. does not
%% contain any placeholders.
@ -322,6 +331,22 @@ parse_deep_term(Term, Opts) when is_binary(Term) ->
parse_deep_term(Term, _Opts) ->
Term.
-spec placeholders_deep(deeptpl()) -> [varname()].
placeholders_deep(Template) when is_map(Template) ->
maps:fold(
fun(KT, VT, Acc) -> placeholders_deep(KT) ++ placeholders_deep(VT) ++ Acc end,
[],
Template
);
placeholders_deep({list, Template}) when is_list(Template) ->
lists:flatmap(fun placeholders_deep/1, Template);
placeholders_deep({tuple, Template}) when is_list(Template) ->
lists:flatmap(fun placeholders_deep/1, Template);
placeholders_deep(Template) when is_list(Template) ->
placeholders(Template);
placeholders_deep(_Term) ->
[].
render_deep(Template, Context, Opts) when is_map(Template) ->
maps:fold(
fun(KT, VT, {Acc, Errors}) ->

17
apps/emqx_utils/src/emqx_variform.erl
View File

@ -69,7 +69,7 @@ render(Expression, Bindings) ->
render(Expression, Bindings, #{}).
render(#{form := Form}, Bindings, Opts) ->
eval_as_string(Form, Bindings, Opts);
eval_render(Form, Bindings, Opts);
render(Expression, Bindings, Opts) ->
case compile(Expression) of
{ok, Compiled} ->
@ -78,9 +78,16 @@ render(Expression, Bindings, Opts) ->
{error, Reason}
end.
eval_as_string(Expr, Bindings, _Opts) ->
eval_render(Expr, Bindings, Opts) ->
EvalAsStr = maps:get(eval_as_string, Opts, true),
try
{ok, return_str(eval(Expr, Bindings, #{}))}
Result = eval(Expr, Bindings, #{}),
case EvalAsStr of
true ->
{ok, return_str(Result)};
false ->
{ok, Result}
end
catch
throw:Reason ->
{error, Reason};
@ -88,7 +95,7 @@ eval_as_string(Expr, Bindings, _Opts) ->
{error, #{exception => C, reason => E, stack_trace => S}}
end.
%% Force the expression to return binary string.
%% Force the expression to return binary string (in most cases).
return_str(Str) when is_binary(Str) -> Str;
return_str(Num) when is_integer(Num) -> integer_to_binary(Num);
return_str(Num) when is_float(Num) -> float_to_binary(Num, [{decimals, 10}, compact]);
@ -313,7 +320,7 @@ assert_module_allowed(Mod) ->
ok;
false ->
throw(#{
reason => unallowed_veriform_module,
reason => unallowed_variform_module,
module => Mod
})
end.

1
apps/emqx_utils/src/emqx_variform_parser.yrl
View File

@ -22,6 +22,7 @@ Rootsymbol
expr -> call_or_var : '$1'.
%% Function call or variable
call_or_var -> identifier '(' ')' : {call, element(3, '$1'), []}.
call_or_var -> identifier '(' args ')' : {call, element(3, '$1'), '$3'}.
call_or_var -> identifier : {var, element(3, '$1')}.

18
apps/emqx_utils/test/emqx_template_SUITE.erl
View File

@ -128,6 +128,14 @@ t_render_custom_bindings(_) ->
render_string(Template, {?MODULE, []})
).
t_placeholders(_) ->
TString = <<"a:${a},b:${b},c:$${c},d:{${d.d1}},e:${$}{e},lit:${$}{$}">>,
Template = emqx_template:parse(TString),
?assertEqual(
["a", "b", "c", "d.d1"],
emqx_template:placeholders(Template)
).
t_unparse(_) ->
TString = <<"a:${a},b:${b},c:$${c},d:{${d.d1}},e:${$}{e},lit:${$}{$}">>,
Template = emqx_template:parse(TString),
@ -337,6 +345,16 @@ t_unparse_tmpl_deep(_) ->
Template = emqx_template:parse_deep(Term),
?assertEqual(Term, emqx_template:unparse(Template)).
t_allow_this(_) ->
?assertEqual(
{error, [{"", disallowed}]},
emqx_template:validate(["d"], emqx_template:parse(<<"this:${}">>))
),
?assertEqual(
{error, [{"", disallowed}]},
emqx_template:validate(["d"], emqx_template:parse(<<"this:${.}">>))
).
t_allow_var_by_namespace(_) ->
Context = #{d => #{d1 => <<"hi">>}},
Template = emqx_template:parse(<<"d.d1:${d.d1}">>),

10
apps/emqx_utils/test/emqx_variform_tests.erl
View File

@ -126,7 +126,7 @@ inject_allowed_module_test() ->
render(atom_to_list(?MODULE) ++ ".concat('a','b')", #{})
),
?assertMatch(
{error, #{reason := unallowed_veriform_module, module := emqx}},
{error, #{reason := unallowed_variform_module, module := emqx}},
render("emqx.concat('a','b')", #{})
)
after
@ -231,8 +231,12 @@ syntax_error_test_() ->
{"const string single quote", fun() -> ?assertMatch(?SYNTAX_ERROR, render("'a'", #{})) end},
{"const string double quote", fun() ->
?assertMatch(?SYNTAX_ERROR, render(<<"\"a\"">>, #{}))
end},
{"no arity", fun() -> ?assertMatch(?SYNTAX_ERROR, render("concat()", #{})) end}
end}
].
maps_test_() ->
[
{"arity zero", ?_assertEqual({ok, <<"0">>}, render(<<"maps.size(maps.new())">>, #{}))}
].
render(Expression, Bindings) ->

4
build
View File

@ -397,9 +397,9 @@ function is_ecr_and_enterprise() {
## Build the default docker image based on debian 12.
make_docker() {
local EMQX_BUILDER_VERSION="${EMQX_BUILDER_VERSION:-5.3-7}"
local EMQX_BUILDER_VERSION="${EMQX_BUILDER_VERSION:-5.3-8}"
local EMQX_BUILDER_PLATFORM="${EMQX_BUILDER_PLATFORM:-debian12}"
local OTP_VSN="${OTP_VSN:-26.2.5-1}"
local OTP_VSN="${OTP_VSN:-26.2.5-2}"
local ELIXIR_VSN="${ELIXIR_VSN:-1.15.7}"
local EMQX_BUILDER=${EMQX_BUILDER:-ghcr.io/emqx/emqx-builder/${EMQX_BUILDER_VERSION}:${ELIXIR_VSN}-${OTP_VSN}-${EMQX_BUILDER_PLATFORM}}
local EMQX_RUNNER="${EMQX_RUNNER:-${EMQX_DEFAULT_RUNNER}}"

1
changes/ce/fix-13156.en.md Normal file
View File

@ -0,0 +1 @@
Fix crashes on monitor dashboard page happening after update to v5.7.0.

10
changes/ce/fix-13216.en.md Normal file
View File

@ -0,0 +1,10 @@
Respcet `clientid_prefix` config for MQTT bridges.
As of version 5.4.1, EMQX limits MQTT Client ID lengths to 23 bytes.
Previously, the system included the `clientid_prefix` in the hash calculation of the original, excessively long Client ID, thereby impacting the resulting shortened ID.
Change Details:
- Without Prefix: Behavior remains unchanged; EMQX will hash the entire Client ID into a 23-byte space (when longer than 23 bytes).
- With Prefix:
- Prefix no more than 19 bytes: The prefix is preserved, and the remaining suffix is hashed into a 4-byte space.
- Prefix is 20 or more bytes: EMQX no longer attempts to shorten the Client ID, respecting the configured prefix in its entirety.

1
changes/ee/feat-13210.en.md Normal file
View File

@ -0,0 +1 @@
Now, when inserting or updating a Schema Validation, EMQX will check if the referenced schemas and message types exist in Schema Registry.

1
changes/ee/fix-13227.en.md Normal file
View File

@ -0,0 +1 @@
Fixed an issue with S3 Bridge when running in aggregated mode, where invalid key template in the configuration haven't been reported as an error during bridge setup, but instead caused a storm of hard to recover crashes later.

2
deploy/docker/Dockerfile
View File

@ -1,4 +1,4 @@
ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.3-7:1.15.7-26.2.5-1-debian12
ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.3-8:1.15.7-26.2.5-2-debian12
ARG RUN_FROM=public.ecr.aws/debian/debian:12-slim
ARG SOURCE_TYPE=src # tgz

3
mix.exs
View File

@ -67,7 +67,7 @@ defmodule EMQXUmbrella.MixProject do
github: "emqx/emqtt", tag: "1.10.1", override: true, system_env: maybe_no_quic_env()},
{:rulesql, github: "emqx/rulesql", tag: "0.2.1"},
{:observer_cli, "1.7.1"},
{:system_monitor, github: "ieQu1/system_monitor", tag: "3.0.3"},
{:system_monitor, github: "ieQu1/system_monitor", tag: "3.0.5"},
{:telemetry, "1.1.0"},
# in conflict by emqtt and hocon
{:getopt, "1.0.2", override: true},
@ -191,6 +191,7 @@ defmodule EMQXUmbrella.MixProject do
:emqx_bridge_azure_blob_storage,
:emqx_schema_registry,
:emqx_schema_validation,
:emqx_message_transformation,
:emqx_enterprise,
:emqx_bridge_kinesis,
:emqx_bridge_azure_event_hub,

2
rebar.config
View File

@ -94,7 +94,7 @@
{rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.2.1"}}},
% NOTE: depends on recon 2.5.x
{observer_cli, "1.7.1"},
{system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}},
{system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.5"}}},
{getopt, "1.0.2"},
{snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.10"}}},
{hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.42.2"}}},

1
rebar.config.erl
View File

@ -118,6 +118,7 @@ is_community_umbrella_app("apps/emqx_gateway_ocpp") -> false;
is_community_umbrella_app("apps/emqx_gateway_jt808") -> false;
is_community_umbrella_app("apps/emqx_bridge_syskeeper") -> false;
is_community_umbrella_app("apps/emqx_schema_validation") -> false;
is_community_umbrella_app("apps/emqx_message_transformation") -> false;
is_community_umbrella_app("apps/emqx_eviction_agent") -> false;
is_community_umbrella_app("apps/emqx_node_rebalance") -> false;
is_community_umbrella_app("apps/emqx_ds_shared_sub") -> false;

36
rel/i18n/emqx_message_transformation_http_api.hocon Normal file
View File

@ -0,0 +1,36 @@
emqx_message_transformation_http_api {
list_transformations.desc:
"""List transformations"""
lookup_transformation.desc:
"""Lookup a transformation"""
update_transformation.desc:
"""Update a transformation"""
delete_transformation.desc:
"""Delete a transformation"""
append_transformation.desc:
"""Append a new transformation to the list of transformations"""
reorder_transformations.desc:
"""Reorder of all transformations"""
enable_disable_transformation.desc:
"""Enable or disable a particular transformation"""
get_transformation_metrics.desc:
"""Get metrics for a particular transformation"""
reset_transformation_metrics.desc:
"""Reset metrics for a particular transformation"""
param_path_name.desc:
"""Transformation name"""
param_path_enable.desc:
"""Enable or disable transformation"""
}

5
rel/i18n/emqx_prometheus_api.hocon
View File

@ -30,4 +30,9 @@ get_prom_schema_validation.desc:
get_prom_schema_validation.label:
"""Prometheus Metrics for Schema Validation"""
get_prom_message_transformation.desc:
"""Get Prometheus Metrics for Message Validation"""
get_prom_message_transformation.label:
"""Prometheus Metrics for Message Validation"""
}

4
scripts/buildx.sh
View File

@ -9,7 +9,7 @@
## example:
## ./scripts/buildx.sh --profile emqx --pkgtype tgz --arch arm64 \
## --builder ghcr.io/emqx/emqx-builder/5.3-7:1.15.7-26.2.5-1-debian12
## --builder ghcr.io/emqx/emqx-builder/5.3-8:1.15.7-26.2.5-2-debian12
set -euo pipefail
@ -24,7 +24,7 @@ help() {
echo "--arch amd64|arm64: Target arch to build the EMQX package for"
echo "--src_dir <SRC_DIR>: EMQX source code in this dir, default to PWD"
echo "--builder <BUILDER>: Builder image to pull"
echo " E.g. ghcr.io/emqx/emqx-builder/5.3-7:1.15.7-26.2.5-1-debian12"
echo " E.g. ghcr.io/emqx/emqx-builder/5.3-8:1.15.7-26.2.5-2-debian12"
}
die() {

4
scripts/pr-sanity-checks.sh
View File

@ -12,8 +12,8 @@ if ! type "yq" > /dev/null; then
exit 1
fi
EMQX_BUILDER_VERSION=${EMQX_BUILDER_VERSION:-5.3-7}
OTP_VSN=${OTP_VSN:-26.2.5-1}
EMQX_BUILDER_VERSION=${EMQX_BUILDER_VERSION:-5.3-8}
OTP_VSN=${OTP_VSN:-26.2.5-2}
ELIXIR_VSN=${ELIXIR_VSN:-1.15.7}
EMQX_BUILDER_PLATFORM=${EMQX_BUILDER_PLATFORM:-ubuntu22.04}
EMQX_BUILDER=${EMQX_BUILDER:-ghcr.io/emqx/emqx-builder/${EMQX_BUILDER_VERSION}:${ELIXIR_VSN}-${OTP_VSN}-${EMQX_BUILDER_PLATFORM}}

2
scripts/relup-test/start-relup-test-cluster.sh
View File

@ -22,7 +22,7 @@ WEBHOOK="webhook.$NET"
BENCH="bench.$NET"
COOKIE='this-is-a-secret'
## Erlang image is needed to run webhook server and emqtt-bench
ERLANG_IMAGE="ghcr.io/emqx/emqx-builder/5.3-7:1.15.7-26.2.5-1-ubuntu22.04"
ERLANG_IMAGE="ghcr.io/emqx/emqx-builder/5.3-8:1.15.7-26.2.5-2-ubuntu22.04"
# builder has emqtt-bench installed
BENCH_IMAGE="$ERLANG_IMAGE"