From 651f72eabcfcde7d4d60956eb159d95d59c91e31 Mon Sep 17 00:00:00 2001 From: Kinplemelon Date: Thu, 16 Mar 2023 11:39:29 +0800 Subject: [PATCH 001/196] docs: correct the chinese label for the `socket_nodelay` field --- lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf index c41b95c3a..716ddb13a 100644 --- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf +++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf @@ -224,7 +224,7 @@ emqx_ee_bridge_kafka { } label { en: "No Delay" - zh: "是否延迟发送" + zh: "是否关闭延迟发送" } } kafka_topic { From 20414d737323aa6a35906f49697fd72fbe990ef5 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 16 Mar 2023 13:33:52 -0300 Subject: [PATCH 002/196] fix(buffer_worker): check request timeout and health check interval Fixes https://emqx.atlassian.net/browse/EMQX-9099 The default value for `request_timeout` is 15 seconds, and the default resume interval is also 15 seconds (the health check timeout, if `resume_interval` is not explicitly given). This means that, in practice, if a buffer worker ever gets into the blocked state, then almost all requests will timeout. Proposed improvement: - `request_timeout` should by default be twice as much as health_check_interval. - Emit a alarm if `request_timeout` is not greater than `health_check_interval`. --- .../i18n/emqx_resource_schema_i18n.conf | 4 +- apps/emqx_resource/include/emqx_resource.hrl | 5 +- .../src/emqx_resource_buffer_worker.erl | 38 ++++++ .../src/emqx_resource_manager.erl | 1 + .../test/emqx_resource_SUITE.erl | 126 ++++++++++++++++++ changes/ce/fix-10154.en.md | 7 + 6 files changed, 178 insertions(+), 3 deletions(-) create mode 100644 changes/ce/fix-10154.en.md diff --git a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf index fb6b2eb06..2e5cf96e8 100644 --- a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf +++ b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf @@ -102,8 +102,8 @@ For bridges only have ingress direction data flow, it can be set to 0 otherwise request_timeout { desc { - en: """Starting from the moment when the request enters the buffer, if the request remains in the buffer for the specified time or is sent but does not receive a response or acknowledgement in time, the request is considered expired.""" - zh: """从请求进入缓冲区开始计时，如果请求在规定的时间内仍停留在缓冲区内或者已发送但未能及时收到响应或确认，该请求将被视为过期。""" + en: """Starting from the moment when the request enters the buffer, if the request remains in the buffer for the specified time or is sent but does not receive a response or acknowledgement in time, the request is considered expired. We recommend setting this timeout to be at least twice the health check interval, so that the buffer has the chance to recover if too many requests get enqueued.""" + zh: """从请求进入缓冲区开始计时，如果请求在规定的时间内仍停留在缓冲区内或者已发送但未能及时收到响应或确认，该请求将被视为过期。我们建议将这个超时设置为健康检查间隔的至少两倍，这样，如果有太多的请求被排队，缓冲区就有机会恢复。""" } label { en: """Request Expiry""" diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl index be570e694..8033ed660 100644 --- a/apps/emqx_resource/include/emqx_resource.hrl +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -91,7 +91,10 @@ -define(DEFAULT_QUEUE_SIZE, 100 * 1024 * 1024). -define(DEFAULT_QUEUE_SIZE_RAW, <<"100MB">>). --define(DEFAULT_REQUEST_TIMEOUT, timer:seconds(15)). +%% Note: this should be greater than the health check timeout; +%% otherwise, if the buffer worker is ever blocked, than all queued +%% requests will basically fail without being attempted. +-define(DEFAULT_REQUEST_TIMEOUT, timer:seconds(30)). %% count -define(DEFAULT_BATCH_SIZE, 1). diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl index 8bfd77e61..05622bdd7 100644 --- a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl @@ -56,6 +56,8 @@ -export([clear_disk_queue_dir/2]). +-export([deactivate_bad_request_timeout_alarm/1]). + -elvis([{elvis_style, dont_repeat_yourself, disable}]). -define(COLLECT_REQ_LIMIT, 1000). @@ -88,6 +90,8 @@ -type queue_query() :: ?QUERY(reply_fun(), request(), HasBeenSent :: boolean(), expire_at()). -type request() :: term(). -type request_from() :: undefined | gen_statem:from(). +-type request_timeout() :: infinity | timer:time(). +-type health_check_interval() :: timer:time(). -type state() :: blocked | running. -type inflight_key() :: integer(). -type data() :: #{ @@ -199,6 +203,7 @@ init({Id, Index, Opts}) -> RequestTimeout = maps:get(request_timeout, Opts, ?DEFAULT_REQUEST_TIMEOUT), BatchTime0 = maps:get(batch_time, Opts, ?DEFAULT_BATCH_TIME), BatchTime = adjust_batch_time(Id, RequestTimeout, BatchTime0), + maybe_toggle_bad_request_timeout_alarm(Id, RequestTimeout, HealthCheckInterval), Data = #{ id => Id, index => Index, @@ -1679,6 +1684,39 @@ adjust_batch_time(Id, RequestTimeout, BatchTime0) -> end, BatchTime. +%% The request timeout should be greater than the health check +%% timeout, health timeout defines how often the buffer worker tries +%% to unblock. If request timeout is <= health check timeout and the +%% buffer worker is ever blocked, than all queued requests will +%% basically fail without being attempted. +-spec maybe_toggle_bad_request_timeout_alarm( + resource_id(), request_timeout(), health_check_interval() +) -> ok. +maybe_toggle_bad_request_timeout_alarm(Id, _RequestTimeout = infinity, _HealthCheckInterval) -> + deactivate_bad_request_timeout_alarm(Id), + ok; +maybe_toggle_bad_request_timeout_alarm(Id, RequestTimeout, HealthCheckInterval) -> + case RequestTimeout > HealthCheckInterval of + true -> + deactivate_bad_request_timeout_alarm(Id), + ok; + false -> + _ = emqx_alarm:activate( + bad_request_timeout_alarm_id(Id), + #{resource_id => Id, reason => bad_request_timeout}, + <<"Request timeout should be greater than health check timeout: ", Id/binary>> + ), + ok + end. + +-spec deactivate_bad_request_timeout_alarm(resource_id()) -> ok. +deactivate_bad_request_timeout_alarm(Id) -> + _ = emqx_alarm:ensure_deactivated(bad_request_timeout_alarm_id(Id)), + ok. + +bad_request_timeout_alarm_id(Id) -> + <<"bad_request_timeout:", Id/binary>>. + -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). adjust_batch_time_test_() -> diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index 40f9fe1ab..2bdc67a4d 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -506,6 +506,7 @@ handle_remove_event(From, ClearMetrics, Data) -> true -> ok = emqx_metrics_worker:clear_metrics(?RES_METRICS, Data#data.id); false -> ok end, + emqx_resource_buffer_worker:deactivate_bad_request_timeout_alarm(Data#data.id), {stop_and_reply, {shutdown, removed}, [{reply, From, ok}]}. start_resource(Data, From) -> diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index ff7e1d347..25f4a6d77 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -2554,6 +2554,132 @@ do_t_recursive_flush() -> ), ok. +%% Check that we raise an alarm if a bad request timeout config is +%% issued. Request timeout should be greater than health check +%% timeout. +t_bad_request_timeout_alarm(_Config) -> + emqx_connector_demo:set_callback_mode(async_if_possible), + + %% 1) Same values. + {ok, _} = emqx_resource:create( + ?ID, + ?DEFAULT_RESOURCE_GROUP, + ?TEST_RESOURCE, + #{name => test_resource}, + #{ + query_mode => async, + request_timeout => 1_000, + health_check_interval => 1_000, + worker_pool_size => 2 + } + ), + ExpectedMessage = + <<"Request timeout should be greater than health check timeout: ", ?ID/binary>>, + ?assertMatch( + [ + #{ + message := ExpectedMessage, + details := #{reason := bad_request_timeout, resource_id := ?ID}, + deactivate_at := infinity + } + ], + emqx_alarm:get_alarms(activated) + ), + %% The unexpected termination of one of the buffer workers should + %% not turn the alarm off. + [Pid, _ | _] = emqx_resource_buffer_worker_sup:worker_pids(?ID), + MRef = monitor(process, Pid), + exit(Pid, kill), + receive + {'DOWN', MRef, process, Pid, _} -> + ok + after 300 -> + ct:fail("buffer worker didn't die") + end, + ?assertMatch( + [ + #{ + message := ExpectedMessage, + details := #{reason := bad_request_timeout, resource_id := ?ID}, + deactivate_at := infinity + } + ], + emqx_alarm:get_alarms(activated) + ), + ok = emqx_resource:remove_local(?ID), + ?assertEqual([], emqx_alarm:get_alarms(activated)), + + %% 2) Request timeout < health check interval. + {ok, _} = emqx_resource:create( + ?ID, + ?DEFAULT_RESOURCE_GROUP, + ?TEST_RESOURCE, + #{name => test_resource}, + #{ + query_mode => async, + request_timeout => 999, + health_check_interval => 1_000, + worker_pool_size => 2 + } + ), + ?assertMatch( + [ + #{ + message := ExpectedMessage, + details := #{reason := bad_request_timeout, resource_id := ?ID}, + deactivate_at := infinity + } + ], + emqx_alarm:get_alarms(activated) + ), + ok = emqx_resource:remove_local(?ID), + ?assertEqual([], emqx_alarm:get_alarms(activated)), + + %% 2) Request timeout < health check interval. + {ok, _} = emqx_resource:create( + ?ID, + ?DEFAULT_RESOURCE_GROUP, + ?TEST_RESOURCE, + #{name => test_resource}, + #{ + query_mode => async, + request_timeout => 999, + health_check_interval => 1_000, + worker_pool_size => 2 + } + ), + ?assertMatch( + [ + #{ + message := ExpectedMessage, + details := #{reason := bad_request_timeout, resource_id := ?ID}, + deactivate_at := infinity + } + ], + emqx_alarm:get_alarms(activated) + ), + ok = emqx_resource:remove_local(?ID), + ?assertEqual([], emqx_alarm:get_alarms(activated)), + + %% 3) Request timeout > health check interval. + {ok, _} = emqx_resource:create( + ?ID, + ?DEFAULT_RESOURCE_GROUP, + ?TEST_RESOURCE, + #{name => test_resource}, + #{ + query_mode => async, + request_timeout => 1_001, + health_check_interval => 1_000, + worker_pool_size => 2 + } + ), + ?assertEqual([], emqx_alarm:get_alarms(activated)), + ok = emqx_resource:remove_local(?ID), + ?assertEqual([], emqx_alarm:get_alarms(activated)), + + ok. + %%------------------------------------------------------------------------------ %% Helpers %%------------------------------------------------------------------------------ diff --git a/changes/ce/fix-10154.en.md b/changes/ce/fix-10154.en.md new file mode 100644 index 000000000..83a729360 --- /dev/null +++ b/changes/ce/fix-10154.en.md @@ -0,0 +1,7 @@ +Change the default `request_timeout` for bridges and connectors to be +twice the default `health_check_interval`. + +Before this change, the default values for those two options meant +that, if a buffer ever got blocked due to resource errors or high +message volumes, then, by the time the buffer would try to resume its +normal operations, almost all requests would have timed out. From c0a216a7401610fd8be60f9aeb6c4ee8069e79df Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 15 Mar 2023 10:56:53 +0800 Subject: [PATCH 003/196] feat(bridge): support cassandra bridge --- .ci/docker-compose-file/cassandra/Dockerfile | 4 + .../cassandra/Dockerfile-tls | 4 + .../cassandra/cassandra-tls.yaml | 1236 ++++++++++++++++ .../cassandra/cassandra.yaml | 1237 +++++++++++++++++ .../docker-compose-cassandra-tcp.yaml | 27 + .../docker-compose-toxiproxy.yaml | 1 + .ci/docker-compose-file/toxiproxy.json | 12 + apps/emqx_bridge/src/emqx_bridge.erl | 3 +- lib-ee/emqx_ee_bridge/docker-ct | 1 + .../i18n/emqx_ee_bridge_cassa.conf | 72 + lib-ee/emqx_ee_bridge/rebar.config | 1 + lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl | 17 +- .../src/emqx_ee_bridge_cassa.erl | 133 ++ .../test/emqx_ee_bridge_cassa_SUITE.erl | 540 +++++++ .../i18n/emqx_ee_connector_cassa.conf | 28 + .../include/emqx_ee_connector.hrl | 1 + .../src/emqx_ee_connector.app.src | 3 +- .../src/emqx_ee_connector_cassa.erl | 415 ++++++ .../test/emqx_ee_connector_cassa_SUITE.erl | 192 +++ scripts/ct/run.sh | 3 + 20 files changed, 3925 insertions(+), 5 deletions(-) create mode 100644 .ci/docker-compose-file/cassandra/Dockerfile create mode 100644 .ci/docker-compose-file/cassandra/Dockerfile-tls create mode 100644 .ci/docker-compose-file/cassandra/cassandra-tls.yaml create mode 100644 .ci/docker-compose-file/cassandra/cassandra.yaml create mode 100644 .ci/docker-compose-file/docker-compose-cassandra-tcp.yaml create mode 100644 lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_cassa.conf create mode 100644 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl create mode 100644 lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl create mode 100644 lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_cassa.conf create mode 100644 lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl create mode 100644 lib-ee/emqx_ee_connector/test/emqx_ee_connector_cassa_SUITE.erl diff --git a/.ci/docker-compose-file/cassandra/Dockerfile b/.ci/docker-compose-file/cassandra/Dockerfile new file mode 100644 index 000000000..f974c1b6f --- /dev/null +++ b/.ci/docker-compose-file/cassandra/Dockerfile @@ -0,0 +1,4 @@ +ARG CASSANDRA_TAG=3.11.6 +FROM cassandra:${CASSANDRA_TAG} +COPY cassandra.yaml /etc/cassandra/cassandra.yaml +CMD ["cassandra", "-f"] diff --git a/.ci/docker-compose-file/cassandra/Dockerfile-tls b/.ci/docker-compose-file/cassandra/Dockerfile-tls new file mode 100644 index 000000000..434584ca6 --- /dev/null +++ b/.ci/docker-compose-file/cassandra/Dockerfile-tls @@ -0,0 +1,4 @@ +ARG CASSANDRA_TAG=3.11.6 +FROM cassandra:${CASSANDRA_TAG} +COPY cassandra-tls.yaml /etc/cassandra/cassandra.yaml +CMD ["cassandra", "-f"] diff --git a/.ci/docker-compose-file/cassandra/cassandra-tls.yaml b/.ci/docker-compose-file/cassandra/cassandra-tls.yaml new file mode 100644 index 000000000..d2d2a5d70 --- /dev/null +++ b/.ci/docker-compose-file/cassandra/cassandra-tls.yaml @@ -0,0 +1,1236 @@ +# Cassandra storage config YAML + +# NOTE: +# See http://wiki.apache.org/cassandra/StorageConfiguration for +# full explanations of configuration directives +# /NOTE + +# The name of the cluster. This is mainly used to prevent machines in +# one logical cluster from joining another. +cluster_name: 'Test Cluster' + +# This defines the number of tokens randomly assigned to this node on the ring +# The more tokens, relative to other nodes, the larger the proportion of data +# that this node will store. You probably want all nodes to have the same number +# of tokens assuming they have equal hardware capability. +# +# If you leave this unspecified, Cassandra will use the default of 1 token for legacy compatibility, +# and will use the initial_token as described below. +# +# Specifying initial_token will override this setting on the node's initial start, +# on subsequent starts, this setting will apply even if initial token is set. +# +# If you already have a cluster with 1 token per node, and wish to migrate to +# multiple tokens per node, see http://wiki.apache.org/cassandra/Operations +num_tokens: 256 + +# Triggers automatic allocation of num_tokens tokens for this node. The allocation +# algorithm attempts to choose tokens in a way that optimizes replicated load over +# the nodes in the datacenter for the replication strategy used by the specified +# keyspace. +# +# The load assigned to each node will be close to proportional to its number of +# vnodes. +# +# Only supported with the Murmur3Partitioner. +# allocate_tokens_for_keyspace: KEYSPACE + +# initial_token allows you to specify tokens manually. While you can use it with +# vnodes (num_tokens > 1, above) -- in which case you should provide a +# comma-separated list -- it's primarily used when adding nodes to legacy clusters +# that do not have vnodes enabled. +# initial_token: + +# See http://wiki.apache.org/cassandra/HintedHandoff +# May either be "true" or "false" to enable globally +hinted_handoff_enabled: true + +# When hinted_handoff_enabled is true, a black list of data centers that will not +# perform hinted handoff +# hinted_handoff_disabled_datacenters: +# - DC1 +# - DC2 + +# this defines the maximum amount of time a dead host will have hints +# generated. After it has been dead this long, new hints for it will not be +# created until it has been seen alive and gone down again. +max_hint_window_in_ms: 10800000 # 3 hours + +# Maximum throttle in KBs per second, per delivery thread. This will be +# reduced proportionally to the number of nodes in the cluster. (If there +# are two nodes in the cluster, each delivery thread will use the maximum +# rate; if there are three, each will throttle to half of the maximum, +# since we expect two nodes to be delivering hints simultaneously.) +hinted_handoff_throttle_in_kb: 1024 + +# Number of threads with which to deliver hints; +# Consider increasing this number when you have multi-dc deployments, since +# cross-dc handoff tends to be slower +max_hints_delivery_threads: 2 + +# Directory where Cassandra should store hints. +# If not set, the default directory is $CASSANDRA_HOME/data/hints. +# hints_directory: /var/lib/cassandra/hints + +# How often hints should be flushed from the internal buffers to disk. +# Will *not* trigger fsync. +hints_flush_period_in_ms: 10000 + +# Maximum size for a single hints file, in megabytes. +max_hints_file_size_in_mb: 128 + +# Compression to apply to the hint files. If omitted, hints files +# will be written uncompressed. LZ4, Snappy, and Deflate compressors +# are supported. +#hints_compression: +# - class_name: LZ4Compressor +# parameters: +# - + +# Maximum throttle in KBs per second, total. This will be +# reduced proportionally to the number of nodes in the cluster. +batchlog_replay_throttle_in_kb: 1024 + +# Authentication backend, implementing IAuthenticator; used to identify users +# Out of the box, Cassandra provides org.apache.cassandra.auth.{AllowAllAuthenticator, +# PasswordAuthenticator}. +# +# - AllowAllAuthenticator performs no checks - set it to disable authentication. +# - PasswordAuthenticator relies on username/password pairs to authenticate +# users. It keeps usernames and hashed passwords in system_auth.roles table. +# Please increase system_auth keyspace replication factor if you use this authenticator. +# If using PasswordAuthenticator, CassandraRoleManager must also be used (see below) +authenticator: PasswordAuthenticator + +# Authorization backend, implementing IAuthorizer; used to limit access/provide permissions +# Out of the box, Cassandra provides org.apache.cassandra.auth.{AllowAllAuthorizer, +# CassandraAuthorizer}. +# +# - AllowAllAuthorizer allows any action to any user - set it to disable authorization. +# - CassandraAuthorizer stores permissions in system_auth.role_permissions table. Please +# increase system_auth keyspace replication factor if you use this authorizer. +authorizer: CassandraAuthorizer + +# Part of the Authentication & Authorization backend, implementing IRoleManager; used +# to maintain grants and memberships between roles. +# Out of the box, Cassandra provides org.apache.cassandra.auth.CassandraRoleManager, +# which stores role information in the system_auth keyspace. Most functions of the +# IRoleManager require an authenticated login, so unless the configured IAuthenticator +# actually implements authentication, most of this functionality will be unavailable. +# +# - CassandraRoleManager stores role data in the system_auth keyspace. Please +# increase system_auth keyspace replication factor if you use this role manager. +role_manager: CassandraRoleManager + +# Validity period for roles cache (fetching granted roles can be an expensive +# operation depending on the role manager, CassandraRoleManager is one example) +# Granted roles are cached for authenticated sessions in AuthenticatedUser and +# after the period specified here, become eligible for (async) reload. +# Defaults to 2000, set to 0 to disable caching entirely. +# Will be disabled automatically for AllowAllAuthenticator. +roles_validity_in_ms: 2000 + +# Refresh interval for roles cache (if enabled). +# After this interval, cache entries become eligible for refresh. Upon next +# access, an async reload is scheduled and the old value returned until it +# completes. If roles_validity_in_ms is non-zero, then this must be +# also. +# Defaults to the same value as roles_validity_in_ms. +# roles_update_interval_in_ms: 2000 + +# Validity period for permissions cache (fetching permissions can be an +# expensive operation depending on the authorizer, CassandraAuthorizer is +# one example). Defaults to 2000, set to 0 to disable. +# Will be disabled automatically for AllowAllAuthorizer. +permissions_validity_in_ms: 2000 + +# Refresh interval for permissions cache (if enabled). +# After this interval, cache entries become eligible for refresh. Upon next +# access, an async reload is scheduled and the old value returned until it +# completes. If permissions_validity_in_ms is non-zero, then this must be +# also. +# Defaults to the same value as permissions_validity_in_ms. +# permissions_update_interval_in_ms: 2000 + +# Validity period for credentials cache. This cache is tightly coupled to +# the provided PasswordAuthenticator implementation of IAuthenticator. If +# another IAuthenticator implementation is configured, this cache will not +# be automatically used and so the following settings will have no effect. +# Please note, credentials are cached in their encrypted form, so while +# activating this cache may reduce the number of queries made to the +# underlying table, it may not bring a significant reduction in the +# latency of individual authentication attempts. +# Defaults to 2000, set to 0 to disable credentials caching. +credentials_validity_in_ms: 2000 + +# Refresh interval for credentials cache (if enabled). +# After this interval, cache entries become eligible for refresh. Upon next +# access, an async reload is scheduled and the old value returned until it +# completes. If credentials_validity_in_ms is non-zero, then this must be +# also. +# Defaults to the same value as credentials_validity_in_ms. +# credentials_update_interval_in_ms: 2000 + +# The partitioner is responsible for distributing groups of rows (by +# partition key) across nodes in the cluster. You should leave this +# alone for new clusters. The partitioner can NOT be changed without +# reloading all data, so when upgrading you should set this to the +# same partitioner you were already using. +# +# Besides Murmur3Partitioner, partitioners included for backwards +# compatibility include RandomPartitioner, ByteOrderedPartitioner, and +# OrderPreservingPartitioner. +# +partitioner: org.apache.cassandra.dht.Murmur3Partitioner + +# Directories where Cassandra should store data on disk. Cassandra +# will spread data evenly across them, subject to the granularity of +# the configured compaction strategy. +# If not set, the default directory is $CASSANDRA_HOME/data/data. +data_file_directories: + - /var/lib/cassandra/data + +# commit log. when running on magnetic HDD, this should be a +# separate spindle than the data directories. +# If not set, the default directory is $CASSANDRA_HOME/data/commitlog. +commitlog_directory: /var/lib/cassandra/commitlog + +# Enable / disable CDC functionality on a per-node basis. This modifies the logic used +# for write path allocation rejection (standard: never reject. cdc: reject Mutation +# containing a CDC-enabled table if at space limit in cdc_raw_directory). +cdc_enabled: false + +# CommitLogSegments are moved to this directory on flush if cdc_enabled: true and the +# segment contains mutations for a CDC-enabled table. This should be placed on a +# separate spindle than the data directories. If not set, the default directory is +# $CASSANDRA_HOME/data/cdc_raw. +# cdc_raw_directory: /var/lib/cassandra/cdc_raw + +# Policy for data disk failures: +# +# die +# shut down gossip and client transports and kill the JVM for any fs errors or +# single-sstable errors, so the node can be replaced. +# +# stop_paranoid +# shut down gossip and client transports even for single-sstable errors, +# kill the JVM for errors during startup. +# +# stop +# shut down gossip and client transports, leaving the node effectively dead, but +# can still be inspected via JMX, kill the JVM for errors during startup. +# +# best_effort +# stop using the failed disk and respond to requests based on +# remaining available sstables. This means you WILL see obsolete +# data at CL.ONE! +# +# ignore +# ignore fatal errors and let requests fail, as in pre-1.2 Cassandra +disk_failure_policy: stop + +# Policy for commit disk failures: +# +# die +# shut down gossip and Thrift and kill the JVM, so the node can be replaced. +# +# stop +# shut down gossip and Thrift, leaving the node effectively dead, but +# can still be inspected via JMX. +# +# stop_commit +# shutdown the commit log, letting writes collect but +# continuing to service reads, as in pre-2.0.5 Cassandra +# +# ignore +# ignore fatal errors and let the batches fail +commit_failure_policy: stop + +# Maximum size of the native protocol prepared statement cache +# +# Valid values are either "auto" (omitting the value) or a value greater 0. +# +# Note that specifying a too large value will result in long running GCs and possbily +# out-of-memory errors. Keep the value at a small fraction of the heap. +# +# If you constantly see "prepared statements discarded in the last minute because +# cache limit reached" messages, the first step is to investigate the root cause +# of these messages and check whether prepared statements are used correctly - +# i.e. use bind markers for variable parts. +# +# Do only change the default value, if you really have more prepared statements than +# fit in the cache. In most cases it is not neccessary to change this value. +# Constantly re-preparing statements is a performance penalty. +# +# Default value ("auto") is 1/256th of the heap or 10MB, whichever is greater +prepared_statements_cache_size_mb: + +# Maximum size of the Thrift prepared statement cache +# +# If you do not use Thrift at all, it is safe to leave this value at "auto". +# +# See description of 'prepared_statements_cache_size_mb' above for more information. +# +# Default value ("auto") is 1/256th of the heap or 10MB, whichever is greater +thrift_prepared_statements_cache_size_mb: + +# Maximum size of the key cache in memory. +# +# Each key cache hit saves 1 seek and each row cache hit saves 2 seeks at the +# minimum, sometimes more. The key cache is fairly tiny for the amount of +# time it saves, so it's worthwhile to use it at large numbers. +# The row cache saves even more time, but must contain the entire row, +# so it is extremely space-intensive. It's best to only use the +# row cache if you have hot rows or static rows. +# +# NOTE: if you reduce the size, you may not get you hottest keys loaded on startup. +# +# Default value is empty to make it "auto" (min(5% of Heap (in MB), 100MB)). Set to 0 to disable key cache. +key_cache_size_in_mb: + +# Duration in seconds after which Cassandra should +# save the key cache. Caches are saved to saved_caches_directory as +# specified in this configuration file. +# +# Saved caches greatly improve cold-start speeds, and is relatively cheap in +# terms of I/O for the key cache. Row cache saving is much more expensive and +# has limited use. +# +# Default is 14400 or 4 hours. +key_cache_save_period: 14400 + +# Number of keys from the key cache to save +# Disabled by default, meaning all keys are going to be saved +# key_cache_keys_to_save: 100 + +# Row cache implementation class name. Available implementations: +# +# org.apache.cassandra.cache.OHCProvider +# Fully off-heap row cache implementation (default). +# +# org.apache.cassandra.cache.SerializingCacheProvider +# This is the row cache implementation availabile +# in previous releases of Cassandra. +# row_cache_class_name: org.apache.cassandra.cache.OHCProvider + +# Maximum size of the row cache in memory. +# Please note that OHC cache implementation requires some additional off-heap memory to manage +# the map structures and some in-flight memory during operations before/after cache entries can be +# accounted against the cache capacity. This overhead is usually small compared to the whole capacity. +# Do not specify more memory that the system can afford in the worst usual situation and leave some +# headroom for OS block level cache. Do never allow your system to swap. +# +# Default value is 0, to disable row caching. +row_cache_size_in_mb: 0 + +# Duration in seconds after which Cassandra should save the row cache. +# Caches are saved to saved_caches_directory as specified in this configuration file. +# +# Saved caches greatly improve cold-start speeds, and is relatively cheap in +# terms of I/O for the key cache. Row cache saving is much more expensive and +# has limited use. +# +# Default is 0 to disable saving the row cache. +row_cache_save_period: 0 + +# Number of keys from the row cache to save. +# Specify 0 (which is the default), meaning all keys are going to be saved +# row_cache_keys_to_save: 100 + +# Maximum size of the counter cache in memory. +# +# Counter cache helps to reduce counter locks' contention for hot counter cells. +# In case of RF = 1 a counter cache hit will cause Cassandra to skip the read before +# write entirely. With RF > 1 a counter cache hit will still help to reduce the duration +# of the lock hold, helping with hot counter cell updates, but will not allow skipping +# the read entirely. Only the local (clock, count) tuple of a counter cell is kept +# in memory, not the whole counter, so it's relatively cheap. +# +# NOTE: if you reduce the size, you may not get you hottest keys loaded on startup. +# +# Default value is empty to make it "auto" (min(2.5% of Heap (in MB), 50MB)). Set to 0 to disable counter cache. +# NOTE: if you perform counter deletes and rely on low gcgs, you should disable the counter cache. +counter_cache_size_in_mb: + +# Duration in seconds after which Cassandra should +# save the counter cache (keys only). Caches are saved to saved_caches_directory as +# specified in this configuration file. +# +# Default is 7200 or 2 hours. +counter_cache_save_period: 7200 + +# Number of keys from the counter cache to save +# Disabled by default, meaning all keys are going to be saved +# counter_cache_keys_to_save: 100 + +# saved caches +# If not set, the default directory is $CASSANDRA_HOME/data/saved_caches. +saved_caches_directory: /var/lib/cassandra/saved_caches + +# commitlog_sync may be either "periodic" or "batch." +# +# When in batch mode, Cassandra won't ack writes until the commit log +# has been fsynced to disk. It will wait +# commitlog_sync_batch_window_in_ms milliseconds between fsyncs. +# This window should be kept short because the writer threads will +# be unable to do extra work while waiting. (You may need to increase +# concurrent_writes for the same reason.) +# +# commitlog_sync: batch +# commitlog_sync_batch_window_in_ms: 2 +# +# the other option is "periodic" where writes may be acked immediately +# and the CommitLog is simply synced every commitlog_sync_period_in_ms +# milliseconds. +commitlog_sync: periodic +commitlog_sync_period_in_ms: 10000 + +# The size of the individual commitlog file segments. A commitlog +# segment may be archived, deleted, or recycled once all the data +# in it (potentially from each columnfamily in the system) has been +# flushed to sstables. +# +# The default size is 32, which is almost always fine, but if you are +# archiving commitlog segments (see commitlog_archiving.properties), +# then you probably want a finer granularity of archiving; 8 or 16 MB +# is reasonable. +# Max mutation size is also configurable via max_mutation_size_in_kb setting in +# cassandra.yaml. The default is half the size commitlog_segment_size_in_mb * 1024. +# This should be positive and less than 2048. +# +# NOTE: If max_mutation_size_in_kb is set explicitly then commitlog_segment_size_in_mb must +# be set to at least twice the size of max_mutation_size_in_kb / 1024 +# +commitlog_segment_size_in_mb: 32 + +# Compression to apply to the commit log. If omitted, the commit log +# will be written uncompressed. LZ4, Snappy, and Deflate compressors +# are supported. +# commitlog_compression: +# - class_name: LZ4Compressor +# parameters: +# - + +# any class that implements the SeedProvider interface and has a +# constructor that takes a Map of parameters will do. +seed_provider: + # Addresses of hosts that are deemed contact points. + # Cassandra nodes use this list of hosts to find each other and learn + # the topology of the ring. You must change this if you are running + # multiple nodes! + - class_name: org.apache.cassandra.locator.SimpleSeedProvider + parameters: + # seeds is actually a comma-delimited list of addresses. + # Ex: ",," + - seeds: "127.0.0.1" + +# For workloads with more data than can fit in memory, Cassandra's +# bottleneck will be reads that need to fetch data from +# disk. "concurrent_reads" should be set to (16 * number_of_drives) in +# order to allow the operations to enqueue low enough in the stack +# that the OS and drives can reorder them. Same applies to +# "concurrent_counter_writes", since counter writes read the current +# values before incrementing and writing them back. +# +# On the other hand, since writes are almost never IO bound, the ideal +# number of "concurrent_writes" is dependent on the number of cores in +# your system; (8 * number_of_cores) is a good rule of thumb. +concurrent_reads: 32 +concurrent_writes: 32 +concurrent_counter_writes: 32 + +# For materialized view writes, as there is a read involved, so this should +# be limited by the less of concurrent reads or concurrent writes. +concurrent_materialized_view_writes: 32 + +# Maximum memory to use for sstable chunk cache and buffer pooling. +# 32MB of this are reserved for pooling buffers, the rest is used as an +# cache that holds uncompressed sstable chunks. +# Defaults to the smaller of 1/4 of heap or 512MB. This pool is allocated off-heap, +# so is in addition to the memory allocated for heap. The cache also has on-heap +# overhead which is roughly 128 bytes per chunk (i.e. 0.2% of the reserved size +# if the default 64k chunk size is used). +# Memory is only allocated when needed. +# file_cache_size_in_mb: 512 + +# Flag indicating whether to allocate on or off heap when the sstable buffer +# pool is exhausted, that is when it has exceeded the maximum memory +# file_cache_size_in_mb, beyond which it will not cache buffers but allocate on request. + +# buffer_pool_use_heap_if_exhausted: true + +# The strategy for optimizing disk read +# Possible values are: +# ssd (for solid state disks, the default) +# spinning (for spinning disks) +# disk_optimization_strategy: ssd + +# Total permitted memory to use for memtables. Cassandra will stop +# accepting writes when the limit is exceeded until a flush completes, +# and will trigger a flush based on memtable_cleanup_threshold +# If omitted, Cassandra will set both to 1/4 the size of the heap. +# memtable_heap_space_in_mb: 2048 +# memtable_offheap_space_in_mb: 2048 + +# memtable_cleanup_threshold is deprecated. The default calculation +# is the only reasonable choice. See the comments on memtable_flush_writers +# for more information. +# +# Ratio of occupied non-flushing memtable size to total permitted size +# that will trigger a flush of the largest memtable. Larger mct will +# mean larger flushes and hence less compaction, but also less concurrent +# flush activity which can make it difficult to keep your disks fed +# under heavy write load. +# +# memtable_cleanup_threshold defaults to 1 / (memtable_flush_writers + 1) +# memtable_cleanup_threshold: 0.11 + +# Specify the way Cassandra allocates and manages memtable memory. +# Options are: +# +# heap_buffers +# on heap nio buffers +# +# offheap_buffers +# off heap (direct) nio buffers +# +# offheap_objects +# off heap objects +memtable_allocation_type: heap_buffers + +# Total space to use for commit logs on disk. +# +# If space gets above this value, Cassandra will flush every dirty CF +# in the oldest segment and remove it. So a small total commitlog space +# will tend to cause more flush activity on less-active columnfamilies. +# +# The default value is the smaller of 8192, and 1/4 of the total space +# of the commitlog volume. +# +# commitlog_total_space_in_mb: 8192 + +# This sets the number of memtable flush writer threads per disk +# as well as the total number of memtables that can be flushed concurrently. +# These are generally a combination of compute and IO bound. +# +# Memtable flushing is more CPU efficient than memtable ingest and a single thread +# can keep up with the ingest rate of a whole server on a single fast disk +# until it temporarily becomes IO bound under contention typically with compaction. +# At that point you need multiple flush threads. At some point in the future +# it may become CPU bound all the time. +# +# You can tell if flushing is falling behind using the MemtablePool.BlockedOnAllocation +# metric which should be 0, but will be non-zero if threads are blocked waiting on flushing +# to free memory. +# +# memtable_flush_writers defaults to two for a single data directory. +# This means that two memtables can be flushed concurrently to the single data directory. +# If you have multiple data directories the default is one memtable flushing at a time +# but the flush will use a thread per data directory so you will get two or more writers. +# +# Two is generally enough to flush on a fast disk [array] mounted as a single data directory. +# Adding more flush writers will result in smaller more frequent flushes that introduce more +# compaction overhead. +# +# There is a direct tradeoff between number of memtables that can be flushed concurrently +# and flush size and frequency. More is not better you just need enough flush writers +# to never stall waiting for flushing to free memory. +# +#memtable_flush_writers: 2 + +# Total space to use for change-data-capture logs on disk. +# +# If space gets above this value, Cassandra will throw WriteTimeoutException +# on Mutations including tables with CDC enabled. A CDCCompactor is responsible +# for parsing the raw CDC logs and deleting them when parsing is completed. +# +# The default value is the min of 4096 mb and 1/8th of the total space +# of the drive where cdc_raw_directory resides. +# cdc_total_space_in_mb: 4096 + +# When we hit our cdc_raw limit and the CDCCompactor is either running behind +# or experiencing backpressure, we check at the following interval to see if any +# new space for cdc-tracked tables has been made available. Default to 250ms +# cdc_free_space_check_interval_ms: 250 + +# A fixed memory pool size in MB for for SSTable index summaries. If left +# empty, this will default to 5% of the heap size. If the memory usage of +# all index summaries exceeds this limit, SSTables with low read rates will +# shrink their index summaries in order to meet this limit. However, this +# is a best-effort process. In extreme conditions Cassandra may need to use +# more than this amount of memory. +index_summary_capacity_in_mb: + +# How frequently index summaries should be resampled. This is done +# periodically to redistribute memory from the fixed-size pool to sstables +# proportional their recent read rates. Setting to -1 will disable this +# process, leaving existing index summaries at their current sampling level. +index_summary_resize_interval_in_minutes: 60 + +# Whether to, when doing sequential writing, fsync() at intervals in +# order to force the operating system to flush the dirty +# buffers. Enable this to avoid sudden dirty buffer flushing from +# impacting read latencies. Almost always a good idea on SSDs; not +# necessarily on platters. +trickle_fsync: false +trickle_fsync_interval_in_kb: 10240 + +# TCP port, for commands and data +# For security reasons, you should not expose this port to the internet. Firewall it if needed. +storage_port: 7000 + +# SSL port, for encrypted communication. Unused unless enabled in +# encryption_options +# For security reasons, you should not expose this port to the internet. Firewall it if needed. +ssl_storage_port: 7001 + +# Address or interface to bind to and tell other Cassandra nodes to connect to. +# You _must_ change this if you want multiple nodes to be able to communicate! +# +# Set listen_address OR listen_interface, not both. +# +# Leaving it blank leaves it up to InetAddress.getLocalHost(). This +# will always do the Right Thing _if_ the node is properly configured +# (hostname, name resolution, etc), and the Right Thing is to use the +# address associated with the hostname (it might not be). +# +# Setting listen_address to 0.0.0.0 is always wrong. +# +listen_address: localhost + +# Set listen_address OR listen_interface, not both. Interfaces must correspond +# to a single address, IP aliasing is not supported. +# listen_interface: eth0 + +# If you choose to specify the interface by name and the interface has an ipv4 and an ipv6 address +# you can specify which should be chosen using listen_interface_prefer_ipv6. If false the first ipv4 +# address will be used. If true the first ipv6 address will be used. Defaults to false preferring +# ipv4. If there is only one address it will be selected regardless of ipv4/ipv6. +# listen_interface_prefer_ipv6: false + +# Address to broadcast to other Cassandra nodes +# Leaving this blank will set it to the same value as listen_address +# broadcast_address: 1.2.3.4 + +# When using multiple physical network interfaces, set this +# to true to listen on broadcast_address in addition to +# the listen_address, allowing nodes to communicate in both +# interfaces. +# Ignore this property if the network configuration automatically +# routes between the public and private networks such as EC2. +# listen_on_broadcast_address: false + +# Internode authentication backend, implementing IInternodeAuthenticator; +# used to allow/disallow connections from peer nodes. +# internode_authenticator: org.apache.cassandra.auth.AllowAllInternodeAuthenticator + +# Whether to start the native transport server. +# Please note that the address on which the native transport is bound is the +# same as the rpc_address. The port however is different and specified below. +start_native_transport: true +# port for the CQL native transport to listen for clients on +# For security reasons, you should not expose this port to the internet. Firewall it if needed. +native_transport_port: 9042 +# Enabling native transport encryption in client_encryption_options allows you to either use +# encryption for the standard port or to use a dedicated, additional port along with the unencrypted +# standard native_transport_port. +# Enabling client encryption and keeping native_transport_port_ssl disabled will use encryption +# for native_transport_port. Setting native_transport_port_ssl to a different value +# from native_transport_port will use encryption for native_transport_port_ssl while +# keeping native_transport_port unencrypted. +native_transport_port_ssl: 9142 +# The maximum threads for handling requests when the native transport is used. +# This is similar to rpc_max_threads though the default differs slightly (and +# there is no native_transport_min_threads, idle threads will always be stopped +# after 30 seconds). +# native_transport_max_threads: 128 +# +# The maximum size of allowed frame. Frame (requests) larger than this will +# be rejected as invalid. The default is 256MB. If you're changing this parameter, +# you may want to adjust max_value_size_in_mb accordingly. This should be positive and less than 2048. +# native_transport_max_frame_size_in_mb: 256 + +# The maximum number of concurrent client connections. +# The default is -1, which means unlimited. +# native_transport_max_concurrent_connections: -1 + +# The maximum number of concurrent client connections per source ip. +# The default is -1, which means unlimited. +# native_transport_max_concurrent_connections_per_ip: -1 + +# Whether to start the thrift rpc server. +start_rpc: true + +# The address or interface to bind the Thrift RPC service and native transport +# server to. +# +# Set rpc_address OR rpc_interface, not both. +# +# Leaving rpc_address blank has the same effect as on listen_address +# (i.e. it will be based on the configured hostname of the node). +# +# Note that unlike listen_address, you can specify 0.0.0.0, but you must also +# set broadcast_rpc_address to a value other than 0.0.0.0. +# +# For security reasons, you should not expose this port to the internet. Firewall it if needed. +rpc_address: 0.0.0.0 + +# Set rpc_address OR rpc_interface, not both. Interfaces must correspond +# to a single address, IP aliasing is not supported. +# rpc_interface: eth1 + +# If you choose to specify the interface by name and the interface has an ipv4 and an ipv6 address +# you can specify which should be chosen using rpc_interface_prefer_ipv6. If false the first ipv4 +# address will be used. If true the first ipv6 address will be used. Defaults to false preferring +# ipv4. If there is only one address it will be selected regardless of ipv4/ipv6. +# rpc_interface_prefer_ipv6: false + +# port for Thrift to listen for clients on +rpc_port: 9160 + +# RPC address to broadcast to drivers and other Cassandra nodes. This cannot +# be set to 0.0.0.0. If left blank, this will be set to the value of +# rpc_address. If rpc_address is set to 0.0.0.0, broadcast_rpc_address must +# be set. +broadcast_rpc_address: 1.2.3.4 + +# enable or disable keepalive on rpc/native connections +rpc_keepalive: true + +# Cassandra provides two out-of-the-box options for the RPC Server: +# +# sync +# One thread per thrift connection. For a very large number of clients, memory +# will be your limiting factor. On a 64 bit JVM, 180KB is the minimum stack size +# per thread, and that will correspond to your use of virtual memory (but physical memory +# may be limited depending on use of stack space). +# +# hsha +# Stands for "half synchronous, half asynchronous." All thrift clients are handled +# asynchronously using a small number of threads that does not vary with the amount +# of thrift clients (and thus scales well to many clients). The rpc requests are still +# synchronous (one thread per active request). If hsha is selected then it is essential +# that rpc_max_threads is changed from the default value of unlimited. +# +# The default is sync because on Windows hsha is about 30% slower. On Linux, +# sync/hsha performance is about the same, with hsha of course using less memory. +# +# Alternatively, can provide your own RPC server by providing the fully-qualified class name +# of an o.a.c.t.TServerFactory that can create an instance of it. +rpc_server_type: sync + +# Uncomment rpc_min|max_thread to set request pool size limits. +# +# Regardless of your choice of RPC server (see above), the number of maximum requests in the +# RPC thread pool dictates how many concurrent requests are possible (but if you are using the sync +# RPC server, it also dictates the number of clients that can be connected at all). +# +# The default is unlimited and thus provides no protection against clients overwhelming the server. You are +# encouraged to set a maximum that makes sense for you in production, but do keep in mind that +# rpc_max_threads represents the maximum number of client requests this server may execute concurrently. +# +# rpc_min_threads: 16 +# rpc_max_threads: 2048 + +# uncomment to set socket buffer sizes on rpc connections +# rpc_send_buff_size_in_bytes: +# rpc_recv_buff_size_in_bytes: + +# Uncomment to set socket buffer size for internode communication +# Note that when setting this, the buffer size is limited by net.core.wmem_max +# and when not setting it it is defined by net.ipv4.tcp_wmem +# See also: +# /proc/sys/net/core/wmem_max +# /proc/sys/net/core/rmem_max +# /proc/sys/net/ipv4/tcp_wmem +# /proc/sys/net/ipv4/tcp_wmem +# and 'man tcp' +# internode_send_buff_size_in_bytes: + +# Uncomment to set socket buffer size for internode communication +# Note that when setting this, the buffer size is limited by net.core.wmem_max +# and when not setting it it is defined by net.ipv4.tcp_wmem +# internode_recv_buff_size_in_bytes: + +# Frame size for thrift (maximum message length). +thrift_framed_transport_size_in_mb: 15 + +# Set to true to have Cassandra create a hard link to each sstable +# flushed or streamed locally in a backups/ subdirectory of the +# keyspace data. Removing these links is the operator's +# responsibility. +incremental_backups: false + +# Whether or not to take a snapshot before each compaction. Be +# careful using this option, since Cassandra won't clean up the +# snapshots for you. Mostly useful if you're paranoid when there +# is a data format change. +snapshot_before_compaction: false + +# Whether or not a snapshot is taken of the data before keyspace truncation +# or dropping of column families. The STRONGLY advised default of true +# should be used to provide data safety. If you set this flag to false, you will +# lose data on truncation or drop. +auto_snapshot: true + +# Granularity of the collation index of rows within a partition. +# Increase if your rows are large, or if you have a very large +# number of rows per partition. The competing goals are these: +# +# - a smaller granularity means more index entries are generated +# and looking up rows withing the partition by collation column +# is faster +# - but, Cassandra will keep the collation index in memory for hot +# rows (as part of the key cache), so a larger granularity means +# you can cache more hot rows +column_index_size_in_kb: 64 + +# Per sstable indexed key cache entries (the collation index in memory +# mentioned above) exceeding this size will not be held on heap. +# This means that only partition information is held on heap and the +# index entries are read from disk. +# +# Note that this size refers to the size of the +# serialized index information and not the size of the partition. +column_index_cache_size_in_kb: 2 + +# Number of simultaneous compactions to allow, NOT including +# validation "compactions" for anti-entropy repair. Simultaneous +# compactions can help preserve read performance in a mixed read/write +# workload, by mitigating the tendency of small sstables to accumulate +# during a single long running compactions. The default is usually +# fine and if you experience problems with compaction running too +# slowly or too fast, you should look at +# compaction_throughput_mb_per_sec first. +# +# concurrent_compactors defaults to the smaller of (number of disks, +# number of cores), with a minimum of 2 and a maximum of 8. +# +# If your data directories are backed by SSD, you should increase this +# to the number of cores. +#concurrent_compactors: 1 + +# Throttles compaction to the given total throughput across the entire +# system. The faster you insert data, the faster you need to compact in +# order to keep the sstable count down, but in general, setting this to +# 16 to 32 times the rate you are inserting data is more than sufficient. +# Setting this to 0 disables throttling. Note that this account for all types +# of compaction, including validation compaction. +compaction_throughput_mb_per_sec: 16 + +# When compacting, the replacement sstable(s) can be opened before they +# are completely written, and used in place of the prior sstables for +# any range that has been written. This helps to smoothly transfer reads +# between the sstables, reducing page cache churn and keeping hot rows hot +sstable_preemptive_open_interval_in_mb: 50 + +# Throttles all outbound streaming file transfers on this node to the +# given total throughput in Mbps. This is necessary because Cassandra does +# mostly sequential IO when streaming data during bootstrap or repair, which +# can lead to saturating the network connection and degrading rpc performance. +# When unset, the default is 200 Mbps or 25 MB/s. +# stream_throughput_outbound_megabits_per_sec: 200 + +# Throttles all streaming file transfer between the datacenters, +# this setting allows users to throttle inter dc stream throughput in addition +# to throttling all network stream traffic as configured with +# stream_throughput_outbound_megabits_per_sec +# When unset, the default is 200 Mbps or 25 MB/s +# inter_dc_stream_throughput_outbound_megabits_per_sec: 200 + +# How long the coordinator should wait for read operations to complete +read_request_timeout_in_ms: 5000 +# How long the coordinator should wait for seq or index scans to complete +range_request_timeout_in_ms: 10000 +# How long the coordinator should wait for writes to complete +write_request_timeout_in_ms: 2000 +# How long the coordinator should wait for counter writes to complete +counter_write_request_timeout_in_ms: 5000 +# How long a coordinator should continue to retry a CAS operation +# that contends with other proposals for the same row +cas_contention_timeout_in_ms: 1000 +# How long the coordinator should wait for truncates to complete +# (This can be much longer, because unless auto_snapshot is disabled +# we need to flush first so we can snapshot before removing the data.) +truncate_request_timeout_in_ms: 60000 +# The default timeout for other, miscellaneous operations +request_timeout_in_ms: 10000 + +# How long before a node logs slow queries. Select queries that take longer than +# this timeout to execute, will generate an aggregated log message, so that slow queries +# can be identified. Set this value to zero to disable slow query logging. +slow_query_log_timeout_in_ms: 500 + +# Enable operation timeout information exchange between nodes to accurately +# measure request timeouts. If disabled, replicas will assume that requests +# were forwarded to them instantly by the coordinator, which means that +# under overload conditions we will waste that much extra time processing +# already-timed-out requests. +# +# Warning: before enabling this property make sure to ntp is installed +# and the times are synchronized between the nodes. +cross_node_timeout: false + +# Set keep-alive period for streaming +# This node will send a keep-alive message periodically with this period. +# If the node does not receive a keep-alive message from the peer for +# 2 keep-alive cycles the stream session times out and fail +# Default value is 300s (5 minutes), which means stalled stream +# times out in 10 minutes by default +# streaming_keep_alive_period_in_secs: 300 + +# phi value that must be reached for a host to be marked down. +# most users should never need to adjust this. +# phi_convict_threshold: 8 + +# endpoint_snitch -- Set this to a class that implements +# IEndpointSnitch. The snitch has two functions: +# +# - it teaches Cassandra enough about your network topology to route +# requests efficiently +# - it allows Cassandra to spread replicas around your cluster to avoid +# correlated failures. It does this by grouping machines into +# "datacenters" and "racks." Cassandra will do its best not to have +# more than one replica on the same "rack" (which may not actually +# be a physical location) +# +# CASSANDRA WILL NOT ALLOW YOU TO SWITCH TO AN INCOMPATIBLE SNITCH +# ONCE DATA IS INSERTED INTO THE CLUSTER. This would cause data loss. +# This means that if you start with the default SimpleSnitch, which +# locates every node on "rack1" in "datacenter1", your only options +# if you need to add another datacenter are GossipingPropertyFileSnitch +# (and the older PFS). From there, if you want to migrate to an +# incompatible snitch like Ec2Snitch you can do it by adding new nodes +# under Ec2Snitch (which will locate them in a new "datacenter") and +# decommissioning the old ones. +# +# Out of the box, Cassandra provides: +# +# SimpleSnitch: +# Treats Strategy order as proximity. This can improve cache +# locality when disabling read repair. Only appropriate for +# single-datacenter deployments. +# +# GossipingPropertyFileSnitch +# This should be your go-to snitch for production use. The rack +# and datacenter for the local node are defined in +# cassandra-rackdc.properties and propagated to other nodes via +# gossip. If cassandra-topology.properties exists, it is used as a +# fallback, allowing migration from the PropertyFileSnitch. +# +# PropertyFileSnitch: +# Proximity is determined by rack and data center, which are +# explicitly configured in cassandra-topology.properties. +# +# Ec2Snitch: +# Appropriate for EC2 deployments in a single Region. Loads Region +# and Availability Zone information from the EC2 API. The Region is +# treated as the datacenter, and the Availability Zone as the rack. +# Only private IPs are used, so this will not work across multiple +# Regions. +# +# Ec2MultiRegionSnitch: +# Uses public IPs as broadcast_address to allow cross-region +# connectivity. (Thus, you should set seed addresses to the public +# IP as well.) You will need to open the storage_port or +# ssl_storage_port on the public IP firewall. (For intra-Region +# traffic, Cassandra will switch to the private IP after +# establishing a connection.) +# +# RackInferringSnitch: +# Proximity is determined by rack and data center, which are +# assumed to correspond to the 3rd and 2nd octet of each node's IP +# address, respectively. Unless this happens to match your +# deployment conventions, this is best used as an example of +# writing a custom Snitch class and is provided in that spirit. +# +# You can use a custom Snitch by setting this to the full class name +# of the snitch, which will be assumed to be on your classpath. +endpoint_snitch: SimpleSnitch + +# controls how often to perform the more expensive part of host score +# calculation +dynamic_snitch_update_interval_in_ms: 100 +# controls how often to reset all host scores, allowing a bad host to +# possibly recover +dynamic_snitch_reset_interval_in_ms: 600000 +# if set greater than zero and read_repair_chance is < 1.0, this will allow +# 'pinning' of replicas to hosts in order to increase cache capacity. +# The badness threshold will control how much worse the pinned host has to be +# before the dynamic snitch will prefer other replicas over it. This is +# expressed as a double which represents a percentage. Thus, a value of +# 0.2 means Cassandra would continue to prefer the static snitch values +# until the pinned host was 20% worse than the fastest. +dynamic_snitch_badness_threshold: 0.1 + +# request_scheduler -- Set this to a class that implements +# RequestScheduler, which will schedule incoming client requests +# according to the specific policy. This is useful for multi-tenancy +# with a single Cassandra cluster. +# NOTE: This is specifically for requests from the client and does +# not affect inter node communication. +# org.apache.cassandra.scheduler.NoScheduler - No scheduling takes place +# org.apache.cassandra.scheduler.RoundRobinScheduler - Round robin of +# client requests to a node with a separate queue for each +# request_scheduler_id. The scheduler is further customized by +# request_scheduler_options as described below. +request_scheduler: org.apache.cassandra.scheduler.NoScheduler + +# Scheduler Options vary based on the type of scheduler +# +# NoScheduler +# Has no options +# +# RoundRobin +# throttle_limit +# The throttle_limit is the number of in-flight +# requests per client. Requests beyond +# that limit are queued up until +# running requests can complete. +# The value of 80 here is twice the number of +# concurrent_reads + concurrent_writes. +# default_weight +# default_weight is optional and allows for +# overriding the default which is 1. +# weights +# Weights are optional and will default to 1 or the +# overridden default_weight. The weight translates into how +# many requests are handled during each turn of the +# RoundRobin, based on the scheduler id. +# +# request_scheduler_options: +# throttle_limit: 80 +# default_weight: 5 +# weights: +# Keyspace1: 1 +# Keyspace2: 5 + +# request_scheduler_id -- An identifier based on which to perform +# the request scheduling. Currently the only valid option is keyspace. +# request_scheduler_id: keyspace + +# Enable or disable inter-node encryption +# JVM defaults for supported SSL socket protocols and cipher suites can +# be replaced using custom encryption options. This is not recommended +# unless you have policies in place that dictate certain settings, or +# need to disable vulnerable ciphers or protocols in case the JVM cannot +# be updated. +# FIPS compliant settings can be configured at JVM level and should not +# involve changing encryption settings here: +# https://docs.oracle.com/javase/8/docs/technotes/guides/security/jsse/FIPS.html +# *NOTE* No custom encryption options are enabled at the moment +# The available internode options are : all, none, dc, rack +# +# If set to dc cassandra will encrypt the traffic between the DCs +# If set to rack cassandra will encrypt the traffic between the racks +# +# The passwords used in these options must match the passwords used when generating +# the keystore and truststore. For instructions on generating these files, see: +# http://download.oracle.com/javase/6/docs/technotes/guides/security/jsse/JSSERefGuide.html#CreateKeystore +# +server_encryption_options: + internode_encryption: none + keystore: conf/.keystore + keystore_password: cassandra + truststore: conf/.truststore + truststore_password: cassandra + # More advanced defaults below: + # protocol: TLS + # algorithm: SunX509 + # store_type: JKS + # cipher_suites: [TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_DHE_RSA_WITH_AES_128_CBC_SHA,TLS_DHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA] + # require_client_auth: false + # require_endpoint_verification: false + +# enable or disable client/server encryption. +client_encryption_options: + enabled: true + # If enabled and optional is set to true encrypted and unencrypted connections are handled. + optional: true + keystore: /certs/cass.jks + keystore_password: nosecret + require_client_auth: true + # Set trustore and truststore_password if require_client_auth is true + truststore: /certs/truststore.jks + truststore_password: nosecret + # More advanced defaults below: + protocol: TLS + algorithm: SunX509 + store_type: JKS + cipher_suites: [TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_DHE_RSA_WITH_AES_128_CBC_SHA,TLS_DHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA] + +# internode_compression controls whether traffic between nodes is +# compressed. +# Can be: +# +# all +# all traffic is compressed +# +# dc +# traffic between different datacenters is compressed +# +# none +# nothing is compressed. +internode_compression: dc + +# Enable or disable tcp_nodelay for inter-dc communication. +# Disabling it will result in larger (but fewer) network packets being sent, +# reducing overhead from the TCP protocol itself, at the cost of increasing +# latency if you block for cross-datacenter responses. +inter_dc_tcp_nodelay: false + +# TTL for different trace types used during logging of the repair process. +tracetype_query_ttl: 86400 +tracetype_repair_ttl: 604800 + +# By default, Cassandra logs GC Pauses greater than 200 ms at INFO level +# This threshold can be adjusted to minimize logging if necessary +# gc_log_threshold_in_ms: 200 + +# If unset, all GC Pauses greater than gc_log_threshold_in_ms will log at +# INFO level +# UDFs (user defined functions) are disabled by default. +# As of Cassandra 3.0 there is a sandbox in place that should prevent execution of evil code. +enable_user_defined_functions: false + +# Enables scripted UDFs (JavaScript UDFs). +# Java UDFs are always enabled, if enable_user_defined_functions is true. +# Enable this option to be able to use UDFs with "language javascript" or any custom JSR-223 provider. +# This option has no effect, if enable_user_defined_functions is false. +enable_scripted_user_defined_functions: false + +# Enables materialized view creation on this node. +# Materialized views are considered experimental and are not recommended for production use. +enable_materialized_views: true + +# The default Windows kernel timer and scheduling resolution is 15.6ms for power conservationLowering this value on Windows can provide much tighter latency and better throughput, however +# some virtualized environments may see a negative performance impact from changing this setting +# below their system default. The sysinternals 'clockres' tool can confirm your system's default +# setting. +windows_timer_interval: 1 + + +# Enables encrypting data at-rest (on disk). Different key providers can be plugged in, but the default reads from +# a JCE-style keystore. A single keystore can hold multiple keys, but the one referenced by +# the "key_alias" is the only key that will be used for encrypt opertaions; previously used keys +# can still (and should!) be in the keystore and will be used on decrypt operations +# (to handle the case of key rotation). +# +# It is strongly recommended to download and install Java Cryptography Extension (JCE) +# Unlimited Strength Jurisdiction Policy Files for your version of the JDK. +# (current link: http://www.oracle.com/technetwork/java/javase/downloads/jce8-download-2133166.html) +# +# Currently, only the following file types are supported for transparent data encryption, although +# more are coming in future cassandra releases: commitlog, hints +transparent_data_encryption_options: + enabled: false + chunk_length_kb: 64 + cipher: AES/CBC/PKCS5Padding + key_alias: testing:1 + # CBC IV length for AES needs to be 16 bytes (which is also the default size) + # iv_length: 16 + key_provider: + - class_name: org.apache.cassandra.security.JKSKeyProvider + parameters: + - keystore: conf/.keystore + keystore_password: cassandra + store_type: JCEKS + key_password: cassandra + + +##################### +# SAFETY THRESHOLDS # +##################### + +# When executing a scan, within or across a partition, we need to keep the +# tombstones seen in memory so we can return them to the coordinator, which +# will use them to make sure other replicas also know about the deleted rows. +# With workloads that generate a lot of tombstones, this can cause performance +# problems and even exaust the server heap. +# (http://www.datastax.com/dev/blog/cassandra-anti-patterns-queues-and-queue-like-datasets) +# Adjust the thresholds here if you understand the dangers and want to +# scan more tombstones anyway. These thresholds may also be adjusted at runtime +# using the StorageService mbean. +tombstone_warn_threshold: 1000 +tombstone_failure_threshold: 100000 + +# Log WARN on any multiple-partition batch size exceeding this value. 5kb per batch by default. +# Caution should be taken on increasing the size of this threshold as it can lead to node instability. +batch_size_warn_threshold_in_kb: 5 + +# Fail any multiple-partition batch exceeding this value. 50kb (10x warn threshold) by default. +batch_size_fail_threshold_in_kb: 50 + +# Log WARN on any batches not of type LOGGED than span across more partitions than this limit +unlogged_batch_across_partitions_warn_threshold: 10 + +# Log a warning when compacting partitions larger than this value +compaction_large_partition_warning_threshold_mb: 100 + +# GC Pauses greater than gc_warn_threshold_in_ms will be logged at WARN level +# Adjust the threshold based on your application throughput requirement +# By default, Cassandra logs GC Pauses greater than 200 ms at INFO level +gc_warn_threshold_in_ms: 1000 + +# Maximum size of any value in SSTables. Safety measure to detect SSTable corruption +# early. Any value size larger than this threshold will result into marking an SSTable +# as corrupted. This should be positive and less than 2048. +# max_value_size_in_mb: 256 + +# Back-pressure settings # +# If enabled, the coordinator will apply the back-pressure strategy specified below to each mutation +# sent to replicas, with the aim of reducing pressure on overloaded replicas. +back_pressure_enabled: false +# The back-pressure strategy applied. +# The default implementation, RateBasedBackPressure, takes three arguments: +# high ratio, factor, and flow type, and uses the ratio between incoming mutation responses and outgoing mutation requests. +# If below high ratio, outgoing mutations are rate limited according to the incoming rate decreased by the given factor; +# if above high ratio, the rate limiting is increased by the given factor; +# such factor is usually best configured between 1 and 10, use larger values for a faster recovery +# at the expense of potentially more dropped mutations; +# the rate limiting is applied according to the flow type: if FAST, it's rate limited at the speed of the fastest replica, +# if SLOW at the speed of the slowest one. +# New strategies can be added. Implementors need to implement org.apache.cassandra.net.BackpressureStrategy and +# provide a public constructor accepting a Map. +back_pressure_strategy: + - class_name: org.apache.cassandra.net.RateBasedBackPressure + parameters: + - high_ratio: 0.90 + factor: 5 + flow: FAST + +# Coalescing Strategies # +# Coalescing multiples messages turns out to significantly boost message processing throughput (think doubling or more). +# On bare metal, the floor for packet processing throughput is high enough that many applications won't notice, but in +# virtualized environments, the point at which an application can be bound by network packet processing can be +# surprisingly low compared to the throughput of task processing that is possible inside a VM. It's not that bare metal +# doesn't benefit from coalescing messages, it's that the number of packets a bare metal network interface can process +# is sufficient for many applications such that no load starvation is experienced even without coalescing. +# There are other benefits to coalescing network messages that are harder to isolate with a simple metric like messages +# per second. By coalescing multiple tasks together, a network thread can process multiple messages for the cost of one +# trip to read from a socket, and all the task submission work can be done at the same time reducing context switching +# and increasing cache friendliness of network message processing. +# See CASSANDRA-8692 for details. + +# Strategy to use for coalescing messages in OutboundTcpConnection. +# Can be fixed, movingaverage, timehorizon, disabled (default). +# You can also specify a subclass of CoalescingStrategies.CoalescingStrategy by name. +# otc_coalescing_strategy: DISABLED + +# How many microseconds to wait for coalescing. For fixed strategy this is the amount of time after the first +# message is received before it will be sent with any accompanying messages. For moving average this is the +# maximum amount of time that will be waited as well as the interval at which messages must arrive on average +# for coalescing to be enabled. +# otc_coalescing_window_us: 200 + +# Do not try to coalesce messages if we already got that many messages. This should be more than 2 and less than 128. +# otc_coalescing_enough_coalesced_messages: 8 + +# How many milliseconds to wait between two expiration runs on the backlog (queue) of the OutboundTcpConnection. +# Expiration is done if messages are piling up in the backlog. Droppable messages are expired to free the memory +# taken by expired messages. The interval should be between 0 and 1000, and in most installations the default value +# will be appropriate. A smaller value could potentially expire messages slightly sooner at the expense of more CPU +# time and queue contention while iterating the backlog of messages. +# An interval of 0 disables any wait time, which is the behavior of former Cassandra versions. +# +# otc_backlog_expiration_interval_ms: 200 diff --git a/.ci/docker-compose-file/cassandra/cassandra.yaml b/.ci/docker-compose-file/cassandra/cassandra.yaml new file mode 100644 index 000000000..6059e9d64 --- /dev/null +++ b/.ci/docker-compose-file/cassandra/cassandra.yaml @@ -0,0 +1,1237 @@ +# Cassandra storage config YAML + +# NOTE: +# See http://wiki.apache.org/cassandra/StorageConfiguration for +# full explanations of configuration directives +# /NOTE + +# The name of the cluster. This is mainly used to prevent machines in +# one logical cluster from joining another. +cluster_name: 'Test Cluster' + +# This defines the number of tokens randomly assigned to this node on the ring +# The more tokens, relative to other nodes, the larger the proportion of data +# that this node will store. You probably want all nodes to have the same number +# of tokens assuming they have equal hardware capability. +# +# If you leave this unspecified, Cassandra will use the default of 1 token for legacy compatibility, +# and will use the initial_token as described below. +# +# Specifying initial_token will override this setting on the node's initial start, +# on subsequent starts, this setting will apply even if initial token is set. +# +# If you already have a cluster with 1 token per node, and wish to migrate to +# multiple tokens per node, see http://wiki.apache.org/cassandra/Operations +num_tokens: 256 + +# Triggers automatic allocation of num_tokens tokens for this node. The allocation +# algorithm attempts to choose tokens in a way that optimizes replicated load over +# the nodes in the datacenter for the replication strategy used by the specified +# keyspace. +# +# The load assigned to each node will be close to proportional to its number of +# vnodes. +# +# Only supported with the Murmur3Partitioner. +# allocate_tokens_for_keyspace: KEYSPACE + +# initial_token allows you to specify tokens manually. While you can use it with +# vnodes (num_tokens > 1, above) -- in which case you should provide a +# comma-separated list -- it's primarily used when adding nodes to legacy clusters +# that do not have vnodes enabled. +# initial_token: + +# See http://wiki.apache.org/cassandra/HintedHandoff +# May either be "true" or "false" to enable globally +hinted_handoff_enabled: true + +# When hinted_handoff_enabled is true, a black list of data centers that will not +# perform hinted handoff +# hinted_handoff_disabled_datacenters: +# - DC1 +# - DC2 + +# this defines the maximum amount of time a dead host will have hints +# generated. After it has been dead this long, new hints for it will not be +# created until it has been seen alive and gone down again. +max_hint_window_in_ms: 10800000 # 3 hours + +# Maximum throttle in KBs per second, per delivery thread. This will be +# reduced proportionally to the number of nodes in the cluster. (If there +# are two nodes in the cluster, each delivery thread will use the maximum +# rate; if there are three, each will throttle to half of the maximum, +# since we expect two nodes to be delivering hints simultaneously.) +hinted_handoff_throttle_in_kb: 1024 + +# Number of threads with which to deliver hints; +# Consider increasing this number when you have multi-dc deployments, since +# cross-dc handoff tends to be slower +max_hints_delivery_threads: 2 + +# Directory where Cassandra should store hints. +# If not set, the default directory is $CASSANDRA_HOME/data/hints. +# hints_directory: /var/lib/cassandra/hints + +# How often hints should be flushed from the internal buffers to disk. +# Will *not* trigger fsync. +hints_flush_period_in_ms: 10000 + +# Maximum size for a single hints file, in megabytes. +max_hints_file_size_in_mb: 128 + +# Compression to apply to the hint files. If omitted, hints files +# will be written uncompressed. LZ4, Snappy, and Deflate compressors +# are supported. +#hints_compression: +# - class_name: LZ4Compressor +# parameters: +# - + +# Maximum throttle in KBs per second, total. This will be +# reduced proportionally to the number of nodes in the cluster. +batchlog_replay_throttle_in_kb: 1024 + +# Authentication backend, implementing IAuthenticator; used to identify users +# Out of the box, Cassandra provides org.apache.cassandra.auth.{AllowAllAuthenticator, +# PasswordAuthenticator}. +# +# - AllowAllAuthenticator performs no checks - set it to disable authentication. +# - PasswordAuthenticator relies on username/password pairs to authenticate +# users. It keeps usernames and hashed passwords in system_auth.roles table. +# Please increase system_auth keyspace replication factor if you use this authenticator. +# If using PasswordAuthenticator, CassandraRoleManager must also be used (see below) +authenticator: AllowAllAuthenticator + +# Authorization backend, implementing IAuthorizer; used to limit access/provide permissions +# Out of the box, Cassandra provides org.apache.cassandra.auth.{AllowAllAuthorizer, +# CassandraAuthorizer}. +# +# - AllowAllAuthorizer allows any action to any user - set it to disable authorization. +# - CassandraAuthorizer stores permissions in system_auth.role_permissions table. Please +# increase system_auth keyspace replication factor if you use this authorizer. +authorizer: AllowAllAuthorizer + +# Part of the Authentication & Authorization backend, implementing IRoleManager; used +# to maintain grants and memberships between roles. +# Out of the box, Cassandra provides org.apache.cassandra.auth.CassandraRoleManager, +# which stores role information in the system_auth keyspace. Most functions of the +# IRoleManager require an authenticated login, so unless the configured IAuthenticator +# actually implements authentication, most of this functionality will be unavailable. +# +# - CassandraRoleManager stores role data in the system_auth keyspace. Please +# increase system_auth keyspace replication factor if you use this role manager. +role_manager: CassandraRoleManager + +# Validity period for roles cache (fetching granted roles can be an expensive +# operation depending on the role manager, CassandraRoleManager is one example) +# Granted roles are cached for authenticated sessions in AuthenticatedUser and +# after the period specified here, become eligible for (async) reload. +# Defaults to 2000, set to 0 to disable caching entirely. +# Will be disabled automatically for AllowAllAuthenticator. +roles_validity_in_ms: 2000 + +# Refresh interval for roles cache (if enabled). +# After this interval, cache entries become eligible for refresh. Upon next +# access, an async reload is scheduled and the old value returned until it +# completes. If roles_validity_in_ms is non-zero, then this must be +# also. +# Defaults to the same value as roles_validity_in_ms. +# roles_update_interval_in_ms: 2000 + +# Validity period for permissions cache (fetching permissions can be an +# expensive operation depending on the authorizer, CassandraAuthorizer is +# one example). Defaults to 2000, set to 0 to disable. +# Will be disabled automatically for AllowAllAuthorizer. +permissions_validity_in_ms: 2000 + +# Refresh interval for permissions cache (if enabled). +# After this interval, cache entries become eligible for refresh. Upon next +# access, an async reload is scheduled and the old value returned until it +# completes. If permissions_validity_in_ms is non-zero, then this must be +# also. +# Defaults to the same value as permissions_validity_in_ms. +# permissions_update_interval_in_ms: 2000 + +# Validity period for credentials cache. This cache is tightly coupled to +# the provided PasswordAuthenticator implementation of IAuthenticator. If +# another IAuthenticator implementation is configured, this cache will not +# be automatically used and so the following settings will have no effect. +# Please note, credentials are cached in their encrypted form, so while +# activating this cache may reduce the number of queries made to the +# underlying table, it may not bring a significant reduction in the +# latency of individual authentication attempts. +# Defaults to 2000, set to 0 to disable credentials caching. +credentials_validity_in_ms: 2000 + +# Refresh interval for credentials cache (if enabled). +# After this interval, cache entries become eligible for refresh. Upon next +# access, an async reload is scheduled and the old value returned until it +# completes. If credentials_validity_in_ms is non-zero, then this must be +# also. +# Defaults to the same value as credentials_validity_in_ms. +# credentials_update_interval_in_ms: 2000 + +# The partitioner is responsible for distributing groups of rows (by +# partition key) across nodes in the cluster. You should leave this +# alone for new clusters. The partitioner can NOT be changed without +# reloading all data, so when upgrading you should set this to the +# same partitioner you were already using. +# +# Besides Murmur3Partitioner, partitioners included for backwards +# compatibility include RandomPartitioner, ByteOrderedPartitioner, and +# OrderPreservingPartitioner. +# +partitioner: org.apache.cassandra.dht.Murmur3Partitioner + +# Directories where Cassandra should store data on disk. Cassandra +# will spread data evenly across them, subject to the granularity of +# the configured compaction strategy. +# If not set, the default directory is $CASSANDRA_HOME/data/data. +data_file_directories: + - /var/lib/cassandra/data + +# commit log. when running on magnetic HDD, this should be a +# separate spindle than the data directories. +# If not set, the default directory is $CASSANDRA_HOME/data/commitlog. +commitlog_directory: /var/lib/cassandra/commitlog + +# Enable / disable CDC functionality on a per-node basis. This modifies the logic used +# for write path allocation rejection (standard: never reject. cdc: reject Mutation +# containing a CDC-enabled table if at space limit in cdc_raw_directory). +cdc_enabled: false + +# CommitLogSegments are moved to this directory on flush if cdc_enabled: true and the +# segment contains mutations for a CDC-enabled table. This should be placed on a +# separate spindle than the data directories. If not set, the default directory is +# $CASSANDRA_HOME/data/cdc_raw. +# cdc_raw_directory: /var/lib/cassandra/cdc_raw + +# Policy for data disk failures: +# +# die +# shut down gossip and client transports and kill the JVM for any fs errors or +# single-sstable errors, so the node can be replaced. +# +# stop_paranoid +# shut down gossip and client transports even for single-sstable errors, +# kill the JVM for errors during startup. +# +# stop +# shut down gossip and client transports, leaving the node effectively dead, but +# can still be inspected via JMX, kill the JVM for errors during startup. +# +# best_effort +# stop using the failed disk and respond to requests based on +# remaining available sstables. This means you WILL see obsolete +# data at CL.ONE! +# +# ignore +# ignore fatal errors and let requests fail, as in pre-1.2 Cassandra +disk_failure_policy: stop + +# Policy for commit disk failures: +# +# die +# shut down gossip and Thrift and kill the JVM, so the node can be replaced. +# +# stop +# shut down gossip and Thrift, leaving the node effectively dead, but +# can still be inspected via JMX. +# +# stop_commit +# shutdown the commit log, letting writes collect but +# continuing to service reads, as in pre-2.0.5 Cassandra +# +# ignore +# ignore fatal errors and let the batches fail +commit_failure_policy: stop + +# Maximum size of the native protocol prepared statement cache +# +# Valid values are either "auto" (omitting the value) or a value greater 0. +# +# Note that specifying a too large value will result in long running GCs and possbily +# out-of-memory errors. Keep the value at a small fraction of the heap. +# +# If you constantly see "prepared statements discarded in the last minute because +# cache limit reached" messages, the first step is to investigate the root cause +# of these messages and check whether prepared statements are used correctly - +# i.e. use bind markers for variable parts. +# +# Do only change the default value, if you really have more prepared statements than +# fit in the cache. In most cases it is not neccessary to change this value. +# Constantly re-preparing statements is a performance penalty. +# +# Default value ("auto") is 1/256th of the heap or 10MB, whichever is greater +prepared_statements_cache_size_mb: + +# Maximum size of the Thrift prepared statement cache +# +# If you do not use Thrift at all, it is safe to leave this value at "auto". +# +# See description of 'prepared_statements_cache_size_mb' above for more information. +# +# Default value ("auto") is 1/256th of the heap or 10MB, whichever is greater +thrift_prepared_statements_cache_size_mb: + +# Maximum size of the key cache in memory. +# +# Each key cache hit saves 1 seek and each row cache hit saves 2 seeks at the +# minimum, sometimes more. The key cache is fairly tiny for the amount of +# time it saves, so it's worthwhile to use it at large numbers. +# The row cache saves even more time, but must contain the entire row, +# so it is extremely space-intensive. It's best to only use the +# row cache if you have hot rows or static rows. +# +# NOTE: if you reduce the size, you may not get you hottest keys loaded on startup. +# +# Default value is empty to make it "auto" (min(5% of Heap (in MB), 100MB)). Set to 0 to disable key cache. +key_cache_size_in_mb: + +# Duration in seconds after which Cassandra should +# save the key cache. Caches are saved to saved_caches_directory as +# specified in this configuration file. +# +# Saved caches greatly improve cold-start speeds, and is relatively cheap in +# terms of I/O for the key cache. Row cache saving is much more expensive and +# has limited use. +# +# Default is 14400 or 4 hours. +key_cache_save_period: 14400 + +# Number of keys from the key cache to save +# Disabled by default, meaning all keys are going to be saved +# key_cache_keys_to_save: 100 + +# Row cache implementation class name. Available implementations: +# +# org.apache.cassandra.cache.OHCProvider +# Fully off-heap row cache implementation (default). +# +# org.apache.cassandra.cache.SerializingCacheProvider +# This is the row cache implementation availabile +# in previous releases of Cassandra. +# row_cache_class_name: org.apache.cassandra.cache.OHCProvider + +# Maximum size of the row cache in memory. +# Please note that OHC cache implementation requires some additional off-heap memory to manage +# the map structures and some in-flight memory during operations before/after cache entries can be +# accounted against the cache capacity. This overhead is usually small compared to the whole capacity. +# Do not specify more memory that the system can afford in the worst usual situation and leave some +# headroom for OS block level cache. Do never allow your system to swap. +# +# Default value is 0, to disable row caching. +row_cache_size_in_mb: 0 + +# Duration in seconds after which Cassandra should save the row cache. +# Caches are saved to saved_caches_directory as specified in this configuration file. +# +# Saved caches greatly improve cold-start speeds, and is relatively cheap in +# terms of I/O for the key cache. Row cache saving is much more expensive and +# has limited use. +# +# Default is 0 to disable saving the row cache. +row_cache_save_period: 0 + +# Number of keys from the row cache to save. +# Specify 0 (which is the default), meaning all keys are going to be saved +# row_cache_keys_to_save: 100 + +# Maximum size of the counter cache in memory. +# +# Counter cache helps to reduce counter locks' contention for hot counter cells. +# In case of RF = 1 a counter cache hit will cause Cassandra to skip the read before +# write entirely. With RF > 1 a counter cache hit will still help to reduce the duration +# of the lock hold, helping with hot counter cell updates, but will not allow skipping +# the read entirely. Only the local (clock, count) tuple of a counter cell is kept +# in memory, not the whole counter, so it's relatively cheap. +# +# NOTE: if you reduce the size, you may not get you hottest keys loaded on startup. +# +# Default value is empty to make it "auto" (min(2.5% of Heap (in MB), 50MB)). Set to 0 to disable counter cache. +# NOTE: if you perform counter deletes and rely on low gcgs, you should disable the counter cache. +counter_cache_size_in_mb: + +# Duration in seconds after which Cassandra should +# save the counter cache (keys only). Caches are saved to saved_caches_directory as +# specified in this configuration file. +# +# Default is 7200 or 2 hours. +counter_cache_save_period: 7200 + +# Number of keys from the counter cache to save +# Disabled by default, meaning all keys are going to be saved +# counter_cache_keys_to_save: 100 + +# saved caches +# If not set, the default directory is $CASSANDRA_HOME/data/saved_caches. +saved_caches_directory: /var/lib/cassandra/saved_caches + +# commitlog_sync may be either "periodic" or "batch." +# +# When in batch mode, Cassandra won't ack writes until the commit log +# has been fsynced to disk. It will wait +# commitlog_sync_batch_window_in_ms milliseconds between fsyncs. +# This window should be kept short because the writer threads will +# be unable to do extra work while waiting. (You may need to increase +# concurrent_writes for the same reason.) +# +# commitlog_sync: batch +# commitlog_sync_batch_window_in_ms: 2 +# +# the other option is "periodic" where writes may be acked immediately +# and the CommitLog is simply synced every commitlog_sync_period_in_ms +# milliseconds. +commitlog_sync: periodic +commitlog_sync_period_in_ms: 10000 + +# The size of the individual commitlog file segments. A commitlog +# segment may be archived, deleted, or recycled once all the data +# in it (potentially from each columnfamily in the system) has been +# flushed to sstables. +# +# The default size is 32, which is almost always fine, but if you are +# archiving commitlog segments (see commitlog_archiving.properties), +# then you probably want a finer granularity of archiving; 8 or 16 MB +# is reasonable. +# Max mutation size is also configurable via max_mutation_size_in_kb setting in +# cassandra.yaml. The default is half the size commitlog_segment_size_in_mb * 1024. +# This should be positive and less than 2048. +# +# NOTE: If max_mutation_size_in_kb is set explicitly then commitlog_segment_size_in_mb must +# be set to at least twice the size of max_mutation_size_in_kb / 1024 +# +commitlog_segment_size_in_mb: 32 + +# Compression to apply to the commit log. If omitted, the commit log +# will be written uncompressed. LZ4, Snappy, and Deflate compressors +# are supported. +# commitlog_compression: +# - class_name: LZ4Compressor +# parameters: +# - + +# any class that implements the SeedProvider interface and has a +# constructor that takes a Map of parameters will do. +seed_provider: + # Addresses of hosts that are deemed contact points. + # Cassandra nodes use this list of hosts to find each other and learn + # the topology of the ring. You must change this if you are running + # multiple nodes! + - class_name: org.apache.cassandra.locator.SimpleSeedProvider + parameters: + # seeds is actually a comma-delimited list of addresses. + # Ex: ",," + - seeds: "127.0.0.1" + +# For workloads with more data than can fit in memory, Cassandra's +# bottleneck will be reads that need to fetch data from +# disk. "concurrent_reads" should be set to (16 * number_of_drives) in +# order to allow the operations to enqueue low enough in the stack +# that the OS and drives can reorder them. Same applies to +# "concurrent_counter_writes", since counter writes read the current +# values before incrementing and writing them back. +# +# On the other hand, since writes are almost never IO bound, the ideal +# number of "concurrent_writes" is dependent on the number of cores in +# your system; (8 * number_of_cores) is a good rule of thumb. +concurrent_reads: 32 +concurrent_writes: 32 +concurrent_counter_writes: 32 + +# For materialized view writes, as there is a read involved, so this should +# be limited by the less of concurrent reads or concurrent writes. +concurrent_materialized_view_writes: 32 + +# Maximum memory to use for sstable chunk cache and buffer pooling. +# 32MB of this are reserved for pooling buffers, the rest is used as an +# cache that holds uncompressed sstable chunks. +# Defaults to the smaller of 1/4 of heap or 512MB. This pool is allocated off-heap, +# so is in addition to the memory allocated for heap. The cache also has on-heap +# overhead which is roughly 128 bytes per chunk (i.e. 0.2% of the reserved size +# if the default 64k chunk size is used). +# Memory is only allocated when needed. +# file_cache_size_in_mb: 512 + +# Flag indicating whether to allocate on or off heap when the sstable buffer +# pool is exhausted, that is when it has exceeded the maximum memory +# file_cache_size_in_mb, beyond which it will not cache buffers but allocate on request. + +# buffer_pool_use_heap_if_exhausted: true + +# The strategy for optimizing disk read +# Possible values are: +# ssd (for solid state disks, the default) +# spinning (for spinning disks) +# disk_optimization_strategy: ssd + +# Total permitted memory to use for memtables. Cassandra will stop +# accepting writes when the limit is exceeded until a flush completes, +# and will trigger a flush based on memtable_cleanup_threshold +# If omitted, Cassandra will set both to 1/4 the size of the heap. +# memtable_heap_space_in_mb: 2048 +# memtable_offheap_space_in_mb: 2048 + +# memtable_cleanup_threshold is deprecated. The default calculation +# is the only reasonable choice. See the comments on memtable_flush_writers +# for more information. +# +# Ratio of occupied non-flushing memtable size to total permitted size +# that will trigger a flush of the largest memtable. Larger mct will +# mean larger flushes and hence less compaction, but also less concurrent +# flush activity which can make it difficult to keep your disks fed +# under heavy write load. +# +# memtable_cleanup_threshold defaults to 1 / (memtable_flush_writers + 1) +# memtable_cleanup_threshold: 0.11 + +# Specify the way Cassandra allocates and manages memtable memory. +# Options are: +# +# heap_buffers +# on heap nio buffers +# +# offheap_buffers +# off heap (direct) nio buffers +# +# offheap_objects +# off heap objects +memtable_allocation_type: heap_buffers + +# Total space to use for commit logs on disk. +# +# If space gets above this value, Cassandra will flush every dirty CF +# in the oldest segment and remove it. So a small total commitlog space +# will tend to cause more flush activity on less-active columnfamilies. +# +# The default value is the smaller of 8192, and 1/4 of the total space +# of the commitlog volume. +# +# commitlog_total_space_in_mb: 8192 + +# This sets the number of memtable flush writer threads per disk +# as well as the total number of memtables that can be flushed concurrently. +# These are generally a combination of compute and IO bound. +# +# Memtable flushing is more CPU efficient than memtable ingest and a single thread +# can keep up with the ingest rate of a whole server on a single fast disk +# until it temporarily becomes IO bound under contention typically with compaction. +# At that point you need multiple flush threads. At some point in the future +# it may become CPU bound all the time. +# +# You can tell if flushing is falling behind using the MemtablePool.BlockedOnAllocation +# metric which should be 0, but will be non-zero if threads are blocked waiting on flushing +# to free memory. +# +# memtable_flush_writers defaults to two for a single data directory. +# This means that two memtables can be flushed concurrently to the single data directory. +# If you have multiple data directories the default is one memtable flushing at a time +# but the flush will use a thread per data directory so you will get two or more writers. +# +# Two is generally enough to flush on a fast disk [array] mounted as a single data directory. +# Adding more flush writers will result in smaller more frequent flushes that introduce more +# compaction overhead. +# +# There is a direct tradeoff between number of memtables that can be flushed concurrently +# and flush size and frequency. More is not better you just need enough flush writers +# to never stall waiting for flushing to free memory. +# +#memtable_flush_writers: 2 + +# Total space to use for change-data-capture logs on disk. +# +# If space gets above this value, Cassandra will throw WriteTimeoutException +# on Mutations including tables with CDC enabled. A CDCCompactor is responsible +# for parsing the raw CDC logs and deleting them when parsing is completed. +# +# The default value is the min of 4096 mb and 1/8th of the total space +# of the drive where cdc_raw_directory resides. +# cdc_total_space_in_mb: 4096 + +# When we hit our cdc_raw limit and the CDCCompactor is either running behind +# or experiencing backpressure, we check at the following interval to see if any +# new space for cdc-tracked tables has been made available. Default to 250ms +# cdc_free_space_check_interval_ms: 250 + +# A fixed memory pool size in MB for for SSTable index summaries. If left +# empty, this will default to 5% of the heap size. If the memory usage of +# all index summaries exceeds this limit, SSTables with low read rates will +# shrink their index summaries in order to meet this limit. However, this +# is a best-effort process. In extreme conditions Cassandra may need to use +# more than this amount of memory. +index_summary_capacity_in_mb: + +# How frequently index summaries should be resampled. This is done +# periodically to redistribute memory from the fixed-size pool to sstables +# proportional their recent read rates. Setting to -1 will disable this +# process, leaving existing index summaries at their current sampling level. +index_summary_resize_interval_in_minutes: 60 + +# Whether to, when doing sequential writing, fsync() at intervals in +# order to force the operating system to flush the dirty +# buffers. Enable this to avoid sudden dirty buffer flushing from +# impacting read latencies. Almost always a good idea on SSDs; not +# necessarily on platters. +trickle_fsync: false +trickle_fsync_interval_in_kb: 10240 + +# TCP port, for commands and data +# For security reasons, you should not expose this port to the internet. Firewall it if needed. +storage_port: 7000 + +# SSL port, for encrypted communication. Unused unless enabled in +# encryption_options +# For security reasons, you should not expose this port to the internet. Firewall it if needed. +ssl_storage_port: 7001 + +# Address or interface to bind to and tell other Cassandra nodes to connect to. +# You _must_ change this if you want multiple nodes to be able to communicate! +# +# Set listen_address OR listen_interface, not both. +# +# Leaving it blank leaves it up to InetAddress.getLocalHost(). This +# will always do the Right Thing _if_ the node is properly configured +# (hostname, name resolution, etc), and the Right Thing is to use the +# address associated with the hostname (it might not be). +# +# Setting listen_address to 0.0.0.0 is always wrong. +# +listen_address: localhost + +# Set listen_address OR listen_interface, not both. Interfaces must correspond +# to a single address, IP aliasing is not supported. +# listen_interface: eth0 + +# If you choose to specify the interface by name and the interface has an ipv4 and an ipv6 address +# you can specify which should be chosen using listen_interface_prefer_ipv6. If false the first ipv4 +# address will be used. If true the first ipv6 address will be used. Defaults to false preferring +# ipv4. If there is only one address it will be selected regardless of ipv4/ipv6. +# listen_interface_prefer_ipv6: false + +# Address to broadcast to other Cassandra nodes +# Leaving this blank will set it to the same value as listen_address +# broadcast_address: 1.2.3.4 + +# When using multiple physical network interfaces, set this +# to true to listen on broadcast_address in addition to +# the listen_address, allowing nodes to communicate in both +# interfaces. +# Ignore this property if the network configuration automatically +# routes between the public and private networks such as EC2. +# listen_on_broadcast_address: false + +# Internode authentication backend, implementing IInternodeAuthenticator; +# used to allow/disallow connections from peer nodes. +# internode_authenticator: org.apache.cassandra.auth.AllowAllInternodeAuthenticator + +# Whether to start the native transport server. +# Please note that the address on which the native transport is bound is the +# same as the rpc_address. The port however is different and specified below. +start_native_transport: true +# port for the CQL native transport to listen for clients on +# For security reasons, you should not expose this port to the internet. Firewall it if needed. +native_transport_port: 9042 +# Enabling native transport encryption in client_encryption_options allows you to either use +# encryption for the standard port or to use a dedicated, additional port along with the unencrypted +# standard native_transport_port. +# Enabling client encryption and keeping native_transport_port_ssl disabled will use encryption +# for native_transport_port. Setting native_transport_port_ssl to a different value +# from native_transport_port will use encryption for native_transport_port_ssl while +# keeping native_transport_port unencrypted. +# native_transport_port_ssl: 9142 +# The maximum threads for handling requests when the native transport is used. +# This is similar to rpc_max_threads though the default differs slightly (and +# there is no native_transport_min_threads, idle threads will always be stopped +# after 30 seconds). +# native_transport_max_threads: 128 +# +# The maximum size of allowed frame. Frame (requests) larger than this will +# be rejected as invalid. The default is 256MB. If you're changing this parameter, +# you may want to adjust max_value_size_in_mb accordingly. This should be positive and less than 2048. +# native_transport_max_frame_size_in_mb: 256 + +# The maximum number of concurrent client connections. +# The default is -1, which means unlimited. +# native_transport_max_concurrent_connections: -1 + +# The maximum number of concurrent client connections per source ip. +# The default is -1, which means unlimited. +# native_transport_max_concurrent_connections_per_ip: -1 + +# Whether to start the thrift rpc server. +start_rpc: true + +# The address or interface to bind the Thrift RPC service and native transport +# server to. +# +# Set rpc_address OR rpc_interface, not both. +# +# Leaving rpc_address blank has the same effect as on listen_address +# (i.e. it will be based on the configured hostname of the node). +# +# Note that unlike listen_address, you can specify 0.0.0.0, but you must also +# set broadcast_rpc_address to a value other than 0.0.0.0. +# +# For security reasons, you should not expose this port to the internet. Firewall it if needed. +rpc_address: 0.0.0.0 + +# Set rpc_address OR rpc_interface, not both. Interfaces must correspond +# to a single address, IP aliasing is not supported. +# rpc_interface: eth1 + +# If you choose to specify the interface by name and the interface has an ipv4 and an ipv6 address +# you can specify which should be chosen using rpc_interface_prefer_ipv6. If false the first ipv4 +# address will be used. If true the first ipv6 address will be used. Defaults to false preferring +# ipv4. If there is only one address it will be selected regardless of ipv4/ipv6. +# rpc_interface_prefer_ipv6: false + +# port for Thrift to listen for clients on +rpc_port: 9160 + +# RPC address to broadcast to drivers and other Cassandra nodes. This cannot +# be set to 0.0.0.0. If left blank, this will be set to the value of +# rpc_address. If rpc_address is set to 0.0.0.0, broadcast_rpc_address must +# be set. +broadcast_rpc_address: 1.2.3.4 + +# enable or disable keepalive on rpc/native connections +rpc_keepalive: true + +# Cassandra provides two out-of-the-box options for the RPC Server: +# +# sync +# One thread per thrift connection. For a very large number of clients, memory +# will be your limiting factor. On a 64 bit JVM, 180KB is the minimum stack size +# per thread, and that will correspond to your use of virtual memory (but physical memory +# may be limited depending on use of stack space). +# +# hsha +# Stands for "half synchronous, half asynchronous." All thrift clients are handled +# asynchronously using a small number of threads that does not vary with the amount +# of thrift clients (and thus scales well to many clients). The rpc requests are still +# synchronous (one thread per active request). If hsha is selected then it is essential +# that rpc_max_threads is changed from the default value of unlimited. +# +# The default is sync because on Windows hsha is about 30% slower. On Linux, +# sync/hsha performance is about the same, with hsha of course using less memory. +# +# Alternatively, can provide your own RPC server by providing the fully-qualified class name +# of an o.a.c.t.TServerFactory that can create an instance of it. +rpc_server_type: sync + +# Uncomment rpc_min|max_thread to set request pool size limits. +# +# Regardless of your choice of RPC server (see above), the number of maximum requests in the +# RPC thread pool dictates how many concurrent requests are possible (but if you are using the sync +# RPC server, it also dictates the number of clients that can be connected at all). +# +# The default is unlimited and thus provides no protection against clients overwhelming the server. You are +# encouraged to set a maximum that makes sense for you in production, but do keep in mind that +# rpc_max_threads represents the maximum number of client requests this server may execute concurrently. +# +# rpc_min_threads: 16 +# rpc_max_threads: 2048 + +# uncomment to set socket buffer sizes on rpc connections +# rpc_send_buff_size_in_bytes: +# rpc_recv_buff_size_in_bytes: + +# Uncomment to set socket buffer size for internode communication +# Note that when setting this, the buffer size is limited by net.core.wmem_max +# and when not setting it it is defined by net.ipv4.tcp_wmem +# See also: +# /proc/sys/net/core/wmem_max +# /proc/sys/net/core/rmem_max +# /proc/sys/net/ipv4/tcp_wmem +# /proc/sys/net/ipv4/tcp_wmem +# and 'man tcp' +# internode_send_buff_size_in_bytes: + +# Uncomment to set socket buffer size for internode communication +# Note that when setting this, the buffer size is limited by net.core.wmem_max +# and when not setting it it is defined by net.ipv4.tcp_wmem +# internode_recv_buff_size_in_bytes: + +# Frame size for thrift (maximum message length). +thrift_framed_transport_size_in_mb: 15 + +# Set to true to have Cassandra create a hard link to each sstable +# flushed or streamed locally in a backups/ subdirectory of the +# keyspace data. Removing these links is the operator's +# responsibility. +incremental_backups: false + +# Whether or not to take a snapshot before each compaction. Be +# careful using this option, since Cassandra won't clean up the +# snapshots for you. Mostly useful if you're paranoid when there +# is a data format change. +snapshot_before_compaction: false + +# Whether or not a snapshot is taken of the data before keyspace truncation +# or dropping of column families. The STRONGLY advised default of true +# should be used to provide data safety. If you set this flag to false, you will +# lose data on truncation or drop. +auto_snapshot: true + +# Granularity of the collation index of rows within a partition. +# Increase if your rows are large, or if you have a very large +# number of rows per partition. The competing goals are these: +# +# - a smaller granularity means more index entries are generated +# and looking up rows withing the partition by collation column +# is faster +# - but, Cassandra will keep the collation index in memory for hot +# rows (as part of the key cache), so a larger granularity means +# you can cache more hot rows +column_index_size_in_kb: 64 + +# Per sstable indexed key cache entries (the collation index in memory +# mentioned above) exceeding this size will not be held on heap. +# This means that only partition information is held on heap and the +# index entries are read from disk. +# +# Note that this size refers to the size of the +# serialized index information and not the size of the partition. +column_index_cache_size_in_kb: 2 + +# Number of simultaneous compactions to allow, NOT including +# validation "compactions" for anti-entropy repair. Simultaneous +# compactions can help preserve read performance in a mixed read/write +# workload, by mitigating the tendency of small sstables to accumulate +# during a single long running compactions. The default is usually +# fine and if you experience problems with compaction running too +# slowly or too fast, you should look at +# compaction_throughput_mb_per_sec first. +# +# concurrent_compactors defaults to the smaller of (number of disks, +# number of cores), with a minimum of 2 and a maximum of 8. +# +# If your data directories are backed by SSD, you should increase this +# to the number of cores. +#concurrent_compactors: 1 + +# Throttles compaction to the given total throughput across the entire +# system. The faster you insert data, the faster you need to compact in +# order to keep the sstable count down, but in general, setting this to +# 16 to 32 times the rate you are inserting data is more than sufficient. +# Setting this to 0 disables throttling. Note that this account for all types +# of compaction, including validation compaction. +compaction_throughput_mb_per_sec: 16 + +# When compacting, the replacement sstable(s) can be opened before they +# are completely written, and used in place of the prior sstables for +# any range that has been written. This helps to smoothly transfer reads +# between the sstables, reducing page cache churn and keeping hot rows hot +sstable_preemptive_open_interval_in_mb: 50 + +# Throttles all outbound streaming file transfers on this node to the +# given total throughput in Mbps. This is necessary because Cassandra does +# mostly sequential IO when streaming data during bootstrap or repair, which +# can lead to saturating the network connection and degrading rpc performance. +# When unset, the default is 200 Mbps or 25 MB/s. +# stream_throughput_outbound_megabits_per_sec: 200 + +# Throttles all streaming file transfer between the datacenters, +# this setting allows users to throttle inter dc stream throughput in addition +# to throttling all network stream traffic as configured with +# stream_throughput_outbound_megabits_per_sec +# When unset, the default is 200 Mbps or 25 MB/s +# inter_dc_stream_throughput_outbound_megabits_per_sec: 200 + +# How long the coordinator should wait for read operations to complete +read_request_timeout_in_ms: 5000 +# How long the coordinator should wait for seq or index scans to complete +range_request_timeout_in_ms: 10000 +# How long the coordinator should wait for writes to complete +write_request_timeout_in_ms: 2000 +# How long the coordinator should wait for counter writes to complete +counter_write_request_timeout_in_ms: 5000 +# How long a coordinator should continue to retry a CAS operation +# that contends with other proposals for the same row +cas_contention_timeout_in_ms: 1000 +# How long the coordinator should wait for truncates to complete +# (This can be much longer, because unless auto_snapshot is disabled +# we need to flush first so we can snapshot before removing the data.) +truncate_request_timeout_in_ms: 60000 +# The default timeout for other, miscellaneous operations +request_timeout_in_ms: 10000 + +# How long before a node logs slow queries. Select queries that take longer than +# this timeout to execute, will generate an aggregated log message, so that slow queries +# can be identified. Set this value to zero to disable slow query logging. +slow_query_log_timeout_in_ms: 500 + +# Enable operation timeout information exchange between nodes to accurately +# measure request timeouts. If disabled, replicas will assume that requests +# were forwarded to them instantly by the coordinator, which means that +# under overload conditions we will waste that much extra time processing +# already-timed-out requests. +# +# Warning: before enabling this property make sure to ntp is installed +# and the times are synchronized between the nodes. +cross_node_timeout: false + +# Set keep-alive period for streaming +# This node will send a keep-alive message periodically with this period. +# If the node does not receive a keep-alive message from the peer for +# 2 keep-alive cycles the stream session times out and fail +# Default value is 300s (5 minutes), which means stalled stream +# times out in 10 minutes by default +# streaming_keep_alive_period_in_secs: 300 + +# phi value that must be reached for a host to be marked down. +# most users should never need to adjust this. +# phi_convict_threshold: 8 + +# endpoint_snitch -- Set this to a class that implements +# IEndpointSnitch. The snitch has two functions: +# +# - it teaches Cassandra enough about your network topology to route +# requests efficiently +# - it allows Cassandra to spread replicas around your cluster to avoid +# correlated failures. It does this by grouping machines into +# "datacenters" and "racks." Cassandra will do its best not to have +# more than one replica on the same "rack" (which may not actually +# be a physical location) +# +# CASSANDRA WILL NOT ALLOW YOU TO SWITCH TO AN INCOMPATIBLE SNITCH +# ONCE DATA IS INSERTED INTO THE CLUSTER. This would cause data loss. +# This means that if you start with the default SimpleSnitch, which +# locates every node on "rack1" in "datacenter1", your only options +# if you need to add another datacenter are GossipingPropertyFileSnitch +# (and the older PFS). From there, if you want to migrate to an +# incompatible snitch like Ec2Snitch you can do it by adding new nodes +# under Ec2Snitch (which will locate them in a new "datacenter") and +# decommissioning the old ones. +# +# Out of the box, Cassandra provides: +# +# SimpleSnitch: +# Treats Strategy order as proximity. This can improve cache +# locality when disabling read repair. Only appropriate for +# single-datacenter deployments. +# +# GossipingPropertyFileSnitch +# This should be your go-to snitch for production use. The rack +# and datacenter for the local node are defined in +# cassandra-rackdc.properties and propagated to other nodes via +# gossip. If cassandra-topology.properties exists, it is used as a +# fallback, allowing migration from the PropertyFileSnitch. +# +# PropertyFileSnitch: +# Proximity is determined by rack and data center, which are +# explicitly configured in cassandra-topology.properties. +# +# Ec2Snitch: +# Appropriate for EC2 deployments in a single Region. Loads Region +# and Availability Zone information from the EC2 API. The Region is +# treated as the datacenter, and the Availability Zone as the rack. +# Only private IPs are used, so this will not work across multiple +# Regions. +# +# Ec2MultiRegionSnitch: +# Uses public IPs as broadcast_address to allow cross-region +# connectivity. (Thus, you should set seed addresses to the public +# IP as well.) You will need to open the storage_port or +# ssl_storage_port on the public IP firewall. (For intra-Region +# traffic, Cassandra will switch to the private IP after +# establishing a connection.) +# +# RackInferringSnitch: +# Proximity is determined by rack and data center, which are +# assumed to correspond to the 3rd and 2nd octet of each node's IP +# address, respectively. Unless this happens to match your +# deployment conventions, this is best used as an example of +# writing a custom Snitch class and is provided in that spirit. +# +# You can use a custom Snitch by setting this to the full class name +# of the snitch, which will be assumed to be on your classpath. +endpoint_snitch: SimpleSnitch + +# controls how often to perform the more expensive part of host score +# calculation +dynamic_snitch_update_interval_in_ms: 100 +# controls how often to reset all host scores, allowing a bad host to +# possibly recover +dynamic_snitch_reset_interval_in_ms: 600000 +# if set greater than zero and read_repair_chance is < 1.0, this will allow +# 'pinning' of replicas to hosts in order to increase cache capacity. +# The badness threshold will control how much worse the pinned host has to be +# before the dynamic snitch will prefer other replicas over it. This is +# expressed as a double which represents a percentage. Thus, a value of +# 0.2 means Cassandra would continue to prefer the static snitch values +# until the pinned host was 20% worse than the fastest. +dynamic_snitch_badness_threshold: 0.1 + +# request_scheduler -- Set this to a class that implements +# RequestScheduler, which will schedule incoming client requests +# according to the specific policy. This is useful for multi-tenancy +# with a single Cassandra cluster. +# NOTE: This is specifically for requests from the client and does +# not affect inter node communication. +# org.apache.cassandra.scheduler.NoScheduler - No scheduling takes place +# org.apache.cassandra.scheduler.RoundRobinScheduler - Round robin of +# client requests to a node with a separate queue for each +# request_scheduler_id. The scheduler is further customized by +# request_scheduler_options as described below. +request_scheduler: org.apache.cassandra.scheduler.NoScheduler + +# Scheduler Options vary based on the type of scheduler +# +# NoScheduler +# Has no options +# +# RoundRobin +# throttle_limit +# The throttle_limit is the number of in-flight +# requests per client. Requests beyond +# that limit are queued up until +# running requests can complete. +# The value of 80 here is twice the number of +# concurrent_reads + concurrent_writes. +# default_weight +# default_weight is optional and allows for +# overriding the default which is 1. +# weights +# Weights are optional and will default to 1 or the +# overridden default_weight. The weight translates into how +# many requests are handled during each turn of the +# RoundRobin, based on the scheduler id. +# +# request_scheduler_options: +# throttle_limit: 80 +# default_weight: 5 +# weights: +# Keyspace1: 1 +# Keyspace2: 5 + +# request_scheduler_id -- An identifier based on which to perform +# the request scheduling. Currently the only valid option is keyspace. +# request_scheduler_id: keyspace + +# Enable or disable inter-node encryption +# JVM defaults for supported SSL socket protocols and cipher suites can +# be replaced using custom encryption options. This is not recommended +# unless you have policies in place that dictate certain settings, or +# need to disable vulnerable ciphers or protocols in case the JVM cannot +# be updated. +# FIPS compliant settings can be configured at JVM level and should not +# involve changing encryption settings here: +# https://docs.oracle.com/javase/8/docs/technotes/guides/security/jsse/FIPS.html +# *NOTE* No custom encryption options are enabled at the moment +# The available internode options are : all, none, dc, rack +# +# If set to dc cassandra will encrypt the traffic between the DCs +# If set to rack cassandra will encrypt the traffic between the racks +# +# The passwords used in these options must match the passwords used when generating +# the keystore and truststore. For instructions on generating these files, see: +# http://download.oracle.com/javase/6/docs/technotes/guides/security/jsse/JSSERefGuide.html#CreateKeystore +# +server_encryption_options: + internode_encryption: none + keystore: conf/.keystore + keystore_password: cassandra + truststore: conf/.truststore + truststore_password: cassandra + # More advanced defaults below: + # protocol: TLS + # algorithm: SunX509 + # store_type: JKS + # cipher_suites: [TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_DHE_RSA_WITH_AES_128_CBC_SHA,TLS_DHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA] + # require_client_auth: false + # require_endpoint_verification: false + +# enable or disable client/server encryption. +client_encryption_options: + enabled: false + # If enabled and optional is set to true encrypted and unencrypted connections are handled. + optional: false + keystore: conf/.keystore + keystore_password: cassandra + # require_client_auth: false + # Set trustore and truststore_password if require_client_auth is true + # truststore: conf/.truststore + # truststore_password: cassandra + # More advanced defaults below: + # protocol: TLS + # algorithm: SunX509 + # store_type: JKS + # cipher_suites: [TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_DHE_RSA_WITH_AES_128_CBC_SHA,TLS_DHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA] + +# internode_compression controls whether traffic between nodes is +# compressed. +# Can be: +# +# all +# all traffic is compressed +# +# dc +# traffic between different datacenters is compressed +# +# none +# nothing is compressed. +internode_compression: dc + +# Enable or disable tcp_nodelay for inter-dc communication. +# Disabling it will result in larger (but fewer) network packets being sent, +# reducing overhead from the TCP protocol itself, at the cost of increasing +# latency if you block for cross-datacenter responses. +inter_dc_tcp_nodelay: false + +# TTL for different trace types used during logging of the repair process. +tracetype_query_ttl: 86400 +tracetype_repair_ttl: 604800 + +# By default, Cassandra logs GC Pauses greater than 200 ms at INFO level +# This threshold can be adjusted to minimize logging if necessary +# gc_log_threshold_in_ms: 200 + +# If unset, all GC Pauses greater than gc_log_threshold_in_ms will log at +# INFO level +# UDFs (user defined functions) are disabled by default. +# As of Cassandra 3.0 there is a sandbox in place that should prevent execution of evil code. +enable_user_defined_functions: false + +# Enables scripted UDFs (JavaScript UDFs). +# Java UDFs are always enabled, if enable_user_defined_functions is true. +# Enable this option to be able to use UDFs with "language javascript" or any custom JSR-223 provider. +# This option has no effect, if enable_user_defined_functions is false. +enable_scripted_user_defined_functions: false + +# Enables materialized view creation on this node. +# Materialized views are considered experimental and are not recommended for production use. +enable_materialized_views: true + +# The default Windows kernel timer and scheduling resolution is 15.6ms for power conservation. +# Lowering this value on Windows can provide much tighter latency and better throughput, however +# some virtualized environments may see a negative performance impact from changing this setting +# below their system default. The sysinternals 'clockres' tool can confirm your system's default +# setting. +windows_timer_interval: 1 + + +# Enables encrypting data at-rest (on disk). Different key providers can be plugged in, but the default reads from +# a JCE-style keystore. A single keystore can hold multiple keys, but the one referenced by +# the "key_alias" is the only key that will be used for encrypt opertaions; previously used keys +# can still (and should!) be in the keystore and will be used on decrypt operations +# (to handle the case of key rotation). +# +# It is strongly recommended to download and install Java Cryptography Extension (JCE) +# Unlimited Strength Jurisdiction Policy Files for your version of the JDK. +# (current link: http://www.oracle.com/technetwork/java/javase/downloads/jce8-download-2133166.html) +# +# Currently, only the following file types are supported for transparent data encryption, although +# more are coming in future cassandra releases: commitlog, hints +transparent_data_encryption_options: + enabled: false + chunk_length_kb: 64 + cipher: AES/CBC/PKCS5Padding + key_alias: testing:1 + # CBC IV length for AES needs to be 16 bytes (which is also the default size) + # iv_length: 16 + key_provider: + - class_name: org.apache.cassandra.security.JKSKeyProvider + parameters: + - keystore: conf/.keystore + keystore_password: cassandra + store_type: JCEKS + key_password: cassandra + + +##################### +# SAFETY THRESHOLDS # +##################### + +# When executing a scan, within or across a partition, we need to keep the +# tombstones seen in memory so we can return them to the coordinator, which +# will use them to make sure other replicas also know about the deleted rows. +# With workloads that generate a lot of tombstones, this can cause performance +# problems and even exaust the server heap. +# (http://www.datastax.com/dev/blog/cassandra-anti-patterns-queues-and-queue-like-datasets) +# Adjust the thresholds here if you understand the dangers and want to +# scan more tombstones anyway. These thresholds may also be adjusted at runtime +# using the StorageService mbean. +tombstone_warn_threshold: 1000 +tombstone_failure_threshold: 100000 + +# Log WARN on any multiple-partition batch size exceeding this value. 5kb per batch by default. +# Caution should be taken on increasing the size of this threshold as it can lead to node instability. +batch_size_warn_threshold_in_kb: 5 + +# Fail any multiple-partition batch exceeding this value. 50kb (10x warn threshold) by default. +batch_size_fail_threshold_in_kb: 50 + +# Log WARN on any batches not of type LOGGED than span across more partitions than this limit +unlogged_batch_across_partitions_warn_threshold: 10 + +# Log a warning when compacting partitions larger than this value +compaction_large_partition_warning_threshold_mb: 100 + +# GC Pauses greater than gc_warn_threshold_in_ms will be logged at WARN level +# Adjust the threshold based on your application throughput requirement +# By default, Cassandra logs GC Pauses greater than 200 ms at INFO level +gc_warn_threshold_in_ms: 1000 + +# Maximum size of any value in SSTables. Safety measure to detect SSTable corruption +# early. Any value size larger than this threshold will result into marking an SSTable +# as corrupted. This should be positive and less than 2048. +# max_value_size_in_mb: 256 + +# Back-pressure settings # +# If enabled, the coordinator will apply the back-pressure strategy specified below to each mutation +# sent to replicas, with the aim of reducing pressure on overloaded replicas. +back_pressure_enabled: false +# The back-pressure strategy applied. +# The default implementation, RateBasedBackPressure, takes three arguments: +# high ratio, factor, and flow type, and uses the ratio between incoming mutation responses and outgoing mutation requests. +# If below high ratio, outgoing mutations are rate limited according to the incoming rate decreased by the given factor; +# if above high ratio, the rate limiting is increased by the given factor; +# such factor is usually best configured between 1 and 10, use larger values for a faster recovery +# at the expense of potentially more dropped mutations; +# the rate limiting is applied according to the flow type: if FAST, it's rate limited at the speed of the fastest replica, +# if SLOW at the speed of the slowest one. +# New strategies can be added. Implementors need to implement org.apache.cassandra.net.BackpressureStrategy and +# provide a public constructor accepting a Map. +back_pressure_strategy: + - class_name: org.apache.cassandra.net.RateBasedBackPressure + parameters: + - high_ratio: 0.90 + factor: 5 + flow: FAST + +# Coalescing Strategies # +# Coalescing multiples messages turns out to significantly boost message processing throughput (think doubling or more). +# On bare metal, the floor for packet processing throughput is high enough that many applications won't notice, but in +# virtualized environments, the point at which an application can be bound by network packet processing can be +# surprisingly low compared to the throughput of task processing that is possible inside a VM. It's not that bare metal +# doesn't benefit from coalescing messages, it's that the number of packets a bare metal network interface can process +# is sufficient for many applications such that no load starvation is experienced even without coalescing. +# There are other benefits to coalescing network messages that are harder to isolate with a simple metric like messages +# per second. By coalescing multiple tasks together, a network thread can process multiple messages for the cost of one +# trip to read from a socket, and all the task submission work can be done at the same time reducing context switching +# and increasing cache friendliness of network message processing. +# See CASSANDRA-8692 for details. + +# Strategy to use for coalescing messages in OutboundTcpConnection. +# Can be fixed, movingaverage, timehorizon, disabled (default). +# You can also specify a subclass of CoalescingStrategies.CoalescingStrategy by name. +# otc_coalescing_strategy: DISABLED + +# How many microseconds to wait for coalescing. For fixed strategy this is the amount of time after the first +# message is received before it will be sent with any accompanying messages. For moving average this is the +# maximum amount of time that will be waited as well as the interval at which messages must arrive on average +# for coalescing to be enabled. +# otc_coalescing_window_us: 200 + +# Do not try to coalesce messages if we already got that many messages. This should be more than 2 and less than 128. +# otc_coalescing_enough_coalesced_messages: 8 + +# How many milliseconds to wait between two expiration runs on the backlog (queue) of the OutboundTcpConnection. +# Expiration is done if messages are piling up in the backlog. Droppable messages are expired to free the memory +# taken by expired messages. The interval should be between 0 and 1000, and in most installations the default value +# will be appropriate. A smaller value could potentially expire messages slightly sooner at the expense of more CPU +# time and queue contention while iterating the backlog of messages. +# An interval of 0 disables any wait time, which is the behavior of former Cassandra versions. +# +# otc_backlog_expiration_interval_ms: 200 diff --git a/.ci/docker-compose-file/docker-compose-cassandra-tcp.yaml b/.ci/docker-compose-file/docker-compose-cassandra-tcp.yaml new file mode 100644 index 000000000..ce0cb4de5 --- /dev/null +++ b/.ci/docker-compose-file/docker-compose-cassandra-tcp.yaml @@ -0,0 +1,27 @@ +version: '3.9' + +services: + cassandra_server: + container_name: cassa_tcp + build: + context: ./cassandra + args: + CASSANDRA_TAG: ${CASSANDRA_TAG} + image: emqx-cassandra + restart: always + environment: + CASSANDRA_BROADCAST_ADDRESS: "1.2.3.4" + CASSANDRA_RPC_ADDRESS: "0.0.0.0" + ports: + - "9042:9042" + command: + - /bin/bash + - -c + - | + /opt/cassandra/bin/cassandra -f -R > /cassandra.log & + /opt/cassandra/bin/cqlsh -e "CREATE KEYSPACE mqtt WITH REPLICATION = { 'class':'SimpleStrategy','replication_factor':1};" + while [[ $$? -ne 0 ]];do sleep 5; /opt/cassandra/bin/cqlsh -e "CREATE KEYSPACE mqtt WITH REPLICATION = { 'class':'SimpleStrategy','replication_factor':1};"; done + /opt/cassandra/bin/cqlsh -e "describe keyspaces;" + tail -f /cassandra.log + networks: + - emqx_bridge diff --git a/.ci/docker-compose-file/docker-compose-toxiproxy.yaml b/.ci/docker-compose-file/docker-compose-toxiproxy.yaml index 3dd30af52..dd8b91252 100644 --- a/.ci/docker-compose-file/docker-compose-toxiproxy.yaml +++ b/.ci/docker-compose-file/docker-compose-toxiproxy.yaml @@ -19,6 +19,7 @@ services: - 15433:5433 - 16041:6041 - 18000:8000 + - 19042:9042 command: - "-host=0.0.0.0" - "-config=/config/toxiproxy.json" diff --git a/.ci/docker-compose-file/toxiproxy.json b/.ci/docker-compose-file/toxiproxy.json index 6188eab17..79589c14a 100644 --- a/.ci/docker-compose-file/toxiproxy.json +++ b/.ci/docker-compose-file/toxiproxy.json @@ -53,5 +53,17 @@ "listen": "0.0.0.0:8000", "upstream": "dynamo:8000", "enabled": true + }, + { + "name": "cassa_tcp", + "listen": "0.0.0.0:9042", + "upstream": "cassa_tcp:9042", + "enabled": true + }, + { + "name": "cassa_tls", + "listen": "0.0.0.0:9043", + "upstream": "cassa_tls:9043", + "enabled": false } ] diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index ddf24d380..4b51a32b1 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -63,7 +63,8 @@ T == timescale; T == matrix; T == tdengine; - T == dynamo + T == dynamo; + T == cassandra ). load() -> diff --git a/lib-ee/emqx_ee_bridge/docker-ct b/lib-ee/emqx_ee_bridge/docker-ct index ac1728ad2..34ae9111f 100644 --- a/lib-ee/emqx_ee_bridge/docker-ct +++ b/lib-ee/emqx_ee_bridge/docker-ct @@ -10,3 +10,4 @@ pgsql tdengine clickhouse dynamo +cassandra diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_cassa.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_cassa.conf new file mode 100644 index 000000000..3e9a9845e --- /dev/null +++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_cassa.conf @@ -0,0 +1,72 @@ +emqx_ee_bridge_cassa { + + local_topic { + desc { + en: """The MQTT topic filter to be forwarded to Cassandra. All MQTT 'PUBLISH' messages with the topic +matching the local_topic will be forwarded.
+NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is +configured, then both the data got from the rule and the MQTT messages that match local_topic +will be forwarded.""" + zh: """发送到 'local_topic' 的消息都会转发到 Cassandra。
+注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。""" + } + label { + en: "Local Topic" + zh: "本地 Topic" + } + } + + sql_template { + desc { + en: """SQL Template""" + zh: """SQL 模板""" + } + label { + en: "SQL Template" + zh: "SQL 模板" + } + } + config_enable { + desc { + en: """Enable or disable this bridge""" + zh: """启用/禁用桥接""" + } + label { + en: "Enable Or Disable Bridge" + zh: "启用/禁用桥接" + } + } + + desc_config { + desc { + en: """Configuration for an Cassandra bridge.""" + zh: """Cassandra 桥接配置""" + } + label: { + en: "Cassandra Bridge Configuration" + zh: "Cassandra 桥接配置" + } + } + + desc_type { + desc { + en: """The Bridge Type""" + zh: """Bridge 类型""" + } + label { + en: "Bridge Type" + zh: "桥接类型" + } + } + + desc_name { + desc { + en: """Bridge name.""" + zh: """桥接名字""" + } + label { + en: "Bridge Name" + zh: "桥接名字" + } + } +} diff --git a/lib-ee/emqx_ee_bridge/rebar.config b/lib-ee/emqx_ee_bridge/rebar.config index fa6dd560e..ba9ae07aa 100644 --- a/lib-ee/emqx_ee_bridge/rebar.config +++ b/lib-ee/emqx_ee_bridge/rebar.config @@ -3,6 +3,7 @@ , {kafka_protocol, {git, "https://github.com/kafka4beam/kafka_protocol.git", {tag, "4.1.2"}}} , {brod_gssapi, {git, "https://github.com/kafka4beam/brod_gssapi.git", {tag, "v0.1.0-rc1"}}} , {brod, {git, "https://github.com/kafka4beam/brod.git", {tag, "3.16.7"}}} + , {ecql, {git, "https://github.com/emqx/ecql.git", {tag, "v0.4.2"}}} , {emqx_connector, {path, "../../apps/emqx_connector"}} , {emqx_resource, {path, "../../apps/emqx_resource"}} , {emqx_bridge, {path, "../../apps/emqx_bridge"}} diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl index b5c656291..d89dc0bcc 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl @@ -31,7 +31,8 @@ api_schemas(Method) -> ref(emqx_ee_bridge_matrix, Method), ref(emqx_ee_bridge_tdengine, Method), ref(emqx_ee_bridge_clickhouse, Method), - ref(emqx_ee_bridge_dynamo, Method) + ref(emqx_ee_bridge_dynamo, Method), + ref(emqx_ee_bridge_cassa, Method) ]. schema_modules() -> @@ -48,7 +49,8 @@ schema_modules() -> emqx_ee_bridge_matrix, emqx_ee_bridge_tdengine, emqx_ee_bridge_clickhouse, - emqx_ee_bridge_dynamo + emqx_ee_bridge_dynamo, + emqx_ee_bridge_cassa ]. examples(Method) -> @@ -81,7 +83,8 @@ resource_type(timescale) -> emqx_connector_pgsql; resource_type(matrix) -> emqx_connector_pgsql; resource_type(tdengine) -> emqx_ee_connector_tdengine; resource_type(clickhouse) -> emqx_ee_connector_clickhouse; -resource_type(dynamo) -> emqx_ee_connector_dynamo. +resource_type(dynamo) -> emqx_ee_connector_dynamo; +resource_type(cassandra) -> emqx_ee_connector_cassa. fields(bridges) -> [ @@ -132,6 +135,14 @@ fields(bridges) -> desc => <<"Dynamo Bridge Config">>, required => false } + )}, + {cassandra, + mk( + hoconsc:map(name, ref(emqx_ee_bridge_cassa, "config")), + #{ + desc => <<"Cassandra Bridge Config">>, + required => false + } )} ] ++ mongodb_structs() ++ influxdb_structs() ++ redis_structs() ++ pgsql_structs() ++ clickhouse_structs(). diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl new file mode 100644 index 000000000..295d2c3b5 --- /dev/null +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl @@ -0,0 +1,133 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_ee_bridge_cassa). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("emqx_bridge/include/emqx_bridge.hrl"). +-include_lib("emqx_resource/include/emqx_resource.hrl"). + +-import(hoconsc, [mk/2, enum/1, ref/2]). + +%% schema examples +-export([ + conn_bridge_examples/1, + values/2, + fields/2 +]). + +%% schema +-export([ + namespace/0, + roots/0, + fields/1, + desc/1 +]). + +-define(DEFAULT_SQL, << + "insert into mqtt_msg(topic, msgid, sender, qos, payload, arrived, retain) " + "values (${topic}, ${id}, ${clientid}, ${qos}, ${payload}, ${timestamp}, ${flags.retain})" +>>). + +%%-------------------------------------------------------------------- +%% schema examples + +conn_bridge_examples(Method) -> + [ + #{ + <<"cassa">> => #{ + summary => <<"Cassandra Bridge">>, + value => values(Method, cassandra) + } + } + ]. + +values(get, Type) -> + maps:merge(values(post, Type), ?METRICS_EXAMPLE); +values(post, Type) -> + #{ + enable => true, + type => Type, + name => <<"foo">>, + servers => <<"127.0.0.1:9042">>, + keyspace => <<"mqtt">>, + pool_size => 8, + username => <<"root">>, + password => <<"public">>, + sql => ?DEFAULT_SQL, + local_topic => <<"local/topic/#">>, + resource_opts => #{ + worker_pool_size => 8, + health_check_interval => ?HEALTHCHECK_INTERVAL_RAW, + auto_restart_interval => ?AUTO_RESTART_INTERVAL_RAW, + batch_size => ?DEFAULT_BATCH_SIZE, + batch_time => ?DEFAULT_BATCH_TIME, + query_mode => sync, + max_queue_bytes => ?DEFAULT_QUEUE_SIZE + } + }; +values(put, Type) -> + values(post, Type). + +%%-------------------------------------------------------------------- +%% schema + +namespace() -> "bridge_cassa". + +roots() -> []. + +fields("config") -> + [ + {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})}, + {sql, + mk( + binary(), + #{desc => ?DESC("sql_template"), default => ?DEFAULT_SQL, format => <<"sql">>} + )}, + {local_topic, + mk( + binary(), + #{desc => ?DESC("local_topic"), default => undefined} + )}, + {resource_opts, + mk( + ref(?MODULE, "creation_opts"), + #{ + required => false, + default => #{}, + desc => ?DESC(emqx_resource_schema, <<"resource_opts">>) + } + )} + ] ++ + (emqx_ee_connector_cassa:fields(config) -- + emqx_connector_schema_lib:prepare_statement_fields()); +fields("creation_opts") -> + emqx_resource_schema:fields("creation_opts_sync_only"); +fields("post") -> + fields("post", cassa); +fields("put") -> + fields("config"); +fields("get") -> + emqx_bridge_schema:status_fields() ++ fields("post"). + +fields("post", Type) -> + [type_field(Type), name_field() | fields("config")]. + +desc("config") -> + ?DESC("desc_config"); +desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> + ["Configuration for Cassandra using `", string:to_upper(Method), "` method."]; +desc("creation_opts" = Name) -> + emqx_resource_schema:desc(Name); +desc(_) -> + undefined. + +%%-------------------------------------------------------------------- +%% utils + +type_field(Type) -> + {type, mk(enum([Type]), #{required => true, desc => ?DESC("desc_type")})}. + +name_field() -> + {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})}. diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl new file mode 100644 index 000000000..b509b537b --- /dev/null +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl @@ -0,0 +1,540 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_ee_bridge_cassa_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +% SQL definitions +-define(SQL_BRIDGE, + "insert into mqtt_msg_test(topic, payload, arrived) " + "values (${topic}, ${payload}, ${timestamp})" +). +-define(SQL_CREATE_TABLE, + "" + "\n" + "CREATE TABLE mqtt.mqtt_msg_test (\n" + " topic text,\n" + " payload text,\n" + " arrived timestamp,\n" + " PRIMARY KEY (topic)\n" + ");\n" + "" +). +-define(SQL_DROP_TABLE, "DROP TABLE mqtt_msg_test"). +-define(SQL_DELETE, "TRUNCATE mqtt_msg_test"). +-define(SQL_SELECT, "SELECT payload FROM mqtt_msg_test"). + +% DB defaults +-define(CASSA_KEYSPACE, "mqtt"). +-define(CASSA_USERNAME, "root"). +-define(CASSA_PASSWORD, "public"). +-define(BATCH_SIZE, 10). + +%%------------------------------------------------------------------------------ +%% CT boilerplate +%%------------------------------------------------------------------------------ + +all() -> + [ + {group, tcp}, + {group, tls} + ]. + +groups() -> + TCs = emqx_common_test_helpers:all(?MODULE), + NonBatchCases = [t_write_timeout], + [ + {tcp, [ + %{group, with_batch}, + {group, without_batch} + ]}, + {tls, [ + %{group, with_batch}, + {group, without_batch} + ]}, + {with_batch, TCs -- NonBatchCases}, + {without_batch, TCs} + ]. + +init_per_group(tcp, Config) -> + Host = os:getenv("CASSA_TCP_HOST", "toxiproxy"), + Port = list_to_integer(os:getenv("CASSA_TCP_PORT", "9042")), + [ + {cassa_host, Host}, + {cassa_port, Port}, + {enable_tls, false}, + {query_mode, sync}, + {proxy_name, "cassa_tcp"} + | Config + ]; +init_per_group(tls, Config) -> + Host = os:getenv("CASSA_TLS_HOST", "toxiproxy"), + Port = list_to_integer(os:getenv("CASSA_TLS_PORT", "9043")), + [ + {cassa_host, Host}, + {cassa_port, Port}, + {enable_tls, true}, + {query_mode, sync}, + {proxy_name, "cassa_tls"} + | Config + ]; +init_per_group(with_batch, Config0) -> + Config = [{enable_batch, true} | Config0], + common_init(Config); +init_per_group(without_batch, Config0) -> + Config = [{enable_batch, false} | Config0], + common_init(Config); +init_per_group(_Group, Config) -> + Config. + +end_per_group(Group, Config) when + Group == without_batch; Group == without_batch +-> + connect_and_drop_table(Config), + ProxyHost = ?config(proxy_host, Config), + ProxyPort = ?config(proxy_port, Config), + emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), + ok; +end_per_group(_Group, _Config) -> + ok. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + emqx_mgmt_api_test_util:end_suite(), + ok = emqx_common_test_helpers:stop_apps([emqx_bridge, emqx_conf]), + ok. + +init_per_testcase(_Testcase, Config) -> + connect_and_clear_table(Config), + delete_bridge(Config), + Config. + +end_per_testcase(_Testcase, Config) -> + ProxyHost = ?config(proxy_host, Config), + ProxyPort = ?config(proxy_port, Config), + emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), + connect_and_clear_table(Config), + ok = snabbkaffe:stop(), + delete_bridge(Config), + ok. + +%%------------------------------------------------------------------------------ +%% Helper fns +%%------------------------------------------------------------------------------ + +common_init(Config0) -> + BridgeType = proplists:get_value(bridge_type, Config0, <<"cassandra">>), + Host = ?config(cassa_host, Config0), + Port = ?config(cassa_port, Config0), + case emqx_common_test_helpers:is_tcp_server_available(Host, Port) of + true -> + % Setup toxiproxy + ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"), + ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")), + emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), + % Ensure EE bridge module is loaded + _ = application:load(emqx_ee_bridge), + _ = emqx_ee_bridge:module_info(), + ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]), + emqx_mgmt_api_test_util:init_suite(), + % Connect to cassnadra directly and create the table + connect_and_create_table(Config0), + {Name, CassaConf} = cassa_config(BridgeType, Config0), + Config = + [ + {cassa_config, CassaConf}, + {cassa_bridge_type, BridgeType}, + {cassa_name, Name}, + {proxy_host, ProxyHost}, + {proxy_port, ProxyPort} + | Config0 + ], + Config; + false -> + case os:getenv("IS_CI") of + "yes" -> + throw(no_cassandra); + _ -> + {skip, no_cassandra} + end + end. + +cassa_config(BridgeType, Config) -> + Port = integer_to_list(?config(cassa_port, Config)), + Server = ?config(cassa_host, Config) ++ ":" ++ Port, + Name = atom_to_binary(?MODULE), + BatchSize = + case ?config(enable_batch, Config) of + true -> ?BATCH_SIZE; + false -> 1 + end, + QueryMode = ?config(query_mode, Config), + TlsEnabled = ?config(enable_tls, Config), + ConfigString = + io_lib:format( + "bridges.~s.~s {\n" + " enable = true\n" + " servers = ~p\n" + " keyspace = ~p\n" + " username = ~p\n" + " password = ~p\n" + " sql = ~p\n" + " resource_opts = {\n" + " request_timeout = 500ms\n" + " batch_size = ~b\n" + " query_mode = ~s\n" + " }\n" + " ssl = {\n" + " enable = ~w\n" + " }\n" + "}", + [ + BridgeType, + Name, + Server, + ?CASSA_KEYSPACE, + ?CASSA_USERNAME, + ?CASSA_PASSWORD, + ?SQL_BRIDGE, + BatchSize, + QueryMode, + TlsEnabled + ] + ), + {Name, parse_and_check(ConfigString, BridgeType, Name)}. + +parse_and_check(ConfigString, BridgeType, Name) -> + {ok, RawConf} = hocon:binary(ConfigString, #{format => map}), + hocon_tconf:check_plain(emqx_bridge_schema, RawConf, #{required => false, atom_key => false}), + #{<<"bridges">> := #{BridgeType := #{Name := Config}}} = RawConf, + Config. + +create_bridge(Config) -> + BridgeType = ?config(cassa_bridge_type, Config), + Name = ?config(cassa_name, Config), + PGConfig = ?config(cassa_config, Config), + emqx_bridge:create(BridgeType, Name, PGConfig). + +delete_bridge(Config) -> + BridgeType = ?config(cassa_bridge_type, Config), + Name = ?config(cassa_name, Config), + emqx_bridge:remove(BridgeType, Name). + +create_bridge_http(Params) -> + Path = emqx_mgmt_api_test_util:api_path(["bridges"]), + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of + {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + Error -> Error + end. + +send_message(Config, Payload) -> + Name = ?config(cassa_name, Config), + BridgeType = ?config(cassa_bridge_type, Config), + BridgeID = emqx_bridge_resource:bridge_id(BridgeType, Name), + emqx_bridge:send_message(BridgeID, Payload). + +query_resource(Config, Request) -> + Name = ?config(cassa_name, Config), + BridgeType = ?config(cassa_bridge_type, Config), + ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), + emqx_resource:query(ResourceID, Request, #{timeout => 1_000}). + +connect_direct_cassa(Config) -> + Opts = #{ + host => ?config(cassa_host, Config), + port => ?config(cassa_port, Config), + username => ?CASSA_USERNAME, + password => ?CASSA_PASSWORD, + keyspace => ?CASSA_KEYSPACE + }, + + SslOpts = + case ?config(enable_tls, Config) of + true -> + Opts#{ + ssl => true, + ssl_opts => emqx_tls_lib:to_client_opts(#{enable => true}) + }; + false -> + Opts + end, + {ok, Con} = ecql:connect(maps:to_list(SslOpts)), + Con. + +% These funs connect and then stop the cassandra connection +connect_and_create_table(Config) -> + Con = connect_direct_cassa(Config), + {ok, _} = ecql:query(Con, ?SQL_CREATE_TABLE), + ok = ecql:close(Con). + +connect_and_drop_table(Config) -> + Con = connect_direct_cassa(Config), + {ok, _} = ecql:query(Con, ?SQL_DROP_TABLE), + ok = ecql:close(Con). + +connect_and_clear_table(Config) -> + Con = connect_direct_cassa(Config), + ok = ecql:query(Con, ?SQL_DELETE), + ok = ecql:close(Con). + +connect_and_get_payload(Config) -> + Con = connect_direct_cassa(Config), + {ok, {_Keyspace, _ColsSpec, [[Result]]}} = ecql:query(Con, ?SQL_SELECT), + ok = ecql:close(Con), + Result. + +%%------------------------------------------------------------------------------ +%% Testcases +%%------------------------------------------------------------------------------ + +t_setup_via_config_and_publish(Config) -> + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + Val = integer_to_binary(erlang:unique_integer()), + SentData = #{ + topic => atom_to_binary(?FUNCTION_NAME), + payload => Val, + timestamp => 1668602148000 + }, + ?check_trace( + begin + ?wait_async_action( + ?assertEqual(ok, send_message(Config, SentData)), + #{?snk_kind := cassandra_connector_query_return}, + 10_000 + ), + ?assertMatch( + Val, + connect_and_get_payload(Config) + ), + ok + end, + fun(Trace0) -> + Trace = ?of_kind(cassandra_connector_query_return, Trace0), + case ?config(enable_batch, Config) of + true -> + ?assertMatch([#{result := {_, [ok]}}], Trace); + false -> + ?assertMatch([#{result := ok}], Trace) + end, + ok + end + ), + ok. + +t_setup_via_http_api_and_publish(Config) -> + BridgeType = ?config(cassa_bridge_type, Config), + Name = ?config(cassa_name, Config), + PgsqlConfig0 = ?config(cassa_config, Config), + PgsqlConfig = PgsqlConfig0#{ + <<"name">> => Name, + <<"type">> => BridgeType + }, + ?assertMatch( + {ok, _}, + create_bridge_http(PgsqlConfig) + ), + Val = integer_to_binary(erlang:unique_integer()), + SentData = #{ + topic => atom_to_binary(?FUNCTION_NAME), + payload => Val, + timestamp => 1668602148000 + }, + ?check_trace( + begin + ?wait_async_action( + ?assertEqual(ok, send_message(Config, SentData)), + #{?snk_kind := cassandra_connector_query_return}, + 10_000 + ), + ?assertMatch( + Val, + connect_and_get_payload(Config) + ), + ok + end, + fun(Trace0) -> + Trace = ?of_kind(cassandra_connector_query_return, Trace0), + case ?config(enable_batch, Config) of + true -> + ?assertMatch([#{result := {_, [{ok, 1}]}}], Trace); + false -> + ?assertMatch([#{result := ok}], Trace) + end, + ok + end + ), + ok. + +t_get_status(Config) -> + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + ProxyPort = ?config(proxy_port, Config), + ProxyHost = ?config(proxy_host, Config), + ProxyName = ?config(proxy_name, Config), + + Name = ?config(cassa_name, Config), + BridgeType = ?config(cassa_bridge_type, Config), + ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), + + ?assertEqual({ok, connected}, emqx_resource_manager:health_check(ResourceID)), + emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() -> + ?assertMatch( + {ok, Status} when Status =:= disconnected orelse Status =:= connecting, + emqx_resource_manager:health_check(ResourceID) + ) + end), + ok. + +t_create_disconnected(Config) -> + ProxyPort = ?config(proxy_port, Config), + ProxyHost = ?config(proxy_host, Config), + ProxyName = ?config(proxy_name, Config), + ?check_trace( + emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() -> + ?assertMatch({ok, _}, create_bridge(Config)) + end), + fun(Trace) -> + ?assertMatch( + [#{error := {start_pool_failed, _, _}}], + ?of_kind(cassandra_connector_start_failed, Trace) + ), + ok + end + ), + ok. + +t_write_failure(Config) -> + ProxyName = ?config(proxy_name, Config), + ProxyPort = ?config(proxy_port, Config), + ProxyHost = ?config(proxy_host, Config), + QueryMode = ?config(query_mode, Config), + {ok, _} = create_bridge(Config), + Val = integer_to_binary(erlang:unique_integer()), + SentData = #{ + topic => atom_to_binary(?FUNCTION_NAME), + payload => Val, + timestamp => 1668602148000 + }, + ?check_trace( + emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() -> + {_, {ok, _}} = + ?wait_async_action( + case QueryMode of + sync -> + ?assertMatch({error, _}, send_message(Config, SentData)); + async -> + send_message(Config, SentData) + end, + #{?snk_kind := buffer_worker_flush_nack}, + 1_000 + ) + end), + fun(Trace0) -> + ct:pal("trace: ~p", [Trace0]), + Trace = ?of_kind(buffer_worker_flush_nack, Trace0), + ?assertMatch([#{result := {error, _}} | _], Trace), + [#{result := {error, Error}} | _] = Trace, + case Error of + {resource_error, _} -> + ok; + {recoverable_error, disconnected} -> + ok; + _ -> + ct:fail("unexpected error: ~p", [Error]) + end + end + ), + ok. + +%% This test doesn't work with batch enabled since it is not possible +%% to set the timeout directly for batch queries +%% +%% XXX: parameter with request timeout is not supported yet. +%% +%t_write_timeout(Config) -> +% ProxyName = ?config(proxy_name, Config), +% ProxyPort = ?config(proxy_port, Config), +% ProxyHost = ?config(proxy_host, Config), +% {ok, _} = create_bridge(Config), +% Val = integer_to_binary(erlang:unique_integer()), +% SentData = #{payload => Val, timestamp => 1668602148000}, +% Timeout = 1000, +% emqx_common_test_helpers:with_failure(timeout, ProxyName, ProxyHost, ProxyPort, fun() -> +% ?assertMatch( +% {error, {resource_error, #{reason := timeout}}}, +% query_resource(Config, {send_message, SentData, [], Timeout}) +% ) +% end), +% ok. + +t_simple_sql_query(Config) -> + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + Request = {query, <<"SELECT count(1) AS T FROM system.local">>}, + Result = query_resource(Config, Request), + case ?config(enable_batch, Config) of + true -> ?assertEqual({error, {unrecoverable_error, batch_prepare_not_implemented}}, Result); + false -> ?assertMatch({ok, {<<"system.local">>, _, [[1]]}}, Result) + end, + ok. + +t_missing_data(Config) -> + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + %% emqx_ee_connector_cassa will send missed data as a `null` atom + %% to ecql driver + Result = send_message(Config, #{}), + ?assertMatch( + %% TODO: match error msgs + {error, {unrecoverable_error, {8704, <<"Expected 8 or 0 byte long for date (4)">>}}}, + Result + ), + ok. + +t_bad_sql_parameter(Config) -> + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + Request = {query, <<"">>, [bad_parameter]}, + Result = query_resource(Config, Request), + case ?config(enable_batch, Config) of + true -> + ?assertEqual({error, {unrecoverable_error, invalid_request}}, Result); + false -> + ?assertMatch( + {error, {unrecoverable_error, _}}, Result + ) + end, + ok. + +t_nasty_sql_string(Config) -> + ?assertMatch({ok, _}, create_bridge(Config)), + Payload = list_to_binary(lists:seq(1, 127)), + Message = #{ + topic => atom_to_binary(?FUNCTION_NAME), + payload => Payload, + timestamp => erlang:system_time(millisecond) + }, + %% XXX: why ok instead of {ok, AffectedLines}? + ?assertEqual(ok, send_message(Config, Message)), + ?assertEqual(Payload, connect_and_get_payload(Config)). diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_cassa.conf b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_cassa.conf new file mode 100644 index 000000000..ecf004722 --- /dev/null +++ b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_cassa.conf @@ -0,0 +1,28 @@ +emqx_ee_connector_cassa { + + servers { + desc { + en: """The IPv4 or IPv6 address or the hostname to connect to.
+A host entry has the following form: `Host[:Port][,Host2:Port]`.
+The Cassandra default port 9042 is used if `[:Port]` is not specified.""" + zh: """将要连接的 IPv4 或 IPv6 地址，或者主机名。
+主机名具有以下形式：`Host[:Port][,Host2:Port]`。
+如果未指定 `[:Port]`，则使用 Cassandra 默认端口 9042。""" + } + label: { + en: "Servers" + zh: "Servers" + } + } + + keyspace { + desc { + en: """Keyspace name to connect to.""" + zh: """要连接到的 Keyspace 名称。""" + } + label: { + en: "Keyspace" + zh: "Keyspace" + } + } +} diff --git a/lib-ee/emqx_ee_connector/include/emqx_ee_connector.hrl b/lib-ee/emqx_ee_connector/include/emqx_ee_connector.hrl index 4b6fbbd92..2a91d2524 100644 --- a/lib-ee/emqx_ee_connector/include/emqx_ee_connector.hrl +++ b/lib-ee/emqx_ee_connector/include/emqx_ee_connector.hrl @@ -3,3 +3,4 @@ %%------------------------------------------------------------------- -define(INFLUXDB_DEFAULT_PORT, 8086). +-define(CASSANDRA_DEFAULT_PORT, 9042). diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src index 6f40f7158..ebe43d8d9 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src @@ -11,7 +11,8 @@ wolff, brod, clickhouse, - erlcloud + erlcloud, + ecql ]}, {env, []}, {modules, []}, diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl new file mode 100644 index 000000000..d1901e462 --- /dev/null +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl @@ -0,0 +1,415 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_ee_connector_cassa). + +-behaviour(emqx_resource). + +-include_lib("emqx_connector/include/emqx_connector.hrl"). +-include_lib("emqx_ee_connector/include/emqx_ee_connector.hrl"). +-include_lib("typerefl/include/types.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +%% schema +-export([roots/0, fields/1]). + +%% callbacks of behaviour emqx_resource +-export([ + callback_mode/0, + on_start/2, + on_stop/2, + on_query/3, + %% TODO: now_supported_now + %%on_batch_query/3, + on_get_status/2 +]). + +%% callbacks of ecpool +-export([ + connect/1, + prepare_sql_to_conn/2 +]). + +%% callbacks for query executing +-export([query/3, prepared_query/3]). + +-export([do_get_status/1]). + +-type prepares() :: #{atom() => binary()}. +-type params_tokens() :: #{atom() => list()}. + +-type state() :: + #{ + poolname := atom(), + prepare_sql := prepares(), + params_tokens := params_tokens(), + %% returned by ecql:prepare/2 + prepare_statement := binary() + }. + +-define(DEFAULT_SERVER_OPTION, #{default_port => ?CASSANDRA_DEFAULT_PORT}). + +%%-------------------------------------------------------------------- +%% schema + +roots() -> + [{config, #{type => hoconsc:ref(?MODULE, config)}}]. + +fields(config) -> + cassandra_db_fields() ++ + emqx_connector_schema_lib:ssl_fields() ++ + emqx_connector_schema_lib:prepare_statement_fields(). + +cassandra_db_fields() -> + [ + {servers, servers()}, + {keyspace, fun keyspace/1}, + {pool_size, fun emqx_connector_schema_lib:pool_size/1}, + {username, fun emqx_connector_schema_lib:username/1}, + {password, fun emqx_connector_schema_lib:password/1}, + {auto_reconnect, fun emqx_connector_schema_lib:auto_reconnect/1} + ]. + +servers() -> + Meta = #{desc => ?DESC("servers")}, + emqx_schema:servers_sc(Meta, ?DEFAULT_SERVER_OPTION). + +keyspace(type) -> binary(); +keyspace(desc) -> ?DESC("keyspace"); +keyspace(required) -> true; +keyspace(_) -> undefined. + +%%-------------------------------------------------------------------- +%% callbacks for emqx_resource + +callback_mode() -> always_sync. + +-spec on_start(binary(), hoconsc:config()) -> {ok, state()} | {error, _}. +on_start( + InstId, + #{ + servers := Servers, + keyspace := Keyspace, + username := Username, + pool_size := PoolSize, + ssl := SSL + } = Config +) -> + {ok, _} = application:ensure_all_started(ecpool), + {ok, _} = application:ensure_all_started(ecql), + + ?SLOG(info, #{ + msg => "starting_cassandra_connector", + connector => InstId, + config => emqx_misc:redact(Config) + }), + + Options = [ + {nodes, emqx_schema:parse_servers(Servers, ?DEFAULT_SERVER_OPTION)}, + {username, Username}, + {password, emqx_secret:wrap(maps:get(password, Config, ""))}, + {keyspace, Keyspace}, + {auto_reconnect, ?AUTO_RECONNECT_INTERVAL}, + {pool_size, PoolSize} + ], + + %% FIXME: how to set tls options + SslOpts = + case maps:get(enable, SSL) of + true -> + [ + %% note: type defined at ecql:option/0 + {ssl, emqx_tls_lib:to_client_opts(SSL)} + ]; + false -> + [] + end, + + PoolName = emqx_plugin_libs_pool:pool_name(InstId), + Prepares = parse_prepare_sql(Config), + InitState = #{poolname => PoolName, prepare_statement => #{}}, + State = maps:merge(InitState, Prepares), + case emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Options ++ SslOpts) of + ok -> + {ok, init_prepare(State)}; + {error, Reason} -> + ?tp( + cassandra_connector_start_failed, + #{error => Reason} + ), + {error, Reason} + end. + +on_stop(InstId, #{poolname := PoolName}) -> + ?SLOG(info, #{ + msg => "stopping_cassandra_connector", + connector => InstId + }), + emqx_plugin_libs_pool:stop_pool(PoolName). + +-type request() :: + % emqx_bridge.erl + {send_message, Params :: map()} + % common query + | {query, SQL :: binary()} + | {query, SQL :: binary(), Params :: map()}. + +-spec on_query( + emqx_resource:resource_id(), + request(), + state() +) -> emqx_resource:query_result(). +on_query( + InstId, + Request, + #{poolname := PoolName} = State +) -> + {Type, PreparedKeyOrSQL, Params} = parse_request_to_sql(Request), + ?tp( + debug, + cassandra_connector_received_sql_query, + #{ + connector => InstId, + type => Type, + params => Params, + prepared_key_or_sql => PreparedKeyOrSQL, + state => State + } + ), + {PreparedKeyOrSQL1, Data} = proc_sql_params(Type, PreparedKeyOrSQL, Params, State), + Res = exec_sql_query(InstId, PoolName, Type, PreparedKeyOrSQL1, Data), + handle_result(Res). + +parse_request_to_sql({send_message, Params}) -> + {prepared_query, _Key = send_message, Params}; +parse_request_to_sql({query, SQL}) -> + parse_request_to_sql({query, SQL, #{}}); +parse_request_to_sql({query, SQL, Params}) -> + {query, SQL, Params}. + +proc_sql_params( + prepared_query, + PreparedKey0, + Params, + #{prepare_statement := Prepares, params_tokens := ParamsTokens} +) -> + PreparedKey = maps:get(PreparedKey0, Prepares), + Tokens = maps:get(PreparedKey0, ParamsTokens), + {PreparedKey, assign_type_for_params(emqx_plugin_libs_rule:proc_sql(Tokens, Params))}; +proc_sql_params(query, SQL, Params, _State) -> + {SQL1, Tokens} = emqx_plugin_libs_rule:preproc_sql(SQL, '?'), + {SQL1, assign_type_for_params(emqx_plugin_libs_rule:proc_sql(Tokens, Params))}. + +exec_sql_query(InstId, PoolName, Type, PreparedKey, Data) when + Type == query; Type == prepared_query +-> + case ecpool:pick_and_do(PoolName, {?MODULE, Type, [PreparedKey, Data]}, no_handover) of + {error, Reason} = Result -> + ?tp( + error, + cassandra_connector_query_return, + #{connector => InstId, error => Reason} + ), + Result; + Result -> + ?tp(debug, cassandra_connector_query_return, #{result => Result}), + Result + end. + +on_get_status(_InstId, #{poolname := Pool} = State) -> + case emqx_plugin_libs_pool:health_check_ecpool_workers(Pool, fun ?MODULE:do_get_status/1) of + true -> + case do_check_prepares(State) of + ok -> + connected; + {ok, NState} -> + %% return new state with prepared statements + {connected, NState}; + false -> + %% do not log error, it is logged in prepare_sql_to_conn + connecting + end; + false -> + connecting + end. + +do_get_status(Conn) -> + ok == element(1, ecql:query(Conn, "SELECT count(1) AS T FROM system.local")). + +do_check_prepares(#{prepare_sql := Prepares}) when is_map(Prepares) -> + ok; +do_check_prepares(State = #{poolname := PoolName, prepare_sql := {error, Prepares}}) -> + %% retry to prepare + case prepare_sql(Prepares, PoolName) of + {ok, Sts} -> + %% remove the error + {ok, State#{prepare_sql => Prepares, prepare_statement := Sts}}; + _Error -> + false + end. + +%%-------------------------------------------------------------------- +%% callbacks query + +query(Conn, SQL, Params) -> + ecql:query(Conn, SQL, Params). + +prepared_query(Conn, PreparedKey, Params) -> + ecql:execute(Conn, PreparedKey, Params). + +%%-------------------------------------------------------------------- +%% callbacks for ecpool + +connect(Opts) -> + case ecql:connect(conn_opts(Opts)) of + {ok, _Conn} = Ok -> + Ok; + {error, Reason} -> + {error, Reason} + end. + +conn_opts(Opts) -> + conn_opts(Opts, []). + +conn_opts([], Acc) -> + Acc; +conn_opts([{password, Password} | Opts], Acc) -> + conn_opts(Opts, [{password, emqx_secret:unwrap(Password)} | Acc]); +conn_opts([Opt | Opts], Acc) -> + conn_opts(Opts, [Opt | Acc]). + +%%-------------------------------------------------------------------- +%% prepare + +%% XXX: hardcode +%% note: the `sql` param is passed by emqx_ee_bridge_cassa +parse_prepare_sql(#{sql := SQL}) -> + parse_prepare_sql([{send_message, SQL}], #{}, #{}); +parse_prepare_sql(_) -> + #{prepare_sql => #{}, params_tokens => #{}}. + +parse_prepare_sql([{Key, H} | T], Prepares, Tokens) -> + {PrepareSQL, ParamsTokens} = emqx_plugin_libs_rule:preproc_sql(H, '?'), + parse_prepare_sql( + T, Prepares#{Key => PrepareSQL}, Tokens#{Key => ParamsTokens} + ); +parse_prepare_sql([], Prepares, Tokens) -> + #{ + prepare_sql => Prepares, + params_tokens => Tokens + }. + +init_prepare(State = #{prepare_sql := Prepares, poolname := PoolName}) -> + case maps:size(Prepares) of + 0 -> + State; + _ -> + case prepare_sql(Prepares, PoolName) of + {ok, Sts} -> + State#{prepare_statement := Sts}; + Error -> + ?tp( + error, + cassandra_prepare_sql_failed, + #{prepares => Prepares, reason => Error} + ), + %% mark the prepare_sqlas failed + State#{prepare_sql => {error, Prepares}} + end + end. + +prepare_sql(Prepares, PoolName) when is_map(Prepares) -> + prepare_sql(maps:to_list(Prepares), PoolName); +prepare_sql(Prepares, PoolName) -> + case do_prepare_sql(Prepares, PoolName) of + {ok, _Sts} = Ok -> + %% prepare for reconnect + ecpool:add_reconnect_callback(PoolName, {?MODULE, prepare_sql_to_conn, [Prepares]}), + Ok; + Error -> + Error + end. + +do_prepare_sql(Prepares, PoolName) -> + do_prepare_sql(ecpool:workers(PoolName), Prepares, PoolName, #{}). + +do_prepare_sql([{_Name, Worker} | T], Prepares, PoolName, _LastSts) -> + {ok, Conn} = ecpool_worker:client(Worker), + case prepare_sql_to_conn(Conn, Prepares) of + {ok, Sts} -> + do_prepare_sql(T, Prepares, PoolName, Sts); + Error -> + Error + end; +do_prepare_sql([], _Prepares, _PoolName, LastSts) -> + {ok, LastSts}. + +prepare_sql_to_conn(Conn, Prepares) -> + prepare_sql_to_conn(Conn, Prepares, #{}). + +prepare_sql_to_conn(Conn, [], Statements) when is_pid(Conn) -> {ok, Statements}; +prepare_sql_to_conn(Conn, [{Key, SQL} | PrepareList], Statements) when is_pid(Conn) -> + ?SLOG(info, #{msg => "cassandra_prepare_sql", name => Key, prepare_sql => SQL}), + case ecql:prepare(Conn, Key, SQL) of + {ok, Statement} -> + prepare_sql_to_conn(Conn, PrepareList, Statements#{Key => Statement}); + {error, Error} = Other -> + ?SLOG(error, #{ + msg => "cassandra_prepare_sql_failed", + worker_pid => Conn, + name => Key, + prepare_sql => SQL, + error => Error + }), + Other + end. + +handle_result({error, disconnected}) -> + {error, {recoverable_error, disconnected}}; +handle_result({error, Error}) -> + {error, {unrecoverable_error, Error}}; +handle_result(Res) -> + Res. + +%%-------------------------------------------------------------------- +%% utils + +%% see ecql driver requirements +assign_type_for_params(Params) -> + assign_type_for_params(Params, []). + +assign_type_for_params([], Acc) -> + lists:reverse(Acc); +assign_type_for_params([Param | More], Acc) -> + assign_type_for_params(More, [may_assign_type(Param) | Acc]). + +may_assign_type(V) when is_boolean(V) -> + {int, + if + V -> 1; + true -> 0 + end}; +may_assign_type(V) when is_binary(V); is_list(V); is_atom(V) -> V; +may_assign_type(V) when is_integer(V) -> + %% The max value of signed int(4) is 2147483647 + case V > 2147483647 orelse V < -2147483647 of + true -> {bigint, V}; + false -> {int, V} + end; +may_assign_type(V) when is_float(V) -> {double, V}; +may_assign_type(V) -> + V. diff --git a/lib-ee/emqx_ee_connector/test/emqx_ee_connector_cassa_SUITE.erl b/lib-ee/emqx_ee_connector/test/emqx_ee_connector_cassa_SUITE.erl new file mode 100644 index 000000000..8a001b865 --- /dev/null +++ b/lib-ee/emqx_ee_connector/test/emqx_ee_connector_cassa_SUITE.erl @@ -0,0 +1,192 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ee_connector_cassa_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include("emqx_connector.hrl"). +-include("emqx_ee_connector.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("emqx/include/emqx.hrl"). +-include_lib("stdlib/include/assert.hrl"). + +-define(CASSANDRA_HOST, "127.0.0.1"). +-define(CASSANDRA_RESOURCE_MOD, emqx_ee_connector_cassa). + +%% This test SUITE requires a running cassandra instance. If you don't want to +%% bring up the whole CI infrastuctucture with the `scripts/ct/run.sh` script +%% you can create a cassandra instance with the following command (execute it +%% from root of the EMQX directory.). You also need to set ?CASSANDRA_HOST and +%% ?CASSANDRA_PORT to appropriate values. +%% +%% sudo docker run --rm -d --name cassandra --network host cassandra:3.11.14 + +all() -> + emqx_common_test_helpers:all(?MODULE). + +groups() -> + []. + +cassandra_servers() -> + emqx_schema:parse_servers( + iolist_to_binary([?CASSANDRA_HOST, ":", erlang:integer_to_list(?CASSANDRA_DEFAULT_PORT)]), + #{default_port => ?CASSANDRA_DEFAULT_PORT} + ). + +init_per_suite(Config) -> + case + emqx_common_test_helpers:is_tcp_server_available(?CASSANDRA_HOST, ?CASSANDRA_DEFAULT_PORT) + of + true -> + ok = emqx_common_test_helpers:start_apps([emqx_conf]), + ok = emqx_connector_test_helpers:start_apps([emqx_resource]), + {ok, _} = application:ensure_all_started(emqx_connector), + {ok, _} = application:ensure_all_started(emqx_ee_connector), + %% keyspace `mqtt` must be created in advance + {ok, Conn} = + ecql:connect([ + {nodes, cassandra_servers()}, + {username, <<"admin">>}, + {password, <<"public">>}, + {keyspace, "mqtt"} + ]), + ecql:close(Conn), + Config; + false -> + case os:getenv("IS_CI") of + "yes" -> + throw(no_cassandra); + _ -> + {skip, no_cassandra} + end + end. + +end_per_suite(_Config) -> + ok = emqx_common_test_helpers:stop_apps([emqx_conf]), + ok = emqx_connector_test_helpers:stop_apps([emqx_resource]), + _ = application:stop(emqx_connector). + +init_per_testcase(_, Config) -> + Config. + +end_per_testcase(_, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% cases +%%-------------------------------------------------------------------- + +t_lifecycle(_Config) -> + perform_lifecycle_check( + <<"emqx_connector_cassandra_SUITE">>, + cassandra_config() + ). + +show(X) -> + erlang:display(X), + X. + +show(Label, What) -> + erlang:display({Label, What}), + What. + +perform_lifecycle_check(PoolName, InitialConfig) -> + {ok, #{config := CheckedConfig}} = + emqx_resource:check_config(?CASSANDRA_RESOURCE_MOD, InitialConfig), + {ok, #{ + state := #{poolname := ReturnedPoolName} = State, + status := InitialStatus + }} = + emqx_resource:create_local( + PoolName, + ?CONNECTOR_RESOURCE_GROUP, + ?CASSANDRA_RESOURCE_MOD, + CheckedConfig, + #{} + ), + ?assertEqual(InitialStatus, connected), + % Instance should match the state and status of the just started resource + {ok, ?CONNECTOR_RESOURCE_GROUP, #{ + state := State, + status := InitialStatus + }} = + emqx_resource:get_instance(PoolName), + ?assertEqual({ok, connected}, emqx_resource:health_check(PoolName)), + % % Perform query as further check that the resource is working as expected + (fun() -> + erlang:display({pool_name, PoolName}), + QueryNoParamsResWrapper = emqx_resource:query(PoolName, test_query_no_params()), + ?assertMatch({ok, _}, QueryNoParamsResWrapper) + end)(), + ?assertEqual(ok, emqx_resource:stop(PoolName)), + % Resource will be listed still, but state will be changed and healthcheck will fail + % as the worker no longer exists. + {ok, ?CONNECTOR_RESOURCE_GROUP, #{ + state := State, + status := StoppedStatus + }} = + emqx_resource:get_instance(PoolName), + ?assertEqual(stopped, StoppedStatus), + ?assertEqual({error, resource_is_stopped}, emqx_resource:health_check(PoolName)), + % Resource healthcheck shortcuts things by checking ets. Go deeper by checking pool itself. + ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)), + % Can call stop/1 again on an already stopped instance + ?assertEqual(ok, emqx_resource:stop(PoolName)), + % Make sure it can be restarted and the healthchecks and queries work properly + ?assertEqual(ok, emqx_resource:restart(PoolName)), + % async restart, need to wait resource + timer:sleep(500), + {ok, ?CONNECTOR_RESOURCE_GROUP, #{status := InitialStatus}} = + emqx_resource:get_instance(PoolName), + ?assertEqual({ok, connected}, emqx_resource:health_check(PoolName)), + (fun() -> + QueryNoParamsResWrapper = + emqx_resource:query(PoolName, test_query_no_params()), + ?assertMatch({ok, _}, QueryNoParamsResWrapper) + end)(), + % Stop and remove the resource in one go. + ?assertEqual(ok, emqx_resource:remove_local(PoolName)), + ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)), + % Should not even be able to get the resource data out of ets now unlike just stopping. + ?assertEqual({error, not_found}, emqx_resource:get_instance(PoolName)). + +%%-------------------------------------------------------------------- +%% utils +%%-------------------------------------------------------------------- + +cassandra_config() -> + Config = + #{ + auto_reconnect => true, + keyspace => <<"mqtt">>, + username => <<"default">>, + password => <<"public">>, + pool_size => 8, + servers => iolist_to_binary( + io_lib:format( + "~s:~b", + [ + ?CASSANDRA_HOST, + ?CASSANDRA_DEFAULT_PORT + ] + ) + ) + }, + #{<<"config">> => Config}. + +test_query_no_params() -> + {query, <<"SELECT count(1) AS T FROM system.local">>}. diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index b3c424ea1..01c148c13 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -168,6 +168,9 @@ for dep in ${CT_DEPS}; do dynamo) FILES+=( '.ci/docker-compose-file/docker-compose-dynamo.yaml' ) ;; + cassandra) + FILES+=( '.ci/docker-compose-file/docker-compose-cassandra-tcp.yaml' ) + ;; *) echo "unknown_ct_dependency $dep" exit 1 From 75c1592e7ca4a114ed7d1b3773f1ad6a9d974c9d Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 17 Mar 2023 11:34:12 +0800 Subject: [PATCH 004/196] chore: make static_checks happy --- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl | 6 +++--- .../src/emqx_ee_connector_cassa.erl | 12 +++++------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl index 295d2c3b5..9df41c81e 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl @@ -25,7 +25,7 @@ desc/1 ]). --define(DEFAULT_SQL, << +-define(DEFAULT_CQL, << "insert into mqtt_msg(topic, msgid, sender, qos, payload, arrived, retain) " "values (${topic}, ${id}, ${clientid}, ${qos}, ${payload}, ${timestamp}, ${flags.retain})" >>). @@ -55,7 +55,7 @@ values(post, Type) -> pool_size => 8, username => <<"root">>, password => <<"public">>, - sql => ?DEFAULT_SQL, + sql => ?DEFAULT_CQL, local_topic => <<"local/topic/#">>, resource_opts => #{ worker_pool_size => 8, @@ -83,7 +83,7 @@ fields("config") -> {sql, mk( binary(), - #{desc => ?DESC("sql_template"), default => ?DEFAULT_SQL, format => <<"sql">>} + #{desc => ?DESC("sql_template"), default => ?DEFAULT_CQL, format => <<"sql">>} )}, {local_topic, mk( diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl index d1901e462..6b7084e9e 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl @@ -172,7 +172,7 @@ on_stop(InstId, #{poolname := PoolName}) -> emqx_resource:resource_id(), request(), state() -) -> emqx_resource:query_result(). +) -> ok | {ok, ecql:cql_result()} | {error, {recoverable_error | unrecoverable_error, term()}}. on_query( InstId, Request, @@ -397,12 +397,10 @@ assign_type_for_params([], Acc) -> assign_type_for_params([Param | More], Acc) -> assign_type_for_params(More, [may_assign_type(Param) | Acc]). -may_assign_type(V) when is_boolean(V) -> - {int, - if - V -> 1; - true -> 0 - end}; +may_assign_type(true) -> + {int, 1}; +may_assign_type(false) -> + {int, 0}; may_assign_type(V) when is_binary(V); is_list(V); is_atom(V) -> V; may_assign_type(V) when is_integer(V) -> %% The max value of signed int(4) is 2147483647 From d745e875d872a613f1713117a5daf90d90900415 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 17 Mar 2023 13:10:33 +0800 Subject: [PATCH 005/196] chore: update changes --- changes/ee/feat-10140.en.md | 4 ++++ changes/ee/feat-10140.zh.md | 3 +++ 2 files changed, 7 insertions(+) create mode 100644 changes/ee/feat-10140.en.md create mode 100644 changes/ee/feat-10140.zh.md diff --git a/changes/ee/feat-10140.en.md b/changes/ee/feat-10140.en.md new file mode 100644 index 000000000..f2616fda1 --- /dev/null +++ b/changes/ee/feat-10140.en.md @@ -0,0 +1,4 @@ +Integrate `Cassandra` into `bridges` as a new backend. At the current stage: +- Only support Cassandra version 3.x, not yet 4.x. +- Only support storing data in synchronously, not yet asynchronous and batch + method to store data to Cassandra. diff --git a/changes/ee/feat-10140.zh.md b/changes/ee/feat-10140.zh.md new file mode 100644 index 000000000..5b070133a --- /dev/null +++ b/changes/ee/feat-10140.zh.md @@ -0,0 +1,3 @@ +支持 Cassandra 数据桥接。在当前阶段： +- 仅支持 Cassandra 3.x 版本，暂不支持 4.x。 +- 仅支持以同步的方式存储数据，暂不支持异步和批量的方式来存储数据到 Cassandra。 From a20c39c83a92a738819c762660ee786ccc01317c Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 17 Mar 2023 13:55:26 +0800 Subject: [PATCH 006/196] test: correct CASSANDRA_HOST --- .../emqx_ee_connector/test/emqx_ee_connector_cassa_SUITE.erl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib-ee/emqx_ee_connector/test/emqx_ee_connector_cassa_SUITE.erl b/lib-ee/emqx_ee_connector/test/emqx_ee_connector_cassa_SUITE.erl index 8a001b865..2b4edabcc 100644 --- a/lib-ee/emqx_ee_connector/test/emqx_ee_connector_cassa_SUITE.erl +++ b/lib-ee/emqx_ee_connector/test/emqx_ee_connector_cassa_SUITE.erl @@ -24,7 +24,9 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("stdlib/include/assert.hrl"). --define(CASSANDRA_HOST, "127.0.0.1"). +%% Cassandra server defined at `.ci/docker-compose-file/docker-compose-cassandra-tcp.yaml` +%% You can change it to `127.0.0.1`, if you run this SUITE locally +-define(CASSANDRA_HOST, "cassa_tcp"). -define(CASSANDRA_RESOURCE_MOD, emqx_ee_connector_cassa). %% This test SUITE requires a running cassandra instance. If you don't want to From 5f0828a2ea476555757a9eafcd3f99665d319c04 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 17 Mar 2023 16:39:10 +0800 Subject: [PATCH 007/196] ci: add certs for cassandra tls --- .ci/docker-compose-file/.env | 1 + .../cassandra/Dockerfile-tls | 4 - .../cassandra/cassandra-tls.yaml | 1236 ----------------- .../cassandra/cassandra.yaml | 24 +- .ci/docker-compose-file/certs/README.md | 23 + .ci/docker-compose-file/certs/server.jks | Bin 0 -> 2898 bytes .ci/docker-compose-file/certs/server.p12 | Bin 0 -> 2708 bytes .ci/docker-compose-file/certs/truststore.jks | Bin 0 -> 1622 bytes .../docker-compose-cassandra-tcp.yaml | 5 +- .../docker-compose-toxiproxy.yaml | 1 + .ci/docker-compose-file/toxiproxy.json | 8 +- .../test/emqx_ee_bridge_cassa_SUITE.erl | 9 +- 12 files changed, 49 insertions(+), 1262 deletions(-) delete mode 100644 .ci/docker-compose-file/cassandra/Dockerfile-tls delete mode 100644 .ci/docker-compose-file/cassandra/cassandra-tls.yaml create mode 100644 .ci/docker-compose-file/certs/README.md create mode 100644 .ci/docker-compose-file/certs/server.jks create mode 100644 .ci/docker-compose-file/certs/server.p12 create mode 100644 .ci/docker-compose-file/certs/truststore.jks diff --git a/.ci/docker-compose-file/.env b/.ci/docker-compose-file/.env index 397c44854..956750e00 100644 --- a/.ci/docker-compose-file/.env +++ b/.ci/docker-compose-file/.env @@ -6,5 +6,6 @@ LDAP_TAG=2.4.50 INFLUXDB_TAG=2.5.0 TDENGINE_TAG=3.0.2.4 DYNAMO_TAG=1.21.0 +CASSANDRA_TAG=3.11.6 TARGET=emqx/emqx diff --git a/.ci/docker-compose-file/cassandra/Dockerfile-tls b/.ci/docker-compose-file/cassandra/Dockerfile-tls deleted file mode 100644 index 434584ca6..000000000 --- a/.ci/docker-compose-file/cassandra/Dockerfile-tls +++ /dev/null @@ -1,4 +0,0 @@ -ARG CASSANDRA_TAG=3.11.6 -FROM cassandra:${CASSANDRA_TAG} -COPY cassandra-tls.yaml /etc/cassandra/cassandra.yaml -CMD ["cassandra", "-f"] diff --git a/.ci/docker-compose-file/cassandra/cassandra-tls.yaml b/.ci/docker-compose-file/cassandra/cassandra-tls.yaml deleted file mode 100644 index d2d2a5d70..000000000 --- a/.ci/docker-compose-file/cassandra/cassandra-tls.yaml +++ /dev/null @@ -1,1236 +0,0 @@ -# Cassandra storage config YAML - -# NOTE: -# See http://wiki.apache.org/cassandra/StorageConfiguration for -# full explanations of configuration directives -# /NOTE - -# The name of the cluster. This is mainly used to prevent machines in -# one logical cluster from joining another. -cluster_name: 'Test Cluster' - -# This defines the number of tokens randomly assigned to this node on the ring -# The more tokens, relative to other nodes, the larger the proportion of data -# that this node will store. You probably want all nodes to have the same number -# of tokens assuming they have equal hardware capability. -# -# If you leave this unspecified, Cassandra will use the default of 1 token for legacy compatibility, -# and will use the initial_token as described below. -# -# Specifying initial_token will override this setting on the node's initial start, -# on subsequent starts, this setting will apply even if initial token is set. -# -# If you already have a cluster with 1 token per node, and wish to migrate to -# multiple tokens per node, see http://wiki.apache.org/cassandra/Operations -num_tokens: 256 - -# Triggers automatic allocation of num_tokens tokens for this node. The allocation -# algorithm attempts to choose tokens in a way that optimizes replicated load over -# the nodes in the datacenter for the replication strategy used by the specified -# keyspace. -# -# The load assigned to each node will be close to proportional to its number of -# vnodes. -# -# Only supported with the Murmur3Partitioner. -# allocate_tokens_for_keyspace: KEYSPACE - -# initial_token allows you to specify tokens manually. While you can use it with -# vnodes (num_tokens > 1, above) -- in which case you should provide a -# comma-separated list -- it's primarily used when adding nodes to legacy clusters -# that do not have vnodes enabled. -# initial_token: - -# See http://wiki.apache.org/cassandra/HintedHandoff -# May either be "true" or "false" to enable globally -hinted_handoff_enabled: true - -# When hinted_handoff_enabled is true, a black list of data centers that will not -# perform hinted handoff -# hinted_handoff_disabled_datacenters: -# - DC1 -# - DC2 - -# this defines the maximum amount of time a dead host will have hints -# generated. After it has been dead this long, new hints for it will not be -# created until it has been seen alive and gone down again. -max_hint_window_in_ms: 10800000 # 3 hours - -# Maximum throttle in KBs per second, per delivery thread. This will be -# reduced proportionally to the number of nodes in the cluster. (If there -# are two nodes in the cluster, each delivery thread will use the maximum -# rate; if there are three, each will throttle to half of the maximum, -# since we expect two nodes to be delivering hints simultaneously.) -hinted_handoff_throttle_in_kb: 1024 - -# Number of threads with which to deliver hints; -# Consider increasing this number when you have multi-dc deployments, since -# cross-dc handoff tends to be slower -max_hints_delivery_threads: 2 - -# Directory where Cassandra should store hints. -# If not set, the default directory is $CASSANDRA_HOME/data/hints. -# hints_directory: /var/lib/cassandra/hints - -# How often hints should be flushed from the internal buffers to disk. -# Will *not* trigger fsync. -hints_flush_period_in_ms: 10000 - -# Maximum size for a single hints file, in megabytes. -max_hints_file_size_in_mb: 128 - -# Compression to apply to the hint files. If omitted, hints files -# will be written uncompressed. LZ4, Snappy, and Deflate compressors -# are supported. -#hints_compression: -# - class_name: LZ4Compressor -# parameters: -# - - -# Maximum throttle in KBs per second, total. This will be -# reduced proportionally to the number of nodes in the cluster. -batchlog_replay_throttle_in_kb: 1024 - -# Authentication backend, implementing IAuthenticator; used to identify users -# Out of the box, Cassandra provides org.apache.cassandra.auth.{AllowAllAuthenticator, -# PasswordAuthenticator}. -# -# - AllowAllAuthenticator performs no checks - set it to disable authentication. -# - PasswordAuthenticator relies on username/password pairs to authenticate -# users. It keeps usernames and hashed passwords in system_auth.roles table. -# Please increase system_auth keyspace replication factor if you use this authenticator. -# If using PasswordAuthenticator, CassandraRoleManager must also be used (see below) -authenticator: PasswordAuthenticator - -# Authorization backend, implementing IAuthorizer; used to limit access/provide permissions -# Out of the box, Cassandra provides org.apache.cassandra.auth.{AllowAllAuthorizer, -# CassandraAuthorizer}. -# -# - AllowAllAuthorizer allows any action to any user - set it to disable authorization. -# - CassandraAuthorizer stores permissions in system_auth.role_permissions table. Please -# increase system_auth keyspace replication factor if you use this authorizer. -authorizer: CassandraAuthorizer - -# Part of the Authentication & Authorization backend, implementing IRoleManager; used -# to maintain grants and memberships between roles. -# Out of the box, Cassandra provides org.apache.cassandra.auth.CassandraRoleManager, -# which stores role information in the system_auth keyspace. Most functions of the -# IRoleManager require an authenticated login, so unless the configured IAuthenticator -# actually implements authentication, most of this functionality will be unavailable. -# -# - CassandraRoleManager stores role data in the system_auth keyspace. Please -# increase system_auth keyspace replication factor if you use this role manager. -role_manager: CassandraRoleManager - -# Validity period for roles cache (fetching granted roles can be an expensive -# operation depending on the role manager, CassandraRoleManager is one example) -# Granted roles are cached for authenticated sessions in AuthenticatedUser and -# after the period specified here, become eligible for (async) reload. -# Defaults to 2000, set to 0 to disable caching entirely. -# Will be disabled automatically for AllowAllAuthenticator. -roles_validity_in_ms: 2000 - -# Refresh interval for roles cache (if enabled). -# After this interval, cache entries become eligible for refresh. Upon next -# access, an async reload is scheduled and the old value returned until it -# completes. If roles_validity_in_ms is non-zero, then this must be -# also. -# Defaults to the same value as roles_validity_in_ms. -# roles_update_interval_in_ms: 2000 - -# Validity period for permissions cache (fetching permissions can be an -# expensive operation depending on the authorizer, CassandraAuthorizer is -# one example). Defaults to 2000, set to 0 to disable. -# Will be disabled automatically for AllowAllAuthorizer. -permissions_validity_in_ms: 2000 - -# Refresh interval for permissions cache (if enabled). -# After this interval, cache entries become eligible for refresh. Upon next -# access, an async reload is scheduled and the old value returned until it -# completes. If permissions_validity_in_ms is non-zero, then this must be -# also. -# Defaults to the same value as permissions_validity_in_ms. -# permissions_update_interval_in_ms: 2000 - -# Validity period for credentials cache. This cache is tightly coupled to -# the provided PasswordAuthenticator implementation of IAuthenticator. If -# another IAuthenticator implementation is configured, this cache will not -# be automatically used and so the following settings will have no effect. -# Please note, credentials are cached in their encrypted form, so while -# activating this cache may reduce the number of queries made to the -# underlying table, it may not bring a significant reduction in the -# latency of individual authentication attempts. -# Defaults to 2000, set to 0 to disable credentials caching. -credentials_validity_in_ms: 2000 - -# Refresh interval for credentials cache (if enabled). -# After this interval, cache entries become eligible for refresh. Upon next -# access, an async reload is scheduled and the old value returned until it -# completes. If credentials_validity_in_ms is non-zero, then this must be -# also. -# Defaults to the same value as credentials_validity_in_ms. -# credentials_update_interval_in_ms: 2000 - -# The partitioner is responsible for distributing groups of rows (by -# partition key) across nodes in the cluster. You should leave this -# alone for new clusters. The partitioner can NOT be changed without -# reloading all data, so when upgrading you should set this to the -# same partitioner you were already using. -# -# Besides Murmur3Partitioner, partitioners included for backwards -# compatibility include RandomPartitioner, ByteOrderedPartitioner, and -# OrderPreservingPartitioner. -# -partitioner: org.apache.cassandra.dht.Murmur3Partitioner - -# Directories where Cassandra should store data on disk. Cassandra -# will spread data evenly across them, subject to the granularity of -# the configured compaction strategy. -# If not set, the default directory is $CASSANDRA_HOME/data/data. -data_file_directories: - - /var/lib/cassandra/data - -# commit log. when running on magnetic HDD, this should be a -# separate spindle than the data directories. -# If not set, the default directory is $CASSANDRA_HOME/data/commitlog. -commitlog_directory: /var/lib/cassandra/commitlog - -# Enable / disable CDC functionality on a per-node basis. This modifies the logic used -# for write path allocation rejection (standard: never reject. cdc: reject Mutation -# containing a CDC-enabled table if at space limit in cdc_raw_directory). -cdc_enabled: false - -# CommitLogSegments are moved to this directory on flush if cdc_enabled: true and the -# segment contains mutations for a CDC-enabled table. This should be placed on a -# separate spindle than the data directories. If not set, the default directory is -# $CASSANDRA_HOME/data/cdc_raw. -# cdc_raw_directory: /var/lib/cassandra/cdc_raw - -# Policy for data disk failures: -# -# die -# shut down gossip and client transports and kill the JVM for any fs errors or -# single-sstable errors, so the node can be replaced. -# -# stop_paranoid -# shut down gossip and client transports even for single-sstable errors, -# kill the JVM for errors during startup. -# -# stop -# shut down gossip and client transports, leaving the node effectively dead, but -# can still be inspected via JMX, kill the JVM for errors during startup. -# -# best_effort -# stop using the failed disk and respond to requests based on -# remaining available sstables. This means you WILL see obsolete -# data at CL.ONE! -# -# ignore -# ignore fatal errors and let requests fail, as in pre-1.2 Cassandra -disk_failure_policy: stop - -# Policy for commit disk failures: -# -# die -# shut down gossip and Thrift and kill the JVM, so the node can be replaced. -# -# stop -# shut down gossip and Thrift, leaving the node effectively dead, but -# can still be inspected via JMX. -# -# stop_commit -# shutdown the commit log, letting writes collect but -# continuing to service reads, as in pre-2.0.5 Cassandra -# -# ignore -# ignore fatal errors and let the batches fail -commit_failure_policy: stop - -# Maximum size of the native protocol prepared statement cache -# -# Valid values are either "auto" (omitting the value) or a value greater 0. -# -# Note that specifying a too large value will result in long running GCs and possbily -# out-of-memory errors. Keep the value at a small fraction of the heap. -# -# If you constantly see "prepared statements discarded in the last minute because -# cache limit reached" messages, the first step is to investigate the root cause -# of these messages and check whether prepared statements are used correctly - -# i.e. use bind markers for variable parts. -# -# Do only change the default value, if you really have more prepared statements than -# fit in the cache. In most cases it is not neccessary to change this value. -# Constantly re-preparing statements is a performance penalty. -# -# Default value ("auto") is 1/256th of the heap or 10MB, whichever is greater -prepared_statements_cache_size_mb: - -# Maximum size of the Thrift prepared statement cache -# -# If you do not use Thrift at all, it is safe to leave this value at "auto". -# -# See description of 'prepared_statements_cache_size_mb' above for more information. -# -# Default value ("auto") is 1/256th of the heap or 10MB, whichever is greater -thrift_prepared_statements_cache_size_mb: - -# Maximum size of the key cache in memory. -# -# Each key cache hit saves 1 seek and each row cache hit saves 2 seeks at the -# minimum, sometimes more. The key cache is fairly tiny for the amount of -# time it saves, so it's worthwhile to use it at large numbers. -# The row cache saves even more time, but must contain the entire row, -# so it is extremely space-intensive. It's best to only use the -# row cache if you have hot rows or static rows. -# -# NOTE: if you reduce the size, you may not get you hottest keys loaded on startup. -# -# Default value is empty to make it "auto" (min(5% of Heap (in MB), 100MB)). Set to 0 to disable key cache. -key_cache_size_in_mb: - -# Duration in seconds after which Cassandra should -# save the key cache. Caches are saved to saved_caches_directory as -# specified in this configuration file. -# -# Saved caches greatly improve cold-start speeds, and is relatively cheap in -# terms of I/O for the key cache. Row cache saving is much more expensive and -# has limited use. -# -# Default is 14400 or 4 hours. -key_cache_save_period: 14400 - -# Number of keys from the key cache to save -# Disabled by default, meaning all keys are going to be saved -# key_cache_keys_to_save: 100 - -# Row cache implementation class name. Available implementations: -# -# org.apache.cassandra.cache.OHCProvider -# Fully off-heap row cache implementation (default). -# -# org.apache.cassandra.cache.SerializingCacheProvider -# This is the row cache implementation availabile -# in previous releases of Cassandra. -# row_cache_class_name: org.apache.cassandra.cache.OHCProvider - -# Maximum size of the row cache in memory. -# Please note that OHC cache implementation requires some additional off-heap memory to manage -# the map structures and some in-flight memory during operations before/after cache entries can be -# accounted against the cache capacity. This overhead is usually small compared to the whole capacity. -# Do not specify more memory that the system can afford in the worst usual situation and leave some -# headroom for OS block level cache. Do never allow your system to swap. -# -# Default value is 0, to disable row caching. -row_cache_size_in_mb: 0 - -# Duration in seconds after which Cassandra should save the row cache. -# Caches are saved to saved_caches_directory as specified in this configuration file. -# -# Saved caches greatly improve cold-start speeds, and is relatively cheap in -# terms of I/O for the key cache. Row cache saving is much more expensive and -# has limited use. -# -# Default is 0 to disable saving the row cache. -row_cache_save_period: 0 - -# Number of keys from the row cache to save. -# Specify 0 (which is the default), meaning all keys are going to be saved -# row_cache_keys_to_save: 100 - -# Maximum size of the counter cache in memory. -# -# Counter cache helps to reduce counter locks' contention for hot counter cells. -# In case of RF = 1 a counter cache hit will cause Cassandra to skip the read before -# write entirely. With RF > 1 a counter cache hit will still help to reduce the duration -# of the lock hold, helping with hot counter cell updates, but will not allow skipping -# the read entirely. Only the local (clock, count) tuple of a counter cell is kept -# in memory, not the whole counter, so it's relatively cheap. -# -# NOTE: if you reduce the size, you may not get you hottest keys loaded on startup. -# -# Default value is empty to make it "auto" (min(2.5% of Heap (in MB), 50MB)). Set to 0 to disable counter cache. -# NOTE: if you perform counter deletes and rely on low gcgs, you should disable the counter cache. -counter_cache_size_in_mb: - -# Duration in seconds after which Cassandra should -# save the counter cache (keys only). Caches are saved to saved_caches_directory as -# specified in this configuration file. -# -# Default is 7200 or 2 hours. -counter_cache_save_period: 7200 - -# Number of keys from the counter cache to save -# Disabled by default, meaning all keys are going to be saved -# counter_cache_keys_to_save: 100 - -# saved caches -# If not set, the default directory is $CASSANDRA_HOME/data/saved_caches. -saved_caches_directory: /var/lib/cassandra/saved_caches - -# commitlog_sync may be either "periodic" or "batch." -# -# When in batch mode, Cassandra won't ack writes until the commit log -# has been fsynced to disk. It will wait -# commitlog_sync_batch_window_in_ms milliseconds between fsyncs. -# This window should be kept short because the writer threads will -# be unable to do extra work while waiting. (You may need to increase -# concurrent_writes for the same reason.) -# -# commitlog_sync: batch -# commitlog_sync_batch_window_in_ms: 2 -# -# the other option is "periodic" where writes may be acked immediately -# and the CommitLog is simply synced every commitlog_sync_period_in_ms -# milliseconds. -commitlog_sync: periodic -commitlog_sync_period_in_ms: 10000 - -# The size of the individual commitlog file segments. A commitlog -# segment may be archived, deleted, or recycled once all the data -# in it (potentially from each columnfamily in the system) has been -# flushed to sstables. -# -# The default size is 32, which is almost always fine, but if you are -# archiving commitlog segments (see commitlog_archiving.properties), -# then you probably want a finer granularity of archiving; 8 or 16 MB -# is reasonable. -# Max mutation size is also configurable via max_mutation_size_in_kb setting in -# cassandra.yaml. The default is half the size commitlog_segment_size_in_mb * 1024. -# This should be positive and less than 2048. -# -# NOTE: If max_mutation_size_in_kb is set explicitly then commitlog_segment_size_in_mb must -# be set to at least twice the size of max_mutation_size_in_kb / 1024 -# -commitlog_segment_size_in_mb: 32 - -# Compression to apply to the commit log. If omitted, the commit log -# will be written uncompressed. LZ4, Snappy, and Deflate compressors -# are supported. -# commitlog_compression: -# - class_name: LZ4Compressor -# parameters: -# - - -# any class that implements the SeedProvider interface and has a -# constructor that takes a Map of parameters will do. -seed_provider: - # Addresses of hosts that are deemed contact points. - # Cassandra nodes use this list of hosts to find each other and learn - # the topology of the ring. You must change this if you are running - # multiple nodes! - - class_name: org.apache.cassandra.locator.SimpleSeedProvider - parameters: - # seeds is actually a comma-delimited list of addresses. - # Ex: ",," - - seeds: "127.0.0.1" - -# For workloads with more data than can fit in memory, Cassandra's -# bottleneck will be reads that need to fetch data from -# disk. "concurrent_reads" should be set to (16 * number_of_drives) in -# order to allow the operations to enqueue low enough in the stack -# that the OS and drives can reorder them. Same applies to -# "concurrent_counter_writes", since counter writes read the current -# values before incrementing and writing them back. -# -# On the other hand, since writes are almost never IO bound, the ideal -# number of "concurrent_writes" is dependent on the number of cores in -# your system; (8 * number_of_cores) is a good rule of thumb. -concurrent_reads: 32 -concurrent_writes: 32 -concurrent_counter_writes: 32 - -# For materialized view writes, as there is a read involved, so this should -# be limited by the less of concurrent reads or concurrent writes. -concurrent_materialized_view_writes: 32 - -# Maximum memory to use for sstable chunk cache and buffer pooling. -# 32MB of this are reserved for pooling buffers, the rest is used as an -# cache that holds uncompressed sstable chunks. -# Defaults to the smaller of 1/4 of heap or 512MB. This pool is allocated off-heap, -# so is in addition to the memory allocated for heap. The cache also has on-heap -# overhead which is roughly 128 bytes per chunk (i.e. 0.2% of the reserved size -# if the default 64k chunk size is used). -# Memory is only allocated when needed. -# file_cache_size_in_mb: 512 - -# Flag indicating whether to allocate on or off heap when the sstable buffer -# pool is exhausted, that is when it has exceeded the maximum memory -# file_cache_size_in_mb, beyond which it will not cache buffers but allocate on request. - -# buffer_pool_use_heap_if_exhausted: true - -# The strategy for optimizing disk read -# Possible values are: -# ssd (for solid state disks, the default) -# spinning (for spinning disks) -# disk_optimization_strategy: ssd - -# Total permitted memory to use for memtables. Cassandra will stop -# accepting writes when the limit is exceeded until a flush completes, -# and will trigger a flush based on memtable_cleanup_threshold -# If omitted, Cassandra will set both to 1/4 the size of the heap. -# memtable_heap_space_in_mb: 2048 -# memtable_offheap_space_in_mb: 2048 - -# memtable_cleanup_threshold is deprecated. The default calculation -# is the only reasonable choice. See the comments on memtable_flush_writers -# for more information. -# -# Ratio of occupied non-flushing memtable size to total permitted size -# that will trigger a flush of the largest memtable. Larger mct will -# mean larger flushes and hence less compaction, but also less concurrent -# flush activity which can make it difficult to keep your disks fed -# under heavy write load. -# -# memtable_cleanup_threshold defaults to 1 / (memtable_flush_writers + 1) -# memtable_cleanup_threshold: 0.11 - -# Specify the way Cassandra allocates and manages memtable memory. -# Options are: -# -# heap_buffers -# on heap nio buffers -# -# offheap_buffers -# off heap (direct) nio buffers -# -# offheap_objects -# off heap objects -memtable_allocation_type: heap_buffers - -# Total space to use for commit logs on disk. -# -# If space gets above this value, Cassandra will flush every dirty CF -# in the oldest segment and remove it. So a small total commitlog space -# will tend to cause more flush activity on less-active columnfamilies. -# -# The default value is the smaller of 8192, and 1/4 of the total space -# of the commitlog volume. -# -# commitlog_total_space_in_mb: 8192 - -# This sets the number of memtable flush writer threads per disk -# as well as the total number of memtables that can be flushed concurrently. -# These are generally a combination of compute and IO bound. -# -# Memtable flushing is more CPU efficient than memtable ingest and a single thread -# can keep up with the ingest rate of a whole server on a single fast disk -# until it temporarily becomes IO bound under contention typically with compaction. -# At that point you need multiple flush threads. At some point in the future -# it may become CPU bound all the time. -# -# You can tell if flushing is falling behind using the MemtablePool.BlockedOnAllocation -# metric which should be 0, but will be non-zero if threads are blocked waiting on flushing -# to free memory. -# -# memtable_flush_writers defaults to two for a single data directory. -# This means that two memtables can be flushed concurrently to the single data directory. -# If you have multiple data directories the default is one memtable flushing at a time -# but the flush will use a thread per data directory so you will get two or more writers. -# -# Two is generally enough to flush on a fast disk [array] mounted as a single data directory. -# Adding more flush writers will result in smaller more frequent flushes that introduce more -# compaction overhead. -# -# There is a direct tradeoff between number of memtables that can be flushed concurrently -# and flush size and frequency. More is not better you just need enough flush writers -# to never stall waiting for flushing to free memory. -# -#memtable_flush_writers: 2 - -# Total space to use for change-data-capture logs on disk. -# -# If space gets above this value, Cassandra will throw WriteTimeoutException -# on Mutations including tables with CDC enabled. A CDCCompactor is responsible -# for parsing the raw CDC logs and deleting them when parsing is completed. -# -# The default value is the min of 4096 mb and 1/8th of the total space -# of the drive where cdc_raw_directory resides. -# cdc_total_space_in_mb: 4096 - -# When we hit our cdc_raw limit and the CDCCompactor is either running behind -# or experiencing backpressure, we check at the following interval to see if any -# new space for cdc-tracked tables has been made available. Default to 250ms -# cdc_free_space_check_interval_ms: 250 - -# A fixed memory pool size in MB for for SSTable index summaries. If left -# empty, this will default to 5% of the heap size. If the memory usage of -# all index summaries exceeds this limit, SSTables with low read rates will -# shrink their index summaries in order to meet this limit. However, this -# is a best-effort process. In extreme conditions Cassandra may need to use -# more than this amount of memory. -index_summary_capacity_in_mb: - -# How frequently index summaries should be resampled. This is done -# periodically to redistribute memory from the fixed-size pool to sstables -# proportional their recent read rates. Setting to -1 will disable this -# process, leaving existing index summaries at their current sampling level. -index_summary_resize_interval_in_minutes: 60 - -# Whether to, when doing sequential writing, fsync() at intervals in -# order to force the operating system to flush the dirty -# buffers. Enable this to avoid sudden dirty buffer flushing from -# impacting read latencies. Almost always a good idea on SSDs; not -# necessarily on platters. -trickle_fsync: false -trickle_fsync_interval_in_kb: 10240 - -# TCP port, for commands and data -# For security reasons, you should not expose this port to the internet. Firewall it if needed. -storage_port: 7000 - -# SSL port, for encrypted communication. Unused unless enabled in -# encryption_options -# For security reasons, you should not expose this port to the internet. Firewall it if needed. -ssl_storage_port: 7001 - -# Address or interface to bind to and tell other Cassandra nodes to connect to. -# You _must_ change this if you want multiple nodes to be able to communicate! -# -# Set listen_address OR listen_interface, not both. -# -# Leaving it blank leaves it up to InetAddress.getLocalHost(). This -# will always do the Right Thing _if_ the node is properly configured -# (hostname, name resolution, etc), and the Right Thing is to use the -# address associated with the hostname (it might not be). -# -# Setting listen_address to 0.0.0.0 is always wrong. -# -listen_address: localhost - -# Set listen_address OR listen_interface, not both. Interfaces must correspond -# to a single address, IP aliasing is not supported. -# listen_interface: eth0 - -# If you choose to specify the interface by name and the interface has an ipv4 and an ipv6 address -# you can specify which should be chosen using listen_interface_prefer_ipv6. If false the first ipv4 -# address will be used. If true the first ipv6 address will be used. Defaults to false preferring -# ipv4. If there is only one address it will be selected regardless of ipv4/ipv6. -# listen_interface_prefer_ipv6: false - -# Address to broadcast to other Cassandra nodes -# Leaving this blank will set it to the same value as listen_address -# broadcast_address: 1.2.3.4 - -# When using multiple physical network interfaces, set this -# to true to listen on broadcast_address in addition to -# the listen_address, allowing nodes to communicate in both -# interfaces. -# Ignore this property if the network configuration automatically -# routes between the public and private networks such as EC2. -# listen_on_broadcast_address: false - -# Internode authentication backend, implementing IInternodeAuthenticator; -# used to allow/disallow connections from peer nodes. -# internode_authenticator: org.apache.cassandra.auth.AllowAllInternodeAuthenticator - -# Whether to start the native transport server. -# Please note that the address on which the native transport is bound is the -# same as the rpc_address. The port however is different and specified below. -start_native_transport: true -# port for the CQL native transport to listen for clients on -# For security reasons, you should not expose this port to the internet. Firewall it if needed. -native_transport_port: 9042 -# Enabling native transport encryption in client_encryption_options allows you to either use -# encryption for the standard port or to use a dedicated, additional port along with the unencrypted -# standard native_transport_port. -# Enabling client encryption and keeping native_transport_port_ssl disabled will use encryption -# for native_transport_port. Setting native_transport_port_ssl to a different value -# from native_transport_port will use encryption for native_transport_port_ssl while -# keeping native_transport_port unencrypted. -native_transport_port_ssl: 9142 -# The maximum threads for handling requests when the native transport is used. -# This is similar to rpc_max_threads though the default differs slightly (and -# there is no native_transport_min_threads, idle threads will always be stopped -# after 30 seconds). -# native_transport_max_threads: 128 -# -# The maximum size of allowed frame. Frame (requests) larger than this will -# be rejected as invalid. The default is 256MB. If you're changing this parameter, -# you may want to adjust max_value_size_in_mb accordingly. This should be positive and less than 2048. -# native_transport_max_frame_size_in_mb: 256 - -# The maximum number of concurrent client connections. -# The default is -1, which means unlimited. -# native_transport_max_concurrent_connections: -1 - -# The maximum number of concurrent client connections per source ip. -# The default is -1, which means unlimited. -# native_transport_max_concurrent_connections_per_ip: -1 - -# Whether to start the thrift rpc server. -start_rpc: true - -# The address or interface to bind the Thrift RPC service and native transport -# server to. -# -# Set rpc_address OR rpc_interface, not both. -# -# Leaving rpc_address blank has the same effect as on listen_address -# (i.e. it will be based on the configured hostname of the node). -# -# Note that unlike listen_address, you can specify 0.0.0.0, but you must also -# set broadcast_rpc_address to a value other than 0.0.0.0. -# -# For security reasons, you should not expose this port to the internet. Firewall it if needed. -rpc_address: 0.0.0.0 - -# Set rpc_address OR rpc_interface, not both. Interfaces must correspond -# to a single address, IP aliasing is not supported. -# rpc_interface: eth1 - -# If you choose to specify the interface by name and the interface has an ipv4 and an ipv6 address -# you can specify which should be chosen using rpc_interface_prefer_ipv6. If false the first ipv4 -# address will be used. If true the first ipv6 address will be used. Defaults to false preferring -# ipv4. If there is only one address it will be selected regardless of ipv4/ipv6. -# rpc_interface_prefer_ipv6: false - -# port for Thrift to listen for clients on -rpc_port: 9160 - -# RPC address to broadcast to drivers and other Cassandra nodes. This cannot -# be set to 0.0.0.0. If left blank, this will be set to the value of -# rpc_address. If rpc_address is set to 0.0.0.0, broadcast_rpc_address must -# be set. -broadcast_rpc_address: 1.2.3.4 - -# enable or disable keepalive on rpc/native connections -rpc_keepalive: true - -# Cassandra provides two out-of-the-box options for the RPC Server: -# -# sync -# One thread per thrift connection. For a very large number of clients, memory -# will be your limiting factor. On a 64 bit JVM, 180KB is the minimum stack size -# per thread, and that will correspond to your use of virtual memory (but physical memory -# may be limited depending on use of stack space). -# -# hsha -# Stands for "half synchronous, half asynchronous." All thrift clients are handled -# asynchronously using a small number of threads that does not vary with the amount -# of thrift clients (and thus scales well to many clients). The rpc requests are still -# synchronous (one thread per active request). If hsha is selected then it is essential -# that rpc_max_threads is changed from the default value of unlimited. -# -# The default is sync because on Windows hsha is about 30% slower. On Linux, -# sync/hsha performance is about the same, with hsha of course using less memory. -# -# Alternatively, can provide your own RPC server by providing the fully-qualified class name -# of an o.a.c.t.TServerFactory that can create an instance of it. -rpc_server_type: sync - -# Uncomment rpc_min|max_thread to set request pool size limits. -# -# Regardless of your choice of RPC server (see above), the number of maximum requests in the -# RPC thread pool dictates how many concurrent requests are possible (but if you are using the sync -# RPC server, it also dictates the number of clients that can be connected at all). -# -# The default is unlimited and thus provides no protection against clients overwhelming the server. You are -# encouraged to set a maximum that makes sense for you in production, but do keep in mind that -# rpc_max_threads represents the maximum number of client requests this server may execute concurrently. -# -# rpc_min_threads: 16 -# rpc_max_threads: 2048 - -# uncomment to set socket buffer sizes on rpc connections -# rpc_send_buff_size_in_bytes: -# rpc_recv_buff_size_in_bytes: - -# Uncomment to set socket buffer size for internode communication -# Note that when setting this, the buffer size is limited by net.core.wmem_max -# and when not setting it it is defined by net.ipv4.tcp_wmem -# See also: -# /proc/sys/net/core/wmem_max -# /proc/sys/net/core/rmem_max -# /proc/sys/net/ipv4/tcp_wmem -# /proc/sys/net/ipv4/tcp_wmem -# and 'man tcp' -# internode_send_buff_size_in_bytes: - -# Uncomment to set socket buffer size for internode communication -# Note that when setting this, the buffer size is limited by net.core.wmem_max -# and when not setting it it is defined by net.ipv4.tcp_wmem -# internode_recv_buff_size_in_bytes: - -# Frame size for thrift (maximum message length). -thrift_framed_transport_size_in_mb: 15 - -# Set to true to have Cassandra create a hard link to each sstable -# flushed or streamed locally in a backups/ subdirectory of the -# keyspace data. Removing these links is the operator's -# responsibility. -incremental_backups: false - -# Whether or not to take a snapshot before each compaction. Be -# careful using this option, since Cassandra won't clean up the -# snapshots for you. Mostly useful if you're paranoid when there -# is a data format change. -snapshot_before_compaction: false - -# Whether or not a snapshot is taken of the data before keyspace truncation -# or dropping of column families. The STRONGLY advised default of true -# should be used to provide data safety. If you set this flag to false, you will -# lose data on truncation or drop. -auto_snapshot: true - -# Granularity of the collation index of rows within a partition. -# Increase if your rows are large, or if you have a very large -# number of rows per partition. The competing goals are these: -# -# - a smaller granularity means more index entries are generated -# and looking up rows withing the partition by collation column -# is faster -# - but, Cassandra will keep the collation index in memory for hot -# rows (as part of the key cache), so a larger granularity means -# you can cache more hot rows -column_index_size_in_kb: 64 - -# Per sstable indexed key cache entries (the collation index in memory -# mentioned above) exceeding this size will not be held on heap. -# This means that only partition information is held on heap and the -# index entries are read from disk. -# -# Note that this size refers to the size of the -# serialized index information and not the size of the partition. -column_index_cache_size_in_kb: 2 - -# Number of simultaneous compactions to allow, NOT including -# validation "compactions" for anti-entropy repair. Simultaneous -# compactions can help preserve read performance in a mixed read/write -# workload, by mitigating the tendency of small sstables to accumulate -# during a single long running compactions. The default is usually -# fine and if you experience problems with compaction running too -# slowly or too fast, you should look at -# compaction_throughput_mb_per_sec first. -# -# concurrent_compactors defaults to the smaller of (number of disks, -# number of cores), with a minimum of 2 and a maximum of 8. -# -# If your data directories are backed by SSD, you should increase this -# to the number of cores. -#concurrent_compactors: 1 - -# Throttles compaction to the given total throughput across the entire -# system. The faster you insert data, the faster you need to compact in -# order to keep the sstable count down, but in general, setting this to -# 16 to 32 times the rate you are inserting data is more than sufficient. -# Setting this to 0 disables throttling. Note that this account for all types -# of compaction, including validation compaction. -compaction_throughput_mb_per_sec: 16 - -# When compacting, the replacement sstable(s) can be opened before they -# are completely written, and used in place of the prior sstables for -# any range that has been written. This helps to smoothly transfer reads -# between the sstables, reducing page cache churn and keeping hot rows hot -sstable_preemptive_open_interval_in_mb: 50 - -# Throttles all outbound streaming file transfers on this node to the -# given total throughput in Mbps. This is necessary because Cassandra does -# mostly sequential IO when streaming data during bootstrap or repair, which -# can lead to saturating the network connection and degrading rpc performance. -# When unset, the default is 200 Mbps or 25 MB/s. -# stream_throughput_outbound_megabits_per_sec: 200 - -# Throttles all streaming file transfer between the datacenters, -# this setting allows users to throttle inter dc stream throughput in addition -# to throttling all network stream traffic as configured with -# stream_throughput_outbound_megabits_per_sec -# When unset, the default is 200 Mbps or 25 MB/s -# inter_dc_stream_throughput_outbound_megabits_per_sec: 200 - -# How long the coordinator should wait for read operations to complete -read_request_timeout_in_ms: 5000 -# How long the coordinator should wait for seq or index scans to complete -range_request_timeout_in_ms: 10000 -# How long the coordinator should wait for writes to complete -write_request_timeout_in_ms: 2000 -# How long the coordinator should wait for counter writes to complete -counter_write_request_timeout_in_ms: 5000 -# How long a coordinator should continue to retry a CAS operation -# that contends with other proposals for the same row -cas_contention_timeout_in_ms: 1000 -# How long the coordinator should wait for truncates to complete -# (This can be much longer, because unless auto_snapshot is disabled -# we need to flush first so we can snapshot before removing the data.) -truncate_request_timeout_in_ms: 60000 -# The default timeout for other, miscellaneous operations -request_timeout_in_ms: 10000 - -# How long before a node logs slow queries. Select queries that take longer than -# this timeout to execute, will generate an aggregated log message, so that slow queries -# can be identified. Set this value to zero to disable slow query logging. -slow_query_log_timeout_in_ms: 500 - -# Enable operation timeout information exchange between nodes to accurately -# measure request timeouts. If disabled, replicas will assume that requests -# were forwarded to them instantly by the coordinator, which means that -# under overload conditions we will waste that much extra time processing -# already-timed-out requests. -# -# Warning: before enabling this property make sure to ntp is installed -# and the times are synchronized between the nodes. -cross_node_timeout: false - -# Set keep-alive period for streaming -# This node will send a keep-alive message periodically with this period. -# If the node does not receive a keep-alive message from the peer for -# 2 keep-alive cycles the stream session times out and fail -# Default value is 300s (5 minutes), which means stalled stream -# times out in 10 minutes by default -# streaming_keep_alive_period_in_secs: 300 - -# phi value that must be reached for a host to be marked down. -# most users should never need to adjust this. -# phi_convict_threshold: 8 - -# endpoint_snitch -- Set this to a class that implements -# IEndpointSnitch. The snitch has two functions: -# -# - it teaches Cassandra enough about your network topology to route -# requests efficiently -# - it allows Cassandra to spread replicas around your cluster to avoid -# correlated failures. It does this by grouping machines into -# "datacenters" and "racks." Cassandra will do its best not to have -# more than one replica on the same "rack" (which may not actually -# be a physical location) -# -# CASSANDRA WILL NOT ALLOW YOU TO SWITCH TO AN INCOMPATIBLE SNITCH -# ONCE DATA IS INSERTED INTO THE CLUSTER. This would cause data loss. -# This means that if you start with the default SimpleSnitch, which -# locates every node on "rack1" in "datacenter1", your only options -# if you need to add another datacenter are GossipingPropertyFileSnitch -# (and the older PFS). From there, if you want to migrate to an -# incompatible snitch like Ec2Snitch you can do it by adding new nodes -# under Ec2Snitch (which will locate them in a new "datacenter") and -# decommissioning the old ones. -# -# Out of the box, Cassandra provides: -# -# SimpleSnitch: -# Treats Strategy order as proximity. This can improve cache -# locality when disabling read repair. Only appropriate for -# single-datacenter deployments. -# -# GossipingPropertyFileSnitch -# This should be your go-to snitch for production use. The rack -# and datacenter for the local node are defined in -# cassandra-rackdc.properties and propagated to other nodes via -# gossip. If cassandra-topology.properties exists, it is used as a -# fallback, allowing migration from the PropertyFileSnitch. -# -# PropertyFileSnitch: -# Proximity is determined by rack and data center, which are -# explicitly configured in cassandra-topology.properties. -# -# Ec2Snitch: -# Appropriate for EC2 deployments in a single Region. Loads Region -# and Availability Zone information from the EC2 API. The Region is -# treated as the datacenter, and the Availability Zone as the rack. -# Only private IPs are used, so this will not work across multiple -# Regions. -# -# Ec2MultiRegionSnitch: -# Uses public IPs as broadcast_address to allow cross-region -# connectivity. (Thus, you should set seed addresses to the public -# IP as well.) You will need to open the storage_port or -# ssl_storage_port on the public IP firewall. (For intra-Region -# traffic, Cassandra will switch to the private IP after -# establishing a connection.) -# -# RackInferringSnitch: -# Proximity is determined by rack and data center, which are -# assumed to correspond to the 3rd and 2nd octet of each node's IP -# address, respectively. Unless this happens to match your -# deployment conventions, this is best used as an example of -# writing a custom Snitch class and is provided in that spirit. -# -# You can use a custom Snitch by setting this to the full class name -# of the snitch, which will be assumed to be on your classpath. -endpoint_snitch: SimpleSnitch - -# controls how often to perform the more expensive part of host score -# calculation -dynamic_snitch_update_interval_in_ms: 100 -# controls how often to reset all host scores, allowing a bad host to -# possibly recover -dynamic_snitch_reset_interval_in_ms: 600000 -# if set greater than zero and read_repair_chance is < 1.0, this will allow -# 'pinning' of replicas to hosts in order to increase cache capacity. -# The badness threshold will control how much worse the pinned host has to be -# before the dynamic snitch will prefer other replicas over it. This is -# expressed as a double which represents a percentage. Thus, a value of -# 0.2 means Cassandra would continue to prefer the static snitch values -# until the pinned host was 20% worse than the fastest. -dynamic_snitch_badness_threshold: 0.1 - -# request_scheduler -- Set this to a class that implements -# RequestScheduler, which will schedule incoming client requests -# according to the specific policy. This is useful for multi-tenancy -# with a single Cassandra cluster. -# NOTE: This is specifically for requests from the client and does -# not affect inter node communication. -# org.apache.cassandra.scheduler.NoScheduler - No scheduling takes place -# org.apache.cassandra.scheduler.RoundRobinScheduler - Round robin of -# client requests to a node with a separate queue for each -# request_scheduler_id. The scheduler is further customized by -# request_scheduler_options as described below. -request_scheduler: org.apache.cassandra.scheduler.NoScheduler - -# Scheduler Options vary based on the type of scheduler -# -# NoScheduler -# Has no options -# -# RoundRobin -# throttle_limit -# The throttle_limit is the number of in-flight -# requests per client. Requests beyond -# that limit are queued up until -# running requests can complete. -# The value of 80 here is twice the number of -# concurrent_reads + concurrent_writes. -# default_weight -# default_weight is optional and allows for -# overriding the default which is 1. -# weights -# Weights are optional and will default to 1 or the -# overridden default_weight. The weight translates into how -# many requests are handled during each turn of the -# RoundRobin, based on the scheduler id. -# -# request_scheduler_options: -# throttle_limit: 80 -# default_weight: 5 -# weights: -# Keyspace1: 1 -# Keyspace2: 5 - -# request_scheduler_id -- An identifier based on which to perform -# the request scheduling. Currently the only valid option is keyspace. -# request_scheduler_id: keyspace - -# Enable or disable inter-node encryption -# JVM defaults for supported SSL socket protocols and cipher suites can -# be replaced using custom encryption options. This is not recommended -# unless you have policies in place that dictate certain settings, or -# need to disable vulnerable ciphers or protocols in case the JVM cannot -# be updated. -# FIPS compliant settings can be configured at JVM level and should not -# involve changing encryption settings here: -# https://docs.oracle.com/javase/8/docs/technotes/guides/security/jsse/FIPS.html -# *NOTE* No custom encryption options are enabled at the moment -# The available internode options are : all, none, dc, rack -# -# If set to dc cassandra will encrypt the traffic between the DCs -# If set to rack cassandra will encrypt the traffic between the racks -# -# The passwords used in these options must match the passwords used when generating -# the keystore and truststore. For instructions on generating these files, see: -# http://download.oracle.com/javase/6/docs/technotes/guides/security/jsse/JSSERefGuide.html#CreateKeystore -# -server_encryption_options: - internode_encryption: none - keystore: conf/.keystore - keystore_password: cassandra - truststore: conf/.truststore - truststore_password: cassandra - # More advanced defaults below: - # protocol: TLS - # algorithm: SunX509 - # store_type: JKS - # cipher_suites: [TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_DHE_RSA_WITH_AES_128_CBC_SHA,TLS_DHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA] - # require_client_auth: false - # require_endpoint_verification: false - -# enable or disable client/server encryption. -client_encryption_options: - enabled: true - # If enabled and optional is set to true encrypted and unencrypted connections are handled. - optional: true - keystore: /certs/cass.jks - keystore_password: nosecret - require_client_auth: true - # Set trustore and truststore_password if require_client_auth is true - truststore: /certs/truststore.jks - truststore_password: nosecret - # More advanced defaults below: - protocol: TLS - algorithm: SunX509 - store_type: JKS - cipher_suites: [TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_DHE_RSA_WITH_AES_128_CBC_SHA,TLS_DHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA] - -# internode_compression controls whether traffic between nodes is -# compressed. -# Can be: -# -# all -# all traffic is compressed -# -# dc -# traffic between different datacenters is compressed -# -# none -# nothing is compressed. -internode_compression: dc - -# Enable or disable tcp_nodelay for inter-dc communication. -# Disabling it will result in larger (but fewer) network packets being sent, -# reducing overhead from the TCP protocol itself, at the cost of increasing -# latency if you block for cross-datacenter responses. -inter_dc_tcp_nodelay: false - -# TTL for different trace types used during logging of the repair process. -tracetype_query_ttl: 86400 -tracetype_repair_ttl: 604800 - -# By default, Cassandra logs GC Pauses greater than 200 ms at INFO level -# This threshold can be adjusted to minimize logging if necessary -# gc_log_threshold_in_ms: 200 - -# If unset, all GC Pauses greater than gc_log_threshold_in_ms will log at -# INFO level -# UDFs (user defined functions) are disabled by default. -# As of Cassandra 3.0 there is a sandbox in place that should prevent execution of evil code. -enable_user_defined_functions: false - -# Enables scripted UDFs (JavaScript UDFs). -# Java UDFs are always enabled, if enable_user_defined_functions is true. -# Enable this option to be able to use UDFs with "language javascript" or any custom JSR-223 provider. -# This option has no effect, if enable_user_defined_functions is false. -enable_scripted_user_defined_functions: false - -# Enables materialized view creation on this node. -# Materialized views are considered experimental and are not recommended for production use. -enable_materialized_views: true - -# The default Windows kernel timer and scheduling resolution is 15.6ms for power conservationLowering this value on Windows can provide much tighter latency and better throughput, however -# some virtualized environments may see a negative performance impact from changing this setting -# below their system default. The sysinternals 'clockres' tool can confirm your system's default -# setting. -windows_timer_interval: 1 - - -# Enables encrypting data at-rest (on disk). Different key providers can be plugged in, but the default reads from -# a JCE-style keystore. A single keystore can hold multiple keys, but the one referenced by -# the "key_alias" is the only key that will be used for encrypt opertaions; previously used keys -# can still (and should!) be in the keystore and will be used on decrypt operations -# (to handle the case of key rotation). -# -# It is strongly recommended to download and install Java Cryptography Extension (JCE) -# Unlimited Strength Jurisdiction Policy Files for your version of the JDK. -# (current link: http://www.oracle.com/technetwork/java/javase/downloads/jce8-download-2133166.html) -# -# Currently, only the following file types are supported for transparent data encryption, although -# more are coming in future cassandra releases: commitlog, hints -transparent_data_encryption_options: - enabled: false - chunk_length_kb: 64 - cipher: AES/CBC/PKCS5Padding - key_alias: testing:1 - # CBC IV length for AES needs to be 16 bytes (which is also the default size) - # iv_length: 16 - key_provider: - - class_name: org.apache.cassandra.security.JKSKeyProvider - parameters: - - keystore: conf/.keystore - keystore_password: cassandra - store_type: JCEKS - key_password: cassandra - - -##################### -# SAFETY THRESHOLDS # -##################### - -# When executing a scan, within or across a partition, we need to keep the -# tombstones seen in memory so we can return them to the coordinator, which -# will use them to make sure other replicas also know about the deleted rows. -# With workloads that generate a lot of tombstones, this can cause performance -# problems and even exaust the server heap. -# (http://www.datastax.com/dev/blog/cassandra-anti-patterns-queues-and-queue-like-datasets) -# Adjust the thresholds here if you understand the dangers and want to -# scan more tombstones anyway. These thresholds may also be adjusted at runtime -# using the StorageService mbean. -tombstone_warn_threshold: 1000 -tombstone_failure_threshold: 100000 - -# Log WARN on any multiple-partition batch size exceeding this value. 5kb per batch by default. -# Caution should be taken on increasing the size of this threshold as it can lead to node instability. -batch_size_warn_threshold_in_kb: 5 - -# Fail any multiple-partition batch exceeding this value. 50kb (10x warn threshold) by default. -batch_size_fail_threshold_in_kb: 50 - -# Log WARN on any batches not of type LOGGED than span across more partitions than this limit -unlogged_batch_across_partitions_warn_threshold: 10 - -# Log a warning when compacting partitions larger than this value -compaction_large_partition_warning_threshold_mb: 100 - -# GC Pauses greater than gc_warn_threshold_in_ms will be logged at WARN level -# Adjust the threshold based on your application throughput requirement -# By default, Cassandra logs GC Pauses greater than 200 ms at INFO level -gc_warn_threshold_in_ms: 1000 - -# Maximum size of any value in SSTables. Safety measure to detect SSTable corruption -# early. Any value size larger than this threshold will result into marking an SSTable -# as corrupted. This should be positive and less than 2048. -# max_value_size_in_mb: 256 - -# Back-pressure settings # -# If enabled, the coordinator will apply the back-pressure strategy specified below to each mutation -# sent to replicas, with the aim of reducing pressure on overloaded replicas. -back_pressure_enabled: false -# The back-pressure strategy applied. -# The default implementation, RateBasedBackPressure, takes three arguments: -# high ratio, factor, and flow type, and uses the ratio between incoming mutation responses and outgoing mutation requests. -# If below high ratio, outgoing mutations are rate limited according to the incoming rate decreased by the given factor; -# if above high ratio, the rate limiting is increased by the given factor; -# such factor is usually best configured between 1 and 10, use larger values for a faster recovery -# at the expense of potentially more dropped mutations; -# the rate limiting is applied according to the flow type: if FAST, it's rate limited at the speed of the fastest replica, -# if SLOW at the speed of the slowest one. -# New strategies can be added. Implementors need to implement org.apache.cassandra.net.BackpressureStrategy and -# provide a public constructor accepting a Map. -back_pressure_strategy: - - class_name: org.apache.cassandra.net.RateBasedBackPressure - parameters: - - high_ratio: 0.90 - factor: 5 - flow: FAST - -# Coalescing Strategies # -# Coalescing multiples messages turns out to significantly boost message processing throughput (think doubling or more). -# On bare metal, the floor for packet processing throughput is high enough that many applications won't notice, but in -# virtualized environments, the point at which an application can be bound by network packet processing can be -# surprisingly low compared to the throughput of task processing that is possible inside a VM. It's not that bare metal -# doesn't benefit from coalescing messages, it's that the number of packets a bare metal network interface can process -# is sufficient for many applications such that no load starvation is experienced even without coalescing. -# There are other benefits to coalescing network messages that are harder to isolate with a simple metric like messages -# per second. By coalescing multiple tasks together, a network thread can process multiple messages for the cost of one -# trip to read from a socket, and all the task submission work can be done at the same time reducing context switching -# and increasing cache friendliness of network message processing. -# See CASSANDRA-8692 for details. - -# Strategy to use for coalescing messages in OutboundTcpConnection. -# Can be fixed, movingaverage, timehorizon, disabled (default). -# You can also specify a subclass of CoalescingStrategies.CoalescingStrategy by name. -# otc_coalescing_strategy: DISABLED - -# How many microseconds to wait for coalescing. For fixed strategy this is the amount of time after the first -# message is received before it will be sent with any accompanying messages. For moving average this is the -# maximum amount of time that will be waited as well as the interval at which messages must arrive on average -# for coalescing to be enabled. -# otc_coalescing_window_us: 200 - -# Do not try to coalesce messages if we already got that many messages. This should be more than 2 and less than 128. -# otc_coalescing_enough_coalesced_messages: 8 - -# How many milliseconds to wait between two expiration runs on the backlog (queue) of the OutboundTcpConnection. -# Expiration is done if messages are piling up in the backlog. Droppable messages are expired to free the memory -# taken by expired messages. The interval should be between 0 and 1000, and in most installations the default value -# will be appropriate. A smaller value could potentially expire messages slightly sooner at the expense of more CPU -# time and queue contention while iterating the backlog of messages. -# An interval of 0 disables any wait time, which is the behavior of former Cassandra versions. -# -# otc_backlog_expiration_interval_ms: 200 diff --git a/.ci/docker-compose-file/cassandra/cassandra.yaml b/.ci/docker-compose-file/cassandra/cassandra.yaml index 6059e9d64..1bc724b0b 100644 --- a/.ci/docker-compose-file/cassandra/cassandra.yaml +++ b/.ci/docker-compose-file/cassandra/cassandra.yaml @@ -638,7 +638,7 @@ native_transport_port: 9042 # for native_transport_port. Setting native_transport_port_ssl to a different value # from native_transport_port will use encryption for native_transport_port_ssl while # keeping native_transport_port unencrypted. -# native_transport_port_ssl: 9142 +native_transport_port_ssl: 9142 # The maximum threads for handling requests when the native transport is used. # This is similar to rpc_max_threads though the default differs slightly (and # there is no native_transport_min_threads, idle threads will always be stopped @@ -1044,20 +1044,20 @@ server_encryption_options: # enable or disable client/server encryption. client_encryption_options: - enabled: false + enabled: true # If enabled and optional is set to true encrypted and unencrypted connections are handled. - optional: false - keystore: conf/.keystore - keystore_password: cassandra - # require_client_auth: false + optional: true + keystore: /certs/server.jks + keystore_password: my_password + require_client_auth: true # Set trustore and truststore_password if require_client_auth is true - # truststore: conf/.truststore - # truststore_password: cassandra + truststore: /certs/truststore.jks + truststore_password: my_password # More advanced defaults below: - # protocol: TLS - # algorithm: SunX509 - # store_type: JKS - # cipher_suites: [TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_DHE_RSA_WITH_AES_128_CBC_SHA,TLS_DHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA] + protocol: TLS + algorithm: SunX509 + store_type: JKS + cipher_suites: [TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_DHE_RSA_WITH_AES_128_CBC_SHA,TLS_DHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA] # internode_compression controls whether traffic between nodes is # compressed. diff --git a/.ci/docker-compose-file/certs/README.md b/.ci/docker-compose-file/certs/README.md new file mode 100644 index 000000000..71c389bdd --- /dev/null +++ b/.ci/docker-compose-file/certs/README.md @@ -0,0 +1,23 @@ +Certificate and Key files for testing + +## Cassandra (v3.x) + +### How to convert server PEM to JKS Format + +1. Convert server.crt and server.key to server.p12 + +```bash +openssl pkcs12 -export -in server.crt -inkey server.key -out server.p12 -name "certificate" +``` + +2. Convert server.p12 to server.jks + +```bash +keytool -importkeystore -srckeystore server.p12 -srcstoretype pkcs12 -destkeystore server.jks +``` + +### How to convert CA PEM certificate to truststore.jks + +``` +keytool -import -file ca.pem -keystore truststore.jks +``` diff --git a/.ci/docker-compose-file/certs/server.jks b/.ci/docker-compose-file/certs/server.jks new file mode 100644 index 0000000000000000000000000000000000000000..a0795527824f297a64cf8e24e91d12e15e99ae9f GIT binary patch literal 2898 zcma);XE+;d`^FPuZ$ULAu}iI>8dU9BMJYAfnnkRZwrUrlqN2o(S$oqKCHCH1Rcr5u zP&I2jG+zJX{~mpg_tX1+IFI|f&+9s`5BG=PaU;-l24p~T1e*4QlID82PWUkckP4WE zrX_;Vv}>345&}&@_>UGvF$hhOe`#l5Dua^l-zj=pAXyfgLgms_L^%FQP%t1Y5OV)q z=@FtJHo#R;-P?H2on3QZuq&3VzYFs_hK%eQGZ~NvK~G8j{~O6@C;$j3C5=tE4$y%d z1iTJn{-z%9)7=$<#6&6rgDm~FKxokDY+-oHdb20jD9%%}Ze%S_z(yt#OJZbC=8MWK#-irQD zX|!`@ZlEFMMy$oU_58F>X1L$rrj6Y2HC#9bF26Z4No&II zQCvj#hm|u?fFTw7lp5M8Nm3ph;+dApv(lZnU*)Utc9B~&D$XiRs0f7ri*hw)myfW| zEutO9g(ujgeUymxTbN>>vC!KF2S(&d;w5?aVCs)%;Ikjg0w5!(U9-D{)jkeY)9nJ! z=x(7t@x-gCWPWt-m#S3Zb$Tdbz{EUplM9!DIBA&=w-)u>4<4_15<#=>U z**7f6*~`ZAr%!`_vVG=DC?0+fnh^u+(sIE`cugnq%*^A$QeAHa0&!fA!EOmv)jn;6 zjQ73vXmRG(jPs;#8I&?v6QBK}@~5p&Sf96MY7n}S9%EZI4DJJF)=d3z>7oaH9^5`( zT>Fn$#!z!^sGPYk#y)GW1BsL$g1PZKc_ucHgVBm^6NEm6cn$Gf- z=(f7b^=ABs_Ngc0q%leKj$}^5lK;S63VZq+QTc~nogE*3Q45w;D|(r#`i=`Hz0KLn zc+OvhWx~A^iFvjGj(bvNs|B;P#}{~HxJoB6*weAwpTV^smD#?0Q@U$siKfLQc2XC( zeX+e?Hmp2T@^n-66C^26cI3wFSZGRTNnD0lWi9If9e!#_$MlAgRK20)ZRFXQo?|3m zBzqE`;Kb#=ipvVxRMxmMu3(b2zpuy1Jv%3Ngj{@jN5DmxT#Yo~vay@;d5h9Shdx!* zYBc`?f6nn@uoN>m%*{;*%PHFgc#G7(J6-oZ-eki=CyNK6+az6*=)o)e#{=Ow>ym$1cEnCbg)}ky#$3`MHLLcSz@-y`y!KM6TEPptBNOf&O`M{g95S}Qb>>t@ma-Kd&OdN&%-aye-6ym z>B&%T?fg3LzZz9gU!76;vzJGvTFYsI8m-?pbfsiODFxVWS)a8do8 z13uzMK1S*eLhuhRY1pJVVH^N^fCJz;;3dEr;CLx}fbC`NfZ+XSj9rQi#A@Q~?!Y5; z6Dg07LEex>BIR#fviP4ND#}YdvtC+^WI({>LG>q21^jQo)-bAld&Dn!K4D$wHF0sq ze(zPT`~L!LRMR0td6RYl4)AQ^jxRAG7!AVJ4;2RM#_t5&!cv4v97KtmtGcw&>=rx0 z1`E?dr~;y=gfp2B+JQk1Dr)$gcdYmD(-r*cQ$0@fF7^%$CR)i?m!M)q-ZCX2Bb+UZ zk7v2r3}D_blJh2qMZ z4}4X;skGj#s*R;%ho^-0hHkw$uvnH|meEOpwg!2Rs6gl2l%;d=_ax9avxwaw{_2 zr3NvU2}h&9E}L0Ojnnz`nqb{`r8QN&cU+FbSVXdP~k_&iUc3G7MqC%zyZT>s|H_Vlke zQ1i_4pi!2V{3`H%GMs!vvtzimtJK1Qf8*Gg-Dr{QD6seIuw&H<#y7sEQleaDKQmia zO;IXUs)FhOn&1>Lpe`p9+jFF>#P`Od5nai;L6A0`cY*%CZwEaI8g7uE%CCNrUgc|5 zDz3+S9}7?6jWzUC(tjvWM@)|$YzlsU^95>^e@DcKe9_>@xh8L6aiKpFEA8A&KKj1O zYQI7yr{jc^sf{k6gCq3s_L$z<*T%$|hw_ihgFPtr*sf&4u-wQpa8!Eh#}!vp2e*fa z$_B4)p)lV_xBp^3HR73Yg?ITlV{Ual}M zpek~BTPtTzHTPpJX!Z4{mrq(Io7z_E+pOcdL1Y+0!5V~QV4^s-F#W0xY8Y6aoe%Y= zd`N64`S~Ctfu$d-SO4WujOoSq#W8tsS|l6?04^Ui?)7v}qLA`VQ_mQRqKafb`gfa+(_6U&2s>*gC!>p#F|G V^fZl!Yo2LvM-`la^;8| zVUCtFOd{u8<%oWIy?(#n_xJnbc|FhT^E|KT&o`O?dk6xupb0P=D7#F&Vf;@{ASp5q^M3FAuyg(vUE*~+uTd)=Al~aKz}mu*#*=ujsB@lLilt4w zGwJrLuRwAxksD$(P-@x8H9X;!>b0M}3{gbxZX_mDsqW(A`DzEz4~z9OemNUN%8#;R zdrfbWrfdC!Os@`$gvXj%i@X|C{t`q!L){xoKNP@K`Repe*QyXqz2?xgJliaxVn^aI zWU;>RlOCO6{p)@VE#0Oz26WnCI#0D}_1c&3Eug#Z1Isdl8Rq=5d;#)8#vK`$Hc0xu z4X%LwTi0UOG(2=7dkV*>32#;Y6x@9oNZU%Vsuhd-Zo)4b2*I~1c~nR82{}62NzwIU z<*(c*DD&vn^9f;-OTe>(J0*$v+*-vV5naOgnZ|$w*@;2 zmE>AfnUIfUlZz*P8xvEQ>S4f* z#j%P?3X8|G3i*>zr)RWcT)8|RnbOZDbPRXfb(r`lG?EP)Zw776?fWqK#%c~i2bVJx zDs)P?@%4gBJ(szQ)Avuf*ZiF<`FjC};nf*izrLfbZ$G-7eyl%K7TO@}+0;nKKXr9A zo01$Z-k!Ej%L}6xC{t-I$ns3Gdq#HK>;5uF^woZOkwsAvmY~N>+U}zPHJ0z$F3Ity zb?&|&FUKsg*cy+-UHYt<+o7M<#>$t7!jbM<^j2q&&K+V0L)N@=&FL+~wc4(7m#n|n zzT3^O2xPvnt)FjI@|{KdSia#>GggW|Q54d+8aZJbSP}jWM~0K19Uh4L&{vcpx60q> zZcIur_IH569EPAW1DVG~*jX~=(^_h#3hj)xh$NjT4ZmUcCSsULmU_|(wB|LyU;e_v z15v|yq}1AUSrWdJ9j}(&Mp@fW_#WdgDg+ODL>|EEzJH!*Ganf9U)ZH*_j2pj)g}9cQa4rt7a%J~)VxZCQaZ#Vh zPjahoA3xI8ZyoLc{{jh-^tX!K^k+`8*JBj7xuH$K7b6Bdp`w1{B9%5;m9nEPdg#T9 zDc{Aia^ry?WTtXJZO&ZZIy9o(rA5<7kQA3;AgR7mLX-zaSd*K7;%s!Xn&j~Dc{&C^ z4j3cFe1Q3ta{s^euMkyF6L`@C2=0I2Dj-1Yzyyf(QEYisrqE;mVGITY9eKBU1l{8Q z>BjO$w_Cf=1L?MJ^M7;$6TqEn;yL4iP@2UEF=37QE7;^0Ux75-iICDm-f&14U2>RM zT+%6#dv@PHa-Tx6c6Q*Rw%wew^26;!TqrrK;ug}pos?B~PiIxBp&3q0kP-@D)J(jH zd!o4XlDV8LDRq*z6Nk_a(?CjVJhkVL&|h&&xUl)slAOKM(9QW;UTFSCoq|OBy1WbS z-AZX*E46a{Ql$kcX72cwI~^vYE|77cE|Fq2P(f^YjSQyyTGmxw2xtv{s1f+G19i?U zEf(ks6kq0i=^pkYhQ;Nls8xDb%KOz}$d$5TP-h=Rm%9gc)g-1dT@%G)k^q&A6Pi4* z$OeB+J?Ailfq(VRSkSzkv9Nkr%@r3TX&@p8bYU8Alz)Kl=<`FW0$ zkUFC$`R8i#$5Q6rRdI6@`)-O9`KF@a(~=(+n^}v6W;0u)CQmrsu@{r3SZc7Rt&|S! zJBXlpIeG$DY@Cpp`^U}Zo1lrN5wp%7YCXEebK)BfMm^Zzzjm8S30cz{d*8y#gn6rj zmT_rTk_k~)tTIEY%m-t?a#laSQ1jMc$I!qKde}$h!Oj23kYpDGq9eENBk$=RM{4z0 z7;a06wbVrM2HetS%?qz>vW3ox7V>e@XkMXen*-`ZI{?izgwJ1jYH0R}nZWp=Rxwz$ z{#L%x)a=9Xh5YtiD=*)+EhP!Ob%q8d89QbHxxp3Es-x@XTow%@QJidOcT~^6z%a12 z!f9W}h1JGQ)ivMUOy^i04i>9y>U#UU#-nN{2Q=(L@Egv(&7$k$B^V@F5h(Hkw`5gb zPxAllGjvDpDNFz9K~Mbi+`Mcpiv__a9=EYU7LqfZo3dOV$P3thQ^fUp=5Lutum;bE zLJO>S`8>f6CT+cC$9i_mXHB)l zsONud9F|uKOEnS+t@N?!rxiv;Kgcr{&Zo1be?&$Nq4-!_;g5l?m&)O249hu?^j~R_ zCCN(9EQo>bZ6>Tm)h@vZvG)KMHCamOud}mZm|$Uf4hP_~3D+v+s&?Yc&@9PSY?CU* z3+-*N?Qqh0P;!#xL|FHT)ft^QQp|)g>ovjXToQAlI_h4dc!82s>=U!wXR5T#jfvSj zd`rTq8=|4W_1{@_wwG{JjEZ8C9!!baimK!HsJ**LX&s2L4l*ei)#Yu6`L9nS5gdr- zmFwPYjDm~h0-mpC25S4TR~*&V``_0!BzTz>WrG$Iy3Ql>I9%8cG(21OwE3w{&Z=gH zmqH!zeGeM2Q9tXGM&d19#y;IRd?%nkbBf&6$++`FCUN;>*af$sP44=Vo==(zEnNO4 z;oCV)uDhd|MFGllg(xkgKzXMv)NEAG`Qh=nx^EF)R&JC&2i}zDBC7B#cW?U$JK5aS zQ#*l9w)Xjyoi!d&jYJ#8{Zh){REwwHhcAZeli}WyJQ|0Reel4^52M0sf8p<6L8AKPhO5e~u7m1y1n;TmkL?e?TAr5AZlDSAg@; z?2bN*7DKZ`A+lU7AVCNKj0oIaX8sj-kT9udm^^Ssfz(}lkOXF<=M<-m41Rw62h)E6 D81(h> literal 0 HcmV?d00001 diff --git a/.ci/docker-compose-file/certs/truststore.jks b/.ci/docker-compose-file/certs/truststore.jks new file mode 100644 index 0000000000000000000000000000000000000000..cd97d01588c7befd6cfbbb679a3d02623380a241 GIT binary patch literal 1622 zcmV-c2C4Zlf(B9o0Ru3C1^fmHDuzgg_YDCD0ic2f?F51a=`ex?&LNQU)EOumwXc=d1z(ClCSwATSID2r7n1hW8Bu2?YQ! z9R>+thDZTr0|Wso1P}}uYk(Nh6jHK?jg?1k+sA-{1yD$Kl-+1u^qlu=j17ww4_OPa zu?L5ne^q+7cGq#*5Xi|&{ajOF$)WAC!KY(M0BI=UMvouRiNw(DhAB$# zA+&Q=ymGuZg$Nvghaioh+Jc16NX=0AI?+n-G*b7j4#S{B1yC^s7&^M*z+BxJ?d}B( z3-ED;r1xR|1;L9AXrw^%q2W3<)?s0JbR=dSE$}W?qhht&NdJ=ABWqKB)vR@?UMV8+^Ej(1xvd02A;okOWW4fOFuI` z9Tj<^&^}5H=vaRpF!X9%j?^6TIAeFvmJtpKQxFWB;xq*7_uBZ@{Xpd zyhkM;8T$j;$3p=j=+baW;LpN3hAw)hgQ5cnePyNdq40Le-bx``xEPR|yXrN_gV6s2 zBw?SPBx4|LhYgt|3{=ja?0Z_c`CeRp%lkP!Df7s9<#djTRZ!|H*z5|*u*fd2_8E2} zR71rLUm~sWbvTkC$v{!M+!p(D&IEI<^L(m|xk>xE%-if+E-UqgA@eKb1JIB(8{pcFoZVaW3&qKTSw<0DXz8#;v++xFEZmf;ZP_P+9~1e54nAm`%6+ zA9I16r50Hw*({R|YM7urX13%ida1BC>v*5Z5` zd`X{gNb<9WoueBPG~GP|V5(e{>vlUMTBv-3_KQ2jo~L&OdK9*_5)ZRBm;0fiPf>4O z*ZB-RTpMav7@#Px%KV2Ts#ia`$>r7wBcse$ra)&WtVs@W4Mh&N{M3V<3 ztuQzT&)SV4F7;O-Ue&ujumkp`O}q^f0UzWGh=6lX#)6vy&iUYNI#+~XguJ-l7*vC6 zLS9123Ru6<{FcC{n9(&lddcMFTf8xqdvH{k&3pe`vSq2+5;_qZ@ zg46`Riqw+$b4G8+x=lG~F)AxlR6Ypf^5&^q$M)RGd;!30B2lTJzAp|{x2Ma?c87Qy zZ!sCL+sgo`s@=v%4ahYSD03VP>nJ~RO$>y?uSlk@QruQdZhPS58R>0e82I&raEL52 z`fM5EO5MBXWI|wgh>Fdxht{FhA7I=69I`Al55Ax_%?Gyldir~i_8=M&iSjZR*EX9R zx1-_=cl6Qh#$`7LO5nT7lH!yU8%;$4b$`@-25fxGE*RN9HeCQ|jx|!qJ&ia*z>x&9 zoA}%-lzsD7k5YpenqR5+IQ1I~gbx-4uG8PmfeRRVo*Bz|fW%g& zNC9O71OfpC00bav$px}Peye0XMhtIbMcGUfw-|BX{Ma=O`Y;u3i9R6&6j{DH7Xq8& U_#+cuAFH4*5M$OX#sUH-5HWY@;{X5v literal 0 HcmV?d00001 diff --git a/.ci/docker-compose-file/docker-compose-cassandra-tcp.yaml b/.ci/docker-compose-file/docker-compose-cassandra-tcp.yaml index ce0cb4de5..ac45af02f 100644 --- a/.ci/docker-compose-file/docker-compose-cassandra-tcp.yaml +++ b/.ci/docker-compose-file/docker-compose-cassandra-tcp.yaml @@ -2,7 +2,7 @@ version: '3.9' services: cassandra_server: - container_name: cassa_tcp + container_name: cassandra build: context: ./cassandra args: @@ -12,8 +12,11 @@ services: environment: CASSANDRA_BROADCAST_ADDRESS: "1.2.3.4" CASSANDRA_RPC_ADDRESS: "0.0.0.0" + volumes: + - ./certs:/certs ports: - "9042:9042" + - "9142:9142" command: - /bin/bash - -c diff --git a/.ci/docker-compose-file/docker-compose-toxiproxy.yaml b/.ci/docker-compose-file/docker-compose-toxiproxy.yaml index dd8b91252..89705e668 100644 --- a/.ci/docker-compose-file/docker-compose-toxiproxy.yaml +++ b/.ci/docker-compose-file/docker-compose-toxiproxy.yaml @@ -20,6 +20,7 @@ services: - 16041:6041 - 18000:8000 - 19042:9042 + - 19142:9142 command: - "-host=0.0.0.0" - "-config=/config/toxiproxy.json" diff --git a/.ci/docker-compose-file/toxiproxy.json b/.ci/docker-compose-file/toxiproxy.json index 79589c14a..04b8bf66c 100644 --- a/.ci/docker-compose-file/toxiproxy.json +++ b/.ci/docker-compose-file/toxiproxy.json @@ -57,13 +57,13 @@ { "name": "cassa_tcp", "listen": "0.0.0.0:9042", - "upstream": "cassa_tcp:9042", + "upstream": "cassandra:9042", "enabled": true }, { "name": "cassa_tls", - "listen": "0.0.0.0:9043", - "upstream": "cassa_tls:9043", - "enabled": false + "listen": "0.0.0.0:9142", + "upstream": "cassandra:9142", + "enabled": true } ] diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl index b509b537b..41ac1c33f 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl @@ -76,7 +76,7 @@ init_per_group(tcp, Config) -> ]; init_per_group(tls, Config) -> Host = os:getenv("CASSA_TLS_HOST", "toxiproxy"), - Port = list_to_integer(os:getenv("CASSA_TLS_PORT", "9043")), + Port = list_to_integer(os:getenv("CASSA_TLS_PORT", "9142")), [ {cassa_host, Host}, {cassa_port, Port}, @@ -132,6 +132,7 @@ end_per_testcase(_Testcase, Config) -> %%------------------------------------------------------------------------------ common_init(Config0) -> + ct:pal("commit_init: ~p~n", [Config0]), BridgeType = proplists:get_value(bridge_type, Config0, <<"cassandra">>), Host = ?config(cassa_host, Config0), Port = ?config(cassa_port, Config0), @@ -251,8 +252,7 @@ query_resource(Config, Request) -> connect_direct_cassa(Config) -> Opts = #{ - host => ?config(cassa_host, Config), - port => ?config(cassa_port, Config), + nodes => [{?config(cassa_host, Config), ?config(cassa_port, Config)}], username => ?CASSA_USERNAME, password => ?CASSA_PASSWORD, keyspace => ?CASSA_KEYSPACE @@ -262,8 +262,7 @@ connect_direct_cassa(Config) -> case ?config(enable_tls, Config) of true -> Opts#{ - ssl => true, - ssl_opts => emqx_tls_lib:to_client_opts(#{enable => true}) + ssl => emqx_tls_lib:to_client_opts(#{enable => true}) }; false -> Opts From 678cc937c08aef6118e435e51ce5dabd6f835dea Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 17 Mar 2023 18:23:58 +0800 Subject: [PATCH 008/196] test(bridge): cover ssl testing for cassandra bridge --- .../cassandra/cassandra.yaml | 3 +- .ci/docker-compose-file/certs/client.key | 27 ++++++++++++++++ .ci/docker-compose-file/certs/client.pem | 25 +++++++++++++++ .ci/docker-compose-file/certs/server.jks | Bin 2898 -> 2390 bytes .ci/docker-compose-file/certs/server.p12 | Bin 2708 -> 2708 bytes .ci/docker-compose-file/certs/truststore.jks | Bin 1622 -> 1318 bytes .../test/emqx_ee_bridge_cassa_SUITE.erl | 30 ++++++++++++++++-- .../src/emqx_ee_connector_cassa.erl | 1 - 8 files changed, 80 insertions(+), 6 deletions(-) create mode 100644 .ci/docker-compose-file/certs/client.key create mode 100644 .ci/docker-compose-file/certs/client.pem diff --git a/.ci/docker-compose-file/cassandra/cassandra.yaml b/.ci/docker-compose-file/cassandra/cassandra.yaml index 1bc724b0b..51a24f7a2 100644 --- a/.ci/docker-compose-file/cassandra/cassandra.yaml +++ b/.ci/docker-compose-file/cassandra/cassandra.yaml @@ -1046,7 +1046,7 @@ server_encryption_options: client_encryption_options: enabled: true # If enabled and optional is set to true encrypted and unencrypted connections are handled. - optional: true + optional: false keystore: /certs/server.jks keystore_password: my_password require_client_auth: true @@ -1055,7 +1055,6 @@ client_encryption_options: truststore_password: my_password # More advanced defaults below: protocol: TLS - algorithm: SunX509 store_type: JKS cipher_suites: [TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_DHE_RSA_WITH_AES_128_CBC_SHA,TLS_DHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA] diff --git a/.ci/docker-compose-file/certs/client.key b/.ci/docker-compose-file/certs/client.key new file mode 100644 index 000000000..2989d0d78 --- /dev/null +++ b/.ci/docker-compose-file/certs/client.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAzs74tdftT7xGMGXQSoX/nnFkFAOjNtEVOI3bChzR+w6Xwo8Z +OUiOuOjynKvsJeltdmc0L+cbHZh7j+aHuAqVYxavqaqhFneF0f03t17qju9AixoV +JXgNT3ru56aZFa6Ov6NhfZfRirGnbNrg2RhuNeYZ4TYLH7iMR36exNFP83glXwXM +inMd1tsHL7xHLf3KjCbkusA5ncFWcpIUtpuWVn9aAE402dN7BJWfAbkQ4Y3VToR1 +P/T+W6WBldv0i2WlNbfiuAzuapA3EzJwoyTrG2Qyz7EtXM8XZdOZ6oJmW4s7c4V/ +FBT5knNtmXTt78xBBlIPFas5BAJIeV4eADx9MwIDAQABAoIBAQCZTvcynpJuxIxn +vmItjK5U/4wIBjZNIawQk6BoG7tR2JyJ/1jcjTw4OX/4wr450JRz7MfUJweD5hDb +OTMtLLNXlG6+YR4vsIUEiSlvhy5srVH0jG5Wq2t6mxBVq7vaRd/OkshnuU79+Pq7 +iHqclS7GSACxYkXWyxE6wtPh5aTWP8joK/LvYFiOqKPilUnLZ4hBhmL7CRUCZ0ZA +QGNyEhlmiAL+LNKW2RLXPBxlKX21X78ahUQmkkTM0lBK9x6hm4dD3SpLqmZyQQ9M +UfiMbU6XOYlDva/USZzrvTDlRf9uCG9QOsZzngP1aIy8Cq3QHECOeMIPO9WQLMll +SyY+SpyJAoGBAP4fhnbDpQC6ekd9TNoU9GE/FNNNGKLh82GDgnGcWU/oIzv8GlaR +rkEHTb6aRoPpjTxWIjJpScs9kycC+7N3oNo9rub4s5UvllI+EgQ95+j/5fnZx6gO +la8ousLy1hTYu9C0nTWdTV3YtfC0l0opn7Friv5QafNmhSn74DqrH0BHAoGBANBV +/NhBDAH1PHzYA+XuNLYTLv56Q4osmoen17nPnFNWb1TtWblzb0yWp86GGDFcs8CZ +eH0mXCRUzGMSWtOHe4CbIm2brAYXuL2t6+DZ1A22gsnW5avNrosZRS7eN7BE7DDj +5cp9+Es9UWnArzJU7jSWwAtA6o47WHfHU/pqRB21AoGAGx6eKPqEF2nPNuXmV7e4 +xNAIluw5XtiiMpvoRdubpG1vpS0oWmi9oe73mwm30MgR7Ih8qciWuXvewmENH3/6 +yI+gpMGR2K/1aN166rz4jOMSVfGp3wN/cev00m0774mZsZI03M3mvccs031ST/XV +Nwf1E2Ldi747I9nfeiNc+G0CgYEAslFHD1ntiyd6VGkYPQ978nPM/2dqs7OluILC +tHmslfAfbpOQ/ph9JRK2IqDHyEhOWoWBiazxpO8n2Yx2TSNjZBpkh2h8/uIC7+cT +Q+tuAya6H0ReZISx5sEEZC8zfx4fA2Gs53qWsN+U9W1FB1GGaWC2k2tG1+KXwD3N +9UJLdxkCgYBB96dsfT7nXmy0JLUz0rQ4umBje6H5uvuaevWdVMEptHB+O7+6CAse +OVwqlFLQ4QC7s4/P9FQwfr/0uMRInB1aC043Haa1LbiRcRIlSuBDUezK5xidUbz+ +uB/ABkwwEuqW3Ns1+QieJyyfoNYKZ2v0RtYxBuieKOpUCm3oNFZRWg== +-----END RSA PRIVATE KEY----- diff --git a/.ci/docker-compose-file/certs/client.pem b/.ci/docker-compose-file/certs/client.pem new file mode 100644 index 000000000..454ca4797 --- /dev/null +++ b/.ci/docker-compose-file/certs/client.pem @@ -0,0 +1,25 @@ +-----BEGIN CERTIFICATE----- +MIIEMjCCAhoCFCOrAvLNRztbFFcN0zrCQXoj73cHMA0GCSqGSIb3DQEBCwUAMDQx +EjAQBgNVBAoMCUVNUVggVGVzdDEeMBwGA1UEAwwVQ2VydGlmaWNhdGUgQXV0aG9y +aXR5MB4XDTIzMDMxNzA5MzgzMVoXDTMzMDMxNDA5MzgzMVowdzELMAkGA1UEBhMC +U0UxEjAQBgNVBAgMCVN0b2NraG9sbTESMBAGA1UEBwwJU3RvY2tob2xtMRIwEAYD +VQQKDAlNeU9yZ05hbWUxGDAWBgNVBAsMD015U2VydmljZUNsaWVudDESMBAGA1UE +AwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzs74 +tdftT7xGMGXQSoX/nnFkFAOjNtEVOI3bChzR+w6Xwo8ZOUiOuOjynKvsJeltdmc0 +L+cbHZh7j+aHuAqVYxavqaqhFneF0f03t17qju9AixoVJXgNT3ru56aZFa6Ov6Nh +fZfRirGnbNrg2RhuNeYZ4TYLH7iMR36exNFP83glXwXMinMd1tsHL7xHLf3KjCbk +usA5ncFWcpIUtpuWVn9aAE402dN7BJWfAbkQ4Y3VToR1P/T+W6WBldv0i2WlNbfi +uAzuapA3EzJwoyTrG2Qyz7EtXM8XZdOZ6oJmW4s7c4V/FBT5knNtmXTt78xBBlIP +Fas5BAJIeV4eADx9MwIDAQABMA0GCSqGSIb3DQEBCwUAA4ICAQBHgfJgMjTgWZXG +eyzIVxaqzWTLxrT7zPy09Mw4qsAl1TfWg9/r8nuskq4bjBQuKm0k9H0HQXz//eFC +Qn85qTHyAmZok6c4ljO2P+kTIl3nkKk5zudmeCTy3W9YBdyWvDXQ/GhbywIfO+1Y +fYA82I5rXVg4c9fUVTNczUFyDNcZzoJoqCS8jwFDtNR0N/fptJN14j8pnYvNV+4c +hZ+pcnhSoz7dD8WjyYCc/QCajJdTyb15i072HxuGmhwltjnwIE/2xfeXCCeUTzsJ +8h4/ABRu9VEqjqDQHepXIflYuVhU38SL0f4ly7neMXmytAbXwGLVM+ME81HG60Bw +8hkfSwKBbEkhUmD6+V1bdUz14I6HjWJt/INtFU+O+MYZbIFt4ep9GKLV3nk97CyL +fwDv5b4WXdC68iWMZqSrADAXr+VG3DgHqpNItj0XmhY6ihmt5tA3Z6IZJj45TShA +vRqTCx3Hf6EO3zf4KCrzaPSSSfVLnGKftA/6oz3bl8EK2e2M44lOspRk4l9k+iBR +sfHPmpiWY0hIiFtd3LD/uGDSBcGkKjU/fLvJZXJpVXwmT9pmK9LzkAPOK1rr97e9 +esHqwe1bo3z7IdeREZ0wdxqGL3BNpm4f1NaIzV/stX+vScau0AyFYXzumjeBIpKa +Gt0A+dZnUfWG6qn5NiRENXxFQSppaA== +-----END CERTIFICATE----- diff --git a/.ci/docker-compose-file/certs/server.jks b/.ci/docker-compose-file/certs/server.jks index a0795527824f297a64cf8e24e91d12e15e99ae9f..06c2fe1847400333440b265830e73b39ae041008 100644 GIT binary patch literal 2390 zcmZvcc{J3G8pmfdF}AEDp|M1k_>Cnd+0vBk%UCnQ*!RYgC1hU;*~>O031eT!*0mE! zh(;(|_3|Q1gqZ7n?>*1O9s$364ZRS0`73 zouA{W7Q3-Pz=J?^5FiqE0m(a-C2{FpZ)L9g33_-q1wEg$+Y3KUbdAJ$A8d+)a)@9iAUM%I$@ zb(4Yt;ZEbyz~oJX5!b;lG76KjQQFg<)qUj+K*B!W=U?jBCbv z#a+-1-DAn>e>CVtIx}CPU#-~1ba&&?j0;NTNI6Z}axyZp;f8oTR4xtq-GwC_J7%!E zM4Z9h`GvL#^Vj5h*x4>Z5_jtPakCqKi|30?L_Xe6P6&t3C>rafW#Bc>a}KoUfmyhJFvS$8tHsS{ zO2}bl8@|@Jytiay8~&j~4ivxlh_r_aKkVpa_!jJ(YV?*_kcPhB=2?vBH`?#CG|$SI zb7?)0&ZTeK?sUiwa?aF!_XT|e%DugJd)$<{?x$?Jgf=QCMp25DU0}sM-`#&mPI=Sk zdk+5SfaFGGw)mJBZtKY1YBPi!a z#h%8h^Znnx;mMQlvRaJ-R?>()SspDMqpj9&$ zLU*pbAps#mYnDZN<5dWSElmgD|a%UO7NBypgFnFLC; z!y=ZY=Mz=)pOU_nJuQR%NZY2w)(~5%CEmCL{LvWhNL~{eQ`D=T)%aC~TUaFiowj>& z<5_jGv#C?34Kvf>5lHBIRcdd0S%z0aPT*vsSlgL5)0&yb4s$UfKIHM?s0;-B7AWev z#w+1mWT`-|*AZFq`7QvTMS?=cS`IAkg1OyNDklX9WzSGs1j9zLi?6!n8*(b{FH;?U z`a;DnbQ`jJR5Uy=mPi|npZIzhK%VKp8lSmyY1lorDE4QXMbqx(TGA4w>v6!ENc^7yHO`NfZ;(0*`B&0yRVE!7ER*avo9XteJq9k0PIja zGI$Q&Ow6k)-m!e%eI@?I51O1cCc!MBm*I)c{(=YA819WC^X&>xBT5^>VJNy|{X6@t zlzo?(B4sHxt_pb3ijqHG=Ri+Old`&5gl6XY=KQU%dKUUT9))<;AS?GtT zYVn5zy!T$j>}1oXXZj$)HsemTLY{81Wl9$+MYMESNN{0Qu1qp9LOERQZ#>djNGkvs z5C}$p3PczX3I4MX3IRhP44@Z-2)9OQmOOwN&L9!16}!(21~bxu0C_Y!a25_Vg&~+2 zuzE%oD4e6OA6fw5`%8l|acTaKw zgUy|MP6CgcjC5%@MGnz#R{kb#y36NOSF^sF*M2r2+V#8XBPXaCnW*ppH^!?jv+fHz z$L$WwDxOVG=o6PLUIx;i@KLJINV$d4$fAZpc3sne1H{Kn-(}6pYtgU3tyWnx%VaD* zP~X7H^1~28fw+fmgfT`vG*;`QJ!jKk7h7C83TbujoZ39kOn7EgZ*jf8+fO!-!inZ_F^2*iiF<&6B>>o(Wv z+ot5=Q$)pG+a+_wx2-D89xXN+d7Zy&F=yUh?WUE`s?omqtmw@NMN87DHWsh9wSO3v zBQh)CrMt(NGD--(Sodr#FCO&SwfISK=B$acGmm_}x&KZY$ZB=%PaxN(}Nv zfE< zlUu4UC-EO8t&u!CKk*8gUD%3l3Ug)JTjXzRO<73XA5K@Sf2BLBvt#**IA74NdstJM zzG6@!^}(QY}~!cD@#_!`L|v+$}X}jmMARW40jn^S+*^Pqr*+8 zB?9dJqb47V*Klh*tR)Vb9v;R~$F~@7cESmsCK76R3E4WK#Ep}}fvRx_t*>L5gDhRr xJ;bS4qN+f>I?;eIePX*R8YNVEPl*GspZ_q%11cYS1d|b=9o!i%6XZwn{2K{6N9_Or literal 2898 zcma);XE+;d`^FPuZ$ULAu}iI>8dU9BMJYAfnnkRZwrUrlqN2o(S$oqKCHCH1Rcr5u zP&I2jG+zJX{~mpg_tX1+IFI|f&+9s`5BG=PaU;-l24p~T1e*4QlID82PWUkckP4WE zrX_;Vv}>345&}&@_>UGvF$hhOe`#l5Dua^l-zj=pAXyfgLgms_L^%FQP%t1Y5OV)q z=@FtJHo#R;-P?H2on3QZuq&3VzYFs_hK%eQGZ~NvK~G8j{~O6@C;$j3C5=tE4$y%d z1iTJn{-z%9)7=$<#6&6rgDm~FKxokDY+-oHdb20jD9%%}Ze%S_z(yt#OJZbC=8MWK#-irQD zX|!`@ZlEFMMy$oU_58F>X1L$rrj6Y2HC#9bF26Z4No&II zQCvj#hm|u?fFTw7lp5M8Nm3ph;+dApv(lZnU*)Utc9B~&D$XiRs0f7ri*hw)myfW| zEutO9g(ujgeUymxTbN>>vC!KF2S(&d;w5?aVCs)%;Ikjg0w5!(U9-D{)jkeY)9nJ! z=x(7t@x-gCWPWt-m#S3Zb$Tdbz{EUplM9!DIBA&=w-)u>4<4_15<#=>U z**7f6*~`ZAr%!`_vVG=DC?0+fnh^u+(sIE`cugnq%*^A$QeAHa0&!fA!EOmv)jn;6 zjQ73vXmRG(jPs;#8I&?v6QBK}@~5p&Sf96MY7n}S9%EZI4DJJF)=d3z>7oaH9^5`( zT>Fn$#!z!^sGPYk#y)GW1BsL$g1PZKc_ucHgVBm^6NEm6cn$Gf- z=(f7b^=ABs_Ngc0q%leKj$}^5lK;S63VZq+QTc~nogE*3Q45w;D|(r#`i=`Hz0KLn zc+OvhWx~A^iFvjGj(bvNs|B;P#}{~HxJoB6*weAwpTV^smD#?0Q@U$siKfLQc2XC( zeX+e?Hmp2T@^n-66C^26cI3wFSZGRTNnD0lWi9If9e!#_$MlAgRK20)ZRFXQo?|3m zBzqE`;Kb#=ipvVxRMxmMu3(b2zpuy1Jv%3Ngj{@jN5DmxT#Yo~vay@;d5h9Shdx!* zYBc`?f6nn@uoN>m%*{;*%PHFgc#G7(J6-oZ-eki=CyNK6+az6*=)o)e#{=Ow>ym$1cEnCbg)}ky#$3`MHLLcSz@-y`y!KM6TEPptBNOf&O`M{g95S}Qb>>t@ma-Kd&OdN&%-aye-6ym z>B&%T?fg3LzZz9gU!76;vzJGvTFYsI8m-?pbfsiODFxVWS)a8do8 z13uzMK1S*eLhuhRY1pJVVH^N^fCJz;;3dEr;CLx}fbC`NfZ+XSj9rQi#A@Q~?!Y5; z6Dg07LEex>BIR#fviP4ND#}YdvtC+^WI({>LG>q21^jQo)-bAld&Dn!K4D$wHF0sq ze(zPT`~L!LRMR0td6RYl4)AQ^jxRAG7!AVJ4;2RM#_t5&!cv4v97KtmtGcw&>=rx0 z1`E?dr~;y=gfp2B+JQk1Dr)$gcdYmD(-r*cQ$0@fF7^%$CR)i?m!M)q-ZCX2Bb+UZ zk7v2r3}D_blJh2qMZ z4}4X;skGj#s*R;%ho^-0hHkw$uvnH|meEOpwg!2Rs6gl2l%;d=_ax9avxwaw{_2 zr3NvU2}h&9E}L0Ojnnz`nqb{`r8QN&cU+FbSVXdP~k_&iUc3G7MqC%zyZT>s|H_Vlke zQ1i_4pi!2V{3`H%GMs!vvtzimtJK1Qf8*Gg-Dr{QD6seIuw&H<#y7sEQleaDKQmia zO;IXUs)FhOn&1>Lpe`p9+jFF>#P`Od5nai;L6A0`cY*%CZwEaI8g7uE%CCNrUgc|5 zDz3+S9}7?6jWzUC(tjvWM@)|$YzlsU^95>^e@DcKe9_>@xh8L6aiKpFEA8A&KKj1O zYQI7yr{jc^sf{k6gCq3s_L$z<*T%$|hw_ihgFPtr*sf&4u-wQpa8!Eh#}!vp2e*fa z$_B4)p)lV_xBp^3HR73Yg?ITlV{Ual}M zpek~BTPtTzHTPpJX!Z4{mrq(Io7z_E+pOcdL1Y+0!5V~QV4^s-F#W0xY8Y6aoe%Y= zd`N64`S~Ctfu$d-SO4WujOoSq#W8tsS|l6?04^Ui?)7v}qLA`VQ_mQRqKafb`gfa+(_6U&2s>*gC!>p#F|G V^fZddr>e2uP3V&h594cIH;(?#+74W#4c!XZAh-TU9Ek1z~fb19AJjK28A8g>%8 zCmCnQlNNQ#Fhm&OP+%Qs^L2Nwu1Vf{+re%9>qz3Ki0e3icsX{8kWPZqEw9o0BR}Zg z7ax}d;iDp|m49Zp?7;}gCb;hX&vAi43g3+M*`5q#6@S2;n<}&Bq!qhEWz`+2^!kcO z3|6rw_8&=gXvAplPR+sDTxW%lyB!=x1`GnaqL%&Pa~V3!nA|+uzP;0NyCmag@~3`Q z-Ds2R;F$4cvn1nPYTqMb9|hK(>}*BWE&9NX+KKi)CVyck$_X-#NfXAIRYUa@W{Ad; zYCtyG>O-;&$WO%2gFS+Y4Y|-XtA2)zz%MY-U&UTI?;Q-~Hif0b@dyy0o3Z3ZQv@U4=S zslZZ@`G1oLi7ab>uOTg$SxQjoCj@!PKhJeo-OP__j|JhF!>w|%9h|pt<*mufa5!t% zlWhu6Z8q9uInn1qyTRs>9o%^pXY?FEbNg8y;&$uzvGFFnS!+hil%|WyxA^e?N-kLNn65LtQPN;TDtW3Ud);NXjVW;cbnTpQvzuT zkE$%@nbr(rD&egUa|dXoz=@Qc>M*|mZHpL`>9h==$=n>tI|SD_*fo`ngDL^EtTz}} zS?U9@dE4p3<&7L9<62rzZp#%Xo67Hoo`2`K*3enDwt%qs`{!r0adM<_a&wuX;umle zVFl<+Z|%(c{eFAPDg1Wkgrx=yfwf@^-DqVq5-*lp-T{XCcx(V8Tu1z{UO`cCIvTM|x9$5>WehkJq$lY;&>r z6BI0y^#n$L@q^wp%9Yrqy8S<#A>`2rq9LTeM~Pa4&Fd<{l5BDj|c zQXiDl5xdgyMiqfS8LoHkZ8T^=v6jF{d-^&en&csW*{9dOll9(XceCPLBE$B`uhG+V zs7!91?VaIED_Zvs2~X(ePj#re<)bM4^dN%O9pvPHI>%S;SAo&kx*4=3d3T%C{W*-OH@3GGU@`k+g=KmL&C@?GMkWpmNZUqp99IrMDm7`|EQ)oOrVo=U!($7Qx z?AvO8$sJLwE`uBJa#k0f>pHs0$R8fL2Dm69`e&hUkfXkcHw9O{jMaaQIgD7#FIbl< zxuY>*iMCI7)d=3Ya85YqT>v`XF63{<4vule%ntS}u>YSS!FAxThJ7I_M-E+{TE1 zM`et>uw!+F2Bs%STKj9gA^CT&I)E-wfwF%d3|~Xu*_K5QO;G|jzDzu>t>aZNUNVE~ z@&@5W)0Zfb3C-HI#CCX}<<97khts-pvmWY+Zy&@XJci*O>m+%L%)zS&g(=?7V%!)>m9J^sXH_uFN zMjD^OD~Jy4^whBzyZfV7gLCRxr&-Z+P2RR>QJC`7rH9UAGb$^k3{EZK zB{N0UKk}JC>CZHs%9`=ctf0I%3Q1z8Di{Hq`>^6SmNJC{U=c#&!)U*WGO%lZn=N~n zQkt2q3bEaGg~WB4ot_XModYr7j`W$HZB2x25+MDz*biiFFwvp7*0Dlfd1qe8)Sw#pS1tp&E z7-U_9x!_u&YMb0xCXai|=CyqVM`YP`L-jAtFox31$tFwSTHFOWr+f z?F`^2^C+DEcE&8haItXXaR4dB@Z31vhZ;<5mrEue)#p}KV!Co5_(NqMa8&JoAo(-2Bxl# zBDxRdDa#{2{o=zgK3N+AxOL2{0A*10-SETak=Bb%SCF+*OydMLA51NrBKREby)bce z^BEA>`Qh70BE2o(G`zZD2W>z?x#`WnOv(v8N-;q+smkYvJuz8meCr_VosFN`uzo*Q z2dd~t@_YmTR2eI{Hmht*D&pI}GSm|FAS&2`7dRI^)M#kI50s;sCb1UJ~ delta 2485 zcmV;m2}<^q6_gc_U4LwbJu>@IdQSoZ2mpYB1c=2iGXYBtNQHa&#QHVVVV?BXCAP&A zO2TOXBF$$Q2f+-F5{P@E$mg`DSJ2$QOv-V)tr-pLo4kQMBRZEE?`eXyH&pk{;vf{7 z`F=QfGP*sS=doKL$LhN(ahURxtM;f1`o6b`;J9*iO;gk#e1C&YRUfs|Gv0NZC$#$7 zk^UHHuW&-m;J7w|O>5;a(4AP88>L*6)CKChq18p!;#K^NfzXjwxPbyBTi~5HzV1}s z>A?beWzp;^(vnXWD;0As8%xHLIKu^z`&MY63FgV_$>0!p+?U^I;<9|gGu3v=L;=wB zhf=s9h3QNdAb)fPX~HsQv3?aBTwGZw)8lh0*!+;^%POEy&|+PzfU=iDRO;i+>nFKh3Rvm3OaoPe{m@U%QZ$5%DFX-;_T5%Z;_QaFtp;tC35B%JFW_SNXUnaD>U%)JdONm`5YO8Xh0N} z-I|MM%-R_Z;`pE}RgH`X}cQh%R~>Qk^2{AvaE_G z|LABq6GGbYUHCypTG{oH3(t5f2D}_+zP;9Iqhn)E-zLF!?w?;TrgA5Eg$M29|Luq@X+~{(K-X^mtu;BBtm+`+p0M016WJgg+SLnn#M+t z!v__JBWRh7P|dNI+2#IM(s%D{norijlkd36s$!O`@9A0R?HH7>Si9%KGH~NCZc(=m zI!iKu7ovB)?ta`@bgg{mXqyn5r2hRPZhzM8GX-j@w?p#XCF1+|0d@>X0;oqSoT69b zo2%9Djg&UfCqW-3){`j^URk$&)bN`qpTYrE6Gv%8-UAW^I%sB#xK_zgg6W%FQ>Ses zRa)F|3l^fG1kz@H{R+D2*WJE?^@>BdT$C?lYI7D!g+=m=m?`2C$&!7L7y65s`hUEP z!Kq){ndQ5GdU;oYWEN&nB0oADnrLY@N%lPv*#RUwrYxk4a(?@(t5soJ4z$By$C9eiY6!>(D;QEL3unGDLkWE3?)W??PP~J^RrQ#m-xKO4Yn;C=U%!l zCB^YAVrakZr=7yIuJJ&wP?>?{5%gu&3MxAolKnd+i&D|8lfkwbcGhrFx~@5M!gi26 zbhE}IG)|9$0b>Co>(=V)r0Yk;#k1?$iZIsMG?;&ILmc%L-FWfxVGFBN9x9 z1}24n8{Pd-mjvXEG+Wa+5aeu<=sshU>~AzjUy1?CdcZ)2c;Y1l))(77hi8$At|d- zJPD8Or_=jeA21UP%XIBlUm2A95l-j628d06edA$fI?F?)}ZctCWn7XQj~YEPtt?r4Y8a#vbjh2NJmHp{?4=#Xy^8lCJ7n=Fn;um z8GAz!89&QVNcAWo!Lol7b80~coqV{ySO(=Fp%o3#&}(=)^U*t#Spd*YcNKCo-_zrN zpH3Lt{y(JzH@j+{O5fvmp@SG8lPruS9MS(;SKXE4#%75SI!JHv^X{|iA4%Bw8A607G}7Adg`mf(S22Q zOrY6A6U7R0@!%O03>3iaU2F*ApgpP>XSLzcxXxEeTsu3@#JaqPYfhq<0_um!H5r`@ zVhH^_XTkbG7PMbsH%>@Ecw1@Cj=WcY_q&PMhEZbL)n5L;7)Ie9o5{%HZ}uvP>>YVI zWOefoyQa(4J}$uya!h<-^_ae6`Pr1Bb2H_kBR?4!s>oOdPT56akP(Ht=6!2YWUJ0w z6N{-IHxQO(Y+8L;E1HcS_;g=bTAr);?-C$9h*mHAr>oN6!DmIs-bg#$d~7BXJpLxm z=O8y9KN->85R(%MUm|q*?C^wz{fFH}NQ%)XE}6=>{fPt!)|jP>*wWXW0s;sChj_y8 diff --git a/.ci/docker-compose-file/certs/truststore.jks b/.ci/docker-compose-file/certs/truststore.jks index cd97d01588c7befd6cfbbb679a3d02623380a241..5ea593a39bb7512727b38beb655c3770ee34f4ea 100644 GIT binary patch literal 1318 zcmezO_TO6u1_mY|W(3o$xs}GAiogd~#h86OQ%RWCpdi2fRtdQg8&x4F7)PMQ9<=FDQ>VY|tU50y7 znGbs>tbM`Qxkqqka-h#5D84mg@@E zj$HH4{MQUVyvfn-Q?R_mi77YVPCm~!ahY88Oy+ZTmo%RVKhl&~@>b>Ve6Olys@Kj6 zy__=9*vtIKh2FfMfzOs!hADr{`M5V^f=N%uQO|m(Pm%kKJ49w$_I7Wbb4_;ZV%^ON zN7)TLb@y&rA#(5|>x^)nt#-0{)s0A^?x?)c&x!^tSs4X8EW%dxOLIf_1iQ< z-J6e1wp=3ihwHgYBC8jh{0CdMR}WV_H*rv$bvjZ@meJwW@f|8t_I+7A>G+ZKD>rx9 z>Afjmbnot=WwMq<)2HoU_-U1o>Nh9(_5b(mx)~dKo%2O|#Y#7Gf4^cTW<~}^{i=S+(A{r|6P zx$I%vCA(LaMVS{}78d_yCqHS6uj^c0gOK<&+YWowFO+hA7I!fBW$#Vz3Lc$%*4H#W z1uf5+e7K>1e4ladrWOAUm>%8vv!Fh*Xtr>Kz1-YTmr55dQ z=lhr4He9B(?%_MJhg+*nC%iXV{`c~?=6j7ZOiuK_JKv=q8R)_BobBPZtDQDtbB_Ji z`g7=0V)(JvhAWl_8vnf1p5m8W@^ovlh?7jVRm%j?e%9f+A{@XeSxaf)wZ&;GTkNoNl#KH^_fcxU(7g=Kj+ ziuQ2Ly=1kM<@I&eCNUnxY0>`AU%tBR{bD1}lUs@RnHrZzwQYD{_fh+@)qhSK0JC0G AbN~PV literal 1622 zcmV-c2C4Zlf(B9o0Ru3C1^fmHDuzgg_YDCD0ic2f?F51a=`ex?&LNQU)EOumwXc=d1z(ClCSwATSID2r7n1hW8Bu2?YQ! z9R>+thDZTr0|Wso1P}}uYk(Nh6jHK?jg?1k+sA-{1yD$Kl-+1u^qlu=j17ww4_OPa zu?L5ne^q+7cGq#*5Xi|&{ajOF$)WAC!KY(M0BI=UMvouRiNw(DhAB$# zA+&Q=ymGuZg$Nvghaioh+Jc16NX=0AI?+n-G*b7j4#S{B1yC^s7&^M*z+BxJ?d}B( z3-ED;r1xR|1;L9AXrw^%q2W3<)?s0JbR=dSE$}W?qhht&NdJ=ABWqKB)vR@?UMV8+^Ej(1xvd02A;okOWW4fOFuI` z9Tj<^&^}5H=vaRpF!X9%j?^6TIAeFvmJtpKQxFWB;xq*7_uBZ@{Xpd zyhkM;8T$j;$3p=j=+baW;LpN3hAw)hgQ5cnePyNdq40Le-bx``xEPR|yXrN_gV6s2 zBw?SPBx4|LhYgt|3{=ja?0Z_c`CeRp%lkP!Df7s9<#djTRZ!|H*z5|*u*fd2_8E2} zR71rLUm~sWbvTkC$v{!M+!p(D&IEI<^L(m|xk>xE%-if+E-UqgA@eKb1JIB(8{pcFoZVaW3&qKTSw<0DXz8#;v++xFEZmf;ZP_P+9~1e54nAm`%6+ zA9I16r50Hw*({R|YM7urX13%ida1BC>v*5Z5` zd`X{gNb<9WoueBPG~GP|V5(e{>vlUMTBv-3_KQ2jo~L&OdK9*_5)ZRBm;0fiPf>4O z*ZB-RTpMav7@#Px%KV2Ts#ia`$>r7wBcse$ra)&WtVs@W4Mh&N{M3V<3 ztuQzT&)SV4F7;O-Ue&ujumkp`O}q^f0UzWGh=6lX#)6vy&iUYNI#+~XguJ-l7*vC6 zLS9123Ru6<{FcC{n9(&lddcMFTf8xqdvH{k&3pe`vSq2+5;_qZ@ zg46`Riqw+$b4G8+x=lG~F)AxlR6Ypf^5&^q$M)RGd;!30B2lTJzAp|{x2Ma?c87Qy zZ!sCL+sgo`s@=v%4ahYSD03VP>nJ~RO$>y?uSlk@QruQdZhPS58R>0e82I&raEL52 z`fM5EO5MBXWI|wgh>Fdxht{FhA7I=69I`Al55Ax_%?Gyldir~i_8=M&iSjZR*EX9R zx1-_=cl6Qh#$`7LO5nT7lH!yU8%;$4b$`@-25fxGE*RN9HeCQ|jx|!qJ&ia*z>x&9 zoA}%-lzsD7k5YpenqR5+IQ1I~gbx-4uG8PmfeRRVo*Bz|fW%g& zNC9O71OfpC00bav$px}Peye0XMhtIbMcGUfw-|BX{Ma=O`Y;u3i9R6&6j{DH7Xq8& U_#+cuAFH4*5M$OX#sUH-5HWY@;{X5v diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl index 41ac1c33f..89187bf5b 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl @@ -37,6 +37,15 @@ -define(CASSA_PASSWORD, "public"). -define(BATCH_SIZE, 10). +%% cert files for client +-define(CERT_ROOT, + filename:join([emqx_common_test_helpers:proj_root(), ".ci", "docker-compose-file", "certs"]) +). + +-define(CAFILE, filename:join(?CERT_ROOT, ["ca.crt"])). +-define(CERTFILE, filename:join(?CERT_ROOT, ["client.pem"])). +-define(KEYFILE, filename:join(?CERT_ROOT, ["client.key"])). + %%------------------------------------------------------------------------------ %% CT boilerplate %%------------------------------------------------------------------------------ @@ -196,6 +205,10 @@ cassa_config(BridgeType, Config) -> " }\n" " ssl = {\n" " enable = ~w\n" + " cacertfile = \"~s\"\n" + " certfile = \"~s\"\n" + " keyfile = \"~s\"\n" + " server_name_indication = disable\n" " }\n" "}", [ @@ -208,7 +221,10 @@ cassa_config(BridgeType, Config) -> ?SQL_BRIDGE, BatchSize, QueryMode, - TlsEnabled + TlsEnabled, + ?CAFILE, + ?CERTFILE, + ?KEYFILE ] ), {Name, parse_and_check(ConfigString, BridgeType, Name)}. @@ -257,12 +273,18 @@ connect_direct_cassa(Config) -> password => ?CASSA_PASSWORD, keyspace => ?CASSA_KEYSPACE }, - SslOpts = case ?config(enable_tls, Config) of true -> Opts#{ - ssl => emqx_tls_lib:to_client_opts(#{enable => true}) + ssl => emqx_tls_lib:to_client_opts( + #{ + enable => true, + cacertfile => ?CAFILE, + certfile => ?CERTFILE, + keyfile => ?KEYFILE + } + ) }; false -> Opts @@ -272,6 +294,8 @@ connect_direct_cassa(Config) -> % These funs connect and then stop the cassandra connection connect_and_create_table(Config) -> + %% XXX: drop first + _ = connect_and_drop_table(Config), Con = connect_direct_cassa(Config), {ok, _} = ecql:query(Con, ?SQL_CREATE_TABLE), ok = ecql:close(Con). diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl index 6b7084e9e..cf25cd6d8 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl @@ -127,7 +127,6 @@ on_start( {pool_size, PoolSize} ], - %% FIXME: how to set tls options SslOpts = case maps:get(enable, SSL) of true -> From d8e6e2a1be39e9cba026dc4004b50d8236ac227a Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 17 Mar 2023 18:52:52 +0800 Subject: [PATCH 009/196] test: refine CASSANDRA_HOST name --- lib-ee/emqx_ee_connector/test/emqx_ee_connector_cassa_SUITE.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib-ee/emqx_ee_connector/test/emqx_ee_connector_cassa_SUITE.erl b/lib-ee/emqx_ee_connector/test/emqx_ee_connector_cassa_SUITE.erl index 2b4edabcc..4436038d9 100644 --- a/lib-ee/emqx_ee_connector/test/emqx_ee_connector_cassa_SUITE.erl +++ b/lib-ee/emqx_ee_connector/test/emqx_ee_connector_cassa_SUITE.erl @@ -26,7 +26,7 @@ %% Cassandra server defined at `.ci/docker-compose-file/docker-compose-cassandra-tcp.yaml` %% You can change it to `127.0.0.1`, if you run this SUITE locally --define(CASSANDRA_HOST, "cassa_tcp"). +-define(CASSANDRA_HOST, "cassandra"). -define(CASSANDRA_RESOURCE_MOD, emqx_ee_connector_cassa). %% This test SUITE requires a running cassandra instance. If you don't want to From 39f22585d00adb15e2ba59073df6c8a541e6a17d Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Sun, 19 Mar 2023 01:16:49 +0200 Subject: [PATCH 010/196] chore(mix): fix mix builds --- Makefile | 1 + apps/emqx/rebar.config | 10 +++++- apps/emqx/rebar.config.script | 22 ++++++------ build | 25 ++----------- mix.exs | 68 +++++++++++++++++++++++++++++++---- 5 files changed, 85 insertions(+), 41 deletions(-) diff --git a/Makefile b/Makefile index fe4e6fc68..babd66b85 100644 --- a/Makefile +++ b/Makefile @@ -152,6 +152,7 @@ $(PROFILES:%=clean-%): .PHONY: clean-all clean-all: @rm -f rebar.lock + @rm -rf deps @rm -rf _build .PHONY: deps-all diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 229979f6c..cbd0da109 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -59,4 +59,12 @@ {statistics, true} ]}. -{project_plugins, [erlfmt]}. +{project_plugins, [ + {erlfmt, [ + {files, [ + "{src,include,test}/*.{hrl,erl,app.src}", + "rebar.config", + "rebar.config.script" + ]} + ]} +]}. diff --git a/apps/emqx/rebar.config.script b/apps/emqx/rebar.config.script index 0827570ff..4639cc0db 100644 --- a/apps/emqx/rebar.config.script +++ b/apps/emqx/rebar.config.script @@ -27,17 +27,17 @@ Bcrypt = {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6. Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.113"}}}. Dialyzer = fun(Config) -> - {dialyzer, OldDialyzerConfig} = lists:keyfind(dialyzer, 1, Config), - {plt_extra_apps, OldExtra} = lists:keyfind(plt_extra_apps, 1, OldDialyzerConfig), - Extra = OldExtra ++ [quicer || IsQuicSupp()], - NewDialyzerConfig = [{plt_extra_apps, Extra} | OldDialyzerConfig], - lists:keystore( - dialyzer, - 1, - Config, - {dialyzer, NewDialyzerConfig} - ) - end. + {dialyzer, OldDialyzerConfig} = lists:keyfind(dialyzer, 1, Config), + {plt_extra_apps, OldExtra} = lists:keyfind(plt_extra_apps, 1, OldDialyzerConfig), + Extra = OldExtra ++ [quicer || IsQuicSupp()], + NewDialyzerConfig = [{plt_extra_apps, Extra} | OldDialyzerConfig], + lists:keystore( + dialyzer, + 1, + Config, + {dialyzer, NewDialyzerConfig} + ) +end. ExtraDeps = fun(C) -> {deps, Deps0} = lists:keyfind(deps, 1, C), diff --git a/build b/build index 76298f1ab..3c558c19a 100755 --- a/build +++ b/build @@ -147,7 +147,7 @@ make_rel() { make_elixir_rel() { ./scripts/pre-compile.sh "$PROFILE" - export_release_vars "$PROFILE" + export_elixir_release_vars "$PROFILE" # for some reason, this has to be run outside "do"... mix local.rebar --if-missing --force # shellcheck disable=SC1010 @@ -362,7 +362,7 @@ function join { # used to control the Elixir Mix Release output # see docstring in `mix.exs` -export_release_vars() { +export_elixir_release_vars() { local profile="$1" case "$profile" in emqx|emqx-enterprise) @@ -376,27 +376,6 @@ export_release_vars() { exit 1 esac export MIX_ENV="$profile" - - local erl_opts=() - - case "$(is_enterprise "$profile")" in - 'yes') - erl_opts+=( "{d, 'EMQX_RELEASE_EDITION', ee}" ) - ;; - 'no') - erl_opts+=( "{d, 'EMQX_RELEASE_EDITION', ce}" ) - ;; - esac - - # At this time, Mix provides no easy way to pass `erl_opts' to - # dependencies. The workaround is to set this variable before - # compiling the project, so that `emqx_release.erl' picks up - # `emqx_vsn' as if it was compiled by rebar3. - erl_opts+=( "{compile_info,[{emqx_vsn,\"${PKG_VSN}\"}]}" ) - erl_opts+=( "{d,snk_kind,msg}" ) - - ERL_COMPILER_OPTIONS="[$(join , "${erl_opts[@]}")]" - export ERL_COMPILER_OPTIONS } log "building artifact=$ARTIFACT for profile=$PROFILE" diff --git a/mix.exs b/mix.exs index 42354f8dc..4a16d9cc5 100644 --- a/mix.exs +++ b/mix.exs @@ -31,16 +31,17 @@ defmodule EMQXUmbrella.MixProject do def project() do profile_info = check_profile!() + version = pkg_vsn() [ app: :emqx_mix, - version: pkg_vsn(), - deps: deps(profile_info), + version: version, + deps: deps(profile_info, version), releases: releases() ] end - defp deps(profile_info) do + defp deps(profile_info, version) do # we need several overrides here because dependencies specify # other exact versions, and not ranges. [ @@ -61,7 +62,9 @@ defmodule EMQXUmbrella.MixProject do {:ecpool, github: "emqx/ecpool", tag: "0.5.3", override: true}, {:replayq, github: "emqx/replayq", tag: "0.3.7", override: true}, {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true}, - {:emqtt, github: "emqx/emqtt", tag: "1.8.5", override: true}, + # maybe forbid to fetch quicer + {:emqtt, + github: "emqx/emqtt", tag: "1.8.5", override: true, system_env: maybe_no_quic_env()}, {:rulesql, github: "emqx/rulesql", tag: "0.1.4"}, {:observer_cli, "1.7.1"}, {:system_monitor, github: "ieQu1/system_monitor", tag: "3.0.3"}, @@ -92,11 +95,15 @@ defmodule EMQXUmbrella.MixProject do {:gpb, "4.19.5", override: true, runtime: false}, {:hackney, github: "benoitc/hackney", tag: "1.18.1", override: true} ] ++ - umbrella_apps() ++ - enterprise_apps(profile_info) ++ + emqx_apps(profile_info, version) ++ enterprise_deps(profile_info) ++ bcrypt_dep() ++ jq_dep() ++ quicer_dep() end + defp emqx_apps(profile_info, version) do + apps = umbrella_apps() ++ enterprise_apps(profile_info) + set_emqx_app_system_env(apps, profile_info, version) + end + defp umbrella_apps() do "apps/*" |> Path.wildcard() @@ -145,6 +152,46 @@ defmodule EMQXUmbrella.MixProject do [] end + defp set_emqx_app_system_env(apps, profile_info, version) do + system_env = emqx_app_system_env(profile_info, version) ++ maybe_no_quic_env() + + Enum.map( + apps, + fn {app, opts} -> + {app, + Keyword.update( + opts, + :system_env, + system_env, + &Keyword.merge(&1, system_env) + )} + end + ) + end + + def emqx_app_system_env(profile_info, version) do + erlc_options(profile_info, version) + |> dump_as_erl() + |> then(&[{"ERL_COMPILER_OPTIONS", &1}]) + end + + defp erlc_options(%{edition_type: edition_type}, version) do + [ + :debug_info, + {:compile_info, [{:emqx_vsn, String.to_charlist(version)}]}, + {:d, :EMQX_RELEASE_EDITION, erlang_edition(edition_type)}, + {:d, :snk_kind, :msg} + ] + end + + def maybe_no_quic_env() do + if not enable_quicer?() do + [{"BUILD_WITHOUT_QUIC", "true"}] + else + [] + end + end + defp releases() do [ emqx: fn -> @@ -804,4 +851,13 @@ defmodule EMQXUmbrella.MixProject do |> List.first() end end + + defp dump_as_erl(term) do + term + |> then(&:io_lib.format("~0p", [&1])) + |> :erlang.iolist_to_binary() + end + + defp erlang_edition(:community), do: :ce + defp erlang_edition(:enterprise), do: :ee end From c2c9abd568b1781472b45ad97f30bde1c12bc169 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 20 Mar 2023 09:47:58 +0800 Subject: [PATCH 011/196] chore: add running ct --- lib-ee/emqx_ee_connector/docker-ct | 1 + 1 file changed, 1 insertion(+) diff --git a/lib-ee/emqx_ee_connector/docker-ct b/lib-ee/emqx_ee_connector/docker-ct index 3db090939..446c8ee4e 100644 --- a/lib-ee/emqx_ee_connector/docker-ct +++ b/lib-ee/emqx_ee_connector/docker-ct @@ -1,3 +1,4 @@ toxiproxy influxdb clickhouse +cassandra From 55f427aa4d9519cb656e670bf1cefa7deb285991 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 20 Mar 2023 09:55:25 +0800 Subject: [PATCH 012/196] chore: ignore chekcing nl-at-eof for *.jks files --- scripts/check-nl-at-eof.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/check-nl-at-eof.sh b/scripts/check-nl-at-eof.sh index 88f8f9c2e..8ca110c81 100755 --- a/scripts/check-nl-at-eof.sh +++ b/scripts/check-nl-at-eof.sh @@ -16,6 +16,9 @@ nl_at_eof() { scripts/erlfmt) return ;; + *.jks) + return + ;; esac local lastbyte lastbyte="$(tail -c 1 "$file" 2>&1)" From 377127ce72ff1b3f86ba5d64145dc49f07a6145e Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 20 Mar 2023 13:53:57 +0800 Subject: [PATCH 013/196] fix(acl): fix wrong default ACL rules --- apps/emqx_authz/etc/acl.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_authz/etc/acl.conf b/apps/emqx_authz/etc/acl.conf index d39490d46..32a693ab8 100644 --- a/apps/emqx_authz/etc/acl.conf +++ b/apps/emqx_authz/etc/acl.conf @@ -23,7 +23,7 @@ %% -type(rule() :: {permission(), who(), access(), topics()} | {permission(), all}). %%-------------------------------------------------------------------- -{allow, {username, "^dashboard?"}, subscribe, ["$SYS/#"]}. +{allow, {username, {re, "^dashboard?"}}, subscribe, ["$SYS/#"]}. {allow, {ipaddr, "127.0.0.1"}, all, ["$SYS/#", "#"]}. From 5b9867dc13cac1d917e1b4a47df17c36cf90674c Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 20 Mar 2023 14:01:05 +0800 Subject: [PATCH 014/196] chore: add changes --- changes/ce/fix-10172.en.md | 9 +++++++++ changes/ce/fix-10172.zh.md | 8 ++++++++ 2 files changed, 17 insertions(+) create mode 100644 changes/ce/fix-10172.en.md create mode 100644 changes/ce/fix-10172.zh.md diff --git a/changes/ce/fix-10172.en.md b/changes/ce/fix-10172.en.md new file mode 100644 index 000000000..14757e44d --- /dev/null +++ b/changes/ce/fix-10172.en.md @@ -0,0 +1,9 @@ +Fix the incorrect default ACL rule, which was: +``` +{allow, {username, "^dashboard?"}, subscribe, ["$SYS/#"]}. +``` + +However, it should use `{re, "^dashboard?"}` to perform a regular expression match: +``` +{allow, {username, {re,"^dashboard?"}}, subscribe ,["$SYS/#"]}. +``` diff --git a/changes/ce/fix-10172.zh.md b/changes/ce/fix-10172.zh.md new file mode 100644 index 000000000..b4a405c4c --- /dev/null +++ b/changes/ce/fix-10172.zh.md @@ -0,0 +1,8 @@ +修复错误的默认 ACL 规则，之前是： +``` +{allow, {username, "^dashboard?"}, subscribe, ["$SYS/#"]}. +``` +但执行正则表达式的匹配应该使用 `{re, "^dashboard?”}`： +``` +{allow, {username, {re, "^dashboard?"}}, subscribe, ["$SYS/#"]}. +``` From d1689f6957c81a6bf9b0615b113c6579aed9508b Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 20 Mar 2023 16:05:06 +0800 Subject: [PATCH 015/196] chore: correct api examples follow https://github.com/emqx/emqx/pull/10114 --- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl | 9 +++------ .../emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl | 2 -- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl index 9df41c81e..bcbeb8e82 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl @@ -43,9 +43,8 @@ conn_bridge_examples(Method) -> } ]. -values(get, Type) -> - maps:merge(values(post, Type), ?METRICS_EXAMPLE); -values(post, Type) -> +%% no difference in get/post/put method +values(_Method, Type) -> #{ enable => true, type => Type, @@ -66,9 +65,7 @@ values(post, Type) -> query_mode => sync, max_queue_bytes => ?DEFAULT_QUEUE_SIZE } - }; -values(put, Type) -> - values(post, Type). + }. %%-------------------------------------------------------------------- %% schema diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl index 89187bf5b..b99bd95e0 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl @@ -294,8 +294,6 @@ connect_direct_cassa(Config) -> % These funs connect and then stop the cassandra connection connect_and_create_table(Config) -> - %% XXX: drop first - _ = connect_and_drop_table(Config), Con = connect_direct_cassa(Config), {ok, _} = ecql:query(Con, ?SQL_CREATE_TABLE), ok = ecql:close(Con). From a2ee85acb8cad1b2747ffa1c1c486fb5703bc373 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 20 Mar 2023 13:26:00 +0100 Subject: [PATCH 016/196] build: add script to delete old change log files --- scripts/rel/delete-old-changelog.sh | 39 +++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100755 scripts/rel/delete-old-changelog.sh diff --git a/scripts/rel/delete-old-changelog.sh b/scripts/rel/delete-old-changelog.sh new file mode 100755 index 000000000..4b0f4db2f --- /dev/null +++ b/scripts/rel/delete-old-changelog.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +set -euo pipefail + +[ "${DEBUG:-0}" = 1 ] && set -x + +top_dir="$(git rev-parse --show-toplevel)" +prev_ce_tag="$("$top_dir"/scripts/find-prev-rel-tag.sh 'emqx')" +prev_ee_tag="$("$top_dir"/scripts/find-prev-rel-tag.sh 'emqx-enterprise')" + +## check if a file is included in the previous release +is_released() { + file="$1" + prev_tag="$2" + # check if file exists in the previous release + if git show "$prev_tag:$file" >/dev/null 2>&1; then + return 1 + else + return 0 + fi +} + +## loop over files in $top_dir/changes/ce +## and delete the ones that are included in the previous ce and ee releases +while read -r file; do + if is_released "$file" "$prev_ce_tag" && is_released "$file" "$prev_ee_tag"; then + echo "deleting $file, released in $prev_ce_tag and $prev_ee_tag" + rm -f "$file" + fi +done < <(find "$top_dir/changes/ce" -type f -name '*.md') + +## loop over files in $top_dir/changes/ee +## and delete the ones taht are included in the previous ee release +while read -r file; do + if is_released "$file" "$prev_ee_tag"; then + echo "deleting $file, released in $prev_ee_tag" + rm -f "$file" + fi +done < <(find "$top_dir/changes/ee" -type f -name '*.md') From 2c6f977cac5c2f8fe0c09d21818b7090d6656534 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 20 Mar 2023 13:32:28 +0100 Subject: [PATCH 017/196] chore: delete old changelogs --- changes/ce/feat-10019.en.md | 1 - changes/ce/feat-10019.zh.md | 1 - changes/ce/feat-10022.en.md | 1 - changes/ce/feat-10022.zh.md | 1 - changes/ce/feat-10059.en.md | 1 - changes/ce/feat-10059.zh.md | 1 - changes/ce/feat-10139.en.md | 3 --- changes/ce/feat-10139.zh.md | 3 --- changes/ce/feat-9213.en.md | 1 - changes/ce/feat-9213.zh.md | 1 - changes/ce/feat-9893.en.md | 2 -- changes/ce/feat-9893.zh.md | 2 -- changes/ce/feat-9949.en.md | 2 -- changes/ce/feat-9949.zh.md | 1 - changes/ce/feat-9986.en.md | 1 - changes/ce/feat-9986.zh.md | 1 - changes/ce/fix-10009.en.md | 1 - changes/ce/fix-10009.zh.md | 1 - changes/ce/fix-10013.en.md | 1 - changes/ce/fix-10013.zh.md | 1 - changes/ce/fix-10014.en.md | 1 - changes/ce/fix-10014.zh.md | 1 - changes/ce/fix-10015.en.md | 7 ------- changes/ce/fix-10015.zh.md | 4 ---- changes/ce/fix-10020.en.md | 1 - changes/ce/fix-10020.zh.md | 1 - changes/ce/fix-10021.en.md | 1 - changes/ce/fix-10021.zh.md | 1 - changes/ce/fix-10027.en.md | 2 -- changes/ce/fix-10027.zh.md | 2 -- changes/ce/fix-10032.en.md | 1 - changes/ce/fix-10032.zh.md | 1 - changes/ce/fix-10037.en.md | 2 -- changes/ce/fix-10037.zh.md | 2 -- changes/ce/fix-10041.en.md | 2 -- changes/ce/fix-10041.zh.md | 2 -- changes/ce/fix-10042.en.md | 5 ----- changes/ce/fix-10042.zh.md | 6 ------ changes/ce/fix-10043.en.md | 3 --- changes/ce/fix-10043.zh.md | 3 --- changes/ce/fix-10044.en.md | 1 - changes/ce/fix-10044.zh.md | 1 - changes/ce/fix-10050.en.md | 1 - changes/ce/fix-10050.zh.md | 1 - changes/ce/fix-10052.en.md | 12 ------------ changes/ce/fix-10052.zh.md | 11 ----------- changes/ce/fix-10054.en.md | 1 - changes/ce/fix-10054.zh.md | 1 - changes/ce/fix-10055.en.md | 1 - changes/ce/fix-10055.zh.md | 1 - changes/ce/fix-10056.en.md | 3 --- changes/ce/fix-10056.zh.md | 3 --- changes/ce/fix-10058.en.md | 7 ------- changes/ce/fix-10058.zh.md | 8 -------- changes/ce/fix-10066.en.md | 1 - changes/ce/fix-10066.zh.md | 1 - changes/ce/fix-10074.en.md | 1 - changes/ce/fix-10074.zh.md | 1 - changes/ce/fix-10076.en.md | 2 -- changes/ce/fix-10076.zh.md | 2 -- changes/ce/fix-10078.en.md | 2 -- changes/ce/fix-10078.zh.md | 2 -- changes/ce/fix-10079.en.md | 1 - changes/ce/fix-10079.zh.md | 2 -- changes/ce/fix-10084.en.md | 3 --- changes/ce/fix-10084.zh.md | 3 --- changes/ce/fix-10085.en.md | 1 - changes/ce/fix-10085.zh.md | 1 - changes/ce/fix-10086.en.md | 4 ---- changes/ce/fix-10086.zh.md | 3 --- changes/ce/fix-10098.en.md | 1 - changes/ce/fix-10098.zh.md | 1 - changes/ce/fix-10100.en.md | 2 -- changes/ce/fix-10100.zh.md | 2 -- changes/ce/fix-10107.en.md | 9 --------- changes/ce/fix-10107.zh.md | 8 -------- changes/ce/fix-10117.en.md | 2 -- changes/ce/fix-10118.en.md | 4 ---- changes/ce/fix-10118.zh.md | 4 ---- changes/ce/fix-10119.en.md | 1 - changes/ce/fix-10119.zh.md | 1 - changes/ce/fix-10124.en.md | 1 - changes/ce/fix-10124.zh.md | 1 - changes/ce/fix-10130.en.md | 3 --- changes/ce/fix-10130.zh.md | 3 --- changes/ce/fix-10132.en.md | 1 - changes/ce/fix-10132.zh.md | 1 - changes/ce/fix-10144.en.md | 1 - changes/ce/fix-9939.en.md | 3 --- changes/ce/fix-9939.zh.md | 2 -- changes/ce/fix-9958.en.md | 1 - changes/ce/fix-9958.zh.md | 1 - changes/ce/fix-9961.en.md | 1 - changes/ce/fix-9961.zh.md | 1 - changes/ce/fix-9974.en.md | 2 -- changes/ce/fix-9974.zh.md | 2 -- changes/ce/fix-9978.en.md | 2 -- changes/ce/fix-9978.zh.md | 2 -- changes/ce/fix-9997.en.md | 1 - changes/ce/fix-9997.zh.md | 1 - changes/ce/perf-9967.en.md | 1 - changes/ce/perf-9967.zh.md | 1 - changes/ce/perf-9998.en.md | 1 - changes/ce/perf-9998.zh.md | 1 - changes/ee/feat-10083.en.md | 1 - changes/ee/feat-10083.zh.md | 1 - changes/ee/feat-9564.en.md | 2 -- changes/ee/feat-9564.zh.md | 2 -- changes/ee/feat-9881.en.md | 4 ---- changes/ee/feat-9881.zh.md | 3 --- changes/ee/feat-9932.en.md | 1 - changes/ee/feat-9932.zh.md | 1 - changes/ee/fix-10007.en.md | 5 ----- changes/ee/fix-10007.zh.md | 3 --- changes/ee/fix-10087.en.md | 2 -- changes/ee/fix-10087.zh.md | 2 -- changes/ee/fix-10095.en.md | 3 --- changes/ee/fix-10095.zh.md | 1 - 118 files changed, 257 deletions(-) delete mode 100644 changes/ce/feat-10019.en.md delete mode 100644 changes/ce/feat-10019.zh.md delete mode 100644 changes/ce/feat-10022.en.md delete mode 100644 changes/ce/feat-10022.zh.md delete mode 100644 changes/ce/feat-10059.en.md delete mode 100644 changes/ce/feat-10059.zh.md delete mode 100644 changes/ce/feat-10139.en.md delete mode 100644 changes/ce/feat-10139.zh.md delete mode 100644 changes/ce/feat-9213.en.md delete mode 100644 changes/ce/feat-9213.zh.md delete mode 100644 changes/ce/feat-9893.en.md delete mode 100644 changes/ce/feat-9893.zh.md delete mode 100644 changes/ce/feat-9949.en.md delete mode 100644 changes/ce/feat-9949.zh.md delete mode 100644 changes/ce/feat-9986.en.md delete mode 100644 changes/ce/feat-9986.zh.md delete mode 100644 changes/ce/fix-10009.en.md delete mode 100644 changes/ce/fix-10009.zh.md delete mode 100644 changes/ce/fix-10013.en.md delete mode 100644 changes/ce/fix-10013.zh.md delete mode 100644 changes/ce/fix-10014.en.md delete mode 100644 changes/ce/fix-10014.zh.md delete mode 100644 changes/ce/fix-10015.en.md delete mode 100644 changes/ce/fix-10015.zh.md delete mode 100644 changes/ce/fix-10020.en.md delete mode 100644 changes/ce/fix-10020.zh.md delete mode 100644 changes/ce/fix-10021.en.md delete mode 100644 changes/ce/fix-10021.zh.md delete mode 100644 changes/ce/fix-10027.en.md delete mode 100644 changes/ce/fix-10027.zh.md delete mode 100644 changes/ce/fix-10032.en.md delete mode 100644 changes/ce/fix-10032.zh.md delete mode 100644 changes/ce/fix-10037.en.md delete mode 100644 changes/ce/fix-10037.zh.md delete mode 100644 changes/ce/fix-10041.en.md delete mode 100644 changes/ce/fix-10041.zh.md delete mode 100644 changes/ce/fix-10042.en.md delete mode 100644 changes/ce/fix-10042.zh.md delete mode 100644 changes/ce/fix-10043.en.md delete mode 100644 changes/ce/fix-10043.zh.md delete mode 100644 changes/ce/fix-10044.en.md delete mode 100644 changes/ce/fix-10044.zh.md delete mode 100644 changes/ce/fix-10050.en.md delete mode 100644 changes/ce/fix-10050.zh.md delete mode 100644 changes/ce/fix-10052.en.md delete mode 100644 changes/ce/fix-10052.zh.md delete mode 100644 changes/ce/fix-10054.en.md delete mode 100644 changes/ce/fix-10054.zh.md delete mode 100644 changes/ce/fix-10055.en.md delete mode 100644 changes/ce/fix-10055.zh.md delete mode 100644 changes/ce/fix-10056.en.md delete mode 100644 changes/ce/fix-10056.zh.md delete mode 100644 changes/ce/fix-10058.en.md delete mode 100644 changes/ce/fix-10058.zh.md delete mode 100644 changes/ce/fix-10066.en.md delete mode 100644 changes/ce/fix-10066.zh.md delete mode 100644 changes/ce/fix-10074.en.md delete mode 100644 changes/ce/fix-10074.zh.md delete mode 100644 changes/ce/fix-10076.en.md delete mode 100644 changes/ce/fix-10076.zh.md delete mode 100644 changes/ce/fix-10078.en.md delete mode 100644 changes/ce/fix-10078.zh.md delete mode 100644 changes/ce/fix-10079.en.md delete mode 100644 changes/ce/fix-10079.zh.md delete mode 100644 changes/ce/fix-10084.en.md delete mode 100644 changes/ce/fix-10084.zh.md delete mode 100644 changes/ce/fix-10085.en.md delete mode 100644 changes/ce/fix-10085.zh.md delete mode 100644 changes/ce/fix-10086.en.md delete mode 100644 changes/ce/fix-10086.zh.md delete mode 100644 changes/ce/fix-10098.en.md delete mode 100644 changes/ce/fix-10098.zh.md delete mode 100644 changes/ce/fix-10100.en.md delete mode 100644 changes/ce/fix-10100.zh.md delete mode 100644 changes/ce/fix-10107.en.md delete mode 100644 changes/ce/fix-10107.zh.md delete mode 100644 changes/ce/fix-10117.en.md delete mode 100644 changes/ce/fix-10118.en.md delete mode 100644 changes/ce/fix-10118.zh.md delete mode 100644 changes/ce/fix-10119.en.md delete mode 100644 changes/ce/fix-10119.zh.md delete mode 100644 changes/ce/fix-10124.en.md delete mode 100644 changes/ce/fix-10124.zh.md delete mode 100644 changes/ce/fix-10130.en.md delete mode 100644 changes/ce/fix-10130.zh.md delete mode 100644 changes/ce/fix-10132.en.md delete mode 100644 changes/ce/fix-10132.zh.md delete mode 100644 changes/ce/fix-10144.en.md delete mode 100644 changes/ce/fix-9939.en.md delete mode 100644 changes/ce/fix-9939.zh.md delete mode 100644 changes/ce/fix-9958.en.md delete mode 100644 changes/ce/fix-9958.zh.md delete mode 100644 changes/ce/fix-9961.en.md delete mode 100644 changes/ce/fix-9961.zh.md delete mode 100644 changes/ce/fix-9974.en.md delete mode 100644 changes/ce/fix-9974.zh.md delete mode 100644 changes/ce/fix-9978.en.md delete mode 100644 changes/ce/fix-9978.zh.md delete mode 100644 changes/ce/fix-9997.en.md delete mode 100644 changes/ce/fix-9997.zh.md delete mode 100644 changes/ce/perf-9967.en.md delete mode 100644 changes/ce/perf-9967.zh.md delete mode 100644 changes/ce/perf-9998.en.md delete mode 100644 changes/ce/perf-9998.zh.md delete mode 100644 changes/ee/feat-10083.en.md delete mode 100644 changes/ee/feat-10083.zh.md delete mode 100644 changes/ee/feat-9564.en.md delete mode 100644 changes/ee/feat-9564.zh.md delete mode 100644 changes/ee/feat-9881.en.md delete mode 100644 changes/ee/feat-9881.zh.md delete mode 100644 changes/ee/feat-9932.en.md delete mode 100644 changes/ee/feat-9932.zh.md delete mode 100644 changes/ee/fix-10007.en.md delete mode 100644 changes/ee/fix-10007.zh.md delete mode 100644 changes/ee/fix-10087.en.md delete mode 100644 changes/ee/fix-10087.zh.md delete mode 100644 changes/ee/fix-10095.en.md delete mode 100644 changes/ee/fix-10095.zh.md diff --git a/changes/ce/feat-10019.en.md b/changes/ce/feat-10019.en.md deleted file mode 100644 index b6cc0381c..000000000 --- a/changes/ce/feat-10019.en.md +++ /dev/null @@ -1 +0,0 @@ -Add low level tuning settings for QUIC listeners. diff --git a/changes/ce/feat-10019.zh.md b/changes/ce/feat-10019.zh.md deleted file mode 100644 index b0eb2a673..000000000 --- a/changes/ce/feat-10019.zh.md +++ /dev/null @@ -1 +0,0 @@ -为 QUIC 监听器添加更多底层调优选项。 diff --git a/changes/ce/feat-10022.en.md b/changes/ce/feat-10022.en.md deleted file mode 100644 index 61d027aa2..000000000 --- a/changes/ce/feat-10022.en.md +++ /dev/null @@ -1 +0,0 @@ -Start releasing Rocky Linux 9 (compatible with Enterprise Linux 9) and MacOS 12 packages diff --git a/changes/ce/feat-10022.zh.md b/changes/ce/feat-10022.zh.md deleted file mode 100644 index 970704f55..000000000 --- a/changes/ce/feat-10022.zh.md +++ /dev/null @@ -1 +0,0 @@ -开始发布Rocky Linux 9（与Enterprise Linux 9兼容）和MacOS 12软件包。 diff --git a/changes/ce/feat-10059.en.md b/changes/ce/feat-10059.en.md deleted file mode 100644 index 2c4de015c..000000000 --- a/changes/ce/feat-10059.en.md +++ /dev/null @@ -1 +0,0 @@ -Errors returned by rule engine API are formatted in a more human readable way rather than dumping the raw error including the stacktrace. diff --git a/changes/ce/feat-10059.zh.md b/changes/ce/feat-10059.zh.md deleted file mode 100644 index 99f8fe8ee..000000000 --- a/changes/ce/feat-10059.zh.md +++ /dev/null @@ -1 +0,0 @@ -规则引擎 API 返回用户可读的错误信息而不是原始的栈追踪信息。 diff --git a/changes/ce/feat-10139.en.md b/changes/ce/feat-10139.en.md deleted file mode 100644 index 3c49ceac3..000000000 --- a/changes/ce/feat-10139.en.md +++ /dev/null @@ -1,3 +0,0 @@ -Add `extraVolumeMounts` to EMQX Helm Chart, it will have the ability to mount the user-own files into the EMQX instance, for example, ACL rule files as mentioned in [#9052](https://github.com/emqx/emqx/issues/9052) - -Done of [#10116](https://github.com/emqx/emqx/issues/10116) diff --git a/changes/ce/feat-10139.zh.md b/changes/ce/feat-10139.zh.md deleted file mode 100644 index f7920a127..000000000 --- a/changes/ce/feat-10139.zh.md +++ /dev/null @@ -1,3 +0,0 @@ -将 `extraVolumeMounts` 添加到 EMQX Helm Chart 中，它将能够挂载用户自己的文件到 EMQX 实例中，例如在 [#9052](https://github.com/emqx/emqx/issues/9052) 中提到的 ACL 规则文件。 - -完成了 [#10116](https://github.com/emqx/emqx/issues/10116) diff --git a/changes/ce/feat-9213.en.md b/changes/ce/feat-9213.en.md deleted file mode 100644 index 3266ed836..000000000 --- a/changes/ce/feat-9213.en.md +++ /dev/null @@ -1 +0,0 @@ -Add pod disruption budget to helm chart diff --git a/changes/ce/feat-9213.zh.md b/changes/ce/feat-9213.zh.md deleted file mode 100644 index 66cb2693e..000000000 --- a/changes/ce/feat-9213.zh.md +++ /dev/null @@ -1 +0,0 @@ -在 Helm chart 中添加干扰预算 (disruption budget)。 diff --git a/changes/ce/feat-9893.en.md b/changes/ce/feat-9893.en.md deleted file mode 100644 index 343c3794f..000000000 --- a/changes/ce/feat-9893.en.md +++ /dev/null @@ -1,2 +0,0 @@ -When connecting with the flag `clean_start=false`, EMQX will filter out messages that published by banned clients. -Previously, the messages sent by banned clients may still be delivered to subscribers in this scenario. diff --git a/changes/ce/feat-9893.zh.md b/changes/ce/feat-9893.zh.md deleted file mode 100644 index 426439c3e..000000000 --- a/changes/ce/feat-9893.zh.md +++ /dev/null @@ -1,2 +0,0 @@ -当使用 `clean_start=false` 标志连接时，EMQX 将会从消息队列中过滤出被封禁客户端发出的消息，使它们不能被下发给订阅者。 -此前被封禁客户端发出的消息仍可能在这一场景下被下发给订阅者。 diff --git a/changes/ce/feat-9949.en.md b/changes/ce/feat-9949.en.md deleted file mode 100644 index 3ed9c30b2..000000000 --- a/changes/ce/feat-9949.en.md +++ /dev/null @@ -1,2 +0,0 @@ -QUIC transport Multistreams support and QUIC TLS cacert support. - diff --git a/changes/ce/feat-9949.zh.md b/changes/ce/feat-9949.zh.md deleted file mode 100644 index 6efabac3f..000000000 --- a/changes/ce/feat-9949.zh.md +++ /dev/null @@ -1 +0,0 @@ -QUIC 传输多流支持和 QUIC TLS cacert 支持。 diff --git a/changes/ce/feat-9986.en.md b/changes/ce/feat-9986.en.md deleted file mode 100644 index ee7a6be71..000000000 --- a/changes/ce/feat-9986.en.md +++ /dev/null @@ -1 +0,0 @@ -For helm charts, add MQTT ingress bridge; and removed stale `mgmt` references. diff --git a/changes/ce/feat-9986.zh.md b/changes/ce/feat-9986.zh.md deleted file mode 100644 index a7f418587..000000000 --- a/changes/ce/feat-9986.zh.md +++ /dev/null @@ -1 +0,0 @@ -在 helm chart 中新增了 MQTT 桥接 ingress 的配置参数；并删除了旧版本遗留的 `mgmt` 配置。 diff --git a/changes/ce/fix-10009.en.md b/changes/ce/fix-10009.en.md deleted file mode 100644 index 37f33a958..000000000 --- a/changes/ce/fix-10009.en.md +++ /dev/null @@ -1 +0,0 @@ -Validate `bytes` param to `GET /trace/:name/log` to not exceed signed 32bit integer. diff --git a/changes/ce/fix-10009.zh.md b/changes/ce/fix-10009.zh.md deleted file mode 100644 index bb55ea5b9..000000000 --- a/changes/ce/fix-10009.zh.md +++ /dev/null @@ -1 +0,0 @@ -验证 `GET /trace/:name/log` 的 `bytes` 参数，使其不超过有符号的32位整数。 diff --git a/changes/ce/fix-10013.en.md b/changes/ce/fix-10013.en.md deleted file mode 100644 index ed7fa21eb..000000000 --- a/changes/ce/fix-10013.en.md +++ /dev/null @@ -1 +0,0 @@ -Fix return type structure for error case in API schema for `/gateways/:name/clients`. diff --git a/changes/ce/fix-10013.zh.md b/changes/ce/fix-10013.zh.md deleted file mode 100644 index 171b79538..000000000 --- a/changes/ce/fix-10013.zh.md +++ /dev/null @@ -1 +0,0 @@ -修复 API `/gateways/:name/clients` 返回值的类型结构错误。 diff --git a/changes/ce/fix-10014.en.md b/changes/ce/fix-10014.en.md deleted file mode 100644 index d52452bf9..000000000 --- a/changes/ce/fix-10014.en.md +++ /dev/null @@ -1 +0,0 @@ -In dashboard API for `/monitor(_current)/nodes/:node` return `404` instead of `400` if node does not exist. diff --git a/changes/ce/fix-10014.zh.md b/changes/ce/fix-10014.zh.md deleted file mode 100644 index 5e6a1660f..000000000 --- a/changes/ce/fix-10014.zh.md +++ /dev/null @@ -1 +0,0 @@ -如果 API 查询的节点不存在，将会返回 404 而不再是 400。 diff --git a/changes/ce/fix-10015.en.md b/changes/ce/fix-10015.en.md deleted file mode 100644 index 5727a52cd..000000000 --- a/changes/ce/fix-10015.en.md +++ /dev/null @@ -1,7 +0,0 @@ -To prevent errors caused by an incorrect EMQX node cookie provided from an environment variable, -we have implemented a fail-fast mechanism. -Previously, when an incorrect cookie was provided, the command would still attempt to ping the node, -leading to the error message 'Node xxx not responding to pings'. -With the new implementation, if a mismatched cookie is detected, -a message will be logged to indicate that the cookie is incorrect, -and the command will terminate with an error code of 1 without trying to ping the node. diff --git a/changes/ce/fix-10015.zh.md b/changes/ce/fix-10015.zh.md deleted file mode 100644 index 0f58fa99c..000000000 --- a/changes/ce/fix-10015.zh.md +++ /dev/null @@ -1,4 +0,0 @@ -在 cookie 给错时，快速失败。 -在此修复前，即使 cookie 配置错误，emqx 命令仍然会尝试去 ping EMQX 节点， -并得到一个 "Node xxx not responding to pings" 的错误。 -修复后，如果发现 cookie 不一致，立即打印不一致的错误信息并退出。 diff --git a/changes/ce/fix-10020.en.md b/changes/ce/fix-10020.en.md deleted file mode 100644 index 73615804b..000000000 --- a/changes/ce/fix-10020.en.md +++ /dev/null @@ -1 +0,0 @@ -Fix bridge metrics when running in async mode with batching enabled (`batch_size` > 1). diff --git a/changes/ce/fix-10020.zh.md b/changes/ce/fix-10020.zh.md deleted file mode 100644 index 2fce853e3..000000000 --- a/changes/ce/fix-10020.zh.md +++ /dev/null @@ -1 +0,0 @@ -修复使用异步和批量配置的桥接计数不准确的问题。 diff --git a/changes/ce/fix-10021.en.md b/changes/ce/fix-10021.en.md deleted file mode 100644 index 28302da70..000000000 --- a/changes/ce/fix-10021.en.md +++ /dev/null @@ -1 +0,0 @@ -Fix error message when the target node of `emqx_ctl cluster join` command is not running. diff --git a/changes/ce/fix-10021.zh.md b/changes/ce/fix-10021.zh.md deleted file mode 100644 index 6df64b76d..000000000 --- a/changes/ce/fix-10021.zh.md +++ /dev/null @@ -1 +0,0 @@ -修正当`emqx_ctl cluster join`命令的目标节点未运行时的错误信息。 diff --git a/changes/ce/fix-10027.en.md b/changes/ce/fix-10027.en.md deleted file mode 100644 index 531da1c50..000000000 --- a/changes/ce/fix-10027.en.md +++ /dev/null @@ -1,2 +0,0 @@ -Allow setting node name from `EMQX_NODE__NAME` when running in docker. -Prior to this fix, only `EMQX_NODE_NAME` is allowed. diff --git a/changes/ce/fix-10027.zh.md b/changes/ce/fix-10027.zh.md deleted file mode 100644 index ee7055d6c..000000000 --- a/changes/ce/fix-10027.zh.md +++ /dev/null @@ -1,2 +0,0 @@ -在 docker 中启动时，允许使用 `EMQX_NODE__NAME` 环境变量来配置节点名。 -在此修复前，只能使 `EMQX_NODE_NAME`。 diff --git a/changes/ce/fix-10032.en.md b/changes/ce/fix-10032.en.md deleted file mode 100644 index bd730c96c..000000000 --- a/changes/ce/fix-10032.en.md +++ /dev/null @@ -1 +0,0 @@ -When resources on some nodes in the cluster are still in the 'initializing/connecting' state, the `bridges/` API will crash due to missing Metrics information for those resources. This fix will ignore resources that do not have Metrics information. diff --git a/changes/ce/fix-10032.zh.md b/changes/ce/fix-10032.zh.md deleted file mode 100644 index fc1fb38b6..000000000 --- a/changes/ce/fix-10032.zh.md +++ /dev/null @@ -1 +0,0 @@ -当集群中某些节点上的资源仍处于 '初始化/连接中' 状态时，`bridges/` API 将由于缺少这些资源的 Metrics 信息而崩溃。此修复后将忽略没有 Metrics 信息的资源。 diff --git a/changes/ce/fix-10037.en.md b/changes/ce/fix-10037.en.md deleted file mode 100644 index 73c92d69d..000000000 --- a/changes/ce/fix-10037.en.md +++ /dev/null @@ -1,2 +0,0 @@ -Fix Swagger API doc rendering crash. -In version 5.0.18, a bug was introduced that resulted in duplicated field names in the configuration schema. This, in turn, caused the Swagger schema generated to become invalid. diff --git a/changes/ce/fix-10037.zh.md b/changes/ce/fix-10037.zh.md deleted file mode 100644 index 5bd447c1f..000000000 --- a/changes/ce/fix-10037.zh.md +++ /dev/null @@ -1,2 +0,0 @@ -修复 Swagger API 文档渲染崩溃。 -在版本 5.0.18 中，引入了一个错误，导致配置 schema 中出现了重复的配置名称，进而导致生成了无效的 Swagger spec。 diff --git a/changes/ce/fix-10041.en.md b/changes/ce/fix-10041.en.md deleted file mode 100644 index c1aff24c2..000000000 --- a/changes/ce/fix-10041.en.md +++ /dev/null @@ -1,2 +0,0 @@ -For influxdb bridge, added integer value placeholder annotation hint to `write_syntax` documentation. -Also supported setting a constant value for the `timestamp` field. diff --git a/changes/ce/fix-10041.zh.md b/changes/ce/fix-10041.zh.md deleted file mode 100644 index d197ea81f..000000000 --- a/changes/ce/fix-10041.zh.md +++ /dev/null @@ -1,2 +0,0 @@ -为 influxdb 桥接的配置项 `write_syntax` 描述文档增加了类型标识符的提醒。 -另外在配置中支持 `timestamp` 使用一个常量。 diff --git a/changes/ce/fix-10042.en.md b/changes/ce/fix-10042.en.md deleted file mode 100644 index af9213c06..000000000 --- a/changes/ce/fix-10042.en.md +++ /dev/null @@ -1,5 +0,0 @@ -Improve behavior of the `replicant` nodes when the `core` cluster becomes partitioned (for example when a core node leaves the cluster). -Previously, the replicant nodes were unable to rebalance connections to the core nodes, until the core cluster became whole again. -This was indicated by the error messages: `[error] line: 182, mfa: mria_lb:list_core_nodes/1, msg: mria_lb_core_discovery divergent cluster`. - -[Mria PR](https://github.com/emqx/mria/pull/123/files) diff --git a/changes/ce/fix-10042.zh.md b/changes/ce/fix-10042.zh.md deleted file mode 100644 index 80db204e2..000000000 --- a/changes/ce/fix-10042.zh.md +++ /dev/null @@ -1,6 +0,0 @@ -改进 `core` 集群被分割时 `replicant`节点的行为。 -修复前，如果 `core` 集群分裂成两个小集群（例如一个节点离开集群）时，`replicant` 节点无法重新平衡与核心节点的连接，直到核心集群再次变得完整。 -这种个问题会导致 replicant 节点出现如下日志： -`[error] line: 182, mfa: mria_lb:list_core_nodes/1, msg: mria_lb_core_discovery divergent cluster`。 - -[Mria PR](https://github.com/emqx/mria/pull/123/files) diff --git a/changes/ce/fix-10043.en.md b/changes/ce/fix-10043.en.md deleted file mode 100644 index 4fd46cb4e..000000000 --- a/changes/ce/fix-10043.en.md +++ /dev/null @@ -1,3 +0,0 @@ -Fixed two bugs introduced in v5.0.18. -* The environment varialbe `SSL_DIST_OPTFILE` was not set correctly for non-boot commands. -* When cookie is overridden from environment variable, EMQX node is unable to start. diff --git a/changes/ce/fix-10043.zh.md b/changes/ce/fix-10043.zh.md deleted file mode 100644 index 6b150f6fb..000000000 --- a/changes/ce/fix-10043.zh.md +++ /dev/null @@ -1,3 +0,0 @@ -修复 v5.0.18 引入的 2 个bug。 -* 环境变量 `SSL_DIST_OPTFILE` 的值设置错误导致节点无法为 Erlang distribution 启用 SSL。 -* 当节点的 cookie 从环境变量重载 （而不是设置在配置文件中时），节点无法启动的问题。 diff --git a/changes/ce/fix-10044.en.md b/changes/ce/fix-10044.en.md deleted file mode 100644 index 00668c5cb..000000000 --- a/changes/ce/fix-10044.en.md +++ /dev/null @@ -1 +0,0 @@ -Fix node information formatter for stopped nodes in the cluster. The bug was introduced by v5.0.18. diff --git a/changes/ce/fix-10044.zh.md b/changes/ce/fix-10044.zh.md deleted file mode 100644 index 72759d707..000000000 --- a/changes/ce/fix-10044.zh.md +++ /dev/null @@ -1 +0,0 @@ -修复集群中已停止节点的信息序列化问题，该错误由 v5.0.18 引入。 diff --git a/changes/ce/fix-10050.en.md b/changes/ce/fix-10050.en.md deleted file mode 100644 index c225c380d..000000000 --- a/changes/ce/fix-10050.en.md +++ /dev/null @@ -1 +0,0 @@ -Ensure Bridge API returns `404` status code consistently for resources that don't exist. diff --git a/changes/ce/fix-10050.zh.md b/changes/ce/fix-10050.zh.md deleted file mode 100644 index d7faf9434..000000000 --- a/changes/ce/fix-10050.zh.md +++ /dev/null @@ -1 +0,0 @@ -确保 Bridge API 对不存在的资源一致返回 `404` 状态代码。 diff --git a/changes/ce/fix-10052.en.md b/changes/ce/fix-10052.en.md deleted file mode 100644 index f83c4d40c..000000000 --- a/changes/ce/fix-10052.en.md +++ /dev/null @@ -1,12 +0,0 @@ -Improve daemon mode startup failure logs. - -Before this change, it was difficult for users to understand the reason for EMQX 'start' command failed to boot the node. -The only information they received was that the node did not start within the expected time frame, -and they were instructed to boot the node with 'console' command in the hope of obtaining some logs. -However, the node might actually be running, which could cause 'console' mode to fail for a different reason. - -With this new change, when daemon mode fails to boot, a diagnosis is issued. Here are the possible scenarios: - -* If the node cannot be found from `ps -ef`, the user is instructed to find information in log files `erlang.log.*`. -* If the node is found to be running but not responding to pings, the user is advised to check if the host name is resolvable and reachable. -* If the node is responding to pings, but the EMQX app is not running, it is likely a bug. In this case, the user is advised to report a Github issue. diff --git a/changes/ce/fix-10052.zh.md b/changes/ce/fix-10052.zh.md deleted file mode 100644 index 1c2eff342..000000000 --- a/changes/ce/fix-10052.zh.md +++ /dev/null @@ -1,11 +0,0 @@ -优化 EMQX daemon 模式启动启动失败的日志。 - -在进行此更改之前，当 EMQX 用 `start` 命令启动失败时，用户很难理解出错的原因。 -所知道的仅仅是节点未能在预期时间内启动，然后被指示以 `console` 式引导节点以获取一些日志。 -然而，节点实际上可能正在运行，这可能会导致 `console` 模式因不同的原因而失败。 - -此次修复后，启动脚本会发出诊断： - -* 如果无法从 `ps -ef` 中找到节点，则指示用户在 `erlang.log.*` 中查找信息。 -* 如果发现节点正在运行但不响应 ping，则建议用户检查节点主机名是否有效并可达。 -* 如果节点响应 ping 但 EMQX 应用程序未运行，则很可能是一个错误。在这种情况下，建议用户报告一个Github issue。 diff --git a/changes/ce/fix-10054.en.md b/changes/ce/fix-10054.en.md deleted file mode 100644 index 5efa73314..000000000 --- a/changes/ce/fix-10054.en.md +++ /dev/null @@ -1 +0,0 @@ -Fix the problem that the obfuscated password is used when using the `/bridges_probe` API to test the connection in Data-Bridge. diff --git a/changes/ce/fix-10054.zh.md b/changes/ce/fix-10054.zh.md deleted file mode 100644 index 45a80dc45..000000000 --- a/changes/ce/fix-10054.zh.md +++ /dev/null @@ -1 +0,0 @@ -修复数据桥接中使用 `/bridges_probe` API 进行测试连接时密码被混淆的问题。 diff --git a/changes/ce/fix-10055.en.md b/changes/ce/fix-10055.en.md deleted file mode 100644 index 4ffaae195..000000000 --- a/changes/ce/fix-10055.en.md +++ /dev/null @@ -1 +0,0 @@ -Fix `mqtt.max_awaiting_rel` change does not work. diff --git a/changes/ce/fix-10055.zh.md b/changes/ce/fix-10055.zh.md deleted file mode 100644 index 4da371c51..000000000 --- a/changes/ce/fix-10055.zh.md +++ /dev/null @@ -1 +0,0 @@ -修复 `mqtt.max_awaiting_rel` 更新不生效问题。 diff --git a/changes/ce/fix-10056.en.md b/changes/ce/fix-10056.en.md deleted file mode 100644 index 55449294d..000000000 --- a/changes/ce/fix-10056.en.md +++ /dev/null @@ -1,3 +0,0 @@ -Fix `/bridges` API status code. -- Return `400` instead of `403` in case of removing a data bridge that is dependent on an active rule. -- Return `400` instead of `403` in case of calling operations (start|stop|restart) when Data-Bridging is not enabled. diff --git a/changes/ce/fix-10056.zh.md b/changes/ce/fix-10056.zh.md deleted file mode 100644 index ec5982137..000000000 --- a/changes/ce/fix-10056.zh.md +++ /dev/null @@ -1,3 +0,0 @@ -修复 `/bridges` API 的 HTTP 状态码。 -- 当删除被活动中的规则依赖的数据桥接时，将返回 `400` 而不是 `403` 。 -- 当数据桥接未启用时，调用操作（启动|停止|重启）将返回 `400` 而不是 `403`。 diff --git a/changes/ce/fix-10058.en.md b/changes/ce/fix-10058.en.md deleted file mode 100644 index 337ac5d47..000000000 --- a/changes/ce/fix-10058.en.md +++ /dev/null @@ -1,7 +0,0 @@ -Deprecate unused QUIC TLS options. -Only following TLS options are kept for the QUIC listeners: - -- cacertfile -- certfile -- keyfile -- verify diff --git a/changes/ce/fix-10058.zh.md b/changes/ce/fix-10058.zh.md deleted file mode 100644 index d1dea37c3..000000000 --- a/changes/ce/fix-10058.zh.md +++ /dev/null @@ -1,8 +0,0 @@ -废弃未使用的 QUIC TLS 选项。 -QUIC 监听器只保留以下 TLS 选项: - -- cacertfile -- certfile -- keyfile -- verify - diff --git a/changes/ce/fix-10066.en.md b/changes/ce/fix-10066.en.md deleted file mode 100644 index 87e253aca..000000000 --- a/changes/ce/fix-10066.en.md +++ /dev/null @@ -1 +0,0 @@ -Improve error messages for `/briges_probe` and `[/node/:node]/bridges/:id/:operation` API calls to make them more readable. And set HTTP status code to `400` instead of `500`. diff --git a/changes/ce/fix-10066.zh.md b/changes/ce/fix-10066.zh.md deleted file mode 100644 index e5e3c2113..000000000 --- a/changes/ce/fix-10066.zh.md +++ /dev/null @@ -1 +0,0 @@ -改进 `/briges_probe` 和 `[/node/:node]/bridges/:id/:operation` API 调用的错误信息，使之更加易读。并将 HTTP 状态代码设置为 `400` 而不是 `500`。 diff --git a/changes/ce/fix-10074.en.md b/changes/ce/fix-10074.en.md deleted file mode 100644 index 49c52b948..000000000 --- a/changes/ce/fix-10074.en.md +++ /dev/null @@ -1 +0,0 @@ -Check if type in `PUT /authorization/sources/:type` matches `type` given in body of request. diff --git a/changes/ce/fix-10074.zh.md b/changes/ce/fix-10074.zh.md deleted file mode 100644 index 930840cdf..000000000 --- a/changes/ce/fix-10074.zh.md +++ /dev/null @@ -1 +0,0 @@ -检查 `PUT /authorization/sources/:type` 中的类型是否与请求正文中的 `type` 相符。 diff --git a/changes/ce/fix-10076.en.md b/changes/ce/fix-10076.en.md deleted file mode 100644 index 5bbbffa32..000000000 --- a/changes/ce/fix-10076.en.md +++ /dev/null @@ -1,2 +0,0 @@ -Fix webhook bridge error handling: connection timeout should be a retriable error. -Prior to this fix, connection timeout was classified as unrecoverable error and led to request being dropped. diff --git a/changes/ce/fix-10076.zh.md b/changes/ce/fix-10076.zh.md deleted file mode 100644 index 516345f92..000000000 --- a/changes/ce/fix-10076.zh.md +++ /dev/null @@ -1,2 +0,0 @@ -修复 HTTP 桥接的一个异常处理：连接超时错误发生后，发生错误的请求可以被重试。 -在此修复前，连接超时后，被当作不可重试类型的错误处理，导致请求被丢弃。 diff --git a/changes/ce/fix-10078.en.md b/changes/ce/fix-10078.en.md deleted file mode 100644 index afb7bcbe0..000000000 --- a/changes/ce/fix-10078.en.md +++ /dev/null @@ -1,2 +0,0 @@ -Fix an issue that invalid QUIC listener setting could casue segfault. - diff --git a/changes/ce/fix-10078.zh.md b/changes/ce/fix-10078.zh.md deleted file mode 100644 index 47a774d1e..000000000 --- a/changes/ce/fix-10078.zh.md +++ /dev/null @@ -1,2 +0,0 @@ -修复了无效的 QUIC 监听器设置可能导致 segfault 的问题。 - diff --git a/changes/ce/fix-10079.en.md b/changes/ce/fix-10079.en.md deleted file mode 100644 index 440351753..000000000 --- a/changes/ce/fix-10079.en.md +++ /dev/null @@ -1 +0,0 @@ -Fix description of `shared_subscription_strategy`. diff --git a/changes/ce/fix-10079.zh.md b/changes/ce/fix-10079.zh.md deleted file mode 100644 index ca2ab9173..000000000 --- a/changes/ce/fix-10079.zh.md +++ /dev/null @@ -1,2 +0,0 @@ -修正对 `shared_subscription_strategy` 的描述。 - diff --git a/changes/ce/fix-10084.en.md b/changes/ce/fix-10084.en.md deleted file mode 100644 index 90da7d660..000000000 --- a/changes/ce/fix-10084.en.md +++ /dev/null @@ -1,3 +0,0 @@ -Fix problem when joining core nodes running different EMQX versions into a cluster. - -[Mria PR](https://github.com/emqx/mria/pull/127) diff --git a/changes/ce/fix-10084.zh.md b/changes/ce/fix-10084.zh.md deleted file mode 100644 index dd44533cf..000000000 --- a/changes/ce/fix-10084.zh.md +++ /dev/null @@ -1,3 +0,0 @@ -修正将运行不同 EMQX 版本的核心节点加入集群的问题。 - -[Mria PR](https://github.com/emqx/mria/pull/127) diff --git a/changes/ce/fix-10085.en.md b/changes/ce/fix-10085.en.md deleted file mode 100644 index e539a04b4..000000000 --- a/changes/ce/fix-10085.en.md +++ /dev/null @@ -1 +0,0 @@ -Consistently return `404` for all requests on non existent source in `/authorization/sources/:source[/*]`. diff --git a/changes/ce/fix-10085.zh.md b/changes/ce/fix-10085.zh.md deleted file mode 100644 index 059680efa..000000000 --- a/changes/ce/fix-10085.zh.md +++ /dev/null @@ -1 +0,0 @@ -如果向 `/authorization/sources/:source[/*]` 请求的 `source` 不存在，将一致地返回 `404`。 diff --git a/changes/ce/fix-10086.en.md b/changes/ce/fix-10086.en.md deleted file mode 100644 index d337a57c7..000000000 --- a/changes/ce/fix-10086.en.md +++ /dev/null @@ -1,4 +0,0 @@ -Upgrade HTTP client ehttpc to `0.4.7`. -Prior to this upgrade, HTTP clients for authentication, authorization and webhook may crash -if `Body` is empty but `Content-Type` HTTP header is set. -For more details see [ehttpc PR#44](https://github.com/emqx/ehttpc/pull/44). diff --git a/changes/ce/fix-10086.zh.md b/changes/ce/fix-10086.zh.md deleted file mode 100644 index c083d6055..000000000 --- a/changes/ce/fix-10086.zh.md +++ /dev/null @@ -1,3 +0,0 @@ -HTTP 客户端库 `ehttpc` 升级到 0.4.7。 -在升级前，如果 HTTP 客户端，例如 '认证'、'授权'、'WebHook' 等配置中使用了 `Content-Type` HTTP 头，但是没有配置 `Body`，则可能会发生异常。 -详情见 [ehttpc PR#44](https://github.com/emqx/ehttpc/pull/44)。 diff --git a/changes/ce/fix-10098.en.md b/changes/ce/fix-10098.en.md deleted file mode 100644 index 61058da0a..000000000 --- a/changes/ce/fix-10098.en.md +++ /dev/null @@ -1 +0,0 @@ -A crash with an error in the log file that happened when the MongoDB authorization module queried the database has been fixed. diff --git a/changes/ce/fix-10098.zh.md b/changes/ce/fix-10098.zh.md deleted file mode 100644 index 6b6d86159..000000000 --- a/changes/ce/fix-10098.zh.md +++ /dev/null @@ -1 +0,0 @@ -当MongoDB授权模块查询数据库时，在日志文件中发生的崩溃与错误已经被修复。 diff --git a/changes/ce/fix-10100.en.md b/changes/ce/fix-10100.en.md deleted file mode 100644 index e16ee5efc..000000000 --- a/changes/ce/fix-10100.en.md +++ /dev/null @@ -1,2 +0,0 @@ -Fix channel crash for slow clients with enhanced authentication. -Previously, when the client was using enhanced authentication, but the Auth message was sent slowly or the Auth message was lost, the client process would crash. diff --git a/changes/ce/fix-10100.zh.md b/changes/ce/fix-10100.zh.md deleted file mode 100644 index ac2483a27..000000000 --- a/changes/ce/fix-10100.zh.md +++ /dev/null @@ -1,2 +0,0 @@ -修复响应较慢的客户端在使用增强认证时可能出现崩溃的问题。 -此前，当客户端使用增强认证功能，但发送 Auth 报文较慢或 Auth 报文丢失时会导致客户端进程崩溃。 diff --git a/changes/ce/fix-10107.en.md b/changes/ce/fix-10107.en.md deleted file mode 100644 index 1bcbbad60..000000000 --- a/changes/ce/fix-10107.en.md +++ /dev/null @@ -1,9 +0,0 @@ -For operations on `bridges API` if `bridge-id` is unknown we now return `404` -instead of `400`. Also a bug was fixed that caused a crash if that was a node -operation. Additionally we now also check if the given bridge is enabled when -doing the cluster operation `start` . Affected endpoints: - * [cluster] `/bridges/:id/:operation`, - * [node] `/nodes/:node/bridges/:id/:operation`, where `operation` is one of -`[start|stop|restart]`. -Moreover, for a node operation, EMQX checks if node name is in our cluster and -return `404` instead of `501`. diff --git a/changes/ce/fix-10107.zh.md b/changes/ce/fix-10107.zh.md deleted file mode 100644 index e541a834f..000000000 --- a/changes/ce/fix-10107.zh.md +++ /dev/null @@ -1,8 +0,0 @@ -现在对桥接的 API 进行调用时，如果 `bridge-id` 不存在，将会返回 `404`，而不再是`400`。 -然后，还修复了这种情况下，在节点级别上进行 API 调用时，可能导致崩溃的问题。 -另外，在启动某个桥接时，会先检查指定桥接是否已启用。 -受影响的接口有： - * [cluster] `/bridges/:id/:operation`, - * [node] `/nodes/:node/bridges/:id/:operation`, -其中 `operation` 是 `[start|stop|restart]` 之一。 -此外，对于节点操作，EMQX 将检查节点是否存在于集群中，如果不在，则会返回`404`，而不再是`501`。 diff --git a/changes/ce/fix-10117.en.md b/changes/ce/fix-10117.en.md deleted file mode 100644 index 711d739ca..000000000 --- a/changes/ce/fix-10117.en.md +++ /dev/null @@ -1,2 +0,0 @@ -Fix an error occurring when a joining node doesn't have plugins that are installed on other nodes in the cluster. -After this change, the joining node will copy all the necessary plugins from other nodes. diff --git a/changes/ce/fix-10118.en.md b/changes/ce/fix-10118.en.md deleted file mode 100644 index f6db758f3..000000000 --- a/changes/ce/fix-10118.en.md +++ /dev/null @@ -1,4 +0,0 @@ -Fix problems related to manual joining of EMQX replicant nodes to the cluster. -Previously, after manually executing joining and then leaving the cluster, the `replicant` node can only run normally after restarting the node after joining the cluster again. - -[Mria PR](https://github.com/emqx/mria/pull/128) diff --git a/changes/ce/fix-10118.zh.md b/changes/ce/fix-10118.zh.md deleted file mode 100644 index a037215f0..000000000 --- a/changes/ce/fix-10118.zh.md +++ /dev/null @@ -1,4 +0,0 @@ -修复 `replicant` 节点因为手动加入 EMQX 集群导致的相关问题。 -此前，手动执行 `加入集群-离开集群` 后，`replicant` 节点再次加入集群后只有重启节点才能正常运行。 - -[Mria PR](https://github.com/emqx/mria/pull/128) diff --git a/changes/ce/fix-10119.en.md b/changes/ce/fix-10119.en.md deleted file mode 100644 index c23a9dcdb..000000000 --- a/changes/ce/fix-10119.en.md +++ /dev/null @@ -1 +0,0 @@ -Fix crash when `statsd.server` is set to an empty string. diff --git a/changes/ce/fix-10119.zh.md b/changes/ce/fix-10119.zh.md deleted file mode 100644 index c77b99025..000000000 --- a/changes/ce/fix-10119.zh.md +++ /dev/null @@ -1 +0,0 @@ -修复 `statsd.server` 配置为空字符串时启动崩溃的问题。 diff --git a/changes/ce/fix-10124.en.md b/changes/ce/fix-10124.en.md deleted file mode 100644 index 1a4aca3d9..000000000 --- a/changes/ce/fix-10124.en.md +++ /dev/null @@ -1 +0,0 @@ -The default heartbeat period for MongoDB has been increased to reduce the risk of too excessive logging to the MongoDB log file. diff --git a/changes/ce/fix-10124.zh.md b/changes/ce/fix-10124.zh.md deleted file mode 100644 index 7605f2da3..000000000 --- a/changes/ce/fix-10124.zh.md +++ /dev/null @@ -1 +0,0 @@ -增加了MongoDB的默认心跳周期，以减少对MongoDB日志文件的过多记录的风险。 diff --git a/changes/ce/fix-10130.en.md b/changes/ce/fix-10130.en.md deleted file mode 100644 index 98484e38f..000000000 --- a/changes/ce/fix-10130.en.md +++ /dev/null @@ -1,3 +0,0 @@ -Fix garbled config display in dashboard when the value is originally from environment variables. -For example, `env EMQX_STATSD__SERVER='127.0.0.1:8124' . /bin/emqx start` results in unreadable string (not '127.0.0.1:8124') displayed in Dashboard's Statsd settings page. -Related PR: [HOCON#234](https://github.com/emqx/hocon/pull/234). diff --git a/changes/ce/fix-10130.zh.md b/changes/ce/fix-10130.zh.md deleted file mode 100644 index 19c092fdf..000000000 --- a/changes/ce/fix-10130.zh.md +++ /dev/null @@ -1,3 +0,0 @@ -修复通过环境变量配置启动的 EMQX 节点无法通过HTTP API获取到正确的配置信息。 -比如：`EMQX_STATSD__SERVER='127.0.0.1:8124' ./bin/emqx start` 后通过 Dashboard看到的 Statsd 配置信息是乱码。 -相关 PR: [HOCON:234](https://github.com/emqx/hocon/pull/234). diff --git a/changes/ce/fix-10132.en.md b/changes/ce/fix-10132.en.md deleted file mode 100644 index ceb617d11..000000000 --- a/changes/ce/fix-10132.en.md +++ /dev/null @@ -1 +0,0 @@ -Fix `systemctl stop emqx` command not stopping jq, os_mon application properly, generating some error logs. diff --git a/changes/ce/fix-10132.zh.md b/changes/ce/fix-10132.zh.md deleted file mode 100644 index 36811e1bf..000000000 --- a/changes/ce/fix-10132.zh.md +++ /dev/null @@ -1 +0,0 @@ -修复`systemctl stop emqx` 命令没有正常停止 jq, os_mon 组件，产生一些错误日志。 diff --git a/changes/ce/fix-10144.en.md b/changes/ce/fix-10144.en.md deleted file mode 100644 index d5a84b24c..000000000 --- a/changes/ce/fix-10144.en.md +++ /dev/null @@ -1 +0,0 @@ -Add -setcookie emulator flag when invoking emqx ctl to prevent problems with emqx cli when home directory is read only. Fixes [#10142](https://github.com/emqx/emqx/issues/10142) diff --git a/changes/ce/fix-9939.en.md b/changes/ce/fix-9939.en.md deleted file mode 100644 index 83e84c493..000000000 --- a/changes/ce/fix-9939.en.md +++ /dev/null @@ -1,3 +0,0 @@ -Allow 'emqx ctl cluster' command to be issued before Mnesia starts. -Prior to this change, EMQX `replicant` could not use `manual` discovery strategy. -Now it's possible to join cluster using 'manual' strategy. diff --git a/changes/ce/fix-9939.zh.md b/changes/ce/fix-9939.zh.md deleted file mode 100644 index 4b150c5fc..000000000 --- a/changes/ce/fix-9939.zh.md +++ /dev/null @@ -1,2 +0,0 @@ -允许 'emqx ctl cluster join' 命令在 Mnesia 启动前就可以调用。 -在此修复前， EMQX 的 `replicant` 类型节点无法使用 `manual` 集群发现策略。 diff --git a/changes/ce/fix-9958.en.md b/changes/ce/fix-9958.en.md deleted file mode 100644 index 821934ad0..000000000 --- a/changes/ce/fix-9958.en.md +++ /dev/null @@ -1 +0,0 @@ -Fix bad http response format when client ID is not found in `clients` APIs diff --git a/changes/ce/fix-9958.zh.md b/changes/ce/fix-9958.zh.md deleted file mode 100644 index a26fbb7fe..000000000 --- a/changes/ce/fix-9958.zh.md +++ /dev/null @@ -1 +0,0 @@ -修复 `clients` API 在 Client ID 不存在时返回的错误的 HTTP 应答格式。 diff --git a/changes/ce/fix-9961.en.md b/changes/ce/fix-9961.en.md deleted file mode 100644 index 6185a64ea..000000000 --- a/changes/ce/fix-9961.en.md +++ /dev/null @@ -1 +0,0 @@ -Avoid parsing config files for node name and cookie when executing non-boot commands in bin/emqx. diff --git a/changes/ce/fix-9961.zh.md b/changes/ce/fix-9961.zh.md deleted file mode 100644 index edd90b2ca..000000000 --- a/changes/ce/fix-9961.zh.md +++ /dev/null @@ -1 +0,0 @@ -在 bin/emqx 脚本中，避免在运行非启动命令时解析 emqx.conf 来获取节点名称和 cookie。 diff --git a/changes/ce/fix-9974.en.md b/changes/ce/fix-9974.en.md deleted file mode 100644 index 97223e03f..000000000 --- a/changes/ce/fix-9974.en.md +++ /dev/null @@ -1,2 +0,0 @@ -Report memory usage to statsd and prometheus using the same data source as dashboard. -Prior to this fix, the memory usage data source was collected from an outdated source which did not work well in containers. diff --git a/changes/ce/fix-9974.zh.md b/changes/ce/fix-9974.zh.md deleted file mode 100644 index 8358204f3..000000000 --- a/changes/ce/fix-9974.zh.md +++ /dev/null @@ -1,2 +0,0 @@ -Statsd 和 prometheus 使用跟 Dashboard 相同的内存用量数据源。 -在此修复前，内存的总量和用量统计使用了过时的（在容器环境中不准确）的数据源。 diff --git a/changes/ce/fix-9978.en.md b/changes/ce/fix-9978.en.md deleted file mode 100644 index 6750d136f..000000000 --- a/changes/ce/fix-9978.en.md +++ /dev/null @@ -1,2 +0,0 @@ -Fixed configuration issue when choosing to use SSL for a Postgres connection (`authn`, `authz` and bridge). -The connection could fail to complete with a previously working configuration after an upgrade from 5.0.13 to newer EMQX versions. diff --git a/changes/ce/fix-9978.zh.md b/changes/ce/fix-9978.zh.md deleted file mode 100644 index 75eed3600..000000000 --- a/changes/ce/fix-9978.zh.md +++ /dev/null @@ -1,2 +0,0 @@ -修正了在Postgres连接中选择使用SSL时的配置问题（`authn`, `authz` 和 bridge）。 -从5.0.13升级到较新的EMQX版本后，连接可能无法完成之前的配置。 diff --git a/changes/ce/fix-9997.en.md b/changes/ce/fix-9997.en.md deleted file mode 100644 index be0344ec1..000000000 --- a/changes/ce/fix-9997.en.md +++ /dev/null @@ -1 +0,0 @@ -Fix Swagger API schema generation. `deprecated` metadata field is now always boolean, as [Swagger specification](https://swagger.io/specification/) suggests. diff --git a/changes/ce/fix-9997.zh.md b/changes/ce/fix-9997.zh.md deleted file mode 100644 index 6f1a0b779..000000000 --- a/changes/ce/fix-9997.zh.md +++ /dev/null @@ -1 +0,0 @@ -修复 Swagger API 生成时，`deprecated` 元数据字段未按照[标准](https://swagger.io/specification/)建议的那样始终为布尔值的问题。 diff --git a/changes/ce/perf-9967.en.md b/changes/ce/perf-9967.en.md deleted file mode 100644 index fadba24c9..000000000 --- a/changes/ce/perf-9967.en.md +++ /dev/null @@ -1 +0,0 @@ -New common TLS option 'hibernate_after' to reduce memory footprint per idle connecion, default: 5s. diff --git a/changes/ce/perf-9967.zh.md b/changes/ce/perf-9967.zh.md deleted file mode 100644 index 7b73f9bd0..000000000 --- a/changes/ce/perf-9967.zh.md +++ /dev/null @@ -1 +0,0 @@ -新的通用 TLS 选项 'hibernate_after'， 以减少空闲连接的内存占用，默认： 5s 。 diff --git a/changes/ce/perf-9998.en.md b/changes/ce/perf-9998.en.md deleted file mode 100644 index e9e23a25e..000000000 --- a/changes/ce/perf-9998.en.md +++ /dev/null @@ -1 +0,0 @@ -Redact the HTTP request body in the authentication error logs for security reasons. diff --git a/changes/ce/perf-9998.zh.md b/changes/ce/perf-9998.zh.md deleted file mode 100644 index 146eb858f..000000000 --- a/changes/ce/perf-9998.zh.md +++ /dev/null @@ -1 +0,0 @@ -出于安全原因，在身份验证错误日志中模糊 HTTP 请求正文。 diff --git a/changes/ee/feat-10083.en.md b/changes/ee/feat-10083.en.md deleted file mode 100644 index f4331faf9..000000000 --- a/changes/ee/feat-10083.en.md +++ /dev/null @@ -1 +0,0 @@ -Add `DynamoDB` support for Data-Brdige. diff --git a/changes/ee/feat-10083.zh.md b/changes/ee/feat-10083.zh.md deleted file mode 100644 index 8274e62c2..000000000 --- a/changes/ee/feat-10083.zh.md +++ /dev/null @@ -1 +0,0 @@ -为数据桥接增加 `DynamoDB` 支持。 diff --git a/changes/ee/feat-9564.en.md b/changes/ee/feat-9564.en.md deleted file mode 100644 index 4405e3e07..000000000 --- a/changes/ee/feat-9564.en.md +++ /dev/null @@ -1,2 +0,0 @@ -Implemented Kafka Consumer bridge. -Now it's possible to consume messages from Kafka and publish them to MQTT topics. diff --git a/changes/ee/feat-9564.zh.md b/changes/ee/feat-9564.zh.md deleted file mode 100644 index 01a7ffe58..000000000 --- a/changes/ee/feat-9564.zh.md +++ /dev/null @@ -1,2 +0,0 @@ -实现了 Kafka 消费者桥接。 -现在可以从 Kafka 消费消息并将其发布到 MQTT 主题。 diff --git a/changes/ee/feat-9881.en.md b/changes/ee/feat-9881.en.md deleted file mode 100644 index 546178965..000000000 --- a/changes/ee/feat-9881.en.md +++ /dev/null @@ -1,4 +0,0 @@ -In this pull request, we have enhanced the error logs related to InfluxDB connectivity health checks. -Previously, if InfluxDB failed to pass the health checks using the specified parameters, the only message provided was "timed out waiting for it to become healthy". -With the updated implementation, the error message will be displayed in both the logs and the dashboard, enabling easier identification and resolution of the issue. - diff --git a/changes/ee/feat-9881.zh.md b/changes/ee/feat-9881.zh.md deleted file mode 100644 index 9746a4c0a..000000000 --- a/changes/ee/feat-9881.zh.md +++ /dev/null @@ -1,3 +0,0 @@ -增强了与 InfluxDB 连接健康检查相关的错误日志。 -在此更改之前，如果使用配置的参数 InfluxDB 未能通过健康检查，用户仅能获得一个“超时”的信息。 -现在，详细的错误消息将显示在日志和控制台，从而让用户更容易地识别和解决问题。 diff --git a/changes/ee/feat-9932.en.md b/changes/ee/feat-9932.en.md deleted file mode 100644 index f4f9ce40d..000000000 --- a/changes/ee/feat-9932.en.md +++ /dev/null @@ -1 +0,0 @@ -Integrate `TDengine` into `bridges` as a new backend. diff --git a/changes/ee/feat-9932.zh.md b/changes/ee/feat-9932.zh.md deleted file mode 100644 index 1fbf7bf34..000000000 --- a/changes/ee/feat-9932.zh.md +++ /dev/null @@ -1 +0,0 @@ -在 `桥接` 中集成 `TDengine`。 diff --git a/changes/ee/fix-10007.en.md b/changes/ee/fix-10007.en.md deleted file mode 100644 index 1adab8e9b..000000000 --- a/changes/ee/fix-10007.en.md +++ /dev/null @@ -1,5 +0,0 @@ -Change Kafka bridge's config `memory_overload_protection` default value from `true` to `false`. -EMQX logs cases when messages get dropped due to overload protection, and this is also reflected in counters. -However, since there is by default no alerting based on the logs and counters, -setting it to `true` may cause messages being dropped without noticing. -At the time being, the better option is to let sysadmin set it explicitly so they are fully aware of the benefits and risks. diff --git a/changes/ee/fix-10007.zh.md b/changes/ee/fix-10007.zh.md deleted file mode 100644 index 0c08f20d0..000000000 --- a/changes/ee/fix-10007.zh.md +++ /dev/null @@ -1,3 +0,0 @@ -Kafka 桥接的配置参数 `memory_overload_protection` 默认值从 `true` 改成了 `false`。 -尽管内存过载后消息被丢弃会产生日志和计数，如果没有基于这些日志或计数的告警，系统管理员可能无法及时发现消息被丢弃。 -当前更好的选择是：让管理员显式的配置该项，迫使他们理解这个配置的好处以及风险。 diff --git a/changes/ee/fix-10087.en.md b/changes/ee/fix-10087.en.md deleted file mode 100644 index fd6e10b7b..000000000 --- a/changes/ee/fix-10087.en.md +++ /dev/null @@ -1,2 +0,0 @@ -Use default template `${timestamp}` if the `timestamp` config is empty (undefined) when inserting data in InfluxDB. -Prior to this change, InfluxDB bridge inserted a wrong timestamp when template is not provided. diff --git a/changes/ee/fix-10087.zh.md b/changes/ee/fix-10087.zh.md deleted file mode 100644 index e08e61f37..000000000 --- a/changes/ee/fix-10087.zh.md +++ /dev/null @@ -1,2 +0,0 @@ -在 InfluxDB 中插入数据时，如果时间戳为空（未定义），则使用默认的占位符 `${timestamp}`。 -在此修复前，如果时间戳字段没有设置，InfluxDB 桥接使用了一个错误的时间戳。 diff --git a/changes/ee/fix-10095.en.md b/changes/ee/fix-10095.en.md deleted file mode 100644 index 49c588345..000000000 --- a/changes/ee/fix-10095.en.md +++ /dev/null @@ -1,3 +0,0 @@ -Stop MySQL client from bombarding server repeatedly with unnecessary `PREPARE` queries on every batch, trashing the server and exhausting its internal limits. This was happening when the MySQL bridge was in the batch mode. - -Ensure safer and more careful escaping of strings and binaries in batch insert queries when the MySQL bridge is in the batch mode. diff --git a/changes/ee/fix-10095.zh.md b/changes/ee/fix-10095.zh.md deleted file mode 100644 index 5a62ccfca..000000000 --- a/changes/ee/fix-10095.zh.md +++ /dev/null @@ -1 +0,0 @@ -优化 MySQL 桥接在批量模式下能更高效的使用预处理语句 ，减少了对 MySQL 服务器的查询压力， 并确保对 SQL 语句进行更安全和谨慎的转义。 From f9044a09599019c1d957ae93b83df710cc428387 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 16 Mar 2023 14:03:37 -0300 Subject: [PATCH 018/196] docs(changelog): fix en changelog --- changes/ce/feat-10128.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ce/feat-10128.en.md diff --git a/changes/ce/feat-10128.en.md b/changes/ce/feat-10128.en.md new file mode 100644 index 000000000..ab3e5ba3e --- /dev/null +++ b/changes/ce/feat-10128.en.md @@ -0,0 +1 @@ +Add support for OCSP stapling for SSL MQTT listeners. From 3a46681dde991f301d77adb4de3bb6447b682517 Mon Sep 17 00:00:00 2001 From: Serge Tupchii Date: Thu, 16 Mar 2023 22:21:04 +0200 Subject: [PATCH 019/196] feat: handle escaped characters in InfluxDB data bridge write_syntax Closes: EMQX-7834 --- changes/ee/feat-10165.en.md | 2 + .../src/emqx_ee_bridge_influxdb.erl | 184 +++++++--- .../test/emqx_ee_bridge_influxdb_tests.erl | 328 ++++++++++++++++++ 3 files changed, 471 insertions(+), 43 deletions(-) create mode 100644 changes/ee/feat-10165.en.md create mode 100644 lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_tests.erl diff --git a/changes/ee/feat-10165.en.md b/changes/ee/feat-10165.en.md new file mode 100644 index 000000000..199d45707 --- /dev/null +++ b/changes/ee/feat-10165.en.md @@ -0,0 +1,2 @@ +Support escaped special characters in InfluxDB data bridge write_syntax. +This update allows to use escaped special characters in string elements in accordance with InfluxDB line protocol. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl index 14f53b5e7..62a9b4e80 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl @@ -3,6 +3,7 @@ %%-------------------------------------------------------------------- -module(emqx_ee_bridge_influxdb). +-include_lib("emqx/include/logger.hrl"). -include_lib("emqx_bridge/include/emqx_bridge.hrl"). -include_lib("emqx_connector/include/emqx_connector.hrl"). -include_lib("typerefl/include/types.hrl"). @@ -169,53 +170,150 @@ write_syntax(_) -> undefined. to_influx_lines(RawLines) -> - Lines = string:tokens(str(RawLines), "\n"), - lists:reverse(lists:foldl(fun converter_influx_line/2, [], Lines)). - -converter_influx_line(Line, AccIn) -> - case string:tokens(str(Line), " ") of - [MeasurementAndTags, Fields, Timestamp] -> - append_influx_item(MeasurementAndTags, Fields, Timestamp, AccIn); - [MeasurementAndTags, Fields] -> - append_influx_item(MeasurementAndTags, Fields, undefined, AccIn); - _ -> - throw("Bad InfluxDB Line Protocol schema") + try + influx_lines(str(RawLines), []) + catch + _:Reason:Stacktrace -> + Msg = lists:flatten( + io_lib:format("Unable to parse InfluxDB line protocol: ~p", [RawLines]) + ), + ?SLOG(error, #{msg => Msg, error_reason => Reason, stacktrace => Stacktrace}), + throw(Msg) end. -append_influx_item(MeasurementAndTags, Fields, Timestamp, Acc) -> - {Measurement, Tags} = split_measurement_and_tags(MeasurementAndTags), - [ - #{ - measurement => Measurement, - tags => kv_pairs(Tags), - fields => kv_pairs(string:tokens(Fields, ",")), - timestamp => Timestamp - } - | Acc - ]. +-define(MEASUREMENT_ESC_CHARS, [$,, $\s]). +-define(TAG_FIELD_KEY_ESC_CHARS, [$,, $=, $\s]). +-define(FIELD_VAL_ESC_CHARS, [$", $\\]). +% Common separator for both tags and fields +-define(SEP, $\s). +-define(MEASUREMENT_TAG_SEP, $,). +-define(KEY_SEP, $=). +-define(VAL_SEP, $,). +-define(NON_EMPTY, [_ | _]). -split_measurement_and_tags(Subject) -> - case string:tokens(Subject, ",") of - [] -> - throw("Bad Measurement schema"); - [Measurement] -> - {Measurement, []}; - [Measurement | Tags] -> - {Measurement, Tags} - end. +influx_lines([] = _RawLines, Acc) -> + ?NON_EMPTY = lists:reverse(Acc); +influx_lines(RawLines, Acc) -> + {Acc1, RawLines1} = influx_line(string:trim(RawLines, leading, "\s\n"), Acc), + influx_lines(RawLines1, Acc1). -kv_pairs(Pairs) -> - kv_pairs(Pairs, []). -kv_pairs([], Acc) -> - lists:reverse(Acc); -kv_pairs([Pair | Rest], Acc) -> - case string:tokens(Pair, "=") of - [K, V] -> - %% Reduplicated keys will be overwritten. Follows InfluxDB Line Protocol. - kv_pairs(Rest, [{K, V} | Acc]); - _ -> - throw(io_lib:format("Bad InfluxDB Line Protocol Key Value pair: ~p", Pair)) - end. +influx_line([], Acc) -> + {Acc, []}; +influx_line(Line, Acc) -> + {?NON_EMPTY = Measurement, Line1} = measurement(Line), + {Tags, Line2} = tags(Line1), + {?NON_EMPTY = Fields, Line3} = influx_fields(Line2), + {Timestamp, Line4} = timestamp(Line3), + { + [ + #{ + measurement => Measurement, + tags => Tags, + fields => Fields, + timestamp => Timestamp + } + | Acc + ], + Line4 + }. + +measurement(Line) -> + unescape(?MEASUREMENT_ESC_CHARS, [?MEASUREMENT_TAG_SEP, ?SEP], Line, []). + +tags([?MEASUREMENT_TAG_SEP | Line]) -> + tags1(Line, []); +tags(Line) -> + {[], Line}. + +%% Empty line is invalid as fields are required after tags, +%% need to break recursion here and fail later on parsing fields +tags1([] = Line, Acc) -> + {lists:reverse(Acc), Line}; +%% Matching non empty Acc treats lines like "m, field=field_val" invalid +tags1([?SEP | _] = Line, ?NON_EMPTY = Acc) -> + {lists:reverse(Acc), Line}; +tags1(Line, Acc) -> + {Tag, Line1} = tag(Line), + tags1(Line1, [Tag | Acc]). + +tag(Line) -> + {?NON_EMPTY = Key, Line1} = key(Line), + {?NON_EMPTY = Val, Line2} = tag_val(Line1), + {{Key, Val}, Line2}. + +tag_val(Line) -> + {Val, Line1} = unescape(?TAG_FIELD_KEY_ESC_CHARS, [?VAL_SEP, ?SEP], Line, []), + {Val, strip_l(Line1, ?VAL_SEP)}. + +influx_fields([?SEP | Line]) -> + fields1(string:trim(Line, leading, "\s"), []). + +%% Timestamp is optional, so fields may be at the very end of the line +fields1([Ch | _] = Line, Acc) when Ch =:= ?SEP; Ch =:= $\n -> + {lists:reverse(Acc), Line}; +fields1([] = Line, Acc) -> + {lists:reverse(Acc), Line}; +fields1(Line, Acc) -> + {Field, Line1} = field(Line), + fields1(Line1, [Field | Acc]). + +field(Line) -> + {?NON_EMPTY = Key, Line1} = key(Line), + {Val, Line2} = field_val(Line1), + {{Key, Val}, Line2}. + +field_val([$" | Line]) -> + {Val, [$" | Line1]} = unescape(?FIELD_VAL_ESC_CHARS, [$"], Line, []), + %% Quoted val can be empty + {Val, strip_l(Line1, ?VAL_SEP)}; +field_val(Line) -> + %% Unquoted value should not be un-escaped according to InfluxDB protocol, + %% as it can only hold float, integer, uinteger or boolean value. + %% However, as templates are possible, un-escaping is applied here, + %% which also helps to detect some invalid lines, e.g.: "m,tag=1 field= ${timestamp}" + {Val, Line1} = unescape(?TAG_FIELD_KEY_ESC_CHARS, [?VAL_SEP, ?SEP, $\n], Line, []), + {?NON_EMPTY = Val, strip_l(Line1, ?VAL_SEP)}. + +timestamp([?SEP | Line]) -> + Line1 = string:trim(Line, leading, "\s"), + %% Similarly to unquoted field value, un-escape a timestamp to validate and handle + %% potentially escaped characters in a template + {T, Line2} = unescape(?TAG_FIELD_KEY_ESC_CHARS, [?SEP, $\n], Line1, []), + {timestamp1(T), Line2}; +timestamp(Line) -> + {undefined, Line}. + +timestamp1(?NON_EMPTY = Ts) -> Ts; +timestamp1(_Ts) -> undefined. + +%% Common for both tag and field keys +key(Line) -> + {Key, Line1} = unescape(?TAG_FIELD_KEY_ESC_CHARS, [?KEY_SEP], Line, []), + {Key, strip_l(Line1, ?KEY_SEP)}. + +%% Only strip a character between pairs, don't strip it(and let it fail) +%% if the char to be stripped is at the end, e.g.: m,tag=val, field=val +strip_l([Ch, Ch1 | Str], Ch) when Ch1 =/= ?SEP -> + [Ch1 | Str]; +strip_l(Str, _Ch) -> + Str. + +unescape(EscapeChars, SepChars, [$\\, Char | T], Acc) -> + ShouldEscapeBackslash = lists:member($\\, EscapeChars), + Acc1 = + case lists:member(Char, EscapeChars) of + true -> [Char | Acc]; + false when not ShouldEscapeBackslash -> [Char, $\\ | Acc] + end, + unescape(EscapeChars, SepChars, T, Acc1); +unescape(EscapeChars, SepChars, [Char | T] = L, Acc) -> + IsEscapeChar = lists:member(Char, EscapeChars), + case lists:member(Char, SepChars) of + true -> {lists:reverse(Acc), L}; + false when not IsEscapeChar -> unescape(EscapeChars, SepChars, T, [Char | Acc]) + end; +unescape(_EscapeChars, _SepChars, [] = L, Acc) -> + {lists:reverse(Acc), L}. str(A) when is_atom(A) -> atom_to_list(A); diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_tests.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_tests.erl new file mode 100644 index 000000000..ce3a0b06f --- /dev/null +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_tests.erl @@ -0,0 +1,328 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_ee_bridge_influxdb_tests). + +-include_lib("eunit/include/eunit.hrl"). + +-import(emqx_ee_bridge_influxdb, [to_influx_lines/1]). + +-define(INVALID_LINES, [ + " ", + " \n", + " \n\n\n ", + "\n", + " \n\n \n \n", + "measurement", + "measurement ", + "measurement,tag", + "measurement field", + "measurement,tag field", + "measurement,tag field ${timestamp}", + "measurement,tag=", + "measurement,tag=tag1", + "measurement,tag =", + "measurement field=", + "measurement field= ", + "measurement field = ", + "measurement, tag = field = ", + "measurement, tag = field = ", + "measurement, tag = tag_val field = field_val", + "measurement, tag = tag_val field = field_val ${timestamp}", + "measurement,= = ${timestamp}", + "measurement,t=a, f=a, ${timestamp}", + "measurement,t=a,t1=b, f=a,f1=b, ${timestamp}", + "measurement,t=a,t1=b, f=a,f1=b,", + "measurement,t=a, t1=b, f=a,f1=b,", + "measurement,t=a,,t1=b, f=a,f1=b,", + "measurement,t=a,,t1=b f=a,,f1=b", + "measurement,t=a,,t1=b f=a,f1=b ${timestamp}", + "measurement, f=a,f1=b", + "measurement, f=a,f1=b ${timestamp}", + "measurement,, f=a,f1=b ${timestamp}", + "measurement,, f=a,f1=b", + "measurement,, f=a,f1=b,, ${timestamp}", + "measurement f=a,f1=b,, ${timestamp}", + "measurement,t=a f=a,f1=b,, ${timestamp}", + "measurement,t=a f=a,f1=b,, ", + "measurement,t=a f=a,f1=b,,", + "measurement, t=a f=a,f1=b", + "measurement,t=a f=a, f1=b", + "measurement,t=a f=a, f1=b ${timestamp}", + "measurement, t=a f=a, f1=b ${timestamp}", + "measurement,t= a f=a,f1=b ${timestamp}", + "measurement,t= a f=a,f1 =b ${timestamp}", + "measurement, t = a f = a,f1 = b ${timestamp}", + "measurement,t=a f=a,f1=b \n ${timestamp}", + "measurement,t=a \n f=a,f1=b \n ${timestamp}", + "measurement,t=a \n f=a,f1=b \n ", + "\n measurement,t=a \n f=a,f1=b \n ${timestamp}", + "\n measurement,t=a \n f=a,f1=b \n", + %% not escaped backslash in a quoted field value is invalid + "measurement,tag=1 field=\"val\\1\"" +]). + +-define(VALID_LINE_PARSED_PAIRS, [ + {"m1,tag=tag1 field=field1 ${timestamp1}", #{ + measurement => "m1", + tags => [{"tag", "tag1"}], + fields => [{"field", "field1"}], + timestamp => "${timestamp1}" + }}, + {"m2,tag=tag2 field=field2", #{ + measurement => "m2", + tags => [{"tag", "tag2"}], + fields => [{"field", "field2"}], + timestamp => undefined + }}, + {"m3 field=field3 ${timestamp3}", #{ + measurement => "m3", + tags => [], + fields => [{"field", "field3"}], + timestamp => "${timestamp3}" + }}, + {"m4 field=field4", #{ + measurement => "m4", + tags => [], + fields => [{"field", "field4"}], + timestamp => undefined + }}, + {"m5,tag=tag5,tag_a=tag5a,tag_b=tag5b field=field5,field_a=field5a,field_b=field5b ${timestamp5}", + #{ + measurement => "m5", + tags => [{"tag", "tag5"}, {"tag_a", "tag5a"}, {"tag_b", "tag5b"}], + fields => [{"field", "field5"}, {"field_a", "field5a"}, {"field_b", "field5b"}], + timestamp => "${timestamp5}" + }}, + {"m6,tag=tag6,tag_a=tag6a,tag_b=tag6b field=field6,field_a=field6a,field_b=field6b", #{ + measurement => "m6", + tags => [{"tag", "tag6"}, {"tag_a", "tag6a"}, {"tag_b", "tag6b"}], + fields => [{"field", "field6"}, {"field_a", "field6a"}, {"field_b", "field6b"}], + timestamp => undefined + }}, + {"m7,tag=tag7,tag_a=\"tag7a\",tag_b=tag7b field=\"field7\",field_a=field7a,field_b=\"field7b\"", + #{ + measurement => "m7", + tags => [{"tag", "tag7"}, {"tag_a", "\"tag7a\""}, {"tag_b", "tag7b"}], + fields => [{"field", "field7"}, {"field_a", "field7a"}, {"field_b", "field7b"}], + timestamp => undefined + }}, + {"m8,tag=tag8,tag_a=\"tag8a\",tag_b=tag8b field=\"field8\",field_a=field8a,field_b=\"field8b\" ${timestamp8}", + #{ + measurement => "m8", + tags => [{"tag", "tag8"}, {"tag_a", "\"tag8a\""}, {"tag_b", "tag8b"}], + fields => [{"field", "field8"}, {"field_a", "field8a"}, {"field_b", "field8b"}], + timestamp => "${timestamp8}" + }}, + {"m9,tag=tag9,tag_a=\"tag9a\",tag_b=tag9b field=\"field9\",field_a=field9a,field_b=\"\" ${timestamp9}", + #{ + measurement => "m9", + tags => [{"tag", "tag9"}, {"tag_a", "\"tag9a\""}, {"tag_b", "tag9b"}], + fields => [{"field", "field9"}, {"field_a", "field9a"}, {"field_b", ""}], + timestamp => "${timestamp9}" + }}, + {"m10 field=\"\" ${timestamp10}", #{ + measurement => "m10", + tags => [], + fields => [{"field", ""}], + timestamp => "${timestamp10}" + }} +]). + +-define(VALID_LINE_EXTRA_SPACES_PARSED_PAIRS, [ + {"\n m1,tag=tag1 field=field1 ${timestamp1} \n", #{ + measurement => "m1", + tags => [{"tag", "tag1"}], + fields => [{"field", "field1"}], + timestamp => "${timestamp1}" + }}, + {" m2,tag=tag2 field=field2 ", #{ + measurement => "m2", + tags => [{"tag", "tag2"}], + fields => [{"field", "field2"}], + timestamp => undefined + }}, + {" m3 field=field3 ${timestamp3} ", #{ + measurement => "m3", + tags => [], + fields => [{"field", "field3"}], + timestamp => "${timestamp3}" + }}, + {" \n m4 field=field4\n ", #{ + measurement => "m4", + tags => [], + fields => [{"field", "field4"}], + timestamp => undefined + }}, + {" \n m5,tag=tag5,tag_a=tag5a,tag_b=tag5b field=field5,field_a=field5a,field_b=field5b ${timestamp5} \n", + #{ + measurement => "m5", + tags => [{"tag", "tag5"}, {"tag_a", "tag5a"}, {"tag_b", "tag5b"}], + fields => [{"field", "field5"}, {"field_a", "field5a"}, {"field_b", "field5b"}], + timestamp => "${timestamp5}" + }}, + {" m6,tag=tag6,tag_a=tag6a,tag_b=tag6b field=field6,field_a=field6a,field_b=field6b\n ", #{ + measurement => "m6", + tags => [{"tag", "tag6"}, {"tag_a", "tag6a"}, {"tag_b", "tag6b"}], + fields => [{"field", "field6"}, {"field_a", "field6a"}, {"field_b", "field6b"}], + timestamp => undefined + }} +]). + +-define(VALID_LINE_PARSED_ESCAPED_CHARS_PAIRS, [ + {"m\\ =1\\,,\\,tag\\ \\==\\=tag\\ 1\\, \\,fie\\ ld\\ =\\ field\\,1 ${timestamp1}", #{ + measurement => "m =1,", + tags => [{",tag =", "=tag 1,"}], + fields => [{",fie ld ", " field,1"}], + timestamp => "${timestamp1}" + }}, + {"m2,tag=tag2 field=\"field \\\"2\\\",\n\"", #{ + measurement => "m2", + tags => [{"tag", "tag2"}], + fields => [{"field", "field \"2\",\n"}], + timestamp => undefined + }}, + {"m\\ 3 field=\"field3\" ${payload.timestamp\\ 3}", #{ + measurement => "m 3", + tags => [], + fields => [{"field", "field3"}], + timestamp => "${payload.timestamp 3}" + }}, + {"m4 field=\"\\\"field\\\\4\\\"\"", #{ + measurement => "m4", + tags => [], + fields => [{"field", "\"field\\4\""}], + timestamp => undefined + }}, + {"m5\\,mA,tag=\\=tag5\\=,\\,tag_a\\,=tag\\ 5a,tag_b=tag5b \\ field\\ =field5,field\\ _\\ a=field5a,\\,field_b\\ =\\=\\,\\ field5b ${timestamp5}", + #{ + measurement => "m5,mA", + tags => [{"tag", "=tag5="}, {",tag_a,", "tag 5a"}, {"tag_b", "tag5b"}], + fields => [ + {" field ", "field5"}, {"field _ a", "field5a"}, {",field_b ", "=, field5b"} + ], + timestamp => "${timestamp5}" + }}, + {"m6,tag=tag6,tag_a=tag6a,tag_b=tag6b field=\"field6\",field_a=\"field6a\",field_b=\"field6b\"", + #{ + measurement => "m6", + tags => [{"tag", "tag6"}, {"tag_a", "tag6a"}, {"tag_b", "tag6b"}], + fields => [{"field", "field6"}, {"field_a", "field6a"}, {"field_b", "field6b"}], + timestamp => undefined + }}, + {"\\ \\ m7\\ \\ ,tag=\\ tag\\,7\\ ,tag_a=\"tag7a\",tag_b\\,tag1=tag7b field=\"field7\",field_a=field7a,field_b=\"field7b\\\\\n\"", + #{ + measurement => " m7 ", + tags => [{"tag", " tag,7 "}, {"tag_a", "\"tag7a\""}, {"tag_b,tag1", "tag7b"}], + fields => [{"field", "field7"}, {"field_a", "field7a"}, {"field_b", "field7b\\\n"}], + timestamp => undefined + }}, + {"m8,tag=tag8,tag_a=\"tag8a\",tag_b=tag8b field=\"field8\",field_a=field8a,field_b=\"\\\"field\\\" = 8b\" ${timestamp8}", + #{ + measurement => "m8", + tags => [{"tag", "tag8"}, {"tag_a", "\"tag8a\""}, {"tag_b", "tag8b"}], + fields => [{"field", "field8"}, {"field_a", "field8a"}, {"field_b", "\"field\" = 8b"}], + timestamp => "${timestamp8}" + }}, + {"m\\9,tag=tag9,tag_a=\"tag9a\",tag_b=tag9b field\\=field=\"field9\",field_a=field9a,field_b=\"\" ${timestamp9}", + #{ + measurement => "m\\9", + tags => [{"tag", "tag9"}, {"tag_a", "\"tag9a\""}, {"tag_b", "tag9b"}], + fields => [{"field=field", "field9"}, {"field_a", "field9a"}, {"field_b", ""}], + timestamp => "${timestamp9}" + }}, + {"m\\,10 \"field\\\\\"=\"\" ${timestamp10}", #{ + measurement => "m,10", + tags => [], + %% backslash should not be un-escaped in tag key + fields => [{"\"field\\\\\"", ""}], + timestamp => "${timestamp10}" + }} +]). + +-define(VALID_LINE_PARSED_ESCAPED_CHARS_EXTRA_SPACES_PAIRS, [ + {" \n m\\ =1\\,,\\,tag\\ \\==\\=tag\\ 1\\, \\,fie\\ ld\\ =\\ field\\,1 ${timestamp1} ", #{ + measurement => "m =1,", + tags => [{",tag =", "=tag 1,"}], + fields => [{",fie ld ", " field,1"}], + timestamp => "${timestamp1}" + }}, + {" m2,tag=tag2 field=\"field \\\"2\\\",\n\" ", #{ + measurement => "m2", + tags => [{"tag", "tag2"}], + fields => [{"field", "field \"2\",\n"}], + timestamp => undefined + }}, + {" m\\ 3 field=\"field3\" ${payload.timestamp\\ 3} ", #{ + measurement => "m 3", + tags => [], + fields => [{"field", "field3"}], + timestamp => "${payload.timestamp 3}" + }}, + {" m4 field=\"\\\"field\\\\4\\\"\" ", #{ + measurement => "m4", + tags => [], + fields => [{"field", "\"field\\4\""}], + timestamp => undefined + }}, + {" m5\\,mA,tag=\\=tag5\\=,\\,tag_a\\,=tag\\ 5a,tag_b=tag5b \\ field\\ =field5,field\\ _\\ a=field5a,\\,field_b\\ =\\=\\,\\ field5b ${timestamp5} ", + #{ + measurement => "m5,mA", + tags => [{"tag", "=tag5="}, {",tag_a,", "tag 5a"}, {"tag_b", "tag5b"}], + fields => [ + {" field ", "field5"}, {"field _ a", "field5a"}, {",field_b ", "=, field5b"} + ], + timestamp => "${timestamp5}" + }}, + {" m6,tag=tag6,tag_a=tag6a,tag_b=tag6b field=\"field6\",field_a=\"field6a\",field_b=\"field6b\" ", + #{ + measurement => "m6", + tags => [{"tag", "tag6"}, {"tag_a", "tag6a"}, {"tag_b", "tag6b"}], + fields => [{"field", "field6"}, {"field_a", "field6a"}, {"field_b", "field6b"}], + timestamp => undefined + }} +]). + +invalid_write_syntax_line_test_() -> + [?_assertThrow(_, to_influx_lines(L)) || L <- ?INVALID_LINES]. + +invalid_write_syntax_multiline_test_() -> + LinesList = [ + join("\n", ?INVALID_LINES), + join("\n\n\n", ?INVALID_LINES), + join("\n\n", lists:reverse(?INVALID_LINES)) + ], + [?_assertThrow(_, to_influx_lines(Lines)) || Lines <- LinesList]. + +valid_write_syntax_test_() -> + test_pairs(?VALID_LINE_PARSED_PAIRS). + +valid_write_syntax_with_extra_spaces_test_() -> + test_pairs(?VALID_LINE_EXTRA_SPACES_PARSED_PAIRS). + +valid_write_syntax_escaped_chars_test_() -> + test_pairs(?VALID_LINE_PARSED_ESCAPED_CHARS_PAIRS). + +valid_write_syntax_escaped_chars_with_extra_spaces_test_() -> + test_pairs(?VALID_LINE_PARSED_ESCAPED_CHARS_EXTRA_SPACES_PAIRS). + +test_pairs(PairsList) -> + {Lines, AllExpected} = lists:unzip(PairsList), + JoinedLines = join("\n", Lines), + JoinedLines1 = join("\n\n\n", Lines), + JoinedLines2 = join("\n\n", lists:reverse(Lines)), + SingleLineTests = + [ + ?_assertEqual([Expected], to_influx_lines(Line)) + || {Line, Expected} <- PairsList + ], + JoinedLinesTests = + [ + ?_assertEqual(AllExpected, to_influx_lines(JoinedLines)), + ?_assertEqual(AllExpected, to_influx_lines(JoinedLines1)), + ?_assertEqual(lists:reverse(AllExpected), to_influx_lines(JoinedLines2)) + ], + SingleLineTests ++ JoinedLinesTests. + +join(Sep, LinesList) -> + lists:flatten(lists:join(Sep, LinesList)). From 08e249dc4d57b9e0288e07f9a9e5b12f3a4379a0 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Mon, 20 Mar 2023 16:53:55 +0100 Subject: [PATCH 020/196] docs: refine README.md for the emqx_rule_engine app This commit adds more content to the readme file of the `emqx_rule_engine` app. The aim of the changes are to make it easier to understand what the rule engine is and what it does. Fixes: https://emqx.atlassian.net/browse/EMQX-9229 --- apps/emqx_rule_engine/README.md | 77 ++++++++++++++++++++++++++------- 1 file changed, 61 insertions(+), 16 deletions(-) diff --git a/apps/emqx_rule_engine/README.md b/apps/emqx_rule_engine/README.md index 2485ff534..f1533a0e3 100644 --- a/apps/emqx_rule_engine/README.md +++ b/apps/emqx_rule_engine/README.md @@ -1,23 +1,46 @@ -# emqx-rule-engine +# Emqx Rule Engine -IoT Rule Engine +The rule engine's goal is to provide a simple and flexible way to transform and +reroute the messages coming to the EMQX broker. For example, one message +containing measurements from multiple sensors of different types can be +transformed into multiple messages. + + +## Concepts + +A rule is quite simple. A rule describes which messages it affects by +specifying a topic filter and a set of conditions that need to be met. If a +message matches the topic filter and all the conditions are met, the rule is +triggered. The rule can then transform the message and route it to a different +topic, or send it to another service (defined by an EMQX bridge). The rule +engine's message data transformation is designed to work well with JSON +formatted data. + + +A rule consists of the three parts **MATCH**, **TRANSFORM** and **ACTIONS** that are +described below: + +* **MATCH** - The rule's trigger condition. The rule is triggered when a message + arrives that matches the topic filter and all the specified conditions are met. +* **TRANSFORM** - The rule's data transformation. The rule can select data from the + incoming message and transform it into a new message. +* **ACTIONS** - The rule's action(s). The rule can have one or more actions. The + actions are executed when the rule is triggered. The actions can be to route + the message to a different topic, or send it to another service (defined by + an EMQX bridge). -## Concept -``` -iot rule "Rule Name" - when - match TopicFilters and Conditions - select - para1 = val1 - para2 = val2 - then - take action(#{para2 => val1, #para2 => val2}) -``` ## Architecture +The following diagram shows how the rule engine is integrated with the EMQX +message broker. Incoming messages are checked against the rules, and if a rule +matches, it is triggered with the message as input. The rule can then transform +or split the message and/or route it to a different topic, or send it to another +service (defined by an EMQX bridge). + + ``` |-----------------| Pub ---->| Message Routing |----> Sub @@ -28,11 +51,33 @@ iot rule "Rule Name" | Rule Engine | |-----------------| | | - Backends Services Bridges + Services Bridges (defined by EMQX bridges) ``` -## SQL for Rule query statement +## Domain Specific Language for Rules + +The **MATCH** and **TRANSFORM** parts of the rule are specified using a domain +specific language that looks similar to SQL. The following is an example of a +rule engine statement. The `from "topic/a"` part specifies the topic filter +(only messages to the topic `topic/a` will be considered). The `where t > 50` +part specifies the condition that needs to be met for the rule to be triggered. +The `select id, time, temperature as t` part specifies the data transformation +(the selected fields will remain in the transformed message payload). The `as +t` part specifies that the `temperature` field name is changed to `t` in the +output message. The name `t` can also be used in the where part of the rule as +an alias for `t`. + ``` -select id, time, temperature as t from "topic/a" where t > 50; +select id, time, temperature as t from "topic/a" where t > 50 ``` + + This just scratches the surface of what is possible with the rule engine. The + full documentation is available at [EMQX Rule + Engine](https://www.emqx.io/docs/en/v5.0/data-integration/rules.html). For + example, there are many built-in functions that can be used in the rule engine + language to help in doing transformations and matching. One of the [built-in + functions allows you to run JQ + queries](https://www.emqx.io/docs/en/v5.0/data-integration/rule-sql-jq.html) + which allows you to do complex transformations of the message. + From 6f71898546977403f4354a1ac025b0d2b1cd73a7 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 20 Mar 2023 12:48:41 +0100 Subject: [PATCH 021/196] fix: upgrade esockd from 5.9.4 to 5.9.6 changes in esockd are * 5.9.5: added API to retrieve SNI * 5.9.6: avoid error log if socket is closed before sending proxy protocol headers --- apps/emqx/rebar.config | 2 +- changes/ce/fix-10174.en.md | 2 ++ changes/ce/fix-10174.zh.md | 2 ++ mix.exs | 2 +- rebar.config | 2 +- 5 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 changes/ce/fix-10174.en.md create mode 100644 changes/ce/fix-10174.zh.md diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 229979f6c..bb66280f0 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -26,7 +26,7 @@ {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}}, {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}, {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}}, - {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.4"}}}, + {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}}, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.14.5"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}, {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.37.0"}}}, diff --git a/changes/ce/fix-10174.en.md b/changes/ce/fix-10174.en.md new file mode 100644 index 000000000..213af19da --- /dev/null +++ b/changes/ce/fix-10174.en.md @@ -0,0 +1,2 @@ +Upgrade library `esockd` from 5.9.4 to 5.9.6. +Fix an unnecessary error level logging when a connection is closed before proxy protocol header is sent by the proxy. diff --git a/changes/ce/fix-10174.zh.md b/changes/ce/fix-10174.zh.md new file mode 100644 index 000000000..435056280 --- /dev/null +++ b/changes/ce/fix-10174.zh.md @@ -0,0 +1,2 @@ +依赖库 `esockd` 从 5.9.4 升级到 5.9.6。 +修复了一个不必要的错误日志。如果连接在 proxy protocol 包头还没有发送前就关闭了， 则不打印错误日志。 diff --git a/mix.exs b/mix.exs index 42354f8dc..c4677dd1d 100644 --- a/mix.exs +++ b/mix.exs @@ -52,7 +52,7 @@ defmodule EMQXUmbrella.MixProject do {:gproc, github: "uwiger/gproc", tag: "0.8.0", override: true}, {:jiffy, github: "emqx/jiffy", tag: "1.0.5", override: true}, {:cowboy, github: "emqx/cowboy", tag: "2.9.0", override: true}, - {:esockd, github: "emqx/esockd", tag: "5.9.4", override: true}, + {:esockd, github: "emqx/esockd", tag: "5.9.6", override: true}, {:rocksdb, github: "emqx/erlang-rocksdb", tag: "1.7.2-emqx-9", override: true}, {:ekka, github: "emqx/ekka", tag: "0.14.5", override: true}, {:gen_rpc, github: "emqx/gen_rpc", tag: "2.8.1", override: true}, diff --git a/rebar.config b/rebar.config index 5ce9138ce..62fadbe51 100644 --- a/rebar.config +++ b/rebar.config @@ -54,7 +54,7 @@ , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}} - , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.4"}}} + , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}} , {rocksdb, {git, "https://github.com/emqx/erlang-rocksdb", {tag, "1.7.2-emqx-9"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.14.5"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}} From c739d9957c6257f5ec0e0d1da74dfc5120922a35 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 20 Mar 2023 18:25:35 +0100 Subject: [PATCH 022/196] docs: add readme for emqx_ctl app --- apps/emqx_ctl/README.md | 43 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/apps/emqx_ctl/README.md b/apps/emqx_ctl/README.md index a91342606..2638031e6 100644 --- a/apps/emqx_ctl/README.md +++ b/apps/emqx_ctl/README.md @@ -1,4 +1,41 @@ -emqx_ctl -===== +# emqx_ctl -Backend module for `emqx_ctl` command. +This application accepts dynamic `emqx ctl` command registrations so plugins can add their own commands. +Please note that the 'proxy' command `emqx_ctl` is considered deprecated, going forward, please use `emqx ctl` instead. + +## Add a new command + +To add a new command, the application must implement a callback function to handle the command, and register the command with `emqx_ctl:register_command/2` API. + +### Register + +To add a new command which can be executed from `emqx ctl`, the application must call `emqx_ctl:register_command/2` API to register the command. + +For example, to add a new command `myplugin` which is to be executed as `emqx ctl myplugin`, the application must call `emqx_ctl:register_command/2` API as follows: + +```erlang +emqx_ctl:register_command(mypluin, {myplugin_cli, cmd}). +``` + +### Callback + +The callback function must be exported by the application and must have the following signature: + +```erlang +cmd([Arg1, Arg2, ...]) -> ok. +``` + +It must also implement a special clause to handle the `usage` argument: + +```erlang +cmd([usage]) -> "myplugin [arg1] [arg2] ..."; +``` + +### Utility + +The `emqx_ctl` application provides some utility functions which help to format the output of the command. +For example `emqx_ctl:print/2` and `emqx_ctl:usage/1`. + +## Reference + +[emqx_management_cli](../emqx_management/src/emqx_mgmt_cli.erl) can be taken as a reference for how to implement a command. From 7a71d64686ea989b4a221a10778d1d15ba837003 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 16 Mar 2023 13:58:52 -0300 Subject: [PATCH 023/196] feat(crl): add crl check support --- apps/emqx/i18n/emqx_schema_i18n.conf | 50 + apps/emqx/src/emqx_crl_cache.erl | 314 +++++ apps/emqx/src/emqx_kernel_sup.erl | 3 +- apps/emqx/src/emqx_listeners.erl | 17 +- apps/emqx/src/emqx_schema.erl | 64 +- apps/emqx/src/emqx_ssl_crl_cache.erl | 237 ++++ apps/emqx/test/emqx_common_test_helpers.erl | 15 +- apps/emqx/test/emqx_crl_cache_SUITE.erl | 1057 +++++++++++++++++ .../ca-chain.cert.pem | 68 ++ .../client-no-dist-points.cert.pem | 32 + .../client-no-dist-points.key.pem | 28 + .../client-revoked.cert.pem | 32 + .../client-revoked.key.pem | 28 + .../emqx_crl_cache_SUITE_data/client.cert.pem | 32 + .../emqx_crl_cache_SUITE_data/client.key.pem | 28 + .../client1.cert.pem | 32 + .../emqx_crl_cache_SUITE_data/client1.key.pem | 28 + .../client2.cert.pem | 32 + .../emqx_crl_cache_SUITE_data/client2.key.pem | 28 + .../client3.cert.pem | 32 + .../emqx_crl_cache_SUITE_data/client3.key.pem | 28 + .../test/emqx_crl_cache_SUITE_data/crl.pem | 20 + .../test/emqx_crl_cache_SUITE_data/emqx.conf | 12 + .../emqx_crl_cache_http_server.erl | 67 ++ .../emqx_just_verify.conf | 12 + .../intermediate-not-revoked.crl.pem | 19 + .../intermediate-revoked-no-dp.crl.pem | 19 + .../intermediate-revoked.crl.pem | 20 + .../intermediate.crl.pem | 20 + .../emqx_crl_cache_SUITE_data/server.cert.pem | 35 + .../emqx_crl_cache_SUITE_data/server.key.pem | 28 + apps/emqx/test/emqx_ocsp_cache_SUITE.erl | 2 +- changes/ce/feat-10164.en.md | 1 + rebar.config | 8 +- 34 files changed, 2436 insertions(+), 12 deletions(-) create mode 100644 apps/emqx/src/emqx_crl_cache.erl create mode 100644 apps/emqx/src/emqx_ssl_crl_cache.erl create mode 100644 apps/emqx/test/emqx_crl_cache_SUITE.erl create mode 100644 apps/emqx/test/emqx_crl_cache_SUITE_data/ca-chain.cert.pem create mode 100644 apps/emqx/test/emqx_crl_cache_SUITE_data/client-no-dist-points.cert.pem create mode 100644 apps/emqx/test/emqx_crl_cache_SUITE_data/client-no-dist-points.key.pem create mode 100644 apps/emqx/test/emqx_crl_cache_SUITE_data/client-revoked.cert.pem create mode 100644 apps/emqx/test/emqx_crl_cache_SUITE_data/client-revoked.key.pem create mode 100644 apps/emqx/test/emqx_crl_cache_SUITE_data/client.cert.pem create mode 100644 apps/emqx/test/emqx_crl_cache_SUITE_data/client.key.pem create mode 100644 apps/emqx/test/emqx_crl_cache_SUITE_data/client1.cert.pem create mode 100644 apps/emqx/test/emqx_crl_cache_SUITE_data/client1.key.pem create mode 100644 apps/emqx/test/emqx_crl_cache_SUITE_data/client2.cert.pem create mode 100644 apps/emqx/test/emqx_crl_cache_SUITE_data/client2.key.pem create mode 100644 apps/emqx/test/emqx_crl_cache_SUITE_data/client3.cert.pem create mode 100644 apps/emqx/test/emqx_crl_cache_SUITE_data/client3.key.pem create mode 100644 apps/emqx/test/emqx_crl_cache_SUITE_data/crl.pem create mode 100644 apps/emqx/test/emqx_crl_cache_SUITE_data/emqx.conf create mode 100644 apps/emqx/test/emqx_crl_cache_SUITE_data/emqx_crl_cache_http_server.erl create mode 100644 apps/emqx/test/emqx_crl_cache_SUITE_data/emqx_just_verify.conf create mode 100644 apps/emqx/test/emqx_crl_cache_SUITE_data/intermediate-not-revoked.crl.pem create mode 100644 apps/emqx/test/emqx_crl_cache_SUITE_data/intermediate-revoked-no-dp.crl.pem create mode 100644 apps/emqx/test/emqx_crl_cache_SUITE_data/intermediate-revoked.crl.pem create mode 100644 apps/emqx/test/emqx_crl_cache_SUITE_data/intermediate.crl.pem create mode 100644 apps/emqx/test/emqx_crl_cache_SUITE_data/server.cert.pem create mode 100644 apps/emqx/test/emqx_crl_cache_SUITE_data/server.key.pem create mode 100644 changes/ce/feat-10164.en.md diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/apps/emqx/i18n/emqx_schema_i18n.conf index 5a48c218a..e05ee4c35 100644 --- a/apps/emqx/i18n/emqx_schema_i18n.conf +++ b/apps/emqx/i18n/emqx_schema_i18n.conf @@ -1810,6 +1810,56 @@ server_ssl_opts_schema_ocsp_refresh_http_timeout { } } +server_ssl_opts_schema_enable_crl_check { + desc { + en: "Whether to enable CRL verification for this listener." + zh: "是否为该监听器启用 CRL 检查。" + } + label: { + en: "Enable CRL Check" + zh: "启用 CRL 检查" + } +} + +crl_cache_refresh_http_timeout { + desc { + en: "The timeout for the HTTP request when fetching CRLs. This is" + " global for all listeners." + zh: "获取 CRLs 时 HTTP 请求的超时。 这对所有监听器来说是全局的。" + } + label: { + en: "CRL Cache Refresh HTTP Timeout" + zh: "CRL 缓存刷新 HTTP 超时" + } +} + +crl_cache_refresh_interval { + desc { + en: "The period to refresh the CRLs from the servers. This is global" + " for all URLs and listeners." + zh: "从服务器刷新CRL的周期。 这对所有URL和监听器来说是全局性的。" + } + label: { + en: "CRL Cache Refresh Interval" + zh: "CRL 缓存刷新间隔" + } +} + +crl_cache_capacity { + desc { + en: "The maximum number of CRL URLs that can be held in cache. If the cache is at" + " full capacity and a new URL must be fetched, then it'll evict the oldest" + " inserted URL in the cache." + zh: "缓存中可容纳的 CRL URL 的最大数量。" + " 如果缓存的容量已满，并且必须获取一个新的 URL，" + "那么它将驱逐缓存中插入的最老的 URL。" + } + label: { + en: "CRL Cache Capacity" + zh: "CRL 缓存容量" + } +} + fields_listeners_tcp { desc { en: """TCP listeners.""" diff --git a/apps/emqx/src/emqx_crl_cache.erl b/apps/emqx/src/emqx_crl_cache.erl new file mode 100644 index 000000000..79e47a6dc --- /dev/null +++ b/apps/emqx/src/emqx_crl_cache.erl @@ -0,0 +1,314 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% @doc EMQX CRL cache. +%%-------------------------------------------------------------------- + +-module(emqx_crl_cache). + +%% API +-export([ + start_link/0, + start_link/1, + register_der_crls/2, + refresh/1, + evict/1 +]). + +%% gen_server callbacks +-export([ + init/1, + handle_call/3, + handle_cast/2, + handle_info/2 +]). + +%% internal exports +-export([http_get/2]). + +-behaviour(gen_server). + +-include("logger.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-define(HTTP_TIMEOUT, timer:seconds(15)). +-define(RETRY_TIMEOUT, 5_000). +-ifdef(TEST). +-define(MIN_REFRESH_PERIOD, timer:seconds(5)). +-else. +-define(MIN_REFRESH_PERIOD, timer:minutes(1)). +-endif. +-define(DEFAULT_REFRESH_INTERVAL, timer:minutes(15)). +-define(DEFAULT_CACHE_CAPACITY, 100). + +-record(state, { + refresh_timers = #{} :: #{binary() => timer:tref()}, + refresh_interval = timer:minutes(15) :: timer:time(), + http_timeout = ?HTTP_TIMEOUT :: timer:time(), + %% keeps track of URLs by insertion time + insertion_times = gb_trees:empty() :: gb_trees:tree(timer:time(), url()), + %% the set of cached URLs, for testing if an URL is already + %% registered. + cached_urls = sets:new([{version, 2}]) :: sets:set(url()), + cache_capacity = 100 :: pos_integer(), + %% for future use + extra = #{} :: map() +}). +-type url() :: uri_string:uri_string(). +-type state() :: #state{}. + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- + +start_link() -> + Config = gather_config(), + start_link(Config). + +start_link(Config = #{cache_capacity := _, refresh_interval := _, http_timeout := _}) -> + gen_server:start_link({local, ?MODULE}, ?MODULE, Config, []). + +-spec refresh(url()) -> ok. +refresh(URL) -> + gen_server:cast(?MODULE, {refresh, URL}). + +-spec evict(url()) -> ok. +evict(URL) -> + gen_server:cast(?MODULE, {evict, URL}). + +%% Adds CRLs in DER format to the cache and register them for periodic +%% refresh. +-spec register_der_crls(url(), [public_key:der_encoded()]) -> ok. +register_der_crls(URL, CRLs) when is_list(CRLs) -> + gen_server:cast(?MODULE, {register_der_crls, URL, CRLs}). + +%%-------------------------------------------------------------------- +%% gen_server behaviour +%%-------------------------------------------------------------------- + +init(Config) -> + #{ + cache_capacity := CacheCapacity, + refresh_interval := RefreshIntervalMS, + http_timeout := HTTPTimeoutMS + } = Config, + State = #state{ + cache_capacity = CacheCapacity, + refresh_interval = RefreshIntervalMS, + http_timeout = HTTPTimeoutMS + }, + {ok, State}. + +handle_call(Call, _From, State) -> + {reply, {error, {bad_call, Call}}, State}. + +handle_cast({evict, URL}, State0 = #state{refresh_timers = RefreshTimers0}) -> + emqx_ssl_crl_cache:delete(URL), + MTimer = maps:get(URL, RefreshTimers0, undefined), + emqx_misc:cancel_timer(MTimer), + RefreshTimers = maps:without([URL], RefreshTimers0), + State = State0#state{refresh_timers = RefreshTimers}, + ?tp( + crl_cache_evict, + #{url => URL} + ), + {noreply, State}; +handle_cast({register_der_crls, URL, CRLs}, State0) -> + handle_register_der_crls(State0, URL, CRLs); +handle_cast({refresh, URL}, State0) -> + case do_http_fetch_and_cache(URL, State0#state.http_timeout) of + {error, Error} -> + ?tp(crl_refresh_failure, #{error => Error, url => URL}), + ?SLOG(error, #{ + msg => "failed_to_fetch_crl_response", + url => URL, + error => Error + }), + {noreply, ensure_timer(URL, State0, ?RETRY_TIMEOUT)}; + {ok, _CRLs} -> + ?SLOG(debug, #{ + msg => "fetched_crl_response", + url => URL + }), + {noreply, ensure_timer(URL, State0)} + end; +handle_cast(_Cast, State) -> + {noreply, State}. + +handle_info( + {timeout, TRef, {refresh, URL}}, + State = #state{ + refresh_timers = RefreshTimers, + http_timeout = HTTPTimeoutMS + } +) -> + case maps:get(URL, RefreshTimers, undefined) of + TRef -> + ?tp(debug, crl_refresh_timer, #{url => URL}), + case do_http_fetch_and_cache(URL, HTTPTimeoutMS) of + {error, Error} -> + ?SLOG(error, #{ + msg => "failed_to_fetch_crl_response", + url => URL, + error => Error + }), + {noreply, ensure_timer(URL, State, ?RETRY_TIMEOUT)}; + {ok, _CRLs} -> + ?tp(debug, crl_refresh_timer_done, #{url => URL}), + {noreply, ensure_timer(URL, State)} + end; + _ -> + {noreply, State} + end; +handle_info(_Info, State) -> + {noreply, State}. + +%%-------------------------------------------------------------------- +%% internal functions +%%-------------------------------------------------------------------- + +http_get(URL, HTTPTimeout) -> + httpc:request( + get, + {URL, [{"connection", "close"}]}, + [{timeout, HTTPTimeout}], + [{body_format, binary}] + ). + +do_http_fetch_and_cache(URL, HTTPTimeoutMS) -> + ?tp(crl_http_fetch, #{crl_url => URL}), + Resp = ?MODULE:http_get(URL, HTTPTimeoutMS), + case Resp of + {ok, {{_, 200, _}, _, Body}} -> + case parse_crls(Body) of + error -> + {error, invalid_crl}; + CRLs -> + %% Note: must ensure it's a string and not a + %% binary because that's what the ssl manager uses + %% when doing lookups. + emqx_ssl_crl_cache:insert(to_string(URL), {der, CRLs}), + ?tp(crl_cache_insert, #{url => URL, crls => CRLs}), + {ok, CRLs} + end; + {ok, {{_, Code, _}, _, Body}} -> + {error, {bad_response, #{code => Code, body => Body}}}; + {error, Error} -> + {error, {http_error, Error}} + end. + +parse_crls(Bin) -> + try + [CRL || {'CertificateList', CRL, not_encrypted} <- public_key:pem_decode(Bin)] + catch + _:_ -> + error + end. + +ensure_timer(URL, State = #state{refresh_interval = Timeout}) -> + ensure_timer(URL, State, Timeout). + +ensure_timer(URL, State = #state{refresh_timers = RefreshTimers0}, Timeout) -> + ?tp(crl_cache_ensure_timer, #{url => URL, timeout => Timeout}), + MTimer = maps:get(URL, RefreshTimers0, undefined), + emqx_misc:cancel_timer(MTimer), + RefreshTimers = RefreshTimers0#{ + URL => emqx_misc:start_timer( + Timeout, + {refresh, URL} + ) + }, + State#state{refresh_timers = RefreshTimers}. + +-spec gather_config() -> + #{ + cache_capacity := pos_integer(), + refresh_interval := timer:time(), + http_timeout := timer:time() + }. +gather_config() -> + %% TODO: add a config handler to refresh the config when those + %% globals change? + CacheCapacity = emqx_config:get([crl_cache, capacity], ?DEFAULT_CACHE_CAPACITY), + RefreshIntervalMS0 = emqx_config:get([crl_cache, refresh_interval], ?DEFAULT_REFRESH_INTERVAL), + MinimumRefreshInverval = ?MIN_REFRESH_PERIOD, + RefreshIntervalMS = max(RefreshIntervalMS0, MinimumRefreshInverval), + HTTPTimeoutMS = emqx_config:get([crl_cache, http_timeout], ?HTTP_TIMEOUT), + #{ + cache_capacity => CacheCapacity, + refresh_interval => RefreshIntervalMS, + http_timeout => HTTPTimeoutMS + }. + +-spec handle_register_der_crls(state(), url(), [public_key:der_encoded()]) -> {noreply, state()}. +handle_register_der_crls(State0, URL0, CRLs) -> + #state{cached_urls = CachedURLs0} = State0, + URL = to_string(URL0), + case sets:is_element(URL, CachedURLs0) of + true -> + {noreply, State0}; + false -> + emqx_ssl_crl_cache:insert(URL, {der, CRLs}), + ?tp(debug, new_crl_url_inserted, #{url => URL}), + State1 = do_register_url(State0, URL), + State2 = handle_cache_overflow(State1), + State = ensure_timer(URL, State2), + {noreply, State} + end. + +-spec do_register_url(state(), url()) -> state(). +do_register_url(State0, URL) -> + #state{ + cached_urls = CachedURLs0, + insertion_times = InsertionTimes0 + } = State0, + Now = erlang:monotonic_time(), + CachedURLs = sets:add_element(URL, CachedURLs0), + InsertionTimes = gb_trees:enter(Now, URL, InsertionTimes0), + State0#state{ + cached_urls = CachedURLs, + insertion_times = InsertionTimes + }. + +-spec handle_cache_overflow(state()) -> state(). +handle_cache_overflow(State0) -> + #state{ + cached_urls = CachedURLs0, + insertion_times = InsertionTimes0, + cache_capacity = CacheCapacity, + refresh_timers = RefreshTimers0 + } = State0, + case sets:size(CachedURLs0) > CacheCapacity of + false -> + State0; + true -> + {_Time, OldestURL, InsertionTimes} = gb_trees:take_smallest(InsertionTimes0), + emqx_ssl_crl_cache:delete(OldestURL), + MTimer = maps:get(OldestURL, RefreshTimers0, undefined), + emqx_misc:cancel_timer(MTimer), + RefreshTimers = maps:remove(OldestURL, RefreshTimers0), + CachedURLs = sets:del_element(OldestURL, CachedURLs0), + ?tp(debug, crl_cache_overflow, #{oldest_url => OldestURL}), + State0#state{ + insertion_times = InsertionTimes, + cached_urls = CachedURLs, + refresh_timers = RefreshTimers + } + end. + +to_string(B) when is_binary(B) -> + binary_to_list(B); +to_string(L) when is_list(L) -> + L. diff --git a/apps/emqx/src/emqx_kernel_sup.erl b/apps/emqx/src/emqx_kernel_sup.erl index 9d2f71068..1027ef639 100644 --- a/apps/emqx/src/emqx_kernel_sup.erl +++ b/apps/emqx/src/emqx_kernel_sup.erl @@ -36,7 +36,8 @@ init([]) -> child_spec(emqx_stats, worker), child_spec(emqx_metrics, worker), child_spec(emqx_authn_authz_metrics_sup, supervisor), - child_spec(emqx_ocsp_cache, worker) + child_spec(emqx_ocsp_cache, worker), + child_spec(emqx_crl_cache, worker) ] }}. diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 97bc15ad3..b351212a7 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -487,7 +487,8 @@ esockd_opts(ListenerId, Type, Opts0) -> tcp -> Opts3#{tcp_options => tcp_opts(Opts0)}; ssl -> - OptsWithSNI = inject_sni_fun(ListenerId, Opts0), + OptsWithCRL = inject_crl_config(Opts0), + OptsWithSNI = inject_sni_fun(ListenerId, OptsWithCRL), SSLOpts = ssl_opts(OptsWithSNI), Opts3#{ssl_options => SSLOpts, tcp_options => tcp_opts(Opts0)} end @@ -794,3 +795,17 @@ inject_sni_fun(ListenerId, Conf = #{ssl_options := #{ocsp := #{enable_ocsp_stapl emqx_ocsp_cache:inject_sni_fun(ListenerId, Conf); inject_sni_fun(_ListenerId, Conf) -> Conf. + +inject_crl_config( + Conf = #{ssl_options := #{enable_crl_check := true} = SSLOpts} +) -> + HTTPTimeout = emqx_config:get([crl_cache, http_timeout], timer:seconds(15)), + Conf#{ + ssl_options := SSLOpts#{ + %% `crl_check => true' doesn't work + crl_check => peer, + crl_cache => {emqx_ssl_crl_cache, {internal, [{http, HTTPTimeout}]}} + } + }; +inject_crl_config(Conf) -> + Conf. diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index b18534a42..25b1dd1d6 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -226,6 +226,11 @@ roots(low) -> sc( ref("trace"), #{} + )}, + {"crl_cache", + sc( + ref("crl_cache"), + #{} )} ]. @@ -794,6 +799,40 @@ fields("listeners") -> } )} ]; +fields("crl_cache") -> + %% Note: we make the refresh interval and HTTP timeout global (not + %% per-listener) because multiple SSL listeners might point to the + %% same URL. If they had diverging timeout options, it would be + %% confusing. + [ + {"refresh_interval", + sc( + duration(), + #{ + hidden => true, + default => <<"15m">>, + desc => ?DESC("crl_cache_refresh_interval") + } + )}, + {"http_timeout", + sc( + duration(), + #{ + hidden => true, + default => <<"15s">>, + desc => ?DESC("crl_cache_refresh_http_timeout") + } + )}, + {"capacity", + sc( + pos_integer(), + #{ + hidden => true, + default => 100, + desc => ?DESC("crl_cache_capacity") + } + )} + ]; fields("mqtt_tcp_listener") -> mqtt_listener(1883) ++ [ @@ -2063,6 +2102,8 @@ desc("shared_subscription_group") -> "Per group dispatch strategy for shared subscription"; desc("ocsp") -> "Per listener OCSP Stapling configuration."; +desc("crl_cache") -> + "Global CRL cache options."; desc(_) -> undefined. @@ -2260,13 +2301,22 @@ server_ssl_opts_schema(Defaults, IsRanchListener) -> required => false, validator => fun ocsp_inner_validator/1 } + )}, + {"enable_crl_check", + sc( + boolean(), + #{ + default => false, + desc => ?DESC("server_ssl_opts_schema_enable_crl_check") + } )} ] ]. mqtt_ssl_listener_ssl_options_validator(Conf) -> Checks = [ - fun ocsp_outer_validator/1 + fun ocsp_outer_validator/1, + fun crl_outer_validator/1 ], case emqx_misc:pipeline(Checks, Conf, not_used) of {ok, _, _} -> @@ -2301,6 +2351,18 @@ ocsp_inner_validator(#{<<"enable_ocsp_stapling">> := true} = Conf) -> ), ok. +crl_outer_validator( + #{<<"enable_crl_check">> := true} = SSLOpts +) -> + case maps:get(<<"verify">>, SSLOpts) of + verify_peer -> + ok; + _ -> + {error, "verify must be verify_peer when CRL check is enabled"} + end; +crl_outer_validator(_SSLOpts) -> + ok. + %% @doc Make schema for SSL client. -spec client_ssl_opts_schema(map()) -> hocon_schema:field_schema(). client_ssl_opts_schema(Defaults) -> diff --git a/apps/emqx/src/emqx_ssl_crl_cache.erl b/apps/emqx/src/emqx_ssl_crl_cache.erl new file mode 100644 index 000000000..13eccbd83 --- /dev/null +++ b/apps/emqx/src/emqx_ssl_crl_cache.erl @@ -0,0 +1,237 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2015-2022. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% + +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%---------------------------------------------------------------------- +% Based on `otp/lib/ssl/src/ssl_crl_cache.erl' +%---------------------------------------------------------------------- + +%---------------------------------------------------------------------- +%% Purpose: Simple default CRL cache +%%---------------------------------------------------------------------- + +-module(emqx_ssl_crl_cache). + +-include_lib("ssl/src/ssl_internal.hrl"). +-include_lib("public_key/include/public_key.hrl"). + +-behaviour(ssl_crl_cache_api). + +-export_type([crl_src/0, uri/0]). +-type crl_src() :: {file, file:filename()} | {der, public_key:der_encoded()}. +-type uri() :: uri_string:uri_string(). + +-export([lookup/3, select/2, fresh_crl/2]). +-export([insert/1, insert/2, delete/1]). + +%% Allow usage of OTP certificate record fields (camelCase). +-elvis([ + {elvis_style, atom_naming_convention, #{ + regex => "^([a-z][a-z0-9]*_?)([a-zA-Z0-9]*_?)*$", + enclosed_atoms => ".*" + }} +]). + +%%==================================================================== +%% Cache callback API +%%==================================================================== + +lookup( + #'DistributionPoint'{distributionPoint = {fullName, Names}}, + _Issuer, + CRLDbInfo +) -> + get_crls(Names, CRLDbInfo); +lookup(_, _, _) -> + not_available. + +select(GenNames, CRLDbHandle) when is_list(GenNames) -> + lists:flatmap( + fun + ({directoryName, Issuer}) -> + select(Issuer, CRLDbHandle); + (_) -> + [] + end, + GenNames + ); +select(Issuer, {{_Cache, Mapping}, _}) -> + case ssl_pkix_db:lookup(Issuer, Mapping) of + undefined -> + []; + CRLs -> + CRLs + end. + +fresh_crl(#'DistributionPoint'{distributionPoint = {fullName, Names}}, CRL) -> + case get_crls(Names, undefined) of + not_available -> + CRL; + NewCRL -> + NewCRL + end. + +%%==================================================================== +%% API +%%==================================================================== + +insert(CRLs) -> + insert(?NO_DIST_POINT, CRLs). + +insert(URI, {file, File}) when is_list(URI) -> + case file:read_file(File) of + {ok, PemBin} -> + PemEntries = public_key:pem_decode(PemBin), + CRLs = [ + CRL + || {'CertificateList', CRL, not_encrypted} <- + PemEntries + ], + do_insert(URI, CRLs); + Error -> + Error + end; +insert(URI, {der, CRLs}) -> + do_insert(URI, CRLs). + +delete({file, File}) -> + case file:read_file(File) of + {ok, PemBin} -> + PemEntries = public_key:pem_decode(PemBin), + CRLs = [ + CRL + || {'CertificateList', CRL, not_encrypted} <- + PemEntries + ], + ssl_manager:delete_crls({?NO_DIST_POINT, CRLs}); + Error -> + Error + end; +delete({der, CRLs}) -> + ssl_manager:delete_crls({?NO_DIST_POINT, CRLs}); +delete(URI) -> + case uri_string:normalize(URI, [return_map]) of + #{scheme := "http", path := Path} -> + ssl_manager:delete_crls(string:trim(Path, leading, "/")); + _ -> + {error, {only_http_distribution_points_supported, URI}} + end. + +%%-------------------------------------------------------------------- +%%% Internal functions +%%-------------------------------------------------------------------- +do_insert(URI, CRLs) -> + case uri_string:normalize(URI, [return_map]) of + #{scheme := "http", path := Path} -> + ssl_manager:insert_crls(string:trim(Path, leading, "/"), CRLs); + _ -> + {error, {only_http_distribution_points_supported, URI}} + end. + +get_crls([], _) -> + not_available; +get_crls( + [{uniformResourceIdentifier, "http" ++ _ = URL} | Rest], + CRLDbInfo +) -> + case cache_lookup(URL, CRLDbInfo) of + [] -> + handle_http(URL, Rest, CRLDbInfo); + CRLs -> + CRLs + end; +get_crls([_ | Rest], CRLDbInfo) -> + %% unsupported CRL location + get_crls(Rest, CRLDbInfo). + +http_lookup(URL, Rest, CRLDbInfo, Timeout) -> + case application:ensure_started(inets) of + ok -> + http_get(URL, Rest, CRLDbInfo, Timeout); + _ -> + get_crls(Rest, CRLDbInfo) + end. + +http_get(URL, Rest, CRLDbInfo, Timeout) -> + case emqx_crl_cache:http_get(URL, Timeout) of + {ok, {_Status, _Headers, Body}} -> + case Body of + <<"-----BEGIN", _/binary>> -> + Pem = public_key:pem_decode(Body), + CRLs = lists:filtermap( + fun + ({'CertificateList', CRL, not_encrypted}) -> + {true, CRL}; + (_) -> + false + end, + Pem + ), + emqx_crl_cache:register_der_crls(URL, CRLs), + CRLs; + _ -> + try public_key:der_decode('CertificateList', Body) of + _ -> + CRLs = [Body], + emqx_crl_cache:register_der_crls(URL, CRLs), + CRLs + catch + _:_ -> + get_crls(Rest, CRLDbInfo) + end + end; + {error, _Reason} -> + get_crls(Rest, CRLDbInfo) + end. + +cache_lookup(_, undefined) -> + []; +cache_lookup(URL, {{Cache, _}, _}) -> + #{path := Path} = uri_string:normalize(URL, [return_map]), + case ssl_pkix_db:lookup(string:trim(Path, leading, "/"), Cache) of + undefined -> + []; + [CRLs] -> + CRLs + end. + +handle_http(URI, Rest, {_, [{http, Timeout}]} = CRLDbInfo) -> + CRLs = http_lookup(URI, Rest, CRLDbInfo, Timeout), + %% Uncomment to improve performance, but need to + %% implement cache limit and or cleaning to prevent + %% DoS attack possibilities + %%insert(URI, {der, CRLs}), + CRLs; +handle_http(_, Rest, CRLDbInfo) -> + get_crls(Rest, CRLDbInfo). diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index 38f30b8c5..79dc4cd76 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -262,12 +262,13 @@ app_schema(App) -> end. mustache_vars(App, Opts) -> - ExtraMustacheVars = maps:get(extra_mustache_vars, Opts, []), - [ - {platform_data_dir, app_path(App, "data")}, - {platform_etc_dir, app_path(App, "etc")}, - {platform_log_dir, app_path(App, "log")} - ] ++ ExtraMustacheVars. + ExtraMustacheVars = maps:get(extra_mustache_vars, Opts, #{}), + Defaults = #{ + platform_data_dir => app_path(App, "data"), + platform_etc_dir => app_path(App, "etc"), + platform_log_dir => app_path(App, "log") + }, + maps:merge(Defaults, ExtraMustacheVars). render_config_file(ConfigFile, Vars0) -> Temp = @@ -275,7 +276,7 @@ render_config_file(ConfigFile, Vars0) -> {ok, T} -> T; {error, Reason} -> error({failed_to_read_config_template, ConfigFile, Reason}) end, - Vars = [{atom_to_list(N), iolist_to_binary(V)} || {N, V} <- Vars0], + Vars = [{atom_to_list(N), iolist_to_binary(V)} || {N, V} <- maps:to_list(Vars0)], Targ = bbmustache:render(Temp, Vars), NewName = ConfigFile ++ ".rendered", ok = file:write_file(NewName, Targ), diff --git a/apps/emqx/test/emqx_crl_cache_SUITE.erl b/apps/emqx/test/emqx_crl_cache_SUITE.erl new file mode 100644 index 000000000..7a61f7835 --- /dev/null +++ b/apps/emqx/test/emqx_crl_cache_SUITE.erl @@ -0,0 +1,1057 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_crl_cache_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +%% from ssl_manager.erl +-record(state, { + session_cache_client, + session_cache_client_cb, + session_lifetime, + certificate_db, + session_validation_timer, + session_cache_client_max, + session_client_invalidator, + options, + client_session_order +}). + +-define(DEFAULT_URL, "http://localhost:9878/intermediate.crl.pem"). + +%%-------------------------------------------------------------------- +%% CT boilerplate +%%-------------------------------------------------------------------- + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + application:load(emqx), + emqx_config:save_schema_mod_and_names(emqx_schema), + emqx_common_test_helpers:boot_modules(all), + Config. + +end_per_suite(_Config) -> + ok. + +init_per_testcase(TestCase, Config) when + TestCase =:= t_cache; + TestCase =:= t_filled_cache; + TestCase =:= t_revoked +-> + ct:timetrap({seconds, 30}), + DataDir = ?config(data_dir, Config), + CRLFile = filename:join([DataDir, "intermediate-revoked.crl.pem"]), + {ok, CRLPem} = file:read_file(CRLFile), + [{'CertificateList', CRLDer, not_encrypted}] = public_key:pem_decode(CRLPem), + ok = snabbkaffe:start_trace(), + ServerPid = start_crl_server(CRLPem), + IsCached = lists:member(TestCase, [t_filled_cache, t_revoked]), + ok = setup_crl_options(Config, #{is_cached => IsCached}), + [ + {crl_pem, CRLPem}, + {crl_der, CRLDer}, + {http_server, ServerPid} + | Config + ]; +init_per_testcase(t_revoke_then_refresh, Config) -> + ct:timetrap({seconds, 120}), + DataDir = ?config(data_dir, Config), + CRLFileNotRevoked = filename:join([DataDir, "intermediate-not-revoked.crl.pem"]), + {ok, CRLPemNotRevoked} = file:read_file(CRLFileNotRevoked), + [{'CertificateList', CRLDerNotRevoked, not_encrypted}] = public_key:pem_decode( + CRLPemNotRevoked + ), + CRLFileRevoked = filename:join([DataDir, "intermediate-revoked.crl.pem"]), + {ok, CRLPemRevoked} = file:read_file(CRLFileRevoked), + [{'CertificateList', CRLDerRevoked, not_encrypted}] = public_key:pem_decode(CRLPemRevoked), + ok = snabbkaffe:start_trace(), + ServerPid = start_crl_server(CRLPemNotRevoked), + ExtraVars = #{refresh_interval => <<"10s">>}, + ok = setup_crl_options(Config, #{is_cached => true, extra_vars => ExtraVars}), + [ + {crl_pem_not_revoked, CRLPemNotRevoked}, + {crl_der_not_revoked, CRLDerNotRevoked}, + {crl_pem_revoked, CRLPemRevoked}, + {crl_der_revoked, CRLDerRevoked}, + {http_server, ServerPid} + | Config + ]; +init_per_testcase(t_cache_overflow, Config) -> + ct:timetrap({seconds, 120}), + DataDir = ?config(data_dir, Config), + CRLFileRevoked = filename:join([DataDir, "intermediate-revoked.crl.pem"]), + {ok, CRLPemRevoked} = file:read_file(CRLFileRevoked), + ok = snabbkaffe:start_trace(), + ServerPid = start_crl_server(CRLPemRevoked), + ExtraVars = #{cache_capacity => <<"2">>}, + ok = setup_crl_options(Config, #{is_cached => false, extra_vars => ExtraVars}), + [ + {http_server, ServerPid} + | Config + ]; +init_per_testcase(t_not_cached_and_unreachable, Config) -> + ct:timetrap({seconds, 30}), + DataDir = ?config(data_dir, Config), + CRLFile = filename:join([DataDir, "intermediate-revoked.crl.pem"]), + {ok, CRLPem} = file:read_file(CRLFile), + [{'CertificateList', CRLDer, not_encrypted}] = public_key:pem_decode(CRLPem), + ok = snabbkaffe:start_trace(), + application:stop(cowboy), + ok = setup_crl_options(Config, #{is_cached => false}), + [ + {crl_pem, CRLPem}, + {crl_der, CRLDer} + | Config + ]; +init_per_testcase(t_refresh_config, Config) -> + ct:timetrap({seconds, 30}), + DataDir = ?config(data_dir, Config), + CRLFile = filename:join([DataDir, "intermediate-revoked.crl.pem"]), + {ok, CRLPem} = file:read_file(CRLFile), + [{'CertificateList', CRLDer, not_encrypted}] = public_key:pem_decode(CRLPem), + TestPid = self(), + ok = meck:new(emqx_crl_cache, [non_strict, passthrough, no_history, no_link]), + meck:expect( + emqx_crl_cache, + http_get, + fun(URL, _HTTPTimeout) -> + ct:pal("http get crl ~p", [URL]), + TestPid ! {http_get, URL}, + {ok, {{"HTTP/1.0", 200, "OK"}, [], CRLPem}} + end + ), + ok = snabbkaffe:start_trace(), + ok = setup_crl_options(Config, #{is_cached => false}), + [ + {crl_pem, CRLPem}, + {crl_der, CRLDer} + | Config + ]; +init_per_testcase(TestCase, Config) when + TestCase =:= t_update_listener; + TestCase =:= t_validations +-> + %% when running emqx standalone tests, we can't use those + %% features. + case does_module_exist(emqx_mgmt_api_test_util) of + true -> + ct:timetrap({seconds, 30}), + DataDir = ?config(data_dir, Config), + PrivDir = ?config(priv_dir, Config), + CRLFile = filename:join([DataDir, "intermediate-revoked.crl.pem"]), + {ok, CRLPem} = file:read_file(CRLFile), + ok = snabbkaffe:start_trace(), + ServerPid = start_crl_server(CRLPem), + ConfFilePath = filename:join([DataDir, "emqx_just_verify.conf"]), + emqx_mgmt_api_test_util:init_suite( + [emqx_conf], + fun emqx_mgmt_api_test_util:set_special_configs/1, + #{ + extra_mustache_vars => #{ + test_data_dir => DataDir, + test_priv_dir => PrivDir + }, + conf_file_path => ConfFilePath + } + ), + [ + {http_server, ServerPid} + | Config + ]; + false -> + [{skip_does_not_apply, true} | Config] + end; +init_per_testcase(_TestCase, Config) -> + ct:timetrap({seconds, 30}), + DataDir = ?config(data_dir, Config), + CRLFile = filename:join([DataDir, "intermediate-revoked.crl.pem"]), + {ok, CRLPem} = file:read_file(CRLFile), + [{'CertificateList', CRLDer, not_encrypted}] = public_key:pem_decode(CRLPem), + TestPid = self(), + ok = meck:new(emqx_crl_cache, [non_strict, passthrough, no_history, no_link]), + meck:expect( + emqx_crl_cache, + http_get, + fun(URL, _HTTPTimeout) -> + ct:pal("http get crl ~p", [URL]), + TestPid ! {http_get, URL}, + {ok, {{"HTTP/1.0", 200, 'OK'}, [], CRLPem}} + end + ), + ok = snabbkaffe:start_trace(), + [ + {crl_pem, CRLPem}, + {crl_der, CRLDer} + | Config + ]. + +end_per_testcase(TestCase, Config) when + TestCase =:= t_cache; + TestCase =:= t_filled_cache; + TestCase =:= t_revoked +-> + ServerPid = ?config(http_server, Config), + emqx_crl_cache_http_server:stop(ServerPid), + emqx_common_test_helpers:stop_apps([]), + clear_listeners(), + application:stop(cowboy), + clear_crl_cache(), + ok = snabbkaffe:stop(), + ok; +end_per_testcase(TestCase, Config) when + TestCase =:= t_revoke_then_refresh; + TestCase =:= t_cache_overflow +-> + ServerPid = ?config(http_server, Config), + emqx_crl_cache_http_server:stop(ServerPid), + emqx_common_test_helpers:stop_apps([]), + clear_listeners(), + clear_crl_cache(), + application:stop(cowboy), + ok = snabbkaffe:stop(), + ok; +end_per_testcase(t_not_cached_and_unreachable, _Config) -> + emqx_common_test_helpers:stop_apps([]), + clear_listeners(), + clear_crl_cache(), + ok = snabbkaffe:stop(), + ok; +end_per_testcase(t_refresh_config, _Config) -> + meck:unload([emqx_crl_cache]), + clear_crl_cache(), + emqx_common_test_helpers:stop_apps([]), + clear_listeners(), + clear_crl_cache(), + application:stop(cowboy), + ok = snabbkaffe:stop(), + ok; +end_per_testcase(TestCase, Config) when + TestCase =:= t_update_listener; + TestCase =:= t_validations +-> + Skip = proplists:get_bool(skip_does_not_apply, Config), + case Skip of + true -> + ok; + false -> + ServerPid = ?config(http_server, Config), + emqx_crl_cache_http_server:stop(ServerPid), + emqx_mgmt_api_test_util:end_suite([emqx_conf]), + clear_listeners(), + ok = snabbkaffe:stop(), + clear_crl_cache(), + ok + end; +end_per_testcase(_TestCase, _Config) -> + meck:unload([emqx_crl_cache]), + clear_crl_cache(), + ok = snabbkaffe:stop(), + ok. + +%%-------------------------------------------------------------------- +%% Helper functions +%%-------------------------------------------------------------------- + +does_module_exist(Mod) -> + case erlang:module_loaded(Mod) of + true -> + true; + false -> + case code:ensure_loaded(Mod) of + ok -> + true; + {module, Mod} -> + true; + _ -> + false + end + end. + +clear_listeners() -> + emqx_config:put([listeners], #{}), + emqx_config:put_raw([listeners], #{}), + ok. + +assert_http_get(URL) -> + receive + {http_get, URL} -> + ok + after 1000 -> + ct:pal("mailbox: ~p", [process_info(self(), messages)]), + error({should_have_requested, URL}) + end. + +get_crl_cache_table() -> + #state{certificate_db = [_, _, _, {Ref, _}]} = sys:get_state(ssl_manager), + Ref. + +start_crl_server(Port, CRLPem) -> + {ok, LSock} = gen_tcp:listen(Port, [binary, {active, true}, reusedaddr]), + spawn_link(fun() -> accept_loop(LSock, CRLPem) end), + ok. + +accept_loop(LSock, CRLPem) -> + case gen_tcp:accept(LSock) of + {ok, Sock} -> + Worker = spawn_link(fun() -> crl_loop(Sock, CRLPem) end), + gen_tcp:controlling_process(Sock, Worker), + accept_loop(LSock, CRLPem); + {error, Reason} -> + error({accept_error, Reason}) + end. + +crl_loop(Sock, CRLPem) -> + receive + {tcp, Sock, _Data} -> + gen_tcp:send(Sock, CRLPem), + crl_loop(Sock, CRLPem); + _Msg -> + ok + end. + +drain_msgs() -> + receive + _Msg -> + drain_msgs() + after 0 -> + ok + end. + +clear_crl_cache() -> + %% reset the CRL cache + exit(whereis(ssl_manager), kill), + ok. + +force_cacertfile(Cacertfile) -> + {SSLListeners0, OtherListeners} = lists:partition( + fun(#{proto := Proto}) -> Proto =:= ssl end, + emqx:get_env(listeners) + ), + SSLListeners = + lists:map( + fun(Listener = #{opts := Opts0}) -> + SSLOpts0 = proplists:get_value(ssl_options, Opts0), + %% it injects some garbage... + SSLOpts1 = lists:keydelete(cacertfile, 1, lists:keydelete(cacertfile, 1, SSLOpts0)), + SSLOpts2 = [{cacertfile, Cacertfile} | SSLOpts1], + Opts1 = lists:keyreplace(ssl_options, 1, Opts0, {ssl_options, SSLOpts2}), + Listener#{opts => Opts1} + end, + SSLListeners0 + ), + application:set_env(emqx, listeners, SSLListeners ++ OtherListeners), + ok. + +setup_crl_options(Config, #{is_cached := IsCached} = Opts) -> + DataDir = ?config(data_dir, Config), + ConfFilePath = filename:join([DataDir, "emqx.conf"]), + Defaults = #{ + refresh_interval => <<"11m">>, + cache_capacity => <<"100">>, + test_data_dir => DataDir + }, + ExtraVars0 = maps:get(extra_vars, Opts, #{}), + ExtraVars = maps:merge(Defaults, ExtraVars0), + emqx_common_test_helpers:start_apps( + [], + fun(_) -> ok end, + #{ + extra_mustache_vars => ExtraVars, + conf_file_path => ConfFilePath + } + ), + case IsCached of + true -> + %% wait the cache to be filled + emqx_crl_cache:refresh(?DEFAULT_URL), + receive + {http_get, <>} -> ok + after 1_000 -> + ct:pal("mailbox: ~p", [process_info(self(), messages)]), + error(crl_cache_not_filled) + end; + false -> + %% ensure cache is empty + clear_crl_cache(), + ct:sleep(200), + ok + end, + drain_msgs(), + ok. + +start_crl_server(CRLPem) -> + application:ensure_all_started(cowboy), + {ok, ServerPid} = emqx_crl_cache_http_server:start_link(self(), 9878, CRLPem, []), + receive + {ServerPid, ready} -> ok + after 1000 -> error(timeout_starting_http_server) + end, + ServerPid. + +request(Method, Url, QueryParams, Body) -> + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + Opts = #{return_all => true}, + case emqx_mgmt_api_test_util:request_api(Method, Url, QueryParams, AuthHeader, Body, Opts) of + {ok, {Reason, Headers, BodyR}} -> + {ok, {Reason, Headers, emqx_json:decode(BodyR, [return_maps])}}; + Error -> + Error + end. + +get_listener_via_api(ListenerId) -> + Path = emqx_mgmt_api_test_util:api_path(["listeners", ListenerId]), + request(get, Path, [], []). + +update_listener_via_api(ListenerId, NewConfig) -> + Path = emqx_mgmt_api_test_util:api_path(["listeners", ListenerId]), + request(put, Path, [], NewConfig). + +assert_successful_connection(Config) -> + assert_successful_connection(Config, default). + +assert_successful_connection(Config, ClientNum) -> + DataDir = ?config(data_dir, Config), + Num = + case ClientNum of + default -> ""; + _ -> integer_to_list(ClientNum) + end, + ClientCert = filename:join(DataDir, "client" ++ Num ++ ".cert.pem"), + ClientKey = filename:join(DataDir, "client" ++ Num ++ ".key.pem"), + %% 1) At first, the cache is empty, and the CRL is fetched and + %% cached on the fly. + {ok, C0} = emqtt:start_link([ + {ssl, true}, + {ssl_opts, [ + {certfile, ClientCert}, + {keyfile, ClientKey} + ]}, + {port, 8883} + ]), + ?tp_span( + mqtt_client_connection, + #{client_num => ClientNum}, + begin + {ok, _} = emqtt:connect(C0), + emqtt:stop(C0), + ok + end + ). + +trace_between(Trace0, Marker1, Marker2) -> + {Trace1, [_ | _]} = ?split_trace_at(#{?snk_kind := Marker2}, Trace0), + {[_ | _], [_ | Trace2]} = ?split_trace_at(#{?snk_kind := Marker1}, Trace1), + Trace2. + +of_kinds(Trace0, Kinds0) -> + Kinds = sets:from_list(Kinds0, [{version, 2}]), + lists:filter( + fun(#{?snk_kind := K}) -> sets:is_element(K, Kinds) end, + Trace0 + ). + +%%-------------------------------------------------------------------- +%% Test cases +%%-------------------------------------------------------------------- + +t_init_empty_urls(_Config) -> + Ref = get_crl_cache_table(), + ?assertEqual([], ets:tab2list(Ref)), + ?assertMatch({ok, _}, emqx_crl_cache:start_link()), + receive + {http_get, _} -> + error(should_not_make_http_request) + after 1000 -> ok + end, + ?assertEqual([], ets:tab2list(Ref)), + ok. + +t_manual_refresh(Config) -> + CRLDer = ?config(crl_der, Config), + Ref = get_crl_cache_table(), + ?assertEqual([], ets:tab2list(Ref)), + {ok, _} = emqx_crl_cache:start_link(), + URL = "http://localhost/crl.pem", + ok = snabbkaffe:start_trace(), + ?wait_async_action( + ?assertEqual(ok, emqx_crl_cache:refresh(URL)), + #{?snk_kind := crl_cache_insert}, + 5_000 + ), + ok = snabbkaffe:stop(), + ?assertEqual( + [{"crl.pem", [CRLDer]}], + ets:tab2list(Ref) + ), + ok. + +t_refresh_request_error(_Config) -> + meck:expect( + emqx_crl_cache, + http_get, + fun(_URL, _HTTPTimeout) -> + {ok, {{"HTTP/1.0", 404, 'Not Found'}, [], <<"not found">>}} + end + ), + {ok, _} = emqx_crl_cache:start_link(), + URL = "http://localhost/crl.pem", + ?check_trace( + ?wait_async_action( + ?assertEqual(ok, emqx_crl_cache:refresh(URL)), + #{?snk_kind := crl_cache_insert}, + 5_000 + ), + fun(Trace) -> + ?assertMatch( + [#{error := {bad_response, #{code := 404}}}], + ?of_kind(crl_refresh_failure, Trace) + ), + ok + end + ), + ok = snabbkaffe:stop(), + ok. + +t_refresh_invalid_response(_Config) -> + meck:expect( + emqx_crl_cache, + http_get, + fun(_URL, _HTTPTimeout) -> + {ok, {{"HTTP/1.0", 200, 'OK'}, [], <<"not a crl">>}} + end + ), + {ok, _} = emqx_crl_cache:start_link(), + URL = "http://localhost/crl.pem", + ?check_trace( + ?wait_async_action( + ?assertEqual(ok, emqx_crl_cache:refresh(URL)), + #{?snk_kind := crl_cache_insert}, + 5_000 + ), + fun(Trace) -> + ?assertMatch( + [#{crls := []}], + ?of_kind(crl_cache_insert, Trace) + ), + ok + end + ), + ok = snabbkaffe:stop(), + ok. + +t_refresh_http_error(_Config) -> + meck:expect( + emqx_crl_cache, + http_get, + fun(_URL, _HTTPTimeout) -> + {error, timeout} + end + ), + {ok, _} = emqx_crl_cache:start_link(), + URL = "http://localhost/crl.pem", + ?check_trace( + ?wait_async_action( + ?assertEqual(ok, emqx_crl_cache:refresh(URL)), + #{?snk_kind := crl_cache_insert}, + 5_000 + ), + fun(Trace) -> + ?assertMatch( + [#{error := {http_error, timeout}}], + ?of_kind(crl_refresh_failure, Trace) + ), + ok + end + ), + ok = snabbkaffe:stop(), + ok. + +t_unknown_messages(_Config) -> + {ok, Server} = emqx_crl_cache:start_link(), + gen_server:call(Server, foo), + gen_server:cast(Server, foo), + Server ! foo, + ok. + +t_evict(_Config) -> + {ok, _} = emqx_crl_cache:start_link(), + URL = "http://localhost/crl.pem", + ?wait_async_action( + ?assertEqual(ok, emqx_crl_cache:refresh(URL)), + #{?snk_kind := crl_cache_insert}, + 5_000 + ), + Ref = get_crl_cache_table(), + ?assertMatch([{"crl.pem", _}], ets:tab2list(Ref)), + {ok, {ok, _}} = ?wait_async_action( + emqx_crl_cache:evict(URL), + #{?snk_kind := crl_cache_evict} + ), + ?assertEqual([], ets:tab2list(Ref)), + ok. + +t_cache(Config) -> + DataDir = ?config(data_dir, Config), + ClientCert = filename:join(DataDir, "client.cert.pem"), + ClientKey = filename:join(DataDir, "client.key.pem"), + %% 1) At first, the cache is empty, and the CRL is fetched and + %% cached on the fly. + {ok, C0} = emqtt:start_link([ + {ssl, true}, + {ssl_opts, [ + {certfile, ClientCert}, + {keyfile, ClientKey} + ]}, + {port, 8883} + ]), + {ok, _} = emqtt:connect(C0), + receive + {http_get, _} -> ok + after 500 -> + emqtt:stop(C0), + error(should_have_checked_server) + end, + emqtt:stop(C0), + %% 2) When another client using the cached CRL URL connects later, + %% it uses the cache. + {ok, C1} = emqtt:start_link([ + {ssl, true}, + {ssl_opts, [ + {certfile, ClientCert}, + {keyfile, ClientKey} + ]}, + {port, 8883} + ]), + {ok, _} = emqtt:connect(C1), + receive + {http_get, _} -> + emqtt:stop(C1), + error(should_not_have_checked_server) + after 500 -> ok + end, + emqtt:stop(C1), + + ok. + +t_cache_overflow(Config) -> + %% we have capacity = 2 here. + ?check_trace( + begin + %% First and second connections goes into the cache + ?tp(first_connections, #{}), + assert_successful_connection(Config, 1), + assert_successful_connection(Config, 2), + %% These should be cached + ?tp(first_reconnections, #{}), + assert_successful_connection(Config, 1), + assert_successful_connection(Config, 2), + %% A third client connects and evicts the oldest URL (1) + ?tp(first_eviction, #{}), + assert_successful_connection(Config, 3), + assert_successful_connection(Config, 3), + %% URL (1) connects again and needs to be re-cached; this + %% time, (2) gets evicted + ?tp(second_eviction, #{}), + assert_successful_connection(Config, 1), + %% TODO: force race condition where the same URL is fetched + %% at the same time and tries to be registered + ?tp(test_end, #{}), + ok + end, + fun(Trace) -> + URL1 = "http://localhost:9878/intermediate1.crl.pem", + URL2 = "http://localhost:9878/intermediate2.crl.pem", + URL3 = "http://localhost:9878/intermediate3.crl.pem", + Kinds = [ + mqtt_client_connection, + new_crl_url_inserted, + crl_cache_ensure_timer, + crl_cache_overflow + ], + Trace1 = of_kinds( + trace_between(Trace, first_connections, first_reconnections), + Kinds + ), + ?assertMatch( + [ + #{ + ?snk_kind := mqtt_client_connection, + ?snk_span := start, + client_num := 1 + }, + #{ + ?snk_kind := new_crl_url_inserted, + url := URL1 + }, + #{ + ?snk_kind := crl_cache_ensure_timer, + url := URL1 + }, + #{ + ?snk_kind := mqtt_client_connection, + ?snk_span := {complete, ok}, + client_num := 1 + }, + #{ + ?snk_kind := mqtt_client_connection, + ?snk_span := start, + client_num := 2 + }, + #{ + ?snk_kind := new_crl_url_inserted, + url := URL2 + }, + #{ + ?snk_kind := crl_cache_ensure_timer, + url := URL2 + }, + #{ + ?snk_kind := mqtt_client_connection, + ?snk_span := {complete, ok}, + client_num := 2 + } + ], + Trace1 + ), + Trace2 = of_kinds( + trace_between(Trace, first_reconnections, first_eviction), + Kinds + ), + ?assertMatch( + [ + #{ + ?snk_kind := mqtt_client_connection, + ?snk_span := start, + client_num := 1 + }, + #{ + ?snk_kind := mqtt_client_connection, + ?snk_span := {complete, ok}, + client_num := 1 + }, + #{ + ?snk_kind := mqtt_client_connection, + ?snk_span := start, + client_num := 2 + }, + #{ + ?snk_kind := mqtt_client_connection, + ?snk_span := {complete, ok}, + client_num := 2 + } + ], + Trace2 + ), + Trace3 = of_kinds( + trace_between(Trace, first_eviction, second_eviction), + Kinds + ), + ?assertMatch( + [ + #{ + ?snk_kind := mqtt_client_connection, + ?snk_span := start, + client_num := 3 + }, + #{ + ?snk_kind := new_crl_url_inserted, + url := URL3 + }, + #{ + ?snk_kind := crl_cache_overflow, + oldest_url := URL1 + }, + #{ + ?snk_kind := crl_cache_ensure_timer, + url := URL3 + }, + #{ + ?snk_kind := mqtt_client_connection, + ?snk_span := {complete, ok}, + client_num := 3 + }, + #{ + ?snk_kind := mqtt_client_connection, + ?snk_span := start, + client_num := 3 + }, + #{ + ?snk_kind := mqtt_client_connection, + ?snk_span := {complete, ok}, + client_num := 3 + } + ], + Trace3 + ), + Trace4 = of_kinds( + trace_between(Trace, second_eviction, test_end), + Kinds + ), + ?assertMatch( + [ + #{ + ?snk_kind := mqtt_client_connection, + ?snk_span := start, + client_num := 1 + }, + #{ + ?snk_kind := new_crl_url_inserted, + url := URL1 + }, + #{ + ?snk_kind := crl_cache_overflow, + oldest_url := URL2 + }, + #{ + ?snk_kind := crl_cache_ensure_timer, + url := URL1 + }, + #{ + ?snk_kind := mqtt_client_connection, + ?snk_span := {complete, ok}, + client_num := 1 + } + ], + Trace4 + ), + ok + end + ). + +%% check that the URL in the certificate is *not* checked if the cache +%% contains that URL. +t_filled_cache(Config) -> + DataDir = ?config(data_dir, Config), + ClientCert = filename:join(DataDir, "client.cert.pem"), + ClientKey = filename:join(DataDir, "client.key.pem"), + {ok, C} = emqtt:start_link([ + {ssl, true}, + {ssl_opts, [ + {certfile, ClientCert}, + {keyfile, ClientKey} + ]}, + {port, 8883} + ]), + {ok, _} = emqtt:connect(C), + receive + http_get -> + emqtt:stop(C), + error(should_have_used_cache) + after 500 -> ok + end, + emqtt:stop(C), + ok. + +%% If the CRL is not cached when the client tries to connect and the +%% CRL server is unreachable, the client will be denied connection. +t_not_cached_and_unreachable(Config) -> + DataDir = ?config(data_dir, Config), + ClientCert = filename:join(DataDir, "client.cert.pem"), + ClientKey = filename:join(DataDir, "client.key.pem"), + {ok, C} = emqtt:start_link([ + {ssl, true}, + {ssl_opts, [ + {certfile, ClientCert}, + {keyfile, ClientKey} + ]}, + {port, 8883} + ]), + Ref = get_crl_cache_table(), + ?assertEqual([], ets:tab2list(Ref)), + process_flag(trap_exit, true), + ?assertMatch({error, {{shutdown, {tls_alert, {bad_certificate, _}}}, _}}, emqtt:connect(C)), + ok. + +t_revoked(Config) -> + DataDir = ?config(data_dir, Config), + ClientCert = filename:join(DataDir, "client-revoked.cert.pem"), + ClientKey = filename:join(DataDir, "client-revoked.key.pem"), + {ok, C} = emqtt:start_link([ + {ssl, true}, + {ssl_opts, [ + {certfile, ClientCert}, + {keyfile, ClientKey} + ]}, + {port, 8883} + ]), + process_flag(trap_exit, true), + ?assertMatch({error, {{shutdown, {tls_alert, {certificate_revoked, _}}}, _}}, emqtt:connect(C)), + ok. + +t_revoke_then_refresh(Config) -> + DataDir = ?config(data_dir, Config), + CRLPemRevoked = ?config(crl_pem_revoked, Config), + ClientCert = filename:join(DataDir, "client-revoked.cert.pem"), + ClientKey = filename:join(DataDir, "client-revoked.key.pem"), + {ok, C0} = emqtt:start_link([ + {ssl, true}, + {ssl_opts, [ + {certfile, ClientCert}, + {keyfile, ClientKey} + ]}, + {port, 8883} + ]), + %% At first, the CRL contains no revoked entries, so the client + %% should be allowed connection. + ?assertMatch({ok, _}, emqtt:connect(C0)), + emqtt:stop(C0), + + %% Now we update the CRL on the server and wait for the cache to + %% be refreshed. + {true, {ok, _}} = + ?wait_async_action( + emqx_crl_cache_http_server:set_crl(CRLPemRevoked), + #{?snk_kind := crl_refresh_timer_done}, + 70_000 + ), + + %% The *same client* should now be denied connection. + {ok, C1} = emqtt:start_link([ + {ssl, true}, + {ssl_opts, [ + {certfile, ClientCert}, + {keyfile, ClientKey} + ]}, + {port, 8883} + ]), + process_flag(trap_exit, true), + ?assertMatch( + {error, {{shutdown, {tls_alert, {certificate_revoked, _}}}, _}}, emqtt:connect(C1) + ), + ok. + +%% check that we can start with a non-crl listener and restart it with +%% the new crl config. +t_update_listener(Config) -> + case proplists:get_bool(skip_does_not_apply, Config) of + true -> + ok; + false -> + do_t_update_listener(Config) + end. + +do_t_update_listener(Config) -> + DataDir = ?config(data_dir, Config), + Keyfile = filename:join([DataDir, "server.key.pem"]), + Certfile = filename:join([DataDir, "server.cert.pem"]), + Cacertfile = filename:join([DataDir, "ca-chain.cert.pem"]), + ClientCert = filename:join(DataDir, "client-revoked.cert.pem"), + ClientKey = filename:join(DataDir, "client-revoked.key.pem"), + + %% no crl at first + ListenerId = "ssl:default", + {ok, {{_, 200, _}, _, ListenerData0}} = get_listener_via_api(ListenerId), + ?assertMatch( + #{ + <<"ssl_options">> := + #{ + <<"enable_crl_check">> := false, + <<"verify">> := <<"verify_peer">> + } + }, + ListenerData0 + ), + {ok, C0} = emqtt:start_link([ + {ssl, true}, + {ssl_opts, [ + {certfile, ClientCert}, + {keyfile, ClientKey} + ]}, + {port, 8883} + ]), + %% At first, the CRL contains no revoked entries, so the client + %% should be allowed connection. + ?assertMatch({ok, _}, emqtt:connect(C0)), + emqtt:stop(C0), + + %% configure crl + CRLConfig = + #{ + <<"ssl_options">> => + #{ + <<"keyfile">> => Keyfile, + <<"certfile">> => Certfile, + <<"cacertfile">> => Cacertfile, + <<"enable_crl_check">> => true + } + }, + ListenerData1 = emqx_map_lib:deep_merge(ListenerData0, CRLConfig), + {ok, {_, _, ListenerData2}} = update_listener_via_api(ListenerId, ListenerData1), + ?assertMatch( + #{ + <<"ssl_options">> := + #{ + <<"enable_crl_check">> := true, + <<"verify">> := <<"verify_peer">> + } + }, + ListenerData2 + ), + + %% Now should use CRL information to block connection + process_flag(trap_exit, true), + {ok, C1} = emqtt:start_link([ + {ssl, true}, + {ssl_opts, [ + {certfile, ClientCert}, + {keyfile, ClientKey} + ]}, + {port, 8883} + ]), + ?assertMatch( + {error, {{shutdown, {tls_alert, {certificate_revoked, _}}}, _}}, emqtt:connect(C1) + ), + assert_http_get(<>), + + ok. + +t_validations(Config) -> + case proplists:get_bool(skip_does_not_apply, Config) of + true -> + ok; + false -> + do_t_validations(Config) + end. + +do_t_validations(_Config) -> + ListenerId = <<"ssl:default">>, + {ok, {{_, 200, _}, _, ListenerData0}} = get_listener_via_api(ListenerId), + + ListenerData1 = + emqx_map_lib:deep_merge( + ListenerData0, + #{ + <<"ssl_options">> => + #{ + <<"enable_crl_check">> => true, + <<"verify">> => <<"verify_none">> + } + } + ), + {error, {_, _, ResRaw1}} = update_listener_via_api(ListenerId, ListenerData1), + #{<<"code">> := <<"BAD_REQUEST">>, <<"message">> := MsgRaw1} = + emqx_json:decode(ResRaw1, [return_maps]), + ?assertMatch( + #{ + <<"mismatches">> := + #{ + <<"listeners:ssl_not_required_bind">> := + #{ + <<"reason">> := + <<"verify must be verify_peer when CRL check is enabled">> + } + } + }, + emqx_json:decode(MsgRaw1, [return_maps]) + ), + + ok. diff --git a/apps/emqx/test/emqx_crl_cache_SUITE_data/ca-chain.cert.pem b/apps/emqx/test/emqx_crl_cache_SUITE_data/ca-chain.cert.pem new file mode 100644 index 000000000..eaabd2445 --- /dev/null +++ b/apps/emqx/test/emqx_crl_cache_SUITE_data/ca-chain.cert.pem @@ -0,0 +1,68 @@ +-----BEGIN CERTIFICATE----- +MIIF+zCCA+OgAwIBAgICEAAwDQYJKoZIhvcNAQELBQAwbzELMAkGA1UEBhMCU0Ux +EjAQBgNVBAgMCVN0b2NraG9sbTESMBAGA1UEBwwJU3RvY2tob2xtMRIwEAYDVQQK +DAlNeU9yZ05hbWUxETAPBgNVBAsMCE15Um9vdENBMREwDwYDVQQDDAhNeVJvb3RD +QTAeFw0yMzAxMTIxMzA4MTZaFw0zMzAxMDkxMzA4MTZaMGsxCzAJBgNVBAYTAlNF +MRIwEAYDVQQIDAlTdG9ja2hvbG0xEjAQBgNVBAoMCU15T3JnTmFtZTEZMBcGA1UE +CwwQTXlJbnRlcm1lZGlhdGVDQTEZMBcGA1UEAwwQTXlJbnRlcm1lZGlhdGVDQTCC +AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALQG7dMeU/y9HDNHzhydR0bm +wN9UGplqJOJPwqJRaZZcrn9umgJ9SU2il2ceEVxMDwzBWCRKJO5/H9A9k13SqsXM +2c2c9xXfIF1kb820lCm1Uow5hZ/auDjxliNk9kNJDigCRi3QoIs/dVeWzFsgEC2l +gxRqauN2eNFb6/yXY788YALHBsCRV2NFOFXxtPsvLXpD9Q/8EqYsSMuLARRdHVNU +ryaEF5lhShpcuz0TlIuTy2TiuXJUtJ+p7a4Z7friZ6JsrmQWsVQBj44F8TJRHWzW +C7vm9c+dzEX9eqbr5iPL+L4ctMW9Lz6ePcYfIXne6CElusRUf8G+xM1uwovF9bpV ++9IqY7tAu9G1iY9iNtJgNNDKOCcOGKcZCx6Cg1XYOEKReNnUMazvYeqRrrjV5WQ0 +vOcD5zcBRNTXCddCLa7U0guXP9mQrfuk4NTH1Bt77JieTJ8cfDXHwtaKf6aGbmZP +wl1Xi/GuXNUP/xeog78RKyFwBmjt2JKwvWzMpfmH4mEkG9moh2alva+aEz6LIJuP +16g6s0Q6c793/OvUtpNcewHw4Vjn39LD9o6VLp854G4n8dVpUWSbWS+sXD1ZE69H +g/sMNMyq+09ufkbewY8xoCm/rQ1pqDZAVMWsstJEaYu7b/eb7R+RGOj1YECCV/Yp +EZPdDotbSNRkIi2d/a1NAgMBAAGjgaQwgaEwHQYDVR0OBBYEFExwhjsVUom6tQ+S +qq6xMUETvnPzMB8GA1UdIwQYMBaAFD90kfU5pc5l48THu0Ayj9SNpHuhMBIGA1Ud +EwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgGGMDsGA1UdHwQ0MDIwMKAuoCyG +Kmh0dHA6Ly9sb2NhbGhvc3Q6OTg3OC9pbnRlcm1lZGlhdGUuY3JsLnBlbTANBgkq +hkiG9w0BAQsFAAOCAgEAK6NgdWQYtPNKQNBGjsgtgqTRh+k30iqSO6Y3yE1KGABO +EuQdVqkC2qUIbCB0M0qoV0ab50KNLfU6cbshggW4LDpcMpoQpI05fukNh1jm3ZuZ +0xsB7vlmlsv00tpqmfIl/zykPDynHKOmFh/hJP/KetMy4+wDv4/+xP31UdEj5XvG +HvMtuqOS23A+H6WPU7ol7KzKBnU2zz/xekvPbUD3JqV+ynP5bgbIZHAndd0o9T8e +NFX23Us4cTenU2/ZlOq694bRzGaK+n3Ksz995Nbtzv5fbUgqmf7Mcq4iHGRVtV11 +MRyBrsXZp2vbF63c4hrf2Zd6SWRoaDKRhP2DMhajpH9zZASSTlfejg/ZRO2s+Clh +YrSTkeMAdnRt6i/q4QRcOTCfsX75RFM5v67njvTXsSaSTnAwaPi78tRtf+WSh0EP +VVPzy++BszBVlJ1VAf7soWZHCjZxZ8ZPqVTy5okoHwWQ09WmYe8GfulDh1oj0wbK +3FjN7bODWHJN+bFf5aQfK+tumYKoPG8RXL6QxpEzjFWjxhIMJHHMKfDWnAV1o1+7 +/1/aDzq7MzEYBbrgQR7oE5ZHtyqhCf9LUgw0Kr7/8QWuNAdeDCJzjXRROU0hJczp +dOyfRlLbHmLLmGOnROlx6LsGNQ17zuz6SPi7ei8/ylhykawDOAGkM1+xFakmQhM= +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIFzzCCA7egAwIBAgIUYjc7hD7/UJ0/VPADfNfp/WpOwRowDQYJKoZIhvcNAQEL +BQAwbzELMAkGA1UEBhMCU0UxEjAQBgNVBAgMCVN0b2NraG9sbTESMBAGA1UEBwwJ +U3RvY2tob2xtMRIwEAYDVQQKDAlNeU9yZ05hbWUxETAPBgNVBAsMCE15Um9vdENB +MREwDwYDVQQDDAhNeVJvb3RDQTAeFw0yMzAxMTIxMzA4MTRaFw00MzAxMDcxMzA4 +MTRaMG8xCzAJBgNVBAYTAlNFMRIwEAYDVQQIDAlTdG9ja2hvbG0xEjAQBgNVBAcM +CVN0b2NraG9sbTESMBAGA1UECgwJTXlPcmdOYW1lMREwDwYDVQQLDAhNeVJvb3RD +QTERMA8GA1UEAwwITXlSb290Q0EwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK +AoICAQCnBwSOYVJw47IoMHMXTVDtOYvUt3rqsurEhFcB4O8xmf2mmwr6m7s8A5Ft +AvAehg1GvnXT3t/KiyU7BK+acTwcErGyZwS2wvdB0lpHWSpOn/u5y+4ZETvQefcj +ZTdDOM9VN5nutpitgNb+1yL8sqSexfVbY7DnYYvFjOVBYoP/SGvM9jVjCad+0WL3 +FhuD+L8QAxzCieX3n9UMymlFwINQuEc+TDjuNcEqt+0J5EgS1fwzxb2RCVL0TNv4 +9a71hFGCNRj20AeZm99hbdufm7+0AFO7ocV5q43rLrWFUoBzqKPYIjga/cv/UdWZ +c5RLRXw3JDSrCqkf/mOlaEhNPlmWRF9MSus5Da3wuwgGCaVzmrf30rWR5aHHcscG +e+AOgJ4HayvBUQeb6ZlRXc0YlACiLToMKxuyxDyUcDfVEXpUIsDILF8dkiVQxEU3 +j9g6qjXiqPVdNiwpqXfBKObj8vNCzORnoHYs8cCgib3RgDVWeqkDmlSwlZE7CvQh +U4Loj4l7813xxzYEKkVaT1JdXPWu42CG/b4Y/+f4V+3rkJkYzUwndX6kZNksIBai +phmtvKt+CTdP1eAbT+C9AWWF3PT31+BIhuT0u9tR8BVSkXdQB8dG4M/AAJcTo640 +0mdYYOXT153gEKHJuUBm750ZTy+r6NjNvpw8VrMAakJwHqnIdQIDAQABo2MwYTAd +BgNVHQ4EFgQUP3SR9TmlzmXjxMe7QDKP1I2ke6EwHwYDVR0jBBgwFoAUP3SR9Tml +zmXjxMe7QDKP1I2ke6EwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAYYw +DQYJKoZIhvcNAQELBQADggIBAFMFv4C+I0+xOAb9v6G/IOpfPBZ1ez31EXKJJBra +lulP4nRHQMeb310JS8BIeQ3dl+7+PkSxPABZSwc3jkxdSMvhc+Z4MQtTgos+Qsjs +gH7sTqwWeeQ0lHYxWmkXijrh5OPRZwTKzYQlkcn85BCUXl2KDuNEdiqPbDTao+lc +lA0/UAvC6NCyFKq/jqf4CmW5Kx6yG1v1LaE+IXn7cbIXj+DaehocVXi0wsXqj03Q +DDUHuLHZP+LBsg4e91/0Jy2ekNRTYJifSqr+9ufHl0ZX1pFDZyf396IgZ5CQZ0PJ +nRxZHlCfsxWxmxxdy3FQSE6YwXhdTjjoAa1ApZcKkkt1beJa6/oRLze/ux5x+5q+ +4QczufHd6rjoKBi6BM3FgFQ8As5iNohHXlMHd/xITo1Go3CWw2j9TGH5vzksOElK +B0mcwwt2zwNEjvfytc+tI5jcfGN3tiT5fVHS8hw9dWKevypLL+55Ua9G8ZgDHasT +XFRJHgmnbyFcaAe26D2dSKmhC9u2mHBH+MaI8dj3e7wNBfpxNgp41aFIk+QTmiFW +VXFED6DHQ/Mxq93ACalHdYg18PlIYClbT6Pf2xXBnn33YPhn5xzoTZ+cDH/RpaQp +s0UUTSJT1UTXgtXPnZWQfvKlMjJEIiVFiLEC0sgZRlWuZDRAY0CdZJJxvQp59lqu +cbTm +-----END CERTIFICATE----- diff --git a/apps/emqx/test/emqx_crl_cache_SUITE_data/client-no-dist-points.cert.pem b/apps/emqx/test/emqx_crl_cache_SUITE_data/client-no-dist-points.cert.pem new file mode 100644 index 000000000..038eec790 --- /dev/null +++ b/apps/emqx/test/emqx_crl_cache_SUITE_data/client-no-dist-points.cert.pem @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFdTCCA12gAwIBAgICEAUwDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCU0Ux +EjAQBgNVBAgMCVN0b2NraG9sbTESMBAGA1UECgwJTXlPcmdOYW1lMRkwFwYDVQQL +DBBNeUludGVybWVkaWF0ZUNBMRkwFwYDVQQDDBBNeUludGVybWVkaWF0ZUNBMB4X +DTIzMDExODEyMzY1NloXDTMzMDQyNTEyMzY1NlowgYQxCzAJBgNVBAYTAlNFMRIw +EAYDVQQIDAlTdG9ja2hvbG0xEjAQBgNVBAcMCVN0b2NraG9sbTESMBAGA1UECgwJ +TXlPcmdOYW1lMRkwFwYDVQQLDBBNeUludGVybWVkaWF0ZUNBMR4wHAYDVQQDDBVj +bGllbnQtbm8tZGlzdC1wb2ludHMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK +AoIBAQCYQqNF7o20tEwyXphDgtwkZ628baYzQoCmmaufR+5SPQWdTN+GFeApv0dP +4y/ncZV24rgButMo73e4+wPsILwSGhaVIU0mMaCmexyC4W6INBkQsVB5FAd/YM0O +gdxS6A42h9HZTaAJ+4ftgFdOOHiP3lwicXeIYykAE7Y5ikxlnHgi8p1PTLowN4Q+ +AjuXChRzmU16cUEAevZKkTVf7VCcK66aJsxBsxfykkGHhc6qLqmlMt6Te6DPCi/R +KP/kARTDWNEkp6qtpvzByYFYAKPSZxPuryajAC3RLuGNkVSB+PZ6NnZW6ASeTdra +Lwuiwsi5XPBeFb0147naQOBzSGG/AgMBAAGjggEHMIIBAzAJBgNVHRMEAjAAMBEG +CWCGSAGG+EIBAQQEAwIFoDBBBglghkgBhvhCAQ0ENBYyT3BlblNTTCBHZW5lcmF0 +ZWQgQ2xpZW50IENlcnRpZmljYXRlIChubyBDUkwgaW5mbykwHQYDVR0OBBYEFBiV +sjDe46MixvftT/wej1mxGuN7MB8GA1UdIwQYMBaAFExwhjsVUom6tQ+Sqq6xMUET +vnPzMA4GA1UdDwEB/wQEAwIF4DAdBgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUH +AwQwMQYIKwYBBQUHAQEEJTAjMCEGCCsGAQUFBzABhhVodHRwOi8vbG9jYWxob3N0 +Ojk4NzcwDQYJKoZIhvcNAQELBQADggIBAKBEnKYVLFtZb3MI0oMJkrWBssVCq5ja +OYomZ61I13QLEeyPevTSWAcWFQ4zQDF/SWBsXjsrC+JIEjx2xac6XCpxcx3jDUgo +46u/hx2rT8tMKa60hW0V1Dk6w8ZHiCe94BlFLsWFKnn6dVzoJd2u3vgUaleh3uxF +hug8XY+wmHd36rO0kVe3DrsqdIdOfhMiJLDxU0cBA79vI5kCvqB8DIwCWtOzkA82 +EPl3Iws5NPhuFAR9u0xOQu0akzmSJFcEGLZ4qfatHD/tZGRduyFvMKy5iIeMzuEs +2etm01tfLHqgKGOKp5LjPm7Aoac/GeVoTvctGF+wayvOuYE7inlGZToz3kQMMzHZ +ZGBBgOhXbR2y74QoFv6DUqmmTRbGfiLYyErA5r881ntgciQi02xrGjoAFntvKb+H +HNB22Qprz16OmdC9dJKF2RhO6Cketdhv65wFWw6xlhRMCWYPY3CI8tWkxS4A4yit +RZQZg3yaeHXMaCAu5HxuqAQXKGjz+7w7N6diwbT7o7CfKk8iHUrGfkQ5nCS0GZ1r +lU1vgKtdzVvJ6HmBrCRcdNqh/L/wdIltwI/52j+TKRtELM1qHuLAYmhcRBW+2wuH +ewaNA9KEgEk6JC+iR8uOBi0ZLkMIm47j+ZLJRJVUfgkVEEFjyiYSFfpwwcgT+/Aw +EczVZOdUEbDM +-----END CERTIFICATE----- diff --git a/apps/emqx/test/emqx_crl_cache_SUITE_data/client-no-dist-points.key.pem b/apps/emqx/test/emqx_crl_cache_SUITE_data/client-no-dist-points.key.pem new file mode 100644 index 000000000..02b865f5e --- /dev/null +++ b/apps/emqx/test/emqx_crl_cache_SUITE_data/client-no-dist-points.key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCYQqNF7o20tEwy +XphDgtwkZ628baYzQoCmmaufR+5SPQWdTN+GFeApv0dP4y/ncZV24rgButMo73e4 ++wPsILwSGhaVIU0mMaCmexyC4W6INBkQsVB5FAd/YM0OgdxS6A42h9HZTaAJ+4ft +gFdOOHiP3lwicXeIYykAE7Y5ikxlnHgi8p1PTLowN4Q+AjuXChRzmU16cUEAevZK +kTVf7VCcK66aJsxBsxfykkGHhc6qLqmlMt6Te6DPCi/RKP/kARTDWNEkp6qtpvzB +yYFYAKPSZxPuryajAC3RLuGNkVSB+PZ6NnZW6ASeTdraLwuiwsi5XPBeFb0147na +QOBzSGG/AgMBAAECggEACSMuozq+vFJ5pCgzIRIQXgruzTkTWU4rZFQijYuGjN7m +oFsFqwlTC45UHEI5FL2nR5wxiMEKfRFp8Or3gEsyni98nXSDKcCesH8A5gXbWUcv +HeZWOv3tuUI47B709vDAMZuTB2R2L0MuFB24n5QaACBLDTIcB05UHpIQRIG9NffH +MhxqFB2kuakp67VekYGZkBCNkqfL3VQZIGRpQC8SvpnRXELqZgI4MyJgvkK6myWj +Vtpwm8YiOQoJHJx4raoVfS2NWTsCwL0M0aXMMtmM2QfMP/xB9OifxnmDDBs7Tie8 +0Wri845xLTCYthaU8B06rhoQdKXoqKmQMoF2doPm8QKBgQDN+0E0PtPkyxIho8pV +CsQnmif91EQQqWxOdkHbE96lT0UKu6ziBSbB4ClRHYil5c8p7INxRpj7pruOY3Kw +MAcacIMMBNhLBJL4R0hr/pwr18WOZxCIMcLHTaCfbVqL71TKp4/6C+GexZfaYJ46 +IZEpLU5RPmD4f9MPIDDm6KcPxwKBgQC9O9TOor93g+A4sU54CGOqvVDrdi5TnGF8 +YdimvUsT20gl2WGX5vq3OohzZi7U8FuxKHWpbgh2efqGLcFsRNFZ/T0ZXX4DDafN +Gzyu/DMVuFO4ccgFJNnl45w3/yFG40kL6yS8kss/iEYu550/uOZ1FjH+kJ0vjV6G +JD8q0PgOSQKBgG2i9cLcSia2nBEBwFlhoKS/ndeyWwRPWZGtykHUoqZ0ufgLiurG ++SkqqnM9eBVta8YR2Ki7fgQ8bApPDqWO+sjs6CPGlGXhqmSydG7fF7sSX1n7q8YC +Tn2M6RjSuOZQ3l37sFvUZSQAYmJfGPkyErTLI6uEu1KpnuqnJMBTR1DTAoGAIGQn +bx9oirqmHM4s0lsNRGKXgVZ/Y4x3G2VcQl5QhZuZY/ErxWaiL87zIF2zUnu6Fj8I +tPHCvRTwDxux6ih1dWPlm3vnX/psaK1q28ELtYIRwpanWEoQiktFqEghmBK7pDCh +3y15YOygptK6lfe+avhboml6nnMiZO+7aEbQzxECgYALuUM4fo1dQYmYuZIqZoFJ +TXGyzMkNGs61SMiD6mW6XgXj5h5T8Q0MdpmHkwsm+z9A/1of5cxkE6d8HCCz+dt5 +tnY7OC0gYB1+gDld8MZgFgP6k0qklreLVhzEz11TbMldifa1EE4VjUDG/NeAEtbq +GbLaw0NhGJtRCgL9Bc7i7g== +-----END PRIVATE KEY----- diff --git a/apps/emqx/test/emqx_crl_cache_SUITE_data/client-revoked.cert.pem b/apps/emqx/test/emqx_crl_cache_SUITE_data/client-revoked.cert.pem new file mode 100644 index 000000000..d0a23bf2f --- /dev/null +++ b/apps/emqx/test/emqx_crl_cache_SUITE_data/client-revoked.cert.pem @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFnDCCA4SgAwIBAgICEAIwDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCU0Ux +EjAQBgNVBAgMCVN0b2NraG9sbTESMBAGA1UECgwJTXlPcmdOYW1lMRkwFwYDVQQL +DBBNeUludGVybWVkaWF0ZUNBMRkwFwYDVQQDDBBNeUludGVybWVkaWF0ZUNBMB4X +DTIzMDExMjEzMDgxNloXDTMzMDQxOTEzMDgxNlowfTELMAkGA1UEBhMCU0UxEjAQ +BgNVBAgMCVN0b2NraG9sbTESMBAGA1UEBwwJU3RvY2tob2xtMRIwEAYDVQQKDAlN +eU9yZ05hbWUxGTAXBgNVBAsMEE15SW50ZXJtZWRpYXRlQ0ExFzAVBgNVBAMMDmNs +aWVudC1yZXZva2VkMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAs+R6 +PDtIxVlUoLYbDBbaVcxgoLjnWcvqL8wSqyWuqi/Y3cjuNYCziR9nR5dWajtkBjzJ +HyhgAr6gBVSRt4RRmDXoOcprK3GcpowAr65UAmC4hdH0af6FdKjKCnFw67byUg52 +f7ueXZ6t/XuuKxlU/f2rjXVwmmnlhBi5EHDkXxvfgWXJekDfsPbW9j0kaCUWCpfj +rzGbfkXqrPkslO41PYlCbPxoiRItJjindFjcQySYvRq7A2uYMGsrxv4n3rzo5NGt +goBmnGj61ii9WOdopcFxKirhIB9zrxC4x0opRfIaF/n1ZXk6NOnaDxu1LTZ18wfC +ZB979ge6pleeKoPf7QIDAQABo4IBNjCCATIwCQYDVR0TBAIwADARBglghkgBhvhC +AQEEBAMCBaAwMwYJYIZIAYb4QgENBCYWJE9wZW5TU0wgR2VuZXJhdGVkIENsaWVu +dCBDZXJ0aWZpY2F0ZTAdBgNVHQ4EFgQUQeItXr3nc6CZ++G9UCoq1YlQ9oowHwYD +VR0jBBgwFoAUTHCGOxVSibq1D5KqrrExQRO+c/MwDgYDVR0PAQH/BAQDAgXgMB0G +A1UdJQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDBDA7BgNVHR8ENDAyMDCgLqAshipo +dHRwOi8vbG9jYWxob3N0Ojk4NzgvaW50ZXJtZWRpYXRlLmNybC5wZW0wMQYIKwYB +BQUHAQEEJTAjMCEGCCsGAQUFBzABhhVodHRwOi8vbG9jYWxob3N0Ojk4NzcwDQYJ +KoZIhvcNAQELBQADggIBAIFuhokODd54/1B2JiNyG6FMq/2z8B+UquC2iw3p2pyM +g/Jz4Ouvg6gGwUwmykEua06FRCxx5vJ5ahdhXvKst/zH/0qmYTFNMhNsDy76J/Ot +Ss+VwQ8ddpEG3EIUI9BQxB3xL7z7kRQzploQjakNcDWtDt1BmN05Iy2vz4lnYJky +Kss6ya9jEkNibHekhxJuchJ0fVGlVe74MO7RNDFG7+O3tMlxu0zH/LpW093V7BI2 +snXNAwQBizvWTrDKWLDu5JsX8KKkrmDtFTs9gegnxDCOYdtG5GbbMq+H1SjWUJPV +wiXTF8/eE02s4Jzm7ZAxre4bRt/hAg7xTGmDQ1Hn+LzLn18I9LaW5ZWqSwwpgv+g +Z/jiLO9DJ/y525Cl7DLCpSFoDTWlQXouKhcgALcVay/cXCsZ3oFZCustburLiJi/ +zgBeEk1gVpwljriJLeZifyfWtJx6yfgB/h6fid8XLsGRD+Yc8Tzs8J1LIgi+j4ZT +UzKX3B85Kht/dr43UDMtWOF3edkOMaJu7rcg5tTsK+LIyHtXvebKPVvvA9f27Dz/ +4gmhAwwqS87Xv3FMVhZ03DNOJ6XAF+T6OTEqwYs+iK56IMSl1Jy+bCzo0j5jZVbl +XFwGxUHzM7pfM6PDx657oUxG1QwM/fIWA18F+kY/yigXxq6pYMeAiQsPanOThgHp +-----END CERTIFICATE----- diff --git a/apps/emqx/test/emqx_crl_cache_SUITE_data/client-revoked.key.pem b/apps/emqx/test/emqx_crl_cache_SUITE_data/client-revoked.key.pem new file mode 100644 index 000000000..0b7698da9 --- /dev/null +++ b/apps/emqx/test/emqx_crl_cache_SUITE_data/client-revoked.key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCz5Ho8O0jFWVSg +thsMFtpVzGCguOdZy+ovzBKrJa6qL9jdyO41gLOJH2dHl1ZqO2QGPMkfKGACvqAF +VJG3hFGYNeg5ymsrcZymjACvrlQCYLiF0fRp/oV0qMoKcXDrtvJSDnZ/u55dnq39 +e64rGVT9/auNdXCaaeWEGLkQcORfG9+BZcl6QN+w9tb2PSRoJRYKl+OvMZt+Reqs ++SyU7jU9iUJs/GiJEi0mOKd0WNxDJJi9GrsDa5gwayvG/ifevOjk0a2CgGacaPrW +KL1Y52ilwXEqKuEgH3OvELjHSilF8hoX+fVleTo06doPG7UtNnXzB8JkH3v2B7qm +V54qg9/tAgMBAAECggEAAml+HRgjZ+gEezot3yngSBW7NvR7v6e9DmKDXpGdB7Go +DANBdGyzG5PU9/AGy9pbgzzl6nnJXcgOD7w8TvRifrK8WCgHa1f05IPMj458GGMR +HlQ8HX647eFEgkLWo4Z6tdB1VM2geDtkNFmn8nJ+wgAYgIdSWPOyDOUi+B43ZbIN +eaLWkP2fiX9tcJp41cytW+ng2YIm4s90Nt4FJPNBNzOrhVm35jciId02MmEjCEnr +0YbK9uoMDC2YLg8vhRcjtsUHV2rREkwEAQj8nCWvWWheIwk943d6OicGAD/yebpV +PTjtlZlpIbrovfvuMcoTxJg3WS8LTg/+cNWAX5a3eQKBgQDcRY7nVSJusYyN0Bij +YWc9H47wU+YucaGT25xKe26w1pl6s4fmr1Sc3NcaN2iyUv4BuAvaQzymHe4g9deU +D9Ws/NCQ9EjHJJsklNyn2KCgkSp7oPKhPwyl64XfPdV2gr5AD6MILf7Rkyib5sSf +1WK8i25KatT7M4mCtrBVJYHNpQKBgQDREjwPIaQBPXouVpnHhSwRHfKD0B1a2koq +4VE6Fnf3ogkiGfV9kqXwIfPHL0tfotFraM3FFmld8RcxhKUPr4oj+K9KTxmMD9lm +9Hal0ANXYmHs5a1iHyoNmTpBGHALWLT9fCoeg+EIYabi2+P1c7cDIdUPkEzo4GmI +nCIpv7hGqQKBgEFUC+8GK+EinWoN1tDV+ZWCP5V9fJ43q1E7592bQBgIfZqLlnnP +dEvVn6Ix3sZMoPMHj9Ra7qjh5Zc28ooCLEBS9tSW7uLJM44k7FCHihQ1GaFy+aLj +HTA0aw7rutycKCq9uH+bjKDBgWDDj3tMAS2kOMCvcJ1UCquO3TtTlWzVAoGBAIDN +8yJ/X0NEVNnnkKZTbWq+QILk3LD0e20fk6Nt5Es0ENxpkczjZEglIsM8Z/trnAnI +b71UqWWu+tMPHYIka77tn1DwmpSnzxCW2+Ib3XMgsaP5fHBPMuFd3X3tSFo1NIxW +yrwyE5nOT7rELhUyTTYoydLk2/09BMedKY7/BtDBAoGAXeX1pX74K1i/uWyYKwYZ +sskRueSo9whDJuZWgNiUovArr57eA+oA+bKdFpiE419348bkFF8jNoGFQ6MXMedD +LqHAYIj+ZPIC4+rObHqO5EaIyblgutwx3citkQp7HXDBxojnOKA9mKQXj1vxCaL1 +/1fFNJQCzEqwnKwnhI2MJ28= +-----END PRIVATE KEY----- diff --git a/apps/emqx/test/emqx_crl_cache_SUITE_data/client.cert.pem b/apps/emqx/test/emqx_crl_cache_SUITE_data/client.cert.pem new file mode 100644 index 000000000..b37d1b0ba --- /dev/null +++ b/apps/emqx/test/emqx_crl_cache_SUITE_data/client.cert.pem @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFljCCA36gAwIBAgICEAEwDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCU0Ux +EjAQBgNVBAgMCVN0b2NraG9sbTESMBAGA1UECgwJTXlPcmdOYW1lMRkwFwYDVQQL +DBBNeUludGVybWVkaWF0ZUNBMRkwFwYDVQQDDBBNeUludGVybWVkaWF0ZUNBMB4X +DTIzMDExMjEzMDgxNloXDTMzMDQxOTEzMDgxNlowdzELMAkGA1UEBhMCU0UxEjAQ +BgNVBAgMCVN0b2NraG9sbTESMBAGA1UEBwwJU3RvY2tob2xtMRIwEAYDVQQKDAlN +eU9yZ05hbWUxGTAXBgNVBAsMEE15SW50ZXJtZWRpYXRlQ0ExETAPBgNVBAMMCE15 +Q2xpZW50MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvGuAShewEo8V +/+aWVO/MuUt92m8K0Ut4nC2gOvpjMjf8mhSSf6KfnxPklsFwP4fdyPOjOiXwCsf3 +1QO5fjVr8to3iGTHhEyZpzRcRqmw1eYJC7iDh3BqtYLAT30R+Kq6Mk+f4tXB5Lp/ +2jXgdi0wshWagCPgJO3CtiwGyE8XSa+Q6EBYwzgh3NFbgYdJma4x+S86Y/5WfmXP +zF//UipsFp4gFUqwGuj6kJrN9NnA1xCiuOxCyN4JuFNMfM/tkeh26jAp0OHhJGsT +s3YiUm9Dpt7Rs7o0so9ov9K+hgDFuQw9HZW3WIJI99M5a9QZ4ZEQqKpABtYBl/Nb +VPXcr+T3fQIDAQABo4IBNjCCATIwCQYDVR0TBAIwADARBglghkgBhvhCAQEEBAMC +BaAwMwYJYIZIAYb4QgENBCYWJE9wZW5TU0wgR2VuZXJhdGVkIENsaWVudCBDZXJ0 +aWZpY2F0ZTAdBgNVHQ4EFgQUOIChBA5aZB0dPWEtALfMIfSopIIwHwYDVR0jBBgw +FoAUTHCGOxVSibq1D5KqrrExQRO+c/MwDgYDVR0PAQH/BAQDAgXgMB0GA1UdJQQW +MBQGCCsGAQUFBwMCBggrBgEFBQcDBDA7BgNVHR8ENDAyMDCgLqAshipodHRwOi8v +bG9jYWxob3N0Ojk4NzgvaW50ZXJtZWRpYXRlLmNybC5wZW0wMQYIKwYBBQUHAQEE +JTAjMCEGCCsGAQUFBzABhhVodHRwOi8vbG9jYWxob3N0Ojk4NzcwDQYJKoZIhvcN +AQELBQADggIBAE0qTL5WIWcxRPU9oTrzJ+oxMTp1JZ7oQdS+ZekLkQ8mP7T6C/Ew +6YftjvkopnHUvn842+PTRXSoEtlFiTccmA60eMAai2tn5asxWBsLIRC9FH3LzOgV +/jgyY7HXuh8XyDBCDD+Sj9QityO+accTHijYAbHPAVBwmZU8nO5D/HsxLjRrCfQf +qf4OQpX3l1ryOi19lqoRXRGwcoZ95dqq3YgTMlLiEqmerQZSR6iSPELw3bcwnAV1 +hoYYzeKps3xhwszCTz2+WaSsUO2sQlcFEsZ9oHex/02UiM4a8W6hGFJl5eojErxH +7MqaSyhwwyX6yt8c75RlNcUThv+4+TLkUTbTnWgC9sFjYfd5KSfAdIMp3jYzw3zw +XEMTX5FaLaOCAfUDttPzn+oNezWZ2UyFTQXQE2CazpRdJoDd04qVg9WLpQxLYRP7 +xSFEHulOPccdAYF2C45yNtJAZyWKfGaAZIxrgEXbMkcdDMlYphpRwpjS8SIBNZ31 +KFE8BczKrg2qO0ywIjanPaRgrFVmeSvBKeU/YLQVx6fZMgOk6vtidLGZLyDXy0Ff +yaZSoj+on++RDz1IXb96Y8scuNlfcYI8QeoNjwiLtf80BV8SRJiG4e/jTvMf/z9L +kWrnDWvx4xkUmxFg4TK42dkNp7sEYBTlVVq9fjKE92ha7FGZRqsxOLNQ +-----END CERTIFICATE----- diff --git a/apps/emqx/test/emqx_crl_cache_SUITE_data/client.key.pem b/apps/emqx/test/emqx_crl_cache_SUITE_data/client.key.pem new file mode 100644 index 000000000..2e767d81f --- /dev/null +++ b/apps/emqx/test/emqx_crl_cache_SUITE_data/client.key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQC8a4BKF7ASjxX/ +5pZU78y5S33abwrRS3icLaA6+mMyN/yaFJJ/op+fE+SWwXA/h93I86M6JfAKx/fV +A7l+NWvy2jeIZMeETJmnNFxGqbDV5gkLuIOHcGq1gsBPfRH4qroyT5/i1cHkun/a +NeB2LTCyFZqAI+Ak7cK2LAbITxdJr5DoQFjDOCHc0VuBh0mZrjH5Lzpj/lZ+Zc/M +X/9SKmwWniAVSrAa6PqQms302cDXEKK47ELI3gm4U0x8z+2R6HbqMCnQ4eEkaxOz +diJSb0Om3tGzujSyj2i/0r6GAMW5DD0dlbdYgkj30zlr1BnhkRCoqkAG1gGX81tU +9dyv5Pd9AgMBAAECggEAAifx6dZKIeNkQ8OaNp5V2IKIPSqBOV4/h/xKMkUZXisV +eDmTCf8du0PR7hfLqrt9xYsGDv+6FQ1/8K231l8qR0tP/6CTl/0ynM4qqEAGeFXN +3h2LvM4liFbdjImechrcwcnVaNKg/DogT5zHUYSMtB/rokaG0VBO3IX/+SGz0aXi +LOLAx6SPaLOVX9GYUCiigTSEDwaQA+F3F6J2fR4u8PrXo+OQUqxjQ/fGXWp+4IfA +6djlpvzO2849/WPB1tL20iLXJlL2OL0UgQNtbKWTjexMe+wgCR5BzCwTyPsQvMwX +YOQrTOwgF3b6O+gLks5wSRT0ivq1sKgzA534+X4M+wKBgQDirPTLlrYobOO8KUpV +LOJU8x9leiRNU9CZWrW/mOw/BXGXikqNWvgL595vvADsjYciuRxSqEE7lClB8Pp9 +20TMlES9orx7gdoQJCodpNV1BuBJhE9YtUiXzWAj+7m3D9LsXM1ewW/2A7Vvopj3 +4zKY7uHAFlo3nXwLOfChG5/i9wKBgQDUy5fPFa58xmn7Elb6x4vmUDHg6P4pf75E +XHRQvNA8I7DTrpqfcsF1N4WuJ3Lm//RSpw7bnyqP20GoEfGHu/iCUPf29B7CuXhO +vvD+I8uPdn8EcKUBWV+V0xNQN/gCe0TzrEjAkZcO2Lq0j93R8HVl3BbowxgRvQV9 +GmxQG/boKwKBgFeV8uSzsGEAaiKrZbBxrmaappgEUQCcES8gULfes/JJ/TFL2zCx +ZMTc7CMKZuUAbqXpFtuNbd9CiYqUPYXh8ryF0eXgeqnSa9ruzmMz7NLSPFnLyQkC +yzD0x2BABOuKLrrrxOMHJWbO2g1vq2GlJUjYjNw3BtcUf/iqg6MM1IPTAoGAWYWJ +SSqS7JVAcsrFYt1eIrdsNHVwr565OeM3X9v/Mr3FH1jeXeQWNSz1hU29Ticx7y+u +1YBBlKGmHoHl/bd7lb9ggjkzU7JZRa+YjSIb+i/cwc5t7IJf7xUMk/vnz4tyd5zs +Qm89gJZ2/Y1kwXSKvx53WNbyokvGKlpaZN1O418CgYACliGux77pe4bWeXSFFd9N +50ipxDLVghw1c5AiZn25GR5YHJZaV4R0wmFcHdZvogLKi0jDMPvU69PaiT8eX/A1 +COkxv7jY1vtKlEtb+gugMjMN8wvb2va4kyFamjqnleiZlBSqIF/Y17wBoMvaWgZ0 +bEPCN//ts5hBwgb1TwGrrg== +-----END PRIVATE KEY----- diff --git a/apps/emqx/test/emqx_crl_cache_SUITE_data/client1.cert.pem b/apps/emqx/test/emqx_crl_cache_SUITE_data/client1.cert.pem new file mode 100644 index 000000000..4e41c15bb --- /dev/null +++ b/apps/emqx/test/emqx_crl_cache_SUITE_data/client1.cert.pem @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFljCCA36gAwIBAgICEAowDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCU0Ux +EjAQBgNVBAgMCVN0b2NraG9sbTESMBAGA1UECgwJTXlPcmdOYW1lMRkwFwYDVQQL +DBBNeUludGVybWVkaWF0ZUNBMRkwFwYDVQQDDBBNeUludGVybWVkaWF0ZUNBMB4X +DTIzMDMxNjIwMjAzMloXDTMzMDYyMTIwMjAzMlowdjELMAkGA1UEBhMCU0UxEjAQ +BgNVBAgMCVN0b2NraG9sbTESMBAGA1UEBwwJU3RvY2tob2xtMRIwEAYDVQQKDAlN +eU9yZ05hbWUxGTAXBgNVBAsMEE15SW50ZXJtZWRpYXRlQ0ExEDAOBgNVBAMMB0Ns +aWVudDEwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDcDhlEvUIYc9uA +ocOBXt5thKrovs+8V0Eus/WrHMTKBk0Kw4X+7HBaRBoZj2sZpYfN63lVaO75kW4I +uJuorGj5PAXYWJj+4uAsCc95xAN/liCuHJnxE5togWVt8W+z0Zll98RIpiCohqiE +FLDL4X6FREL07GLgQZ/BFORvAwU+Gog05AFh43iZDnJl8MmrG2HBSRXtSZ6vQj9A +NrOSqz5eK4YIHEEsgwTWQmhtNwu3Y+GzrAPWCA4TeYrSRwIrnGh20fOWXkAMldS4 +eRXmBztEsXMGqbe6oYO1QPYOlmoGO8EaaDPJ2sFIuM0zn98Alq3kCnRhM5Bi9RpJ +7IpudIopAgMBAAGjggE3MIIBMzAJBgNVHRMEAjAAMBEGCWCGSAGG+EIBAQQEAwIF +oDAzBglghkgBhvhCAQ0EJhYkT3BlblNTTCBHZW5lcmF0ZWQgQ2xpZW50IENlcnRp +ZmljYXRlMB0GA1UdDgQWBBQoIuXq3wG6JEzAEj9wPe7am0OVgjAfBgNVHSMEGDAW +gBRMcIY7FVKJurUPkqqusTFBE75z8zAOBgNVHQ8BAf8EBAMCBeAwHQYDVR0lBBYw +FAYIKwYBBQUHAwIGCCsGAQUFBwMEMDwGA1UdHwQ1MDMwMaAvoC2GK2h0dHA6Ly9s +b2NhbGhvc3Q6OTg3OC9pbnRlcm1lZGlhdGUxLmNybC5wZW0wMQYIKwYBBQUHAQEE +JTAjMCEGCCsGAQUFBzABhhVodHRwOi8vbG9jYWxob3N0Ojk4NzcwDQYJKoZIhvcN +AQELBQADggIBAHqKYcwkm3ODPD7Mqxq3bsswSXregWfc8tqfIBc5FZg2F+IzhxcJ +kINB0lmcNdLALK6ka0sDs1Nrj1KB96NcHUqE+WY/qPS1Yksr34yFatb1ddlKQ9HK +VRrIsi0ZfjBpHpvoQ0GsLeyRKm7iN/Fm5H9u8rw6RBu0Oe/l20FVSQIDzldYw51L +uV/E9No8ZhdQ2Dffujs8madI7b7I1NMXS+Z1pZ+gYrz6O60tDEprE+rYuYWypURr +fK+DnLLl+KQ+eekTPynw7LRpFzI/1cOMmd4BRnsBHCbCObfNp7WPasemZOEXGIlZ +CQwZS62DYOJE4u4Nz5pSF+JgXfr6X/Im6Y1SV900xVHfoL0GpFDI9k+0Y5ncHfSH ++V9HlRWB3zqQF+yla32XOpBbER0vFDH52gp8/o1ZGg7rr6KrP4QKxnqywNLiAPDX +txaAykZhON7uG8j+Lbjx5Ik91NRn9Fd5NH/vtT33a4uig2TP9EWd7EPcD2z8ONuD +yiK3S37XAnmSKKX4HcCpEb+LedtqQo/+sqWyWXkpKdpkUSozvcYS4J/ob3z9N2IE +qIH5I+Mty1I4EB4W89Pem8DHNq86Lt0Ea6TBtPTV8NwR5aG2vvLzb5lNdpANXYcp +nGr57mTWaHnQh+yqgy66J++k+WokWkAkwE989AvUfNoQ+Jr6cTH8nKo2 +-----END CERTIFICATE----- diff --git a/apps/emqx/test/emqx_crl_cache_SUITE_data/client1.key.pem b/apps/emqx/test/emqx_crl_cache_SUITE_data/client1.key.pem new file mode 100644 index 000000000..b355a3814 --- /dev/null +++ b/apps/emqx/test/emqx_crl_cache_SUITE_data/client1.key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDcDhlEvUIYc9uA +ocOBXt5thKrovs+8V0Eus/WrHMTKBk0Kw4X+7HBaRBoZj2sZpYfN63lVaO75kW4I +uJuorGj5PAXYWJj+4uAsCc95xAN/liCuHJnxE5togWVt8W+z0Zll98RIpiCohqiE +FLDL4X6FREL07GLgQZ/BFORvAwU+Gog05AFh43iZDnJl8MmrG2HBSRXtSZ6vQj9A +NrOSqz5eK4YIHEEsgwTWQmhtNwu3Y+GzrAPWCA4TeYrSRwIrnGh20fOWXkAMldS4 +eRXmBztEsXMGqbe6oYO1QPYOlmoGO8EaaDPJ2sFIuM0zn98Alq3kCnRhM5Bi9RpJ +7IpudIopAgMBAAECggEARcly2gnrXDXh9vlWN0EO6UyZpxZcay6AzX7k+k81WZyF +8lPvutjhCL9wR4rkPE3ys6tp31xX7W3hp4JkWynSYLhYYjQ20R7CWTUDR2qScXP7 +CTyo1XuSXaIruKJI+o4OR/g7l46X7NpHtxuYtg/dQAZV9bbB5LzrHSCzEUGz9+17 +jV//cBgLBiMdlbdLuQoGt4NQpBkNrauBVFq7Nq648uKkICmUo3Bzn/dfn3ehB+Zc ++580S+tawYd224j19tFQmd5oK8tfjqKuHenNGjp/gsRoY86N7qAtc3VIQ0yjE6ez +tgREo/ftCb8kGfwRJOAQIeeDamBv+FWNT6QzcOtbwQKBgQDzWhY9BUgI8JVzjYg0 +oWfU90On81BtckKsEo//8MTlgwOD2PnUF0hTZF3RcSPYT+HybouTqHT8EOLBAzqy +1+koH06MnAc/Y2ipaAe2fGaVH3SuXAsV/b8VcWWl4Qx7tYJDhE7sKmdl3/+jHZ7A +hZQzgOQnxxCANBo3pwF9KboDbwKBgQDnfglSpgYdGzFpWp1hZnPl2RKIfX/4M2z2 +s+hVN1tp+1VySPrBRUC3J6hKPQUzzeUzPICclHOnO+kP7jAos/rlJ9VcNdAQTbTL +7Ds9Em1KJTBphE038YbW3e93rydQpukXh50wRD9RI/3F3A/1rKhab92DXZZr6Wqu +JouhNV8f5wKBgQCLQ3XQi/Iyc4QDse5NuETUgoCsX7kaOTZghOr1nFMByV08mfI2 +5vAUES8DigzqYKS8eXjVEqWIDx3FOVThPmCG/ouUOkKHixs9P3SSgVSvaGX81l3d +wu4UlmWGbWkYbsJSYyhLTOUJTwxby7qrEIbEhrGK9gfCZo7OZHucpkF2bwKBgFhl +1qWK5JbExY+XnLWO6/7/b4ZTdkSPTrK+bJ/t7aiA41Yq7CZVjarjJ+6BcrUfkMCK +AArK3Yck55C/wgApCkvrdBwsKHGxWrLsWIqvuLAxl1UTwnD0eCsgwMsRRZAUzLnB +fZLq3MrdVZDywd1suzUdtpbta/11OtmZuoQq31JNAoGAIzmevuPaUZRqnjDpLXAm +Bo11q6CunhG5qZX4wifeZ9Fp5AaQu97F36igZ5/eDxFkDCrFRq6teMiNjRQZSLA3 +5tMBkq6BtN2Ozzm/6D135c4BF14ODHqAMPUy4sXbw5PS/kRFs4fKAH/+LcAOPgyI +N/jJIY1LfM7PzrG2NdyscMU= +-----END PRIVATE KEY----- diff --git a/apps/emqx/test/emqx_crl_cache_SUITE_data/client2.cert.pem b/apps/emqx/test/emqx_crl_cache_SUITE_data/client2.cert.pem new file mode 100644 index 000000000..0cba3fb26 --- /dev/null +++ b/apps/emqx/test/emqx_crl_cache_SUITE_data/client2.cert.pem @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFljCCA36gAwIBAgICEAswDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCU0Ux +EjAQBgNVBAgMCVN0b2NraG9sbTESMBAGA1UECgwJTXlPcmdOYW1lMRkwFwYDVQQL +DBBNeUludGVybWVkaWF0ZUNBMRkwFwYDVQQDDBBNeUludGVybWVkaWF0ZUNBMB4X +DTIzMDMxNjIwMjAzMloXDTMzMDYyMTIwMjAzMlowdjELMAkGA1UEBhMCU0UxEjAQ +BgNVBAgMCVN0b2NraG9sbTESMBAGA1UEBwwJU3RvY2tob2xtMRIwEAYDVQQKDAlN +eU9yZ05hbWUxGTAXBgNVBAsMEE15SW50ZXJtZWRpYXRlQ0ExEDAOBgNVBAMMB0Ns +aWVudDIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDFLcCjzNhfY6Sk +2nSdrB/6UPPeTCCH5NBBVLvu1hdlqLS4qEdq8+EjyMDZTtspRtYPkBfjpOrlBWUO +lKyxw2mZOjZ8iWvd4sJaAI/6KZl5X0Rdsg1RjzW03kUdLx9XJCyrYY0YFrT1dgJo +Ih56jk2SJX7wrz0NCJ05VPIdpaOF6CcziA+YhdVHcE6xyHagsYI0JdDWxFZrl9zT +LyhaDgBUN/yUQBnxKzxs8TMT4YVSi73ouh5IP9Xvs52hd6HO8ZGVr+YthQZKo95p +OlwFF+AQWxdDIKoPYUPFo8XMOXvOeQ9iUJarxrYSrelLXtGkaGLBolAvqo/YKE7j +rcJWjRGHAgMBAAGjggE3MIIBMzAJBgNVHRMEAjAAMBEGCWCGSAGG+EIBAQQEAwIF +oDAzBglghkgBhvhCAQ0EJhYkT3BlblNTTCBHZW5lcmF0ZWQgQ2xpZW50IENlcnRp +ZmljYXRlMB0GA1UdDgQWBBTOo9YSgx1h5k/imP7nOfRfzQrRxjAfBgNVHSMEGDAW +gBRMcIY7FVKJurUPkqqusTFBE75z8zAOBgNVHQ8BAf8EBAMCBeAwHQYDVR0lBBYw +FAYIKwYBBQUHAwIGCCsGAQUFBwMEMDwGA1UdHwQ1MDMwMaAvoC2GK2h0dHA6Ly9s +b2NhbGhvc3Q6OTg3OC9pbnRlcm1lZGlhdGUyLmNybC5wZW0wMQYIKwYBBQUHAQEE +JTAjMCEGCCsGAQUFBzABhhVodHRwOi8vbG9jYWxob3N0Ojk4NzcwDQYJKoZIhvcN +AQELBQADggIBAFo91lLqjPY67Wmj2yWxZuTTuUwXdXXUQxL6sEUUnfkECvRhNyBA +eCHkfVopNbXZ5tdLfsUvXF0ulaC76GCK/P7gHOG9D/RJX/85VzhuJcqa4dsEEifg +IiKIG7viYxSA6HFXuyzHvwNco3FqTBHbY46lKf1lWRVLhiAtcwcyPP34/RWcPfQi +6NZfLyitu5U7Z9XVN5wCp8sg0ayaO5Ib2ejIYuBCUddV1gV//tSDf+rKCgtAbm/X +K64Bf3GdaX3h6EhoqMZ+Z2f4XpKSXTabsWAU44xdVxisI82eo+NwT8KleE65GpOv +nPvr/dLq5fQ6VtHbRL3wWqhzB1VKVCtd8a6RE2k8HVWflU3qgwJ+woF19ed921eq +OZxc+KzjsGFyW1D2fPdgoZFmePadSstIME7qtCNEi7D3im01/1KKzE2m/nosrHeW +ePjY2YrXu0w47re/N2kBJL2xRbj+fAjBsfNn9RhvQsWheXG6mgg8w1ac6y72ZA2W +72pWoDkgXQMX5XBBj/zMnmwtrX9zTILFjNGFuWMPYgBRI0xOf2FoqqZ67cQ2yTW/ +1T/6Mp0FSh4cIo/ENiNSdvlt3BIo84EyOm3iHHy28Iv5SiFjF0pkwtXlYYvjM3+R +BeWqlPsVCZXcVC1rPVDzfWZE219yghldY4I3QPJ7dlmszi8eI0HtzhTK +-----END CERTIFICATE----- diff --git a/apps/emqx/test/emqx_crl_cache_SUITE_data/client2.key.pem b/apps/emqx/test/emqx_crl_cache_SUITE_data/client2.key.pem new file mode 100644 index 000000000..29196b1e2 --- /dev/null +++ b/apps/emqx/test/emqx_crl_cache_SUITE_data/client2.key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDFLcCjzNhfY6Sk +2nSdrB/6UPPeTCCH5NBBVLvu1hdlqLS4qEdq8+EjyMDZTtspRtYPkBfjpOrlBWUO +lKyxw2mZOjZ8iWvd4sJaAI/6KZl5X0Rdsg1RjzW03kUdLx9XJCyrYY0YFrT1dgJo +Ih56jk2SJX7wrz0NCJ05VPIdpaOF6CcziA+YhdVHcE6xyHagsYI0JdDWxFZrl9zT +LyhaDgBUN/yUQBnxKzxs8TMT4YVSi73ouh5IP9Xvs52hd6HO8ZGVr+YthQZKo95p +OlwFF+AQWxdDIKoPYUPFo8XMOXvOeQ9iUJarxrYSrelLXtGkaGLBolAvqo/YKE7j +rcJWjRGHAgMBAAECggEABJYUCcyJcnbagytBxfnaNQUuAp8AIypFG3kipq0l5Stk +gGaTJq5F4OTGS4ofRsqeu07IgBSAfqJcJH8toPkDQqfvs6ftO1Mso2UzakMOcP51 +Ywxd91Kjm+LKOyHkHGDirPGnutUg/YpLLrrMvTk/bJHDZCM4i/WP1WTREVFjUgl7 +4L6Y53x2Lk5shJJhv0MzTGaoZzQcW0EbhNH1AI6MBv5/CN5m/7/+HCPlHSNKnozl +o3PXD6l0XNfOY2Hi6MgS/Vd70s3VmDT9UCJNsDjdFpKNHmI7vr9FScOLN8EwbqWe +maFa0TPknmPDmVjEGMtgGlJWL7Sm0MpNW+WsEXcDPQKBgQDv3sp0nVML9pxdzX/w +rGebFaZaMYDWmV9w0V1uXYh4ZkpFmqrWkq/QSTGpwIP/x8WH9FBDUZqspLpPBNgG +ft1XhuY34y3hoCxOyRhQcR/1dY+lgCzuN4G4MG3seq/cAXhrmkPljma/iO8KzoRK +Pa+uaKFGHy1vWY2AmOhT20zr4wKBgQDScA3478TFHg9THlSFzqpzvVn5eAvmmrCQ +RMYIZKFWPortqzeJHdA5ShVF1XBBar1yNMid7E7FXqi/P8Oh+E6Nuc7JxyVIJWlV +mcBE1ceTKdZn7A0nuQIaU6hcn7xz/UHmxGur1ZcNQm3diFJ2CPn11lzZlkSZLSCN +V86nndA9DQKBgQCWsUxXPo7xsRhDBdseg/ECyPMdLoRWTTxcT+t2bmRR31FBsQ0q +iDTTkWgV0NAcXJCH/MB/ykB1vXceNVjRm9nKJwFyktI8MLglNsiDoM4HErgPrRqM +/WoNIL+uFNVuTa4tS1jkWjXKlmg2Tc9mJKK92xWWS/frQENZSraKF/eXKQKBgGR9 +ni6CUTTQZgELOtGrHzql8ZFwAj7dH/PE48yeQW0t8KoOWTbhRc4V0pLGmhSjJFSl +YCgJ8JPP4EVz7bgrG1gSou04bFVHiEWYZnh4nhVopTp7Psz5TEfGK2AP5658Ajxx +D/m+xaNPVae0sawsHTGIbE57s8ZyBll41Pa2JfsBAoGBANtS7SOehkflSdry0eAZ +50Ec3CmY+fArC044hQCmXxol5SiTUnXf/OIQH8y+RZUjF4F3PbbrFNLm/J6wuUPw +XUIb4gAL75srakL8DXqyTYfO02aNrFEHhXzMs+GoAnRkFy16IAAFwpjbYSPanfed +PfHCTWz6Y0pGdh1hUJAFV/3v +-----END PRIVATE KEY----- diff --git a/apps/emqx/test/emqx_crl_cache_SUITE_data/client3.cert.pem b/apps/emqx/test/emqx_crl_cache_SUITE_data/client3.cert.pem new file mode 100644 index 000000000..94092fad9 --- /dev/null +++ b/apps/emqx/test/emqx_crl_cache_SUITE_data/client3.cert.pem @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFljCCA36gAwIBAgICEAwwDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCU0Ux +EjAQBgNVBAgMCVN0b2NraG9sbTESMBAGA1UECgwJTXlPcmdOYW1lMRkwFwYDVQQL +DBBNeUludGVybWVkaWF0ZUNBMRkwFwYDVQQDDBBNeUludGVybWVkaWF0ZUNBMB4X +DTIzMDMxNjIwMjAzMloXDTMzMDYyMTIwMjAzMlowdjELMAkGA1UEBhMCU0UxEjAQ +BgNVBAgMCVN0b2NraG9sbTESMBAGA1UEBwwJU3RvY2tob2xtMRIwEAYDVQQKDAlN +eU9yZ05hbWUxGTAXBgNVBAsMEE15SW50ZXJtZWRpYXRlQ0ExEDAOBgNVBAMMB0Ns +aWVudDMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDEOZ6fYNjZDNXX +eOyapHMOMeNeYM3b7vsWXAbiJIt4utVrTS0A+/G640t/U0g8F9jbKgbjEEPtgPJ7 +GltjLWObfqDWKSO2D9/ei2+NauqgiN/HX+dQnSKHob0McXBXvLfrA4tn4braKrbg +p1fZB8bAECuT/bUhVBqWlzrUwDMpqjMJWDab48ixezb2gnc/ePE6wq/d3ecDb0/k +cYWQ0LX4JiQBgaTGhwczyoGfL1z2vx5kJqptK+r0Hc2jNCn6kFvoZUCYjCWgWNxZ +sQk7fObQQkUb/XQyqRaKJBWDyqsNcuK2gOg3LGeolAlgtMiEqGhHv77XdJnJug/w +3OiHpP/7AgMBAAGjggE3MIIBMzAJBgNVHRMEAjAAMBEGCWCGSAGG+EIBAQQEAwIF +oDAzBglghkgBhvhCAQ0EJhYkT3BlblNTTCBHZW5lcmF0ZWQgQ2xpZW50IENlcnRp +ZmljYXRlMB0GA1UdDgQWBBRxZFdIkSg6zDZCakXmIest5a6dBzAfBgNVHSMEGDAW +gBRMcIY7FVKJurUPkqqusTFBE75z8zAOBgNVHQ8BAf8EBAMCBeAwHQYDVR0lBBYw +FAYIKwYBBQUHAwIGCCsGAQUFBwMEMDwGA1UdHwQ1MDMwMaAvoC2GK2h0dHA6Ly9s +b2NhbGhvc3Q6OTg3OC9pbnRlcm1lZGlhdGUzLmNybC5wZW0wMQYIKwYBBQUHAQEE +JTAjMCEGCCsGAQUFBzABhhVodHRwOi8vbG9jYWxob3N0Ojk4NzcwDQYJKoZIhvcN +AQELBQADggIBAEntkhiPpQtModUF/ffnxruq+cqopPhIdMXhMD8gtU5e4e7o3EHX +lfZKIbxyw56v6dFPrl4TuHBiBudqIvBCsPtllWKixWvg6FV3CrEeTcg4shUIaJcD +pqv1qHLwS4pue6oau/lb8jv1GuzuBXoMFQwlmiOXO7xXqXjV2GdmkFJCDdB/0BW1 +VHvh0DXgotaxITWKhCpSNB7F7LSvegRwZIAN6JXrLDpue7tgqLqBB1EzpmS6ALbn +uZDdikOs/tGAFB3un/3Gl7jEPL8UGOoSj/H9PUT5AFHrHJDH72+QSXu09agz8RWJ +V939njYFCAxQ8Jt2mOK8BJQDJgPtLfIIb1iYicQV13Eypt8uIUYvp0i0Wq8WxPbq +rOEvQYpcGUsreS5XqZ7y68hgq6ePiR18Fnc3GyTV5o6qT3W7IOvPArTzNV5fFCwM +lx8xSEm+ebJrJPphp6Uc/h8evohvAN8R/Z7FSo9OL6V+F3ywPqWTXaqiIiRc9PS0 +0vxsYZ96EeZY5HzjN6LzHxmkv4KYM5I1qmXlviQlaU+sotp3tzegADlM4K78nUFh +HuXamecEcS73eAgjk+FGqJ9E25B0TLlQMcP6tCKdaUIGn6ZsF5wT87GzqT99wL/5 +foHCYIkyG7ZmAQmoaKBd4q6xqVOUHovmsPza69FuSrsBxoRR39PtAnrY +-----END CERTIFICATE----- diff --git a/apps/emqx/test/emqx_crl_cache_SUITE_data/client3.key.pem b/apps/emqx/test/emqx_crl_cache_SUITE_data/client3.key.pem new file mode 100644 index 000000000..6ede63fd2 --- /dev/null +++ b/apps/emqx/test/emqx_crl_cache_SUITE_data/client3.key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDEOZ6fYNjZDNXX +eOyapHMOMeNeYM3b7vsWXAbiJIt4utVrTS0A+/G640t/U0g8F9jbKgbjEEPtgPJ7 +GltjLWObfqDWKSO2D9/ei2+NauqgiN/HX+dQnSKHob0McXBXvLfrA4tn4braKrbg +p1fZB8bAECuT/bUhVBqWlzrUwDMpqjMJWDab48ixezb2gnc/ePE6wq/d3ecDb0/k +cYWQ0LX4JiQBgaTGhwczyoGfL1z2vx5kJqptK+r0Hc2jNCn6kFvoZUCYjCWgWNxZ +sQk7fObQQkUb/XQyqRaKJBWDyqsNcuK2gOg3LGeolAlgtMiEqGhHv77XdJnJug/w +3OiHpP/7AgMBAAECggEADSe89sig5E63SKAlFXcGw0H2XgqIzDP/TGMnqPvNoYhX +eSXUgxDhBptpB9e9a4RaKwaFxxPjlSXEdYFX9O22YSN1RMMl6Q8Zl9g3edhcDR6W +b7Qbx2x8qj6Rjibnlh8JiFPiaDjN2wUeSDBss/9D98NkKiJ9Ue2YCYmJAOA3B3w9 +2t4Co5+3YrxkdzkvibTQCUSEwHFeB1Nim21126fknMPxyrf+AezRBRc8JNAHqzWb +4QEeMnmIJDOzc3Oh7+P85tNyejOeRm9T7X3EQ0jKXgLYe+HUzXclBQ66b9x9Nc9b +tNn6XkMlLlsQ3f149Th6PtHksH3hM+GF8bMuCp9yxQKBgQDGk0PYPkLqTD8jHjJW +s8wBNhozigZPGaynxdTsD7L6UtDdOl1sSW/jFOj9UIs1duBce9dP1IjFc0jY+Kin +lMLv3qCtk5ZjxRglOoLipR9hdClcM69rDoRZdoQK8KYa+QHcOTSazIp3fnw4gWSX +nscelMfd1rtVP0dOGTuqE/73/QKBgQD8+F5WAi2IOVPHnBxAAOP+6XTs9Ntn1sDi +L5wNgm+QA28aJJ4KRAwdXIc3IFPlHxZI77c2K1L9dKDu9X4UcyZIZYDvGVLuOOt5 +twaRaGuJW03cjbgOWC7rGyfzfZ49YlCZi2YuxERclBkbqgWD9hfa8twUfKNguF2Y +AyiOhohtVwKBgQCJB8zUp7pzhqQ3LrpcHHzWBSi1kjTiVvxPVnSlZfwDRCz/zSv0 +8wRz9tUFIZS/E0ama4tcenTblL+bgpSX+E9BSiclQOiR9su/vQ3fK0Vpccis6LnP +rdflCKT8C68Eg/slppBHloijBzTfpWLuQlJ0JwV5b5ocrKsfGMiUiHH1XQKBgQDg +RnakfEPP7TtY0g+9ssxwOJxAZImM0zmojpsk4wpzvIeovuQap9+xvFHoztFyZhBE +07oz3U8zhE4V7TI9gSVktBEOaf47U914yIqbKd+FJJywODkBBq96I1ZVKn67X0mk +B5GtTrZo+agU/bTsHKdjp0L1KtdSLcJUviAb1Cxp+wKBgDrGqS01CCgxSUwMaZe4 +8HFWp/oMSyVDG9lTSC3uP/VL76zNFI55T3X06Q87hDN3gCJGUOmHzDZ/oCOgM4/S +SU55M4lXeSEdFe84tMXJKOv5JXTkulzBYzATJ5J8DeS/4YZxMKyPDLXX8wgwmU+l +i6Imd3qCPhh5eI3z9eSNDX+6 +-----END PRIVATE KEY----- diff --git a/apps/emqx/test/emqx_crl_cache_SUITE_data/crl.pem b/apps/emqx/test/emqx_crl_cache_SUITE_data/crl.pem new file mode 100644 index 000000000..a119cede2 --- /dev/null +++ b/apps/emqx/test/emqx_crl_cache_SUITE_data/crl.pem @@ -0,0 +1,20 @@ +-----BEGIN X509 CRL----- +MIIDPDCCASQCAQEwDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCU0UxEjAQBgNV +BAgMCVN0b2NraG9sbTESMBAGA1UECgwJTXlPcmdOYW1lMRkwFwYDVQQLDBBNeUlu +dGVybWVkaWF0ZUNBMRkwFwYDVQQDDBBNeUludGVybWVkaWF0ZUNBFw0yMjA3MjAy +MDIzNTNaFw0zMjEwMjUyMDIzNTNaMBUwEwICEAIXDTIyMDYxMzEyNDIwNVqgbjBs +MB8GA1UdIwQYMBaAFCuv1TkzC1fSgTfzE1m1u5pRCJsVMDwGA1UdHAQ1MDOgLqAs +hipodHRwOi8vbG9jYWxob3N0Ojk4NzgvaW50ZXJtZWRpYXRlLmNybC5wZW2EAf8w +CwYDVR0UBAQCAhADMA0GCSqGSIb3DQEBCwUAA4ICAQBbWdqRFsIrG6coL6ln1RL+ +uhgW9l3XMmjNlyiYHHNzOgnlBok9xu9UdaVCOKC6GEthWSzSlBY1AZugje57DQQd +RkIJol9am94lKMTjF/qhzFLiSjho8fwZGDGyES5YeZXkLqNMOf6m/drKaI3iofWf +l63qU9jY8dnSrVDkwgCguUL2FTx60v5H9NPxSctQ3VDxDvDj0sTAcHFknQcZbfvY +ZWpOYNS0FAJlQPVK9wUoDxI0LhrWDq5h/T1jcGO34fPT8RUA5HRtFVUevqSuOLWx +WTfTx5oDeMZPJTvHWUcg4yMElHty4tEvtkFxLSYbZqj7qTU+mi/LAN3UKBH/gBEN +y2OsJvFhVRgHf+zPYegf3WzBSoeaXNAJZ4UnRo34P9AL3Mrh+4OOUP++oYRKjWno +pYtAmTrIwEYoLyisEhhZ6aD92f/Op3dIYsxwhHt0n0lKrbTmUfiJUAe7kUZ4PMn4 +Gg/OHlbEDaDxW1dCymjyRGl+3/8kjy7bkYUXCf7w6JBeL2Hw2dFp1Gh13NRjre93 +PYlSOvI6QNisYGscfuYPwefXogVrNjf/ttCksMa51tUk+ylw7ZMZqQjcPPSzmwKc +5CqpnQHfolvRuN0xIVZiAn5V6/MdHm7ocrXxOkzWQyaoNODTq4js8h8eYXgAkt1w +p1PTEFBucGud7uBDE6Ub6A== +-----END X509 CRL----- diff --git a/apps/emqx/test/emqx_crl_cache_SUITE_data/emqx.conf b/apps/emqx/test/emqx_crl_cache_SUITE_data/emqx.conf new file mode 100644 index 000000000..f34ab1456 --- /dev/null +++ b/apps/emqx/test/emqx_crl_cache_SUITE_data/emqx.conf @@ -0,0 +1,12 @@ +crl_cache.refresh_interval = {{ refresh_interval }} +crl_cache.http_timeout = 17s +crl_cache.capacity = {{ cache_capacity }} +listeners.ssl.default { + ssl_options { + keyfile = "{{ test_data_dir }}/server.key.pem" + certfile = "{{ test_data_dir }}/server.cert.pem" + cacertfile = "{{ test_data_dir }}/ca-chain.cert.pem" + verify = verify_peer + enable_crl_check = true + } +} diff --git a/apps/emqx/test/emqx_crl_cache_SUITE_data/emqx_crl_cache_http_server.erl b/apps/emqx/test/emqx_crl_cache_SUITE_data/emqx_crl_cache_http_server.erl new file mode 100644 index 000000000..4e8b989fa --- /dev/null +++ b/apps/emqx/test/emqx_crl_cache_SUITE_data/emqx_crl_cache_http_server.erl @@ -0,0 +1,67 @@ +-module(emqx_crl_cache_http_server). + +-behaviour(gen_server). +-compile([nowarn_export_all, export_all]). + +set_crl(CRLPem) -> + ets:insert(?MODULE, {crl, CRLPem}). + +%%-------------------------------------------------------------------- +%% `gen_server' APIs +%%-------------------------------------------------------------------- + +start_link(Parent, BasePort, CRLPem, Opts) -> + process_flag(trap_exit, true), + stop_http(), + timer:sleep(100), + gen_server:start_link(?MODULE, {Parent, BasePort, CRLPem, Opts}, []). + +init({Parent, BasePort, CRLPem, Opts}) -> + Tab = ets:new(?MODULE, [named_table, ordered_set, public]), + ets:insert(Tab, {crl, CRLPem}), + ok = start_http(Parent, [{port, BasePort} | Opts]), + Parent ! {self(), ready}, + {ok, #{parent => Parent}}. + +handle_call(_Request, _From, State) -> + {reply, ignored, State}. + +handle_cast(_Msg, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + stop_http(). + +stop(Pid) -> + ok = gen_server:stop(Pid). + +%%-------------------------------------------------------------------- +%% Callbacks +%%-------------------------------------------------------------------- + +start_http(Parent, Opts) -> + {ok, _Pid1} = cowboy:start_clear(http, Opts, #{ + env => #{dispatch => compile_router(Parent)} + }), + ok. + +stop_http() -> + cowboy:stop_listener(http), + ok. + +compile_router(Parent) -> + {ok, _} = application:ensure_all_started(cowboy), + cowboy_router:compile([ + {'_', [{'_', ?MODULE, #{parent => Parent}}]} + ]). + +init(Req, #{parent := Parent} = State) -> + %% assert + <<"GET">> = cowboy_req:method(Req), + [{crl, CRLPem}] = ets:lookup(?MODULE, crl), + Parent ! {http_get, iolist_to_binary(cowboy_req:uri(Req))}, + Reply = reply(Req, CRLPem), + {ok, Reply, State}. + +reply(Req, CRLPem) -> + cowboy_req:reply(200, #{<<"content-type">> => <<"text/plain">>}, CRLPem, Req). diff --git a/apps/emqx/test/emqx_crl_cache_SUITE_data/emqx_just_verify.conf b/apps/emqx/test/emqx_crl_cache_SUITE_data/emqx_just_verify.conf new file mode 100644 index 000000000..8b9549823 --- /dev/null +++ b/apps/emqx/test/emqx_crl_cache_SUITE_data/emqx_just_verify.conf @@ -0,0 +1,12 @@ +node.name = test@127.0.0.1 +node.cookie = emqxsecretcookie +node.data_dir = "{{ test_priv_dir }}" +listeners.ssl.default { + ssl_options { + keyfile = "{{ test_data_dir }}/server.key.pem" + certfile = "{{ test_data_dir }}/server.cert.pem" + cacertfile = "{{ test_data_dir }}/ca-chain.cert.pem" + verify = verify_peer + enable_crl_check = false + } +} diff --git a/apps/emqx/test/emqx_crl_cache_SUITE_data/intermediate-not-revoked.crl.pem b/apps/emqx/test/emqx_crl_cache_SUITE_data/intermediate-not-revoked.crl.pem new file mode 100644 index 000000000..e484b44c0 --- /dev/null +++ b/apps/emqx/test/emqx_crl_cache_SUITE_data/intermediate-not-revoked.crl.pem @@ -0,0 +1,19 @@ +-----BEGIN X509 CRL----- +MIIDJTCCAQ0CAQEwDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCU0UxEjAQBgNV +BAgMCVN0b2NraG9sbTESMBAGA1UECgwJTXlPcmdOYW1lMRkwFwYDVQQLDBBNeUlu +dGVybWVkaWF0ZUNBMRkwFwYDVQQDDBBNeUludGVybWVkaWF0ZUNBFw0yMzAxMTIx +MzA4MTZaFw0zMzAxMDkxMzA4MTZaoG4wbDAfBgNVHSMEGDAWgBRMcIY7FVKJurUP +kqqusTFBE75z8zA8BgNVHRwENTAzoC6gLIYqaHR0cDovL2xvY2FsaG9zdDo5ODc4 +L2ludGVybWVkaWF0ZS5jcmwucGVthAH/MAsGA1UdFAQEAgIQADANBgkqhkiG9w0B +AQsFAAOCAgEAJGOZuqZL4m7zUaRyBrxeT6Tqo+XKz7HeD5zvO4BTNX+0E0CRyki4 +HhIGbxjv2NKWoaUv0HYbGAiZdO4TaPu3w3tm4+pGEDBclBj2KTdbB+4Hlzv956gD +KXZ//ziNwx1SCoxxkxB+TALxReN0shE7Mof9GlB5HPskhLorZgg/pmgJtIykEpsq +QAjJo4aq+f2/L+9dzRM205fVFegtsHvgEVNKz6iK6skt+kDhj/ks9BKsnfCDIGr+ +XnPYwS9yDnnhFdoJ40AQQDtomxggAjfgcSnqtHCxZwKJohuztbSWUgD/4yxzlrwP +Dk1cT/Ajjjqb2dXVOfTLK1VB2168uuouArxZ7KYbXwBjHduYWGGkA6FfkNJO/jpF +SL9qhX3oxcRF3hDhWigN1ZRD7NpDKwVal3Y9tmvO5bWhb5VF+3qv0HGeSGp6V0dp +sjwhIj+78bkUrcXxrivACLAXgSTGonx1uXD+T4P4NCt148dgRAbgd8sUXK5FcgU2 +cdBl8Kv2ZUjEaod5gUzDtf22VGSoO9lHvfHdpG9o2H3wC7s4tyLTidNrduIguJff +IIgc44Y252iV0sOmZ5S0jjTRiF1YUUPy9qA/6bOnr2LohbwbNZv9tDlNj8cdhxUz +cKiS+c7Qsz+YCcrp19QRiJoQae/gUqz7kmUZQgyPmDd+ArE0V+kDZEE= +-----END X509 CRL----- diff --git a/apps/emqx/test/emqx_crl_cache_SUITE_data/intermediate-revoked-no-dp.crl.pem b/apps/emqx/test/emqx_crl_cache_SUITE_data/intermediate-revoked-no-dp.crl.pem new file mode 100644 index 000000000..4d3611d49 --- /dev/null +++ b/apps/emqx/test/emqx_crl_cache_SUITE_data/intermediate-revoked-no-dp.crl.pem @@ -0,0 +1,19 @@ +-----BEGIN X509 CRL----- +MIIC/TCB5gIBATANBgkqhkiG9w0BAQsFADBrMQswCQYDVQQGEwJTRTESMBAGA1UE +CAwJU3RvY2tob2xtMRIwEAYDVQQKDAlNeU9yZ05hbWUxGTAXBgNVBAsMEE15SW50 +ZXJtZWRpYXRlQ0ExGTAXBgNVBAMMEE15SW50ZXJtZWRpYXRlQ0EXDTIzMDExODEz +Mjc1M1oXDTMzMDExNTEzMjc1M1owFTATAgIQAhcNMjMwMTEyMTMwODE2WqAwMC4w +HwYDVR0jBBgwFoAUTHCGOxVSibq1D5KqrrExQRO+c/MwCwYDVR0UBAQCAhACMA0G +CSqGSIb3DQEBCwUAA4ICAQCxoRYDc5MaBpDI+HQUX60+obFeZJdBkPO2wMb6HBQq +e0lZM2ukS+4n5oGhRelsvmEz0qKvnYS6ISpuFzv4Qy6Vaun/KwIYAdXsEQVwDHsu +Br4m1V01igjFnujowwR/7F9oPnZOmBaBdiyYbjgGV0YMF7sOfl4UO2MqI2GSGqVk +63wELT1AXjx31JVoyATQOQkq1A5HKFYLEbFmdF/8lNfbxSCBY2tuJ+uWVQtzjM0y +i+/owz5ez1BZ/Swx8akYhuvs8DVVTbjXydidVSrxt/QEf3+oJCzTA9qFqt4MH7gL +6BAglCGtRiYTHqeMHrwddaHF2hzR61lHJlkMCL61yhVuL8WsEJ/AxVX0W3MfQ4Cw +x/A6xIkgqtu+HtQnPyDcJxyaFHtFC+U67nSbEQySFvHfMw42DGdIGojKQCeUer9W +ECFC8OATQwN2h//f8QkY7D0H3k/brrNYDfdFIcCti9iZiFrrPFxO7NbOTfkeKCt3 +7IwYduRc8DWKmS8c7j2re1KkdYnfE1sfwbn3trImkcET5tvDlVCZ1glnBQzk82PS +HvKmSjD2pZI7upfLkoMgMhYyYJhYk7Mw2o4JXuddYGKmmw3bJyHkG/Ot5NAKjb7g +k1QCeWzxO1xXm8PNDDFWMn351twUGDQ/cwrUw0ODeUZpfL0BtTn4YnfCLLTvZDxo +Vg== +-----END X509 CRL----- diff --git a/apps/emqx/test/emqx_crl_cache_SUITE_data/intermediate-revoked.crl.pem b/apps/emqx/test/emqx_crl_cache_SUITE_data/intermediate-revoked.crl.pem new file mode 100644 index 000000000..4c5cdd441 --- /dev/null +++ b/apps/emqx/test/emqx_crl_cache_SUITE_data/intermediate-revoked.crl.pem @@ -0,0 +1,20 @@ +-----BEGIN X509 CRL----- +MIIDPDCCASQCAQEwDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCU0UxEjAQBgNV +BAgMCVN0b2NraG9sbTESMBAGA1UECgwJTXlPcmdOYW1lMRkwFwYDVQQLDBBNeUlu +dGVybWVkaWF0ZUNBMRkwFwYDVQQDDBBNeUludGVybWVkaWF0ZUNBFw0yMzAxMTIx +MzA4MTZaFw0zMzAxMDkxMzA4MTZaMBUwEwICEAIXDTIzMDExMjEzMDgxNlqgbjBs +MB8GA1UdIwQYMBaAFExwhjsVUom6tQ+Sqq6xMUETvnPzMDwGA1UdHAQ1MDOgLqAs +hipodHRwOi8vbG9jYWxob3N0Ojk4NzgvaW50ZXJtZWRpYXRlLmNybC5wZW2EAf8w +CwYDVR0UBAQCAhABMA0GCSqGSIb3DQEBCwUAA4ICAQCPadbaehEqLv4pwqF8em8T +CW8TOQ4Vjz02uiVk9Bo0za1dQqQmwCBA6UE1BcOh+aWzQxBRz56NeUcfhgDxTntG +xLs896N9MHIG6UxpqJH8cH+DXKHsQjvvCjXtiObmBQR1RiG5C1vEMkfzTt/WSrq5 +7blowLDs4NP6YbtqXEyyUkF7DQSUEUuIDWPQdx1f++nSpVaHWW4xpoO4umesaJco +FuxaXQnZpTHHQfqUJVIL2Mmzvez9thgfKTV3vgkYrGiSLW2m2+Tfga30pUc0qaVI +RrBVORVbcu9m1sV0aJyk96b2T/+i2FRR/np4TOcLgckBpHKeK2FH69lHFr0W/71w +CErNTxahoh82Yi8POenu+S1m2sDnrF1FMf+ZG/i2wr0nW6/+zVGQsEOw77Spbmei +dbEchu3iWF1XEO/n4zVBzl6a1o2RyVg+1pItYd5C5bPwcrfZnBrm4WECPxO+6rbW +2/wz9Iku4XznTLqLEpXLAtenAdo73mLGC7riviX7mhcxfN2UjNfLuVGHmG8XwIsM +Lgpr6DKaxHwpHgW3wA3SGJrY5dj0TvGWaoInrNt1cOMnIpoxRNy5+ko71Ubx3yrV +RhbUMggd1GG1ct9uZn82v74RYF6J8Xcxn9vDFJu5LLT5kvfy414kdJeTXKqfKXA/ +atdUgFa0otoccn5FzyUuzg== +-----END X509 CRL----- diff --git a/apps/emqx/test/emqx_crl_cache_SUITE_data/intermediate.crl.pem b/apps/emqx/test/emqx_crl_cache_SUITE_data/intermediate.crl.pem new file mode 100644 index 000000000..a119cede2 --- /dev/null +++ b/apps/emqx/test/emqx_crl_cache_SUITE_data/intermediate.crl.pem @@ -0,0 +1,20 @@ +-----BEGIN X509 CRL----- +MIIDPDCCASQCAQEwDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCU0UxEjAQBgNV +BAgMCVN0b2NraG9sbTESMBAGA1UECgwJTXlPcmdOYW1lMRkwFwYDVQQLDBBNeUlu +dGVybWVkaWF0ZUNBMRkwFwYDVQQDDBBNeUludGVybWVkaWF0ZUNBFw0yMjA3MjAy +MDIzNTNaFw0zMjEwMjUyMDIzNTNaMBUwEwICEAIXDTIyMDYxMzEyNDIwNVqgbjBs +MB8GA1UdIwQYMBaAFCuv1TkzC1fSgTfzE1m1u5pRCJsVMDwGA1UdHAQ1MDOgLqAs +hipodHRwOi8vbG9jYWxob3N0Ojk4NzgvaW50ZXJtZWRpYXRlLmNybC5wZW2EAf8w +CwYDVR0UBAQCAhADMA0GCSqGSIb3DQEBCwUAA4ICAQBbWdqRFsIrG6coL6ln1RL+ +uhgW9l3XMmjNlyiYHHNzOgnlBok9xu9UdaVCOKC6GEthWSzSlBY1AZugje57DQQd +RkIJol9am94lKMTjF/qhzFLiSjho8fwZGDGyES5YeZXkLqNMOf6m/drKaI3iofWf +l63qU9jY8dnSrVDkwgCguUL2FTx60v5H9NPxSctQ3VDxDvDj0sTAcHFknQcZbfvY +ZWpOYNS0FAJlQPVK9wUoDxI0LhrWDq5h/T1jcGO34fPT8RUA5HRtFVUevqSuOLWx +WTfTx5oDeMZPJTvHWUcg4yMElHty4tEvtkFxLSYbZqj7qTU+mi/LAN3UKBH/gBEN +y2OsJvFhVRgHf+zPYegf3WzBSoeaXNAJZ4UnRo34P9AL3Mrh+4OOUP++oYRKjWno +pYtAmTrIwEYoLyisEhhZ6aD92f/Op3dIYsxwhHt0n0lKrbTmUfiJUAe7kUZ4PMn4 +Gg/OHlbEDaDxW1dCymjyRGl+3/8kjy7bkYUXCf7w6JBeL2Hw2dFp1Gh13NRjre93 +PYlSOvI6QNisYGscfuYPwefXogVrNjf/ttCksMa51tUk+ylw7ZMZqQjcPPSzmwKc +5CqpnQHfolvRuN0xIVZiAn5V6/MdHm7ocrXxOkzWQyaoNODTq4js8h8eYXgAkt1w +p1PTEFBucGud7uBDE6Ub6A== +-----END X509 CRL----- diff --git a/apps/emqx/test/emqx_crl_cache_SUITE_data/server.cert.pem b/apps/emqx/test/emqx_crl_cache_SUITE_data/server.cert.pem new file mode 100644 index 000000000..38cc63534 --- /dev/null +++ b/apps/emqx/test/emqx_crl_cache_SUITE_data/server.cert.pem @@ -0,0 +1,35 @@ +-----BEGIN CERTIFICATE----- +MIIGCTCCA/GgAwIBAgICEAAwDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCU0Ux +EjAQBgNVBAgMCVN0b2NraG9sbTESMBAGA1UECgwJTXlPcmdOYW1lMRkwFwYDVQQL +DBBNeUludGVybWVkaWF0ZUNBMRkwFwYDVQQDDBBNeUludGVybWVkaWF0ZUNBMB4X +DTIzMDExMjEzMDgxNloXDTMzMDQxOTEzMDgxNloweDELMAkGA1UEBhMCU0UxEjAQ +BgNVBAgMCVN0b2NraG9sbTESMBAGA1UEBwwJU3RvY2tob2xtMRIwEAYDVQQKDAlN +eU9yZ05hbWUxGTAXBgNVBAsMEE15SW50ZXJtZWRpYXRlQ0ExEjAQBgNVBAMMCWxv +Y2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKdU9FaA/n0Z +TXkd10XA9l+UV9xKR65ZTy2ApCFlw2gGWLiUh96a6hX+GQZFUV7ECIDDf+7nC85o +xo1Xyf0rHGABQ0uHlhqSemc12F9APIzRLlQkhtV4vMBBbGQFekje4F9bhY9JQtGd +XJGmwsR+XWo6SUY7K5l9FuSSSRXC0kSYYQfSTPR/LrF6efdHf+ZN4huP7lM2qIFd +afX+qBOI1/Y2LtITo2TaU/hXyKh9wEiuynoq0RZ2KkYQll5cKD9fSD+pW3Xm0XWX +TQy4RZEe3WoYEQsklNw3NC92ocA/PQB9BGNO1fKhzDn6kW2HxDxruDKOuO/meGek +ApCayu3e/I0CAwEAAaOCAagwggGkMAkGA1UdEwQCMAAwEQYJYIZIAYb4QgEBBAQD +AgZAMDMGCWCGSAGG+EIBDQQmFiRPcGVuU1NMIEdlbmVyYXRlZCBTZXJ2ZXIgQ2Vy +dGlmaWNhdGUwHQYDVR0OBBYEFGy5LQPzIelruJl7mL0mtUXM57XhMIGaBgNVHSME +gZIwgY+AFExwhjsVUom6tQ+Sqq6xMUETvnPzoXOkcTBvMQswCQYDVQQGEwJTRTES +MBAGA1UECAwJU3RvY2tob2xtMRIwEAYDVQQHDAlTdG9ja2hvbG0xEjAQBgNVBAoM +CU15T3JnTmFtZTERMA8GA1UECwwITXlSb290Q0ExETAPBgNVBAMMCE15Um9vdENB +ggIQADAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwOwYDVR0f +BDQwMjAwoC6gLIYqaHR0cDovL2xvY2FsaG9zdDo5ODc4L2ludGVybWVkaWF0ZS5j +cmwucGVtMDEGCCsGAQUFBwEBBCUwIzAhBggrBgEFBQcwAYYVaHR0cDovL2xvY2Fs +aG9zdDo5ODc3MA0GCSqGSIb3DQEBCwUAA4ICAQCX3EQgiCVqLhnCNd0pmptxXPxo +l1KyZkpdrFa/NgSqRhkuZSAkszwBDDS/gzkHFKEUhmqs6/UZwN4+Rr3LzrHonBiN +aQ6GeNNXZ/3xAQfUCwjjGmz9Sgw6kaX19Gnk2CjI6xP7T+O5UmsMI9hHUepC9nWa +XX2a0hsO/KOVu5ZZckI16Ek/jxs2/HEN0epYdvjKFAaVmzZZ5PATNjrPQXvPmq2r +x++La+3bXZsrH8P2FhPpM5t/IxKKW/Tlpgz92c2jVSIHF5khSA/MFDC+dk80OFmm +v4ZTPIMuZ//Q+wo0f9P48rsL9D27qS7CA+8pn9wu+cfnBDSt7JD5Yipa1gHz71fy +YTa9qRxIAPpzW2v7TFZE8eSKFUY9ipCeM2BbdmCQGmq4+v36b5TZoyjH4k0UVWGo +Gclos2cic5Vxi8E6hb7b7yZpjEfn/5lbCiGMfAnI6aoOyrWg6keaRA33kaLUEZiK +OgFNbPkjiTV0ZQyLXf7uK9YFhpVzJ0dv0CFNse8rZb7A7PLn8VrV/ZFnJ9rPoawn +t7ZGxC0d5BRSEyEeEgsQdxuY4m8OkE18zwhCkt2Qs3uosOWlIrYmqSEa0i/sPSQP +jiwB4nEdBrf8ZygzuYjT5T9YRSwhVox4spS/Av8Ells5JnkuKAhCVv9gHxYwbj0c +CzyLJgE1z9Tq63m+gQ== +-----END CERTIFICATE----- diff --git a/apps/emqx/test/emqx_crl_cache_SUITE_data/server.key.pem b/apps/emqx/test/emqx_crl_cache_SUITE_data/server.key.pem new file mode 100644 index 000000000..d456ece72 --- /dev/null +++ b/apps/emqx/test/emqx_crl_cache_SUITE_data/server.key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCnVPRWgP59GU15 +HddFwPZflFfcSkeuWU8tgKQhZcNoBli4lIfemuoV/hkGRVFexAiAw3/u5wvOaMaN +V8n9KxxgAUNLh5YaknpnNdhfQDyM0S5UJIbVeLzAQWxkBXpI3uBfW4WPSULRnVyR +psLEfl1qOklGOyuZfRbkkkkVwtJEmGEH0kz0fy6xenn3R3/mTeIbj+5TNqiBXWn1 +/qgTiNf2Ni7SE6Nk2lP4V8iofcBIrsp6KtEWdipGEJZeXCg/X0g/qVt15tF1l00M +uEWRHt1qGBELJJTcNzQvdqHAPz0AfQRjTtXyocw5+pFth8Q8a7gyjrjv5nhnpAKQ +msrt3vyNAgMBAAECggEABnWvIQ/Fw0qQxRYz00uJt1LguW5cqgxklBsdOvTUwFVO +Y4HIZP2R/9tZV/ahF4l10pK5g52DxSoiUB6Ne6qIY+RolqfbUZdKBmX7vmGadM02 +fqUSV3dbwghEiO/1Mo74FnZQB6IKZFEw26aWakN+k7VAUufB3SEJGzXSgHaO63ru +dFGSiYI8U+q+YnhUJjCnmI12fycNfy451TdUQtGZb6pNmm5HRUF6hpAV8Le9LojP +Ql9eacPpsrzU15X5ElCQZ/f9iNh1bplcISuhrULgKUKOvAVrBlEK67uRVy6g98xA +c/rgNLkbL/jZEsAc3/vHAyFgd3lABfwpBGLHej3QgQKBgQDFNYmfBNQr89HC5Zc+ +M6jXcAT/R+0GNczBTfC4iyNemwqsumSSRelNZ748UefKuS3F6Mvb2CBqE2LbB61G +hrnCffG2pARjZ491SefRwghhWWVGLP1p8KliLgOGBehA1REgJb+XULncjuHZuh4O +LVn3HVnWGxeBGg+yKa6Z4YQi3QKBgQDZN0O8ZcZY74lRJ0UjscD9mJ1yHlsssZag +njkX/f0GR/iVpfaIxQNC3gvWUy2LsU0He9sidcB0cfej0j/qZObQyFsCB0+utOgy ++hX7gokV2pes27WICbNWE2lJL4QZRJgvf82OaEy57kfDrm+eK1XaSZTZ10P82C9u +gAmMnontcQKBgGu29lhY9tqa7jOZ26Yp6Uri8JfO3XPK5u+edqEVvlfqL0Zw+IW8 +kdWpmIqx4f0kcA/tO4v03J+TvycLZmVjKQtGZ0PvCkaRRhY2K9yyMomZnmtaH4BB +5wKtR1do2pauyg/ZDnDDswD5OfsGYWw08TK8YVlEqu3lIjWZ9rguKVIxAoGAZYUk +zVqr10ks3pcCA2rCjkPT4lA5wKvHgI4ylPoKVfMxRY/pp4acvZXV5ne9o7pcDBFh +G7v5FPNnEFPlt4EtN4tMragJH9hBZgHoYEJkG6islweg0lHmVWaBIMlqbfzXO+v5 +gINSyNuLAvP2CvCqEXmubhnkFrpbgMOqsuQuBqECgYB3ss2PDhBF+5qoWgqymFof +1ovRPuQ9sPjWBn5IrCdoYITDnbBzBZERx7GLs6A/PUlWgST7jkb1PY/TxYSUfXzJ +SNd47q0mCQ+IUdqUbHgpK9b1ncwLMsnexpYZdHJWRLgnUhOx7OMjJc/4iLCAFCoN +3KJ7/V1keo7GBHOwnsFcCA== +-----END PRIVATE KEY----- diff --git a/apps/emqx/test/emqx_ocsp_cache_SUITE.erl b/apps/emqx/test/emqx_ocsp_cache_SUITE.erl index c45bc15ef..90cb5fd4d 100644 --- a/apps/emqx/test/emqx_ocsp_cache_SUITE.erl +++ b/apps/emqx/test/emqx_ocsp_cache_SUITE.erl @@ -76,7 +76,7 @@ init_per_testcase(t_openssl_client, Config) -> [], Handler, #{ - extra_mustache_vars => [{test_data_dir, DataDir}], + extra_mustache_vars => #{test_data_dir => DataDir}, conf_file_path => ConfFilePath } ), diff --git a/changes/ce/feat-10164.en.md b/changes/ce/feat-10164.en.md new file mode 100644 index 000000000..9acea755f --- /dev/null +++ b/changes/ce/feat-10164.en.md @@ -0,0 +1 @@ +Add CRL check support for TLS MQTT listeners. diff --git a/rebar.config b/rebar.config index 5ce9138ce..343c6be69 100644 --- a/rebar.config +++ b/rebar.config @@ -37,7 +37,13 @@ {cover_opts, [verbose]}. {cover_export_enabled, true}. -{cover_excl_mods, [emqx_exproto_pb, emqx_exhook_pb]}. +{cover_excl_mods, + [ %% generated protobuf modules + emqx_exproto_pb, + emqx_exhook_pb, + %% taken almost as-is from OTP + emqx_ssl_crl_cache + ]}. {provider_hooks, [{pre, [{release, {relup_helper, gen_appups}}]}]}. From f2b54588e79d57a87e04d822d75964daddbeef7f Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 20 Mar 2023 11:16:32 -0300 Subject: [PATCH 024/196] docs: improve descriptions Co-authored-by: Zaiming (Stone) Shi --- apps/emqx/i18n/emqx_schema_i18n.conf | 8 ++++---- apps/emqx/src/emqx_schema.erl | 5 +---- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/apps/emqx/i18n/emqx_schema_i18n.conf index e05ee4c35..6f926ec39 100644 --- a/apps/emqx/i18n/emqx_schema_i18n.conf +++ b/apps/emqx/i18n/emqx_schema_i18n.conf @@ -1824,8 +1824,8 @@ server_ssl_opts_schema_enable_crl_check { crl_cache_refresh_http_timeout { desc { en: "The timeout for the HTTP request when fetching CRLs. This is" - " global for all listeners." - zh: "获取 CRLs 时 HTTP 请求的超时。 这对所有监听器来说是全局的。" + " a global setting for all listeners." + zh: "获取 CRLs 时 HTTP 请求的超时。 该配置对所有启用 CRL 检查的监听器监听器有效。" } label: { en: "CRL Cache Refresh HTTP Timeout" @@ -1835,9 +1835,9 @@ crl_cache_refresh_http_timeout { crl_cache_refresh_interval { desc { - en: "The period to refresh the CRLs from the servers. This is global" + en: "The period to refresh the CRLs from the servers. This is a global setting" " for all URLs and listeners." - zh: "从服务器刷新CRL的周期。 这对所有URL和监听器来说是全局性的。" + zh: "从服务器刷新CRL的周期。 该配置对所有 URL 和监听器有效。" } label: { en: "CRL Cache Refresh Interval" diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 25b1dd1d6..433fb20e5 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -230,7 +230,7 @@ roots(low) -> {"crl_cache", sc( ref("crl_cache"), - #{} + #{hidden => true} )} ]. @@ -809,7 +809,6 @@ fields("crl_cache") -> sc( duration(), #{ - hidden => true, default => <<"15m">>, desc => ?DESC("crl_cache_refresh_interval") } @@ -818,7 +817,6 @@ fields("crl_cache") -> sc( duration(), #{ - hidden => true, default => <<"15s">>, desc => ?DESC("crl_cache_refresh_http_timeout") } @@ -827,7 +825,6 @@ fields("crl_cache") -> sc( pos_integer(), #{ - hidden => true, default => 100, desc => ?DESC("crl_cache_capacity") } From b059bad08aca12315a6ebeb3149f27114b111e8a Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 21 Mar 2023 09:06:46 +0800 Subject: [PATCH 025/196] chore(acl): fully match `dashboard` username Co-authored-by: Ilya Averyanov --- apps/emqx_authz/etc/acl.conf | 2 +- changes/ce/fix-10172.en.md | 4 ++-- changes/ce/fix-10172.zh.md | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/emqx_authz/etc/acl.conf b/apps/emqx_authz/etc/acl.conf index 32a693ab8..a64287a4a 100644 --- a/apps/emqx_authz/etc/acl.conf +++ b/apps/emqx_authz/etc/acl.conf @@ -23,7 +23,7 @@ %% -type(rule() :: {permission(), who(), access(), topics()} | {permission(), all}). %%-------------------------------------------------------------------- -{allow, {username, {re, "^dashboard?"}}, subscribe, ["$SYS/#"]}. +{allow, {username, {re, "^dashboard$"}}, subscribe, ["$SYS/#"]}. {allow, {ipaddr, "127.0.0.1"}, all, ["$SYS/#", "#"]}. diff --git a/changes/ce/fix-10172.en.md b/changes/ce/fix-10172.en.md index 14757e44d..821bfbfe0 100644 --- a/changes/ce/fix-10172.en.md +++ b/changes/ce/fix-10172.en.md @@ -1,9 +1,9 @@ Fix the incorrect default ACL rule, which was: ``` -{allow, {username, "^dashboard?"}, subscribe, ["$SYS/#"]}. +{allow, {username, "^dashboard$"}, subscribe, ["$SYS/#"]}. ``` -However, it should use `{re, "^dashboard?"}` to perform a regular expression match: +However, it should use `{re, "^dashboard$"}` to perform a regular expression match: ``` {allow, {username, {re,"^dashboard?"}}, subscribe ,["$SYS/#"]}. ``` diff --git a/changes/ce/fix-10172.zh.md b/changes/ce/fix-10172.zh.md index b4a405c4c..1111c7613 100644 --- a/changes/ce/fix-10172.zh.md +++ b/changes/ce/fix-10172.zh.md @@ -1,6 +1,6 @@ 修复错误的默认 ACL 规则，之前是： ``` -{allow, {username, "^dashboard?"}, subscribe, ["$SYS/#"]}. +{allow, {username, "^dashboard$"}, subscribe, ["$SYS/#"]}. ``` 但执行正则表达式的匹配应该使用 `{re, "^dashboard?”}`： ``` From 84636a35752adeabead6308b4a0fe347b1b414ea Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 21 Mar 2023 09:10:20 +0800 Subject: [PATCH 026/196] chore: improve changes --- changes/ce/fix-10172.en.md | 4 ++-- changes/ce/fix-10172.zh.md | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/changes/ce/fix-10172.en.md b/changes/ce/fix-10172.en.md index 821bfbfe0..8a0c5c96e 100644 --- a/changes/ce/fix-10172.en.md +++ b/changes/ce/fix-10172.en.md @@ -1,9 +1,9 @@ Fix the incorrect default ACL rule, which was: ``` -{allow, {username, "^dashboard$"}, subscribe, ["$SYS/#"]}. +{allow, {username, "^dashboard?"}, subscribe, ["$SYS/#"]}. ``` However, it should use `{re, "^dashboard$"}` to perform a regular expression match: ``` -{allow, {username, {re,"^dashboard?"}}, subscribe ,["$SYS/#"]}. +{allow, {username, {re,"^dashboard$"}}, subscribe ,["$SYS/#"]}. ``` diff --git a/changes/ce/fix-10172.zh.md b/changes/ce/fix-10172.zh.md index 1111c7613..bfdfab60c 100644 --- a/changes/ce/fix-10172.zh.md +++ b/changes/ce/fix-10172.zh.md @@ -1,8 +1,8 @@ 修复错误的默认 ACL 规则，之前是： ``` -{allow, {username, "^dashboard$"}, subscribe, ["$SYS/#"]}. +{allow, {username, "^dashboard?"}, subscribe, ["$SYS/#"]}. ``` -但执行正则表达式的匹配应该使用 `{re, "^dashboard?”}`： +但执行正则表达式的匹配应该使用 `{re, "^dashboard$"}`： ``` -{allow, {username, {re, "^dashboard?"}}, subscribe, ["$SYS/#"]}. +{allow, {username, {re, "^dashboard$"}}, subscribe, ["$SYS/#"]}. ``` From ae5c24445bd7c890706bd86cc53ea7670dc6f127 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 21 Mar 2023 10:34:37 +0800 Subject: [PATCH 027/196] chore: make shellcheck happy --- lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_cassa.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_cassa.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_cassa.conf index 3e9a9845e..b8d810413 100644 --- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_cassa.conf +++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_cassa.conf @@ -39,7 +39,7 @@ will be forwarded.""" desc_config { desc { - en: """Configuration for an Cassandra bridge.""" + en: """Configuration for a Cassandra bridge.""" zh: """Cassandra 桥接配置""" } label: { From 89782e6d2113e794b6931e5e4d75c9d3d15a46e2 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Tue, 21 Mar 2023 06:26:04 +0100 Subject: [PATCH 028/196] docs: add info about that rule engine can handle different types of structured data Co-authored-by: Zaiming (Stone) Shi --- apps/emqx_rule_engine/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_rule_engine/README.md b/apps/emqx_rule_engine/README.md index f1533a0e3..2c2e43db3 100644 --- a/apps/emqx_rule_engine/README.md +++ b/apps/emqx_rule_engine/README.md @@ -14,8 +14,8 @@ specifying a topic filter and a set of conditions that need to be met. If a message matches the topic filter and all the conditions are met, the rule is triggered. The rule can then transform the message and route it to a different topic, or send it to another service (defined by an EMQX bridge). The rule -engine's message data transformation is designed to work well with JSON -formatted data. +engine's message data transformation is designed to work well with structured data +such as JSON, avro, and protobuf. A rule consists of the three parts **MATCH**, **TRANSFORM** and **ACTIONS** that are From 539ec2f774f3c73f2c214023b1785d00a5f8318b Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 21 Mar 2023 13:55:53 +0800 Subject: [PATCH 029/196] chore(bridge): cover username/password auth for cassandra bridges --- .ci/docker-compose-file/cassandra/cassandra.yaml | 2 +- ...andra-tcp.yaml => docker-compose-cassandra.yaml} | 6 +++--- .../test/emqx_ee_bridge_cassa_SUITE.erl | 4 ++-- .../test/emqx_ee_connector_cassa_SUITE.erl | 13 +++++++++---- scripts/ct/run.sh | 2 +- 5 files changed, 16 insertions(+), 11 deletions(-) rename .ci/docker-compose-file/{docker-compose-cassandra-tcp.yaml => docker-compose-cassandra.yaml} (63%) diff --git a/.ci/docker-compose-file/cassandra/cassandra.yaml b/.ci/docker-compose-file/cassandra/cassandra.yaml index 51a24f7a2..968efe5f6 100644 --- a/.ci/docker-compose-file/cassandra/cassandra.yaml +++ b/.ci/docker-compose-file/cassandra/cassandra.yaml @@ -100,7 +100,7 @@ batchlog_replay_throttle_in_kb: 1024 # users. It keeps usernames and hashed passwords in system_auth.roles table. # Please increase system_auth keyspace replication factor if you use this authenticator. # If using PasswordAuthenticator, CassandraRoleManager must also be used (see below) -authenticator: AllowAllAuthenticator +authenticator: PasswordAuthenticator # Authorization backend, implementing IAuthorizer; used to limit access/provide permissions # Out of the box, Cassandra provides org.apache.cassandra.auth.{AllowAllAuthorizer, diff --git a/.ci/docker-compose-file/docker-compose-cassandra-tcp.yaml b/.ci/docker-compose-file/docker-compose-cassandra.yaml similarity index 63% rename from .ci/docker-compose-file/docker-compose-cassandra-tcp.yaml rename to .ci/docker-compose-file/docker-compose-cassandra.yaml index ac45af02f..393a5cac7 100644 --- a/.ci/docker-compose-file/docker-compose-cassandra-tcp.yaml +++ b/.ci/docker-compose-file/docker-compose-cassandra.yaml @@ -22,9 +22,9 @@ services: - -c - | /opt/cassandra/bin/cassandra -f -R > /cassandra.log & - /opt/cassandra/bin/cqlsh -e "CREATE KEYSPACE mqtt WITH REPLICATION = { 'class':'SimpleStrategy','replication_factor':1};" - while [[ $$? -ne 0 ]];do sleep 5; /opt/cassandra/bin/cqlsh -e "CREATE KEYSPACE mqtt WITH REPLICATION = { 'class':'SimpleStrategy','replication_factor':1};"; done - /opt/cassandra/bin/cqlsh -e "describe keyspaces;" + /opt/cassandra/bin/cqlsh -u cassandra -p cassandra -e "CREATE KEYSPACE mqtt WITH REPLICATION = { 'class':'SimpleStrategy','replication_factor':1};" + while [[ $$? -ne 0 ]];do sleep 5; /opt/cassandra/bin/cqlsh -u cassandra -p cassandra -e "CREATE KEYSPACE mqtt WITH REPLICATION = { 'class':'SimpleStrategy','replication_factor':1};"; done + /opt/cassandra/bin/cqlsh -u cassandra -p cassandra -e "describe keyspaces;" tail -f /cassandra.log networks: - emqx_bridge diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl index b99bd95e0..666cd0caf 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl @@ -33,8 +33,8 @@ % DB defaults -define(CASSA_KEYSPACE, "mqtt"). --define(CASSA_USERNAME, "root"). --define(CASSA_PASSWORD, "public"). +-define(CASSA_USERNAME, "cassandra"). +-define(CASSA_PASSWORD, "cassandra"). -define(BATCH_SIZE, 10). %% cert files for client diff --git a/lib-ee/emqx_ee_connector/test/emqx_ee_connector_cassa_SUITE.erl b/lib-ee/emqx_ee_connector/test/emqx_ee_connector_cassa_SUITE.erl index 4436038d9..81b9e3859 100644 --- a/lib-ee/emqx_ee_connector/test/emqx_ee_connector_cassa_SUITE.erl +++ b/lib-ee/emqx_ee_connector/test/emqx_ee_connector_cassa_SUITE.erl @@ -37,6 +37,11 @@ %% %% sudo docker run --rm -d --name cassandra --network host cassandra:3.11.14 +%% Cassandra default username & password once enable `authenticator: PasswordAuthenticator` +%% in cassandra config +-define(CASSA_USERNAME, <<"cassandra">>). +-define(CASSA_PASSWORD, <<"cassandra">>). + all() -> emqx_common_test_helpers:all(?MODULE). @@ -62,8 +67,8 @@ init_per_suite(Config) -> {ok, Conn} = ecql:connect([ {nodes, cassandra_servers()}, - {username, <<"admin">>}, - {password, <<"public">>}, + {username, ?CASSA_USERNAME}, + {password, ?CASSA_PASSWORD}, {keyspace, "mqtt"} ]), ecql:close(Conn), @@ -175,8 +180,8 @@ cassandra_config() -> #{ auto_reconnect => true, keyspace => <<"mqtt">>, - username => <<"default">>, - password => <<"public">>, + username => ?CASSA_USERNAME, + password => ?CASSA_PASSWORD, pool_size => 8, servers => iolist_to_binary( io_lib:format( diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index 10eeff727..38d838985 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -171,7 +171,7 @@ for dep in ${CT_DEPS}; do FILES+=( '.ci/docker-compose-file/docker-compose-dynamo.yaml' ) ;; cassandra) - FILES+=( '.ci/docker-compose-file/docker-compose-cassandra-tcp.yaml' ) + FILES+=( '.ci/docker-compose-file/docker-compose-cassandra.yaml' ) ;; *) echo "unknown_ct_dependency $dep" From b6e583143fae9e5cbb5483e731226ef573819e88 Mon Sep 17 00:00:00 2001 From: William Yang Date: Tue, 21 Mar 2023 09:01:25 +0100 Subject: [PATCH 030/196] chore: bump quicer 0.0.114 for fetching quicer binary for macOS M1 builds. --- apps/emqx/rebar.config.script | 2 +- mix.exs | 2 +- rebar.config.erl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/rebar.config.script b/apps/emqx/rebar.config.script index 0827570ff..12e6124ac 100644 --- a/apps/emqx/rebar.config.script +++ b/apps/emqx/rebar.config.script @@ -24,7 +24,7 @@ IsQuicSupp = fun() -> end, Bcrypt = {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}}, -Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.113"}}}. +Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.114"}}}. Dialyzer = fun(Config) -> {dialyzer, OldDialyzerConfig} = lists:keyfind(dialyzer, 1, Config), diff --git a/mix.exs b/mix.exs index 42354f8dc..74be81bcd 100644 --- a/mix.exs +++ b/mix.exs @@ -651,7 +651,7 @@ defmodule EMQXUmbrella.MixProject do defp quicer_dep() do if enable_quicer?(), # in conflict with emqx and emqtt - do: [{:quicer, github: "emqx/quic", tag: "0.0.113", override: true}], + do: [{:quicer, github: "emqx/quic", tag: "0.0.114", override: true}], else: [] end diff --git a/rebar.config.erl b/rebar.config.erl index e976d7729..98cd30570 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -39,7 +39,7 @@ bcrypt() -> {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}}. quicer() -> - {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.113"}}}. + {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.114"}}}. jq() -> {jq, {git, "https://github.com/emqx/jq", {tag, "v0.3.9"}}}. From 092daf409a8956c5edee7ec13bc8be4871f56ce2 Mon Sep 17 00:00:00 2001 From: William Yang Date: Tue, 21 Mar 2023 09:03:31 +0100 Subject: [PATCH 031/196] chore(testlib): update include for CI of emqtt lib --- apps/emqx/test/emqx_common_test_helpers.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index 38f30b8c5..01a01b1fc 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -16,7 +16,7 @@ -module(emqx_common_test_helpers). --include("emqx_authentication.hrl"). +-include_lib("emqx/include/emqx_authentication.hrl"). -type special_config_handler() :: fun(). From b62e9bba807d29eb0b65c707476a6438bbc84cb9 Mon Sep 17 00:00:00 2001 From: William Yang Date: Tue, 21 Mar 2023 10:04:02 +0100 Subject: [PATCH 032/196] ci: rerun failed checks on master branch --- .github/workflows/geen_master.yaml | 26 ++++++ scripts/rerun-failed-checks.py | 128 +++++++++++++++++++++++++++++ 2 files changed, 154 insertions(+) create mode 100644 .github/workflows/geen_master.yaml create mode 100644 scripts/rerun-failed-checks.py diff --git a/.github/workflows/geen_master.yaml b/.github/workflows/geen_master.yaml new file mode 100644 index 000000000..1161ca7d4 --- /dev/null +++ b/.github/workflows/geen_master.yaml @@ -0,0 +1,26 @@ +--- + +name: Keep master green + +on: + schedule: + # run hourly + - cron: "0 * * * *" + workflow_dispatch: + +jobs: + rerun-failed-jobs: + runs-on: ubuntu-22.04 + if: github.repository_owner == 'emqx' + permissions: + checks: read + actions: write + steps: + - uses: actions/checkout@v3 + + - name: run script + shell: bash + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + python3 scripts/rerun-failed-checks.py diff --git a/scripts/rerun-failed-checks.py b/scripts/rerun-failed-checks.py new file mode 100644 index 000000000..167d8e81c --- /dev/null +++ b/scripts/rerun-failed-checks.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +# Usage: python3 rerun-failed-checks.py -t -r -b +# +# Description: This script will fetch the latest commit from a branch, and check the status of all check runs of the commit. +# If any check run is not successful, it will trigger a rerun of the failed jobs. +# +# Default branch is master, default repo is emqx/emqx +# +# Limitation: only works for upstream repo, not for forked. +import requests +import http.client +import json +import os +import sys +import time +from optparse import OptionParser + +job_black_list = [ + 'windows', + 'publish_artifacts', + 'stale' +] + +def fetch_latest_commit(token: str, repo: str, branch: str): + url = f'https://api.github.com/repos/{repo}/commits/{branch}' + headers = {'Accept': 'application/vnd.github+json', + 'Authorization': f'Bearer {token}', + 'X-GitHub-Api-Version': '2022-11-28', + 'User-Agent': 'python3' + } + r = requests.get(url, headers=headers) + if r.status_code == 200: + res = r.json() + return res + else: + print( + f'Failed to fetch latest commit from {branch} branch, code: {r.status_code}') + sys.exit(1) + + +''' +fetch check runs of a commit. +@note, only works for public repos +''' +def fetch_check_runs(token: str, repo: str, ref: str): + failed_checks = [] + url = f'https://api.github.com/repos/{repo}/commits/{ref}/check-runs?per_page=100' + headers = {'Accept': 'application/vnd.github.v3+json', + 'Authorization': f'Bearer {token}' + } + r = requests.get(url, headers=headers) + if r.status_code == 200: + for crun in r.json()['check_runs']: + if crun['status'] == 'completed' and crun['conclusion'] != 'success': + print('Failed check: ', crun['name']) + failed_checks.append( + {'id': crun['id'], 'name': crun['name'], 'url': crun['url']}) + else: + # pretty print crun + # print(json.dumps(crun, indent=4)) + print('successed:', crun['id'], crun['name'], + crun['status'], crun['conclusion']) + else: + print(f'Failed to fetch check runs {r.status_code}') + sys.exit(1) + return failed_checks + + +''' +rerquest a check-run +''' +def trigger_build(failed_checks: list, repo: str, token: str): + reruns = [] + for crun in failed_checks: + if crun['name'] in job_black_list: + print(f'Skip black listed job {crun["name"]}') + continue + + r = requests.get(crun['url'], headers={'Accept': 'application/vnd.github.v3+json', + 'User-Agent': 'python3', + 'Authorization': f'Bearer {token}'} + ) + if r.status_code == 200: + # url example: https://github.com/qzhuyan/emqx/actions/runs/4469557961/jobs/7852858687 + run_id = r.json()['details_url'].split('/')[-3] + reruns.append(run_id) + else: + print(f'failed to fetch check run {crun["name"]}') + + # remove duplicates + for run_id in set(reruns): + url = f'https://api.github.com/repos/{repo}/actions/runs/{run_id}/rerun-failed-jobs' + + r = requests.post(url, headers={'Accept': 'application/vnd.github.v3+json', + 'User-Agent': 'python3', + 'Authorization': f'Bearer {token}'} + ) + if r.status_code == 201: + print(f'Successfully triggered build for {crun["name"]}') + + else: + # Only complain but not exit. + print( + f'Failed to trigger build for {crun["name"]} : {r.status_code} : {r.text}') + + +def main(): + parser = OptionParser() + parser.add_option("-r", "--repo", dest="repo", + help="github repo", default="emqx/emqx") + parser.add_option("-t", "--token", dest="gh_token", + help="github API token") + parser.add_option("-b", "--branch", dest="branch", default='master', + help="Branch that workflow runs on") + (options, args) = parser.parse_args() + + # Get gh token from env var GITHUB_TOKEN if provided, else use the one from command line + token = os.environ['GITHUB_TOKEN'] if 'GITHUB_TOKEN' in os.environ else options.gh_token + + target_commit = fetch_latest_commit(token, options.repo, options.branch) + + failed_checks = fetch_check_runs(token, options.repo, target_commit['sha']) + + trigger_build(failed_checks, options.repo, token) + + +if __name__ == '__main__': + main() From 97e7c439f515239b9edc1d2735d328af65d0e51b Mon Sep 17 00:00:00 2001 From: William Yang Date: Tue, 21 Mar 2023 10:49:18 +0100 Subject: [PATCH 033/196] ci(gree-master): handle pagination --- scripts/rerun-failed-checks.py | 59 +++++++++++++++++++++------------- 1 file changed, 37 insertions(+), 22 deletions(-) diff --git a/scripts/rerun-failed-checks.py b/scripts/rerun-failed-checks.py index 167d8e81c..ff9b9f33e 100644 --- a/scripts/rerun-failed-checks.py +++ b/scripts/rerun-failed-checks.py @@ -13,6 +13,7 @@ import json import os import sys import time +import math from optparse import OptionParser job_black_list = [ @@ -43,28 +44,42 @@ fetch check runs of a commit. @note, only works for public repos ''' def fetch_check_runs(token: str, repo: str, ref: str): + all_checks = [] + page = 1 + total_pages = 1 + per_page = 100 failed_checks = [] - url = f'https://api.github.com/repos/{repo}/commits/{ref}/check-runs?per_page=100' - headers = {'Accept': 'application/vnd.github.v3+json', - 'Authorization': f'Bearer {token}' - } - r = requests.get(url, headers=headers) - if r.status_code == 200: - for crun in r.json()['check_runs']: - if crun['status'] == 'completed' and crun['conclusion'] != 'success': - print('Failed check: ', crun['name']) - failed_checks.append( - {'id': crun['id'], 'name': crun['name'], 'url': crun['url']}) - else: - # pretty print crun - # print(json.dumps(crun, indent=4)) - print('successed:', crun['id'], crun['name'], - crun['status'], crun['conclusion']) - else: - print(f'Failed to fetch check runs {r.status_code}') - sys.exit(1) - return failed_checks + while page <= total_pages: + print(f'Fetching check runs for page {page} of {total_pages} pages') + url = f'https://api.github.com/repos/{repo}/commits/{ref}/check-runs?per_page={per_page}&page={page}' + headers = {'Accept': 'application/vnd.github.v3+json', + 'Authorization': f'Bearer {token}' + } + r = requests.get(url, headers=headers) + if r.status_code == 200: + resp = r.json() + all_checks.extend(resp['check_runs']) + page += 1 + if 'total_count' in resp and resp['total_count'] > per_page: + total_pages = math.ceil(resp['total_count'] / per_page) + else: + print(f'Failed to fetch check runs {r.status_code}') + sys.exit(1) + + + for crun in all_checks: + if crun['status'] == 'completed' and crun['conclusion'] != 'success': + print('Failed check: ', crun['name']) + failed_checks.append( + {'id': crun['id'], 'name': crun['name'], 'url': crun['url']}) + else: + # pretty print crun + # print(json.dumps(crun, indent=4)) + print('successed:', crun['id'], crun['name'], + crun['status'], crun['conclusion']) + + return failed_checks ''' rerquest a check-run @@ -72,7 +87,7 @@ rerquest a check-run def trigger_build(failed_checks: list, repo: str, token: str): reruns = [] for crun in failed_checks: - if crun['name'] in job_black_list: + if crun['name'].strip() in job_black_list: print(f'Skip black listed job {crun["name"]}') continue @@ -101,7 +116,7 @@ def trigger_build(failed_checks: list, repo: str, token: str): else: # Only complain but not exit. print( - f'Failed to trigger build for {crun["name"]} : {r.status_code} : {r.text}') + f'Failed to trigger rerun for {run_id}, {crun["name"]}: {r.status_code} : {r.text}') def main(): From 53825b9abace7225d52458c9e0c04aaa4b934ff9 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Wed, 15 Mar 2023 14:50:40 +0100 Subject: [PATCH 034/196] fix(emqx_bridge): propagate connection error to resource status --- apps/emqx_bridge/i18n/emqx_bridge_schema.conf | 11 ++ apps/emqx_bridge/src/emqx_bridge_api.erl | 102 +++++++++-------- .../src/schema/emqx_bridge_schema.erl | 14 ++- .../test/emqx_bridge_api_SUITE.erl | 107 +++++++++++------- apps/emqx_resource/include/emqx_resource.hrl | 1 + .../src/emqx_resource_manager.erl | 1 + changes/ce/fix-10145.en.md | 1 + 7 files changed, 152 insertions(+), 85 deletions(-) create mode 100644 changes/ce/fix-10145.en.md diff --git a/apps/emqx_bridge/i18n/emqx_bridge_schema.conf b/apps/emqx_bridge/i18n/emqx_bridge_schema.conf index 901f25455..de4ceb0d5 100644 --- a/apps/emqx_bridge/i18n/emqx_bridge_schema.conf +++ b/apps/emqx_bridge/i18n/emqx_bridge_schema.conf @@ -54,6 +54,17 @@ emqx_bridge_schema { } } + desc_status_reason { + desc { + en: "This is the reason given in case a bridge is failing to connect." + zh: "桥接连接失败的原因。" + } + label: { + en: "Failure reason" + zh: "失败原因" + } + } + desc_node_status { desc { en: """The status of the bridge for each node. diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index 8ac3e476a..c0e69ce83 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -748,7 +748,7 @@ pick_bridges_by_id(Type, Name, BridgesAllNodes) -> format_bridge_info_with_metrics([FirstBridge | _] = Bridges) -> Res = maps:remove(node, FirstBridge), - NodeStatus = collect_status(Bridges), + NodeStatus = node_status(Bridges), NodeMetrics = collect_metrics(Bridges), redact(Res#{ status => aggregate_status(NodeStatus), @@ -765,8 +765,8 @@ format_bridge_metrics(Bridges) -> Res = format_bridge_info_with_metrics(Bridges), maps:with([metrics, node_metrics], Res). -collect_status(Bridges) -> - [maps:with([node, status], B) || B <- Bridges]. +node_status(Bridges) -> + [maps:with([node, status, status_reason], B) || B <- Bridges]. aggregate_status(AllStatus) -> Head = fun([A | _]) -> A end, @@ -837,52 +837,63 @@ format_resource( ) ). -format_resource_data(#{status := Status, metrics := Metrics}) -> - #{status => Status, metrics => format_metrics(Metrics)}; -format_resource_data(#{status := Status}) -> - #{status => Status}. +format_resource_data(ResData) -> + maps:fold(fun format_resource_data/3, #{}, maps:with([status, metrics, error], ResData)). -format_metrics(#{ - counters := #{ - 'dropped' := Dropped, - 'dropped.other' := DroppedOther, - 'dropped.expired' := DroppedExpired, - 'dropped.queue_full' := DroppedQueueFull, - 'dropped.resource_not_found' := DroppedResourceNotFound, - 'dropped.resource_stopped' := DroppedResourceStopped, - 'matched' := Matched, - 'retried' := Retried, - 'late_reply' := LateReply, - 'failed' := SentFailed, - 'success' := SentSucc, - 'received' := Rcvd +format_resource_data(error, undefined, Result) -> + Result; +format_resource_data(error, Error, Result) -> + Result#{status_reason => emqx_misc:readable_error_msg(Error)}; +format_resource_data( + metrics, + #{ + counters := #{ + 'dropped' := Dropped, + 'dropped.other' := DroppedOther, + 'dropped.expired' := DroppedExpired, + 'dropped.queue_full' := DroppedQueueFull, + 'dropped.resource_not_found' := DroppedResourceNotFound, + 'dropped.resource_stopped' := DroppedResourceStopped, + 'matched' := Matched, + 'retried' := Retried, + 'late_reply' := LateReply, + 'failed' := SentFailed, + 'success' := SentSucc, + 'received' := Rcvd + }, + gauges := Gauges, + rate := #{ + matched := #{current := Rate, last5m := Rate5m, max := RateMax} + } }, - gauges := Gauges, - rate := #{ - matched := #{current := Rate, last5m := Rate5m, max := RateMax} - } -}) -> + Result +) -> Queued = maps:get('queuing', Gauges, 0), SentInflight = maps:get('inflight', Gauges, 0), - ?METRICS( - Dropped, - DroppedOther, - DroppedExpired, - DroppedQueueFull, - DroppedResourceNotFound, - DroppedResourceStopped, - Matched, - Queued, - Retried, - LateReply, - SentFailed, - SentInflight, - SentSucc, - Rate, - Rate5m, - RateMax, - Rcvd - ). + Result#{ + metrics => + ?METRICS( + Dropped, + DroppedOther, + DroppedExpired, + DroppedQueueFull, + DroppedResourceNotFound, + DroppedResourceStopped, + Matched, + Queued, + Retried, + LateReply, + SentFailed, + SentInflight, + SentSucc, + Rate, + Rate5m, + RateMax, + Rcvd + ) + }; +format_resource_data(K, V, Result) -> + Result#{K => V}. fill_defaults(Type, RawConf) -> PackedConf = pack_bridge_conf(Type, RawConf), @@ -924,6 +935,7 @@ filter_out_request_body(Conf) -> <<"type">>, <<"name">>, <<"status">>, + <<"error">>, <<"node_status">>, <<"node_metrics">>, <<"metrics">>, diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl index 74d2a5ca1..6c278a5ec 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl @@ -106,6 +106,12 @@ common_bridge_fields() -> status_fields() -> [ {"status", mk(status(), #{desc => ?DESC("desc_status")})}, + {"status_reason", + mk(binary(), #{ + required => false, + desc => ?DESC("desc_status_reason"), + example => <<"Connection refused">> + })}, {"node_status", mk( hoconsc:array(ref(?MODULE, "node_status")), @@ -190,7 +196,13 @@ fields("node_metrics") -> fields("node_status") -> [ node_name(), - {"status", mk(status(), #{})} + {"status", mk(status(), #{})}, + {"status_reason", + mk(binary(), #{ + required => false, + desc => ?DESC("desc_status_reason"), + example => <<"Connection refused">> + })} ]. desc(bridges) -> diff --git a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl index 8b388a771..b919dba6b 100644 --- a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl @@ -23,7 +23,7 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). -define(CONF_DEFAULT, <<"bridges: {}">>). --define(BRIDGE_TYPE, <<"webhook">>). +-define(BRIDGE_TYPE_HTTP, <<"webhook">>). -define(BRIDGE_NAME, (atom_to_binary(?FUNCTION_NAME))). -define(URL(PORT, PATH), list_to_binary( @@ -48,7 +48,7 @@ }). -define(MQTT_BRIDGE(SERVER), ?MQTT_BRIDGE(SERVER, <<"mqtt_egress_test_bridge">>)). --define(HTTP_BRIDGE(URL, TYPE, NAME), ?BRIDGE(NAME, TYPE)#{ +-define(HTTP_BRIDGE(URL, NAME), ?BRIDGE(NAME, ?BRIDGE_TYPE_HTTP)#{ <<"url">> => URL, <<"local_topic">> => <<"emqx_webhook/#">>, <<"method">> => <<"post">>, @@ -57,6 +57,7 @@ <<"content-type">> => <<"application/json">> } }). +-define(HTTP_BRIDGE(URL), ?HTTP_BRIDGE(URL, ?BRIDGE_NAME)). all() -> emqx_common_test_helpers:all(?MODULE). @@ -206,12 +207,12 @@ t_http_crud_apis(Config) -> {ok, 201, Bridge} = request( post, uri(["bridges"]), - ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, Name) + ?HTTP_BRIDGE(URL1, Name) ), %ct:pal("---bridge: ~p", [Bridge]), #{ - <<"type">> := ?BRIDGE_TYPE, + <<"type">> := ?BRIDGE_TYPE_HTTP, <<"name">> := Name, <<"enable">> := true, <<"status">> := _, @@ -219,7 +220,7 @@ t_http_crud_apis(Config) -> <<"url">> := URL1 } = emqx_json:decode(Bridge, [return_maps]), - BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, Name), + BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_HTTP, Name), %% send an message to emqx and the message should be forwarded to the HTTP server Body = <<"my msg">>, emqx:publish(emqx_message:make(<<"emqx_webhook/1">>, Body)), @@ -243,11 +244,11 @@ t_http_crud_apis(Config) -> {ok, 200, Bridge2} = request( put, uri(["bridges", BridgeID]), - ?HTTP_BRIDGE(URL2, ?BRIDGE_TYPE, Name) + ?HTTP_BRIDGE(URL2, Name) ), ?assertMatch( #{ - <<"type">> := ?BRIDGE_TYPE, + <<"type">> := ?BRIDGE_TYPE_HTTP, <<"name">> := Name, <<"enable">> := true, <<"status">> := _, @@ -262,7 +263,7 @@ t_http_crud_apis(Config) -> ?assertMatch( [ #{ - <<"type">> := ?BRIDGE_TYPE, + <<"type">> := ?BRIDGE_TYPE_HTTP, <<"name">> := Name, <<"enable">> := true, <<"status">> := _, @@ -279,7 +280,7 @@ t_http_crud_apis(Config) -> {ok, 200, Bridge3Str} = request(get, uri(["bridges", BridgeID]), []), ?assertMatch( #{ - <<"type">> := ?BRIDGE_TYPE, + <<"type">> := ?BRIDGE_TYPE_HTTP, <<"name">> := Name, <<"enable">> := true, <<"status">> := _, @@ -311,7 +312,7 @@ t_http_crud_apis(Config) -> {ok, 404, ErrMsg2} = request( put, uri(["bridges", BridgeID]), - ?HTTP_BRIDGE(URL2, ?BRIDGE_TYPE, Name) + ?HTTP_BRIDGE(URL2, Name) ), ?assertMatch( #{ @@ -340,6 +341,34 @@ t_http_crud_apis(Config) -> }, emqx_json:decode(ErrMsg3, [return_maps]) ), + + %% Create non working bridge + BrokenURL = ?URL(Port + 1, "/foo"), + {ok, 201, BrokenBridge} = request( + post, + uri(["bridges"]), + ?HTTP_BRIDGE(BrokenURL, Name) + ), + #{ + <<"type">> := ?BRIDGE_TYPE_HTTP, + <<"name">> := Name, + <<"enable">> := true, + <<"status">> := <<"disconnected">>, + <<"status_reason">> := <<"Connection refused">>, + <<"node_status">> := [ + #{<<"status">> := <<"disconnected">>, <<"status_reason">> := <<"Connection refused">>} + | _ + ], + <<"url">> := BrokenURL + } = emqx_json:decode(BrokenBridge, [return_maps]), + {ok, 200, FixedBridgeResponse} = request(put, uri(["bridges", BridgeID]), ?HTTP_BRIDGE(URL1)), + #{ + <<"status">> := <<"connected">>, + <<"node_status">> := [FixedNodeStatus = #{<<"status">> := <<"connected">>} | _] + } = FixedBridge = emqx_json:decode(FixedBridgeResponse, [return_maps]), + ?assert(not maps:is_key(<<"status_reason">>, FixedBridge)), + ?assert(not maps:is_key(<<"status_reason">>, FixedNodeStatus)), + {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), []), ok. t_http_bridges_local_topic(Config) -> @@ -356,16 +385,16 @@ t_http_bridges_local_topic(Config) -> {ok, 201, _} = request( post, uri(["bridges"]), - ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, Name1) + ?HTTP_BRIDGE(URL1, Name1) ), %% and we create another one without local_topic {ok, 201, _} = request( post, uri(["bridges"]), - maps:remove(<<"local_topic">>, ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, Name2)) + maps:remove(<<"local_topic">>, ?HTTP_BRIDGE(URL1, Name2)) ), - BridgeID1 = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, Name1), - BridgeID2 = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, Name2), + BridgeID1 = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_HTTP, Name1), + BridgeID2 = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_HTTP, Name2), %% Send an message to emqx and the message should be forwarded to the HTTP server. %% This is to verify we can have 2 bridges with and without local_topic fields %% at the same time. @@ -400,11 +429,11 @@ t_check_dependent_actions_on_delete(Config) -> %% POST /bridges/ will create a bridge URL1 = ?URL(Port, "path1"), Name = <<"t_http_crud_apis">>, - BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, Name), + BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_HTTP, Name), {ok, 201, _} = request( post, uri(["bridges"]), - ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, Name) + ?HTTP_BRIDGE(URL1, Name) ), {ok, 201, Rule} = request( post, @@ -438,11 +467,11 @@ t_cascade_delete_actions(Config) -> %% POST /bridges/ will create a bridge URL1 = ?URL(Port, "path1"), Name = <<"t_http_crud_apis">>, - BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, Name), + BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_HTTP, Name), {ok, 201, _} = request( post, uri(["bridges"]), - ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, Name) + ?HTTP_BRIDGE(URL1, Name) ), {ok, 201, Rule} = request( post, @@ -472,7 +501,7 @@ t_cascade_delete_actions(Config) -> {ok, 201, _} = request( post, uri(["bridges"]), - ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, Name) + ?HTTP_BRIDGE(URL1, Name) ), {ok, 201, _} = request( post, @@ -496,9 +525,9 @@ t_broken_bpapi_vsn(Config) -> {ok, 201, _Bridge} = request( post, uri(["bridges"]), - ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, Name) + ?HTTP_BRIDGE(URL1, Name) ), - BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, Name), + BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_HTTP, Name), %% still works since we redirect to 'restart' {ok, 501, <<>>} = request(post, operation_path(cluster, start, BridgeID), <<"">>), {ok, 501, <<>>} = request(post, operation_path(node, start, BridgeID), <<"">>), @@ -511,9 +540,9 @@ t_old_bpapi_vsn(Config) -> {ok, 201, _Bridge} = request( post, uri(["bridges"]), - ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, Name) + ?HTTP_BRIDGE(URL1, Name) ), - BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, Name), + BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_HTTP, Name), {ok, 204, <<>>} = request(post, operation_path(cluster, stop, BridgeID), <<"">>), {ok, 204, <<>>} = request(post, operation_path(node, stop, BridgeID), <<"">>), %% still works since we redirect to 'restart' @@ -551,18 +580,18 @@ do_start_stop_bridges(Type, Config) -> {ok, 201, Bridge} = request( post, uri(["bridges"]), - ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, Name) + ?HTTP_BRIDGE(URL1, Name) ), %ct:pal("the bridge ==== ~p", [Bridge]), #{ - <<"type">> := ?BRIDGE_TYPE, + <<"type">> := ?BRIDGE_TYPE_HTTP, <<"name">> := Name, <<"enable">> := true, <<"status">> := <<"connected">>, <<"node_status">> := [_ | _], <<"url">> := URL1 } = emqx_json:decode(Bridge, [return_maps]), - BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, Name), + BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_HTTP, Name), %% stop it {ok, 204, <<>>} = request(post, operation_path(Type, stop, BridgeID), <<"">>), {ok, 200, Bridge2} = request(get, uri(["bridges", BridgeID]), []), @@ -633,18 +662,18 @@ t_enable_disable_bridges(Config) -> {ok, 201, Bridge} = request( post, uri(["bridges"]), - ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, Name) + ?HTTP_BRIDGE(URL1, Name) ), %ct:pal("the bridge ==== ~p", [Bridge]), #{ - <<"type">> := ?BRIDGE_TYPE, + <<"type">> := ?BRIDGE_TYPE_HTTP, <<"name">> := Name, <<"enable">> := true, <<"status">> := <<"connected">>, <<"node_status">> := [_ | _], <<"url">> := URL1 } = emqx_json:decode(Bridge, [return_maps]), - BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, Name), + BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_HTTP, Name), %% disable it {ok, 204, <<>>} = request(put, enable_path(false, BridgeID), <<"">>), {ok, 200, Bridge2} = request(get, uri(["bridges", BridgeID]), []), @@ -690,18 +719,18 @@ t_reset_bridges(Config) -> {ok, 201, Bridge} = request( post, uri(["bridges"]), - ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, Name) + ?HTTP_BRIDGE(URL1, Name) ), %ct:pal("the bridge ==== ~p", [Bridge]), #{ - <<"type">> := ?BRIDGE_TYPE, + <<"type">> := ?BRIDGE_TYPE_HTTP, <<"name">> := Name, <<"enable">> := true, <<"status">> := <<"connected">>, <<"node_status">> := [_ | _], <<"url">> := URL1 } = emqx_json:decode(Bridge, [return_maps]), - BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, Name), + BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_HTTP, Name), {ok, 204, <<>>} = request(put, uri(["bridges", BridgeID, "metrics/reset"]), []), %% delete the bridge @@ -748,20 +777,20 @@ t_bridges_probe(Config) -> {ok, 204, <<>>} = request( post, uri(["bridges_probe"]), - ?HTTP_BRIDGE(URL, ?BRIDGE_TYPE, ?BRIDGE_NAME) + ?HTTP_BRIDGE(URL) ), %% second time with same name is ok since no real bridge created {ok, 204, <<>>} = request( post, uri(["bridges_probe"]), - ?HTTP_BRIDGE(URL, ?BRIDGE_TYPE, ?BRIDGE_NAME) + ?HTTP_BRIDGE(URL) ), {ok, 400, NxDomain} = request( post, uri(["bridges_probe"]), - ?HTTP_BRIDGE(<<"http://203.0.113.3:1234/foo">>, ?BRIDGE_TYPE, ?BRIDGE_NAME) + ?HTTP_BRIDGE(<<"http://203.0.113.3:1234/foo">>) ), ?assertMatch( #{ @@ -882,12 +911,12 @@ t_metrics(Config) -> {ok, 201, Bridge} = request( post, uri(["bridges"]), - ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, Name) + ?HTTP_BRIDGE(URL1, Name) ), %ct:pal("---bridge: ~p", [Bridge]), #{ - <<"type">> := ?BRIDGE_TYPE, + <<"type">> := ?BRIDGE_TYPE_HTTP, <<"name">> := Name, <<"enable">> := true, <<"status">> := _, @@ -895,7 +924,7 @@ t_metrics(Config) -> <<"url">> := URL1 } = emqx_json:decode(Bridge, [return_maps]), - BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, Name), + BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_HTTP, Name), %% check for empty bridge metrics {ok, 200, Bridge1Str} = request(get, uri(["bridges", BridgeID, "metrics"]), []), @@ -963,7 +992,7 @@ t_inconsistent_webhook_request_timeouts(Config) -> Name = ?BRIDGE_NAME, BadBridgeParams = emqx_map_lib:deep_merge( - ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, Name), + ?HTTP_BRIDGE(URL1, Name), #{ <<"request_timeout">> => <<"1s">>, <<"resource_opts">> => #{<<"request_timeout">> => <<"2s">>} diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl index 41be9e8a0..ae22e27e0 100644 --- a/apps/emqx_resource/include/emqx_resource.hrl +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -41,6 +41,7 @@ callback_mode := callback_mode(), query_mode := query_mode(), config := resource_config(), + error := term(), state := resource_state(), status := resource_status(), metrics => emqx_metrics_worker:metrics() diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index 40f9fe1ab..1bc6a3b6a 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -666,6 +666,7 @@ maybe_reply(Actions, From, Reply) -> data_record_to_external_map(Data) -> #{ id => Data#data.id, + error => Data#data.error, mod => Data#data.mod, callback_mode => Data#data.callback_mode, query_mode => Data#data.query_mode, diff --git a/changes/ce/fix-10145.en.md b/changes/ce/fix-10145.en.md new file mode 100644 index 000000000..bddbd9085 --- /dev/null +++ b/changes/ce/fix-10145.en.md @@ -0,0 +1 @@ +Fix `bridges` API to report error conditions for a failing bridge as `status_reason`. From c1384b6e6ed805985e93b13d5083a974dbc55fa8 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Wed, 15 Mar 2023 14:52:03 +0100 Subject: [PATCH 035/196] feat(emqx_resource): include error with alarm for resource_down --- .../emqx_resource/src/emqx_resource_manager.erl | 17 +++++++++++------ changes/ce/fix-10145.en.md | 4 +++- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index 1bc6a3b6a..b21ffcae3 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -522,7 +522,7 @@ start_resource(Data, From) -> id => Data#data.id, reason => Reason }), - _ = maybe_alarm(disconnected, Data#data.id), + _ = maybe_alarm(disconnected, Data#data.id, Data#data.error), %% Keep track of the error reason why the connection did not work %% so that the Reason can be returned when the verification call is made. UpdatedData = Data#data{status = disconnected, error = Reason}, @@ -597,7 +597,7 @@ with_health_check(Data, Func) -> ResId = Data#data.id, HCRes = emqx_resource:call_health_check(Data#data.manager_id, Data#data.mod, Data#data.state), {Status, NewState, Err} = parse_health_check_result(HCRes, Data), - _ = maybe_alarm(Status, ResId), + _ = maybe_alarm(Status, ResId, Err), ok = maybe_resume_resource_workers(ResId, Status), UpdatedData = Data#data{ state = NewState, status = Status, error = Err @@ -616,15 +616,20 @@ update_state(Data, _DataWas) -> health_check_interval(Opts) -> maps:get(health_check_interval, Opts, ?HEALTHCHECK_INTERVAL). -maybe_alarm(connected, _ResId) -> +maybe_alarm(connected, _ResId, _Error) -> ok; -maybe_alarm(_Status, <>) -> +maybe_alarm(_Status, <>, _Error) -> ok; -maybe_alarm(_Status, ResId) -> +maybe_alarm(_Status, ResId, Error) -> + HrError = + case Error of + undefined -> <<"Unknown reason">>; + _Else -> emqx_misc:readable_error_msg(Error) + end, emqx_alarm:activate( ResId, #{resource_id => ResId, reason => resource_down}, - <<"resource down: ", ResId/binary>> + <<"resource down: ", HrError/binary>> ). maybe_resume_resource_workers(ResId, connected) -> diff --git a/changes/ce/fix-10145.en.md b/changes/ce/fix-10145.en.md index bddbd9085..eaa896793 100644 --- a/changes/ce/fix-10145.en.md +++ b/changes/ce/fix-10145.en.md @@ -1 +1,3 @@ -Fix `bridges` API to report error conditions for a failing bridge as `status_reason`. +Fix `bridges` API to report error conditions for a failing bridge as +`status_reason`. Also when creating an alarm for a failing resource we include +this error condition with the alarm's message. From 8af3fb4ee7536170159001eb8282db69e96d15b4 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Wed, 15 Mar 2023 14:53:18 +0100 Subject: [PATCH 036/196] feat: move human readable error translations to emqx_misc --- apps/emqx/src/emqx_misc.erl | 15 ++++++++++++++- apps/emqx_bridge/src/emqx_bridge_api.erl | 17 ++--------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/apps/emqx/src/emqx_misc.erl b/apps/emqx/src/emqx_misc.erl index 18ecc644a..59b5b1be9 100644 --- a/apps/emqx/src/emqx_misc.erl +++ b/apps/emqx/src/emqx_misc.erl @@ -545,10 +545,23 @@ readable_error_msg(Error) -> {ok, Msg} -> Msg; false -> - iolist_to_binary(io_lib:format("~0p", [Error])) + to_hr_error(Error) end end. +to_hr_error(nxdomain) -> + <<"Host not found">>; +to_hr_error(econnrefused) -> + <<"Connection refused">>; +to_hr_error({unauthorized_client, _}) -> + <<"Unauthorized client">>; +to_hr_error({not_authorized, _}) -> + <<"Not authorized">>; +to_hr_error({malformed_username_or_password, _}) -> + <<"Malformed username or password">>; +to_hr_error(Error) -> + iolist_to_binary(io_lib:format("~0p", [Error])). + try_to_existing_atom(Convert, Data, Encoding) -> try Convert(Data, Encoding) of Atom -> diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index c0e69ce83..e466b418d 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -568,7 +568,7 @@ schema("/bridges_probe") -> ok -> 204; {error, Reason} when not is_tuple(Reason); element(1, Reason) =/= 'exit' -> - {400, error_msg('TEST_FAILED', to_hr_reason(Reason))} + {400, error_msg('TEST_FAILED', emqx_misc:readable_error_msg(Reason))} end; BadRequest -> BadRequest @@ -979,7 +979,7 @@ call_operation(NodeOrAll, OperFunc, Args = [_Nodes, BridgeType, BridgeName]) -> {error, {node_not_found, Node}} -> ?NOT_FOUND(<<"Node not found: ", (atom_to_binary(Node))/binary>>); {error, Reason} when not is_tuple(Reason); element(1, Reason) =/= 'exit' -> - ?BAD_REQUEST(to_hr_reason(Reason)) + ?BAD_REQUEST(emqx_misc:readable_error_msg(Reason)) end. maybe_try_restart(all, start_bridges_to_all_nodes, Args) -> @@ -1018,19 +1018,6 @@ supported_versions(start_bridge_to_node) -> [2, 3]; supported_versions(start_bridges_to_all_nodes) -> [2, 3]; supported_versions(_Call) -> [1, 2, 3]. -to_hr_reason(nxdomain) -> - <<"Host not found">>; -to_hr_reason(econnrefused) -> - <<"Connection refused">>; -to_hr_reason({unauthorized_client, _}) -> - <<"Unauthorized client">>; -to_hr_reason({not_authorized, _}) -> - <<"Not authorized">>; -to_hr_reason({malformed_username_or_password, _}) -> - <<"Malformed username or password">>; -to_hr_reason(Reason) -> - Reason. - redact(Term) -> emqx_misc:redact(Term). From 84fc64822ef532ac51c53eeac0cff08195e883e4 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Tue, 21 Mar 2023 10:29:24 +0100 Subject: [PATCH 037/196] style: fix wording for nxdomain and malformed_username_or_password Co-authored-by: Zaiming (Stone) Shi --- apps/emqx/src/emqx_misc.erl | 4 ++-- apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/emqx/src/emqx_misc.erl b/apps/emqx/src/emqx_misc.erl index 59b5b1be9..cdd62df11 100644 --- a/apps/emqx/src/emqx_misc.erl +++ b/apps/emqx/src/emqx_misc.erl @@ -550,7 +550,7 @@ readable_error_msg(Error) -> end. to_hr_error(nxdomain) -> - <<"Host not found">>; + <<"Could not resolve host">>; to_hr_error(econnrefused) -> <<"Connection refused">>; to_hr_error({unauthorized_client, _}) -> @@ -558,7 +558,7 @@ to_hr_error({unauthorized_client, _}) -> to_hr_error({not_authorized, _}) -> <<"Not authorized">>; to_hr_error({malformed_username_or_password, _}) -> - <<"Malformed username or password">>; + <<"Bad username or password">>; to_hr_error(Error) -> iolist_to_binary(io_lib:format("~0p", [Error])). diff --git a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl index b919dba6b..986c0d29d 100644 --- a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl @@ -819,7 +819,7 @@ t_bridges_probe(Config) -> emqx_json:decode(ConnRefused, [return_maps]) ), - {ok, 400, HostNotFound} = request( + {ok, 400, CouldNotResolveHost} = request( post, uri(["bridges_probe"]), ?MQTT_BRIDGE(<<"nohost:2883">>) @@ -827,9 +827,9 @@ t_bridges_probe(Config) -> ?assertMatch( #{ <<"code">> := <<"TEST_FAILED">>, - <<"message">> := <<"Host not found">> + <<"message">> := <<"Could not resolve host">> }, - emqx_json:decode(HostNotFound, [return_maps]) + emqx_json:decode(CouldNotResolveHost, [return_maps]) ), AuthnConfig = #{ @@ -873,7 +873,7 @@ t_bridges_probe(Config) -> ?assertMatch( #{ <<"code">> := <<"TEST_FAILED">>, - <<"message">> := <<"Malformed username or password">> + <<"message">> := <<"Bad username or password">> }, emqx_json:decode(Malformed, [return_maps]) ), From 4b0ea562a28e0ac8ae874998fd8b12c40855b04c Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Tue, 21 Mar 2023 14:21:30 +0100 Subject: [PATCH 038/196] refactor(emqx_bridge): consistently use macros for http response --- apps/emqx_bridge/src/emqx_bridge_api.erl | 79 +++++++++++++----------- 1 file changed, 42 insertions(+), 37 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index e466b418d..917b44096 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -46,18 +46,33 @@ -export([lookup_from_local_node/2]). --define(BAD_REQUEST(Reason), {400, error_msg('BAD_REQUEST', Reason)}). +%% [TODO] Move those to a commonly shared header file +-define(ERROR_MSG(CODE, REASON), #{code => CODE, message => emqx_misc:readable_error_msg(REASON)}). + +-define(OK(CONTENT), {200, CONTENT}). + +-define(NO_CONTENT, 204). + +-define(BAD_REQUEST(CODE, REASON), {400, ?ERROR_MSG(CODE, REASON)}). +-define(BAD_REQUEST(REASON), ?BAD_REQUEST('BAD_REQUEST', REASON)). + +-define(NOT_FOUND(REASON), {404, ?ERROR_MSG('NOT_FOUND', REASON)}). + +-define(INTERNAL_ERROR(REASON), {500, ?ERROR_MSG('INTERNAL_ERROR', REASON)}). + +-define(NOT_IMPLEMENTED, 501). + +-define(SERVICE_UNAVAILABLE(REASON), {503, ?ERROR_MSG('SERVICE_UNAVAILABLE', REASON)}). +%% End TODO -define(BRIDGE_NOT_ENABLED, ?BAD_REQUEST(<<"Forbidden operation, bridge not enabled">>) ). --define(NOT_FOUND(Reason), {404, error_msg('NOT_FOUND', Reason)}). - --define(BRIDGE_NOT_FOUND(BridgeType, BridgeName), +-define(BRIDGE_NOT_FOUND(BRIDGE_TYPE, BRIDGE_NAME), ?NOT_FOUND( - <<"Bridge lookup failed: bridge named '", BridgeName/binary, "' of type ", - (atom_to_binary(BridgeType))/binary, " does not exist.">> + <<"Bridge lookup failed: bridge named '", BRIDGE_NAME/binary, "' of type ", + (atom_to_binary(BRIDGE_TYPE))/binary, " does not exist.">> ) ). @@ -480,7 +495,7 @@ schema("/bridges_probe") -> '/bridges'(post, #{body := #{<<"type">> := BridgeType, <<"name">> := BridgeName} = Conf0}) -> case emqx_bridge:lookup(BridgeType, BridgeName) of {ok, _} -> - {400, error_msg('ALREADY_EXISTS', <<"bridge already exists">>)}; + ?BAD_REQUEST('ALREADY_EXISTS', <<"bridge already exists">>); {error, not_found} -> Conf = filter_out_request_body(Conf0), {ok, _} = emqx_bridge:create(BridgeType, BridgeName, Conf), @@ -495,9 +510,9 @@ schema("/bridges_probe") -> format_resource(Data, Node) || {Node, Bridges} <- lists:zip(Nodes, NodeBridges), Data <- Bridges ], - {200, zip_bridges([AllBridges])}; + ?OK(zip_bridges([AllBridges])); {error, Reason} -> - {500, error_msg('INTERNAL_ERROR', Reason)} + ?INTERNAL_ERROR(Reason) end. '/bridges/:id'(get, #{bindings := #{id := Id}}) -> @@ -529,16 +544,16 @@ schema("/bridges_probe") -> end, case emqx_bridge:check_deps_and_remove(BridgeType, BridgeName, AlsoDeleteActs) of {ok, _} -> - 204; + ?NO_CONTENT; {error, {rules_deps_on_this_bridge, RuleIds}} -> ?BAD_REQUEST( {<<"Cannot delete bridge while active rules are defined for this bridge">>, RuleIds} ); {error, timeout} -> - {503, error_msg('SERVICE_UNAVAILABLE', <<"request timeout">>)}; + ?SERVICE_UNAVAILABLE(<<"request timeout">>); {error, Reason} -> - {500, error_msg('INTERNAL_ERROR', Reason)} + ?INTERNAL_ERROR(Reason) end; {error, not_found} -> ?BRIDGE_NOT_FOUND(BridgeType, BridgeName) @@ -555,7 +570,7 @@ schema("/bridges_probe") -> ok = emqx_bridge_resource:reset_metrics( emqx_bridge_resource:resource_id(BridgeType, BridgeName) ), - {204} + ?NO_CONTENT end ). @@ -566,9 +581,9 @@ schema("/bridges_probe") -> Params1 = maybe_deobfuscate_bridge_probe(Params), case emqx_bridge_resource:create_dry_run(ConnType, maps:remove(<<"type">>, Params1)) of ok -> - 204; + ?NO_CONTENT; {error, Reason} when not is_tuple(Reason); element(1, Reason) =/= 'exit' -> - {400, error_msg('TEST_FAILED', emqx_misc:readable_error_msg(Reason))} + ?BAD_REQUEST('TEST_FAILED', Reason) end; BadRequest -> BadRequest @@ -602,7 +617,7 @@ do_lookup_from_all_nodes(BridgeType, BridgeName, SuccCode, FormatFun) -> {ok, [{error, not_found} | _]} -> ?BRIDGE_NOT_FOUND(BridgeType, BridgeName); {error, Reason} -> - {500, error_msg('INTERNAL_ERROR', Reason)} + ?INTERNAL_ERROR(Reason) end. lookup_from_local_node(BridgeType, BridgeName) -> @@ -620,15 +635,15 @@ lookup_from_local_node(BridgeType, BridgeName) -> OperFunc -> case emqx_bridge:disable_enable(OperFunc, BridgeType, BridgeName) of {ok, _} -> - 204; + ?NO_CONTENT; {error, {pre_config_update, _, bridge_not_found}} -> ?BRIDGE_NOT_FOUND(BridgeType, BridgeName); {error, {_, _, timeout}} -> - {503, error_msg('SERVICE_UNAVAILABLE', <<"request timeout">>)}; + ?SERVICE_UNAVAILABLE(<<"request timeout">>); {error, timeout} -> - {503, error_msg('SERVICE_UNAVAILABLE', <<"request timeout">>)}; + ?SERVICE_UNAVAILABLE(<<"request timeout">>); {error, Reason} -> - {500, error_msg('INTERNAL_ERROR', Reason)} + ?INTERNAL_ERROR(Reason) end end ). @@ -943,9 +958,6 @@ filter_out_request_body(Conf) -> ], maps:without(ExtraConfs, Conf). -error_msg(Code, Msg) -> - #{code => Code, message => emqx_misc:readable_error_msg(Msg)}. - bin(S) when is_list(S) -> list_to_binary(S); bin(S) when is_atom(S) -> @@ -956,30 +968,23 @@ bin(S) when is_binary(S) -> call_operation(NodeOrAll, OperFunc, Args = [_Nodes, BridgeType, BridgeName]) -> case is_ok(do_bpapi_call(NodeOrAll, OperFunc, Args)) of Ok when Ok =:= ok; is_tuple(Ok), element(1, Ok) =:= ok -> - 204; + ?NO_CONTENT; {error, not_implemented} -> %% Should only happen if we call `start` on a node that is %% still on an older bpapi version that doesn't support it. maybe_try_restart(NodeOrAll, OperFunc, Args); {error, timeout} -> - {503, error_msg('SERVICE_UNAVAILABLE', <<"request timeout">>)}; + ?SERVICE_UNAVAILABLE(<<"Request timeout">>); {error, {start_pool_failed, Name, Reason}} -> - {503, - error_msg( - 'SERVICE_UNAVAILABLE', - bin( - io_lib:format( - "failed to start ~p pool for reason ~p", - [Name, Reason] - ) - ) - )}; + ?SERVICE_UNAVAILABLE( + bin(io_lib:format("Failed to start ~p pool for reason ~p", [Name, Reason])) + ); {error, not_found} -> ?BRIDGE_NOT_FOUND(BridgeType, BridgeName); {error, {node_not_found, Node}} -> ?NOT_FOUND(<<"Node not found: ", (atom_to_binary(Node))/binary>>); {error, Reason} when not is_tuple(Reason); element(1, Reason) =/= 'exit' -> - ?BAD_REQUEST(emqx_misc:readable_error_msg(Reason)) + ?BAD_REQUEST(Reason) end. maybe_try_restart(all, start_bridges_to_all_nodes, Args) -> @@ -987,7 +992,7 @@ maybe_try_restart(all, start_bridges_to_all_nodes, Args) -> maybe_try_restart(Node, start_bridge_to_node, Args) -> call_operation(Node, restart_bridge_to_node, Args); maybe_try_restart(_, _, _) -> - 501. + ?NOT_IMPLEMENTED. do_bpapi_call(all, Call, Args) -> maybe_unwrap( From 3880862c810b386b35ebcd83fe0c596cb10b3809 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Tue, 21 Mar 2023 14:22:04 +0100 Subject: [PATCH 039/196] fix(emqx_bridge): return 503 for inconsistency in bridge setup --- apps/emqx_bridge/src/emqx_bridge_api.erl | 10 ++++++- .../test/emqx_bridge_api_SUITE.erl | 30 +++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index 917b44096..f7a1bb345 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -980,7 +980,15 @@ call_operation(NodeOrAll, OperFunc, Args = [_Nodes, BridgeType, BridgeName]) -> bin(io_lib:format("Failed to start ~p pool for reason ~p", [Name, Reason])) ); {error, not_found} -> - ?BRIDGE_NOT_FOUND(BridgeType, BridgeName); + BridgeId = emqx_bridge_resource:bridge_id(BridgeType, BridgeName), + ?SLOG(warning, #{ + msg => "bridge_inconsistent_in_cluster_for_call_operation", + reason => not_found, + type => BridgeType, + name => BridgeName, + bridge => BridgeId + }), + ?SERVICE_UNAVAILABLE(<<"Bridge not found on remote node: ", BridgeId/binary>>); {error, {node_not_found, Node}} -> ?NOT_FOUND(<<"Node not found: ", (atom_to_binary(Node))/binary>>); {error, Reason} when not is_tuple(Reason); element(1, Reason) =/= 'exit' -> diff --git a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl index 986c0d29d..45ab2b623 100644 --- a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl @@ -98,6 +98,20 @@ init_per_testcase(t_old_bpapi_vsn, Config) -> meck:expect(emqx_bpapi, supported_version, 1, 1), meck:expect(emqx_bpapi, supported_version, 2, 1), init_per_testcase(common, Config); +init_per_testcase(StartStop, Config) when + StartStop == t_start_stop_bridges_cluster; + StartStop == t_start_stop_bridges_node +-> + meck:new(emqx_bridge_resource, [passthrough]), + meck:expect( + emqx_bridge_resource, + stop, + fun + (_, <<"bridge_not_found">>) -> {error, not_found}; + (Type, Name) -> meck:passthrough([Type, Name]) + end + ), + init_per_testcase(common, Config); init_per_testcase(_, Config) -> {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), {Port, Sock, Acceptor} = start_http_server(fun handle_fun_200_ok/2), @@ -109,6 +123,12 @@ end_per_testcase(t_broken_bpapi_vsn, Config) -> end_per_testcase(t_old_bpapi_vsn, Config) -> meck:unload([emqx_bpapi]), end_per_testcase(common, Config); +end_per_testcase(StartStop, Config) when + StartStop == t_start_stop_bridges_cluster; + StartStop == t_start_stop_bridges_node +-> + meck:unload([emqx_bridge_resource]), + end_per_testcase(common, Config); end_per_testcase(_, Config) -> Sock = ?config(sock, Config), Acceptor = ?config(acceptor, Config), @@ -626,6 +646,16 @@ do_start_stop_bridges(Type, Config) -> %% Looks ok but doesn't exist {ok, 404, _} = request(post, operation_path(Type, start, <<"webhook:cptn_hook">>), <<"">>), + %% + {ok, 201, _Bridge} = request( + post, + uri(["bridges"]), + ?HTTP_BRIDGE(URL1, <<"bridge_not_found">>) + ), + {ok, 503, _} = request( + post, operation_path(Type, stop, <<"webhook:bridge_not_found">>), <<"">> + ), + %% Create broken bridge {ListenPort, Sock} = listen_on_random_port(), %% Connecting to this endpoint should always timeout From d2c5cbbcaaa3063358ca46bfd8d6eaf4f01e2ab3 Mon Sep 17 00:00:00 2001 From: Ingmar Delsink Date: Tue, 21 Mar 2023 15:21:08 +0100 Subject: [PATCH 040/196] fix(helm): helm chart extraVolumeMounts and extraVolumes indentation --- .../emqx-enterprise/templates/StatefulSet.yaml | 12 ++++++------ deploy/charts/emqx/templates/StatefulSet.yaml | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/deploy/charts/emqx-enterprise/templates/StatefulSet.yaml b/deploy/charts/emqx-enterprise/templates/StatefulSet.yaml index 00751aceb..3e9e39f2c 100644 --- a/deploy/charts/emqx-enterprise/templates/StatefulSet.yaml +++ b/deploy/charts/emqx-enterprise/templates/StatefulSet.yaml @@ -74,9 +74,9 @@ spec: secret: secretName: {{ .Values.emqxLicenseSecretName }} {{- end }} - {{- if .Values.extraVolumes }} - {{- toYaml .Values.extraVolumes | nindent 8 }} - {{- end }} + {{- if .Values.extraVolumes }} + {{- toYaml .Values.extraVolumes | nindent 6 }} + {{- end }} {{- if .Values.podSecurityContext.enabled }} securityContext: {{- omit .Values.podSecurityContext "enabled" | toYaml | nindent 8 }} {{- end }} @@ -141,9 +141,9 @@ spec: subPath: "emqx.lic" readOnly: true {{- end }} - {{- if .Values.extraVolumeMounts }} - {{- toYaml .Values.extraVolumeMounts | nindent 12 }} - {{- end }} + {{- if .Values.extraVolumeMounts }} + {{- toYaml .Values.extraVolumeMounts | nindent 10 }} + {{- end }} readinessProbe: httpGet: path: /status diff --git a/deploy/charts/emqx/templates/StatefulSet.yaml b/deploy/charts/emqx/templates/StatefulSet.yaml index 00751aceb..3e9e39f2c 100644 --- a/deploy/charts/emqx/templates/StatefulSet.yaml +++ b/deploy/charts/emqx/templates/StatefulSet.yaml @@ -74,9 +74,9 @@ spec: secret: secretName: {{ .Values.emqxLicenseSecretName }} {{- end }} - {{- if .Values.extraVolumes }} - {{- toYaml .Values.extraVolumes | nindent 8 }} - {{- end }} + {{- if .Values.extraVolumes }} + {{- toYaml .Values.extraVolumes | nindent 6 }} + {{- end }} {{- if .Values.podSecurityContext.enabled }} securityContext: {{- omit .Values.podSecurityContext "enabled" | toYaml | nindent 8 }} {{- end }} @@ -141,9 +141,9 @@ spec: subPath: "emqx.lic" readOnly: true {{- end }} - {{- if .Values.extraVolumeMounts }} - {{- toYaml .Values.extraVolumeMounts | nindent 12 }} - {{- end }} + {{- if .Values.extraVolumeMounts }} + {{- toYaml .Values.extraVolumeMounts | nindent 10 }} + {{- end }} readinessProbe: httpGet: path: /status From 981268911a3cca621cc83506588a4377cce5d467 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Tue, 21 Mar 2023 16:56:37 +0100 Subject: [PATCH 041/196] style: make summary lower-case --- apps/emqx_bridge/src/emqx_bridge_api.erl | 20 +++++----- .../emqx_dashboard/src/emqx_dashboard_api.erl | 2 +- apps/emqx_gateway/src/emqx_gateway_api.erl | 24 +++++------ .../src/emqx_gateway_api_authn.erl | 20 +++++----- .../emqx_gateway_api_authn_user_import.erl | 4 +- .../src/emqx_gateway_api_clients.erl | 28 ++++++------- .../src/emqx_gateway_api_listeners.erl | 40 +++++++++---------- .../src/emqx_rule_engine_api.erl | 18 ++++----- 8 files changed, 78 insertions(+), 78 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index 8ac3e476a..532daf448 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -301,7 +301,7 @@ schema("/bridges") -> 'operationId' => '/bridges', get => #{ tags => [<<"bridges">>], - summary => <<"List Bridges">>, + summary => <<"List bridges">>, description => ?DESC("desc_api1"), responses => #{ 200 => emqx_dashboard_swagger:schema_with_example( @@ -312,7 +312,7 @@ schema("/bridges") -> }, post => #{ tags => [<<"bridges">>], - summary => <<"Create Bridge">>, + summary => <<"Create bridge">>, description => ?DESC("desc_api2"), 'requestBody' => emqx_dashboard_swagger:schema_with_examples( emqx_bridge_schema:post_request(), @@ -329,7 +329,7 @@ schema("/bridges/:id") -> 'operationId' => '/bridges/:id', get => #{ tags => [<<"bridges">>], - summary => <<"Get Bridge">>, + summary => <<"Get bridge">>, description => ?DESC("desc_api3"), parameters => [param_path_id()], responses => #{ @@ -339,7 +339,7 @@ schema("/bridges/:id") -> }, put => #{ tags => [<<"bridges">>], - summary => <<"Update Bridge">>, + summary => <<"Update bridge">>, description => ?DESC("desc_api4"), parameters => [param_path_id()], 'requestBody' => emqx_dashboard_swagger:schema_with_examples( @@ -354,7 +354,7 @@ schema("/bridges/:id") -> }, delete => #{ tags => [<<"bridges">>], - summary => <<"Delete Bridge">>, + summary => <<"Delete bridge">>, description => ?DESC("desc_api5"), parameters => [param_path_id()], responses => #{ @@ -373,7 +373,7 @@ schema("/bridges/:id/metrics") -> 'operationId' => '/bridges/:id/metrics', get => #{ tags => [<<"bridges">>], - summary => <<"Get Bridge Metrics">>, + summary => <<"Get bridge metrics">>, description => ?DESC("desc_bridge_metrics"), parameters => [param_path_id()], responses => #{ @@ -387,7 +387,7 @@ schema("/bridges/:id/metrics/reset") -> 'operationId' => '/bridges/:id/metrics/reset', put => #{ tags => [<<"bridges">>], - summary => <<"Reset Bridge Metrics">>, + summary => <<"Reset bridge metrics">>, description => ?DESC("desc_api6"), parameters => [param_path_id()], responses => #{ @@ -402,7 +402,7 @@ schema("/bridges/:id/enable/:enable") -> put => #{ tags => [<<"bridges">>], - summary => <<"Enable or Disable Bridge">>, + summary => <<"Enable or disable bridge">>, desc => ?DESC("desc_enable_bridge"), parameters => [param_path_id(), param_path_enable()], responses => @@ -418,7 +418,7 @@ schema("/bridges/:id/:operation") -> 'operationId' => '/bridges/:id/:operation', post => #{ tags => [<<"bridges">>], - summary => <<"Stop or Restart Bridge">>, + summary => <<"Stop or restart bridge">>, description => ?DESC("desc_api7"), parameters => [ param_path_id(), @@ -440,7 +440,7 @@ schema("/nodes/:node/bridges/:id/:operation") -> 'operationId' => '/nodes/:node/bridges/:id/:operation', post => #{ tags => [<<"bridges">>], - summary => <<"Stop/Restart Bridge">>, + summary => <<"Stop/Restart bridge">>, description => ?DESC("desc_api8"), parameters => [ param_path_node(), diff --git a/apps/emqx_dashboard/src/emqx_dashboard_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_api.erl index cc2a1337d..d5655d99d 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_api.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_api.erl @@ -74,7 +74,7 @@ schema("/login") -> post => #{ tags => [<<"dashboard">>], desc => ?DESC(login_api), - summary => <<"Dashboard Auth">>, + summary => <<"Dashboard authentication">>, 'requestBody' => fields([username, password]), responses => #{ 200 => fields([token, version, license]), diff --git a/apps/emqx_gateway/src/emqx_gateway_api.erl b/apps/emqx_gateway/src/emqx_gateway_api.erl index 1c43340e2..62f723d59 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api.erl @@ -180,7 +180,7 @@ schema("/gateways") -> #{ tags => ?TAGS, desc => ?DESC(list_gateway), - summary => <<"List All Gateways">>, + summary => <<"List all gateways">>, parameters => params_gateway_status_in_qs(), responses => #{ @@ -201,7 +201,7 @@ schema("/gateways/:name") -> #{ tags => ?TAGS, desc => ?DESC(get_gateway), - summary => <<"Get the Gateway">>, + summary => <<"Get gateway">>, parameters => params_gateway_name_in_path(), responses => #{ @@ -608,7 +608,7 @@ examples_gateway_confs() -> #{ stomp_gateway => #{ - summary => <<"A simple STOMP gateway configs">>, + summary => <<"A simple STOMP gateway config">>, value => #{ enable => true, @@ -636,7 +636,7 @@ examples_gateway_confs() -> }, mqttsn_gateway => #{ - summary => <<"A simple MQTT-SN gateway configs">>, + summary => <<"A simple MQTT-SN gateway config">>, value => #{ enable => true, @@ -672,7 +672,7 @@ examples_gateway_confs() -> }, coap_gateway => #{ - summary => <<"A simple CoAP gateway configs">>, + summary => <<"A simple CoAP gateway config">>, value => #{ enable => true, @@ -699,7 +699,7 @@ examples_gateway_confs() -> }, lwm2m_gateway => #{ - summary => <<"A simple LwM2M gateway configs">>, + summary => <<"A simple LwM2M gateway config">>, value => #{ enable => true, @@ -735,7 +735,7 @@ examples_gateway_confs() -> }, exproto_gateway => #{ - summary => <<"A simple ExProto gateway configs">>, + summary => <<"A simple ExProto gateway config">>, value => #{ enable => true, @@ -765,7 +765,7 @@ examples_update_gateway_confs() -> #{ stomp_gateway => #{ - summary => <<"A simple STOMP gateway configs">>, + summary => <<"A simple STOMP gateway config">>, value => #{ enable => true, @@ -782,7 +782,7 @@ examples_update_gateway_confs() -> }, mqttsn_gateway => #{ - summary => <<"A simple MQTT-SN gateway configs">>, + summary => <<"A simple MQTT-SN gateway config">>, value => #{ enable => true, @@ -803,7 +803,7 @@ examples_update_gateway_confs() -> }, coap_gateway => #{ - summary => <<"A simple CoAP gateway configs">>, + summary => <<"A simple CoAP gateway config">>, value => #{ enable => true, @@ -819,7 +819,7 @@ examples_update_gateway_confs() -> }, lwm2m_gateway => #{ - summary => <<"A simple LwM2M gateway configs">>, + summary => <<"A simple LwM2M gateway config">>, value => #{ enable => true, @@ -844,7 +844,7 @@ examples_update_gateway_confs() -> }, exproto_gateway => #{ - summary => <<"A simple ExProto gateway configs">>, + summary => <<"A simple ExProto gateway config">>, value => #{ enable => true, diff --git a/apps/emqx_gateway/src/emqx_gateway_api_authn.erl b/apps/emqx_gateway/src/emqx_gateway_api_authn.erl index f52b26cd2..41b1b11d5 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_authn.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_authn.erl @@ -185,13 +185,13 @@ schema("/gateways/:name/authentication") -> #{ tags => ?TAGS, desc => ?DESC(get_authn), - summary => <<"Get Authenticator Configuration">>, + summary => <<"Get authenticator configuration">>, parameters => params_gateway_name_in_path(), responses => ?STANDARD_RESP( #{ 200 => schema_authn(), - 204 => <<"Authenticator doesn't initiated">> + 204 => <<"Authenticator not initialized">> } ) }, @@ -199,7 +199,7 @@ schema("/gateways/:name/authentication") -> #{ tags => ?TAGS, desc => ?DESC(update_authn), - summary => <<"Update Authenticator Configuration">>, + summary => <<"Update authenticator configuration">>, parameters => params_gateway_name_in_path(), 'requestBody' => schema_authn(), responses => @@ -209,7 +209,7 @@ schema("/gateways/:name/authentication") -> #{ tags => ?TAGS, desc => ?DESC(add_authn), - summary => <<"Create an Authenticator for a Gateway">>, + summary => <<"Create authenticator for gateway">>, parameters => params_gateway_name_in_path(), 'requestBody' => schema_authn(), responses => @@ -219,7 +219,7 @@ schema("/gateways/:name/authentication") -> #{ tags => ?TAGS, desc => ?DESC(delete_authn), - summary => <<"Delete the Gateway Authenticator">>, + summary => <<"Delete gateway authenticator">>, parameters => params_gateway_name_in_path(), responses => ?STANDARD_RESP(#{204 => <<"Deleted">>}) @@ -232,7 +232,7 @@ schema("/gateways/:name/authentication/users") -> #{ tags => ?TAGS, desc => ?DESC(list_users), - summary => <<"List users for a Gateway Authenticator">>, + summary => <<"List users for gateway authenticator">>, parameters => params_gateway_name_in_path() ++ params_paging_in_qs() ++ params_fuzzy_in_qs(), @@ -250,7 +250,7 @@ schema("/gateways/:name/authentication/users") -> #{ tags => ?TAGS, desc => ?DESC(add_user), - summary => <<"Add User for a Gateway Authenticator">>, + summary => <<"Add user for gateway authenticator">>, parameters => params_gateway_name_in_path(), 'requestBody' => emqx_dashboard_swagger:schema_with_examples( ref(emqx_authn_api, request_user_create), @@ -274,7 +274,7 @@ schema("/gateways/:name/authentication/users/:uid") -> #{ tags => ?TAGS, desc => ?DESC(get_user), - summary => <<"Get User Info for a Gateway Authenticator">>, + summary => <<"Get user info for gateway authenticator">>, parameters => params_gateway_name_in_path() ++ params_userid_in_path(), responses => @@ -291,7 +291,7 @@ schema("/gateways/:name/authentication/users/:uid") -> #{ tags => ?TAGS, desc => ?DESC(update_user), - summary => <<"Update User Info for a Gateway Authenticator">>, + summary => <<"Update user info for gateway authenticator">>, parameters => params_gateway_name_in_path() ++ params_userid_in_path(), 'requestBody' => emqx_dashboard_swagger:schema_with_examples( @@ -312,7 +312,7 @@ schema("/gateways/:name/authentication/users/:uid") -> #{ tags => ?TAGS, desc => ?DESC(delete_user), - summary => <<"Delete User for a Gateway Authenticator">>, + summary => <<"Delete user for gateway authenticator">>, parameters => params_gateway_name_in_path() ++ params_userid_in_path(), responses => diff --git a/apps/emqx_gateway/src/emqx_gateway_api_authn_user_import.erl b/apps/emqx_gateway/src/emqx_gateway_api_authn_user_import.erl index 705fccf90..68f392923 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_authn_user_import.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_authn_user_import.erl @@ -126,7 +126,7 @@ schema("/gateways/:name/authentication/import_users") -> #{ tags => ?TAGS, desc => ?DESC(emqx_gateway_api_authn, import_users), - summary => <<"Import Users">>, + summary => <<"Import users">>, parameters => params_gateway_name_in_path(), 'requestBody' => emqx_dashboard_swagger:file_schema(filename), responses => @@ -140,7 +140,7 @@ schema("/gateways/:name/listeners/:id/authentication/import_users") -> #{ tags => ?TAGS, desc => ?DESC(emqx_gateway_api_listeners, import_users), - summary => <<"Import Users">>, + summary => <<"Import users">>, parameters => params_gateway_name_in_path() ++ params_listener_id_in_path(), 'requestBody' => emqx_dashboard_swagger:file_schema(filename), diff --git a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl index b30de3a3e..e64e918b4 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl @@ -460,7 +460,7 @@ schema("/gateways/:name/clients") -> #{ tags => ?TAGS, desc => ?DESC(list_clients), - summary => <<"List Gateway's Clients">>, + summary => <<"List gateway's clients">>, parameters => params_client_query(), responses => ?STANDARD_RESP(#{ @@ -478,7 +478,7 @@ schema("/gateways/:name/clients/:clientid") -> #{ tags => ?TAGS, desc => ?DESC(get_client), - summary => <<"Get Client Info">>, + summary => <<"Get client info">>, parameters => params_client_insta(), responses => ?STANDARD_RESP(#{200 => schema_client()}) @@ -487,7 +487,7 @@ schema("/gateways/:name/clients/:clientid") -> #{ tags => ?TAGS, desc => ?DESC(kick_client), - summary => <<"Kick out Client">>, + summary => <<"Kick out client">>, parameters => params_client_insta(), responses => ?STANDARD_RESP(#{204 => <<"Kicked">>}) @@ -500,7 +500,7 @@ schema("/gateways/:name/clients/:clientid/subscriptions") -> #{ tags => ?TAGS, desc => ?DESC(list_subscriptions), - summary => <<"List Client's Subscription">>, + summary => <<"List client's subscription">>, parameters => params_client_insta(), responses => ?STANDARD_RESP( @@ -516,7 +516,7 @@ schema("/gateways/:name/clients/:clientid/subscriptions") -> #{ tags => ?TAGS, desc => ?DESC(add_subscription), - summary => <<"Add Subscription for Client">>, + summary => <<"Add subscription for client">>, parameters => params_client_insta(), 'requestBody' => emqx_dashboard_swagger:schema_with_examples( ref(subscription), @@ -540,7 +540,7 @@ schema("/gateways/:name/clients/:clientid/subscriptions/:topic") -> #{ tags => ?TAGS, desc => ?DESC(delete_subscription), - summary => <<"Delete Client's Subscription">>, + summary => <<"Delete client's subscription">>, parameters => params_topic_name_in_path() ++ params_client_insta(), responses => ?STANDARD_RESP(#{204 => <<"Unsubscribed">>}) @@ -1020,12 +1020,12 @@ examples_client_list() -> #{ general_client_list => #{ - summary => <<"General Client List">>, + summary => <<"General client list">>, value => [example_general_client()] }, lwm2m_client_list => #{ - summary => <<"LwM2M Client List">>, + summary => <<"LwM2M client list">>, value => [example_lwm2m_client()] } }. @@ -1034,12 +1034,12 @@ examples_client() -> #{ general_client => #{ - summary => <<"General Client Info">>, + summary => <<"General client info">>, value => example_general_client() }, lwm2m_client => #{ - summary => <<"LwM2M Client Info">>, + summary => <<"LwM2M client info">>, value => example_lwm2m_client() } }. @@ -1048,12 +1048,12 @@ examples_subscription_list() -> #{ general_subscription_list => #{ - summary => <<"A General Subscription List">>, + summary => <<"A general subscription list">>, value => [example_general_subscription()] }, stomp_subscription_list => #{ - summary => <<"The Stomp Subscription List">>, + summary => <<"The STOMP subscription list">>, value => [example_stomp_subscription] } }. @@ -1062,12 +1062,12 @@ examples_subscription() -> #{ general_subscription => #{ - summary => <<"A General Subscription">>, + summary => <<"A general subscription">>, value => example_general_subscription() }, stomp_subscription => #{ - summary => <<"A Stomp Subscription">>, + summary => <<"A STOMP subscription">>, value => example_stomp_subscription() } }. diff --git a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl index 43c8156d6..14b80a500 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl @@ -362,7 +362,7 @@ schema("/gateways/:name/listeners") -> #{ tags => ?TAGS, desc => ?DESC(list_listeners), - summary => <<"List All Listeners">>, + summary => <<"List all listeners">>, parameters => params_gateway_name_in_path(), responses => ?STANDARD_RESP( @@ -378,7 +378,7 @@ schema("/gateways/:name/listeners") -> #{ tags => ?TAGS, desc => ?DESC(add_listener), - summary => <<"Add a Listener">>, + summary => <<"Add listener">>, parameters => params_gateway_name_in_path(), %% XXX: How to distinguish the different listener supported by %% different types of gateways? @@ -404,7 +404,7 @@ schema("/gateways/:name/listeners/:id") -> #{ tags => ?TAGS, desc => ?DESC(get_listener), - summary => <<"Get the Listener Configs">>, + summary => <<"Get listener config">>, parameters => params_gateway_name_in_path() ++ params_listener_id_in_path(), responses => @@ -421,7 +421,7 @@ schema("/gateways/:name/listeners/:id") -> #{ tags => ?TAGS, desc => ?DESC(delete_listener), - summary => <<"Delete the Listener">>, + summary => <<"Delete listener">>, parameters => params_gateway_name_in_path() ++ params_listener_id_in_path(), responses => @@ -431,7 +431,7 @@ schema("/gateways/:name/listeners/:id") -> #{ tags => ?TAGS, desc => ?DESC(update_listener), - summary => <<"Update the Listener Configs">>, + summary => <<"Update listener config">>, parameters => params_gateway_name_in_path() ++ params_listener_id_in_path(), 'requestBody' => emqx_dashboard_swagger:schema_with_examples( @@ -456,7 +456,7 @@ schema("/gateways/:name/listeners/:id/authentication") -> #{ tags => ?TAGS, desc => ?DESC(get_listener_authn), - summary => <<"Get the Listener's Authenticator">>, + summary => <<"Get the listener's authenticator">>, parameters => params_gateway_name_in_path() ++ params_listener_id_in_path(), responses => @@ -471,7 +471,7 @@ schema("/gateways/:name/listeners/:id/authentication") -> #{ tags => ?TAGS, desc => ?DESC(add_listener_authn), - summary => <<"Create an Authenticator for a Listener">>, + summary => <<"Create authenticator for listener">>, parameters => params_gateway_name_in_path() ++ params_listener_id_in_path(), 'requestBody' => schema_authn(), @@ -482,7 +482,7 @@ schema("/gateways/:name/listeners/:id/authentication") -> #{ tags => ?TAGS, desc => ?DESC(update_listener_authn), - summary => <<"Update the Listener Authenticator configs">>, + summary => <<"Update config of authenticator for listener">>, parameters => params_gateway_name_in_path() ++ params_listener_id_in_path(), 'requestBody' => schema_authn(), @@ -493,7 +493,7 @@ schema("/gateways/:name/listeners/:id/authentication") -> #{ tags => ?TAGS, desc => ?DESC(delete_listener_authn), - summary => <<"Delete the Listener's Authenticator">>, + summary => <<"Delete the listener's authenticator">>, parameters => params_gateway_name_in_path() ++ params_listener_id_in_path(), responses => @@ -507,7 +507,7 @@ schema("/gateways/:name/listeners/:id/authentication/users") -> #{ tags => ?TAGS, desc => ?DESC(list_users), - summary => <<"List Authenticator's Users">>, + summary => <<"List authenticator's users">>, parameters => params_gateway_name_in_path() ++ params_listener_id_in_path() ++ params_paging_in_qs(), @@ -525,7 +525,7 @@ schema("/gateways/:name/listeners/:id/authentication/users") -> #{ tags => ?TAGS, desc => ?DESC(add_user), - summary => <<"Add User for an Authenticator">>, + summary => <<"Add user for an authenticator">>, parameters => params_gateway_name_in_path() ++ params_listener_id_in_path(), 'requestBody' => emqx_dashboard_swagger:schema_with_examples( @@ -550,7 +550,7 @@ schema("/gateways/:name/listeners/:id/authentication/users/:uid") -> #{ tags => ?TAGS, desc => ?DESC(get_user), - summary => <<"Get User Info">>, + summary => <<"Get user info">>, parameters => params_gateway_name_in_path() ++ params_listener_id_in_path() ++ params_userid_in_path(), @@ -568,7 +568,7 @@ schema("/gateways/:name/listeners/:id/authentication/users/:uid") -> #{ tags => ?TAGS, desc => ?DESC(update_user), - summary => <<"Update User Info">>, + summary => <<"Update user info">>, parameters => params_gateway_name_in_path() ++ params_listener_id_in_path() ++ params_userid_in_path(), @@ -590,7 +590,7 @@ schema("/gateways/:name/listeners/:id/authentication/users/:uid") -> #{ tags => ?TAGS, desc => ?DESC(delete_user), - summary => <<"Delete User">>, + summary => <<"Delete user">>, parameters => params_gateway_name_in_path() ++ params_listener_id_in_path() ++ params_userid_in_path(), @@ -712,7 +712,7 @@ examples_listener() -> #{ tcp_listener => #{ - summary => <<"A simple tcp listener example">>, + summary => <<"A simple TCP listener example">>, value => #{ name => <<"tcp-def">>, @@ -738,7 +738,7 @@ examples_listener() -> }, ssl_listener => #{ - summary => <<"A simple ssl listener example">>, + summary => <<"A simple SSL listener example">>, value => #{ name => <<"ssl-def">>, @@ -771,7 +771,7 @@ examples_listener() -> }, udp_listener => #{ - summary => <<"A simple udp listener example">>, + summary => <<"A simple UDP listener example">>, value => #{ name => <<"udp-def">>, @@ -789,7 +789,7 @@ examples_listener() -> }, dtls_listener => #{ - summary => <<"A simple dtls listener example">>, + summary => <<"A simple DTLS listener example">>, value => #{ name => <<"dtls-def">>, @@ -817,7 +817,7 @@ examples_listener() -> }, dtls_listener_with_psk_ciphers => #{ - summary => <<"A dtls listener with PSK example">>, + summary => <<"A DTLS listener with PSK example">>, value => #{ name => <<"dtls-psk">>, @@ -845,7 +845,7 @@ examples_listener() -> }, lisetner_with_authn => #{ - summary => <<"A tcp listener with authentication example">>, + summary => <<"A TCP listener with authentication example">>, value => #{ name => <<"tcp-with-authn">>, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index 30de3e8e8..106693a0a 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -180,7 +180,7 @@ schema("/rules") -> ref(emqx_dashboard_swagger, page), ref(emqx_dashboard_swagger, limit) ], - summary => <<"List Rules">>, + summary => <<"List rules">>, responses => #{ 200 => [ @@ -193,7 +193,7 @@ schema("/rules") -> post => #{ tags => [<<"rules">>], description => ?DESC("api2"), - summary => <<"Create a Rule">>, + summary => <<"Create a rule">>, 'requestBody' => rule_creation_schema(), responses => #{ 400 => error_schema('BAD_REQUEST', "Invalid Parameters"), @@ -207,7 +207,7 @@ schema("/rule_events") -> get => #{ tags => [<<"rules">>], description => ?DESC("api3"), - summary => <<"List Events">>, + summary => <<"List rule events">>, responses => #{ 200 => mk(ref(emqx_rule_api_schema, "rule_events"), #{}) } @@ -219,7 +219,7 @@ schema("/rules/:id") -> get => #{ tags => [<<"rules">>], description => ?DESC("api4"), - summary => <<"Get a Rule">>, + summary => <<"Get rule">>, parameters => param_path_id(), responses => #{ 404 => error_schema('NOT_FOUND', "Rule not found"), @@ -229,7 +229,7 @@ schema("/rules/:id") -> put => #{ tags => [<<"rules">>], description => ?DESC("api5"), - summary => <<"Update a Rule">>, + summary => <<"Update rule">>, parameters => param_path_id(), 'requestBody' => rule_creation_schema(), responses => #{ @@ -240,7 +240,7 @@ schema("/rules/:id") -> delete => #{ tags => [<<"rules">>], description => ?DESC("api6"), - summary => <<"Delete a Rule">>, + summary => <<"Delete rule">>, parameters => param_path_id(), responses => #{ 204 => <<"Delete rule successfully">> @@ -253,7 +253,7 @@ schema("/rules/:id/metrics") -> get => #{ tags => [<<"rules">>], description => ?DESC("api4_1"), - summary => <<"Get a Rule's Metrics">>, + summary => <<"Get rule metrics">>, parameters => param_path_id(), responses => #{ 404 => error_schema('NOT_FOUND', "Rule not found"), @@ -267,7 +267,7 @@ schema("/rules/:id/metrics/reset") -> put => #{ tags => [<<"rules">>], description => ?DESC("api7"), - summary => <<"Reset a Rule Metrics">>, + summary => <<"Reset rule metrics">>, parameters => param_path_id(), responses => #{ 404 => error_schema('NOT_FOUND', "Rule not found"), @@ -281,7 +281,7 @@ schema("/rule_test") -> post => #{ tags => [<<"rules">>], description => ?DESC("api8"), - summary => <<"Test a Rule">>, + summary => <<"Test a rule">>, 'requestBody' => rule_test_schema(), responses => #{ 400 => error_schema('BAD_REQUEST', "Invalid Parameters"), From d9b1b1c802b8247dbabb1523c2f757d40f415067 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 21 Mar 2023 15:23:50 +0300 Subject: [PATCH 042/196] fix(bridge-api): unflatten nodes' bridge lists back Bridge lists were erroneously flattened in cad6492c. This causes bridge listing fail in emqx clusters consisting of more than 1 node. --- apps/emqx_bridge/src/emqx_bridge_api.erl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index f7a1bb345..f780aa16d 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -71,8 +71,8 @@ -define(BRIDGE_NOT_FOUND(BRIDGE_TYPE, BRIDGE_NAME), ?NOT_FOUND( - <<"Bridge lookup failed: bridge named '", BRIDGE_NAME/binary, "' of type ", - (atom_to_binary(BRIDGE_TYPE))/binary, " does not exist.">> + <<"Bridge lookup failed: bridge named '", (BRIDGE_NAME)/binary, "' of type ", + (bin(BRIDGE_TYPE))/binary, " does not exist.">> ) ). @@ -507,10 +507,10 @@ schema("/bridges_probe") -> case is_ok(NodeReplies) of {ok, NodeBridges} -> AllBridges = [ - format_resource(Data, Node) - || {Node, Bridges} <- lists:zip(Nodes, NodeBridges), Data <- Bridges + [format_resource(Data, Node) || Data <- Bridges] + || {Node, Bridges} <- lists:zip(Nodes, NodeBridges) ], - ?OK(zip_bridges([AllBridges])); + ?OK(zip_bridges(AllBridges)); {error, Reason} -> ?INTERNAL_ERROR(Reason) end. From e2ff8c1c8f071d549d5041460c956091ea7d2fad Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 21 Mar 2023 17:22:23 +0300 Subject: [PATCH 043/196] chore: add changelog entry --- changes/ce/fix-10190.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ce/fix-10190.en.md diff --git a/changes/ce/fix-10190.en.md b/changes/ce/fix-10190.en.md new file mode 100644 index 000000000..bffd9ca00 --- /dev/null +++ b/changes/ce/fix-10190.en.md @@ -0,0 +1 @@ +Fix the issue where nodes responses to the list bridges RPC were incorrectly flattened, which caused List Bridges API HTTP handler to crash when there was more than 1 node in the cluster. From 327b78221027b430db6f9c6658e2b2e2987ab7e9 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Tue, 21 Mar 2023 17:04:42 +0100 Subject: [PATCH 044/196] style: lower case in descriptions --- .../emqx_modules/i18n/emqx_topic_metrics_api_i18n.conf | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/emqx_modules/i18n/emqx_topic_metrics_api_i18n.conf b/apps/emqx_modules/i18n/emqx_topic_metrics_api_i18n.conf index 623884f31..22f038d4e 100644 --- a/apps/emqx_modules/i18n/emqx_topic_metrics_api_i18n.conf +++ b/apps/emqx_modules/i18n/emqx_topic_metrics_api_i18n.conf @@ -1,7 +1,7 @@ emqx_topic_metrics_api { get_topic_metrics_api { desc { - en: """List Topic metrics""" + en: """List topic metrics""" zh: """获取主题监控数据""" } } @@ -15,21 +15,21 @@ emqx_topic_metrics_api { post_topic_metrics_api { desc { - en: """Create Topic metrics""" + en: """Create topic metrics""" zh: """创建主题监控数据""" } } gat_topic_metrics_data_api { desc { - en: """Get Topic metrics""" + en: """Get topic metrics""" zh: """获取主题监控数据""" } } delete_topic_metrics_data_api { desc { - en: """Delete Topic metrics""" + en: """Delete topic metrics""" zh: """删除主题监控数据""" } } @@ -43,7 +43,7 @@ emqx_topic_metrics_api { topic_metrics_api_response400 { desc { - en: """Bad Request. Already exists or bad topic name""" + en: """Bad request. Already exists or bad topic name""" zh: """错误请求。已存在或错误的主题名称""" } } From 394a4559035b6855f37a3094a87ded234a5e1773 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Tue, 21 Mar 2023 17:07:13 +0100 Subject: [PATCH 045/196] chore: add changelog --- changes/ce/fix-10196.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ce/fix-10196.en.md diff --git a/changes/ce/fix-10196.en.md b/changes/ce/fix-10196.en.md new file mode 100644 index 000000000..58ff01d8e --- /dev/null +++ b/changes/ce/fix-10196.en.md @@ -0,0 +1 @@ +Use lower-case for schema summaries and descritptions to be used in menu of generated online documentation. From 27b84453371fe99c3a8b0e2e0e0aaed274b1d31c Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Tue, 21 Mar 2023 16:30:45 +0100 Subject: [PATCH 046/196] fix: add inflight window setting to the clickhouse bridge This commit makes sure the inflight window setting is present for the clickhouse bridge. It also changes emqx_resource_schema that previously removed the inflight window setting from resources with query mode `always_sync`. We don't need to do that because all bridges that uses the buffer worker queue will get async call handling even if the bridge don't support the async callback. Co-authored-by: Zaiming (Stone) Shi --- .../i18n/emqx_resource_schema_i18n.conf | 10 +++++----- .../src/schema/emqx_resource_schema.erl | 18 +++++++++--------- .../src/emqx_ee_bridge_clickhouse.erl | 7 +------ 3 files changed, 15 insertions(+), 20 deletions(-) diff --git a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf index fb6b2eb06..3a6b50e83 100644 --- a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf +++ b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf @@ -135,14 +135,14 @@ When disabled the messages are buffered in RAM only.""" } } - async_inflight_window { + inflight_window { desc { - en: """Async query inflight window.""" - zh: """异步请求飞行队列窗口大小。""" + en: """Query inflight window. When query_mode is set to async, this config has to be et to 1 if messages from the same MQTT client have to be stricktly ordered.""" + zh: """请求飞行队列窗口大小。当请求模式为异步时，如果需要严格保证来自同一 MQTT 客户端的消息有序，则必须将此值设为 1。""""" } label { - en: """Async inflight window""" - zh: """异步请求飞行队列窗口""" + en: """Inflight window""" + zh: """请求飞行队列窗口""" } } diff --git a/apps/emqx_resource/src/schema/emqx_resource_schema.erl b/apps/emqx_resource/src/schema/emqx_resource_schema.erl index fdd65bc3c..e0f68d639 100644 --- a/apps/emqx_resource/src/schema/emqx_resource_schema.erl +++ b/apps/emqx_resource/src/schema/emqx_resource_schema.erl @@ -39,10 +39,9 @@ fields("resource_opts_sync_only") -> )} ]; fields("creation_opts_sync_only") -> - Fields0 = fields("creation_opts"), - Fields1 = lists:keydelete(async_inflight_window, 1, Fields0), + Fields = fields("creation_opts"), QueryMod = {query_mode, fun query_mode_sync_only/1}, - lists:keyreplace(query_mode, 1, Fields1, QueryMod); + lists:keyreplace(query_mode, 1, Fields, QueryMod); fields("resource_opts") -> [ {resource_opts, @@ -60,7 +59,7 @@ fields("creation_opts") -> {auto_restart_interval, fun auto_restart_interval/1}, {query_mode, fun query_mode/1}, {request_timeout, fun request_timeout/1}, - {async_inflight_window, fun async_inflight_window/1}, + {inflight_window, fun inflight_window/1}, {enable_batch, fun enable_batch/1}, {batch_size, fun batch_size/1}, {batch_time, fun batch_time/1}, @@ -136,11 +135,12 @@ enable_queue(deprecated) -> {since, "v5.0.14"}; enable_queue(desc) -> ?DESC("enable_queue"); enable_queue(_) -> undefined. -async_inflight_window(type) -> pos_integer(); -async_inflight_window(desc) -> ?DESC("async_inflight_window"); -async_inflight_window(default) -> ?DEFAULT_INFLIGHT; -async_inflight_window(required) -> false; -async_inflight_window(_) -> undefined. +inflight_window(type) -> pos_integer(); +inflight_window(aliases) -> [async_inflight_window]; +inflight_window(desc) -> ?DESC("inflight_window"); +inflight_window(default) -> ?DEFAULT_INFLIGHT; +inflight_window(required) -> false; +inflight_window(_) -> undefined. batch_size(type) -> pos_integer(); batch_size(desc) -> ?DESC("batch_size"); diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_clickhouse.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_clickhouse.erl index 9e03aca4a..89b07c6e4 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_clickhouse.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_clickhouse.erl @@ -107,8 +107,7 @@ fields("config") -> ] ++ emqx_ee_connector_clickhouse:fields(config); fields("creation_opts") -> - Opts = emqx_resource_schema:fields("creation_opts"), - [O || {Field, _} = O <- Opts, not is_hidden_opts(Field)]; + emqx_resource_schema:fields("creation_opts"); fields("post") -> fields("post", clickhouse); fields("put") -> @@ -131,10 +130,6 @@ desc(_) -> %% ------------------------------------------------------------------------------------------------- %% internal %% ------------------------------------------------------------------------------------------------- -is_hidden_opts(Field) -> - lists:member(Field, [ - async_inflight_window - ]). type_field(Type) -> {type, mk(enum([Type]), #{required => true, desc => ?DESC("desc_type")})}. From 7e6f52e8fea00d3f2f5553e49ff24dbb7b1a1ff4 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 21 Mar 2023 17:45:58 -0300 Subject: [PATCH 047/196] test: attempt to fix flaky kafka consumer test It might need some time for the metrics to be set. --- .../test/emqx_bridge_impl_kafka_consumer_SUITE.erl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl index 15b4fbe40..be6494cb2 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl @@ -1623,7 +1623,11 @@ t_bridge_rule_action_source(Config) -> }, emqx_json:decode(RawPayload, [return_maps]) ), - ?assertEqual(1, emqx_resource_metrics:received_get(ResourceId)), + ?retry( + _Interval = 200, + _NAttempts = 20, + ?assertEqual(1, emqx_resource_metrics:received_get(ResourceId)) + ), ok end ), From ed68687208ed161787654eee70b74ed3786f6079 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 22 Mar 2023 09:27:01 +0800 Subject: [PATCH 048/196] chore: add Keyspace to our spellcheck dict --- scripts/spellcheck/dicts/emqx.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/spellcheck/dicts/emqx.txt b/scripts/spellcheck/dicts/emqx.txt index b027f92ec..3ab3e1850 100644 --- a/scripts/spellcheck/dicts/emqx.txt +++ b/scripts/spellcheck/dicts/emqx.txt @@ -271,3 +271,4 @@ nif TDengine clickhouse FormatType +Keyspace From 9853d00cadb9ba446c769353d565ceb2c5d54230 Mon Sep 17 00:00:00 2001 From: firest Date: Mon, 13 Mar 2023 11:17:53 +0800 Subject: [PATCH 049/196] feat(bridges): integrate RocketMQ into data bridges --- apps/emqx_bridge/src/emqx_bridge.erl | 3 +- .../i18n/emqx_ee_bridge_rocketmq.conf | 70 ++++ lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl | 17 +- .../src/emqx_ee_bridge_rocketmq.erl | 120 ++++++ .../i18n/emqx_ee_connector_rocketmq.conf | 66 ++++ lib-ee/emqx_ee_connector/rebar.config | 1 + .../src/emqx_ee_connector.app.src | 3 +- .../src/emqx_ee_connector_rocketmq.erl | 342 ++++++++++++++++++ 8 files changed, 617 insertions(+), 5 deletions(-) create mode 100644 lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_rocketmq.conf create mode 100644 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_rocketmq.erl create mode 100644 lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_rocketmq.conf create mode 100644 lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index 98ce6a8b0..087c72dc3 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -67,7 +67,8 @@ T == timescale; T == matrix; T == tdengine; - T == dynamo + T == dynamo; + T == rocketmq ). load() -> diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_rocketmq.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_rocketmq.conf new file mode 100644 index 000000000..2e33e6c07 --- /dev/null +++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_rocketmq.conf @@ -0,0 +1,70 @@ +emqx_ee_bridge_rocketmq { + + local_topic { + desc { + en: """The MQTT topic filter to be forwarded to RocketMQ. All MQTT `PUBLISH` messages with the topic +matching the `local_topic` will be forwarded.
+NOTE: if the bridge is used as a rule action, `local_topic` should be left empty otherwise the messages will be duplicated.""" + zh: """发送到 'local_topic' 的消息都会转发到 RocketMQ。
+注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。""" + } + label { + en: "Local Topic" + zh: "本地 Topic" + } + } + + template { + desc { + en: """Template, the default value is empty. When this value is empty the whole message will be stored in the RocketMQ""" + zh: """模板, 默认为空，为空时将会将整个消息转发给 RocketMQ""" + } + label { + en: "Template" + zh: "模板" + } + } + config_enable { + desc { + en: """Enable or disable this bridge""" + zh: """启用/禁用桥接""" + } + label { + en: "Enable Or Disable Bridge" + zh: "启用/禁用桥接" + } + } + + desc_config { + desc { + en: """Configuration for a RocketMQ bridge.""" + zh: """RocketMQ 桥接配置""" + } + label: { + en: "RocketMQ Bridge Configuration" + zh: "RocketMQ 桥接配置" + } + } + + desc_type { + desc { + en: """The Bridge Type""" + zh: """Bridge 类型""" + } + label { + en: "Bridge Type" + zh: "桥接类型" + } + } + + desc_name { + desc { + en: """Bridge name.""" + zh: """桥接名字""" + } + label { + en: "Bridge Name" + zh: "桥接名字" + } + } +} diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl index ec81b7935..3989c3ab2 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl @@ -32,7 +32,8 @@ api_schemas(Method) -> ref(emqx_ee_bridge_matrix, Method), ref(emqx_ee_bridge_tdengine, Method), ref(emqx_ee_bridge_clickhouse, Method), - ref(emqx_ee_bridge_dynamo, Method) + ref(emqx_ee_bridge_dynamo, Method), + ref(emqx_ee_bridge_rocketmq, Method) ]. schema_modules() -> @@ -49,7 +50,8 @@ schema_modules() -> emqx_ee_bridge_matrix, emqx_ee_bridge_tdengine, emqx_ee_bridge_clickhouse, - emqx_ee_bridge_dynamo + emqx_ee_bridge_dynamo, + emqx_ee_bridge_rocketmq ]. examples(Method) -> @@ -85,7 +87,8 @@ resource_type(timescale) -> emqx_connector_pgsql; resource_type(matrix) -> emqx_connector_pgsql; resource_type(tdengine) -> emqx_ee_connector_tdengine; resource_type(clickhouse) -> emqx_ee_connector_clickhouse; -resource_type(dynamo) -> emqx_ee_connector_dynamo. +resource_type(dynamo) -> emqx_ee_connector_dynamo; +resource_type(rocketmq) -> emqx_ee_connector_rocketmq. fields(bridges) -> [ @@ -128,6 +131,14 @@ fields(bridges) -> desc => <<"Dynamo Bridge Config">>, required => false } + )}, + {rocketmq, + mk( + hoconsc:map(name, ref(emqx_ee_bridge_rocketmq, "config")), + #{ + desc => <<"RocketMQ Bridge Config">>, + required => false + } )} ] ++ kafka_structs() ++ mongodb_structs() ++ influxdb_structs() ++ redis_structs() ++ pgsql_structs() ++ clickhouse_structs(). diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_rocketmq.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_rocketmq.erl new file mode 100644 index 000000000..d81ffc54c --- /dev/null +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_rocketmq.erl @@ -0,0 +1,120 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_ee_bridge_rocketmq). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("emqx_bridge/include/emqx_bridge.hrl"). +-include_lib("emqx_resource/include/emqx_resource.hrl"). + +-import(hoconsc, [mk/2, enum/1, ref/2]). + +-export([ + conn_bridge_examples/1, + values/1 +]). + +-export([ + namespace/0, + roots/0, + fields/1, + desc/1 +]). + +-define(DEFAULT_TEMPLATE, <<>>). +-define(DEFFAULT_REQ_TIMEOUT, <<"15s">>). + +%% ------------------------------------------------------------------------------------------------- +%% api + +conn_bridge_examples(Method) -> + [ + #{ + <<"rocketmq">> => #{ + summary => <<"RocketMQ Bridge">>, + value => values(Method) + } + } + ]. + +values(get) -> + maps:merge(values(post), ?METRICS_EXAMPLE); +values(post) -> + #{ + enable => true, + type => rocketmq, + name => <<"foo">>, + server => <<"127.0.0.1:9876">>, + topic => <<"TopicTest">>, + template => ?DEFAULT_TEMPLATE, + local_topic => <<"local/topic/#">>, + resource_opts => #{ + worker_pool_size => 1, + health_check_interval => ?HEALTHCHECK_INTERVAL_RAW, + auto_restart_interval => ?AUTO_RESTART_INTERVAL_RAW, + batch_size => ?DEFAULT_BATCH_SIZE, + batch_time => ?DEFAULT_BATCH_TIME, + query_mode => sync, + max_queue_bytes => ?DEFAULT_QUEUE_SIZE + } + }; +values(put) -> + values(post). + +%% ------------------------------------------------------------------------------------------------- +%% Hocon Schema Definitions +namespace() -> "bridge_rocketmq". + +roots() -> []. + +fields("config") -> + [ + {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})}, + {template, + mk( + binary(), + #{desc => ?DESC("template"), default => ?DEFAULT_TEMPLATE} + )}, + {local_topic, + mk( + binary(), + #{desc => ?DESC("local_topic"), required => false} + )}, + {resource_opts, + mk( + ref(?MODULE, "creation_opts"), + #{ + required => false, + default => #{<<"request_timeout">> => ?DEFFAULT_REQ_TIMEOUT}, + desc => ?DESC(emqx_resource_schema, <<"resource_opts">>) + } + )} + ] ++ + (emqx_ee_connector_rocketmq:fields(config) -- + emqx_connector_schema_lib:prepare_statement_fields()); +fields("creation_opts") -> + emqx_resource_schema:fields("creation_opts_sync_only"); +fields("post") -> + [type_field(), name_field() | fields("config")]; +fields("put") -> + fields("config"); +fields("get") -> + emqx_bridge_schema:status_fields() ++ fields("post"). + +desc("config") -> + ?DESC("desc_config"); +desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> + ["Configuration for RocketMQ using `", string:to_upper(Method), "` method."]; +desc("creation_opts" = Name) -> + emqx_resource_schema:desc(Name); +desc(_) -> + undefined. + +%% ------------------------------------------------------------------------------------------------- + +type_field() -> + {type, mk(enum([rocketmq]), #{required => true, desc => ?DESC("desc_type")})}. + +name_field() -> + {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})}. diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_rocketmq.conf b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_rocketmq.conf new file mode 100644 index 000000000..d4a610212 --- /dev/null +++ b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_rocketmq.conf @@ -0,0 +1,66 @@ +emqx_ee_connector_rocketmq { + + server { + desc { + en: """ +The IPv4 or IPv6 address or the hostname to connect to.
+A host entry has the following form: `Host[:Port]`.
+The RocketMQ default port 9876 is used if `[:Port]` is not specified. +""" + zh: """ +将要连接的 IPv4 或 IPv6 地址，或者主机名。
+主机名具有以下形式：`Host[:Port]`。
+如果未指定 `[:Port]`，则使用 RocketMQ 默认端口 9876。 +""" + } + label: { + en: "Server Host" + zh: "服务器地址" + } + } + + topic { + desc { + en: """RocketMQ Topic""" + zh: """RocketMQ 主题""" + } + label: { + en: "RocketMQ Topic" + zh: "RocketMQ 主题" + } + } + + refresh_interval { + desc { + en: """RocketMQ Topic Route Refresh Interval.""" + zh: """RocketMQ 主题路由更新间隔。""" + } + label: { + en: "Topic Route Refresh Interval" + zh: "主题路由更新间隔" + } + } + + send_buffer { + desc { + en: """The socket send buffer size of the RocketMQ driver client.""" + zh: """RocketMQ 驱动的套字节发送消息的缓冲区大小""" + } + label: { + en: "Send Buffer Size" + zh: "发送消息的缓冲区大小" + } + } + + security_token { + desc { + en: """RocketMQ Server Security Token""" + zh: """RocketMQ 服务器安全令牌""" + } + label: { + en: "Security Token" + zh: "安全令牌" + } + } + +} diff --git a/lib-ee/emqx_ee_connector/rebar.config b/lib-ee/emqx_ee_connector/rebar.config index 76f6ccfba..96b3df6a3 100644 --- a/lib-ee/emqx_ee_connector/rebar.config +++ b/lib-ee/emqx_ee_connector/rebar.config @@ -5,6 +5,7 @@ {tdengine, {git, "https://github.com/emqx/tdengine-client-erl", {tag, "0.1.5"}}}, {clickhouse, {git, "https://github.com/emqx/clickhouse-client-erl", {tag, "0.2"}}}, {erlcloud, {git, "https://github.com/emqx/erlcloud.git", {tag,"3.5.16-emqx-1"}}}, + {rocketmq, {git, "https://github.com/emqx/rocketmq-client-erl.git", {tag, "v0.5.1"}}}, {emqx, {path, "../../apps/emqx"}} ]}. diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src index 6f40f7158..d8921198c 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src @@ -11,7 +11,8 @@ wolff, brod, clickhouse, - erlcloud + erlcloud, + rocketmq ]}, {env, []}, {modules, []}, diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl new file mode 100644 index 000000000..d41f83e1d --- /dev/null +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl @@ -0,0 +1,342 @@ +%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_ee_connector_rocketmq). + +-behaviour(emqx_resource). + +-include_lib("emqx_resource/include/emqx_resource.hrl"). +-include_lib("typerefl/include/types.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). + +-export([roots/0, fields/1]). + +%% `emqx_resource' API +-export([ + callback_mode/0, + is_buffer_supported/0, + on_start/2, + on_stop/2, + on_query/3, + on_batch_query/3, + on_get_status/2 +]). + +-import(hoconsc, [mk/2, enum/1, ref/2]). + +-define(ROCKETMQ_HOST_OPTIONS, #{ + default_port => 9876 +}). + +-ifdef(TEST). +-export([execute/2]). +-endif. + +%%===================================================================== +%% Hocon schema +roots() -> + [{config, #{type => hoconsc:ref(?MODULE, config)}}]. + +fields(config) -> + [ + {server, server()}, + {topic, + mk( + binary(), + #{default => <<"TopicTest">>, desc => ?DESC(topic)} + )}, + {refresh_interval, + mk( + emqx_schema:duration(), + #{default => <<"3s">>, desc => ?DESC(refresh_interval)} + )}, + {send_buffer, + mk( + emqx_schema:bytesize(), + #{default => <<"1024KB">>, desc => ?DESC(send_buffer)} + )}, + {security_token, mk(binary(), #{default => <<>>, desc => ?DESC(security_token)})} + | relational_fields() + ]. + +add_default_username(Fields) -> + lists:map( + fun + ({username, OrigUsernameFn}) -> + {username, add_default_fn(OrigUsernameFn, <<"">>)}; + (Field) -> + Field + end, + Fields + ). + +add_default_fn(OrigFn, Default) -> + fun + (default) -> Default; + (Field) -> OrigFn(Field) + end. + +server() -> + Meta = #{desc => ?DESC("server")}, + emqx_schema:servers_sc(Meta, ?ROCKETMQ_HOST_OPTIONS). + +relational_fields() -> + Fields = [username, password, auto_reconnect], + Values = lists:filter( + fun({E, _}) -> lists:member(E, Fields) end, + emqx_connector_schema_lib:relational_db_fields() + ), + add_default_username(Values). + +%%======================================================================================== +%% `emqx_resource' API +%%======================================================================================== + +callback_mode() -> always_sync. + +is_buffer_supported() -> false. + +on_start( + InstanceId, + #{server := Server, topic := Topic} = Config1 +) -> + ?SLOG(info, #{ + msg => "starting_rocketmq_connector", + connector => InstanceId, + config => redact(Config1) + }), + Config = maps:merge(default_security_info(), Config1), + {Host, Port} = emqx_schema:parse_server(Server, ?ROCKETMQ_HOST_OPTIONS), + + Server1 = [{Host, Port}], + ClientId = client_id(InstanceId), + ClientCfg = #{acl_info => #{}}, + + TopicTks = emqx_plugin_libs_rule:preproc_tmpl(Topic), + ProducerOpts = make_producer_opts(Config), + Templates = parse_template(Config), + ProducersMapPID = create_producers_map(ClientId), + State = #{ + client_id => ClientId, + topic_tokens => TopicTks, + config => Config, + templates => Templates, + producers_map_pid => ProducersMapPID, + producers_opts => ProducerOpts + }, + + case rocketmq:ensure_supervised_client(ClientId, Server1, ClientCfg) of + {ok, _Pid} -> + {ok, State}; + {error, _Reason} = Error -> + ?tp( + rocketmq_connector_start_failed, + #{error => _Reason} + ), + Error + end. + +on_stop(InstanceId, #{client_id := ClientId, producers_map_pid := Pid} = _State) -> + ?SLOG(info, #{ + msg => "stopping_rocketmq_connector", + connector => InstanceId + }), + Pid ! ok, + ok = rocketmq:stop_and_delete_supervised_client(ClientId). + +on_query(InstanceId, Query, State) -> + do_query(InstanceId, Query, send_sync, State). + +%% We only support batch inserts and all messages must have the same topic +on_batch_query(InstanceId, [{send_message, _Msg} | _] = Query, State) -> + do_query(InstanceId, Query, batch_send_sync, State); +on_batch_query(_InstanceId, Query, _State) -> + {error, {unrecoverable_error, {invalid_request, Query}}}. + +on_get_status(_InstanceId, #{client_id := ClientId}) -> + case rocketmq_client_sup:find_client(ClientId) of + {ok, _Pid} -> + connected; + _ -> + connecting + end. + +%%======================================================================================== +%% Helper fns +%%======================================================================================== + +do_query( + InstanceId, + Query, + QueryFunc, + #{ + templates := Templates, + client_id := ClientId, + topic_tokens := TopicTks, + producers_opts := ProducerOpts, + config := #{topic := RawTopic, resource_opts := #{request_timeout := RequestTimeout}} + } = State +) -> + ?TRACE( + "QUERY", + "rocketmq_connector_received", + #{connector => InstanceId, query => Query, state => State} + ), + + TopicKey = get_topic_key(Query, RawTopic, TopicTks), + Data = apply_template(Query, Templates), + + Result = safe_do_produce( + InstanceId, QueryFunc, ClientId, TopicKey, Data, ProducerOpts, RequestTimeout + ), + case Result of + {error, Reason} -> + ?tp( + rocketmq_connector_query_return, + #{error => Reason} + ), + ?SLOG(error, #{ + msg => "rocketmq_connector_do_query_failed", + connector => InstanceId, + query => Query, + reason => Reason + }), + Result; + _ -> + ?tp( + rocketmq_connector_query_return, + #{result => Result} + ), + Result + end. + +safe_do_produce(InstanceId, QueryFunc, ClientId, TopicKey, Data, ProducerOpts, RequestTimeout) -> + try + Producers = get_producers(ClientId, TopicKey, ProducerOpts), + produce(InstanceId, QueryFunc, Producers, Data, RequestTimeout) + catch + _Type:Reason -> + {error, {unrecoverable_error, Reason}} + end. + +produce(_InstanceId, QueryFunc, Producers, Data, RequestTimeout) -> + rocketmq:QueryFunc(Producers, Data, RequestTimeout). + +parse_template(Config) -> + Templates = + case maps:get(template, Config, undefined) of + undefined -> #{}; + <<>> -> #{}; + Template -> #{send_message => Template} + end, + + parse_template(maps:to_list(Templates), #{}). + +parse_template([{Key, H} | T], Templates) -> + ParamsTks = emqx_plugin_libs_rule:preproc_tmpl(H), + parse_template( + T, + Templates#{Key => ParamsTks} + ); +parse_template([], Templates) -> + Templates. + +get_topic_key({_, Msg}, RawTopic, TopicTks) -> + {RawTopic, emqx_plugin_libs_rule:proc_tmpl(TopicTks, Msg)}; +get_topic_key([Query | _], RawTopic, TopicTks) -> + get_topic_key(Query, RawTopic, TopicTks). + +apply_template({Key, Msg} = _Req, Templates) -> + case maps:get(Key, Templates, undefined) of + undefined -> + emqx_json:encode(Msg); + Template -> + emqx_plugin_libs_rule:proc_tmpl(Template, Msg) + end; +apply_template([{Key, _} | _] = Reqs, Templates) -> + case maps:get(Key, Templates, undefined) of + undefined -> + [emqx_json:encode(Msg) || {_, Msg} <- Reqs]; + Template -> + [emqx_plugin_libs_rule:proc_tmpl(Template, Msg) || {_, Msg} <- Reqs] + end. + +client_id(InstanceId) -> + Name = emqx_resource_manager:manager_id_to_resource_id(InstanceId), + erlang:binary_to_atom(Name, utf8). + +redact(Msg) -> + emqx_misc:redact(Msg, fun is_sensitive_key/1). + +is_sensitive_key(security_token) -> + true; +is_sensitive_key(_) -> + false. + +make_producer_opts( + #{ + username := Username, + password := Password, + security_token := SecurityToken, + send_buffer := SendBuff, + refresh_interval := RefreshInterval + } +) -> + ACLInfo = acl_info(Username, Password, SecurityToken), + #{ + tcp_opts => [{sndbuf, SendBuff}], + ref_topic_route_interval => RefreshInterval, + acl_info => ACLInfo + }. + +acl_info(<<>>, <<>>, <<>>) -> + #{}; +acl_info(Username, Password, <<>>) when is_binary(Username), is_binary(Password) -> + #{ + access_key => Username, + secret_key => Password + }; +acl_info(Username, Password, SecurityToken) when + is_binary(Username), is_binary(Password), is_binary(SecurityToken) +-> + #{ + access_key => Username, + secret_key => Password, + security_token => SecurityToken + }; +acl_info(_, _, _) -> + #{}. + +create_producers_map(ClientId) -> + erlang:spawn(fun() -> + case ets:whereis(ClientId) of + undefined -> + _ = ets:new(ClientId, [public, named_table]), + ok; + _ -> + ok + end, + receive + _Msg -> + ok + end + end). + +get_producers(ClientId, {_, Topic1} = TopicKey, ProducerOpts) -> + case ets:lookup(ClientId, TopicKey) of + [{_, Producers0}] -> + Producers0; + _ -> + ProducerGroup = iolist_to_binary([atom_to_list(ClientId), "_", Topic1]), + {ok, Producers0} = rocketmq:ensure_supervised_producers( + ClientId, ProducerGroup, Topic1, ProducerOpts + ), + ets:insert(ClientId, {TopicKey, Producers0}), + Producers0 + end. + +default_security_info() -> + #{username => <<>>, password => <<>>, security_token => <<>>}. From 4ad3579966945a40750e9ee2b3c87fe4148a6d89 Mon Sep 17 00:00:00 2001 From: firest Date: Mon, 20 Mar 2023 11:14:25 +0800 Subject: [PATCH 050/196] test(bridges): add test suite for RocketMQ --- .../docker-compose-rocketmq.yaml | 34 +++ .../docker-compose-toxiproxy.yaml | 1 + .../rocketmq/conf/broker.conf | 22 ++ .../rocketmq/logs/.gitkeep | 0 .../rocketmq/store/.gitkeep | 0 .ci/docker-compose-file/toxiproxy.json | 6 + lib-ee/emqx_ee_bridge/docker-ct | 1 + .../test/emqx_ee_bridge_rocketmq_SUITE.erl | 267 ++++++++++++++++++ .../src/emqx_ee_connector_rocketmq.erl | 4 - scripts/ct/run.sh | 3 + 10 files changed, 334 insertions(+), 4 deletions(-) create mode 100644 .ci/docker-compose-file/docker-compose-rocketmq.yaml create mode 100644 .ci/docker-compose-file/rocketmq/conf/broker.conf create mode 100644 .ci/docker-compose-file/rocketmq/logs/.gitkeep create mode 100644 .ci/docker-compose-file/rocketmq/store/.gitkeep create mode 100644 lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_rocketmq_SUITE.erl diff --git a/.ci/docker-compose-file/docker-compose-rocketmq.yaml b/.ci/docker-compose-file/docker-compose-rocketmq.yaml new file mode 100644 index 000000000..3c872a7c2 --- /dev/null +++ b/.ci/docker-compose-file/docker-compose-rocketmq.yaml @@ -0,0 +1,34 @@ +version: '3.9' + +services: + mqnamesrv: + image: apache/rocketmq:4.9.4 + container_name: rocketmq_namesrv +# ports: +# - 9876:9876 + volumes: + - ./rocketmq/logs:/opt/logs + - ./rocketmq/store:/opt/store + command: ./mqnamesrv + networks: + - emqx_bridge + + mqbroker: + image: apache/rocketmq:4.9.4 + container_name: rocketmq_broker +# ports: +# - 10909:10909 +# - 10911:10911 + volumes: + - ./rocketmq/logs:/opt/logs + - ./rocketmq/store:/opt/store + - ./rocketmq/conf/broker.conf:/etc/rocketmq/broker.conf + environment: + NAMESRV_ADDR: "rocketmq_namesrv:9876" + JAVA_OPTS: " -Duser.home=/opt" + JAVA_OPT_EXT: "-server -Xms1024m -Xmx1024m -Xmn1024m" + command: ./mqbroker -c /etc/rocketmq/broker.conf + depends_on: + - mqnamesrv + networks: + - emqx_bridge diff --git a/.ci/docker-compose-file/docker-compose-toxiproxy.yaml b/.ci/docker-compose-file/docker-compose-toxiproxy.yaml index 16f18b6c2..24f1d90b2 100644 --- a/.ci/docker-compose-file/docker-compose-toxiproxy.yaml +++ b/.ci/docker-compose-file/docker-compose-toxiproxy.yaml @@ -22,6 +22,7 @@ services: - 15433:5433 - 16041:6041 - 18000:8000 + - 19876:9876 command: - "-host=0.0.0.0" - "-config=/config/toxiproxy.json" diff --git a/.ci/docker-compose-file/rocketmq/conf/broker.conf b/.ci/docker-compose-file/rocketmq/conf/broker.conf new file mode 100644 index 000000000..c343090e4 --- /dev/null +++ b/.ci/docker-compose-file/rocketmq/conf/broker.conf @@ -0,0 +1,22 @@ +brokerClusterName=DefaultCluster +brokerName=broker-a +brokerId=0 + +brokerIP1=rocketmq_broker + +defaultTopicQueueNums=4 +autoCreateTopicEnable=true +autoCreateSubscriptionGroup=true + +listenPort=10911 +deleteWhen=04 + +fileReservedTime=120 +mapedFileSizeCommitLog=1073741824 +mapedFileSizeConsumeQueue=300000 +diskMaxUsedSpaceRatio=100 +maxMessageSize=65536 + +brokerRole=ASYNC_MASTER + +flushDiskType=ASYNC_FLUSH diff --git a/.ci/docker-compose-file/rocketmq/logs/.gitkeep b/.ci/docker-compose-file/rocketmq/logs/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/.ci/docker-compose-file/rocketmq/store/.gitkeep b/.ci/docker-compose-file/rocketmq/store/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/.ci/docker-compose-file/toxiproxy.json b/.ci/docker-compose-file/toxiproxy.json index 2f8c4341b..e22735091 100644 --- a/.ci/docker-compose-file/toxiproxy.json +++ b/.ci/docker-compose-file/toxiproxy.json @@ -77,5 +77,11 @@ "listen": "0.0.0.0:9295", "upstream": "kafka-1.emqx.net:9295", "enabled": true + }, + { + "name": "rocketmq", + "listen": "0.0.0.0:9876", + "upstream": "rocketmq_namesrv:9876", + "enabled": true } ] diff --git a/lib-ee/emqx_ee_bridge/docker-ct b/lib-ee/emqx_ee_bridge/docker-ct index ac1728ad2..116bc44ad 100644 --- a/lib-ee/emqx_ee_bridge/docker-ct +++ b/lib-ee/emqx_ee_bridge/docker-ct @@ -10,3 +10,4 @@ pgsql tdengine clickhouse dynamo +rocketmq diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_rocketmq_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_rocketmq_SUITE.erl new file mode 100644 index 000000000..cd02b65d0 --- /dev/null +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_rocketmq_SUITE.erl @@ -0,0 +1,267 @@ +%%-------------------------------------------------------------------- +% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_ee_bridge_rocketmq_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +% Bridge defaults +-define(TOPIC, "TopicTest"). +-define(BATCH_SIZE, 10). +-define(PAYLOAD, <<"HELLO">>). + +-define(GET_CONFIG(KEY__, CFG__), proplists:get_value(KEY__, CFG__)). + +%%------------------------------------------------------------------------------ +%% CT boilerplate +%%------------------------------------------------------------------------------ + +all() -> + [ + {group, with_batch}, + {group, without_batch} + ]. + +groups() -> + TCs = emqx_common_test_helpers:all(?MODULE), + [ + {with_batch, TCs}, + {without_batch, TCs} + ]. + +init_per_group(with_batch, Config0) -> + Config = [{batch_size, ?BATCH_SIZE} | Config0], + common_init(Config); +init_per_group(without_batch, Config0) -> + Config = [{batch_size, 1} | Config0], + common_init(Config); +init_per_group(_Group, Config) -> + Config. + +end_per_group(Group, Config) when Group =:= with_batch; Group =:= without_batch -> + ProxyHost = ?config(proxy_host, Config), + ProxyPort = ?config(proxy_port, Config), + emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), + ok; +end_per_group(_Group, _Config) -> + ok. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + emqx_mgmt_api_test_util:end_suite(), + ok = emqx_common_test_helpers:stop_apps([emqx_bridge, emqx_conf]), + ok. + +init_per_testcase(_Testcase, Config) -> + delete_bridge(Config), + Config. + +end_per_testcase(_Testcase, Config) -> + ProxyHost = ?config(proxy_host, Config), + ProxyPort = ?config(proxy_port, Config), + emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), + ok = snabbkaffe:stop(), + delete_bridge(Config), + ok. + +%%------------------------------------------------------------------------------ +%% Helper fns +%%------------------------------------------------------------------------------ + +common_init(ConfigT) -> + BridgeType = <<"rocketmq">>, + Host = os:getenv("ROCKETMQ_HOST", "toxiproxy"), + Port = list_to_integer(os:getenv("ROCKETMQ_PORT", "9876")), + + Config0 = [ + {host, Host}, + {port, Port}, + {query_mode, sync}, + {proxy_name, "rocketmq"} + | ConfigT + ], + + case emqx_common_test_helpers:is_tcp_server_available(Host, Port) of + true -> + % Setup toxiproxy + ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"), + ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")), + emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), + % Ensure EE bridge module is loaded + _ = application:load(emqx_ee_bridge), + _ = emqx_ee_bridge:module_info(), + ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]), + emqx_mgmt_api_test_util:init_suite(), + {Name, RocketMQConf} = rocketmq_config(BridgeType, Config0), + Config = + [ + {rocketmq_config, RocketMQConf}, + {rocketmq_bridge_type, BridgeType}, + {rocketmq_name, Name}, + {proxy_host, ProxyHost}, + {proxy_port, ProxyPort} + | Config0 + ], + Config; + false -> + case os:getenv("IS_CI") of + false -> + {skip, no_rocketmq}; + _ -> + throw(no_rocketmq) + end + end. + +rocketmq_config(BridgeType, Config) -> + Port = integer_to_list(?GET_CONFIG(port, Config)), + Server = ?GET_CONFIG(host, Config) ++ ":" ++ Port, + Name = atom_to_binary(?MODULE), + BatchSize = ?config(batch_size, Config), + QueryMode = ?config(query_mode, Config), + ConfigString = + io_lib:format( + "bridges.~s.~s {\n" + " enable = true\n" + " server = ~p\n" + " topic = ~p\n" + " resource_opts = {\n" + " request_timeout = 1500ms\n" + " batch_size = ~b\n" + " query_mode = ~s\n" + " }\n" + "}", + [ + BridgeType, + Name, + Server, + ?TOPIC, + BatchSize, + QueryMode + ] + ), + {Name, parse_and_check(ConfigString, BridgeType, Name)}. + +parse_and_check(ConfigString, BridgeType, Name) -> + {ok, RawConf} = hocon:binary(ConfigString, #{format => map}), + hocon_tconf:check_plain(emqx_bridge_schema, RawConf, #{required => false, atom_key => false}), + #{<<"bridges">> := #{BridgeType := #{Name := Config}}} = RawConf, + Config. + +create_bridge(Config) -> + BridgeType = ?GET_CONFIG(rocketmq_bridge_type, Config), + Name = ?GET_CONFIG(rocketmq_name, Config), + RocketMQConf = ?GET_CONFIG(rocketmq_config, Config), + emqx_bridge:create(BridgeType, Name, RocketMQConf). + +delete_bridge(Config) -> + BridgeType = ?GET_CONFIG(rocketmq_bridge_type, Config), + Name = ?GET_CONFIG(rocketmq_name, Config), + emqx_bridge:remove(BridgeType, Name). + +create_bridge_http(Params) -> + Path = emqx_mgmt_api_test_util:api_path(["bridges"]), + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of + {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + Error -> Error + end. + +send_message(Config, Payload) -> + Name = ?GET_CONFIG(rocketmq_name, Config), + BridgeType = ?GET_CONFIG(rocketmq_bridge_type, Config), + BridgeID = emqx_bridge_resource:bridge_id(BridgeType, Name), + emqx_bridge:send_message(BridgeID, Payload). + +query_resource(Config, Request) -> + Name = ?GET_CONFIG(rocketmq_name, Config), + BridgeType = ?GET_CONFIG(rocketmq_bridge_type, Config), + ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), + emqx_resource:query(ResourceID, Request, #{timeout => 500}). + +%%------------------------------------------------------------------------------ +%% Testcases +%%------------------------------------------------------------------------------ + +t_setup_via_config_and_publish(Config) -> + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + SentData = #{payload => ?PAYLOAD}, + ?check_trace( + begin + ?wait_async_action( + ?assertEqual(ok, send_message(Config, SentData)), + #{?snk_kind := rocketmq_connector_query_return}, + 10_000 + ), + ok + end, + fun(Trace0) -> + Trace = ?of_kind(rocketmq_connector_query_return, Trace0), + ?assertMatch([#{result := ok}], Trace), + ok + end + ), + ok. + +t_setup_via_http_api_and_publish(Config) -> + BridgeType = ?GET_CONFIG(rocketmq_bridge_type, Config), + Name = ?GET_CONFIG(rocketmq_name, Config), + RocketMQConf = ?GET_CONFIG(rocketmq_config, Config), + RocketMQConf2 = RocketMQConf#{ + <<"name">> => Name, + <<"type">> => BridgeType + }, + ?assertMatch( + {ok, _}, + create_bridge_http(RocketMQConf2) + ), + SentData = #{payload => ?PAYLOAD}, + ?check_trace( + begin + ?wait_async_action( + ?assertEqual(ok, send_message(Config, SentData)), + #{?snk_kind := rocketmq_connector_query_return}, + 10_000 + ), + ok + end, + fun(Trace0) -> + Trace = ?of_kind(rocketmq_connector_query_return, Trace0), + ?assertMatch([#{result := ok}], Trace), + ok + end + ), + ok. + +t_get_status(Config) -> + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + + Name = ?GET_CONFIG(rocketmq_name, Config), + BridgeType = ?GET_CONFIG(rocketmq_bridge_type, Config), + ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), + + ?assertEqual({ok, connected}, emqx_resource_manager:health_check(ResourceID)), + ok. + +t_simple_query(Config) -> + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + Request = {send_message, #{message => <<"Hello">>}}, + Result = query_resource(Config, Request), + ?assertEqual(ok, Result), + ok. diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl index d41f83e1d..84f2e2a89 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl @@ -31,10 +31,6 @@ default_port => 9876 }). --ifdef(TEST). --export([execute/2]). --endif. - %%===================================================================== %% Hocon schema roots() -> diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index bf7b2073d..a4eeb366d 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -170,6 +170,9 @@ for dep in ${CT_DEPS}; do dynamo) FILES+=( '.ci/docker-compose-file/docker-compose-dynamo.yaml' ) ;; + rocketmq) + FILES+=( '.ci/docker-compose-file/docker-compose-rocketmq.yaml' ) + ;; *) echo "unknown_ct_dependency $dep" exit 1 From 17e207cb71dd5247a2718ff0db6c043b1d038b2c Mon Sep 17 00:00:00 2001 From: firest Date: Thu, 16 Mar 2023 22:58:47 +0800 Subject: [PATCH 051/196] chore: fix spellcheck && update changes --- changes/ee/feat-10143.en.md | 1 + changes/ee/feat-10143.zh.md | 1 + scripts/spellcheck/dicts/emqx.txt | 1 + 3 files changed, 3 insertions(+) create mode 100644 changes/ee/feat-10143.en.md create mode 100644 changes/ee/feat-10143.zh.md diff --git a/changes/ee/feat-10143.en.md b/changes/ee/feat-10143.en.md new file mode 100644 index 000000000..67fc13dc2 --- /dev/null +++ b/changes/ee/feat-10143.en.md @@ -0,0 +1 @@ +Add `RocketMQ` data integration bridge. diff --git a/changes/ee/feat-10143.zh.md b/changes/ee/feat-10143.zh.md new file mode 100644 index 000000000..85a13ffa7 --- /dev/null +++ b/changes/ee/feat-10143.zh.md @@ -0,0 +1 @@ +为数据桥接增加 `RocketMQ` 支持。 diff --git a/scripts/spellcheck/dicts/emqx.txt b/scripts/spellcheck/dicts/emqx.txt index b027f92ec..b0d663ead 100644 --- a/scripts/spellcheck/dicts/emqx.txt +++ b/scripts/spellcheck/dicts/emqx.txt @@ -271,3 +271,4 @@ nif TDengine clickhouse FormatType +RocketMQ From 934e46307242a952db1b108b296ef496c0ef14cc Mon Sep 17 00:00:00 2001 From: Kinplemelon Date: Wed, 22 Mar 2023 12:57:39 +0800 Subject: [PATCH 052/196] chore(dashboard): change dashboard repo --- scripts/get-dashboard.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/get-dashboard.sh b/scripts/get-dashboard.sh index c3559865f..ace795aa5 100755 --- a/scripts/get-dashboard.sh +++ b/scripts/get-dashboard.sh @@ -20,7 +20,7 @@ case "$VERSION" in esac DASHBOARD_PATH='apps/emqx_dashboard/priv' -DASHBOARD_REPO='emqx-dashboard-web-new' +DASHBOARD_REPO='emqx-dashboard5' DIRECT_DOWNLOAD_URL="https://github.com/emqx/${DASHBOARD_REPO}/releases/download/${VERSION}/${RELEASE_ASSET_FILE}" case $(uname) in From 5d31f85aec0efcfa95f40a0f333fc412ff70c530 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Wed, 22 Mar 2023 11:43:25 +0300 Subject: [PATCH 053/196] chore: remove irrelevant changelog entry --- changes/ce/fix-10190.en.md | 1 - 1 file changed, 1 deletion(-) delete mode 100644 changes/ce/fix-10190.en.md diff --git a/changes/ce/fix-10190.en.md b/changes/ce/fix-10190.en.md deleted file mode 100644 index bffd9ca00..000000000 --- a/changes/ce/fix-10190.en.md +++ /dev/null @@ -1 +0,0 @@ -Fix the issue where nodes responses to the list bridges RPC were incorrectly flattened, which caused List Bridges API HTTP handler to crash when there was more than 1 node in the cluster. From 208813330a6f5ea7a997cf104ba9edb0bcadeeea Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 22 Mar 2023 10:59:20 -0300 Subject: [PATCH 054/196] fix: remove metrics from rocketmq example --- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_rocketmq.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_rocketmq.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_rocketmq.erl index d81ffc54c..124e18069 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_rocketmq.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_rocketmq.erl @@ -39,7 +39,7 @@ conn_bridge_examples(Method) -> ]. values(get) -> - maps:merge(values(post), ?METRICS_EXAMPLE); + values(post); values(post) -> #{ enable => true, From 61cb03b45a427fe361a79edc807d5e1e791de132 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 21 Mar 2023 14:32:50 -0300 Subject: [PATCH 055/196] fix(buffer_worker): change the default `resume_interval` value and expose it as hidden config Also removes the previously added alarm for request timeout. There are situations where having a short request timeout and a long health check interval make sense, so we don't want to alarm the user for those situations. Instead, we automatically attempt to set a reasonable `resume_interval` value. --- .../i18n/emqx_resource_schema_i18n.conf | 15 ++- apps/emqx_resource/include/emqx_resource.hrl | 5 +- .../src/emqx_resource_buffer_worker.erl | 49 ++----- .../src/emqx_resource_manager.erl | 1 - .../src/schema/emqx_resource_schema.erl | 7 + .../test/emqx_resource_SUITE.erl | 126 ------------------ changes/ce/fix-10154.en.md | 7 +- .../test/emqx_ee_bridge_gcp_pubsub_SUITE.erl | 14 +- 8 files changed, 50 insertions(+), 174 deletions(-) diff --git a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf index 2e5cf96e8..eb4f00ac7 100644 --- a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf +++ b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf @@ -45,6 +45,17 @@ For bridges only have ingress direction data flow, it can be set to 0 otherwise } } + resume_interval { + desc { + en: """The interval at which a resource will retry inflight requests.""" + zh: """资源重试机内请求的间隔时间。""" + } + label { + en: """Resume Interval""" + zh: """复职时间间隔""" + } + } + start_after_created { desc { en: """Whether start the resource right after created.""" @@ -102,8 +113,8 @@ For bridges only have ingress direction data flow, it can be set to 0 otherwise request_timeout { desc { - en: """Starting from the moment when the request enters the buffer, if the request remains in the buffer for the specified time or is sent but does not receive a response or acknowledgement in time, the request is considered expired. We recommend setting this timeout to be at least twice the health check interval, so that the buffer has the chance to recover if too many requests get enqueued.""" - zh: """从请求进入缓冲区开始计时，如果请求在规定的时间内仍停留在缓冲区内或者已发送但未能及时收到响应或确认，该请求将被视为过期。我们建议将这个超时设置为健康检查间隔的至少两倍，这样，如果有太多的请求被排队，缓冲区就有机会恢复。""" + en: """Starting from the moment when the request enters the buffer, if the request remains in the buffer for the specified time or is sent but does not receive a response or acknowledgement in time, the request is considered expired.""" + zh: """从请求进入缓冲区开始计时，如果请求在规定的时间内仍停留在缓冲区内或者已发送但未能及时收到响应或确认，该请求将被视为过期。""" } label { en: """Request Expiry""" diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl index 8033ed660..be570e694 100644 --- a/apps/emqx_resource/include/emqx_resource.hrl +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -91,10 +91,7 @@ -define(DEFAULT_QUEUE_SIZE, 100 * 1024 * 1024). -define(DEFAULT_QUEUE_SIZE_RAW, <<"100MB">>). -%% Note: this should be greater than the health check timeout; -%% otherwise, if the buffer worker is ever blocked, than all queued -%% requests will basically fail without being attempted. --define(DEFAULT_REQUEST_TIMEOUT, timer:seconds(30)). +-define(DEFAULT_REQUEST_TIMEOUT, timer:seconds(15)). %% count -define(DEFAULT_BATCH_SIZE, 1). diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl index 05622bdd7..648587c25 100644 --- a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl @@ -56,8 +56,6 @@ -export([clear_disk_queue_dir/2]). --export([deactivate_bad_request_timeout_alarm/1]). - -elvis([{elvis_style, dont_repeat_yourself, disable}]). -define(COLLECT_REQ_LIMIT, 1000). @@ -203,7 +201,8 @@ init({Id, Index, Opts}) -> RequestTimeout = maps:get(request_timeout, Opts, ?DEFAULT_REQUEST_TIMEOUT), BatchTime0 = maps:get(batch_time, Opts, ?DEFAULT_BATCH_TIME), BatchTime = adjust_batch_time(Id, RequestTimeout, BatchTime0), - maybe_toggle_bad_request_timeout_alarm(Id, RequestTimeout, HealthCheckInterval), + DefaultResumeInterval = default_resume_interval(RequestTimeout, HealthCheckInterval), + ResumeInterval = maps:get(resume_interval, Opts, DefaultResumeInterval), Data = #{ id => Id, index => Index, @@ -212,7 +211,7 @@ init({Id, Index, Opts}) -> batch_size => BatchSize, batch_time => BatchTime, queue => Queue, - resume_interval => maps:get(resume_interval, Opts, HealthCheckInterval), + resume_interval => ResumeInterval, tref => undefined }, ?tp(buffer_worker_init, #{id => Id, index => Index}), @@ -1684,38 +1683,16 @@ adjust_batch_time(Id, RequestTimeout, BatchTime0) -> end, BatchTime. -%% The request timeout should be greater than the health check -%% timeout, health timeout defines how often the buffer worker tries -%% to unblock. If request timeout is <= health check timeout and the -%% buffer worker is ever blocked, than all queued requests will -%% basically fail without being attempted. --spec maybe_toggle_bad_request_timeout_alarm( - resource_id(), request_timeout(), health_check_interval() -) -> ok. -maybe_toggle_bad_request_timeout_alarm(Id, _RequestTimeout = infinity, _HealthCheckInterval) -> - deactivate_bad_request_timeout_alarm(Id), - ok; -maybe_toggle_bad_request_timeout_alarm(Id, RequestTimeout, HealthCheckInterval) -> - case RequestTimeout > HealthCheckInterval of - true -> - deactivate_bad_request_timeout_alarm(Id), - ok; - false -> - _ = emqx_alarm:activate( - bad_request_timeout_alarm_id(Id), - #{resource_id => Id, reason => bad_request_timeout}, - <<"Request timeout should be greater than health check timeout: ", Id/binary>> - ), - ok - end. - --spec deactivate_bad_request_timeout_alarm(resource_id()) -> ok. -deactivate_bad_request_timeout_alarm(Id) -> - _ = emqx_alarm:ensure_deactivated(bad_request_timeout_alarm_id(Id)), - ok. - -bad_request_timeout_alarm_id(Id) -> - <<"bad_request_timeout:", Id/binary>>. +%% The request timeout should be greater than the resume interval, as +%% it defines how often the buffer worker tries to unblock. If request +%% timeout is <= resume interval and the buffer worker is ever +%% blocked, than all queued requests will basically fail without being +%% attempted. +-spec default_resume_interval(request_timeout(), health_check_interval()) -> timer:time(). +default_resume_interval(_RequestTimeout = infinity, HealthCheckInterval) -> + max(1, HealthCheckInterval); +default_resume_interval(RequestTimeout, HealthCheckInterval) -> + max(1, min(HealthCheckInterval, RequestTimeout div 3)). -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index 2bdc67a4d..40f9fe1ab 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -506,7 +506,6 @@ handle_remove_event(From, ClearMetrics, Data) -> true -> ok = emqx_metrics_worker:clear_metrics(?RES_METRICS, Data#data.id); false -> ok end, - emqx_resource_buffer_worker:deactivate_bad_request_timeout_alarm(Data#data.id), {stop_and_reply, {shutdown, removed}, [{reply, From, ok}]}. start_resource(Data, From) -> diff --git a/apps/emqx_resource/src/schema/emqx_resource_schema.erl b/apps/emqx_resource/src/schema/emqx_resource_schema.erl index fdd65bc3c..b9ed176fe 100644 --- a/apps/emqx_resource/src/schema/emqx_resource_schema.erl +++ b/apps/emqx_resource/src/schema/emqx_resource_schema.erl @@ -55,6 +55,7 @@ fields("creation_opts") -> [ {worker_pool_size, fun worker_pool_size/1}, {health_check_interval, fun health_check_interval/1}, + {resume_interval, fun resume_interval/1}, {start_after_created, fun start_after_created/1}, {start_timeout, fun start_timeout/1}, {auto_restart_interval, fun auto_restart_interval/1}, @@ -81,6 +82,12 @@ worker_pool_size(default) -> ?WORKER_POOL_SIZE; worker_pool_size(required) -> false; worker_pool_size(_) -> undefined. +resume_interval(type) -> emqx_schema:duration_ms(); +resume_interval(hidden) -> true; +resume_interval(desc) -> ?DESC("resume_interval"); +resume_interval(required) -> false; +resume_interval(_) -> undefined. + health_check_interval(type) -> emqx_schema:duration_ms(); health_check_interval(desc) -> ?DESC("health_check_interval"); health_check_interval(default) -> ?HEALTHCHECK_INTERVAL_RAW; diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index 25f4a6d77..ff7e1d347 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -2554,132 +2554,6 @@ do_t_recursive_flush() -> ), ok. -%% Check that we raise an alarm if a bad request timeout config is -%% issued. Request timeout should be greater than health check -%% timeout. -t_bad_request_timeout_alarm(_Config) -> - emqx_connector_demo:set_callback_mode(async_if_possible), - - %% 1) Same values. - {ok, _} = emqx_resource:create( - ?ID, - ?DEFAULT_RESOURCE_GROUP, - ?TEST_RESOURCE, - #{name => test_resource}, - #{ - query_mode => async, - request_timeout => 1_000, - health_check_interval => 1_000, - worker_pool_size => 2 - } - ), - ExpectedMessage = - <<"Request timeout should be greater than health check timeout: ", ?ID/binary>>, - ?assertMatch( - [ - #{ - message := ExpectedMessage, - details := #{reason := bad_request_timeout, resource_id := ?ID}, - deactivate_at := infinity - } - ], - emqx_alarm:get_alarms(activated) - ), - %% The unexpected termination of one of the buffer workers should - %% not turn the alarm off. - [Pid, _ | _] = emqx_resource_buffer_worker_sup:worker_pids(?ID), - MRef = monitor(process, Pid), - exit(Pid, kill), - receive - {'DOWN', MRef, process, Pid, _} -> - ok - after 300 -> - ct:fail("buffer worker didn't die") - end, - ?assertMatch( - [ - #{ - message := ExpectedMessage, - details := #{reason := bad_request_timeout, resource_id := ?ID}, - deactivate_at := infinity - } - ], - emqx_alarm:get_alarms(activated) - ), - ok = emqx_resource:remove_local(?ID), - ?assertEqual([], emqx_alarm:get_alarms(activated)), - - %% 2) Request timeout < health check interval. - {ok, _} = emqx_resource:create( - ?ID, - ?DEFAULT_RESOURCE_GROUP, - ?TEST_RESOURCE, - #{name => test_resource}, - #{ - query_mode => async, - request_timeout => 999, - health_check_interval => 1_000, - worker_pool_size => 2 - } - ), - ?assertMatch( - [ - #{ - message := ExpectedMessage, - details := #{reason := bad_request_timeout, resource_id := ?ID}, - deactivate_at := infinity - } - ], - emqx_alarm:get_alarms(activated) - ), - ok = emqx_resource:remove_local(?ID), - ?assertEqual([], emqx_alarm:get_alarms(activated)), - - %% 2) Request timeout < health check interval. - {ok, _} = emqx_resource:create( - ?ID, - ?DEFAULT_RESOURCE_GROUP, - ?TEST_RESOURCE, - #{name => test_resource}, - #{ - query_mode => async, - request_timeout => 999, - health_check_interval => 1_000, - worker_pool_size => 2 - } - ), - ?assertMatch( - [ - #{ - message := ExpectedMessage, - details := #{reason := bad_request_timeout, resource_id := ?ID}, - deactivate_at := infinity - } - ], - emqx_alarm:get_alarms(activated) - ), - ok = emqx_resource:remove_local(?ID), - ?assertEqual([], emqx_alarm:get_alarms(activated)), - - %% 3) Request timeout > health check interval. - {ok, _} = emqx_resource:create( - ?ID, - ?DEFAULT_RESOURCE_GROUP, - ?TEST_RESOURCE, - #{name => test_resource}, - #{ - query_mode => async, - request_timeout => 1_001, - health_check_interval => 1_000, - worker_pool_size => 2 - } - ), - ?assertEqual([], emqx_alarm:get_alarms(activated)), - ok = emqx_resource:remove_local(?ID), - ?assertEqual([], emqx_alarm:get_alarms(activated)), - - ok. - %%------------------------------------------------------------------------------ %% Helpers %%------------------------------------------------------------------------------ diff --git a/changes/ce/fix-10154.en.md b/changes/ce/fix-10154.en.md index 83a729360..24bc4bae1 100644 --- a/changes/ce/fix-10154.en.md +++ b/changes/ce/fix-10154.en.md @@ -1,7 +1,8 @@ -Change the default `request_timeout` for bridges and connectors to be -twice the default `health_check_interval`. +Change the default `resume_interval` for bridges and connectors to be +the minimum of `health_check_interval` and `request_timeout / 3`. +Also exposes it as a hidden configuration to allow fine tuning. -Before this change, the default values for those two options meant +Before this change, the default values for `resume_interval` meant that, if a buffer ever got blocked due to resource errors or high message volumes, then, by the time the buffer would try to resume its normal operations, almost all requests would have timed out. diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl index 452b7a4d2..55dfa5555 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl @@ -520,6 +520,7 @@ wait_until_gauge_is(GaugeName, ExpectedValue, Timeout) -> #{measurements := #{gauge_set := ExpectedValue}} -> ok; #{measurements := #{gauge_set := Value}} -> + ct:pal("events: ~p", [Events]), ct:fail( "gauge ~p didn't reach expected value ~p; last value: ~p", [GaugeName, ExpectedValue, Value] @@ -972,7 +973,13 @@ t_publish_econnrefused(Config) -> ResourceId = ?config(resource_id, Config), %% set pipelining to 1 so that one of the 2 requests is `pending' %% in ehttpc. - {ok, _} = create_bridge(Config, #{<<"pipelining">> => 1}), + {ok, _} = create_bridge( + Config, + #{ + <<"pipelining">> => 1, + <<"resource_opts">> => #{<<"resume_interval">> => <<"15s">>} + } + ), {ok, #{<<"id">> := RuleId}} = create_rule_and_action_http(Config), on_exit(fun() -> ok = emqx_rule_engine:delete_rule(RuleId) end), assert_empty_metrics(ResourceId), @@ -986,7 +993,10 @@ t_publish_timeout(Config) -> %% requests are done separately. {ok, _} = create_bridge(Config, #{ <<"pipelining">> => 1, - <<"resource_opts">> => #{<<"batch_size">> => 1} + <<"resource_opts">> => #{ + <<"batch_size">> => 1, + <<"resume_interval">> => <<"15s">> + } }), {ok, #{<<"id">> := RuleId}} = create_rule_and_action_http(Config), on_exit(fun() -> ok = emqx_rule_engine:delete_rule(RuleId) end), From 2662f79ec0ce26124664227a357fa3952271c998 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Wed, 22 Mar 2023 16:29:51 +0100 Subject: [PATCH 056/196] feat(emqx_dashboard): use label as summary in OpenAPI spec --- apps/emqx_dashboard/src/emqx_dashboard_swagger.erl | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 77fcd4f76..2290dbedb 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -457,7 +457,18 @@ trans_description(Spec, Hocon) -> Spec; Desc -> Desc1 = binary:replace(Desc, [<<"\n">>], <<"
">>, [global]), - Spec#{description => Desc1} + maybe_add_summary_from_label(Spec#{description => Desc1}, Hocon) + end. + +maybe_add_summary_from_label(Spec, Hocon) -> + Label = + case desc_struct(Hocon) of + ?DESC(_, _) = Struct -> get_i18n(<<"label">>, Struct, undefined); + _ -> undefined + end, + case Label of + undefined -> Spec; + _ -> Spec#{summary => Label} end. get_i18n(Key, Struct, Default) -> From 944d456f944ec2b3df863ce16c0cdfeb7d4e119f Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Wed, 22 Mar 2023 16:32:34 +0100 Subject: [PATCH 057/196] chore: add changelog --- changes/ce/feat-10207.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ce/feat-10207.en.md diff --git a/changes/ce/feat-10207.en.md b/changes/ce/feat-10207.en.md new file mode 100644 index 000000000..99ca17944 --- /dev/null +++ b/changes/ce/feat-10207.en.md @@ -0,0 +1 @@ +Use 'label' from i18n file as 'summary' in OpenAPI spec. From 127a075b66c469b0e873934c3b389ed2ebb73aca Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 22 Mar 2023 14:34:37 -0300 Subject: [PATCH 058/196] test(dynamo): attempt to fix dynamo tests Those tests in the `flaky` test are really flaky and require lots of CI retries. Apparently, the flakiness comes from race conditions from restarting bridges with the same name too fast between test cases. Previously, all test cases were sharing the same bridge name (the module name). --- .../src/emqx_resource_manager.erl | 1 + .../test/emqx_ee_bridge_dynamo_SUITE.erl | 50 +++++++++++++------ 2 files changed, 36 insertions(+), 15 deletions(-) diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index b21ffcae3..6a4919b41 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -388,6 +388,7 @@ handle_event(state_timeout, health_check, connecting, Data) -> handle_event(enter, _OldState, connected = State, Data) -> ok = log_state_consistency(State, Data), _ = emqx_alarm:deactivate(Data#data.id), + ?tp(resource_connected_enter, #{}), {keep_state_and_data, health_check_actions(Data)}; handle_event(state_timeout, health_check, connected, Data) -> handle_connected_health_check(Data); diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl index 26666c6d8..183002e61 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl @@ -83,9 +83,10 @@ end_per_suite(_Config) -> ok = emqx_common_test_helpers:stop_apps([emqx_bridge, emqx_conf]), ok. -init_per_testcase(_Testcase, Config) -> +init_per_testcase(TestCase, Config) -> create_table(Config), - Config. + ok = snabbkaffe:start_trace(), + [{dynamo_name, atom_to_binary(TestCase)} | Config]. end_per_testcase(_Testcase, Config) -> ProxyHost = ?config(proxy_host, Config), @@ -93,7 +94,7 @@ end_per_testcase(_Testcase, Config) -> emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), ok = snabbkaffe:stop(), delete_table(Config), - delete_bridge(Config), + delete_all_bridges(), ok. %%------------------------------------------------------------------------------ @@ -186,15 +187,22 @@ parse_and_check(ConfigString, BridgeType, Name) -> Config. create_bridge(Config) -> - BridgeType = ?config(dynamo_bridge_type, Config), - Name = ?config(dynamo_name, Config), - TDConfig = ?config(dynamo_config, Config), - emqx_bridge:create(BridgeType, Name, TDConfig). + create_bridge(Config, _Overrides = #{}). -delete_bridge(Config) -> +create_bridge(Config, Overrides) -> BridgeType = ?config(dynamo_bridge_type, Config), Name = ?config(dynamo_name, Config), - emqx_bridge:remove(BridgeType, Name). + DynamoConfig0 = ?config(dynamo_config, Config), + DynamoConfig = emqx_map_lib:deep_merge(DynamoConfig0, Overrides), + emqx_bridge:create(BridgeType, Name, DynamoConfig). + +delete_all_bridges() -> + lists:foreach( + fun(#{name := Name, type := Type}) -> + emqx_bridge:remove(Type, Name) + end, + emqx_bridge:list() + ). create_bridge_http(Params) -> Path = emqx_mgmt_api_test_util:api_path(["bridges"]), @@ -327,10 +335,12 @@ t_setup_via_http_api_and_publish(Config) -> ok. t_get_status(Config) -> - ?assertMatch( - {ok, _}, - create_bridge(Config) - ), + {{ok, _}, {ok, _}} = + ?wait_async_action( + create_bridge(Config), + #{?snk_kind := resource_connected_enter}, + 20_000 + ), ProxyPort = ?config(proxy_port, Config), ProxyHost = ?config(proxy_host, Config), @@ -359,7 +369,12 @@ t_write_failure(Config) -> ProxyName = ?config(proxy_name, Config), ProxyPort = ?config(proxy_port, Config), ProxyHost = ?config(proxy_host, Config), - {ok, _} = create_bridge(Config), + {{ok, _}, {ok, _}} = + ?wait_async_action( + create_bridge(Config), + #{?snk_kind := resource_connected_enter}, + 20_000 + ), SentData = #{id => emqx_misc:gen_id(), payload => ?PAYLOAD}, emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() -> ?assertMatch( @@ -372,7 +387,12 @@ t_write_timeout(Config) -> ProxyName = ?config(proxy_name, Config), ProxyPort = ?config(proxy_port, Config), ProxyHost = ?config(proxy_host, Config), - {ok, _} = create_bridge(Config), + {{ok, _}, {ok, _}} = + ?wait_async_action( + create_bridge(Config), + #{?snk_kind := resource_connected_enter}, + 20_000 + ), SentData = #{id => emqx_misc:gen_id(), payload => ?PAYLOAD}, emqx_common_test_helpers:with_failure(timeout, ProxyName, ProxyHost, ProxyPort, fun() -> ?assertMatch( From 8844b22c809ae7495243ed8f9164d84cf6303513 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 22 Mar 2023 15:32:09 -0300 Subject: [PATCH 059/196] docs: improve descriptions Co-authored-by: Zaiming (Stone) Shi --- apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf index eb4f00ac7..aedcabc70 100644 --- a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf +++ b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf @@ -47,12 +47,12 @@ For bridges only have ingress direction data flow, it can be set to 0 otherwise resume_interval { desc { - en: """The interval at which a resource will retry inflight requests.""" - zh: """资源重试机内请求的间隔时间。""" + en: """The interval at which the buffer worker attempts to resend failed requests in the inflight window.""" + zh: """在发送失败后尝试重传飞行窗口中的请求的时间间隔。""" } label { en: """Resume Interval""" - zh: """复职时间间隔""" + zh: """重试时间间隔""" } } From cb65cded8825eba47d5fde6f4542e66af7807b04 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 22 Mar 2023 15:22:13 -0300 Subject: [PATCH 060/196] fix(last_will_testament): don't publish LWT if client is banned when kicked Fixes https://emqx.atlassian.net/browse/EMQX-9288 Related issue: https://github.com/emqx/emqx/issues/10192#issuecomment-1478809900 --- apps/emqx/src/emqx_channel.erl | 20 ++++--- apps/emqx_authz/test/emqx_authz_SUITE.erl | 65 +++++++++++++++++++++++ changes/ce/fix-10209.en.md | 2 + 3 files changed, 80 insertions(+), 7 deletions(-) create mode 100644 changes/ce/fix-10209.en.md diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index 9acad4d57..e01a16f83 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -2128,17 +2128,23 @@ publish_will_msg( ClientInfo = #{mountpoint := MountPoint}, Msg = #message{topic = Topic} ) -> - case emqx_access_control:authorize(ClientInfo, publish, Topic) of - allow -> - NMsg = emqx_mountpoint:mount(MountPoint, Msg), - _ = emqx_broker:publish(NMsg), - ok; - deny -> + PublishingDisallowed = emqx_access_control:authorize(ClientInfo, publish, Topic) =/= allow, + ClientBanned = emqx_banned:check(ClientInfo), + case PublishingDisallowed orelse ClientBanned of + true -> ?tp( warning, last_will_testament_publish_denied, - #{topic => Topic} + #{ + topic => Topic, + client_banned => ClientBanned, + publishing_disallowed => PublishingDisallowed + } ), + ok; + false -> + NMsg = emqx_mountpoint:mount(MountPoint, Msg), + _ = emqx_broker:publish(NMsg), ok end. diff --git a/apps/emqx_authz/test/emqx_authz_SUITE.erl b/apps/emqx_authz/test/emqx_authz_SUITE.erl index b3ce04f43..84b1d903e 100644 --- a/apps/emqx_authz/test/emqx_authz_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_SUITE.erl @@ -26,6 +26,8 @@ -include_lib("emqx/include/emqx_placeholder.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). +-import(emqx_common_test_helpers, [on_exit/1]). + all() -> emqx_common_test_helpers:all(?MODULE). @@ -65,6 +67,7 @@ end_per_suite(_Config) -> init_per_testcase(TestCase, Config) when TestCase =:= t_subscribe_deny_disconnect_publishes_last_will_testament; + TestCase =:= t_publish_last_will_testament_banned_client_connecting; TestCase =:= t_publish_deny_disconnect_publishes_last_will_testament -> {ok, _} = emqx_authz:update(?CMD_REPLACE, []), @@ -76,11 +79,15 @@ init_per_testcase(_, Config) -> end_per_testcase(TestCase, _Config) when TestCase =:= t_subscribe_deny_disconnect_publishes_last_will_testament; + TestCase =:= t_publish_last_will_testament_banned_client_connecting; TestCase =:= t_publish_deny_disconnect_publishes_last_will_testament -> {ok, _} = emqx:update_config([authorization, deny_action], ignore), + {ok, _} = emqx_authz:update(?CMD_REPLACE, []), + emqx_common_test_helpers:call_janitor(), ok; end_per_testcase(_TestCase, _Config) -> + emqx_common_test_helpers:call_janitor(), ok. set_special_configs(emqx_authz) -> @@ -396,5 +403,63 @@ t_publish_last_will_testament_denied_topic(_Config) -> ok. +%% client is allowed by ACL to publish to its LWT topic, is connected, +%% and then gets banned and kicked out while connected. Should not +%% publish LWT. +t_publish_last_will_testament_banned_client_connecting(_Config) -> + {ok, _} = emqx_authz:update(?CMD_REPLACE, [?SOURCE7]), + Username = <<"some_client">>, + ClientId = <<"some_clientid">>, + LWTPayload = <<"should not be published">>, + LWTTopic = <<"some_client/lwt">>, + ok = emqx:subscribe(<<"some_client/lwt">>), + {ok, C} = emqtt:start_link([ + {clientid, ClientId}, + {username, Username}, + {will_topic, LWTTopic}, + {will_payload, LWTPayload} + ]), + ?assertMatch({ok, _}, emqtt:connect(C)), + + %% Now we ban the client while it is connected. + Now = erlang:system_time(second), + Who = {username, Username}, + emqx_banned:create(#{ + who => Who, + by => <<"test">>, + reason => <<"test">>, + at => Now, + until => Now + 120 + }), + on_exit(fun() -> emqx_banned:delete(Who) end), + %% Now kick it as we do in the ban API. + process_flag(trap_exit, true), + ?check_trace( + begin + ok = emqx_cm:kick_session(ClientId), + receive + {deliver, LWTTopic, #message{payload = LWTPayload}} -> + error(lwt_should_not_be_published_to_forbidden_topic) + after 2_000 -> ok + end, + ok + end, + fun(Trace) -> + ?assertMatch( + [ + #{ + client_banned := true, + publishing_disallowed := false + } + ], + ?of_kind(last_will_testament_publish_denied, Trace) + ), + ok + end + ), + ok = snabbkaffe:stop(), + + ok. + stop_apps(Apps) -> lists:foreach(fun application:stop/1, Apps). diff --git a/changes/ce/fix-10209.en.md b/changes/ce/fix-10209.en.md new file mode 100644 index 000000000..21ce98e44 --- /dev/null +++ b/changes/ce/fix-10209.en.md @@ -0,0 +1,2 @@ +Fix bug where a last will testament (LWT) message could be published +when kicking out a banned client. From db97b36cd6e25ce8367ab27e80a8af931b87c917 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 23 Mar 2023 11:47:12 +0800 Subject: [PATCH 061/196] chore: update changes/ce/fix-10172.en.md Co-authored-by: Andrew Mayorov --- changes/ce/fix-10172.en.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changes/ce/fix-10172.en.md b/changes/ce/fix-10172.en.md index 8a0c5c96e..d5cec50f8 100644 --- a/changes/ce/fix-10172.en.md +++ b/changes/ce/fix-10172.en.md @@ -5,5 +5,5 @@ Fix the incorrect default ACL rule, which was: However, it should use `{re, "^dashboard$"}` to perform a regular expression match: ``` -{allow, {username, {re,"^dashboard$"}}, subscribe ,["$SYS/#"]}. +{allow, {username, {re,"^dashboard$"}}, subscribe, ["$SYS/#"]}. ``` From ac41c7e6532d366c78cb91bce815190a346948f5 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 23 Mar 2023 11:53:26 +0800 Subject: [PATCH 062/196] chore: format codes --- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl index cc0a73993..1ddc1a110 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl @@ -142,7 +142,7 @@ fields(bridges) -> desc => <<"RocketMQ Bridge Config">>, required => false } - )}, + )}, {cassandra, mk( hoconsc:map(name, ref(emqx_ee_bridge_cassa, "config")), From 9b63bdc1e01e38b3a08281a875039f26113c8aa1 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 23 Mar 2023 15:27:34 +0800 Subject: [PATCH 063/196] chore: apply review suggestions - Rename sql to cql - Add tests for `bridges_probe` API --- .../docker-compose-cassandra.yaml | 6 +- .../i18n/emqx_ee_bridge_cassa.conf | 10 +- .../src/emqx_ee_bridge_cassa.erl | 10 +- .../test/emqx_ee_bridge_cassa_SUITE.erl | 86 +++++++++--- .../src/emqx_ee_connector_cassa.erl | 125 +++++++++--------- .../test/emqx_ee_connector_cassa_SUITE.erl | 3 +- 6 files changed, 140 insertions(+), 100 deletions(-) diff --git a/.ci/docker-compose-file/docker-compose-cassandra.yaml b/.ci/docker-compose-file/docker-compose-cassandra.yaml index 393a5cac7..a54f621c1 100644 --- a/.ci/docker-compose-file/docker-compose-cassandra.yaml +++ b/.ci/docker-compose-file/docker-compose-cassandra.yaml @@ -14,9 +14,9 @@ services: CASSANDRA_RPC_ADDRESS: "0.0.0.0" volumes: - ./certs:/certs - ports: - - "9042:9042" - - "9142:9142" + #ports: + # - "9042:9042" + # - "9142:9142" command: - /bin/bash - -c diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_cassa.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_cassa.conf index b8d810413..3bbac6658 100644 --- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_cassa.conf +++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_cassa.conf @@ -16,14 +16,14 @@ will be forwarded.""" } } - sql_template { + cql_template { desc { - en: """SQL Template""" - zh: """SQL 模板""" + en: """CQL Template""" + zh: """CQL 模板""" } label { - en: "SQL Template" - zh: "SQL 模板" + en: "CQL Template" + zh: "CQL 模板" } } config_enable { diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl index bcbeb8e82..20821dc8f 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl @@ -36,7 +36,7 @@ conn_bridge_examples(Method) -> [ #{ - <<"cassa">> => #{ + <<"cassandra">> => #{ summary => <<"Cassandra Bridge">>, value => values(Method, cassandra) } @@ -54,7 +54,7 @@ values(_Method, Type) -> pool_size => 8, username => <<"root">>, password => <<"public">>, - sql => ?DEFAULT_CQL, + cql => ?DEFAULT_CQL, local_topic => <<"local/topic/#">>, resource_opts => #{ worker_pool_size => 8, @@ -77,10 +77,10 @@ roots() -> []. fields("config") -> [ {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})}, - {sql, + {cql, mk( binary(), - #{desc => ?DESC("sql_template"), default => ?DEFAULT_CQL, format => <<"sql">>} + #{desc => ?DESC("cql_template"), default => ?DEFAULT_CQL, format => <<"sql">>} )}, {local_topic, mk( @@ -102,7 +102,7 @@ fields("config") -> fields("creation_opts") -> emqx_resource_schema:fields("creation_opts_sync_only"); fields("post") -> - fields("post", cassa); + fields("post", cassandra); fields("put") -> fields("config"); fields("get") -> diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl index 666cd0caf..d040000e2 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl @@ -27,9 +27,9 @@ ");\n" "" ). --define(SQL_DROP_TABLE, "DROP TABLE mqtt_msg_test"). --define(SQL_DELETE, "TRUNCATE mqtt_msg_test"). --define(SQL_SELECT, "SELECT payload FROM mqtt_msg_test"). +-define(SQL_DROP_TABLE, "DROP TABLE mqtt.mqtt_msg_test"). +-define(SQL_DELETE, "TRUNCATE mqtt.mqtt_msg_test"). +-define(SQL_SELECT, "SELECT payload FROM mqtt.mqtt_msg_test"). % DB defaults -define(CASSA_KEYSPACE, "mqtt"). @@ -46,6 +46,20 @@ -define(CERTFILE, filename:join(?CERT_ROOT, ["client.pem"])). -define(KEYFILE, filename:join(?CERT_ROOT, ["client.key"])). +%% How to run it locally: +%% 1. Start all deps services +%% sudo docker compose -f .ci/docker-compose-file/docker-compose.yaml \ +%% -f .ci/docker-compose-file/docker-compose-cassandra.yaml \ +%% -f .ci/docker-compose-file/docker-compose-toxiproxy.yaml \ +%% up --build +%% +%% 2. Run use cases with special environment variables +%% CASSA_TCP_HOST=127.0.0.1 CASSA_TCP_PORT=19042 \ +%% CASSA_TLS_HOST=127.0.0.1 CASSA_TLS_PORT=19142 \ +%% PROXY_HOST=127.0.0.1 ./rebar3 as test ct -c -v --name ct@127.0.0.1 \ +%% --suite lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl +%% + %%------------------------------------------------------------------------------ %% CT boilerplate %%------------------------------------------------------------------------------ @@ -197,7 +211,7 @@ cassa_config(BridgeType, Config) -> " keyspace = ~p\n" " username = ~p\n" " password = ~p\n" - " sql = ~p\n" + " cql = ~p\n" " resource_opts = {\n" " request_timeout = 500ms\n" " batch_size = ~b\n" @@ -238,8 +252,8 @@ parse_and_check(ConfigString, BridgeType, Name) -> create_bridge(Config) -> BridgeType = ?config(cassa_bridge_type, Config), Name = ?config(cassa_name, Config), - PGConfig = ?config(cassa_config, Config), - emqx_bridge:create(BridgeType, Name, PGConfig). + BridgeConfig = ?config(cassa_config, Config), + emqx_bridge:create(BridgeType, Name, BridgeConfig). delete_bridge(Config) -> BridgeType = ?config(cassa_bridge_type, Config), @@ -254,6 +268,14 @@ create_bridge_http(Params) -> Error -> Error end. +bridges_probe_http(Params) -> + Path = emqx_mgmt_api_test_util:api_path(["bridges_probe"]), + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of + {ok, _} -> ok; + Error -> Error + end. + send_message(Config, Payload) -> Name = ?config(cassa_name, Config), BridgeType = ?config(cassa_bridge_type, Config), @@ -294,25 +316,33 @@ connect_direct_cassa(Config) -> % These funs connect and then stop the cassandra connection connect_and_create_table(Config) -> - Con = connect_direct_cassa(Config), - {ok, _} = ecql:query(Con, ?SQL_CREATE_TABLE), - ok = ecql:close(Con). + with_direct_conn(Config, fun(Conn) -> + {ok, _} = ecql:query(Conn, ?SQL_CREATE_TABLE) + end). connect_and_drop_table(Config) -> - Con = connect_direct_cassa(Config), - {ok, _} = ecql:query(Con, ?SQL_DROP_TABLE), - ok = ecql:close(Con). + with_direct_conn(Config, fun(Conn) -> + {ok, _} = ecql:query(Conn, ?SQL_DROP_TABLE) + end). connect_and_clear_table(Config) -> - Con = connect_direct_cassa(Config), - ok = ecql:query(Con, ?SQL_DELETE), - ok = ecql:close(Con). + with_direct_conn(Config, fun(Conn) -> + ok = ecql:query(Conn, ?SQL_DELETE) + end). connect_and_get_payload(Config) -> - Con = connect_direct_cassa(Config), - {ok, {_Keyspace, _ColsSpec, [[Result]]}} = ecql:query(Con, ?SQL_SELECT), - ok = ecql:close(Con), - Result. + with_direct_conn(Config, fun(Conn) -> + {ok, {_Keyspace, _ColsSpec, [[Result]]}} = ecql:query(Conn, ?SQL_SELECT), + Result + end). + +with_direct_conn(Config, Fn) -> + Conn = connect_direct_cassa(Config), + try + Fn(Conn) + after + ok = ecql:close(Conn) + end. %%------------------------------------------------------------------------------ %% Testcases @@ -358,14 +388,14 @@ t_setup_via_config_and_publish(Config) -> t_setup_via_http_api_and_publish(Config) -> BridgeType = ?config(cassa_bridge_type, Config), Name = ?config(cassa_name, Config), - PgsqlConfig0 = ?config(cassa_config, Config), - PgsqlConfig = PgsqlConfig0#{ + BridgeConfig0 = ?config(cassa_config, Config), + BridgeConfig = BridgeConfig0#{ <<"name">> => Name, <<"type">> => BridgeType }, ?assertMatch( {ok, _}, - create_bridge_http(PgsqlConfig) + create_bridge_http(BridgeConfig) ), Val = integer_to_binary(erlang:unique_integer()), SentData = #{ @@ -421,6 +451,18 @@ t_get_status(Config) -> end), ok. +t_bridges_probe_via_http(Config) -> + BridgeType = ?config(cassa_bridge_type, Config), + Name = ?config(cassa_name, Config), + BridgeConfig0 = ?config(cassa_config, Config), + BridgeConfig = BridgeConfig0#{ + <<"name">> => Name, + <<"type">> => BridgeType + }, + ?assertMatch(ok, bridges_probe_http(BridgeConfig)), + + ok. + t_create_disconnected(Config) -> ProxyPort = ?config(proxy_port, Config), ProxyHost = ?config(proxy_host, Config), diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl index cf25cd6d8..cdece50f7 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl @@ -33,7 +33,7 @@ on_start/2, on_stop/2, on_query/3, - %% TODO: now_supported_now + %% TODO: not_supported_now %%on_batch_query/3, on_get_status/2 ]). @@ -41,7 +41,7 @@ %% callbacks of ecpool -export([ connect/1, - prepare_sql_to_conn/2 + prepare_cql_to_conn/2 ]). %% callbacks for query executing @@ -55,7 +55,7 @@ -type state() :: #{ poolname := atom(), - prepare_sql := prepares(), + prepare_cql := prepares(), params_tokens := params_tokens(), %% returned by ecql:prepare/2 prepare_statement := binary() @@ -109,9 +109,6 @@ on_start( ssl := SSL } = Config ) -> - {ok, _} = application:ensure_all_started(ecpool), - {ok, _} = application:ensure_all_started(ecql), - ?SLOG(info, #{ msg => "starting_cassandra_connector", connector => InstId, @@ -139,7 +136,7 @@ on_start( end, PoolName = emqx_plugin_libs_pool:pool_name(InstId), - Prepares = parse_prepare_sql(Config), + Prepares = parse_prepare_cql(Config), InitState = #{poolname => PoolName, prepare_statement => #{}}, State = maps:merge(InitState, Prepares), case emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Options ++ SslOpts) of @@ -177,30 +174,30 @@ on_query( Request, #{poolname := PoolName} = State ) -> - {Type, PreparedKeyOrSQL, Params} = parse_request_to_sql(Request), + {Type, PreparedKeyOrSQL, Params} = parse_request_to_cql(Request), ?tp( debug, - cassandra_connector_received_sql_query, + cassandra_connector_received_cql_query, #{ connector => InstId, type => Type, params => Params, - prepared_key_or_sql => PreparedKeyOrSQL, + prepared_key_or_cql => PreparedKeyOrSQL, state => State } ), - {PreparedKeyOrSQL1, Data} = proc_sql_params(Type, PreparedKeyOrSQL, Params, State), - Res = exec_sql_query(InstId, PoolName, Type, PreparedKeyOrSQL1, Data), + {PreparedKeyOrSQL1, Data} = proc_cql_params(Type, PreparedKeyOrSQL, Params, State), + Res = exec_cql_query(InstId, PoolName, Type, PreparedKeyOrSQL1, Data), handle_result(Res). -parse_request_to_sql({send_message, Params}) -> +parse_request_to_cql({send_message, Params}) -> {prepared_query, _Key = send_message, Params}; -parse_request_to_sql({query, SQL}) -> - parse_request_to_sql({query, SQL, #{}}); -parse_request_to_sql({query, SQL, Params}) -> +parse_request_to_cql({query, SQL}) -> + parse_request_to_cql({query, SQL, #{}}); +parse_request_to_cql({query, SQL, Params}) -> {query, SQL, Params}. -proc_sql_params( +proc_cql_params( prepared_query, PreparedKey0, Params, @@ -209,11 +206,11 @@ proc_sql_params( PreparedKey = maps:get(PreparedKey0, Prepares), Tokens = maps:get(PreparedKey0, ParamsTokens), {PreparedKey, assign_type_for_params(emqx_plugin_libs_rule:proc_sql(Tokens, Params))}; -proc_sql_params(query, SQL, Params, _State) -> +proc_cql_params(query, SQL, Params, _State) -> {SQL1, Tokens} = emqx_plugin_libs_rule:preproc_sql(SQL, '?'), {SQL1, assign_type_for_params(emqx_plugin_libs_rule:proc_sql(Tokens, Params))}. -exec_sql_query(InstId, PoolName, Type, PreparedKey, Data) when +exec_cql_query(InstId, PoolName, Type, PreparedKey, Data) when Type == query; Type == prepared_query -> case ecpool:pick_and_do(PoolName, {?MODULE, Type, [PreparedKey, Data]}, no_handover) of @@ -239,7 +236,7 @@ on_get_status(_InstId, #{poolname := Pool} = State) -> %% return new state with prepared statements {connected, NState}; false -> - %% do not log error, it is logged in prepare_sql_to_conn + %% do not log error, it is logged in prepare_cql_to_conn connecting end; false -> @@ -249,14 +246,14 @@ on_get_status(_InstId, #{poolname := Pool} = State) -> do_get_status(Conn) -> ok == element(1, ecql:query(Conn, "SELECT count(1) AS T FROM system.local")). -do_check_prepares(#{prepare_sql := Prepares}) when is_map(Prepares) -> +do_check_prepares(#{prepare_cql := Prepares}) when is_map(Prepares) -> ok; -do_check_prepares(State = #{poolname := PoolName, prepare_sql := {error, Prepares}}) -> +do_check_prepares(State = #{poolname := PoolName, prepare_cql := {error, Prepares}}) -> %% retry to prepare - case prepare_sql(Prepares, PoolName) of + case prepare_cql(Prepares, PoolName) of {ok, Sts} -> %% remove the error - {ok, State#{prepare_sql => Prepares, prepare_statement := Sts}}; + {ok, State#{prepare_cql => Prepares, prepare_statement := Sts}}; _Error -> false end. @@ -295,83 +292,83 @@ conn_opts([Opt | Opts], Acc) -> %% prepare %% XXX: hardcode -%% note: the `sql` param is passed by emqx_ee_bridge_cassa -parse_prepare_sql(#{sql := SQL}) -> - parse_prepare_sql([{send_message, SQL}], #{}, #{}); -parse_prepare_sql(_) -> - #{prepare_sql => #{}, params_tokens => #{}}. +%% note: the `cql` param is passed by emqx_ee_bridge_cassa +parse_prepare_cql(#{cql := SQL}) -> + parse_prepare_cql([{send_message, SQL}], #{}, #{}); +parse_prepare_cql(_) -> + #{prepare_cql => #{}, params_tokens => #{}}. -parse_prepare_sql([{Key, H} | T], Prepares, Tokens) -> +parse_prepare_cql([{Key, H} | T], Prepares, Tokens) -> {PrepareSQL, ParamsTokens} = emqx_plugin_libs_rule:preproc_sql(H, '?'), - parse_prepare_sql( + parse_prepare_cql( T, Prepares#{Key => PrepareSQL}, Tokens#{Key => ParamsTokens} ); -parse_prepare_sql([], Prepares, Tokens) -> +parse_prepare_cql([], Prepares, Tokens) -> #{ - prepare_sql => Prepares, + prepare_cql => Prepares, params_tokens => Tokens }. -init_prepare(State = #{prepare_sql := Prepares, poolname := PoolName}) -> +init_prepare(State = #{prepare_cql := Prepares, poolname := PoolName}) -> case maps:size(Prepares) of 0 -> State; _ -> - case prepare_sql(Prepares, PoolName) of + case prepare_cql(Prepares, PoolName) of {ok, Sts} -> State#{prepare_statement := Sts}; Error -> ?tp( error, - cassandra_prepare_sql_failed, + cassandra_prepare_cql_failed, #{prepares => Prepares, reason => Error} ), - %% mark the prepare_sqlas failed - State#{prepare_sql => {error, Prepares}} + %% mark the prepare_cql as failed + State#{prepare_cql => {error, Prepares}} end end. -prepare_sql(Prepares, PoolName) when is_map(Prepares) -> - prepare_sql(maps:to_list(Prepares), PoolName); -prepare_sql(Prepares, PoolName) -> - case do_prepare_sql(Prepares, PoolName) of +prepare_cql(Prepares, PoolName) when is_map(Prepares) -> + prepare_cql(maps:to_list(Prepares), PoolName); +prepare_cql(Prepares, PoolName) -> + case do_prepare_cql(Prepares, PoolName) of {ok, _Sts} = Ok -> %% prepare for reconnect - ecpool:add_reconnect_callback(PoolName, {?MODULE, prepare_sql_to_conn, [Prepares]}), + ecpool:add_reconnect_callback(PoolName, {?MODULE, prepare_cql_to_conn, [Prepares]}), Ok; Error -> Error end. -do_prepare_sql(Prepares, PoolName) -> - do_prepare_sql(ecpool:workers(PoolName), Prepares, PoolName, #{}). +do_prepare_cql(Prepares, PoolName) -> + do_prepare_cql(ecpool:workers(PoolName), Prepares, PoolName, #{}). -do_prepare_sql([{_Name, Worker} | T], Prepares, PoolName, _LastSts) -> +do_prepare_cql([{_Name, Worker} | T], Prepares, PoolName, _LastSts) -> {ok, Conn} = ecpool_worker:client(Worker), - case prepare_sql_to_conn(Conn, Prepares) of + case prepare_cql_to_conn(Conn, Prepares) of {ok, Sts} -> - do_prepare_sql(T, Prepares, PoolName, Sts); + do_prepare_cql(T, Prepares, PoolName, Sts); Error -> Error end; -do_prepare_sql([], _Prepares, _PoolName, LastSts) -> +do_prepare_cql([], _Prepares, _PoolName, LastSts) -> {ok, LastSts}. -prepare_sql_to_conn(Conn, Prepares) -> - prepare_sql_to_conn(Conn, Prepares, #{}). +prepare_cql_to_conn(Conn, Prepares) -> + prepare_cql_to_conn(Conn, Prepares, #{}). -prepare_sql_to_conn(Conn, [], Statements) when is_pid(Conn) -> {ok, Statements}; -prepare_sql_to_conn(Conn, [{Key, SQL} | PrepareList], Statements) when is_pid(Conn) -> - ?SLOG(info, #{msg => "cassandra_prepare_sql", name => Key, prepare_sql => SQL}), +prepare_cql_to_conn(Conn, [], Statements) when is_pid(Conn) -> {ok, Statements}; +prepare_cql_to_conn(Conn, [{Key, SQL} | PrepareList], Statements) when is_pid(Conn) -> + ?SLOG(info, #{msg => "cassandra_prepare_cql", name => Key, prepare_cql => SQL}), case ecql:prepare(Conn, Key, SQL) of {ok, Statement} -> - prepare_sql_to_conn(Conn, PrepareList, Statements#{Key => Statement}); + prepare_cql_to_conn(Conn, PrepareList, Statements#{Key => Statement}); {error, Error} = Other -> ?SLOG(error, #{ - msg => "cassandra_prepare_sql_failed", + msg => "cassandra_prepare_cql_failed", worker_pid => Conn, name => Key, - prepare_sql => SQL, + prepare_cql => SQL, error => Error }), Other @@ -394,19 +391,19 @@ assign_type_for_params(Params) -> assign_type_for_params([], Acc) -> lists:reverse(Acc); assign_type_for_params([Param | More], Acc) -> - assign_type_for_params(More, [may_assign_type(Param) | Acc]). + assign_type_for_params(More, [maybe_assign_type(Param) | Acc]). -may_assign_type(true) -> +maybe_assign_type(true) -> {int, 1}; -may_assign_type(false) -> +maybe_assign_type(false) -> {int, 0}; -may_assign_type(V) when is_binary(V); is_list(V); is_atom(V) -> V; -may_assign_type(V) when is_integer(V) -> +maybe_assign_type(V) when is_binary(V); is_list(V); is_atom(V) -> V; +maybe_assign_type(V) when is_integer(V) -> %% The max value of signed int(4) is 2147483647 case V > 2147483647 orelse V < -2147483647 of true -> {bigint, V}; false -> {int, V} end; -may_assign_type(V) when is_float(V) -> {double, V}; -may_assign_type(V) -> +maybe_assign_type(V) when is_float(V) -> {double, V}; +maybe_assign_type(V) -> V. diff --git a/lib-ee/emqx_ee_connector/test/emqx_ee_connector_cassa_SUITE.erl b/lib-ee/emqx_ee_connector/test/emqx_ee_connector_cassa_SUITE.erl index 81b9e3859..95b4407cf 100644 --- a/lib-ee/emqx_ee_connector/test/emqx_ee_connector_cassa_SUITE.erl +++ b/lib-ee/emqx_ee_connector/test/emqx_ee_connector_cassa_SUITE.erl @@ -85,7 +85,8 @@ init_per_suite(Config) -> end_per_suite(_Config) -> ok = emqx_common_test_helpers:stop_apps([emqx_conf]), ok = emqx_connector_test_helpers:stop_apps([emqx_resource]), - _ = application:stop(emqx_connector). + _ = application:stop(emqx_connector), + _ = application:stop(emqx_ee_connector). init_per_testcase(_, Config) -> Config. From 1a466fda0b1de59dff06a559c93e244fefa2e86d Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 23 Mar 2023 16:03:25 +0800 Subject: [PATCH 064/196] chore: use binary type InstanceId as PoolName --- lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl index cdece50f7..a6a77f233 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl @@ -134,8 +134,8 @@ on_start( false -> [] end, - - PoolName = emqx_plugin_libs_pool:pool_name(InstId), + %% use InstaId of binary type as Pool name, which is supported in ecpool. + PoolName = InstId, Prepares = parse_prepare_cql(Config), InitState = #{poolname => PoolName, prepare_statement => #{}}, State = maps:merge(InitState, Prepares), From 761847b74aca24f78fe16f10a75c40e154153047 Mon Sep 17 00:00:00 2001 From: Serge Tupchii Date: Wed, 22 Mar 2023 15:09:13 +0200 Subject: [PATCH 065/196] ci: update emqx-builders with OTP 24.3.4.2-3 and 25.1.2-3 These OTP versions implement mnesia_hook:unregister_hook/1 --- .../docker-compose-kafka.yaml | 2 +- .ci/docker-compose-file/docker-compose.yaml | 2 +- .../workflows/build_and_push_docker_images.yaml | 6 +++--- .github/workflows/build_packages.yaml | 16 ++++++++-------- .github/workflows/build_slim_packages.yaml | 12 ++++++------ .github/workflows/check_deps_integrity.yaml | 2 +- .github/workflows/code_style_check.yaml | 2 +- .github/workflows/elixir_apps_check.yaml | 2 +- .github/workflows/elixir_deps_check.yaml | 2 +- .github/workflows/elixir_release.yml | 2 +- .github/workflows/release.yaml | 2 +- .github/workflows/run_emqx_app_tests.yaml | 6 +++--- .github/workflows/run_fvt_tests.yaml | 10 +++++----- .github/workflows/run_relup_tests.yaml | 2 +- .github/workflows/run_test_cases.yaml | 12 ++++++------ .tool-versions | 2 +- deploy/docker/Dockerfile | 2 +- scripts/relup-test/run-relup-lux.sh | 4 ++-- scripts/relup-test/start-relup-test-cluster.sh | 2 +- 19 files changed, 45 insertions(+), 45 deletions(-) diff --git a/.ci/docker-compose-file/docker-compose-kafka.yaml b/.ci/docker-compose-file/docker-compose-kafka.yaml index d4989bd0b..bbfb4080a 100644 --- a/.ci/docker-compose-file/docker-compose-kafka.yaml +++ b/.ci/docker-compose-file/docker-compose-kafka.yaml @@ -18,7 +18,7 @@ services: - /tmp/emqx-ci/emqx-shared-secret:/var/lib/secret kdc: hostname: kdc.emqx.net - image: ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2-ubuntu20.04 + image: ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-24.3.4.2-3-ubuntu20.04 container_name: kdc.emqx.net expose: - 88 # kdc diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml index 5c37d971e..2d1c553e9 100644 --- a/.ci/docker-compose-file/docker-compose.yaml +++ b/.ci/docker-compose-file/docker-compose.yaml @@ -3,7 +3,7 @@ version: '3.9' services: erlang: container_name: erlang - image: ${DOCKER_CT_RUNNER_IMAGE:-ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2-ubuntu20.04} + image: ${DOCKER_CT_RUNNER_IMAGE:-ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-24.3.4.2-3-ubuntu20.04} env_file: - conf.env environment: diff --git a/.github/workflows/build_and_push_docker_images.yaml b/.github/workflows/build_and_push_docker_images.yaml index adf2c2b84..7391adb5c 100644 --- a/.github/workflows/build_and_push_docker_images.yaml +++ b/.github/workflows/build_and_push_docker_images.yaml @@ -25,7 +25,7 @@ jobs: prepare: runs-on: ubuntu-22.04 # prepare source with any OTP version, no need for a matrix - container: "ghcr.io/emqx/emqx-builder/5.0-32:1.13.4-24.3.4.2-2-ubuntu22.04" + container: "ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-24.3.4.2-3-ubuntu22.04" outputs: PROFILE: ${{ steps.get_profile.outputs.PROFILE }} @@ -121,9 +121,9 @@ jobs: # NOTE: 'otp' and 'elixir' are to configure emqx-builder image # only support latest otp and elixir, not a matrix builder: - - 5.0-32 # update to latest + - 5.0-33 # update to latest otp: - - 24.3.4.2-2 # switch to 25 once ready to release 5.1 + - 24.3.4.2-3 # switch to 25 once ready to release 5.1 elixir: - 'no_elixir' - '1.13.4' # update to latest diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 3141b77d5..2afe23f67 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -24,7 +24,7 @@ jobs: prepare: runs-on: ubuntu-22.04 if: (github.repository_owner == 'emqx' && github.event_name == 'schedule') || github.event_name != 'schedule' - container: ghcr.io/emqx/emqx-builder/5.0-32:1.13.4-24.3.4.2-2-ubuntu22.04 + container: ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-24.3.4.2-3-ubuntu22.04 outputs: BUILD_PROFILE: ${{ steps.get_profile.outputs.BUILD_PROFILE }} IS_EXACT_TAG: ${{ steps.get_profile.outputs.IS_EXACT_TAG }} @@ -151,7 +151,7 @@ jobs: profile: - ${{ needs.prepare.outputs.BUILD_PROFILE }} otp: - - 24.3.4.2-2 + - 24.3.4.2-3 os: - macos-11 - macos-12 @@ -203,7 +203,7 @@ jobs: profile: - ${{ needs.prepare.outputs.BUILD_PROFILE }} otp: - - 24.3.4.2-2 + - 24.3.4.2-3 arch: - amd64 - arm64 @@ -221,7 +221,7 @@ jobs: - aws-arm64 - ubuntu-22.04 builder: - - 5.0-32 + - 5.0-33 elixir: - 1.13.4 exclude: @@ -231,19 +231,19 @@ jobs: build_machine: aws-arm64 include: - profile: emqx - otp: 25.1.2-2 + otp: 25.1.2-3 arch: amd64 os: ubuntu22.04 build_machine: ubuntu-22.04 - builder: 5.0-32 + builder: 5.0-33 elixir: 1.13.4 release_with: elixir - profile: emqx - otp: 25.1.2-2 + otp: 25.1.2-3 arch: amd64 os: amzn2 build_machine: ubuntu-22.04 - builder: 5.0-32 + builder: 5.0-33 elixir: 1.13.4 release_with: elixir diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 163956790..30af8bdbc 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -30,12 +30,12 @@ jobs: fail-fast: false matrix: profile: - - ["emqx", "24.3.4.2-2", "el7", "erlang"] - - ["emqx", "25.1.2-2", "ubuntu22.04", "elixir"] - - ["emqx-enterprise", "24.3.4.2-2", "amzn2", "erlang"] - - ["emqx-enterprise", "25.1.2-2", "ubuntu20.04", "erlang"] + - ["emqx", "24.3.4.2-3", "el7", "erlang"] + - ["emqx", "25.1.2-3", "ubuntu22.04", "elixir"] + - ["emqx-enterprise", "24.3.4.2-3", "amzn2", "erlang"] + - ["emqx-enterprise", "25.1.2-3", "ubuntu20.04", "erlang"] builder: - - 5.0-32 + - 5.0-33 elixir: - '1.13.4' @@ -132,7 +132,7 @@ jobs: - emqx - emqx-enterprise otp: - - 24.3.4.2-2 + - 24.3.4.2-3 os: - macos-11 - macos-12-arm64 diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml index 58dd06e30..62dfa24ef 100644 --- a/.github/workflows/check_deps_integrity.yaml +++ b/.github/workflows/check_deps_integrity.yaml @@ -6,7 +6,7 @@ on: jobs: check_deps_integrity: runs-on: ubuntu-22.04 - container: ghcr.io/emqx/emqx-builder/5.0-32:1.13.4-25.1.2-2-ubuntu22.04 + container: ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-25.1.2-3-ubuntu22.04 steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/code_style_check.yaml b/.github/workflows/code_style_check.yaml index de05f7e59..97c6b0c88 100644 --- a/.github/workflows/code_style_check.yaml +++ b/.github/workflows/code_style_check.yaml @@ -5,7 +5,7 @@ on: [pull_request] jobs: code_style_check: runs-on: ubuntu-22.04 - container: "ghcr.io/emqx/emqx-builder/5.0-32:1.13.4-25.1.2-2-ubuntu22.04" + container: "ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-25.1.2-3-ubuntu22.04" steps: - uses: actions/checkout@v3 with: diff --git a/.github/workflows/elixir_apps_check.yaml b/.github/workflows/elixir_apps_check.yaml index 181e81305..247f67a8f 100644 --- a/.github/workflows/elixir_apps_check.yaml +++ b/.github/workflows/elixir_apps_check.yaml @@ -9,7 +9,7 @@ jobs: elixir_apps_check: runs-on: ubuntu-22.04 # just use the latest builder - container: "ghcr.io/emqx/emqx-builder/5.0-32:1.13.4-25.1.2-2-ubuntu22.04" + container: "ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-25.1.2-3-ubuntu22.04" strategy: fail-fast: false diff --git a/.github/workflows/elixir_deps_check.yaml b/.github/workflows/elixir_deps_check.yaml index d753693cc..511639a3c 100644 --- a/.github/workflows/elixir_deps_check.yaml +++ b/.github/workflows/elixir_deps_check.yaml @@ -8,7 +8,7 @@ on: jobs: elixir_deps_check: runs-on: ubuntu-22.04 - container: ghcr.io/emqx/emqx-builder/5.0-32:1.13.4-25.1.2-2-ubuntu22.04 + container: ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-25.1.2-3-ubuntu22.04 steps: - name: Checkout diff --git a/.github/workflows/elixir_release.yml b/.github/workflows/elixir_release.yml index 1647071af..7bd6102ff 100644 --- a/.github/workflows/elixir_release.yml +++ b/.github/workflows/elixir_release.yml @@ -17,7 +17,7 @@ jobs: profile: - emqx - emqx-enterprise - container: ghcr.io/emqx/emqx-builder/5.0-32:1.13.4-25.1.2-2-ubuntu22.04 + container: ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-25.1.2-3-ubuntu22.04 steps: - name: Checkout uses: actions/checkout@v3 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 2f5ddf171..32a45bd51 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -54,7 +54,7 @@ jobs: OUTPUT_DIR=${{ steps.profile.outputs.s3dir }} aws s3 cp --recursive s3://$BUCKET/$OUTPUT_DIR/${{ github.ref_name }} packages cd packages - DEFAULT_BEAM_PLATFORM='otp24.3.4.2-2' + DEFAULT_BEAM_PLATFORM='otp24.3.4.2-3' # all packages including full-name and default-name are uploaded to s3 # but we only upload default-name packages (and elixir) as github artifacts # so we rename (overwrite) non-default packages before uploading diff --git a/.github/workflows/run_emqx_app_tests.yaml b/.github/workflows/run_emqx_app_tests.yaml index 52ba13373..0a15f6c0b 100644 --- a/.github/workflows/run_emqx_app_tests.yaml +++ b/.github/workflows/run_emqx_app_tests.yaml @@ -12,10 +12,10 @@ jobs: strategy: matrix: builder: - - 5.0-32 + - 5.0-33 otp: - - 24.3.4.2-2 - - 25.1.2-2 + - 24.3.4.2-3 + - 25.1.2-3 # no need to use more than 1 version of Elixir, since tests # run using only Erlang code. This is needed just to specify # the base image. diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index f729c8cbd..bb5aa4a1a 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -17,7 +17,7 @@ jobs: prepare: runs-on: ubuntu-22.04 # prepare source with any OTP version, no need for a matrix - container: ghcr.io/emqx/emqx-builder/5.0-32:1.13.4-24.3.4.2-2-debian11 + container: ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-24.3.4.2-3-debian11 steps: - uses: actions/checkout@v3 @@ -50,9 +50,9 @@ jobs: os: - ["debian11", "debian:11-slim"] builder: - - 5.0-32 + - 5.0-33 otp: - - 24.3.4.2-2 + - 24.3.4.2-3 elixir: - 1.13.4 arch: @@ -123,9 +123,9 @@ jobs: os: - ["debian11", "debian:11-slim"] builder: - - 5.0-32 + - 5.0-33 otp: - - 24.3.4.2-2 + - 24.3.4.2-3 elixir: - 1.13.4 arch: diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml index cd969045d..8727f4d9d 100644 --- a/.github/workflows/run_relup_tests.yaml +++ b/.github/workflows/run_relup_tests.yaml @@ -15,7 +15,7 @@ concurrency: jobs: relup_test_plan: runs-on: ubuntu-22.04 - container: "ghcr.io/emqx/emqx-builder/5.0-32:1.13.4-24.3.4.2-2-ubuntu22.04" + container: "ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-24.3.4.2-3-ubuntu22.04" outputs: CUR_EE_VSN: ${{ steps.find-versions.outputs.CUR_EE_VSN }} OLD_VERSIONS: ${{ steps.find-versions.outputs.OLD_VERSIONS }} diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 1efe7a4e7..8702cd849 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -31,13 +31,13 @@ jobs: MATRIX="$(echo "${APPS}" | jq -c ' [ (.[] | select(.profile == "emqx") | . + { - builder: "5.0-32", - otp: "25.1.2-2", + builder: "5.0-33", + otp: "25.1.2-3", elixir: "1.13.4" }), (.[] | select(.profile == "emqx-enterprise") | . + { - builder: "5.0-32", - otp: ["24.3.4.2-2", "25.1.2-2"][], + builder: "5.0-33", + otp: ["24.3.4.2-3", "25.1.2-3"][], elixir: "1.13.4" }) ] @@ -230,12 +230,12 @@ jobs: - ct - ct_docker runs-on: ubuntu-22.04 - container: "ghcr.io/emqx/emqx-builder/5.0-32:1.13.4-24.3.4.2-2-ubuntu22.04" + container: "ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-24.3.4.2-3-ubuntu22.04" steps: - uses: AutoModality/action-clean@v1 - uses: actions/download-artifact@v3 with: - name: source-emqx-enterprise-24.3.4.2-2 + name: source-emqx-enterprise-24.3.4.2-3 path: . - name: unzip source code run: unzip -q source.zip diff --git a/.tool-versions b/.tool-versions index dcf5945a8..b4d8f8675 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,2 +1,2 @@ -erlang 24.3.4.2-2 +erlang 24.3.4.2-3 elixir 1.13.4-otp-24 diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index f26926bce..308c26231 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -1,4 +1,4 @@ -ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2-debian11 +ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-24.3.4.2-3-debian11 ARG RUN_FROM=debian:11-slim FROM ${BUILD_FROM} AS builder diff --git a/scripts/relup-test/run-relup-lux.sh b/scripts/relup-test/run-relup-lux.sh index 570e58340..674eadc45 100755 --- a/scripts/relup-test/run-relup-lux.sh +++ b/scripts/relup-test/run-relup-lux.sh @@ -45,8 +45,8 @@ fi # From now on, no need for the v|e prefix OLD_VSN="${old_vsn#[e|v]}" -OLD_PKG="$(pwd)/_upgrade_base/${profile}-${OLD_VSN}-otp24.3.4.2-2-ubuntu20.04-amd64.tar.gz" -CUR_PKG="$(pwd)/_packages/${profile}/${profile}-${cur_vsn}-otp24.3.4.2-2-ubuntu20.04-amd64.tar.gz" +OLD_PKG="$(pwd)/_upgrade_base/${profile}-${OLD_VSN}-otp24.3.4.2-3-ubuntu20.04-amd64.tar.gz" +CUR_PKG="$(pwd)/_packages/${profile}/${profile}-${cur_vsn}-otp24.3.4.2-3-ubuntu20.04-amd64.tar.gz" if [ ! -f "$OLD_PKG" ]; then echo "$OLD_PKG not found" diff --git a/scripts/relup-test/start-relup-test-cluster.sh b/scripts/relup-test/start-relup-test-cluster.sh index 385137dc7..d22c61680 100755 --- a/scripts/relup-test/start-relup-test-cluster.sh +++ b/scripts/relup-test/start-relup-test-cluster.sh @@ -22,7 +22,7 @@ WEBHOOK="webhook.$NET" BENCH="bench.$NET" COOKIE='this-is-a-secret' ## Erlang image is needed to run webhook server and emqtt-bench -ERLANG_IMAGE="ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2-ubuntu20.04" +ERLANG_IMAGE="ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-24.3.4.2-3-ubuntu20.04" # builder has emqtt-bench installed BENCH_IMAGE="$ERLANG_IMAGE" From 50bbf4f231433d044569568cb8508f6d4c3da000 Mon Sep 17 00:00:00 2001 From: Serge Tupchii Date: Wed, 22 Mar 2023 19:20:27 +0200 Subject: [PATCH 066/196] chore: bump ekka to 0.14.6 ekka 0.14.6 uses mria 0.4.2, which unregisters mnesia_hook upon stopping mria. This fixes mnesia_hook errors on restarting mria. Closes: EMQX-9101 --- apps/emqx/rebar.config | 2 +- changes/ce/feat-10210.en.md | 4 ++++ mix.exs | 2 +- rebar.config | 2 +- 4 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 changes/ce/feat-10210.en.md diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 6875f4003..4c8e2569f 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -27,7 +27,7 @@ {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}, {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}}, {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}}, - {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.14.5"}}}, + {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.14.6"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}, {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.37.0"}}}, {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}}, diff --git a/changes/ce/feat-10210.en.md b/changes/ce/feat-10210.en.md new file mode 100644 index 000000000..2894ee44e --- /dev/null +++ b/changes/ce/feat-10210.en.md @@ -0,0 +1,4 @@ +Unregister Mnesia post commit hook when Mria is being stopped. +This fixes hook failures occasionally occurring on stopping/restarting Mria. + +[Mria PR](https://github.com/emqx/mria/pull/133) diff --git a/mix.exs b/mix.exs index a06bbcfa8..66268e13c 100644 --- a/mix.exs +++ b/mix.exs @@ -55,7 +55,7 @@ defmodule EMQXUmbrella.MixProject do {:cowboy, github: "emqx/cowboy", tag: "2.9.0", override: true}, {:esockd, github: "emqx/esockd", tag: "5.9.6", override: true}, {:rocksdb, github: "emqx/erlang-rocksdb", tag: "1.7.2-emqx-9", override: true}, - {:ekka, github: "emqx/ekka", tag: "0.14.5", override: true}, + {:ekka, github: "emqx/ekka", tag: "0.14.6", override: true}, {:gen_rpc, github: "emqx/gen_rpc", tag: "2.8.1", override: true}, {:grpc, github: "emqx/grpc-erl", tag: "0.6.7", override: true}, {:minirest, github: "emqx/minirest", tag: "1.3.8", override: true}, diff --git a/rebar.config b/rebar.config index f71da084e..9e874e109 100644 --- a/rebar.config +++ b/rebar.config @@ -62,7 +62,7 @@ , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}} , {rocksdb, {git, "https://github.com/emqx/erlang-rocksdb", {tag, "1.7.2-emqx-9"}}} - , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.14.5"}}} + , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.14.6"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}} , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.7"}}} , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.8"}}} From e3541d89479e82d008d2143082281e9f033aa7ce Mon Sep 17 00:00:00 2001 From: Serge Tupchii Date: Wed, 22 Mar 2023 19:25:13 +0200 Subject: [PATCH 067/196] test: move dashboard specific cleanup call from emqx_common_test_helpers to the relevant test suite --- apps/emqx/test/emqx_common_test_helpers.erl | 1 - apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl | 9 +-------- .../emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl | 2 ++ 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index 658b22c56..d08812075 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -202,7 +202,6 @@ start_apps(Apps, SpecAppConfig, Opts) when is_function(SpecAppConfig) -> %% Because, minirest, ekka etc.. application will scan these modules lists:foreach(fun load/1, [emqx | Apps]), ok = start_ekka(), - mnesia:clear_table(emqx_admin), ok = emqx_ratelimiter_SUITE:load_conf(), lists:foreach(fun(App) -> start_app(App, SpecAppConfig, Opts) end, [emqx | Apps]). diff --git a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl index 906d57e9d..18393a40e 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl @@ -56,14 +56,6 @@ all() -> emqx_common_test_helpers:all(?MODULE). -end_suite() -> - end_suite([]). - -end_suite(Apps) -> - application:unload(emqx_management), - mnesia:clear_table(?ADMIN), - emqx_common_test_helpers:stop_apps(Apps ++ [emqx_dashboard]). - init_per_suite(Config) -> emqx_common_test_helpers:start_apps( [emqx_management, emqx_dashboard], @@ -72,6 +64,7 @@ init_per_suite(Config) -> Config. end_per_suite(_Config) -> + mnesia:clear_table(?ADMIN), emqx_common_test_helpers:stop_apps([emqx_dashboard, emqx_management]), mria:stop(). diff --git a/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl index bfbd9b973..fa2373ac3 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl @@ -33,10 +33,12 @@ all() -> init_per_suite(Config) -> application:load(emqx_dashboard), mria:start(), + mnesia:clear_table(?ADMIN), emqx_common_test_helpers:start_apps([emqx_dashboard], fun set_special_configs/1), Config. end_per_suite(Config) -> + mnesia:clear_table(?ADMIN), emqx_common_test_helpers:stop_apps([emqx_dashboard]), Config. From 9d3f369cca524f1d038e9bde053d1abfceeb399b Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Thu, 23 Mar 2023 14:09:57 +0100 Subject: [PATCH 068/196] docs: fix spelling mistake Co-authored-by: Thales Macedo Garitezi --- apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf index 3a6b50e83..9b6990035 100644 --- a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf +++ b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf @@ -137,7 +137,7 @@ When disabled the messages are buffered in RAM only.""" inflight_window { desc { - en: """Query inflight window. When query_mode is set to async, this config has to be et to 1 if messages from the same MQTT client have to be stricktly ordered.""" + en: """Query inflight window. When query_mode is set to async, this config has to be set to 1 if messages from the same MQTT client have to be strictly ordered.""" zh: """请求飞行队列窗口大小。当请求模式为异步时，如果需要严格保证来自同一 MQTT 客户端的消息有序，则必须将此值设为 1。""""" } label { From 35474578ca6f1909da19de4d3928e8ea62fe00f6 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Thu, 23 Mar 2023 14:19:17 +0100 Subject: [PATCH 069/196] refactor: rename async_inflight_window to inflight_window everywhere --- apps/emqx_bridge/src/emqx_bridge_api.erl | 2 +- .../src/schema/emqx_bridge_compatible_config.erl | 2 +- apps/emqx_bridge/test/emqx_bridge_webhook_SUITE.erl | 2 +- apps/emqx_resource/include/emqx_resource.hrl | 2 +- apps/emqx_resource/src/emqx_resource_buffer_worker.erl | 2 +- apps/emqx_resource/test/emqx_resource_SUITE.erl | 10 +++++----- scripts/test/influx/influx-bridge.conf | 2 +- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index 8ac3e476a..ff6842e37 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -238,7 +238,7 @@ info_example_basic(webhook) -> health_check_interval => 15000, auto_restart_interval => 15000, query_mode => async, - async_inflight_window => 100, + inflight_window => 100, max_queue_bytes => 100 * 1024 * 1024 } }; diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_compatible_config.erl b/apps/emqx_bridge/src/schema/emqx_bridge_compatible_config.erl index 1e55d0c0e..fe173fa89 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_compatible_config.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_compatible_config.erl @@ -86,7 +86,7 @@ default_ssl() -> default_resource_opts() -> #{ - <<"async_inflight_window">> => 100, + <<"inflight_window">> => 100, <<"auto_restart_interval">> => <<"60s">>, <<"health_check_interval">> => <<"15s">>, <<"max_queue_bytes">> => <<"1GB">>, diff --git a/apps/emqx_bridge/test/emqx_bridge_webhook_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_webhook_SUITE.erl index f249aa95e..e222190d2 100644 --- a/apps/emqx_bridge/test/emqx_bridge_webhook_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_webhook_SUITE.erl @@ -172,7 +172,7 @@ bridge_async_config(#{port := Port} = Config) -> " request_timeout = \"~ps\"\n" " body = \"${id}\"" " resource_opts {\n" - " async_inflight_window = 100\n" + " inflight_window = 100\n" " auto_restart_interval = \"60s\"\n" " health_check_interval = \"15s\"\n" " max_queue_bytes = \"1GB\"\n" diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl index 41be9e8a0..4e438bb79 100644 --- a/apps/emqx_resource/include/emqx_resource.hrl +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -73,7 +73,7 @@ max_queue_bytes => pos_integer(), query_mode => query_mode(), resume_interval => pos_integer(), - async_inflight_window => pos_integer() + inflight_window => pos_integer() }. -type query_result() :: ok diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl index 8bfd77e61..7934a2a4e 100644 --- a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl @@ -193,7 +193,7 @@ init({Id, Index, Opts}) -> Queue = replayq:open(QueueOpts), emqx_resource_metrics:queuing_set(Id, Index, queue_count(Queue)), emqx_resource_metrics:inflight_set(Id, Index, 0), - InflightWinSize = maps:get(async_inflight_window, Opts, ?DEFAULT_INFLIGHT), + InflightWinSize = maps:get(inflight_window, Opts, ?DEFAULT_INFLIGHT), InflightTID = inflight_new(InflightWinSize, Id, Index), HealthCheckInterval = maps:get(health_check_interval, Opts, ?HEALTHCHECK_INTERVAL), RequestTimeout = maps:get(request_timeout, Opts, ?DEFAULT_REQUEST_TIMEOUT), diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index e7c252fa9..990e5b472 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -369,7 +369,7 @@ t_query_counter_async_callback(_) -> #{ query_mode => async, batch_size => 1, - async_inflight_window => 1000000 + inflight_window => 1000000 } ), ?assertMatch({ok, 0}, emqx_resource:simple_sync_query(?ID, get_counter)), @@ -450,7 +450,7 @@ t_query_counter_async_inflight(_) -> #{ query_mode => async, batch_size => 1, - async_inflight_window => WindowSize, + inflight_window => WindowSize, worker_pool_size => 1, resume_interval => 300 } @@ -634,7 +634,7 @@ t_query_counter_async_inflight_batch(_) -> query_mode => async, batch_size => BatchSize, batch_time => 100, - async_inflight_window => WindowSize, + inflight_window => WindowSize, worker_pool_size => 1, resume_interval => 300 } @@ -1584,7 +1584,7 @@ t_retry_async_inflight_full(_Config) -> #{name => ?FUNCTION_NAME}, #{ query_mode => async, - async_inflight_window => AsyncInflightWindow, + inflight_window => AsyncInflightWindow, batch_size => 1, worker_pool_size => 1, resume_interval => ResumeInterval @@ -1642,7 +1642,7 @@ t_async_reply_multi_eval(_Config) -> #{name => ?FUNCTION_NAME}, #{ query_mode => async, - async_inflight_window => AsyncInflightWindow, + inflight_window => AsyncInflightWindow, batch_size => 3, batch_time => 10, worker_pool_size => 1, diff --git a/scripts/test/influx/influx-bridge.conf b/scripts/test/influx/influx-bridge.conf index df10a0ec6..0416e42b6 100644 --- a/scripts/test/influx/influx-bridge.conf +++ b/scripts/test/influx/influx-bridge.conf @@ -6,7 +6,7 @@ bridges { org = "emqx" precision = "ms" resource_opts { - async_inflight_window = 100 + inflight_window = 100 auto_restart_interval = "60s" batch_size = 100 batch_time = "10ms" From 88731fd145303576a1a2a41eafed0c1387adbf3f Mon Sep 17 00:00:00 2001 From: William Yang Date: Mon, 6 Mar 2023 14:03:28 +0100 Subject: [PATCH 070/196] feat(quic): support TLS password protected keyfile --- apps/emqx/src/emqx_listeners.erl | 6 +- apps/emqx/test/emqx_common_test_helpers.erl | 106 ++++++++++++++++++++ apps/emqx/test/emqx_listeners_SUITE.erl | 36 ++++++- 3 files changed, 142 insertions(+), 6 deletions(-) diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index b351212a7..4e5843166 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -388,7 +388,11 @@ do_start_listener(quic, ListenerName, #{bind := Bind} = Opts) -> ] ++ case maps:get(cacertfile, SSLOpts, undefined) of undefined -> []; - CaCertFile -> [{cacertfile, binary_to_list(CaCertFile)}] + CaCertFile -> [{cacertfile, str(CaCertFile)}] + end ++ + case maps:get(password, SSLOpts, undefined) of + undefined -> []; + Password -> [{password, str(Password)}] end ++ optional_quic_listener_opts(Opts), ConnectionOpts = #{ diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index d08812075..67edd58a7 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -85,6 +85,13 @@ reset_proxy/2 ]). +%% TLS certs API +-export([ + gen_ca/2, + gen_host_cert/3, + gen_host_cert/4 +]). + -define(CERTS_PATH(CertName), filename:join(["etc", "certs", CertName])). -define(MQTT_SSL_CLIENT_CERTS, [ @@ -561,6 +568,7 @@ ensure_quic_listener(Name, UdpPort, ExtraSettings) -> mountpoint => <<>>, zone => default }, + Conf2 = maps:merge(Conf, ExtraSettings), emqx_config:put([listeners, quic, Name], Conf2), case emqx_listeners:start_listener(emqx_listeners:listener_id(quic, Name)) of @@ -1073,6 +1081,104 @@ latency_up_proxy(off, Name, ProxyHost, ProxyPort) -> ). %%------------------------------------------------------------------------------- +%% TLS certs +%%------------------------------------------------------------------------------- +gen_ca(Path, Name) -> + %% Generate ca.pem and ca.key which will be used to generate certs + %% for hosts server and clients + ECKeyFile = filename(Path, "~s-ec.key", [Name]), + filelib:ensure_dir(ECKeyFile), + os:cmd("openssl ecparam -name secp256r1 > " ++ ECKeyFile), + Cmd = lists:flatten( + io_lib:format( + "openssl req -new -x509 -nodes " + "-newkey ec:~s " + "-keyout ~s -out ~s -days 3650 " + "-subj \"/C=SE/O=Internet Widgits Pty Ltd CA\"", + [ + ECKeyFile, + ca_key_name(Path, Name), + ca_cert_name(Path, Name) + ] + ) + ), + os:cmd(Cmd). + +ca_cert_name(Path, Name) -> + filename(Path, "~s.pem", [Name]). +ca_key_name(Path, Name) -> + filename(Path, "~s.key", [Name]). + +gen_host_cert(H, CaName, Path) -> + gen_host_cert(H, CaName, Path, #{}). + +gen_host_cert(H, CaName, Path, Opts) -> + ECKeyFile = filename(Path, "~s-ec.key", [CaName]), + CN = str(H), + HKey = filename(Path, "~s.key", [H]), + HCSR = filename(Path, "~s.csr", [H]), + HPEM = filename(Path, "~s.pem", [H]), + HEXT = filename(Path, "~s.extfile", [H]), + PasswordArg = + case maps:get(password, Opts, undefined) of + undefined -> + " -nodes "; + Password -> + io_lib:format(" -passout pass:'~s' ", [Password]) + end, + CSR_Cmd = + lists:flatten( + io_lib:format( + "openssl req -new ~s -newkey ec:~s " + "-keyout ~s -out ~s " + "-addext \"subjectAltName=DNS:~s\" " + "-addext keyUsage=digitalSignature,keyAgreement " + "-subj \"/C=SE/O=Internet Widgits Pty Ltd/CN=~s\"", + [PasswordArg, ECKeyFile, HKey, HCSR, CN, CN] + ) + ), + create_file( + HEXT, + "keyUsage=digitalSignature,keyAgreement\n" + "subjectAltName=DNS:~s\n", + [CN] + ), + CERT_Cmd = + lists:flatten( + io_lib:format( + "openssl x509 -req " + "-extfile ~s " + "-in ~s -CA ~s -CAkey ~s -CAcreateserial " + "-out ~s -days 500", + [ + HEXT, + HCSR, + ca_cert_name(Path, CaName), + ca_key_name(Path, CaName), + HPEM + ] + ) + ), + os:cmd(CSR_Cmd), + os:cmd(CERT_Cmd), + file:delete(HEXT). + +filename(Path, F, A) -> + filename:join(Path, str(io_lib:format(F, A))). + +str(Arg) -> + binary_to_list(iolist_to_binary(Arg)). + +create_file(Filename, Fmt, Args) -> + filelib:ensure_dir(Filename), + {ok, F} = file:open(Filename, [write]), + try + io:format(F, Fmt, Args) + after + file:close(F) + end, + ok. +%%------------------------------------------------------------------------------- %% Testcase teardown utilities %%------------------------------------------------------------------------------- diff --git a/apps/emqx/test/emqx_listeners_SUITE.erl b/apps/emqx/test/emqx_listeners_SUITE.erl index 015439587..ac4bf6c76 100644 --- a/apps/emqx/test/emqx_listeners_SUITE.erl +++ b/apps/emqx/test/emqx_listeners_SUITE.erl @@ -26,6 +26,8 @@ -define(CERTS_PATH(CertName), filename:join(["../../lib/emqx/etc/certs/", CertName])). +-define(SERVER_KEY_PASSWORD, "sErve7r8Key$!"). + all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> @@ -45,11 +47,6 @@ init_per_testcase(Case, Config) when -> catch emqx_config_handler:stop(), {ok, _} = emqx_config_handler:start_link(), - case emqx_config:get([listeners], undefined) of - undefined -> ok; - Listeners -> emqx_config:put([listeners], maps:remove(quic, Listeners)) - end, - PrevListeners = emqx_config:get([listeners], #{}), PureListeners = remove_default_limiter(PrevListeners), PureListeners2 = PureListeners#{ @@ -185,6 +182,28 @@ t_wss_conn(_) -> {ok, Socket} = ssl:connect({127, 0, 0, 1}, 9998, [{verify, verify_none}], 1000), ok = ssl:close(Socket). +t_quic_conn(Config) -> + DataDir = ?config(data_dir, Config), + generate_quic_tls_certs(Config), + SSLOpts = #{ + password => ?SERVER_KEY_PASSWORD, + certfile => filename:join(DataDir, "server-password.pem"), + cacertfile => filename:join(DataDir, "ca.pem"), + keyfile => filename:join(DataDir, "server-password.key") + }, + emqx_common_test_helpers:ensure_quic_listener(?FUNCTION_NAME, 24568, #{ssl_options => SSLOpts}), + ct:pal("~p", [emqx_listeners:list()]), + {ok, Conn} = quicer:connect( + {127, 0, 0, 1}, + 24568, + [ + {verify, verify_none}, + {alpn, ["mqtt"]} + ], + 1000 + ), + ok = quicer:close_connection(Conn). + t_format_bind(_) -> ?assertEqual( ":1883", @@ -269,3 +288,10 @@ remove_default_limiter(Listeners) -> end, Listeners ). + +generate_quic_tls_certs(Config) -> + DataDir = ?config(data_dir, Config), + emqx_common_test_helpers:gen_ca(DataDir, "ca"), + emqx_common_test_helpers:gen_host_cert("server-password", "ca", DataDir, #{ + password => ?SERVER_KEY_PASSWORD + }). From 7e15f90bf5cfa824faad63ffdef02e5eab063ff6 Mon Sep 17 00:00:00 2001 From: William Yang Date: Sun, 19 Mar 2023 21:48:05 +0100 Subject: [PATCH 071/196] chore(test): check openssl cmd returns --- apps/emqx/test/emqx_common_test_helpers.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index 67edd58a7..bda8d167b 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -1159,8 +1159,8 @@ gen_host_cert(H, CaName, Path, Opts) -> ] ) ), - os:cmd(CSR_Cmd), - os:cmd(CERT_Cmd), + ct:pal(os:cmd(CSR_Cmd)), + ct:pal(os:cmd(CERT_Cmd)), file:delete(HEXT). filename(Path, F, A) -> From ba49316ec5a8f908a6c09d8ce8b5f9c72c29aeaa Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Tue, 21 Mar 2023 19:43:04 +0100 Subject: [PATCH 072/196] fix: use lower-case --- apps/emqx_bridge/src/emqx_bridge_api.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index c71664c87..080670de6 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -438,7 +438,7 @@ schema("/nodes/:node/bridges/:id/:operation") -> 'operationId' => '/nodes/:node/bridges/:id/:operation', post => #{ tags => [<<"bridges">>], - summary => <<"Stop/Restart bridge">>, + summary => <<"Stop/restart bridge">>, description => ?DESC("desc_api8"), parameters => [ param_path_node(), From 678400bc0a198f88ac1011326d7a053f7475f333 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Tue, 21 Mar 2023 16:09:53 +0100 Subject: [PATCH 073/196] fix: add summary to schemas where description has html --- apps/emqx_exhook/i18n/emqx_exhook_api_i18n.conf | 4 ++++ .../i18n/emqx_mgmt_api_publish_i18n.conf | 14 ++++++++++---- .../i18n/emqx_mgmt_api_status_i18n.conf | 4 ++++ .../emqx_management/src/emqx_mgmt_api_configs.erl | 1 + .../emqx_management/src/emqx_mgmt_api_plugins.erl | 15 ++++++++++----- .../emqx_management/src/emqx_mgmt_api_publish.erl | 2 ++ changes/ce/fix-10195.en.md | 1 + 7 files changed, 32 insertions(+), 9 deletions(-) create mode 100644 changes/ce/fix-10195.en.md diff --git a/apps/emqx_exhook/i18n/emqx_exhook_api_i18n.conf b/apps/emqx_exhook/i18n/emqx_exhook_api_i18n.conf index 46854a3db..3ec5367ed 100644 --- a/apps/emqx_exhook/i18n/emqx_exhook_api_i18n.conf +++ b/apps/emqx_exhook/i18n/emqx_exhook_api_i18n.conf @@ -49,6 +49,10 @@ NOTE: The position should be \"front | rear | before:{name} | after:{name}""" zh: """移动 Exhook 服务器顺序。 注意: 移动的参数只能是：front | rear | before:{name} | after:{name}""" } + label { + en: "Change order of execution for registered Exhook server" + zh: "改变已注册的Exhook服务器的执行顺序" + } } move_position { diff --git a/apps/emqx_management/i18n/emqx_mgmt_api_publish_i18n.conf b/apps/emqx_management/i18n/emqx_mgmt_api_publish_i18n.conf index 4123ceefd..f91115df5 100644 --- a/apps/emqx_management/i18n/emqx_mgmt_api_publish_i18n.conf +++ b/apps/emqx_management/i18n/emqx_mgmt_api_publish_i18n.conf @@ -2,8 +2,7 @@ emqx_mgmt_api_publish { publish_api { desc { - en: """Publish one message.
-Possible HTTP status response codes are:
+ en: """Possible HTTP status response codes are:
200: The message is delivered to at least one subscriber;
202: No matched subscribers;
400: Message is invalid. for example bad topic name, or QoS is out of range;
@@ -16,11 +15,14 @@ Possible HTTP status response codes are:
400: 消息编码错误，如非法主题，或 QoS 超出范围等。
503: 服务重启等过程中导致转发失败。""" } + label { + en: "Publish a message" + zh: "发布一条信息" + } } publish_bulk_api { desc { - en: """Publish a batch of messages.
-Possible HTTP response status code are:
+ en: """Possible HTTP response status code are:
200: All messages are delivered to at least one subscriber;
202: At least one message was not delivered to any subscriber;
400: At least one message is invalid. For example bad topic name, or QoS is out of range;
@@ -41,6 +43,10 @@ result of each individual message in the batch.""" /publish 是一样的。 如果所有的消息都是合法的，那么 HTTP 返回的内容是一个 JSON 数组，每个元素代表了该消息转发的状态。""" } + label { + en: "Publish a batch of messages" + zh: "发布一批信息" + } } topic_name { diff --git a/apps/emqx_management/i18n/emqx_mgmt_api_status_i18n.conf b/apps/emqx_management/i18n/emqx_mgmt_api_status_i18n.conf index fae17b35d..d72fd0998 100644 --- a/apps/emqx_management/i18n/emqx_mgmt_api_status_i18n.conf +++ b/apps/emqx_management/i18n/emqx_mgmt_api_status_i18n.conf @@ -22,6 +22,10 @@ emqx_mgmt_api_status { "GET `/status`端点（没有`/api/...`前缀）也是这个端点的一个别名，工作方式相同。" " 这个别名从v5.0.0开始就有了。" } + label { + en: "Service health check" + zh: "服务健康检查" + } } get_status_response200 { diff --git a/apps/emqx_management/src/emqx_mgmt_api_configs.erl b/apps/emqx_management/src/emqx_mgmt_api_configs.erl index 2e6aac849..55cc50597 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_configs.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_configs.erl @@ -119,6 +119,7 @@ schema("/configs_reset/:rootname") -> "- For a config entry that has default value, this resets it to the default value;\n" "- For a config entry that has no default value, an error 400 will be returned" >>, + summary => <<"Reset config entry">>, %% We only return "200" rather than the new configs that has been changed, as %% the schema of the changed configs is depends on the request parameter %% `conf_path`, it cannot be defined here. diff --git a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl index 4930e587c..f744ec0d2 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl @@ -68,10 +68,10 @@ schema("/plugins") -> #{ 'operationId' => list_plugins, get => #{ + summary => <<"List all installed plugins">>, description => - "List all install plugins.
" "Plugins are launched in top-down order.
" - "Using `POST /plugins/{name}/move` to change the boot order.", + "Use `POST /plugins/{name}/move` to change the boot order.", tags => ?TAGS, responses => #{ 200 => hoconsc:array(hoconsc:ref(plugin)) @@ -82,8 +82,9 @@ schema("/plugins/install") -> #{ 'operationId' => upload_install, post => #{ + summary => <<"Install a new plugin">>, description => - "Install a plugin(plugin-vsn.tar.gz)." + "Upload a plugin tarball (plugin-vsn.tar.gz)." "Follow [emqx-plugin-template](https://github.com/emqx/emqx-plugin-template) " "to develop plugin.", tags => ?TAGS, @@ -112,7 +113,8 @@ schema("/plugins/:name") -> #{ 'operationId' => plugin, get => #{ - description => "Describe a plugin according `release.json` and `README.md`.", + summary => <<"Get a plugin description">>, + description => "Describs plugin according to its `release.json` and `README.md`.", tags => ?TAGS, parameters => [hoconsc:ref(name)], responses => #{ @@ -121,7 +123,8 @@ schema("/plugins/:name") -> } }, delete => #{ - description => "Uninstall a plugin package.", + summary => <<"Delete a plugin">>, + description => "Uninstalls a previously uploaded plugin package.", tags => ?TAGS, parameters => [hoconsc:ref(name)], responses => #{ @@ -134,6 +137,7 @@ schema("/plugins/:name/:action") -> #{ 'operationId' => update_plugin, put => #{ + summary => <<"Trigger action on an installed plugin">>, description => "start/stop a installed plugin.
" "- **start**: start the plugin.
" @@ -153,6 +157,7 @@ schema("/plugins/:name/move") -> #{ 'operationId' => update_boot_order, post => #{ + summary => <<"Move plugin within plugin hiearchy">>, description => "Setting the boot order of plugins.", tags => ?TAGS, parameters => [hoconsc:ref(name)], diff --git a/apps/emqx_management/src/emqx_mgmt_api_publish.erl b/apps/emqx_management/src/emqx_mgmt_api_publish.erl index 245b56c1d..ba486ab89 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_publish.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_publish.erl @@ -50,6 +50,7 @@ schema("/publish") -> #{ 'operationId' => publish, post => #{ + summary => <<"Publish a message">>, description => ?DESC(publish_api), tags => [<<"Publish">>], 'requestBody' => hoconsc:mk(hoconsc:ref(?MODULE, publish_message)), @@ -65,6 +66,7 @@ schema("/publish/bulk") -> #{ 'operationId' => publish_batch, post => #{ + summary => <<"Publish a batch of messages">>, description => ?DESC(publish_bulk_api), tags => [<<"Publish">>], 'requestBody' => hoconsc:mk(hoconsc:array(hoconsc:ref(?MODULE, publish_message)), #{}), diff --git a/changes/ce/fix-10195.en.md b/changes/ce/fix-10195.en.md new file mode 100644 index 000000000..35cc7c082 --- /dev/null +++ b/changes/ce/fix-10195.en.md @@ -0,0 +1 @@ +Add labels to API schemas where description contains HTML and breaks formatting of generated documentation otherwise. From f8d5d53908f71518dc3f16f6d07cc077b0fc3aea Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 22 Mar 2023 11:26:15 -0300 Subject: [PATCH 074/196] feat(buffer_worker): decouple query mode from underlying connector call mode Fixes https://emqx.atlassian.net/browse/EMQX-9129 Currently, if an user configures a bridge with query mode sync, then all calls to the underlying driver/connector ("inner calls") will always be synchronous, regardless of its support for async calls. Since buffer workers always support async queries ("outer calls"), we should decouple those two call modes (inner and outer), and avoid exposing the inner call configuration to user to avoid complexity. There are two situations when we want to force synchronous calls to the underlying connector even if it supports async: 1) When using `simple_sync_query`, since we are bypassing the buffer workers; 2) When retrying the inflight window, to avoid overwhelming the driver. --- apps/emqx_resource/src/emqx_resource.erl | 2 +- .../src/emqx_resource_buffer_worker.erl | 26 +++--- .../test/emqx_connector_demo.erl | 15 ++++ .../test/emqx_resource_SUITE.erl | 83 +++++++++++++++++++ changes/ce/feat-10206.en.md | 7 ++ .../test/emqx_ee_bridge_dynamo_SUITE.erl | 4 +- .../test/emqx_ee_bridge_gcp_pubsub_SUITE.erl | 82 ++++-------------- .../test/emqx_ee_bridge_influxdb_SUITE.erl | 40 +++++---- .../test/emqx_ee_connector_influxdb_SUITE.erl | 4 +- 9 files changed, 162 insertions(+), 101 deletions(-) create mode 100644 changes/ce/feat-10206.en.md diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index 1ccb5ca71..2e72c2a28 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -265,7 +265,7 @@ query(ResId, Request, Opts) -> IsBufferSupported = is_buffer_supported(Module), case {IsBufferSupported, QM} of {true, _} -> - %% only Kafka so far + %% only Kafka producer so far Opts1 = Opts#{is_buffer_supported => true}, emqx_resource_buffer_worker:simple_async_query(ResId, Request, Opts1); {false, sync} -> diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl index 8bfd77e61..4151cf430 100644 --- a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl @@ -140,7 +140,7 @@ simple_sync_query(Id, Request) -> QueryOpts = simple_query_opts(), emqx_resource_metrics:matched_inc(Id), Ref = make_request_ref(), - Result = call_query(sync, Id, Index, Ref, ?SIMPLE_QUERY(Request), QueryOpts), + Result = call_query(force_sync, Id, Index, Ref, ?SIMPLE_QUERY(Request), QueryOpts), _ = handle_query_result(Id, Result, _HasBeenSent = false), Result. @@ -152,7 +152,7 @@ simple_async_query(Id, Request, QueryOpts0) -> QueryOpts = maps:merge(simple_query_opts(), QueryOpts0), emqx_resource_metrics:matched_inc(Id), Ref = make_request_ref(), - Result = call_query(async, Id, Index, Ref, ?SIMPLE_QUERY(Request), QueryOpts), + Result = call_query(async_if_possible, Id, Index, Ref, ?SIMPLE_QUERY(Request), QueryOpts), _ = handle_query_result(Id, Result, _HasBeenSent = false), Result. @@ -377,7 +377,7 @@ retry_inflight_sync(Ref, QueryOrBatch, Data0) -> } = Data0, ?tp(buffer_worker_retry_inflight, #{query_or_batch => QueryOrBatch, ref => Ref}), QueryOpts = #{simple_query => false}, - Result = call_query(sync, Id, Index, Ref, QueryOrBatch, QueryOpts), + Result = call_query(force_sync, Id, Index, Ref, QueryOrBatch, QueryOpts), ReplyResult = case QueryOrBatch of ?QUERY(ReplyTo, _, HasBeenSent, _ExpireAt) -> @@ -566,7 +566,7 @@ do_flush( %% unwrap when not batching (i.e., batch size == 1) [?QUERY(ReplyTo, _, HasBeenSent, _ExpireAt) = Request] = Batch, QueryOpts = #{inflight_tid => InflightTID, simple_query => false}, - Result = call_query(configured, Id, Index, Ref, Request, QueryOpts), + Result = call_query(async_if_possible, Id, Index, Ref, Request, QueryOpts), Reply = ?REPLY(ReplyTo, HasBeenSent, Result), case reply_caller(Id, Reply, QueryOpts) of %% Failed; remove the request from the queue, as we cannot pop @@ -651,7 +651,7 @@ do_flush(#{queue := Q1} = Data0, #{ inflight_tid := InflightTID } = Data0, QueryOpts = #{inflight_tid => InflightTID, simple_query => false}, - Result = call_query(configured, Id, Index, Ref, Batch, QueryOpts), + Result = call_query(async_if_possible, Id, Index, Ref, Batch, QueryOpts), case batch_reply_caller(Id, Result, Batch, QueryOpts) of %% Failed; remove the request from the queue, as we cannot pop %% from it again, but we'll retry it using the inflight table. @@ -883,17 +883,13 @@ handle_async_worker_down(Data0, Pid) -> mark_inflight_items_as_retriable(Data, WorkerMRef), {keep_state, Data}. -call_query(QM0, Id, Index, Ref, Query, QueryOpts) -> - ?tp(call_query_enter, #{id => Id, query => Query, query_mode => QM0}), +-spec call_query(force_sync | async_if_possible, _, _, _, _, _) -> _. +call_query(QM, Id, Index, Ref, Query, QueryOpts) -> + ?tp(call_query_enter, #{id => Id, query => Query, query_mode => QM}), case emqx_resource_manager:lookup_cached(Id) of {ok, _Group, #{status := stopped}} -> ?RESOURCE_ERROR(stopped, "resource stopped or disabled"); {ok, _Group, Resource} -> - QM = - case QM0 =:= configured of - true -> maps:get(query_mode, Resource); - false -> QM0 - end, do_call_query(QM, Id, Index, Ref, Query, QueryOpts, Resource); {error, not_found} -> ?RESOURCE_ERROR(not_found, "resource not found") @@ -1511,9 +1507,9 @@ inc_sent_success(Id, _HasBeenSent = true) -> inc_sent_success(Id, _HasBeenSent) -> emqx_resource_metrics:success_inc(Id). -call_mode(sync, _) -> sync; -call_mode(async, always_sync) -> sync; -call_mode(async, async_if_possible) -> async. +call_mode(force_sync, _) -> sync; +call_mode(async_if_possible, always_sync) -> sync; +call_mode(async_if_possible, async_if_possible) -> async. assert_ok_result(ok) -> true; diff --git a/apps/emqx_resource/test/emqx_connector_demo.erl b/apps/emqx_resource/test/emqx_connector_demo.erl index a863dbb78..a1393c574 100644 --- a/apps/emqx_resource/test/emqx_connector_demo.erl +++ b/apps/emqx_resource/test/emqx_connector_demo.erl @@ -146,6 +146,12 @@ on_query(_InstId, {sleep_before_reply, For}, #{pid := Pid}) -> {error, timeout} end. +on_query_async(_InstId, block, ReplyFun, #{pid := Pid}) -> + Pid ! {block, ReplyFun}, + {ok, Pid}; +on_query_async(_InstId, resume, ReplyFun, #{pid := Pid}) -> + Pid ! {resume, ReplyFun}, + {ok, Pid}; on_query_async(_InstId, {inc_counter, N}, ReplyFun, #{pid := Pid}) -> Pid ! {inc, N, ReplyFun}, {ok, Pid}; @@ -274,6 +280,10 @@ counter_loop( block -> ct:pal("counter recv: ~p", [block]), State#{status => blocked}; + {block, ReplyFun} -> + ct:pal("counter recv: ~p", [block]), + apply_reply(ReplyFun, ok), + State#{status => blocked}; {block_now, ReplyFun} -> ct:pal("counter recv: ~p", [block_now]), apply_reply( @@ -284,6 +294,11 @@ counter_loop( {messages, Msgs} = erlang:process_info(self(), messages), ct:pal("counter recv: ~p, buffered msgs: ~p", [resume, length(Msgs)]), State#{status => running}; + {resume, ReplyFun} -> + {messages, Msgs} = erlang:process_info(self(), messages), + ct:pal("counter recv: ~p, buffered msgs: ~p", [resume, length(Msgs)]), + apply_reply(ReplyFun, ok), + State#{status => running}; {inc, N, ReplyFun} when Status == running -> %ct:pal("async counter recv: ~p", [{inc, N}]), apply_reply(ReplyFun, ok), diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index e7c252fa9..ca91ae40d 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -2561,6 +2561,84 @@ do_t_recursive_flush() -> ), ok. +t_call_mode_uncoupled_from_query_mode(_Config) -> + DefaultOpts = #{ + batch_size => 1, + batch_time => 5, + worker_pool_size => 1 + }, + ?check_trace( + begin + %% We check that we can call the buffer workers with async + %% calls, even if the underlying connector itself only + %% supports sync calls. + emqx_connector_demo:set_callback_mode(always_sync), + {ok, _} = emqx_resource:create( + ?ID, + ?DEFAULT_RESOURCE_GROUP, + ?TEST_RESOURCE, + #{name => test_resource}, + DefaultOpts#{query_mode => async} + ), + ?tp_span( + async_query_sync_driver, + #{}, + ?assertMatch( + {ok, {ok, _}}, + ?wait_async_action( + emqx_resource:query(?ID, {inc_counter, 1}), + #{?snk_kind := buffer_worker_flush_ack}, + 500 + ) + ) + ), + ?assertEqual(ok, emqx_resource:remove_local(?ID)), + + %% And we check the converse: a connector that allows async + %% calls can be called synchronously, but the underlying + %% call should be async. + emqx_connector_demo:set_callback_mode(async_if_possible), + {ok, _} = emqx_resource:create( + ?ID, + ?DEFAULT_RESOURCE_GROUP, + ?TEST_RESOURCE, + #{name => test_resource}, + DefaultOpts#{query_mode => sync} + ), + ?tp_span( + sync_query_async_driver, + #{}, + ?assertEqual(ok, emqx_resource:query(?ID, {inc_counter, 2})) + ), + ?assertEqual(ok, emqx_resource:remove_local(?ID)), + ?tp(sync_query_async_driver, #{}), + ok + end, + fun(Trace0) -> + Trace1 = trace_between_span(Trace0, async_query_sync_driver), + ct:pal("async query calling sync driver\n ~p", [Trace1]), + ?assert( + ?strict_causality( + #{?snk_kind := async_query, request := {inc_counter, 1}}, + #{?snk_kind := call_query, call_mode := sync}, + Trace1 + ) + ), + + Trace2 = trace_between_span(Trace0, sync_query_async_driver), + ct:pal("sync query calling async driver\n ~p", [Trace2]), + ?assert( + ?strict_causality( + #{?snk_kind := sync_query, request := {inc_counter, 2}}, + #{?snk_kind := call_query_async}, + Trace2 + ) + ), + + ok + end + ). + %%------------------------------------------------------------------------------ %% Helpers %%------------------------------------------------------------------------------ @@ -2742,3 +2820,8 @@ assert_async_retry_fail_then_succeed_inflight(Trace) -> ) ), ok. + +trace_between_span(Trace0, Marker) -> + {Trace1, [_ | _]} = ?split_trace_at(#{?snk_kind := Marker, ?snk_span := {complete, _}}, Trace0), + {[_ | _], [_ | Trace2]} = ?split_trace_at(#{?snk_kind := Marker, ?snk_span := start}, Trace1), + Trace2. diff --git a/changes/ce/feat-10206.en.md b/changes/ce/feat-10206.en.md new file mode 100644 index 000000000..014ea71f2 --- /dev/null +++ b/changes/ce/feat-10206.en.md @@ -0,0 +1,7 @@ +Decouple the query mode from the underlying call mode for buffer +workers. + +Prior to this change, setting the query mode of a resource +such as a bridge to `sync` would force the buffer to call the +underlying connector in a synchronous way, even if it supports async +calls. diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl index 183002e61..c0d58c4f7 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl @@ -291,7 +291,7 @@ t_setup_via_config_and_publish(Config) -> end, fun(Trace0) -> Trace = ?of_kind(dynamo_connector_query_return, Trace0), - ?assertMatch([#{result := {ok, _}}], Trace), + ?assertMatch([#{result := ok}], Trace), ok end ), @@ -328,7 +328,7 @@ t_setup_via_http_api_and_publish(Config) -> end, fun(Trace0) -> Trace = ?of_kind(dynamo_connector_query_return, Trace0), - ?assertMatch([#{result := {ok, _}}], Trace), + ?assertMatch([#{result := ok}], Trace), ok end ), diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl index 8424ddff0..709955666 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl @@ -1013,7 +1013,6 @@ t_publish_timeout(Config) -> do_econnrefused_or_timeout_test(Config, timeout). do_econnrefused_or_timeout_test(Config, Error) -> - QueryMode = ?config(query_mode, Config), ResourceId = ?config(resource_id, Config), TelemetryTable = ?config(telemetry_table, Config), Topic = <<"t/topic">>, @@ -1021,15 +1020,8 @@ do_econnrefused_or_timeout_test(Config, Error) -> Message = emqx_message:make(Topic, Payload), ?check_trace( begin - case {QueryMode, Error} of - {sync, _} -> - {_, {ok, _}} = - ?wait_async_action( - emqx:publish(Message), - #{?snk_kind := gcp_pubsub_request_failed, recoverable_error := true}, - 15_000 - ); - {async, econnrefused} -> + case Error of + econnrefused -> %% at the time of writing, async requests %% are never considered expired by ehttpc %% (even if they arrive late, or never @@ -1049,7 +1041,7 @@ do_econnrefused_or_timeout_test(Config, Error) -> }, 15_000 ); - {async, timeout} -> + timeout -> %% at the time of writing, async requests %% are never considered expired by ehttpc %% (even if they arrive late, or never @@ -1067,18 +1059,13 @@ do_econnrefused_or_timeout_test(Config, Error) -> end end, fun(Trace) -> - case {QueryMode, Error} of - {sync, _} -> + case Error of + econnrefused -> ?assertMatch( [#{reason := Error, connector := ResourceId} | _], ?of_kind(gcp_pubsub_request_failed, Trace) ); - {async, econnrefused} -> - ?assertMatch( - [#{reason := Error, connector := ResourceId} | _], - ?of_kind(gcp_pubsub_request_failed, Trace) - ); - {async, timeout} -> + timeout -> ?assertMatch( [_, _ | _], ?of_kind(gcp_pubsub_response, Trace) @@ -1088,11 +1075,11 @@ do_econnrefused_or_timeout_test(Config, Error) -> end ), - case {Error, QueryMode} of + case Error of %% apparently, async with disabled queue doesn't mark the %% message as dropped; and since it never considers the %% response expired, this succeeds. - {econnrefused, async} -> + econnrefused -> wait_telemetry_event(TelemetryTable, queuing, ResourceId, #{ timeout => 10_000, n_events => 1 }), @@ -1114,7 +1101,7 @@ do_econnrefused_or_timeout_test(Config, Error) -> } when Matched >= 1 andalso Inflight + Queueing + Dropped + Failed =< 2, CurrentMetrics ); - {timeout, async} -> + timeout -> wait_until_gauge_is(inflight, 0, _Timeout = 400), wait_until_gauge_is(queuing, 0, _Timeout = 400), assert_metrics( @@ -1129,21 +1116,6 @@ do_econnrefused_or_timeout_test(Config, Error) -> late_reply => 2 }, ResourceId - ); - {_, sync} -> - wait_until_gauge_is(queuing, 0, 500), - wait_until_gauge_is(inflight, 1, 500), - assert_metrics( - #{ - dropped => 0, - failed => 0, - inflight => 1, - matched => 1, - queuing => 0, - retried => 0, - success => 0 - }, - ResourceId ) end, @@ -1267,7 +1239,6 @@ t_failure_no_body(Config) -> t_unrecoverable_error(Config) -> ResourceId = ?config(resource_id, Config), - QueryMode = ?config(query_mode, Config), TestPid = self(), FailureNoBodyHandler = fun(Req0, State) -> @@ -1298,33 +1269,16 @@ t_unrecoverable_error(Config) -> Message = emqx_message:make(Topic, Payload), ?check_trace( {_, {ok, _}} = - case QueryMode of - sync -> - ?wait_async_action( - emqx:publish(Message), - #{?snk_kind := gcp_pubsub_request_failed}, - 5_000 - ); - async -> - ?wait_async_action( - emqx:publish(Message), - #{?snk_kind := gcp_pubsub_response}, - 5_000 - ) - end, + ?wait_async_action( + emqx:publish(Message), + #{?snk_kind := gcp_pubsub_response}, + 5_000 + ), fun(Trace) -> - case QueryMode of - sync -> - ?assertMatch( - [#{reason := killed}], - ?of_kind(gcp_pubsub_request_failed, Trace) - ); - async -> - ?assertMatch( - [#{response := {error, killed}}], - ?of_kind(gcp_pubsub_response, Trace) - ) - end, + ?assertMatch( + [#{response := {error, killed}}], + ?of_kind(gcp_pubsub_response, Trace) + ), ok end ), diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl index 2b2214df0..e8dd970f3 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl @@ -532,10 +532,12 @@ t_start_ok(Config) -> }, ?check_trace( begin - ?assertEqual(ok, send_message(Config, SentData)), case QueryMode of - async -> ct:sleep(500); - sync -> ok + async -> + ?assertMatch(ok, send_message(Config, SentData)), + ct:sleep(500); + sync -> + ?assertMatch({ok, 204, _}, send_message(Config, SentData)) end, PersistedData = query_by_clientid(ClientId, Config), Expected = #{ @@ -689,10 +691,12 @@ t_const_timestamp(Config) -> <<"payload">> => Payload, <<"timestamp">> => erlang:system_time(millisecond) }, - ?assertEqual(ok, send_message(Config, SentData)), case QueryMode of - async -> ct:sleep(500); - sync -> ok + async -> + ?assertMatch(ok, send_message(Config, SentData)), + ct:sleep(500); + sync -> + ?assertMatch({ok, 204, _}, send_message(Config, SentData)) end, PersistedData = query_by_clientid(ClientId, Config), Expected = #{foo => <<"123">>}, @@ -745,7 +749,12 @@ t_boolean_variants(Config) -> <<"timestamp">> => erlang:system_time(millisecond), <<"payload">> => Payload }, - ?assertEqual(ok, send_message(Config, SentData)), + case QueryMode of + sync -> + ?assertMatch({ok, 204, _}, send_message(Config, SentData)); + async -> + ?assertMatch(ok, send_message(Config, SentData)) + end, case QueryMode of async -> ct:sleep(500); sync -> ok @@ -841,10 +850,9 @@ t_bad_timestamp(Config) -> ); {sync, false} -> ?assertEqual( - {error, - {unrecoverable_error, [ - {error, {bad_timestamp, <<"bad_timestamp">>}} - ]}}, + {error, [ + {error, {bad_timestamp, <<"bad_timestamp">>}} + ]}, Return ); {sync, true} -> @@ -964,7 +972,7 @@ t_write_failure(Config) -> {error, {resource_error, #{reason := timeout}}}, send_message(Config, SentData) ), - #{?snk_kind := buffer_worker_flush_nack}, + #{?snk_kind := handle_async_reply, action := nack}, 1_000 ); async -> @@ -978,13 +986,13 @@ t_write_failure(Config) -> fun(Trace0) -> case QueryMode of sync -> - Trace = ?of_kind(buffer_worker_flush_nack, Trace0), + Trace = ?of_kind(handle_async_reply, Trace0), ?assertMatch([_ | _], Trace), [#{result := Result} | _] = Trace, ?assert( {error, {error, {closed, "The connection was lost."}}} =:= Result orelse {error, {error, closed}} =:= Result orelse - {error, {recoverable_error, {error, econnrefused}}} =:= Result, + {error, {recoverable_error, econnrefused}} =:= Result, #{got => Result} ); async -> @@ -1006,7 +1014,6 @@ t_write_failure(Config) -> ok. t_missing_field(Config) -> - QueryMode = ?config(query_mode, Config), BatchSize = ?config(batch_size, Config), IsBatch = BatchSize > 1, {ok, _} = @@ -1034,8 +1041,7 @@ t_missing_field(Config) -> {ok, _} = snabbkaffe:block_until( ?match_n_events(NEvents, #{ - ?snk_kind := influxdb_connector_send_query_error, - mode := QueryMode + ?snk_kind := influxdb_connector_send_query_error }), _Timeout1 = 10_000 ), diff --git a/lib-ee/emqx_ee_connector/test/emqx_ee_connector_influxdb_SUITE.erl b/lib-ee/emqx_ee_connector/test/emqx_ee_connector_influxdb_SUITE.erl index 72fc11a67..364821ea0 100644 --- a/lib-ee/emqx_ee_connector/test/emqx_ee_connector_influxdb_SUITE.erl +++ b/lib-ee/emqx_ee_connector/test/emqx_ee_connector_influxdb_SUITE.erl @@ -94,7 +94,7 @@ perform_lifecycle_check(PoolName, InitialConfig) -> emqx_resource:get_instance(PoolName), ?assertEqual({ok, connected}, emqx_resource:health_check(PoolName)), % % Perform query as further check that the resource is working as expected - ?assertMatch(ok, emqx_resource:query(PoolName, test_query())), + ?assertMatch({ok, 204, _}, emqx_resource:query(PoolName, test_query())), ?assertEqual(ok, emqx_resource:stop(PoolName)), % Resource will be listed still, but state will be changed and healthcheck will fail % as the worker no longer exists. @@ -116,7 +116,7 @@ perform_lifecycle_check(PoolName, InitialConfig) -> {ok, ?CONNECTOR_RESOURCE_GROUP, #{status := InitialStatus}} = emqx_resource:get_instance(PoolName), ?assertEqual({ok, connected}, emqx_resource:health_check(PoolName)), - ?assertMatch(ok, emqx_resource:query(PoolName, test_query())), + ?assertMatch({ok, 204, _}, emqx_resource:query(PoolName, test_query())), % Stop and remove the resource in one go. ?assertEqual(ok, emqx_resource:remove_local(PoolName)), ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)), From 65f468f7068365bdf0345f542655700ec1e29cb0 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 23 Mar 2023 16:23:56 +0800 Subject: [PATCH 075/196] fix: for connection used ecpool, let worker do health check fun let it be same with ecpool:pick_and_do/3 for checkfun use format as {M,F,A} --- apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl b/apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl index 289d39032..9b286f360 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl @@ -67,13 +67,14 @@ stop_pool(Name) -> health_check_ecpool_workers(PoolName, CheckFunc) -> health_check_ecpool_workers(PoolName, CheckFunc, ?HEALTH_CHECK_TIMEOUT). -health_check_ecpool_workers(PoolName, CheckFunc, Timeout) when is_function(CheckFunc) -> +health_check_ecpool_workers(PoolName, CheckFunc, Timeout) -> Workers = [Worker || {_WorkerName, Worker} <- ecpool:workers(PoolName)], DoPerWorker = fun(Worker) -> case ecpool_worker:client(Worker) of {ok, Conn} -> - erlang:is_process_alive(Conn) andalso CheckFunc(Conn); + erlang:is_process_alive(Conn) andalso + ecpool_worker:exec(Worker, CheckFunc, Timeout); _ -> false end From d0f0a9388841d4dcec553219a9aa1fe16ce94859 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Fri, 24 Mar 2023 11:15:52 +0800 Subject: [PATCH 076/196] fix: ecpool worker exec dynamo health check fun --- .../src/emqx_ee_connector_dynamo.erl | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl index 957706f6a..4703e0a21 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl @@ -31,8 +31,7 @@ connect/1, do_get_status/1, do_async_reply/2, - worker_do_query/4, - worker_do_get_status/1 + worker_do_query/4 ]). -import(hoconsc, [mk/2, enum/1, ref/2]). @@ -165,18 +164,14 @@ on_get_status(_InstanceId, #{poolname := Pool}) -> Health = emqx_plugin_libs_pool:health_check_ecpool_workers(Pool, fun ?MODULE:do_get_status/1), status_result(Health). -do_get_status(Conn) -> +do_get_status(_Conn) -> %% because the dynamodb driver connection process is the ecpool worker self %% so we must call the checker function inside the worker - ListTables = ecpool_worker:exec(Conn, {?MODULE, worker_do_get_status, []}, infinity), - case ListTables of + case erlcloud_ddb2:list_tables() of {ok, _} -> true; _ -> false end. -worker_do_get_status(_) -> - erlcloud_ddb2:list_tables(). - status_result(_Status = true) -> connected; status_result(_Status = false) -> connecting. From 4d28731176f7d9e8e1b3dd1819580b0ab57a377a Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 24 Mar 2023 08:37:00 +0100 Subject: [PATCH 077/196] docs: delete local-override.conf from config template --- rel/emqx_conf.template.en.md | 3 +-- rel/emqx_conf.template.zh.md | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/rel/emqx_conf.template.en.md b/rel/emqx_conf.template.en.md index bbeac9489..8740e4319 100644 --- a/rel/emqx_conf.template.en.md +++ b/rel/emqx_conf.template.en.md @@ -4,13 +4,12 @@ and a superset of JSON. ## Layered -EMQX configuration consists of 3 layers. +EMQX configuration consists of two layers. From bottom up: 1. Immutable base: `emqx.conf` + `EMQX_` prefixed environment variables.
Changes in this layer require a full node restart to take effect. 1. Cluster overrides: `$EMQX_NODE__DATA_DIR/configs/cluster-override.conf` -1. Local node overrides: `$EMQX_NODE__DATA_DIR/configs/local-override.conf` When environment variable `$EMQX_NODE__DATA_DIR` is not set, config `node.data_dir` is used. diff --git a/rel/emqx_conf.template.zh.md b/rel/emqx_conf.template.zh.md index cfb620c0f..9402760a2 100644 --- a/rel/emqx_conf.template.zh.md +++ b/rel/emqx_conf.template.zh.md @@ -3,12 +3,11 @@ HOCON（Human-Optimized Config Object Notation）是一个JSON的超集，非常 ## 分层结构 -EMQX的配置文件可分为三层，自底向上依次是： +EMQX的配置文件可分为二层，自底向上依次是： 1. 不可变的基础层 `emqx.conf` 加上 `EMQX_` 前缀的环境变量。
修改这一层的配置之后，需要重启节点来使之生效。 1. 集群范围重载层：`$EMQX_NODE__DATA_DIR/configs/cluster-override.conf` -1. 节点本地重载层：`$EMQX_NODE__DATA_DIR/configs/local-override.conf` 如果环境变量 `$EMQX_NODE__DATA_DIR` 没有设置，那么该目录会从 `emqx.conf` 的 `node.data_dir` 配置中读取。 From a8cfc1c7a14cb049ea4f4a7978ebfcb9d5b8f07d Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 24 Mar 2023 11:17:50 +0100 Subject: [PATCH 078/196] fix(emqx_bridge): filter status_reason in config update --- apps/emqx_bridge/src/emqx_bridge_api.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index c71664c87..f70964e47 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -929,7 +929,7 @@ filter_out_request_body(Conf) -> <<"type">>, <<"name">>, <<"status">>, - <<"error">>, + <<"status_reason">>, <<"node_status">>, <<"node_metrics">>, <<"metrics">>, From fbfdaf44e09de51228f97be6aba082437cbdb1bc Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Thu, 23 Mar 2023 17:41:42 +0100 Subject: [PATCH 079/196] fix(emqx_bridge): don't crash on validation error --- apps/emqx_bridge/src/emqx_bridge_api.erl | 25 +++++++++++++--- .../test/emqx_bridge_api_SUITE.erl | 30 +++++++++++++++++++ changes/ce/fix-10226.en.md | 1 + 3 files changed, 52 insertions(+), 4 deletions(-) create mode 100644 changes/ce/fix-10226.en.md diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index f70964e47..9fb8b30c2 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -481,8 +481,7 @@ schema("/bridges_probe") -> ?BAD_REQUEST('ALREADY_EXISTS', <<"bridge already exists">>); {error, not_found} -> Conf = filter_out_request_body(Conf0), - {ok, _} = emqx_bridge:create(BridgeType, BridgeName, Conf), - lookup_from_all_nodes(BridgeType, BridgeName, 201) + create_bridge(BridgeType, BridgeName, Conf) end; '/bridges'(get, _Params) -> Nodes = mria:running_nodes(), @@ -508,8 +507,7 @@ schema("/bridges_probe") -> {ok, _} -> RawConf = emqx:get_raw_config([bridges, BridgeType, BridgeName], #{}), Conf = deobfuscate(Conf1, RawConf), - {ok, _} = emqx_bridge:create(BridgeType, BridgeName, Conf), - lookup_from_all_nodes(BridgeType, BridgeName, 200); + update_bridge(BridgeType, BridgeName, Conf); {error, not_found} -> ?BRIDGE_NOT_FOUND(BridgeType, BridgeName) end @@ -609,6 +607,20 @@ lookup_from_local_node(BridgeType, BridgeName) -> Error -> Error end. +create_bridge(BridgeType, BridgeName, Conf) -> + create_or_update_bridge(BridgeType, BridgeName, Conf, 201). + +update_bridge(BridgeType, BridgeName, Conf) -> + create_or_update_bridge(BridgeType, BridgeName, Conf, 200). + +create_or_update_bridge(BridgeType, BridgeName, Conf, HttpStatusCode) -> + case emqx_bridge:create(BridgeType, BridgeName, Conf) of + {ok, _} -> + lookup_from_all_nodes(BridgeType, BridgeName, HttpStatusCode); + {error, #{kind := validation_error} = Reason} -> + ?BAD_REQUEST(map_to_json(Reason)) + end. + '/bridges/:id/enable/:enable'(put, #{bindings := #{id := Id, enable := Enable}}) -> ?TRY_PARSE_ID( Id, @@ -1033,3 +1045,8 @@ deobfuscate(NewConf, OldConf) -> #{}, NewConf ). + +map_to_json(M) -> + emqx_json:encode( + emqx_map_lib:jsonable_map(M, fun(K, V) -> {K, emqx_map_lib:binary_string(V)} end) + ). diff --git a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl index 932d7261a..47a23e71c 100644 --- a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl @@ -322,6 +322,33 @@ t_http_crud_apis(Config) -> end ), + %% Test bad updates + {ok, 400, PutFail1} = request( + put, + uri(["bridges", BridgeID]), + maps:remove(<<"url">>, ?HTTP_BRIDGE(URL2, Name)) + ), + ?assertMatch( + #{<<"reason">> := <<"required_field">>}, + emqx_json:decode(maps:get(<<"message">>, emqx_json:decode(PutFail1, [return_maps])), [ + return_maps + ]) + ), + {ok, 400, PutFail2} = request( + put, + uri(["bridges", BridgeID]), + maps:put(<<"curl">>, URL2, maps:remove(<<"url">>, ?HTTP_BRIDGE(URL2, Name))) + ), + ?assertMatch( + #{ + <<"reason">> := <<"unknown_fields">>, + <<"unknown">> := <<"curl">> + }, + emqx_json:decode(maps:get(<<"message">>, emqx_json:decode(PutFail2, [return_maps])), [ + return_maps + ]) + ), + %% delete the bridge {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), []), {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []), @@ -387,6 +414,9 @@ t_http_crud_apis(Config) -> ?assert(not maps:is_key(<<"status_reason">>, FixedBridge)), ?assert(not maps:is_key(<<"status_reason">>, FixedNodeStatus)), {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), []), + + %% Try create bridge with bad characters as name + {ok, 400, _} = request(post, uri(["bridges"]), ?HTTP_BRIDGE(URL1, <<"隋达"/utf8>>)), ok. t_http_bridges_local_topic(Config) -> diff --git a/changes/ce/fix-10226.en.md b/changes/ce/fix-10226.en.md new file mode 100644 index 000000000..2d833d2dc --- /dev/null +++ b/changes/ce/fix-10226.en.md @@ -0,0 +1 @@ +Don't crash on validation error in `/bridges` API, return `400` instead. From 9031c398fbe24c218d5a1ea017a42dc620b7f51d Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 23 Mar 2023 12:20:46 -0300 Subject: [PATCH 080/196] feat(helm): add option to customize `clusterIP` Fixes https://github.com/emqx/emqx/issues/10223 Adds an option to set `clusterIP` to a fixed IP if the user wants that. --- changes/ce/feat-10224.en.md | 2 ++ deploy/charts/emqx-enterprise/templates/service.yaml | 2 +- deploy/charts/emqx/templates/service.yaml | 2 +- deploy/charts/emqx/values.yaml | 3 +++ 4 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 changes/ce/feat-10224.en.md diff --git a/changes/ce/feat-10224.en.md b/changes/ce/feat-10224.en.md new file mode 100644 index 000000000..ba6303b16 --- /dev/null +++ b/changes/ce/feat-10224.en.md @@ -0,0 +1,2 @@ +Add the option to customize `clusterIP` in Helm chart, so that an user +may set it to a fixed IP. diff --git a/deploy/charts/emqx-enterprise/templates/service.yaml b/deploy/charts/emqx-enterprise/templates/service.yaml index 401746a51..dea548653 100644 --- a/deploy/charts/emqx-enterprise/templates/service.yaml +++ b/deploy/charts/emqx-enterprise/templates/service.yaml @@ -114,7 +114,7 @@ metadata: spec: type: ClusterIP sessionAffinity: None - clusterIP: None + clusterIP: {{ .Values.service.clusterIP | default "None" }} publishNotReadyAddresses: true ports: - name: mqtt diff --git a/deploy/charts/emqx/templates/service.yaml b/deploy/charts/emqx/templates/service.yaml index 401746a51..dea548653 100644 --- a/deploy/charts/emqx/templates/service.yaml +++ b/deploy/charts/emqx/templates/service.yaml @@ -114,7 +114,7 @@ metadata: spec: type: ClusterIP sessionAffinity: None - clusterIP: None + clusterIP: {{ .Values.service.clusterIP | default "None" }} publishNotReadyAddresses: true ports: - name: mqtt diff --git a/deploy/charts/emqx/values.yaml b/deploy/charts/emqx/values.yaml index f4649cc15..f7c6483fe 100644 --- a/deploy/charts/emqx/values.yaml +++ b/deploy/charts/emqx/values.yaml @@ -128,6 +128,9 @@ service: ## Service type ## type: ClusterIP + ## The cluster IP if one wants to customize it to a fixed value + ## + clusterIP: None ## Port for MQTT ## mqtt: 1883 From b02d0088ea1143b620764588d85863147b760755 Mon Sep 17 00:00:00 2001 From: Serge Tupchii Date: Thu, 23 Mar 2023 18:27:46 +0200 Subject: [PATCH 081/196] fix(emqx_management): resolve plugin name clashes Allow installing a plugin if its name matches the beginning of another (already installed) plugin name. For example: if plugin "emqx_plugin_template_a" is installed, it must not block installing plugin "emqx_plugin_template". Closes: #10213, EMQX-9290 --- .../src/emqx_mgmt_api_plugins.erl | 5 +- .../test/emqx_mgmt_api_plugins_SUITE.erl | 46 ++++++++++++++++++- changes/ce/fix-10225.en.md | 2 + 3 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 changes/ce/fix-10225.en.md diff --git a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl index f744ec0d2..a46584f7f 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl @@ -48,6 +48,9 @@ -define(NAME_RE, "^[A-Za-z]+[A-Za-z0-9-_.]*$"). -define(TAGS, [<<"Plugins">>]). +%% Plugin NameVsn must follow the pattern -, +%% app_name must be a snake_case (no '-' allowed). +-define(VSN_WILDCARD, "-*.tar.gz"). namespace() -> "plugins". @@ -334,7 +337,7 @@ upload_install(post, #{body := #{<<"plugin">> := Plugin}}) when is_map(Plugin) - case emqx_plugins:parse_name_vsn(FileName) of {ok, AppName, _Vsn} -> AppDir = filename:join(emqx_plugins:install_dir(), AppName), - case filelib:wildcard(AppDir ++ "*.tar.gz") of + case filelib:wildcard(AppDir ++ ?VSN_WILDCARD) of [] -> do_install_package(FileName, Bin); OtherVsn -> diff --git a/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl index 0cf15d678..24e55494d 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl @@ -20,6 +20,7 @@ -include_lib("eunit/include/eunit.hrl"). +-define(EMQX_PLUGIN_TEMPLATE_NAME, "emqx_plugin_template"). -define(EMQX_PLUGIN_TEMPLATE_VSN, "5.0.0"). -define(PACKAGE_SUFFIX, ".tar.gz"). @@ -89,6 +90,27 @@ t_plugins(Config) -> {ok, []} = uninstall_plugin(NameVsn), ok. +t_install_plugin_matching_exisiting_name(Config) -> + DemoShDir = proplists:get_value(demo_sh_dir, Config), + PackagePath = get_demo_plugin_package(DemoShDir), + NameVsn = filename:basename(PackagePath, ?PACKAGE_SUFFIX), + ok = emqx_plugins:ensure_uninstalled(NameVsn), + ok = emqx_plugins:delete_package(NameVsn), + NameVsn1 = ?EMQX_PLUGIN_TEMPLATE_NAME ++ "_a" ++ "-" ++ ?EMQX_PLUGIN_TEMPLATE_VSN, + PackagePath1 = create_renamed_package(PackagePath, NameVsn1), + NameVsn1 = filename:basename(PackagePath1, ?PACKAGE_SUFFIX), + ok = emqx_plugins:ensure_uninstalled(NameVsn1), + ok = emqx_plugins:delete_package(NameVsn1), + %% First, install plugin "emqx_plugin_template_a", then: + %% "emqx_plugin_template" which matches the beginning + %% of the previously installed plugin name + ok = install_plugin(PackagePath1), + ok = install_plugin(PackagePath), + {ok, _} = describe_plugins(NameVsn), + {ok, _} = describe_plugins(NameVsn1), + {ok, _} = uninstall_plugin(NameVsn), + {ok, _} = uninstall_plugin(NameVsn1). + t_bad_plugin(Config) -> DemoShDir = proplists:get_value(demo_sh_dir, Config), PackagePathOrig = get_demo_plugin_package(DemoShDir), @@ -160,9 +182,31 @@ uninstall_plugin(Name) -> get_demo_plugin_package(Dir) -> #{package := Pkg} = emqx_plugins_SUITE:get_demo_plugin_package(), - FileName = "emqx_plugin_template-" ++ ?EMQX_PLUGIN_TEMPLATE_VSN ++ ?PACKAGE_SUFFIX, + FileName = ?EMQX_PLUGIN_TEMPLATE_NAME ++ "-" ++ ?EMQX_PLUGIN_TEMPLATE_VSN ++ ?PACKAGE_SUFFIX, PluginPath = "./" ++ FileName, Pkg = filename:join([Dir, FileName]), _ = os:cmd("cp " ++ Pkg ++ " " ++ PluginPath), true = filelib:is_regular(PluginPath), PluginPath. + +create_renamed_package(PackagePath, NewNameVsn) -> + {ok, Content} = erl_tar:extract(PackagePath, [compressed, memory]), + {ok, NewName, _Vsn} = emqx_plugins:parse_name_vsn(NewNameVsn), + NewNameB = atom_to_binary(NewName, utf8), + Content1 = lists:map( + fun({F, B}) -> + [_ | PathPart] = filename:split(F), + B1 = update_release_json(PathPart, B, NewNameB), + {filename:join([NewNameVsn | PathPart]), B1} + end, + Content + ), + NewPackagePath = filename:join(filename:dirname(PackagePath), NewNameVsn ++ ?PACKAGE_SUFFIX), + ok = erl_tar:create(NewPackagePath, Content1, [compressed]), + NewPackagePath. + +update_release_json(["release.json"], FileContent, NewName) -> + ContentMap = emqx_json:decode(FileContent, [return_maps]), + emqx_json:encode(ContentMap#{<<"name">> => NewName}); +update_release_json(_FileName, FileContent, _NewName) -> + FileContent. diff --git a/changes/ce/fix-10225.en.md b/changes/ce/fix-10225.en.md new file mode 100644 index 000000000..20f7dfa47 --- /dev/null +++ b/changes/ce/fix-10225.en.md @@ -0,0 +1,2 @@ +Allow installing a plugin if its name matches the beginning of another (already installed) plugin name. +For example: if plugin "emqx_plugin_template_a" is installed, it must not block installing plugin "emqx_plugin_template". From 9f006d0e2a817f9b0122df41697aa3eed808a01d Mon Sep 17 00:00:00 2001 From: Serge Tupchii Date: Fri, 24 Mar 2023 13:37:05 +0200 Subject: [PATCH 082/196] ci: run start-two-nodes-in-docker.sh test script in CI Closes: EMQX-8884 --- .github/workflows/build_slim_packages.yaml | 17 +++++-- scripts/test/start-two-nodes-in-docker.sh | 57 ++++++++++++++-------- 2 files changed, 50 insertions(+), 24 deletions(-) diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 30af8bdbc..7f7f35e16 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -165,19 +165,21 @@ jobs: fail-fast: false matrix: profile: - - emqx - - emqx-enterprise + - ["emqx", "5.0.16"] + - ["emqx-enterprise", "5.0.1"] steps: - uses: actions/checkout@v3 - name: prepare run: | - EMQX_NAME=${{ matrix.profile }} + EMQX_NAME=${{ matrix.profile[0] }} PKG_VSN=${PKG_VSN:-$(./pkg-vsn.sh $EMQX_NAME)} EMQX_IMAGE_TAG=emqx/$EMQX_NAME:test + EMQX_IMAGE_OLD_VERSION_TAG=emqx/$EMQX_NAME:${{ matrix.profile[1] }} echo "EMQX_NAME=$EMQX_NAME" >> $GITHUB_ENV echo "PKG_VSN=$PKG_VSN" >> $GITHUB_ENV echo "EMQX_IMAGE_TAG=$EMQX_IMAGE_TAG" >> $GITHUB_ENV + echo "EMQX_IMAGE_OLD_VERSION_TAG=$EMQX_IMAGE_OLD_VERSION_TAG" >> $GITHUB_ENV - uses: docker/setup-buildx-action@v2 - name: build and export to Docker uses: docker/build-push-action@v4 @@ -194,12 +196,19 @@ jobs: HTTP_PORT=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "18083/tcp") 0).HostPort}}' $CID) ./scripts/test/emqx-smoke-test.sh localhost $HTTP_PORT docker stop $CID + - name: test two nodes cluster with proto_dist=inet_tls in docker + run: | + ./scripts/test/start-two-nodes-in-docker.sh -P $EMQX_IMAGE_TAG $EMQX_IMAGE_OLD_VERSION_TAG + HTTP_PORT=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "18083/tcp") 0).HostPort}}' haproxy) + ./scripts/test/emqx-smoke-test.sh localhost $HTTP_PORT + # cleanup + ./scripts/test/start-two-nodes-in-docker.sh -c - name: export docker image run: | docker save $EMQX_IMAGE_TAG | gzip > $EMQX_NAME-$PKG_VSN.tar.gz - uses: actions/upload-artifact@v3 with: - name: "${{ matrix.profile }}-docker" + name: "${{ matrix.profile[0] }}-docker" path: "${{ env.EMQX_NAME }}-${{ env.PKG_VSN }}.tar.gz" spellcheck: diff --git a/scripts/test/start-two-nodes-in-docker.sh b/scripts/test/start-two-nodes-in-docker.sh index c174bc630..53689b207 100755 --- a/scripts/test/start-two-nodes-in-docker.sh +++ b/scripts/test/start-two-nodes-in-docker.sh @@ -10,19 +10,36 @@ set -euo pipefail # ensure dir cd -P -- "$(dirname -- "$0")/../../" -IMAGE1="${1}" -IMAGE2="${2:-${IMAGE1}}" +HAPROXY_PORTS=(-p 18083:18083 -p 8883:8883 -p 8084:8084) NET='emqx.io' NODE1="node1.$NET" NODE2="node2.$NET" COOKIE='this-is-a-secret' -## clean up -docker rm -f haproxy >/dev/null 2>&1 || true -docker rm -f "$NODE1" >/dev/null 2>&1 || true -docker rm -f "$NODE2" >/dev/null 2>&1 || true -docker network rm "$NET" >/dev/null 2>&1 || true +cleanup() { + docker rm -f haproxy >/dev/null 2>&1 || true + docker rm -f "$NODE1" >/dev/null 2>&1 || true + docker rm -f "$NODE2" >/dev/null 2>&1 || true + docker network rm "$NET" >/dev/null 2>&1 || true +} + +while getopts ":Pc" opt +do + case $opt in + # -P option is treated similarly to docker run -P: + # publish ports to random available host ports + P) HAPROXY_PORTS=(-p 18083 -p 8883 -p 8084);; + c) cleanup; exit 0;; + *) ;; + esac +done +shift $((OPTIND - 1)) + +IMAGE1="${1}" +IMAGE2="${2:-${IMAGE1}}" + +cleanup docker network create "$NET" @@ -128,18 +145,18 @@ backend emqx_wss_back EOF -docker run -d --name haproxy \ - --net "$NET" \ - -v "$(pwd)/tmp/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg" \ - -v "$(pwd)/apps/emqx/etc/certs:/usr/local/etc/haproxy/certs" \ - -w /usr/local/etc/haproxy \ - -p 18083:18083 \ - -p 8883:8883 \ - -p 8084:8084 \ - "haproxy:2.4" \ - bash -c 'set -euo pipefail; - cat certs/cert.pem certs/key.pem > /tmp/emqx.pem; - haproxy -f haproxy.cfg' +haproxy_cid=$(docker run -d --name haproxy \ + --net "$NET" \ + -v "$(pwd)/tmp/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg" \ + -v "$(pwd)/apps/emqx/etc/certs:/usr/local/etc/haproxy/certs" \ + -w /usr/local/etc/haproxy \ + "${HAPROXY_PORTS[@]}" \ + "haproxy:2.4" \ + bash -c 'set -euo pipefail; + cat certs/cert.pem certs/key.pem > /tmp/emqx.pem; + haproxy -f haproxy.cfg') + +haproxy_ssl_port=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "8084/tcp") 0).HostPort}}' "$haproxy_cid") wait_limit=60 wait_for_emqx() { @@ -165,7 +182,7 @@ wait_for_haproxy() { -CAfile apps/emqx/etc/certs/cacert.pem \ -cert apps/emqx/etc/certs/cert.pem \ -key apps/emqx/etc/certs/key.pem \ - localhost:8084 Date: Fri, 24 Mar 2023 21:46:26 +0100 Subject: [PATCH 083/196] chore: bump app vsns --- apps/emqx/src/emqx.app.src | 2 +- apps/emqx_bridge/src/emqx_bridge.app.src | 2 +- apps/emqx_dashboard/src/emqx_dashboard.app.src | 2 +- apps/emqx_gateway/src/emqx_gateway.app.src | 2 +- apps/emqx_management/src/emqx_management.app.src | 2 +- apps/emqx_resource/src/emqx_resource.app.src | 2 +- apps/emqx_rule_engine/src/emqx_rule_engine.app.src | 2 +- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src | 2 +- lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/apps/emqx/src/emqx.app.src b/apps/emqx/src/emqx.app.src index d2831b74d..1cecd7b61 100644 --- a/apps/emqx/src/emqx.app.src +++ b/apps/emqx/src/emqx.app.src @@ -3,7 +3,7 @@ {id, "emqx"}, {description, "EMQX Core"}, % strict semver, bump manually! - {vsn, "5.0.20"}, + {vsn, "5.0.21"}, {modules, []}, {registered, []}, {applications, [ diff --git a/apps/emqx_bridge/src/emqx_bridge.app.src b/apps/emqx_bridge/src/emqx_bridge.app.src index 99a49f8fd..f5bcb23e2 100644 --- a/apps/emqx_bridge/src/emqx_bridge.app.src +++ b/apps/emqx_bridge/src/emqx_bridge.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_bridge, [ {description, "EMQX bridges"}, - {vsn, "0.1.13"}, + {vsn, "0.1.14"}, {registered, [emqx_bridge_sup]}, {mod, {emqx_bridge_app, []}}, {applications, [ diff --git a/apps/emqx_dashboard/src/emqx_dashboard.app.src b/apps/emqx_dashboard/src/emqx_dashboard.app.src index 3970d76e4..8a4764c84 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.app.src +++ b/apps/emqx_dashboard/src/emqx_dashboard.app.src @@ -2,7 +2,7 @@ {application, emqx_dashboard, [ {description, "EMQX Web Dashboard"}, % strict semver, bump manually! - {vsn, "5.0.15"}, + {vsn, "5.0.16"}, {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [kernel, stdlib, mnesia, minirest, emqx, emqx_ctl]}, diff --git a/apps/emqx_gateway/src/emqx_gateway.app.src b/apps/emqx_gateway/src/emqx_gateway.app.src index 59eed7f3f..ced013497 100644 --- a/apps/emqx_gateway/src/emqx_gateway.app.src +++ b/apps/emqx_gateway/src/emqx_gateway.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_gateway, [ {description, "The Gateway management application"}, - {vsn, "0.1.13"}, + {vsn, "0.1.14"}, {registered, []}, {mod, {emqx_gateway_app, []}}, {applications, [kernel, stdlib, grpc, emqx, emqx_authn, emqx_ctl]}, diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index 966358f47..9863f5cf6 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -2,7 +2,7 @@ {application, emqx_management, [ {description, "EMQX Management API and CLI"}, % strict semver, bump manually! - {vsn, "5.0.16"}, + {vsn, "5.0.17"}, {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel, stdlib, emqx_plugins, minirest, emqx, emqx_ctl]}, diff --git a/apps/emqx_resource/src/emqx_resource.app.src b/apps/emqx_resource/src/emqx_resource.app.src index 7be1bcb1c..fbfe8c1fa 100644 --- a/apps/emqx_resource/src/emqx_resource.app.src +++ b/apps/emqx_resource/src/emqx_resource.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_resource, [ {description, "Manager for all external resources"}, - {vsn, "0.1.10"}, + {vsn, "0.1.11"}, {registered, []}, {mod, {emqx_resource_app, []}}, {applications, [ diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index 1681297ec..8d50f60e3 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -2,7 +2,7 @@ {application, emqx_rule_engine, [ {description, "EMQX Rule Engine"}, % strict semver, bump manually! - {vsn, "5.0.11"}, + {vsn, "5.0.12"}, {modules, []}, {registered, [emqx_rule_engine_sup, emqx_rule_engine]}, {applications, [kernel, stdlib, rulesql, getopt, emqx_ctl]}, diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src index 6647ec212..156c3eeac 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src @@ -1,6 +1,6 @@ {application, emqx_ee_bridge, [ {description, "EMQX Enterprise data bridges"}, - {vsn, "0.1.7"}, + {vsn, "0.1.8"}, {registered, [emqx_ee_bridge_kafka_consumer_sup]}, {applications, [ kernel, diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src index d8921198c..6a9f8f924 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src @@ -1,6 +1,6 @@ {application, emqx_ee_connector, [ {description, "EMQX Enterprise connectors"}, - {vsn, "0.1.7"}, + {vsn, "0.1.8"}, {registered, []}, {applications, [ kernel, From 19e136bb66614342d720649da64b3d32ac67f2de Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sun, 26 Mar 2023 10:22:04 +0200 Subject: [PATCH 084/196] fix: avoid uisng 'pid' as log data field Since it clashes with log metad data field --- apps/emqx/src/emqx_cm.erl | 14 +++++++------- apps/emqx_gateway/src/emqx_gateway_cm.erl | 12 ++++++------ changes/ce/fix-10242.en.md | 2 ++ changes/ce/fix-10242.zh.md | 2 ++ lib-ee/emqx_license/src/emqx_license.app.src | 2 +- lib-ee/emqx_license/src/emqx_license_installer.erl | 6 +++--- 6 files changed, 21 insertions(+), 17 deletions(-) create mode 100644 changes/ce/fix-10242.en.md create mode 100644 changes/ce/fix-10242.zh.md diff --git a/apps/emqx/src/emqx_cm.erl b/apps/emqx/src/emqx_cm.erl index 6de05dabe..f8c510482 100644 --- a/apps/emqx/src/emqx_cm.erl +++ b/apps/emqx/src/emqx_cm.erl @@ -465,23 +465,23 @@ request_stepdown(Action, ConnMod, Pid) -> catch % emqx_ws_connection: call _:noproc -> - ok = ?tp(debug, "session_already_gone", #{pid => Pid, action => Action}), + ok = ?tp(debug, "session_already_gone", #{stale_pid => Pid, action => Action}), {error, noproc}; % emqx_connection: gen_server:call _:{noproc, _} -> - ok = ?tp(debug, "session_already_gone", #{pid => Pid, action => Action}), + ok = ?tp(debug, "session_already_gone", #{stale_pid => Pid, action => Action}), {error, noproc}; _:{shutdown, _} -> - ok = ?tp(debug, "session_already_shutdown", #{pid => Pid, action => Action}), + ok = ?tp(debug, "session_already_shutdown", #{stale_pid => Pid, action => Action}), {error, noproc}; _:{{shutdown, _}, _} -> - ok = ?tp(debug, "session_already_shutdown", #{pid => Pid, action => Action}), + ok = ?tp(debug, "session_already_shutdown", #{stale_pid => Pid, action => Action}), {error, noproc}; _:{timeout, {gen_server, call, _}} -> ?tp( warning, "session_stepdown_request_timeout", - #{pid => Pid, action => Action, stale_channel => stale_channel_info(Pid)} + #{stale_pid => Pid, action => Action, stale_channel => stale_channel_info(Pid)} ), ok = force_kill(Pid), {error, timeout}; @@ -490,7 +490,7 @@ request_stepdown(Action, ConnMod, Pid) -> error, "session_stepdown_request_exception", #{ - pid => Pid, + stale_pid => Pid, action => Action, reason => Error, stacktrace => St, @@ -671,7 +671,7 @@ handle_cast(Msg, State) -> {noreply, State}. handle_info({'DOWN', _MRef, process, Pid, _Reason}, State = #{chan_pmon := PMon}) -> - ?tp(emqx_cm_process_down, #{pid => Pid, reason => _Reason}), + ?tp(emqx_cm_process_down, #{stale_pid => Pid, reason => _Reason}), ChanPids = [Pid | emqx_misc:drain_down(?BATCH_SIZE)], {Items, PMon1} = emqx_pmon:erase_all(ChanPids, PMon), lists:foreach(fun mark_channel_disconnected/1, ChanPids), diff --git a/apps/emqx_gateway/src/emqx_gateway_cm.erl b/apps/emqx_gateway/src/emqx_gateway_cm.erl index 599493d97..71ec4bf59 100644 --- a/apps/emqx_gateway/src/emqx_gateway_cm.erl +++ b/apps/emqx_gateway/src/emqx_gateway_cm.erl @@ -587,24 +587,24 @@ request_stepdown(Action, ConnMod, Pid) -> catch % emqx_ws_connection: call _:noproc -> - ok = ?tp(debug, "session_already_gone", #{pid => Pid, action => Action}), + ok = ?tp(debug, "session_already_gone", #{stale_pid => Pid, action => Action}), {error, noproc}; % emqx_connection: gen_server:call _:{noproc, _} -> - ok = ?tp(debug, "session_already_gone", #{pid => Pid, action => Action}), + ok = ?tp(debug, "session_already_gone", #{stale_pid => Pid, action => Action}), {error, noproc}; _:Reason = {shutdown, _} -> - ok = ?tp(debug, "session_already_shutdown", #{pid => Pid, action => Action}), + ok = ?tp(debug, "session_already_shutdown", #{stale_pid => Pid, action => Action}), {error, Reason}; _:Reason = {{shutdown, _}, _} -> - ok = ?tp(debug, "session_already_shutdown", #{pid => Pid, action => Action}), + ok = ?tp(debug, "session_already_shutdown", #{stale_pid => Pid, action => Action}), {error, Reason}; _:{timeout, {gen_server, call, _}} -> ?tp( warning, "session_stepdown_request_timeout", #{ - pid => Pid, + stale_pid => Pid, action => Action, stale_channel => stale_channel_info(Pid) } @@ -616,7 +616,7 @@ request_stepdown(Action, ConnMod, Pid) -> error, "session_stepdown_request_exception", #{ - pid => Pid, + stale_pid => Pid, action => Action, reason => Error, stacktrace => St, diff --git a/changes/ce/fix-10242.en.md b/changes/ce/fix-10242.en.md new file mode 100644 index 000000000..bc4309b94 --- /dev/null +++ b/changes/ce/fix-10242.en.md @@ -0,0 +1,2 @@ +Fixed a log data field name clash. +Piror to this fix, some debug logs may report a wrong Erlang PID which may affect troubleshooting session takeover issues. diff --git a/changes/ce/fix-10242.zh.md b/changes/ce/fix-10242.zh.md new file mode 100644 index 000000000..36c0a1556 --- /dev/null +++ b/changes/ce/fix-10242.zh.md @@ -0,0 +1,2 @@ +修复log数据字段名称冲突。 +在这个修复之前，一些调试日志可能会报告一个错误的Erlang PID，这可能会影响会话接管问题的故障调查。 diff --git a/lib-ee/emqx_license/src/emqx_license.app.src b/lib-ee/emqx_license/src/emqx_license.app.src index 7a569c402..0a97ee83b 100644 --- a/lib-ee/emqx_license/src/emqx_license.app.src +++ b/lib-ee/emqx_license/src/emqx_license.app.src @@ -1,6 +1,6 @@ {application, emqx_license, [ {description, "EMQX License"}, - {vsn, "5.0.7"}, + {vsn, "5.0.8"}, {modules, []}, {registered, [emqx_license_sup]}, {applications, [kernel, stdlib, emqx_ctl]}, diff --git a/lib-ee/emqx_license/src/emqx_license_installer.erl b/lib-ee/emqx_license/src/emqx_license_installer.erl index 7a6ea5339..58ee6ebcc 100644 --- a/lib-ee/emqx_license/src/emqx_license_installer.erl +++ b/lib-ee/emqx_license/src/emqx_license_installer.erl @@ -74,13 +74,13 @@ ensure_timer(#{interval := Interval} = State) -> check_pid(#{name := Name, pid := OldPid, callback := Callback} = State) -> case whereis(Name) of undefined -> - ?tp(debug, emqx_license_installer_noproc, #{pid => OldPid}), + ?tp(debug, emqx_license_installer_noproc, #{old_pid => OldPid}), State; OldPid -> - ?tp(debug, emqx_license_installer_nochange, #{pid => OldPid}), + ?tp(debug, emqx_license_installer_nochange, #{old_pid => OldPid}), State; NewPid -> _ = Callback(), - ?tp(debug, emqx_license_installer_called, #{pid => OldPid}), + ?tp(debug, emqx_license_installer_called, #{old_pid => OldPid}), State#{pid => NewPid} end. From acb9bcb60309cb10a0369bc1362a9222cf07d2a8 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sun, 26 Mar 2023 10:46:03 +0200 Subject: [PATCH 085/196] build: check git diff against current working dir when checking app vsn but not the HEAD commit --- scripts/apps-version-check.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/apps-version-check.sh b/scripts/apps-version-check.sh index b86101027..2918ff5f7 100755 --- a/scripts/apps-version-check.sh +++ b/scripts/apps-version-check.sh @@ -36,7 +36,7 @@ for app in ${APPS}; do echo "IGNORE: $src_file is newly added" true elif [ "$old_app_version" = "$now_app_version" ]; then - changed_lines="$(git diff "$latest_release"...HEAD --ignore-blank-lines -G "$no_comment_re" \ + changed_lines="$(git diff "$latest_release" --ignore-blank-lines -G "$no_comment_re" \ -- "$app_path/src" \ -- "$app_path/include" \ -- ":(exclude)"$app_path/src/*.appup.src"" \ From 07ac2cd57aaba0f2e1776ad36ddbed85475a753a Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Mon, 27 Mar 2023 09:33:17 +0800 Subject: [PATCH 086/196] chore: remove peercert from the state of connection processes --- apps/emqx/src/emqx_channel.erl | 4 +++- apps/emqx/test/emqx_client_SUITE.erl | 6 ++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index e01a16f83..29a59e482 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -276,7 +276,9 @@ init( ), {NClientInfo, NConnInfo} = take_ws_cookie(ClientInfo, ConnInfo), #channel{ - conninfo = NConnInfo, + %% We remove the peercert because it duplicates to what's stored in the socket, + %% Saving a copy here causes unnecessary wast of memory (about 1KB per connection). + conninfo = maps:put(peercert, undefined, NConnInfo), clientinfo = NClientInfo, topic_aliases = #{ inbound => #{}, diff --git a/apps/emqx/test/emqx_client_SUITE.erl b/apps/emqx/test/emqx_client_SUITE.erl index 2f433c73d..82d4038da 100644 --- a/apps/emqx/test/emqx_client_SUITE.erl +++ b/apps/emqx/test/emqx_client_SUITE.erl @@ -390,4 +390,10 @@ tls_certcn_as_clientid(TLSVsn, RequiredTLSVsn) -> {ok, _} = emqtt:connect(Client), #{clientinfo := #{clientid := CN}} = emqx_cm:get_chan_info(CN), confirm_tls_version(Client, RequiredTLSVsn), + %% verify that the peercert won't be stored in the conninfo + [ChannPid] = emqx_cm:lookup_channels(CN), + SysState = sys:get_state(ChannPid), + ChannelRecord = lists:keyfind(channel, 1, tuple_to_list(SysState)), + ConnInfo = lists:nth(2, tuple_to_list(ChannelRecord)), + ?assertMatch(#{peercert := undefined}, ConnInfo), emqtt:disconnect(Client). From 7085a2d6d7d092224f8096512a3b77ea90905713 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 27 Mar 2023 09:47:51 +0800 Subject: [PATCH 087/196] chore: hidden the password field ref: https://github.com/emqx/emqx/issues/10222 Co-authored-by: Thales Macedo Garitezi --- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl index 20821dc8f..12f86fcf7 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl @@ -53,7 +53,7 @@ values(_Method, Type) -> keyspace => <<"mqtt">>, pool_size => 8, username => <<"root">>, - password => <<"public">>, + password => <<"******">>, cql => ?DEFAULT_CQL, local_topic => <<"local/topic/#">>, resource_opts => #{ From 5c9538086fc7fadfb6cd86d96df96c34dcb39041 Mon Sep 17 00:00:00 2001 From: firest Date: Mon, 27 Mar 2023 13:51:36 +0800 Subject: [PATCH 088/196] chore: bump emqx_plugin_libs.app.src --- apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src index 605fdb346..dcb330df4 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_plugin_libs, [ {description, "EMQX Plugin utility libs"}, - {vsn, "4.3.7"}, + {vsn, "4.3.8"}, {modules, []}, {applications, [kernel, stdlib]}, {env, []} From 4c17a31a4a63812ef095aa67e7cae823998b1fec Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Mon, 27 Mar 2023 11:12:24 +0200 Subject: [PATCH 089/196] chore: setup @Meggielqk as codeowner for i18n --- .github/CODEOWNERS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 4de5a83b3..5db0f4465 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -22,5 +22,8 @@ ## CI /deploy/ @emqx/emqx-review-board @Rory-Z +## @Meggielqk owns all files in any i18n directory anywhere in the project +/i18n/ @Meggielqk + ## no owner for changelogs, anyone can approve /changes From 9a93b745fc6d989aea643f049425e8c291409465 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 23 Mar 2023 00:02:48 +0300 Subject: [PATCH 090/196] fix(dashboard): use correct default for listeners map --- apps/emqx_dashboard/src/emqx_dashboard.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard.erl b/apps/emqx_dashboard/src/emqx_dashboard.erl index f0344dd5a..6f0c8334a 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard.erl @@ -262,7 +262,7 @@ i18n_file() -> end. listeners() -> - emqx_conf:get([dashboard, listeners], []). + emqx_conf:get([dashboard, listeners], #{}). api_key_authorize(Req, Key, Secret) -> Path = cowboy_req:path(Req), From 5cbf038ac14a30a38156592acbb125a478bce784 Mon Sep 17 00:00:00 2001 From: zhouzb Date: Mon, 27 Mar 2023 17:44:21 +0800 Subject: [PATCH 091/196] chore: update readme --- README-CN.md | 3 --- README-RU.md | 1 - README.md | 3 --- 3 files changed, 7 deletions(-) diff --git a/README-CN.md b/README-CN.md index 193e5ab98..3eccc267c 100644 --- a/README-CN.md +++ b/README-CN.md @@ -11,9 +11,6 @@ [![YouTube](https://img.shields.io/badge/Subscribe-EMQ%20中文-FF0000?logo=youtube)](https://www.youtube.com/channel/UCir_r04HIsLjf2qqyZ4A8Cg) - -[English](./README.md) | 简体中文 | [русский](./README-RU.md) - EMQX 是一款全球下载量超千万的大规模分布式物联网 MQTT 服务器，单集群支持 1 亿物联网设备连接，消息分发时延低于 1 毫秒。为高可靠、高性能的物联网实时数据移动、处理和集成提供动力，助力企业构建关键业务的 IoT 平台与应用。 EMQX 自 2013 年在 GitHub 发布开源版本以来，获得了来自 50 多个国家和地区的 20000 余家企业用户的广泛认可，累计连接物联网关键设备超过 1 亿台。 diff --git a/README-RU.md b/README-RU.md index fb5ff9608..b8be19e80 100644 --- a/README-RU.md +++ b/README-RU.md @@ -9,7 +9,6 @@ [![Twitter](https://img.shields.io/badge/Follow-EMQ-1DA1F2?logo=twitter)](https://twitter.com/EMQTech) [![YouTube](https://img.shields.io/badge/Subscribe-EMQ-FF0000?logo=youtube)](https://www.youtube.com/channel/UC5FjR77ErAxvZENEWzQaO5Q) -[English](./README.md) | [简体中文](./README-CN.md) | русский *EMQX* — это самый масштабируемый и популярный высокопроизводительный MQTT брокер с полностью открытым кодом для интернета вещей, межмашинного взаимодействия и мобильных приложений. EMQX может поддерживать более чем 100 миллионов одновременных соединенией на одном кластере с задержкой в 1 миллисекунду, а также принимать и обрабабывать миллионы MQTT сообщений в секунду. diff --git a/README.md b/README.md index 94baba04f..9a3387902 100644 --- a/README.md +++ b/README.md @@ -10,9 +10,6 @@ [![YouTube](https://img.shields.io/badge/Subscribe-EMQ-FF0000?logo=youtube)](https://www.youtube.com/channel/UC5FjR77ErAxvZENEWzQaO5Q) - -English | [简体中文](./README-CN.md) | [русский](./README-RU.md) - EMQX is the world's most scalable open-source MQTT broker with a high performance that connects 100M+ IoT devices in 1 cluster, while maintaining 1M message per second throughput and sub-millisecond latency. EMQX supports multiple open standard protocols like MQTT, HTTP, QUIC, and WebSocket. It’s 100% compliant with MQTT 5.0 and 3.x standard, and secures bi-directional communication with MQTT over TLS/SSL and various authentication mechanisms. From 70a1c25d0fa9288007925fe98aa1e24ce956bce6 Mon Sep 17 00:00:00 2001 From: William Yang Date: Tue, 21 Mar 2023 12:06:18 +0100 Subject: [PATCH 092/196] docs: add changelogs feat-10077 Co-authored-by: Thales Macedo Garitezi --- changes/ce/feat-10077.en.md | 2 ++ changes/ce/feat-10077.zh.md | 1 + 2 files changed, 3 insertions(+) create mode 100644 changes/ce/feat-10077.en.md create mode 100644 changes/ce/feat-10077.zh.md diff --git a/changes/ce/feat-10077.en.md b/changes/ce/feat-10077.en.md new file mode 100644 index 000000000..923e21fa1 --- /dev/null +++ b/changes/ce/feat-10077.en.md @@ -0,0 +1,2 @@ +Add support for QUIC TLS password protected certificate file. + diff --git a/changes/ce/feat-10077.zh.md b/changes/ce/feat-10077.zh.md new file mode 100644 index 000000000..e9c7b5625 --- /dev/null +++ b/changes/ce/feat-10077.zh.md @@ -0,0 +1 @@ +增加对 QUIC TLS 密码保护证书文件的支持。 From 169cc9f822885d44a168f2c06175769419143cbd Mon Sep 17 00:00:00 2001 From: William Yang Date: Tue, 21 Mar 2023 12:11:03 +0100 Subject: [PATCH 093/196] chore(quic): unhide TLS certfile password --- apps/emqx/src/emqx_schema.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 433fb20e5..b7a8aed64 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -3019,9 +3019,9 @@ is_quic_ssl_opts(Name) -> "cacertfile", "certfile", "keyfile", - "verify" + "verify", + "password" %% Followings are planned - %% , "password" %% , "hibernate_after" %% , "fail_if_no_peer_cert" %% , "handshake_timeout" From cec77c2b6529a332add7af71852a966abec25c88 Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 22 Mar 2023 15:14:39 +0100 Subject: [PATCH 094/196] test(quic): chasing flaky tc. --- apps/emqx/test/emqx_quic_multistreams_SUITE.erl | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl index 642b5468c..4afd965bd 100644 --- a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl +++ b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl @@ -1569,7 +1569,7 @@ t_multi_streams_remote_shutdown(Config) -> ok = stop_emqx(), %% Client should be closed - assert_client_die(C). + assert_client_die(C, 100, 50). t_multi_streams_remote_shutdown_with_reconnect(Config) -> erlang:process_flag(trap_exit, true), @@ -2047,14 +2047,15 @@ via_stream({quic, _Conn, Stream}) -> assert_client_die(C) -> assert_client_die(C, 100, 10). assert_client_die(C, _, 0) -> - ct:fail("Client ~p did not die", [C]); + ct:fail("Client ~p did not die: stacktrace: ~p", [C, process_info(C, current_stacktrace)]); assert_client_die(C, Delay, Retries) -> - case catch emqtt:info(C) of - {'EXIT', {noproc, {gen_statem, call, [_, info, infinity]}}} -> - ok; - _Other -> + try emqtt:info(C) of + Info when is_list(Info) -> timer:sleep(Delay), assert_client_die(C, Delay, Retries - 1) + catch + exit:Error -> + ct:comment("client die with ~p", [Error]) end. %% BUILD_WITHOUT_QUIC From 64a1d84a44cf94cb5b5e725f30cc0d2abc54ae50 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 24 Mar 2023 16:42:21 +0100 Subject: [PATCH 095/196] fix: create consistent interface 'with_node' for API access --- apps/emqx/include/emqx_api_lib.hrl | 36 +++++++ apps/emqx/src/emqx_api_lib.erl | 69 ++++++++++++ apps/emqx/test/emqx_api_lib_SUITE.erl | 101 ++++++++++++++++++ apps/emqx_bridge/src/emqx_bridge_api.erl | 20 +--- .../src/emqx_dashboard_monitor_api.erl | 27 ++--- .../src/emqx_mgmt_api_nodes.erl | 62 +++++------ .../test/emqx_mgmt_api_nodes_SUITE.erl | 6 +- changes/ce/fix-10237.en.md | 1 + 8 files changed, 247 insertions(+), 75 deletions(-) create mode 100644 apps/emqx/include/emqx_api_lib.hrl create mode 100644 apps/emqx/src/emqx_api_lib.erl create mode 100644 apps/emqx/test/emqx_api_lib_SUITE.erl create mode 100644 changes/ce/fix-10237.en.md diff --git a/apps/emqx/include/emqx_api_lib.hrl b/apps/emqx/include/emqx_api_lib.hrl new file mode 100644 index 000000000..549b0f94c --- /dev/null +++ b/apps/emqx/include/emqx_api_lib.hrl @@ -0,0 +1,36 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-ifndef(EMQX_API_LIB_HRL). +-define(EMQX_API_LIB_HRL, true). + +-define(ERROR_MSG(CODE, REASON), #{code => CODE, message => emqx_misc:readable_error_msg(REASON)}). + +-define(OK(CONTENT), {200, CONTENT}). + +-define(NO_CONTENT, 204). + +-define(BAD_REQUEST(CODE, REASON), {400, ?ERROR_MSG(CODE, REASON)}). +-define(BAD_REQUEST(REASON), ?BAD_REQUEST('BAD_REQUEST', REASON)). + +-define(NOT_FOUND(REASON), {404, ?ERROR_MSG('NOT_FOUND', REASON)}). + +-define(INTERNAL_ERROR(REASON), {500, ?ERROR_MSG('INTERNAL_ERROR', REASON)}). + +-define(NOT_IMPLEMENTED, 501). + +-define(SERVICE_UNAVAILABLE(REASON), {503, ?ERROR_MSG('SERVICE_UNAVAILABLE', REASON)}). +-endif. diff --git a/apps/emqx/src/emqx_api_lib.erl b/apps/emqx/src/emqx_api_lib.erl new file mode 100644 index 000000000..8c49c57c3 --- /dev/null +++ b/apps/emqx/src/emqx_api_lib.erl @@ -0,0 +1,69 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_api_lib). + +-export([ + with_node/2, + with_node_or_cluster/2 +]). + +-include("emqx_api_lib.hrl"). + +-define(NODE_NOT_FOUND(NODE), ?NOT_FOUND(<<"Node not found: ", NODE/binary>>)). + +%%-------------------------------------------------------------------- +%% exported API +%%-------------------------------------------------------------------- +-spec with_node(binary(), fun((atom()) -> {ok, term()} | {error, term()})) -> + ?OK(term()) | ?NOT_FOUND(binary()) | ?BAD_REQUEST(term()). +with_node(BinNode, Fun) -> + case lookup_node(BinNode) of + {ok, Node} -> + handle_result(Fun(Node)); + not_found -> + ?NODE_NOT_FOUND(BinNode) + end. + +-spec with_node_or_cluster(binary(), fun((atom()) -> {ok, term()} | {error, term()})) -> + ?OK(term()) | ?NOT_FOUND(iolist()) | ?BAD_REQUEST(term()). +with_node_or_cluster(<<"all">>, Fun) -> + handle_result(Fun(all)); +with_node_or_cluster(Node, Fun) -> + with_node(Node, Fun). + +%%-------------------------------------------------------------------- +%% Internal +%%-------------------------------------------------------------------- + +-spec lookup_node(binary()) -> {ok, atom()} | not_found. +lookup_node(BinNode) -> + case emqx_misc:safe_to_existing_atom(BinNode, utf8) of + {ok, Node} -> + case lists:member(Node, mria:running_nodes()) of + true -> + {ok, Node}; + false -> + not_found + end; + _Error -> + not_found + end. + +handle_result({ok, Result}) -> + ?OK(Result); +handle_result({error, Reason}) -> + ?BAD_REQUEST(Reason). diff --git a/apps/emqx/test/emqx_api_lib_SUITE.erl b/apps/emqx/test/emqx_api_lib_SUITE.erl new file mode 100644 index 000000000..29f5c6095 --- /dev/null +++ b/apps/emqx/test/emqx_api_lib_SUITE.erl @@ -0,0 +1,101 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_api_lib_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include("emqx_api_lib.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-define(DUMMY, dummy_module). + +all() -> emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + emqx_common_test_helpers:boot_modules(all), + emqx_common_test_helpers:start_apps([]), + Config. + +end_per_suite(_Config) -> + emqx_common_test_helpers:stop_apps([]). + +init_per_testcase(_Case, Config) -> + meck:new(?DUMMY, [non_strict]), + meck:expect(?DUMMY, expect_not_called, 1, fun(Node) -> throw({blow_this_up, Node}) end), + meck:expect(?DUMMY, expect_success, 1, {ok, success}), + meck:expect(?DUMMY, expect_error, 1, {error, error}), + Config. + +end_per_testcase(_Case, _Config) -> + meck:unload(?DUMMY). + +t_with_node(_) -> + test_with(fun emqx_api_lib:with_node/2, [<<"all">>]). + +t_with_node_or_cluster(_) -> + test_with(fun emqx_api_lib:with_node_or_cluster/2, []), + meck:reset(?DUMMY), + ?assertEqual( + ?OK(success), + emqx_api_lib:with_node_or_cluster( + <<"all">>, + fun ?DUMMY:expect_success/1 + ) + ), + ?assertMatch([{_, {?DUMMY, expect_success, [all]}, {ok, success}}], meck:history(?DUMMY)). + +%% helpers +test_with(TestFun, ExtraBadNodes) -> + % make sure this is an atom + 'unknownnode@unknownnohost', + BadNodes = + [ + <<"undefined">>, + <<"this_should_not_be_an_atom">>, + <<"unknownnode@unknownnohost">> + ] ++ ExtraBadNodes, + [ensure_not_found(TestFun(N, fun ?DUMMY:expect_not_called/1)) || N <- BadNodes], + ensure_not_called(?DUMMY, expect_not_called), + ensure_not_existing_atom(<<"this_should_not_be_an_atom">>), + + GoodNode = node(), + + ?assertEqual( + ?OK(success), + TestFun(GoodNode, fun ?DUMMY:expect_success/1) + ), + + ?assertEqual( + ?BAD_REQUEST(error), + TestFun(GoodNode, fun ?DUMMY:expect_error/1) + ), + ok. + +ensure_not_found(Result) -> + ?assertMatch({404, _}, Result). + +ensure_not_called(Mod, Fun) -> + ?assert(not meck:called(Mod, Fun, '_')). + +ensure_not_existing_atom(Bin) -> + try binary_to_existing_atom(Bin) of + _ -> throw(is_atom) + catch + error:badarg -> + ok + end. diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index 685543c84..3f15d9b91 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -20,6 +20,7 @@ -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). -include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/emqx_api_lib.hrl"). -include_lib("emqx_bridge/include/emqx_bridge.hrl"). -import(hoconsc, [mk/2, array/1, enum/1]). @@ -46,25 +47,6 @@ -export([lookup_from_local_node/2]). -%% [TODO] Move those to a commonly shared header file --define(ERROR_MSG(CODE, REASON), #{code => CODE, message => emqx_misc:readable_error_msg(REASON)}). - --define(OK(CONTENT), {200, CONTENT}). - --define(NO_CONTENT, 204). - --define(BAD_REQUEST(CODE, REASON), {400, ?ERROR_MSG(CODE, REASON)}). --define(BAD_REQUEST(REASON), ?BAD_REQUEST('BAD_REQUEST', REASON)). - --define(NOT_FOUND(REASON), {404, ?ERROR_MSG('NOT_FOUND', REASON)}). - --define(INTERNAL_ERROR(REASON), {500, ?ERROR_MSG('INTERNAL_ERROR', REASON)}). - --define(NOT_IMPLEMENTED, 501). - --define(SERVICE_UNAVAILABLE(REASON), {503, ?ERROR_MSG('SERVICE_UNAVAILABLE', REASON)}). -%% End TODO - -define(BRIDGE_NOT_ENABLED, ?BAD_REQUEST(<<"Forbidden operation, bridge not enabled">>) ). diff --git a/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl index 69f5bf34e..aaee92b8c 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl @@ -121,32 +121,27 @@ fields(sampler_current) -> monitor(get, #{query_string := QS, bindings := Bindings}) -> Latest = maps:get(<<"latest">>, QS, infinity), - RawNode = maps:get(node, Bindings, all), - with_node(RawNode, dashboard_samplers_fun(Latest)). + RawNode = maps:get(node, Bindings, <<"all">>), + emqx_api_lib:with_node_or_cluster(RawNode, dashboard_samplers_fun(Latest)). dashboard_samplers_fun(Latest) -> fun(NodeOrCluster) -> case emqx_dashboard_monitor:samplers(NodeOrCluster, Latest) of - {badrpc, _} = Error -> Error; + {badrpc, _} = Error -> {error, Error}; Samplers -> {ok, Samplers} end end. monitor_current(get, #{bindings := Bindings}) -> - RawNode = maps:get(node, Bindings, all), - with_node(RawNode, fun emqx_dashboard_monitor:current_rate/1). + RawNode = maps:get(node, Bindings, <<"all">>), + emqx_api_lib:with_node_or_cluster(RawNode, fun current_rate/1). -with_node(RawNode, Fun) -> - case emqx_misc:safe_to_existing_atom(RawNode, utf8) of - {ok, NodeOrCluster} -> - case Fun(NodeOrCluster) of - {badrpc, {Node, Reason}} -> - {404, 'NOT_FOUND', io_lib:format("Node not found: ~p (~p)", [Node, Reason])}; - {ok, Result} -> - {200, Result} - end; - _Error -> - {404, 'NOT_FOUND', io_lib:format("Node not found: ~p", [RawNode])} +current_rate(Node) -> + case emqx_dashboard_monitor:current_rate(Node) of + {badrpc, _} = BadRpc -> + {error, BadRpc}; + {ok, _} = OkResult -> + OkResult end. %% ------------------------------------------------------------------------------------------------- diff --git a/apps/emqx_management/src/emqx_mgmt_api_nodes.erl b/apps/emqx_management/src/emqx_mgmt_api_nodes.erl index 21d905331..a4173f5b0 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_nodes.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_nodes.erl @@ -17,7 +17,6 @@ -behaviour(minirest_api). --include_lib("emqx/include/emqx.hrl"). -include_lib("typerefl/include/types.hrl"). -import(hoconsc, [mk/2, ref/1, ref/2, enum/1, array/1]). @@ -25,8 +24,6 @@ -define(NODE_METRICS_MODULE, emqx_mgmt_api_metrics). -define(NODE_STATS_MODULE, emqx_mgmt_api_stats). --define(SOURCE_ERROR, 'SOURCE_ERROR'). - %% Swagger specs from hocon schema -export([ api_spec/0, @@ -88,7 +85,7 @@ schema("/nodes/:node") -> ref(node_info), #{desc => <<"Get node info successfully">>} ), - 400 => node_error() + 404 => not_found() } } }; @@ -106,7 +103,7 @@ schema("/nodes/:node/metrics") -> ref(?NODE_METRICS_MODULE, node_metrics), #{desc => <<"Get node metrics successfully">>} ), - 400 => node_error() + 404 => not_found() } } }; @@ -124,7 +121,7 @@ schema("/nodes/:node/stats") -> ref(?NODE_STATS_MODULE, node_stats_data), #{desc => <<"Get node stats successfully">>} ), - 400 => node_error() + 404 => not_found() } } }. @@ -136,7 +133,7 @@ fields(node_name) -> [ {node, mk( - atom(), + binary(), #{ in => path, description => <<"Node name">>, @@ -250,55 +247,46 @@ nodes(get, _Params) -> list_nodes(#{}). node(get, #{bindings := #{node := NodeName}}) -> - get_node(NodeName). + emqx_api_lib:with_node(NodeName, to_ok_result_fun(fun get_node/1)). node_metrics(get, #{bindings := #{node := NodeName}}) -> - get_metrics(NodeName). + emqx_api_lib:with_node(NodeName, to_ok_result_fun(fun emqx_mgmt:get_metrics/1)). node_stats(get, #{bindings := #{node := NodeName}}) -> - get_stats(NodeName). + emqx_api_lib:with_node(NodeName, to_ok_result_fun(fun emqx_mgmt:get_stats/1)). %%-------------------------------------------------------------------- %% api apply list_nodes(#{}) -> - NodesInfo = [format(Node, NodeInfo) || {Node, NodeInfo} <- emqx_mgmt:list_nodes()], + NodesInfo = [format(NodeInfo) || {_Node, NodeInfo} <- emqx_mgmt:list_nodes()], {200, NodesInfo}. get_node(Node) -> - case emqx_mgmt:lookup_node(Node) of - {error, _} -> - {400, #{code => 'SOURCE_ERROR', message => <<"rpc_failed">>}}; - NodeInfo -> - {200, format(Node, NodeInfo)} - end. - -get_metrics(Node) -> - case emqx_mgmt:get_metrics(Node) of - {error, _} -> - {400, #{code => 'SOURCE_ERROR', message => <<"rpc_failed">>}}; - Metrics -> - {200, Metrics} - end. - -get_stats(Node) -> - case emqx_mgmt:get_stats(Node) of - {error, _} -> - {400, #{code => 'SOURCE_ERROR', message => <<"rpc_failed">>}}; - Stats -> - {200, Stats} - end. + format(emqx_mgmt:lookup_node(Node)). %%-------------------------------------------------------------------- %% internal function -format(_Node, Info = #{memory_total := Total, memory_used := Used}) -> +format(Info = #{memory_total := Total, memory_used := Used}) -> Info#{ memory_total := emqx_mgmt_util:kmg(Total), memory_used := emqx_mgmt_util:kmg(Used) }; -format(_Node, Info) when is_map(Info) -> +format(Info) when is_map(Info) -> Info. -node_error() -> - emqx_dashboard_swagger:error_codes([?SOURCE_ERROR], <<"Node error">>). +to_ok_result({error, _} = Error) -> + Error; +to_ok_result({ok, _} = Ok) -> + Ok; +to_ok_result(Result) -> + {ok, Result}. + +to_ok_result_fun(Fun) when is_function(Fun) -> + fun(Arg) -> + to_ok_result(Fun(Arg)) + end. + +not_found() -> + emqx_dashboard_swagger:error_codes(['NOT_FOUND'], <<"Node not found">>). diff --git a/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl index 03b0ea2d9..30313e555 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl @@ -68,7 +68,7 @@ t_nodes_api(_) -> BadNodePath = emqx_mgmt_api_test_util:api_path(["nodes", "badnode"]), ?assertMatch( - {error, {_, 400, _}}, + {error, {_, 404, _}}, emqx_mgmt_api_test_util:request_api(get, BadNodePath) ). @@ -94,7 +94,7 @@ t_node_stats_api(_) -> BadNodePath = emqx_mgmt_api_test_util:api_path(["nodes", "badnode", "stats"]), ?assertMatch( - {error, {_, 400, _}}, + {error, {_, 404, _}}, emqx_mgmt_api_test_util:request_api(get, BadNodePath) ). @@ -112,7 +112,7 @@ t_node_metrics_api(_) -> BadNodePath = emqx_mgmt_api_test_util:api_path(["nodes", "badnode", "metrics"]), ?assertMatch( - {error, {_, 400, _}}, + {error, {_, 404, _}}, emqx_mgmt_api_test_util:request_api(get, BadNodePath) ). diff --git a/changes/ce/fix-10237.en.md b/changes/ce/fix-10237.en.md new file mode 100644 index 000000000..cf3fc707b --- /dev/null +++ b/changes/ce/fix-10237.en.md @@ -0,0 +1 @@ +Ensure we return `404` status code for unknown node names in `/nodes/:node[/metrics|/stats]` API. From 2f344ec9d351e93611d4af28b6b1272ceade8438 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 27 Mar 2023 11:02:59 -0300 Subject: [PATCH 096/196] docs: improve formatting Co-authored-by: Zaiming (Stone) Shi --- changes/ce/feat-10224.en.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/changes/ce/feat-10224.en.md b/changes/ce/feat-10224.en.md index ba6303b16..7ef3e7f99 100644 --- a/changes/ce/feat-10224.en.md +++ b/changes/ce/feat-10224.en.md @@ -1,2 +1 @@ -Add the option to customize `clusterIP` in Helm chart, so that an user -may set it to a fixed IP. +Add the option to customize `clusterIP` in Helm chart, so that a user may set it to a fixed IP. From 11b5b7b6383bfffd00582846b0989c8296d9605a Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 23 Mar 2023 00:34:10 +0300 Subject: [PATCH 097/196] test(bridge-api): also run testcases in cluster environment Excluding a couple of testcases which does not make much sense running in the cluster. Also try to reduce amount of "noise" in the testcases, making them easier to comprehend. Co-authored-by: Thales Macedo Garitezi --- apps/emqx/test/emqx_common_test_helpers.erl | 7 +- .../test/emqx_bridge_api_SUITE.erl | 1038 ++++++++++------- 2 files changed, 629 insertions(+), 416 deletions(-) diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index d08812075..f445bea94 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -723,7 +723,7 @@ setup_node(Node, Opts) when is_map(Opts) -> ConfigureGenRpc = maps:get(configure_gen_rpc, Opts, true), LoadSchema = maps:get(load_schema, Opts, true), SchemaMod = maps:get(schema_mod, Opts, emqx_schema), - LoadApps = maps:get(load_apps, Opts, [gen_rpc, emqx, ekka, mria] ++ Apps), + LoadApps = maps:get(load_apps, Opts, Apps), Env = maps:get(env, Opts, []), Conf = maps:get(conf, Opts, []), ListenerPorts = maps:get(listener_ports, Opts, [ @@ -741,12 +741,13 @@ setup_node(Node, Opts) when is_map(Opts) -> StartAutocluster = maps:get(start_autocluster, Opts, false), %% Load env before doing anything to avoid overriding - lists:foreach(fun(App) -> rpc:call(Node, ?MODULE, load, [App]) end, LoadApps), + [ok = erpc:call(Node, ?MODULE, load, [App]) || App <- [gen_rpc, ekka, mria, emqx | LoadApps]], + %% Ensure a clean mnesia directory for each run to avoid %% inter-test flakiness. MnesiaDataDir = filename:join([ PrivDataDir, - node(), + Node, integer_to_list(erlang:unique_integer()), "mnesia" ]), diff --git a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl index 47a23e71c..fdbeb1be4 100644 --- a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl @@ -18,11 +18,14 @@ -compile(nowarn_export_all). -compile(export_all). --import(emqx_mgmt_api_test_util, [request/3, uri/1]). +-import(emqx_mgmt_api_test_util, [uri/1]). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). --define(CONF_DEFAULT, <<"bridges: {}">>). +-include_lib("snabbkaffe/include/test_macros.hrl"). + +-define(SUITE_APPS, [emqx_conf, emqx_authn, emqx_management, emqx_rule_engine, emqx_bridge]). + -define(BRIDGE_TYPE_HTTP, <<"webhook">>). -define(BRIDGE_NAME, (atom_to_binary(?FUNCTION_NAME))). -define(URL(PORT, PATH), @@ -54,37 +57,123 @@ <<"method">> => <<"post">>, <<"body">> => <<"${payload}">>, <<"headers">> => #{ - <<"content-type">> => <<"application/json">> + % NOTE + % The Pascal-Case is important here. + % The reason is kinda ridiculous: `emqx_bridge_resource:create_dry_run/2` converts + % bridge config keys into atoms, and the atom 'Content-Type' exists in the ERTS + % when this happens (while the 'content-type' does not). + <<"Content-Type">> => <<"application/json">> } }). -define(HTTP_BRIDGE(URL), ?HTTP_BRIDGE(URL, ?BRIDGE_NAME)). all() -> - emqx_common_test_helpers:all(?MODULE). + [ + {group, single}, + {group, cluster} + ]. groups() -> - []. + SingleOnlyTests = [ + t_broken_bpapi_vsn, + t_old_bpapi_vsn, + t_bridges_probe + ], + [ + {single, [], emqx_common_test_helpers:all(?MODULE)}, + {cluster, [], emqx_common_test_helpers:all(?MODULE) -- SingleOnlyTests} + ]. suite() -> [{timetrap, {seconds, 60}}]. init_per_suite(Config) -> - _ = application:load(emqx_conf), - %% some testcases (may from other app) already get emqx_connector started - _ = application:stop(emqx_resource), - _ = application:stop(emqx_connector), - ok = emqx_mgmt_api_test_util:init_suite( - [emqx_rule_engine, emqx_bridge, emqx_authn] - ), - ok = emqx_common_test_helpers:load_config( - emqx_rule_engine_schema, - <<"rule_engine {rules {}}">> - ), - ok = emqx_common_test_helpers:load_config(emqx_bridge_schema, ?CONF_DEFAULT), Config. end_per_suite(_Config) -> - emqx_mgmt_api_test_util:end_suite([emqx_rule_engine, emqx_bridge, emqx_authn]), + ok. + +init_per_group(cluster, Config) -> + Cluster = mk_cluster_specs(Config), + ct:pal("Starting ~p", [Cluster]), + Nodes = [ + emqx_common_test_helpers:start_slave(Name, Opts) + || {Name, Opts} <- Cluster + ], + [NodePrimary | NodesRest] = Nodes, + ok = erpc:call(NodePrimary, fun() -> init_node(primary) end), + _ = [ok = erpc:call(Node, fun() -> init_node(regular) end) || Node <- NodesRest], + [{group, cluster}, {cluster_nodes, Nodes}, {api_node, NodePrimary} | Config]; +init_per_group(_, Config) -> + ok = emqx_mgmt_api_test_util:init_suite(?SUITE_APPS), + ok = load_suite_config(emqx_rule_engine), + ok = load_suite_config(emqx_bridge), + [{group, single}, {api_node, node()} | Config]. + +mk_cluster_specs(Config) -> + Specs = [ + {core, emqx_bridge_api_SUITE1, #{}}, + {core, emqx_bridge_api_SUITE2, #{}} + ], + CommonOpts = #{ + env => [{emqx, boot_modules, [broker]}], + apps => [], + % NOTE + % We need to start all those apps _after_ the cluster becomes stable, in the + % `init_node/1`. This is because usual order is broken in very subtle way: + % 1. Node starts apps including `mria` and `emqx_conf` which starts `emqx_cluster_rpc`. + % 2. The `emqx_cluster_rpc` sets up a mnesia table subscription during initialization. + % 3. In the meantime `mria` joins the cluster and notices it should restart. + % 4. Mnesia subscription becomes lost during restarts (god knows why). + % Yet we need to load them before, so that mria / mnesia will know which tables + % should be created in the cluster. + % TODO + % We probably should hide these intricacies behind the `emqx_common_test_helpers`. + load_apps => ?SUITE_APPS ++ [emqx_dashboard], + env_handler => fun load_suite_config/1, + load_schema => false, + priv_data_dir => ?config(priv_dir, Config) + }, + emqx_common_test_helpers:emqx_cluster(Specs, CommonOpts). + +init_node(Type) -> + ok = emqx_common_test_helpers:start_apps(?SUITE_APPS, fun load_suite_config/1), + case Type of + primary -> + ok = emqx_config:put( + [dashboard, listeners], + #{http => #{enable => true, bind => 18083}} + ), + ok = emqx_dashboard:start_listeners(), + ready = emqx_dashboard_listener:regenerate_minirest_dispatch(), + emqx_common_test_http:create_default_app(); + regular -> + ok + end. + +load_suite_config(emqx_rule_engine) -> + ok = emqx_common_test_helpers:load_config( + emqx_rule_engine_schema, + <<"rule_engine { rules {} }">> + ); +load_suite_config(emqx_bridge) -> + ok = emqx_common_test_helpers:load_config( + emqx_bridge_schema, + <<"bridges {}">> + ); +load_suite_config(_) -> + ok. + +end_per_group(cluster, Config) -> + ok = lists:foreach( + fun(Node) -> + _ = erpc:call(Node, emqx_common_test_helpers, stop_apps, [?SUITE_APPS]), + emqx_common_test_helpers:stop_slave(Node) + end, + ?config(cluster_nodes, Config) + ); +end_per_group(_, _Config) -> + emqx_mgmt_api_test_util:end_suite(?SUITE_APPS), mria:clear_table(emqx_authn_mnesia), ok. @@ -98,22 +187,7 @@ init_per_testcase(t_old_bpapi_vsn, Config) -> meck:expect(emqx_bpapi, supported_version, 1, 1), meck:expect(emqx_bpapi, supported_version, 2, 1), init_per_testcase(common, Config); -init_per_testcase(StartStop, Config) when - StartStop == t_start_stop_bridges_cluster; - StartStop == t_start_stop_bridges_node --> - meck:new(emqx_bridge_resource, [passthrough]), - meck:expect( - emqx_bridge_resource, - stop, - fun - (_, <<"bridge_not_found">>) -> {error, not_found}; - (Type, Name) -> meck:passthrough([Type, Name]) - end - ), - init_per_testcase(common, Config); init_per_testcase(_, Config) -> - {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), {Port, Sock, Acceptor} = start_http_server(fun handle_fun_200_ok/2), [{port, Port}, {sock, Sock}, {acceptor, Acceptor} | Config]. @@ -123,17 +197,13 @@ end_per_testcase(t_broken_bpapi_vsn, Config) -> end_per_testcase(t_old_bpapi_vsn, Config) -> meck:unload([emqx_bpapi]), end_per_testcase(common, Config); -end_per_testcase(StartStop, Config) when - StartStop == t_start_stop_bridges_cluster; - StartStop == t_start_stop_bridges_node --> - meck:unload([emqx_bridge_resource]), - end_per_testcase(common, Config); end_per_testcase(_, Config) -> Sock = ?config(sock, Config), Acceptor = ?config(acceptor, Config), - stop_http_server(Sock, Acceptor), - clear_resources(), + Node = ?config(api_node, Config), + ok = emqx_common_test_helpers:call_janitor(), + ok = stop_http_server(Sock, Acceptor), + ok = erpc:call(Node, fun clear_resources/0), ok. clear_resources() -> @@ -215,35 +285,36 @@ parse_http_request(ReqStr0) -> t_http_crud_apis(Config) -> Port = ?config(port, Config), %% assert we there's no bridges at first - {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []), + {ok, 200, []} = request_json(get, uri(["bridges"]), Config), - {ok, 404, _} = request(get, uri(["bridges", "foo"]), []), - {ok, 404, _} = request(get, uri(["bridges", "webhook:foo"]), []), + {ok, 404, _} = request(get, uri(["bridges", "foo"]), Config), + {ok, 404, _} = request(get, uri(["bridges", "webhook:foo"]), Config), %% then we add a webhook bridge, using POST %% POST /bridges/ will create a bridge URL1 = ?URL(Port, "path1"), Name = ?BRIDGE_NAME, - {ok, 201, Bridge} = request( - post, - uri(["bridges"]), - ?HTTP_BRIDGE(URL1, Name) + ?assertMatch( + {ok, 201, #{ + <<"type">> := ?BRIDGE_TYPE_HTTP, + <<"name">> := Name, + <<"enable">> := true, + <<"status">> := _, + <<"node_status">> := [_ | _], + <<"url">> := URL1 + }}, + request_json( + post, + uri(["bridges"]), + ?HTTP_BRIDGE(URL1, Name), + Config + ) ), - %ct:pal("---bridge: ~p", [Bridge]), - #{ - <<"type">> := ?BRIDGE_TYPE_HTTP, - <<"name">> := Name, - <<"enable">> := true, - <<"status">> := _, - <<"node_status">> := [_ | _], - <<"url">> := URL1 - } = emqx_json:decode(Bridge, [return_maps]), - BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_HTTP, Name), %% send an message to emqx and the message should be forwarded to the HTTP server Body = <<"my msg">>, - emqx:publish(emqx_message:make(<<"emqx_webhook/1">>, Body)), + _ = publish_message(<<"emqx_webhook/1">>, Body, Config), ?assert( receive {http_server, received, #{ @@ -261,27 +332,26 @@ t_http_crud_apis(Config) -> ), %% update the request-path of the bridge URL2 = ?URL(Port, "path2"), - {ok, 200, Bridge2} = request( - put, - uri(["bridges", BridgeID]), - ?HTTP_BRIDGE(URL2, Name) - ), ?assertMatch( - #{ + {ok, 200, #{ <<"type">> := ?BRIDGE_TYPE_HTTP, <<"name">> := Name, <<"enable">> := true, <<"status">> := _, <<"node_status">> := [_ | _], <<"url">> := URL2 - }, - emqx_json:decode(Bridge2, [return_maps]) + }}, + request_json( + put, + uri(["bridges", BridgeID]), + ?HTTP_BRIDGE(URL2, Name), + Config + ) ), %% list all bridges again, assert Bridge2 is in it - {ok, 200, Bridge2Str} = request(get, uri(["bridges"]), []), ?assertMatch( - [ + {ok, 200, [ #{ <<"type">> := ?BRIDGE_TYPE_HTTP, <<"name">> := Name, @@ -290,26 +360,25 @@ t_http_crud_apis(Config) -> <<"node_status">> := [_ | _], <<"url">> := URL2 } - ], - emqx_json:decode(Bridge2Str, [return_maps]) + ]}, + request_json(get, uri(["bridges"]), Config) ), %% get the bridge by id - {ok, 200, Bridge3Str} = request(get, uri(["bridges", BridgeID]), []), ?assertMatch( - #{ + {ok, 200, #{ <<"type">> := ?BRIDGE_TYPE_HTTP, <<"name">> := Name, <<"enable">> := true, <<"status">> := _, <<"node_status">> := [_ | _], <<"url">> := URL2 - }, - emqx_json:decode(Bridge3Str, [return_maps]) + }}, + request_json(get, uri(["bridges", BridgeID]), Config) ), %% send an message to emqx again, check the path has been changed - emqx:publish(emqx_message:make(<<"emqx_webhook/1">>, Body)), + _ = publish_message(<<"emqx_webhook/1">>, Body, Config), ?assert( receive {http_server, received, #{path := <<"/path2">>}} -> @@ -323,68 +392,64 @@ t_http_crud_apis(Config) -> ), %% Test bad updates - {ok, 400, PutFail1} = request( + {ok, 400, PutFail1} = request_json( put, uri(["bridges", BridgeID]), - maps:remove(<<"url">>, ?HTTP_BRIDGE(URL2, Name)) + maps:remove(<<"url">>, ?HTTP_BRIDGE(URL2, Name)), + Config ), ?assertMatch( #{<<"reason">> := <<"required_field">>}, - emqx_json:decode(maps:get(<<"message">>, emqx_json:decode(PutFail1, [return_maps])), [ - return_maps - ]) + json(maps:get(<<"message">>, PutFail1)) ), - {ok, 400, PutFail2} = request( + {ok, 400, PutFail2} = request_json( put, uri(["bridges", BridgeID]), - maps:put(<<"curl">>, URL2, maps:remove(<<"url">>, ?HTTP_BRIDGE(URL2, Name))) + maps:put(<<"curl">>, URL2, maps:remove(<<"url">>, ?HTTP_BRIDGE(URL2, Name))), + Config ), ?assertMatch( #{ <<"reason">> := <<"unknown_fields">>, <<"unknown">> := <<"curl">> }, - emqx_json:decode(maps:get(<<"message">>, emqx_json:decode(PutFail2, [return_maps])), [ - return_maps - ]) + json(maps:get(<<"message">>, PutFail2)) ), %% delete the bridge - {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), []), - {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []), + {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), Config), + {ok, 200, []} = request_json(get, uri(["bridges"]), Config), %% update a deleted bridge returns an error - {ok, 404, ErrMsg2} = request( - put, - uri(["bridges", BridgeID]), - ?HTTP_BRIDGE(URL2, Name) - ), ?assertMatch( - #{ + {ok, 404, #{ <<"code">> := <<"NOT_FOUND">>, <<"message">> := _ - }, - emqx_json:decode(ErrMsg2, [return_maps]) + }}, + request_json( + put, + uri(["bridges", BridgeID]), + ?HTTP_BRIDGE(URL2, Name), + Config + ) ), %% try delete bad bridge id - {ok, 404, BadId} = request(delete, uri(["bridges", "foo"]), []), ?assertMatch( - #{ + {ok, 404, #{ <<"code">> := <<"NOT_FOUND">>, <<"message">> := <<"Invalid bridge ID", _/binary>> - }, - emqx_json:decode(BadId, [return_maps]) + }}, + request_json(delete, uri(["bridges", "foo"]), Config) ), %% Deleting a non-existing bridge should result in an error - {ok, 404, ErrMsg3} = request(delete, uri(["bridges", BridgeID]), []), ?assertMatch( - #{ + {ok, 404, #{ <<"code">> := <<"NOT_FOUND">>, <<"message">> := _ - }, - emqx_json:decode(ErrMsg3, [return_maps]) + }}, + request_json(delete, uri(["bridges", BridgeID]), Config) ), %% Create non working bridge @@ -392,37 +457,54 @@ t_http_crud_apis(Config) -> {ok, 201, BrokenBridge} = request( post, uri(["bridges"]), - ?HTTP_BRIDGE(BrokenURL, Name) + ?HTTP_BRIDGE(BrokenURL, Name), + fun json/1, + Config + ), + ?assertMatch( + #{ + <<"type">> := ?BRIDGE_TYPE_HTTP, + <<"name">> := Name, + <<"enable">> := true, + <<"status">> := <<"disconnected">>, + <<"status_reason">> := <<"Connection refused">>, + <<"node_status">> := [ + #{ + <<"status">> := <<"disconnected">>, + <<"status_reason">> := <<"Connection refused">> + } + | _ + ], + <<"url">> := BrokenURL + }, + BrokenBridge + ), + + {ok, 200, FixedBridge} = request_json( + put, + uri(["bridges", BridgeID]), + ?HTTP_BRIDGE(URL1), + Config + ), + ?assertMatch( + #{ + <<"status">> := <<"connected">>, + <<"node_status">> := [FixedNodeStatus = #{<<"status">> := <<"connected">>} | _] + } when + not is_map_key(<<"status_reason">>, FixedBridge) andalso + not is_map_key(<<"status_reason">>, FixedNodeStatus), + FixedBridge ), - #{ - <<"type">> := ?BRIDGE_TYPE_HTTP, - <<"name">> := Name, - <<"enable">> := true, - <<"status">> := <<"disconnected">>, - <<"status_reason">> := <<"Connection refused">>, - <<"node_status">> := [ - #{<<"status">> := <<"disconnected">>, <<"status_reason">> := <<"Connection refused">>} - | _ - ], - <<"url">> := BrokenURL - } = emqx_json:decode(BrokenBridge, [return_maps]), - {ok, 200, FixedBridgeResponse} = request(put, uri(["bridges", BridgeID]), ?HTTP_BRIDGE(URL1)), - #{ - <<"status">> := <<"connected">>, - <<"node_status">> := [FixedNodeStatus = #{<<"status">> := <<"connected">>} | _] - } = FixedBridge = emqx_json:decode(FixedBridgeResponse, [return_maps]), - ?assert(not maps:is_key(<<"status_reason">>, FixedBridge)), - ?assert(not maps:is_key(<<"status_reason">>, FixedNodeStatus)), - {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), []), %% Try create bridge with bad characters as name - {ok, 400, _} = request(post, uri(["bridges"]), ?HTTP_BRIDGE(URL1, <<"隋达"/utf8>>)), - ok. + {ok, 400, _} = request(post, uri(["bridges"]), ?HTTP_BRIDGE(URL1, <<"隋达"/utf8>>), Config), + + {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), Config). t_http_bridges_local_topic(Config) -> Port = ?config(port, Config), %% assert we there's no bridges at first - {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []), + {ok, 200, []} = request_json(get, uri(["bridges"]), Config), %% then we add a webhook bridge, using POST %% POST /bridges/ will create a bridge @@ -433,13 +515,15 @@ t_http_bridges_local_topic(Config) -> {ok, 201, _} = request( post, uri(["bridges"]), - ?HTTP_BRIDGE(URL1, Name1) + ?HTTP_BRIDGE(URL1, Name1), + Config ), %% and we create another one without local_topic {ok, 201, _} = request( post, uri(["bridges"]), - maps:remove(<<"local_topic">>, ?HTTP_BRIDGE(URL1, Name2)) + maps:remove(<<"local_topic">>, ?HTTP_BRIDGE(URL1, Name2)), + Config ), BridgeID1 = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_HTTP, Name1), BridgeID2 = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_HTTP, Name2), @@ -447,7 +531,7 @@ t_http_bridges_local_topic(Config) -> %% This is to verify we can have 2 bridges with and without local_topic fields %% at the same time. Body = <<"my msg">>, - emqx:publish(emqx_message:make(<<"emqx_webhook/1">>, Body)), + _ = publish_message(<<"emqx_webhook/1">>, Body, Config), ?assert( receive {http_server, received, #{ @@ -464,14 +548,13 @@ t_http_bridges_local_topic(Config) -> end ), %% delete the bridge - {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID1]), []), - {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID2]), []), - ok. + {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID1]), Config), + {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID2]), Config). t_check_dependent_actions_on_delete(Config) -> Port = ?config(port, Config), %% assert we there's no bridges at first - {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []), + {ok, 200, []} = request_json(get, uri(["bridges"]), Config), %% then we add a webhook bridge, using POST %% POST /bridges/ will create a bridge @@ -481,9 +564,10 @@ t_check_dependent_actions_on_delete(Config) -> {ok, 201, _} = request( post, uri(["bridges"]), - ?HTTP_BRIDGE(URL1, Name) + ?HTTP_BRIDGE(URL1, Name), + Config ), - {ok, 201, Rule} = request( + {ok, 201, #{<<"id">> := RuleId}} = request_json( post, uri(["rules"]), #{ @@ -491,25 +575,23 @@ t_check_dependent_actions_on_delete(Config) -> <<"enable">> => true, <<"actions">> => [BridgeID], <<"sql">> => <<"SELECT * from \"t\"">> - } + }, + Config ), - #{<<"id">> := RuleId} = emqx_json:decode(Rule, [return_maps]), %% deleting the bridge should fail because there is a rule that depends on it {ok, 400, _} = request( - delete, uri(["bridges", BridgeID]) ++ "?also_delete_dep_actions=false", [] + delete, uri(["bridges", BridgeID]) ++ "?also_delete_dep_actions=false", Config ), %% delete the rule first - {ok, 204, <<>>} = request(delete, uri(["rules", RuleId]), []), + {ok, 204, <<>>} = request(delete, uri(["rules", RuleId]), Config), %% then delete the bridge is OK - {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), []), - {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []), - - ok. + {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), Config), + {ok, 200, []} = request_json(get, uri(["bridges"]), Config). t_cascade_delete_actions(Config) -> Port = ?config(port, Config), %% assert we there's no bridges at first - {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []), + {ok, 200, []} = request_json(get, uri(["bridges"]), Config), %% then we add a webhook bridge, using POST %% POST /bridges/ will create a bridge @@ -519,9 +601,10 @@ t_cascade_delete_actions(Config) -> {ok, 201, _} = request( post, uri(["bridges"]), - ?HTTP_BRIDGE(URL1, Name) + ?HTTP_BRIDGE(URL1, Name), + Config ), - {ok, 201, Rule} = request( + {ok, 201, #{<<"id">> := RuleId}} = request_json( post, uri(["rules"]), #{ @@ -529,27 +612,27 @@ t_cascade_delete_actions(Config) -> <<"enable">> => true, <<"actions">> => [BridgeID], <<"sql">> => <<"SELECT * from \"t\"">> - } + }, + Config ), - #{<<"id">> := RuleId} = emqx_json:decode(Rule, [return_maps]), %% delete the bridge will also delete the actions from the rules {ok, 204, _} = request( - delete, uri(["bridges", BridgeID]) ++ "?also_delete_dep_actions=true", [] + delete, + uri(["bridges", BridgeID]) ++ "?also_delete_dep_actions=true", + Config ), - {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []), - {ok, 200, Rule1} = request(get, uri(["rules", RuleId]), []), + {ok, 200, []} = request_json(get, uri(["bridges"]), Config), ?assertMatch( - #{ - <<"actions">> := [] - }, - emqx_json:decode(Rule1, [return_maps]) + {ok, 200, #{<<"actions">> := []}}, + request_json(get, uri(["rules", RuleId]), Config) ), - {ok, 204, <<>>} = request(delete, uri(["rules", RuleId]), []), + {ok, 204, <<>>} = request(delete, uri(["rules", RuleId]), Config), {ok, 201, _} = request( post, uri(["bridges"]), - ?HTTP_BRIDGE(URL1, Name) + ?HTTP_BRIDGE(URL1, Name), + Config ), {ok, 201, _} = request( post, @@ -559,12 +642,16 @@ t_cascade_delete_actions(Config) -> <<"enable">> => true, <<"actions">> => [BridgeID], <<"sql">> => <<"SELECT * from \"t\"">> - } + }, + Config ), - {ok, 204, _} = request(delete, uri(["bridges", BridgeID]) ++ "?also_delete_dep_actions", []), - {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []), - ok. + {ok, 204, _} = request( + delete, + uri(["bridges", BridgeID]) ++ "?also_delete_dep_actions", + Config + ), + {ok, 200, []} = request_json(get, uri(["bridges"]), Config). t_broken_bpapi_vsn(Config) -> Port = ?config(port, Config), @@ -573,12 +660,13 @@ t_broken_bpapi_vsn(Config) -> {ok, 201, _Bridge} = request( post, uri(["bridges"]), - ?HTTP_BRIDGE(URL1, Name) + ?HTTP_BRIDGE(URL1, Name), + Config ), BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_HTTP, Name), %% still works since we redirect to 'restart' - {ok, 501, <<>>} = request(post, operation_path(cluster, start, BridgeID), <<"">>), - {ok, 501, <<>>} = request(post, operation_path(node, start, BridgeID), <<"">>), + {ok, 501, <<>>} = request(post, {operation, cluster, start, BridgeID}, Config), + {ok, 501, <<>>} = request(post, {operation, node, start, BridgeID}, Config), ok. t_old_bpapi_vsn(Config) -> @@ -588,31 +676,34 @@ t_old_bpapi_vsn(Config) -> {ok, 201, _Bridge} = request( post, uri(["bridges"]), - ?HTTP_BRIDGE(URL1, Name) + ?HTTP_BRIDGE(URL1, Name), + Config ), BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_HTTP, Name), - {ok, 204, <<>>} = request(post, operation_path(cluster, stop, BridgeID), <<"">>), - {ok, 204, <<>>} = request(post, operation_path(node, stop, BridgeID), <<"">>), + {ok, 204, <<>>} = request(post, {operation, cluster, stop, BridgeID}, Config), + {ok, 204, <<>>} = request(post, {operation, node, stop, BridgeID}, Config), %% still works since we redirect to 'restart' - {ok, 204, <<>>} = request(post, operation_path(cluster, start, BridgeID), <<"">>), - {ok, 204, <<>>} = request(post, operation_path(node, start, BridgeID), <<"">>), - {ok, 204, <<>>} = request(post, operation_path(cluster, restart, BridgeID), <<"">>), - {ok, 204, <<>>} = request(post, operation_path(node, restart, BridgeID), <<"">>), + {ok, 204, <<>>} = request(post, {operation, cluster, start, BridgeID}, Config), + {ok, 204, <<>>} = request(post, {operation, node, start, BridgeID}, Config), + {ok, 204, <<>>} = request(post, {operation, cluster, restart, BridgeID}, Config), + {ok, 204, <<>>} = request(post, {operation, node, restart, BridgeID}, Config), ok. -t_start_stop_bridges_node(Config) -> +t_start_bridge_unknown_node(Config) -> {ok, 404, _} = request( post, uri(["nodes", "thisbetterbenotanatomyet", "bridges", "webhook:foo", start]), - <<"">> + Config ), {ok, 404, _} = request( post, uri(["nodes", "undefined", "bridges", "webhook:foo", start]), - <<"">> - ), + Config + ). + +t_start_stop_bridges_node(Config) -> do_start_stop_bridges(node, Config). t_start_stop_bridges_cluster(Config) -> @@ -620,182 +711,250 @@ t_start_stop_bridges_cluster(Config) -> do_start_stop_bridges(Type, Config) -> %% assert we there's no bridges at first - {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []), + {ok, 200, []} = request_json(get, uri(["bridges"]), Config), Port = ?config(port, Config), URL1 = ?URL(Port, "abc"), Name = atom_to_binary(Type), - {ok, 201, Bridge} = request( - post, - uri(["bridges"]), - ?HTTP_BRIDGE(URL1, Name) + ?assertMatch( + {ok, 201, #{ + <<"type">> := ?BRIDGE_TYPE_HTTP, + <<"name">> := Name, + <<"enable">> := true, + <<"status">> := <<"connected">>, + <<"node_status">> := [_ | _], + <<"url">> := URL1 + }}, + request_json( + post, + uri(["bridges"]), + ?HTTP_BRIDGE(URL1, Name), + Config + ) ), - %ct:pal("the bridge ==== ~p", [Bridge]), - #{ - <<"type">> := ?BRIDGE_TYPE_HTTP, - <<"name">> := Name, - <<"enable">> := true, - <<"status">> := <<"connected">>, - <<"node_status">> := [_ | _], - <<"url">> := URL1 - } = emqx_json:decode(Bridge, [return_maps]), - BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_HTTP, Name), - %% stop it - {ok, 204, <<>>} = request(post, operation_path(Type, stop, BridgeID), <<"">>), - {ok, 200, Bridge2} = request(get, uri(["bridges", BridgeID]), []), - ?assertMatch(#{<<"status">> := <<"stopped">>}, emqx_json:decode(Bridge2, [return_maps])), - %% start again - {ok, 204, <<>>} = request(post, operation_path(Type, start, BridgeID), <<"">>), - {ok, 200, Bridge3} = request(get, uri(["bridges", BridgeID]), []), - ?assertMatch(#{<<"status">> := <<"connected">>}, emqx_json:decode(Bridge3, [return_maps])), - %% start a started bridge - {ok, 204, <<>>} = request(post, operation_path(Type, start, BridgeID), <<"">>), - {ok, 200, Bridge3_1} = request(get, uri(["bridges", BridgeID]), []), - ?assertMatch(#{<<"status">> := <<"connected">>}, emqx_json:decode(Bridge3_1, [return_maps])), - %% restart an already started bridge - {ok, 204, <<>>} = request(post, operation_path(Type, restart, BridgeID), <<"">>), - {ok, 200, Bridge3} = request(get, uri(["bridges", BridgeID]), []), - ?assertMatch(#{<<"status">> := <<"connected">>}, emqx_json:decode(Bridge3, [return_maps])), - %% stop it again - {ok, 204, <<>>} = request(post, operation_path(Type, stop, BridgeID), <<"">>), - %% restart a stopped bridge - {ok, 204, <<>>} = request(post, operation_path(Type, restart, BridgeID), <<"">>), - {ok, 200, Bridge4} = request(get, uri(["bridges", BridgeID]), []), - ?assertMatch(#{<<"status">> := <<"connected">>}, emqx_json:decode(Bridge4, [return_maps])), - {ok, 404, _} = request(post, operation_path(Type, invalidop, BridgeID), <<"">>), + BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_HTTP, Name), + ExpectedStatus = + case ?config(group, Config) of + cluster when Type == node -> + <<"inconsistent">>; + _ -> + <<"stopped">> + end, + + %% stop it + {ok, 204, <<>>} = request(post, {operation, Type, stop, BridgeID}, Config), + ?assertMatch( + {ok, 200, #{<<"status">> := ExpectedStatus}}, + request_json(get, uri(["bridges", BridgeID]), Config) + ), + %% start again + {ok, 204, <<>>} = request(post, {operation, Type, start, BridgeID}, Config), + ?assertMatch( + {ok, 200, #{<<"status">> := <<"connected">>}}, + request_json(get, uri(["bridges", BridgeID]), Config) + ), + %% start a started bridge + {ok, 204, <<>>} = request(post, {operation, Type, start, BridgeID}, Config), + ?assertMatch( + {ok, 200, #{<<"status">> := <<"connected">>}}, + request_json(get, uri(["bridges", BridgeID]), Config) + ), + %% restart an already started bridge + {ok, 204, <<>>} = request(post, {operation, Type, restart, BridgeID}, Config), + ?assertMatch( + {ok, 200, #{<<"status">> := <<"connected">>}}, + request_json(get, uri(["bridges", BridgeID]), Config) + ), + %% stop it again + {ok, 204, <<>>} = request(post, {operation, Type, stop, BridgeID}, Config), + %% restart a stopped bridge + {ok, 204, <<>>} = request(post, {operation, Type, restart, BridgeID}, Config), + ?assertMatch( + {ok, 200, #{<<"status">> := <<"connected">>}}, + request_json(get, uri(["bridges", BridgeID]), Config) + ), + + {ok, 404, _} = request(post, {operation, Type, invalidop, BridgeID}, Config), %% delete the bridge - {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), []), - {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []), + {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), Config), + {ok, 200, []} = request_json(get, uri(["bridges"]), Config), %% Fail parse-id check - {ok, 404, _} = request(post, operation_path(Type, start, <<"wreckbook_fugazi">>), <<"">>), + {ok, 404, _} = request(post, {operation, Type, start, <<"wreckbook_fugazi">>}, Config), %% Looks ok but doesn't exist - {ok, 404, _} = request(post, operation_path(Type, start, <<"webhook:cptn_hook">>), <<"">>), - - %% - {ok, 201, _Bridge} = request( - post, - uri(["bridges"]), - ?HTTP_BRIDGE(URL1, <<"bridge_not_found">>) - ), - {ok, 503, _} = request( - post, operation_path(Type, stop, <<"webhook:bridge_not_found">>), <<"">> - ), + {ok, 404, _} = request(post, {operation, Type, start, <<"webhook:cptn_hook">>}, Config), %% Create broken bridge {ListenPort, Sock} = listen_on_random_port(), %% Connecting to this endpoint should always timeout BadServer = iolist_to_binary(io_lib:format("localhost:~B", [ListenPort])), BadName = <<"bad_", (atom_to_binary(Type))/binary>>, - {ok, 201, BadBridge1} = request( - post, - uri(["bridges"]), - ?MQTT_BRIDGE(BadServer, BadName) + ?assertMatch( + {ok, 201, #{ + <<"type">> := ?BRIDGE_TYPE_MQTT, + <<"name">> := BadName, + <<"enable">> := true, + <<"server">> := BadServer, + <<"status">> := <<"connecting">>, + <<"node_status">> := [_ | _] + }}, + request_json( + post, + uri(["bridges"]), + ?MQTT_BRIDGE(BadServer, BadName), + Config + ) ), - #{ - <<"type">> := ?BRIDGE_TYPE_MQTT, - <<"name">> := BadName, - <<"enable">> := true, - <<"server">> := BadServer, - <<"status">> := <<"connecting">>, - <<"node_status">> := [_ | _] - } = emqx_json:decode(BadBridge1, [return_maps]), BadBridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_MQTT, BadName), ?assertMatch( {ok, SC, _} when SC == 500 orelse SC == 503, - request(post, operation_path(Type, start, BadBridgeID), <<"">>) + request(post, {operation, Type, start, BadBridgeID}, Config) ), ok = gen_tcp:close(Sock), ok. +t_start_stop_inconsistent_bridge_node(Config) -> + start_stop_inconsistent_bridge(node, Config). + +t_start_stop_inconsistent_bridge_cluster(Config) -> + start_stop_inconsistent_bridge(cluster, Config). + +start_stop_inconsistent_bridge(Type, Config) -> + Port = ?config(port, Config), + URL = ?URL(Port, "abc"), + Node = ?config(api_node, Config), + + erpc:call(Node, fun() -> + meck:new(emqx_bridge_resource, [passthrough, no_link]), + meck:expect( + emqx_bridge_resource, + stop, + fun + (_, <<"bridge_not_found">>) -> {error, not_found}; + (BridgeType, Name) -> meck:passthrough([BridgeType, Name]) + end + ) + end), + + emqx_common_test_helpers:on_exit(fun() -> + erpc:call(Node, fun() -> + meck:unload([emqx_bridge_resource]) + end) + end), + + {ok, 201, _Bridge} = request( + post, + uri(["bridges"]), + ?HTTP_BRIDGE(URL, <<"bridge_not_found">>), + Config + ), + {ok, 503, _} = request( + post, {operation, Type, stop, <<"webhook:bridge_not_found">>}, Config + ). + t_enable_disable_bridges(Config) -> %% assert we there's no bridges at first - {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []), + {ok, 200, []} = request_json(get, uri(["bridges"]), Config), Name = ?BRIDGE_NAME, Port = ?config(port, Config), URL1 = ?URL(Port, "abc"), - {ok, 201, Bridge} = request( - post, - uri(["bridges"]), - ?HTTP_BRIDGE(URL1, Name) + ?assertMatch( + {ok, 201, #{ + <<"type">> := ?BRIDGE_TYPE_HTTP, + <<"name">> := Name, + <<"enable">> := true, + <<"status">> := <<"connected">>, + <<"node_status">> := [_ | _], + <<"url">> := URL1 + }}, + request_json( + post, + uri(["bridges"]), + ?HTTP_BRIDGE(URL1, Name), + Config + ) ), - %ct:pal("the bridge ==== ~p", [Bridge]), - #{ - <<"type">> := ?BRIDGE_TYPE_HTTP, - <<"name">> := Name, - <<"enable">> := true, - <<"status">> := <<"connected">>, - <<"node_status">> := [_ | _], - <<"url">> := URL1 - } = emqx_json:decode(Bridge, [return_maps]), BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_HTTP, Name), %% disable it - {ok, 204, <<>>} = request(put, enable_path(false, BridgeID), <<"">>), - {ok, 200, Bridge2} = request(get, uri(["bridges", BridgeID]), []), - ?assertMatch(#{<<"status">> := <<"stopped">>}, emqx_json:decode(Bridge2, [return_maps])), + {ok, 204, <<>>} = request(put, enable_path(false, BridgeID), Config), + ?assertMatch( + {ok, 200, #{<<"status">> := <<"stopped">>}}, + request_json(get, uri(["bridges", BridgeID]), Config) + ), %% enable again - {ok, 204, <<>>} = request(put, enable_path(true, BridgeID), <<"">>), - {ok, 200, Bridge3} = request(get, uri(["bridges", BridgeID]), []), - ?assertMatch(#{<<"status">> := <<"connected">>}, emqx_json:decode(Bridge3, [return_maps])), + {ok, 204, <<>>} = request(put, enable_path(true, BridgeID), Config), + ?assertMatch( + {ok, 200, #{<<"status">> := <<"connected">>}}, + request_json(get, uri(["bridges", BridgeID]), Config) + ), %% enable an already started bridge - {ok, 204, <<>>} = request(put, enable_path(true, BridgeID), <<"">>), - {ok, 200, Bridge3} = request(get, uri(["bridges", BridgeID]), []), - ?assertMatch(#{<<"status">> := <<"connected">>}, emqx_json:decode(Bridge3, [return_maps])), + {ok, 204, <<>>} = request(put, enable_path(true, BridgeID), Config), + ?assertMatch( + {ok, 200, #{<<"status">> := <<"connected">>}}, + request_json(get, uri(["bridges", BridgeID]), Config) + ), %% disable it again - {ok, 204, <<>>} = request(put, enable_path(false, BridgeID), <<"">>), + {ok, 204, <<>>} = request(put, enable_path(false, BridgeID), Config), %% bad param - {ok, 404, _} = request(put, enable_path(foo, BridgeID), <<"">>), - {ok, 404, _} = request(put, enable_path(true, "foo"), <<"">>), - {ok, 404, _} = request(put, enable_path(true, "webhook:foo"), <<"">>), + {ok, 404, _} = request(put, enable_path(foo, BridgeID), Config), + {ok, 404, _} = request(put, enable_path(true, "foo"), Config), + {ok, 404, _} = request(put, enable_path(true, "webhook:foo"), Config), - {ok, 400, Res} = request(post, operation_path(node, start, BridgeID), <<"">>), + {ok, 400, Res} = request(post, {operation, node, start, BridgeID}, <<>>, fun json/1, Config), ?assertEqual( - <<"{\"code\":\"BAD_REQUEST\",\"message\":\"Forbidden operation, bridge not enabled\"}">>, + #{ + <<"code">> => <<"BAD_REQUEST">>, + <<"message">> => <<"Forbidden operation, bridge not enabled">> + }, Res ), - {ok, 400, Res} = request(post, operation_path(cluster, start, BridgeID), <<"">>), + {ok, 400, Res} = request(post, {operation, cluster, start, BridgeID}, <<>>, fun json/1, Config), %% enable a stopped bridge - {ok, 204, <<>>} = request(put, enable_path(true, BridgeID), <<"">>), - {ok, 200, Bridge4} = request(get, uri(["bridges", BridgeID]), []), - ?assertMatch(#{<<"status">> := <<"connected">>}, emqx_json:decode(Bridge4, [return_maps])), + {ok, 204, <<>>} = request(put, enable_path(true, BridgeID), Config), + ?assertMatch( + {ok, 200, #{<<"status">> := <<"connected">>}}, + request_json(get, uri(["bridges", BridgeID]), Config) + ), %% delete the bridge - {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), []), - {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []). + {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), Config), + {ok, 200, []} = request_json(get, uri(["bridges"]), Config). t_reset_bridges(Config) -> %% assert there's no bridges at first - {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []), + {ok, 200, []} = request_json(get, uri(["bridges"]), Config), Name = ?BRIDGE_NAME, Port = ?config(port, Config), URL1 = ?URL(Port, "abc"), - {ok, 201, Bridge} = request( - post, - uri(["bridges"]), - ?HTTP_BRIDGE(URL1, Name) + ?assertMatch( + {ok, 201, #{ + <<"type">> := ?BRIDGE_TYPE_HTTP, + <<"name">> := Name, + <<"enable">> := true, + <<"status">> := <<"connected">>, + <<"node_status">> := [_ | _], + <<"url">> := URL1 + }}, + request_json( + post, + uri(["bridges"]), + ?HTTP_BRIDGE(URL1, Name), + Config + ) ), - %ct:pal("the bridge ==== ~p", [Bridge]), - #{ - <<"type">> := ?BRIDGE_TYPE_HTTP, - <<"name">> := Name, - <<"enable">> := true, - <<"status">> := <<"connected">>, - <<"node_status">> := [_ | _], - <<"url">> := URL1 - } = emqx_json:decode(Bridge, [return_maps]), BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_HTTP, Name), - {ok, 204, <<>>} = request(put, uri(["bridges", BridgeID, "metrics/reset"]), []), + {ok, 204, <<>>} = request(put, uri(["bridges", BridgeID, "metrics/reset"]), Config), %% delete the bridge - {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), []), - {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []). + {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), Config), + {ok, 200, []} = request_json(get, uri(["bridges"]), Config). -t_with_redact_update(_Config) -> +t_with_redact_update(Config) -> Name = <<"redact_update">>, Type = <<"mqtt">>, Password = <<"123456">>, @@ -812,20 +971,18 @@ t_with_redact_update(_Config) -> {ok, 201, _} = request( post, uri(["bridges"]), - Template + Template, + Config ), %% update with redacted config - Conf = emqx_misc:redact(Template), + BridgeConf = emqx_misc:redact(Template), BridgeID = emqx_bridge_resource:bridge_id(Type, Name), - {ok, 200, _ResBin} = request( - put, - uri(["bridges", BridgeID]), - Conf + {ok, 200, _} = request(put, uri(["bridges", BridgeID]), BridgeConf, Config), + ?assertEqual( + Password, + get_raw_config([bridges, Type, Name, password], Config) ), - RawConf = emqx:get_raw_config([bridges, Type, Name]), - Value = maps:get(<<"password">>, RawConf), - ?assertEqual(Password, Value), ok. t_bridges_probe(Config) -> @@ -835,59 +992,62 @@ t_bridges_probe(Config) -> {ok, 204, <<>>} = request( post, uri(["bridges_probe"]), - ?HTTP_BRIDGE(URL) + ?HTTP_BRIDGE(URL), + Config ), %% second time with same name is ok since no real bridge created {ok, 204, <<>>} = request( post, uri(["bridges_probe"]), - ?HTTP_BRIDGE(URL) + ?HTTP_BRIDGE(URL), + Config ), - {ok, 400, NxDomain} = request( - post, - uri(["bridges_probe"]), - ?HTTP_BRIDGE(<<"http://203.0.113.3:1234/foo">>) - ), ?assertMatch( - #{ + {ok, 400, #{ <<"code">> := <<"TEST_FAILED">>, <<"message">> := _ - }, - emqx_json:decode(NxDomain, [return_maps]) + }}, + request_json( + post, + uri(["bridges_probe"]), + ?HTTP_BRIDGE(<<"http://203.0.113.3:1234/foo">>), + Config + ) ), {ok, 204, _} = request( post, uri(["bridges_probe"]), - ?MQTT_BRIDGE(<<"127.0.0.1:1883">>) + ?MQTT_BRIDGE(<<"127.0.0.1:1883">>), + Config ), - {ok, 400, ConnRefused} = request( - post, - uri(["bridges_probe"]), - ?MQTT_BRIDGE(<<"127.0.0.1:2883">>) - ), ?assertMatch( - #{ + {ok, 400, #{ <<"code">> := <<"TEST_FAILED">>, <<"message">> := <<"Connection refused">> - }, - emqx_json:decode(ConnRefused, [return_maps]) + }}, + request_json( + post, + uri(["bridges_probe"]), + ?MQTT_BRIDGE(<<"127.0.0.1:2883">>), + Config + ) ), - {ok, 400, CouldNotResolveHost} = request( - post, - uri(["bridges_probe"]), - ?MQTT_BRIDGE(<<"nohost:2883">>) - ), ?assertMatch( - #{ + {ok, 400, #{ <<"code">> := <<"TEST_FAILED">>, <<"message">> := <<"Could not resolve host">> - }, - emqx_json:decode(CouldNotResolveHost, [return_maps]) + }}, + request_json( + post, + uri(["bridges_probe"]), + ?MQTT_BRIDGE(<<"nohost:2883">>), + Config + ) ), AuthnConfig = #{ @@ -896,118 +1056,123 @@ t_bridges_probe(Config) -> <<"user_id_type">> => <<"username">> }, Chain = 'mqtt:global', - emqx:update_config( + {ok, _} = update_config( [authentication], - {create_authenticator, Chain, AuthnConfig} + {create_authenticator, Chain, AuthnConfig}, + Config ), User = #{user_id => <<"u">>, password => <<"p">>}, AuthenticatorID = <<"password_based:built_in_database">>, - {ok, _} = emqx_authentication:add_user( + {ok, _} = add_user_auth( Chain, AuthenticatorID, - User + User, + Config ), - {ok, 400, Unauthorized} = request( - post, - uri(["bridges_probe"]), - ?MQTT_BRIDGE(<<"127.0.0.1:1883">>)#{<<"proto_ver">> => <<"v4">>} - ), ?assertMatch( - #{ + {ok, 400, #{ <<"code">> := <<"TEST_FAILED">>, <<"message">> := <<"Unauthorized client">> - }, - emqx_json:decode(Unauthorized, [return_maps]) + }}, + request_json( + post, + uri(["bridges_probe"]), + ?MQTT_BRIDGE(<<"127.0.0.1:1883">>)#{<<"proto_ver">> => <<"v4">>}, + Config + ) ), - {ok, 400, Malformed} = request( - post, - uri(["bridges_probe"]), - ?MQTT_BRIDGE(<<"127.0.0.1:1883">>)#{ - <<"proto_ver">> => <<"v4">>, <<"password">> => <<"mySecret">>, <<"username">> => <<"u">> - } - ), ?assertMatch( - #{ + {ok, 400, #{ <<"code">> := <<"TEST_FAILED">>, <<"message">> := <<"Bad username or password">> - }, - emqx_json:decode(Malformed, [return_maps]) + }}, + request_json( + post, + uri(["bridges_probe"]), + ?MQTT_BRIDGE(<<"127.0.0.1:1883">>)#{ + <<"proto_ver">> => <<"v4">>, + <<"password">> => <<"mySecret">>, + <<"username">> => <<"u">> + }, + Config + ) ), - {ok, 400, NotAuthorized} = request( - post, - uri(["bridges_probe"]), - ?MQTT_BRIDGE(<<"127.0.0.1:1883">>) - ), ?assertMatch( - #{ + {ok, 400, #{ <<"code">> := <<"TEST_FAILED">>, <<"message">> := <<"Not authorized">> - }, - emqx_json:decode(NotAuthorized, [return_maps]) + }}, + request_json( + post, + uri(["bridges_probe"]), + ?MQTT_BRIDGE(<<"127.0.0.1:1883">>), + Config + ) ), - {ok, 400, BadReq} = request( - post, - uri(["bridges_probe"]), - ?BRIDGE(<<"bad_bridge">>, <<"unknown_type">>) + ?assertMatch( + {ok, 400, #{<<"code">> := <<"BAD_REQUEST">>}}, + request_json( + post, + uri(["bridges_probe"]), + ?BRIDGE(<<"bad_bridge">>, <<"unknown_type">>), + Config + ) ), - ?assertMatch(#{<<"code">> := <<"BAD_REQUEST">>}, emqx_json:decode(BadReq, [return_maps])), ok. t_metrics(Config) -> Port = ?config(port, Config), %% assert we there's no bridges at first - {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []), + {ok, 200, []} = request_json(get, uri(["bridges"]), Config), %% then we add a webhook bridge, using POST %% POST /bridges/ will create a bridge URL1 = ?URL(Port, "path1"), Name = ?BRIDGE_NAME, - {ok, 201, Bridge} = request( - post, - uri(["bridges"]), - ?HTTP_BRIDGE(URL1, Name) + ?assertMatch( + {ok, 201, + Bridge = #{ + <<"type">> := ?BRIDGE_TYPE_HTTP, + <<"name">> := Name, + <<"enable">> := true, + <<"status">> := _, + <<"node_status">> := [_ | _], + <<"url">> := URL1 + }} when + %% assert that the bridge return doesn't contain metrics anymore + not is_map_key(<<"metrics">>, Bridge) andalso + not is_map_key(<<"node_metrics">>, Bridge), + request_json( + post, + uri(["bridges"]), + ?HTTP_BRIDGE(URL1, Name), + Config + ) ), - %ct:pal("---bridge: ~p", [Bridge]), - Decoded = emqx_json:decode(Bridge, [return_maps]), - #{ - <<"type">> := ?BRIDGE_TYPE_HTTP, - <<"name">> := Name, - <<"enable">> := true, - <<"status">> := _, - <<"node_status">> := [_ | _], - <<"url">> := URL1 - } = Decoded, - - %% assert that the bridge return doesn't contain metrics anymore - ?assertNot(maps:is_key(<<"metrics">>, Decoded)), - ?assertNot(maps:is_key(<<"node_metrics">>, Decoded)), - BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_HTTP, Name), %% check for empty bridge metrics - {ok, 200, Bridge1Str} = request(get, uri(["bridges", BridgeID, "metrics"]), []), ?assertMatch( - #{ + {ok, 200, #{ <<"metrics">> := #{<<"success">> := 0}, <<"node_metrics">> := [_ | _] - }, - emqx_json:decode(Bridge1Str, [return_maps]) + }}, + request_json(get, uri(["bridges", BridgeID, "metrics"]), Config) ), %% check that the bridge doesn't contain metrics anymore - {ok, 200, Bridge2Str} = request(get, uri(["bridges", BridgeID]), []), - Decoded2 = emqx_json:decode(Bridge2Str, [return_maps]), - ?assertNot(maps:is_key(<<"metrics">>, Decoded2)), - ?assertNot(maps:is_key(<<"node_metrics">>, Decoded2)), + {ok, 200, Bridge} = request_json(get, uri(["bridges", BridgeID]), Config), + ?assertNot(maps:is_key(<<"metrics">>, Bridge)), + ?assertNot(maps:is_key(<<"node_metrics">>, Bridge)), %% send an message to emqx and the message should be forwarded to the HTTP server Body = <<"my msg">>, - emqx:publish(emqx_message:make(<<"emqx_webhook/1">>, Body)), + _ = publish_message(<<"emqx_webhook/1">>, Body, Config), ?assert( receive {http_server, received, #{ @@ -1025,21 +1190,20 @@ t_metrics(Config) -> ), %% check for non-empty bridge metrics - {ok, 200, Bridge3Str} = request(get, uri(["bridges", BridgeID, "metrics"]), []), ?assertMatch( - #{ + {ok, 200, #{ <<"metrics">> := #{<<"success">> := _}, <<"node_metrics">> := [_ | _] - }, - emqx_json:decode(Bridge3Str, [return_maps]) + }}, + request_json(get, uri(["bridges", BridgeID, "metrics"]), Config) ), %% check that metrics isn't returned when listing all bridges - {ok, 200, BridgesStr} = request(get, uri(["bridges"]), []), + {ok, 200, Bridges} = request_json(get, uri(["bridges"]), Config), ?assert( lists:all( fun(E) -> not maps:is_key(<<"metrics">>, E) end, - emqx_json:decode(BridgesStr, [return_maps]) + Bridges ) ), ok. @@ -1058,28 +1222,76 @@ t_inconsistent_webhook_request_timeouts(Config) -> <<"resource_opts">> => #{<<"request_timeout">> => <<"2s">>} } ), - {ok, 201, RawResponse} = request( - post, - uri(["bridges"]), - BadBridgeParams - ), - %% note: same value on both fields ?assertMatch( - #{ + {ok, 201, #{ + %% note: same value on both fields <<"request_timeout">> := <<"2s">>, <<"resource_opts">> := #{<<"request_timeout">> := <<"2s">>} - }, - emqx_json:decode(RawResponse, [return_maps]) + }}, + request_json( + post, + uri(["bridges"]), + BadBridgeParams, + Config + ) ), ok. -operation_path(node, Oper, BridgeID) -> - uri(["nodes", node(), "bridges", BridgeID, Oper]); -operation_path(cluster, Oper, BridgeID) -> +%% + +request(Method, URL, Config) -> + request(Method, URL, [], Config). + +request(Method, {operation, Type, Op, BridgeID}, Body, Config) -> + URL = operation_path(Type, Op, BridgeID, Config), + request(Method, URL, Body, Config); +request(Method, URL, Body, Config) -> + Opts = #{compatible_mode => true, httpc_req_opts => [{body_format, binary}]}, + emqx_mgmt_api_test_util:request_api(Method, URL, [], auth_header(Config), Body, Opts). + +request(Method, URL, Body, Decoder, Config) -> + case request(Method, URL, Body, Config) of + {ok, Code, Response} -> + {ok, Code, Decoder(Response)}; + Otherwise -> + Otherwise + end. + +request_json(Method, URLLike, Config) -> + request(Method, URLLike, [], fun json/1, Config). + +request_json(Method, URLLike, Body, Config) -> + request(Method, URLLike, Body, fun json/1, Config). + +auth_header(Config) -> + erpc:call(?config(api_node, Config), emqx_common_test_http, default_auth_header, []). + +operation_path(node, Oper, BridgeID, Config) -> + uri(["nodes", ?config(api_node, Config), "bridges", BridgeID, Oper]); +operation_path(cluster, Oper, BridgeID, _Config) -> uri(["bridges", BridgeID, Oper]). enable_path(Enable, BridgeID) -> uri(["bridges", BridgeID, "enable", Enable]). +publish_message(Topic, Body, Config) -> + Node = ?config(api_node, Config), + erpc:call(Node, emqx, publish, [emqx_message:make(Topic, Body)]). + +update_config(Path, Value, Config) -> + Node = ?config(api_node, Config), + erpc:call(Node, emqx, update_config, [Path, Value]). + +get_raw_config(Path, Config) -> + Node = ?config(api_node, Config), + erpc:call(Node, emqx, get_raw_config, [Path]). + +add_user_auth(Chain, AuthenticatorID, User, Config) -> + Node = ?config(api_node, Config), + erpc:call(Node, emqx_authentication, add_user, [Chain, AuthenticatorID, User]). + str(S) when is_list(S) -> S; str(S) when is_binary(S) -> binary_to_list(S). + +json(B) when is_binary(B) -> + emqx_json:decode(B, [return_maps]). From d942a531e638310619dd378578b2666ef4932ac7 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 23 Mar 2023 13:23:56 +0300 Subject: [PATCH 098/196] fix(test): avoid unnecessary cleanups This might make some noise in the logs, depending on the testsuite configuration, because some test setup might be missing required applications. --- apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl index fdbeb1be4..8899cd24a 100644 --- a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl @@ -174,7 +174,6 @@ end_per_group(cluster, Config) -> ); end_per_group(_, _Config) -> emqx_mgmt_api_test_util:end_suite(?SUITE_APPS), - mria:clear_table(emqx_authn_mnesia), ok. init_per_testcase(t_broken_bpapi_vsn, Config) -> @@ -1070,6 +1069,10 @@ t_bridges_probe(Config) -> Config ), + emqx_common_test_helpers:on_exit(fun() -> + delete_user_auth(Chain, AuthenticatorID, User, Config) + end), + ?assertMatch( {ok, 400, #{ <<"code">> := <<"TEST_FAILED">>, @@ -1290,6 +1293,10 @@ add_user_auth(Chain, AuthenticatorID, User, Config) -> Node = ?config(api_node, Config), erpc:call(Node, emqx_authentication, add_user, [Chain, AuthenticatorID, User]). +delete_user_auth(Chain, AuthenticatorID, User, Config) -> + Node = ?config(api_node, Config), + erpc:call(Node, emqx_authentication, delete_user, [Chain, AuthenticatorID, User]). + str(S) when is_list(S) -> S; str(S) when is_binary(S) -> binary_to_list(S). From b14a42088534e861bd10cee2dcb36597654bb6f9 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 23 Mar 2023 15:20:32 +0300 Subject: [PATCH 099/196] fix(test): delete mnesia schema in-between test suites When using `emqx_common_test_helpers` at least. This should help reduce the need for superfluous cleanups and test failures due to unclean state. --- apps/emqx/test/emqx_common_test_helpers.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index f445bea94..3575f2815 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -299,6 +299,7 @@ generate_config(SchemaModule, ConfigFile) when is_atom(SchemaModule) -> -spec stop_apps(list()) -> ok. stop_apps(Apps) -> [application:stop(App) || App <- Apps ++ [emqx, ekka, mria, mnesia]], + ok = mria_mnesia:delete_schema(), %% to avoid inter-suite flakiness application:unset_env(emqx, init_config_load_done), persistent_term:erase(?EMQX_AUTHENTICATION_SCHEMA_MODULE_PT_KEY), From 40f7fd691ea092d701b7e730a8a6c8ff1e802765 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 23 Mar 2023 15:23:42 +0300 Subject: [PATCH 100/196] test(dashboard): reuse `emqx_mgmt_api_test_util` in testsuites To reduce the init / teardown complexity. --- .../test/emqx_dashboard_SUITE.erl | 15 ++------------ .../test/emqx_dashboard_admin_SUITE.erl | 20 +++---------------- .../test/emqx_dashboard_bad_api_SUITE.erl | 7 +------ .../test/emqx_dashboard_error_code_SUITE.erl | 19 +++--------------- .../test/emqx_dashboard_haproxy_SUITE.erl | 13 +++--------- .../test/emqx_dashboard_monitor_SUITE.erl | 17 +++------------- .../test/emqx_swagger_parameter_SUITE.erl | 19 +++--------------- .../test/emqx_swagger_response_SUITE.erl | 12 ++--------- 8 files changed, 20 insertions(+), 102 deletions(-) diff --git a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl index 18393a40e..e951a9a2a 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl @@ -57,22 +57,11 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - emqx_common_test_helpers:start_apps( - [emqx_management, emqx_dashboard], - fun set_special_configs/1 - ), + emqx_mgmt_api_test_util:init_suite([emqx_management]), Config. end_per_suite(_Config) -> - mnesia:clear_table(?ADMIN), - emqx_common_test_helpers:stop_apps([emqx_dashboard, emqx_management]), - mria:stop(). - -set_special_configs(emqx_dashboard) -> - emqx_dashboard_api_test_helpers:set_default_config(), - ok; -set_special_configs(_) -> - ok. + emqx_mgmt_api_test_util:end_suite([emqx_management]). t_overview(_) -> mnesia:clear_table(?ADMIN), diff --git a/apps/emqx_dashboard/test/emqx_dashboard_admin_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_admin_SUITE.erl index 9ae5d4418..c12849ac7 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_admin_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_admin_SUITE.erl @@ -19,36 +19,22 @@ -compile(export_all). -include("emqx_dashboard.hrl"). --include_lib("emqx/include/http_api.hrl"). -include_lib("eunit/include/eunit.hrl"). all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - mria:start(), - application:load(emqx_dashboard), - emqx_common_test_helpers:start_apps([emqx_conf, emqx_dashboard], fun set_special_configs/1), + emqx_mgmt_api_test_util:init_suite([emqx_conf]), Config. -set_special_configs(emqx_dashboard) -> - emqx_dashboard_api_test_helpers:set_default_config(), - ok; -set_special_configs(_) -> - ok. - -end_per_suite(Config) -> - end_suite(), - Config. +end_per_suite(_Config) -> + emqx_mgmt_api_test_util:end_suite([emqx_conf]). end_per_testcase(_, _Config) -> All = emqx_dashboard_admin:all_users(), [emqx_dashboard_admin:remove_user(Name) || #{username := Name} <- All]. -end_suite() -> - application:unload(emqx_management), - emqx_common_test_helpers:stop_apps([emqx_dashboard]). - t_check_user(_) -> Username = <<"admin1">>, Password = <<"public_1">>, diff --git a/apps/emqx_dashboard/test/emqx_dashboard_bad_api_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_bad_api_SUITE.erl index a9b448662..92327a7db 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_bad_api_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_bad_api_SUITE.erl @@ -31,15 +31,10 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - mria:start(), emqx_mgmt_api_test_util:init_suite([emqx_conf]), Config. -end_per_suite(Config) -> - end_suite(), - Config. - -end_suite() -> +end_per_suite(_Config) -> emqx_mgmt_api_test_util:end_suite([emqx_conf]). t_bad_api_path(_) -> diff --git a/apps/emqx_dashboard/test/emqx_dashboard_error_code_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_error_code_SUITE.erl index 5def3c9dd..19d3f471e 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_error_code_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_error_code_SUITE.erl @@ -29,24 +29,11 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - mria:start(), - application:load(emqx_dashboard), - emqx_common_test_helpers:start_apps([emqx_conf, emqx_dashboard], fun set_special_configs/1), + emqx_mgmt_api_test_util:init_suite([emqx_conf]), Config. -set_special_configs(emqx_dashboard) -> - emqx_dashboard_api_test_helpers:set_default_config(), - ok; -set_special_configs(_) -> - ok. - -end_per_suite(Config) -> - end_suite(), - Config. - -end_suite() -> - application:unload(emqx_management), - emqx_common_test_helpers:stop_apps([emqx_dashboard]). +end_per_suite(_Config) -> + emqx_mgmt_api_test_util:end_suite([emqx_conf]). t_all_code(_) -> HrlDef = ?ERROR_CODES, diff --git a/apps/emqx_dashboard/test/emqx_dashboard_haproxy_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_haproxy_SUITE.erl index a05de339b..cb6a5a9fd 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_haproxy_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_haproxy_SUITE.erl @@ -26,10 +26,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - emqx_common_test_helpers:start_apps( - [emqx_management, emqx_dashboard], - fun set_special_configs/1 - ), + emqx_mgmt_api_test_util:init_suite([emqx_management], fun set_special_configs/1), Config. set_special_configs(emqx_dashboard) -> @@ -38,12 +35,8 @@ set_special_configs(emqx_dashboard) -> set_special_configs(_) -> ok. -end_per_suite(Config) -> - application:unload(emqx_management), - mnesia:clear_table(?ADMIN), - emqx_common_test_helpers:stop_apps([emqx_dashboard, emqx_management]), - mria:stop(), - Config. +end_per_suite(_Config) -> + emqx_mgmt_api_test_util:end_suite([emqx_management]). t_status(_Config) -> ProxyInfo = #{ diff --git a/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl index fa2373ac3..f35652f8e 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl @@ -31,22 +31,11 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - application:load(emqx_dashboard), - mria:start(), - mnesia:clear_table(?ADMIN), - emqx_common_test_helpers:start_apps([emqx_dashboard], fun set_special_configs/1), + emqx_mgmt_api_test_util:init_suite([]), Config. -end_per_suite(Config) -> - mnesia:clear_table(?ADMIN), - emqx_common_test_helpers:stop_apps([emqx_dashboard]), - Config. - -set_special_configs(emqx_dashboard) -> - emqx_dashboard_api_test_helpers:set_default_config(), - ok; -set_special_configs(_) -> - ok. +end_per_suite(_Config) -> + emqx_mgmt_api_test_util:end_suite([]). t_monitor_samplers_all(_Config) -> timer:sleep(?DEFAULT_SAMPLE_INTERVAL * 2 * 1000 + 20), diff --git a/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl index 5d89fb273..472e90405 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl @@ -63,25 +63,12 @@ groups() -> ]. init_per_suite(Config) -> - mria:start(), - application:load(emqx_dashboard), - emqx_common_test_helpers:start_apps([emqx_conf, emqx_dashboard], fun set_special_configs/1), + emqx_mgmt_api_test_util:init_suite([emqx_conf]), emqx_dashboard:init_i18n(), Config. -set_special_configs(emqx_dashboard) -> - emqx_dashboard_api_test_helpers:set_default_config(), - ok; -set_special_configs(_) -> - ok. - -end_per_suite(Config) -> - end_suite(), - Config. - -end_suite() -> - application:unload(emqx_management), - emqx_common_test_helpers:stop_apps([emqx_dashboard]). +end_per_suite(_Config) -> + emqx_mgmt_api_test_util:end_suite([emqx_conf]). t_in_path(_Config) -> Expect = diff --git a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl index c9cfba254..314154633 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl @@ -32,25 +32,17 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - mria:start(), - application:load(emqx_dashboard), - emqx_common_test_helpers:start_apps([emqx_conf, emqx_dashboard], fun set_special_configs/1), + emqx_mgmt_api_test_util:init_suite([emqx_conf]), emqx_dashboard:init_i18n(), Config. -set_special_configs(emqx_dashboard) -> - emqx_dashboard_api_test_helpers:set_default_config(), - ok; -set_special_configs(_) -> - ok. - end_per_suite(Config) -> end_suite(), Config. end_suite() -> application:unload(emqx_management), - emqx_common_test_helpers:stop_apps([emqx_dashboard]). + emqx_mgmt_api_test_util:end_suite([emqx_conf]). t_simple_binary(_config) -> Path = "/simple/bin", From 6750618d9a69170fab504805f2e91e8079a36b62 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 28 Mar 2023 17:44:33 +0800 Subject: [PATCH 101/196] fix(lwm2m): ignore unknown ObjectId to auto observe --- .../emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl | 3 +- .../src/lwm2m/emqx_lwm2m_session.erl | 23 +++++++++---- .../src/lwm2m/emqx_lwm2m_xml_object_db.erl | 7 ++-- .../src/lwm2m/include/emqx_lwm2m.hrl | 33 ++++++++++++++++++- 4 files changed, 55 insertions(+), 11 deletions(-) diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl index d0b362dda..090af3e87 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl @@ -138,7 +138,7 @@ mqtt_to_coap(AlternatePath, InputCmd = #{<<"msgType">> := <<"discover">>, <<"dat [ {uri_path, FullPathList}, {uri_query, QueryList}, - {'accept', ?LWM2M_FORMAT_LINK} + {accept, ?LWM2M_FORMAT_LINK} ] ), InputCmd @@ -241,6 +241,7 @@ empty_ack_to_mqtt(Ref) -> coap_failure_to_mqtt(Ref, MsgType) -> make_base_response(maps:put(<<"msgType">>, MsgType, Ref)). +%% TODO: application/link-format content_to_mqtt(CoapPayload, <<"text/plain">>, Ref) -> emqx_lwm2m_message:text_to_json(extract_path(Ref), CoapPayload); content_to_mqtt(CoapPayload, <<"application/octet-stream">>, Ref) -> diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_session.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_session.erl index 19cd5c25d..595543046 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_session.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_session.erl @@ -513,12 +513,16 @@ observe_object_list(AlternatePath, ObjectList, Session) -> true -> Acc; false -> - try - emqx_lwm2m_xml_object_db:find_objectid(binary_to_integer(ObjId)), - observe_object(AlternatePath, ObjectPath, Acc) - catch - error:no_xml_definition -> - Acc + case emqx_lwm2m_xml_object_db:find_objectid(binary_to_integer(ObjId)) of + {error, no_xml_definition} -> + ?SLOG(warning, #{ + msg => "ignore_observer_resource", + reason => no_xml_definition, + object_id => ObjId + }), + Acc; + _ -> + observe_object(AlternatePath, ObjectPath, Acc) end end end, @@ -538,15 +542,20 @@ deliver_auto_observe_to_coap(AlternatePath, TermData, Session) -> path => AlternatePath, data => TermData }), - {Req, Ctx} = emqx_lwm2m_cmd:mqtt_to_coap(AlternatePath, TermData), + {Req0, Ctx} = emqx_lwm2m_cmd:mqtt_to_coap(AlternatePath, TermData), + Req = alloc_token(Req0), maybe_do_deliver_to_coap(Ctx, Req, 0, false, Session). is_auto_observe() -> emqx:get_config([gateway, lwm2m, auto_observe]). +alloc_token(Req = #coap_message{}) -> + Req#coap_message{token = crypto:strong_rand_bytes(4)}. + %%-------------------------------------------------------------------- %% Response %%-------------------------------------------------------------------- + handle_coap_response( {Ctx = #{<<"msgType">> := EventType}, #coap_message{ method = CoapMsgMethod, diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl index 19335768f..58373e114 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl @@ -57,6 +57,7 @@ start_link(XmlDir) -> gen_server:start_link({local, ?MODULE}, ?MODULE, [XmlDir], []). +-spec find_objectid(integer()) -> {error, no_xml_definition} | xmerl:xmlElement(). find_objectid(ObjectId) -> ObjectIdInt = case is_list(ObjectId) of @@ -65,9 +66,10 @@ find_objectid(ObjectId) -> end, case ets:lookup(?LWM2M_OBJECT_DEF_TAB, ObjectIdInt) of [] -> {error, no_xml_definition}; - [{ObjectId, Xml}] -> Xml + [{_ObjectId, Xml}] -> Xml end. +-spec find_name(string()) -> {error, no_xml_definition} | xmerl:xmlElement(). find_name(Name) -> NameBinary = case is_list(Name) of @@ -77,10 +79,11 @@ find_name(Name) -> case ets:lookup(?LWM2M_OBJECT_NAME_TO_ID_TAB, NameBinary) of [] -> {error, no_xml_definition}; - [{NameBinary, ObjectId}] -> + [{_NameBinary, ObjectId}] -> find_objectid(ObjectId) end. +-spec stop() -> ok. stop() -> gen_server:stop(?MODULE). diff --git a/apps/emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl b/apps/emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl index 1f02a1637..e1a6ec0d6 100644 --- a/apps/emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl +++ b/apps/emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl @@ -36,8 +36,39 @@ -define(ERR_BAD_REQUEST, <<"Bad Request">>). -define(REG_PREFIX, <<"rd">>). +%%-------------------------------------------------------------------- +%% Data formats for transferring resource information, defined in +%% OMA-TS-LightweightM2M-V1_0_1-20170704-A + +%% 0: Plain text. 0 is numeric value used in CoAP Content-Format option. +%% The plain text format is used for "Read" and "Write" operations on singular +%% Resources. i.e: /3/0/0 +%% +%% This data format has a Media Type of "text/plain". -define(LWM2M_FORMAT_PLAIN_TEXT, 0). + +%% 40: Link format. 40 is numeric value used in CoAP Content-Format option. +%% -define(LWM2M_FORMAT_LINK, 40). + +%% 42: Opaque. 41 is numeric value used in CoAP Content-Format option. +%% The opaque format is used for "Read" and "Write" operations on singular +%% Resources where the value of the Resource is an opaque binary value. +%% i.e: firmware images or opaque value from top layer. +%% +%% This data format has a Media Type of "application/octet-stream". -define(LWM2M_FORMAT_OPAQUE, 42). + +%% 11542: TLV. 11542 is numeric value used in CoAP Content-Format option. +%% For "Read" and "Write" operation, the binary TLV format is used to represent +%% an array of values or a single value using a compact binary representation. +%% +%% This data format has a Media Type of "application/vnd.oma.lwm2m+tlv". -define(LWM2M_FORMAT_TLV, 11542). --define(LWMWM_FORMAT_JSON, 11543). + +%% 11543: JSON. 11543 is numeric value used in CoAP Content-Format option. +%% The client may support the JSON format for "Read" and "Write" operations to +%% represent multiple resource or single resource values. +%% +%% This data format has a Media Type of "application/vnd.oma.lwm2m+json". +-define(LWM2M_FORMAT_OMA_JSON, 11543). From 6db5a4872aa47ce5b0ff69e9126bf1643cd5c36c Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 28 Mar 2023 18:31:20 +0800 Subject: [PATCH 102/196] test(lwm2m): cover auto_observe mechanism --- .../src/lwm2m/emqx_lwm2m_session.erl | 21 ++-- apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl | 116 ++++++++++++++---- 2 files changed, 102 insertions(+), 35 deletions(-) diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_session.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_session.erl index 595543046..8634280e3 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_session.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_session.erl @@ -15,11 +15,12 @@ %%-------------------------------------------------------------------- -module(emqx_lwm2m_session). +-include("src/coap/include/emqx_coap.hrl"). +-include("src/lwm2m/include/emqx_lwm2m.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). --include("src/coap/include/emqx_coap.hrl"). --include("src/lwm2m/include/emqx_lwm2m.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). %% API -export([ @@ -513,13 +514,17 @@ observe_object_list(AlternatePath, ObjectList, Session) -> true -> Acc; false -> - case emqx_lwm2m_xml_object_db:find_objectid(binary_to_integer(ObjId)) of + ObjId1 = binary_to_integer(ObjId), + case emqx_lwm2m_xml_object_db:find_objectid(ObjId1) of {error, no_xml_definition} -> - ?SLOG(warning, #{ - msg => "ignore_observer_resource", - reason => no_xml_definition, - object_id => ObjId - }), + ?tp( + warning, + ignore_observer_resource, + #{ + reason => no_xml_definition, + object_id => ObjId1 + } + ), Acc; _ -> observe_object(AlternatePath, ObjectPath, Acc) diff --git a/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl b/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl index f91bbf16e..fc852709c 100644 --- a/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl @@ -35,29 +35,7 @@ -include("src/coap/include/emqx_coap.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). - --define(CONF_DEFAULT, << - "\n" - "gateway.lwm2m {\n" - " xml_dir = \"../../lib/emqx_gateway/src/lwm2m/lwm2m_xml\"\n" - " lifetime_min = 1s\n" - " lifetime_max = 86400s\n" - " qmode_time_window = 22\n" - " auto_observe = false\n" - " mountpoint = \"lwm2m/${username}\"\n" - " update_msg_publish_condition = contains_object_list\n" - " translators {\n" - " command = {topic = \"/dn/#\", qos = 0}\n" - " response = {topic = \"/up/resp\", qos = 0}\n" - " notify = {topic = \"/up/notify\", qos = 0}\n" - " register = {topic = \"/up/resp\", qos = 0}\n" - " update = {topic = \"/up/resp\", qos = 0}\n" - " }\n" - " listeners.udp.default {\n" - " bind = 5783\n" - " }\n" - "}\n" ->>). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). -record(coap_content, {content_format, payload = <<>>}). @@ -99,7 +77,8 @@ groups() -> %% case06_register_wrong_lifetime, %% now, will ignore wrong lifetime case07_register_alternate_path_01, case07_register_alternate_path_02, - case08_reregister + case08_reregister, + case09_auto_observe ]}, {test_grp_1_read, [RepeatOpt], [ case10_read, @@ -164,8 +143,15 @@ end_per_suite(Config) -> emqx_mgmt_api_test_util:end_suite([emqx_conf, emqx_authn]), Config. -init_per_testcase(_AllTestCase, Config) -> - ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), +init_per_testcase(TestCase, Config) -> + GatewayConfig = + case TestCase of + case09_auto_observe -> + default_config(#{auto_observe => true}); + _ -> + default_config() + end, + ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, GatewayConfig), {ok, _} = application:ensure_all_started(emqx_gateway), {ok, ClientUdpSock} = gen_udp:open(0, [binary, {active, false}]), @@ -187,7 +173,37 @@ end_per_testcase(_AllTestCase, Config) -> ok = application:stop(emqx_gateway). default_config() -> - ?CONF_DEFAULT. + default_config(#{}). + +default_config(Overrides) -> + iolist_to_binary( + io_lib:format( + "\n" + "gateway.lwm2m {\n" + " xml_dir = \"../../lib/emqx_gateway/src/lwm2m/lwm2m_xml\"\n" + " lifetime_min = 1s\n" + " lifetime_max = 86400s\n" + " qmode_time_window = 22\n" + " auto_observe = ~w\n" + " mountpoint = \"lwm2m/${username}\"\n" + " update_msg_publish_condition = contains_object_list\n" + " translators {\n" + " command = {topic = \"/dn/#\", qos = 0}\n" + " response = {topic = \"/up/resp\", qos = 0}\n" + " notify = {topic = \"/up/notify\", qos = 0}\n" + " register = {topic = \"/up/resp\", qos = 0}\n" + " update = {topic = \"/up/resp\", qos = 0}\n" + " }\n" + " listeners.udp.default {\n" + " bind = ~w\n" + " }\n" + "}\n", + [ + maps:get(auto_observe, Overrides, false), + maps:get(bind, Overrides, ?PORT) + ] + ) + ). default_port() -> ?PORT. @@ -762,6 +778,52 @@ case08_reregister(Config) -> %% verify the lwm2m client is still online ?assertEqual(ReadResult, test_recv_mqtt_response(ReportTopic)). +case09_auto_observe(Config) -> + UdpSock = ?config(sock, Config), + Epn = "urn:oma:lwm2m:oma:3", + MsgId1 = 15, + RespTopic = list_to_binary("lwm2m/" ++ Epn ++ "/up/resp"), + emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), + timer:sleep(200), + + ok = snabbkaffe:start_trace(), + + %% step 1, device register ... + test_send_coap_request( + UdpSock, + post, + sprintf("coap://127.0.0.1:~b/rd?ep=~ts<=345&lwm2m=1", [?PORT, Epn]), + #coap_content{ + content_format = <<"text/plain">>, + payload = << + ";rt=\"oma.lwm2m\";ct=11543," + ",,," + >> + }, + [], + MsgId1 + ), + #coap_message{method = Method1} = test_recv_coap_response(UdpSock), + ?assertEqual({ok, created}, Method1), + + #coap_message{ + method = Method2, + token = Token2, + options = Options2 + } = test_recv_coap_request(UdpSock), + ?assertEqual(get, Method2), + ?assertNotEqual(<<>>, Token2), + ?assertMatch( + #{ + observe := 0, + uri_path := [<<"lwm2m">>, <<"3">>, <<"0">>] + }, + Options2 + ), + + {ok, _} = ?block_until(#{?snk_kind := ignore_observer_resource}, 1000), + ok. + case10_read(Config) -> UdpSock = ?config(sock, Config), Epn = "urn:oma:lwm2m:oma:3", From 447f5bf9c55797f524b2f9487563fcbe7fbb3f35 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 28 Mar 2023 18:51:27 +0800 Subject: [PATCH 103/196] chore: add changes --- changes/ce/fix-10257.en.md | 11 +++++++++++ changes/ce/fix-10257.zh.md | 8 ++++++++ 2 files changed, 19 insertions(+) create mode 100644 changes/ce/fix-10257.en.md create mode 100644 changes/ce/fix-10257.zh.md diff --git a/changes/ce/fix-10257.en.md b/changes/ce/fix-10257.en.md new file mode 100644 index 000000000..aa5ed2519 --- /dev/null +++ b/changes/ce/fix-10257.en.md @@ -0,0 +1,11 @@ +Fixed the issue where `auto_observe` was not working in LwM2M Gateway. + +Before the fix, OBSERVE requests were sent without a token, causing failures +that LwM2M clients could not handle. + +After the fix, LwM2M Gateway can correctly observe the resource list carried by +client, furthermore, unknown resources will be ignored and printing the following +warning log: +``` +2023-03-28T18:50:27.771123+08:00 [warning] msg: ignore_observer_resource, mfa: emqx_lwm2m_session:observe_object_list/3, line: 522, peername: 127.0.0.1:56830, clientid: testlwm2mclient, object_id: 31024, reason: no_xml_definition +``` diff --git a/changes/ce/fix-10257.zh.md b/changes/ce/fix-10257.zh.md new file mode 100644 index 000000000..962495f2d --- /dev/null +++ b/changes/ce/fix-10257.zh.md @@ -0,0 +1,8 @@ +修复 LwM2M 网关 `auto_observe` 不工作的问题。 + +在修复之前，下发的 OBSERVE 请求没有 Token 从而导致 LwM2M 客户端无法处理的失败。 + +修复后，能正确监听 LwM2M 携带的资源列表、和会忽略未知的资源，并打印以下日志： +``` +2023-03-28T18:50:27.771123+08:00 [warning] msg: ignore_observer_resource, mfa: emqx_lwm2m_session:observe_object_list/3, line: 522, peername: 127.0.0.1:56830, clientid: testlwm2mclient, object_id: 31024, reason: no_xml_definition +``` From 813ccd7df1357ee7234ee82be2580465b3d1a4e9 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 21 Mar 2023 16:14:12 +0100 Subject: [PATCH 104/196] chore: hide node.perf configs these configs are rarely needed by regular users and if they are to be changed, it should be done by someone who really understands what they mean --- apps/emqx/src/emqx_schema.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 19fc3aae4..bf509b4e5 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -1492,7 +1492,7 @@ fields("broker") -> {"perf", sc( ref("broker_perf"), - #{} + #{hidden => true} )}, {"shared_subscription_group", sc( From c302405408ba4233a97f63a42da7515de36e4b5c Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 22 Mar 2023 20:16:07 +0100 Subject: [PATCH 105/196] fix: fail fast if not able to load config --- apps/emqx_conf/src/emqx_conf_app.erl | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/apps/emqx_conf/src/emqx_conf_app.erl b/apps/emqx_conf/src/emqx_conf_app.erl index 34224c3f2..09478e304 100644 --- a/apps/emqx_conf/src/emqx_conf_app.erl +++ b/apps/emqx_conf/src/emqx_conf_app.erl @@ -28,7 +28,18 @@ -define(DEFAULT_INIT_TXN_ID, -1). start(_StartType, _StartArgs) -> - init_conf(), + try + ok = init_conf() + catch + C:E:St -> + ?SLOG(critical, #{ + msg => failed_to_init_config, + exception => C, + reason => E, + stacktrace => St + }), + init:stop() + end, ok = emqx_config_logger:refresh_config(), emqx_conf_sup:start_link(). @@ -85,9 +96,9 @@ init_load() -> init_conf() -> {ok, TnxId} = copy_override_conf_from_core_node(), - emqx_app:set_init_tnx_id(TnxId), - init_load(), - emqx_app:set_init_config_load_done(). + _ = emqx_app:set_init_tnx_id(TnxId), + ok = init_load(), + ok = emqx_app:set_init_config_load_done(). cluster_nodes() -> maps:get(running_nodes, ekka_cluster:info()) -- [node()]. From fb29f8035bb3107866e71da8f0ef60f1df643d1b Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 22 Mar 2023 22:06:00 +0100 Subject: [PATCH 106/196] feat: make possible to dump full schema (including hidden fields) --- apps/emqx_conf/src/emqx_conf.erl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl index 33214946d..a840c3f1b 100644 --- a/apps/emqx_conf/src/emqx_conf.erl +++ b/apps/emqx_conf/src/emqx_conf.erl @@ -156,7 +156,11 @@ dump_schema(Dir, SchemaModule, I18nFile) -> gen_schema_json(Dir, I18nFile, SchemaModule, Lang) -> SchemaJsonFile = filename:join([Dir, "schema-" ++ Lang ++ ".json"]), io:format(user, "===< Generating: ~s~n", [SchemaJsonFile]), - Opts = #{desc_file => I18nFile, lang => Lang}, + %% EMQX_SCHEMA_FULL_DUMP is quite a hidden API + %% it is used to dump the full schema for EMQX developers and supporters + IncludeHidden = (os:getenv("EMQX_SCHEMA_FULL_DUMP") =:= "1"), + IncludeHidden andalso io:format(user, "===< Including hidden fields~n", []), + Opts = #{desc_file => I18nFile, lang => Lang, include_hidden_fields => IncludeHidden}, JsonMap = hocon_schema_json:gen(SchemaModule, Opts), IoData = jsx:encode(JsonMap, [space, {indent, 4}]), ok = file:write_file(SchemaJsonFile, IoData). From 486352eb6f978a565fea4c9a6d4a966f858f9a6c Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 23 Mar 2023 19:32:00 +0100 Subject: [PATCH 107/196] fix(swagger): do not generate scheam for hidden args --- .../src/emqx_dashboard_swagger.erl | 80 ++++++++++++------- .../test/emqx_swagger_requestBody_SUITE.erl | 2 +- .../test/emqx_swagger_response_SUITE.erl | 2 +- 3 files changed, 52 insertions(+), 32 deletions(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 2290dbedb..18196425f 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -830,36 +830,8 @@ to_bin(X) -> X. parse_object(PropList = [_ | _], Module, Options) when is_list(PropList) -> - {Props, Required, Refs} = - lists:foldl( - fun({Name, Hocon}, {Acc, RequiredAcc, RefsAcc}) -> - NameBin = to_bin(Name), - case hoconsc:is_schema(Hocon) of - true -> - HoconType = hocon_schema:field_schema(Hocon, type), - Init0 = init_prop([default | ?DEFAULT_FIELDS], #{}, Hocon), - SchemaToSpec = schema_converter(Options), - Init = trans_desc(Init0, Hocon, SchemaToSpec, NameBin), - {Prop, Refs1} = SchemaToSpec(HoconType, Module), - NewRequiredAcc = - case is_required(Hocon) of - true -> [NameBin | RequiredAcc]; - false -> RequiredAcc - end, - { - [{NameBin, maps:merge(Prop, Init)} | Acc], - NewRequiredAcc, - Refs1 ++ RefsAcc - }; - false -> - {SubObject, SubRefs} = parse_object(Hocon, Module, Options), - {[{NameBin, SubObject} | Acc], RequiredAcc, SubRefs ++ RefsAcc} - end - end, - {[], [], []}, - PropList - ), - Object = #{<<"type">> => object, <<"properties">> => lists:reverse(Props)}, + {Props, Required, Refs} = parse_object_loop(PropList, Module, Options), + Object = #{<<"type">> => object, <<"properties">> => Props}, case Required of [] -> {Object, Refs}; _ -> {maps:put(required, Required, Object), Refs} @@ -874,6 +846,54 @@ parse_object(Other, Module, Options) -> }} ). +parse_object_loop(PropList0, Module, Options) -> + PropList = lists:filter( + fun({_, Hocon}) -> + case hoconsc:is_schema(Hocon) andalso is_hidden(Hocon) of + true -> false; + false -> true + end + end, + PropList0 + ), + parse_object_loop(PropList, Module, Options, _Props = [], _Required = [], _Refs = []). + +parse_object_loop([], _Modlue, _Options, Props, Required, Refs) -> + {lists:reverse(Props), lists:usort(Required), Refs}; +parse_object_loop([{Name, Hocon} | Rest], Module, Options, Props, Required, Refs) -> + NameBin = to_bin(Name), + case hoconsc:is_schema(Hocon) of + true -> + HoconType = hocon_schema:field_schema(Hocon, type), + Init0 = init_prop([default | ?DEFAULT_FIELDS], #{}, Hocon), + SchemaToSpec = schema_converter(Options), + Init = trans_desc(Init0, Hocon, SchemaToSpec, NameBin), + {Prop, Refs1} = SchemaToSpec(HoconType, Module), + NewRequiredAcc = + case is_required(Hocon) of + true -> [NameBin | Required]; + false -> Required + end, + parse_object_loop( + Rest, + Module, + Options, + [{NameBin, maps:merge(Prop, Init)} | Props], + NewRequiredAcc, + Refs1 ++ Refs + ); + false -> + %% TODO: there is only a handful of such + %% refactor the schema to unify the two cases + {SubObject, SubRefs} = parse_object(Hocon, Module, Options), + parse_object_loop( + Rest, Module, Options, [{NameBin, SubObject} | Props], Required, SubRefs ++ Refs + ) + end. + +is_hidden(Hocon) -> + hocon_schema:is_hidden(Hocon). + is_required(Hocon) -> hocon_schema:field_schema(Hocon, required) =:= true. diff --git a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl index 717a7d4ca..3150ed097 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl @@ -61,7 +61,7 @@ t_object(_Config) -> #{ <<"schema">> => #{ - required => [<<"timeout">>, <<"per_page">>], + required => [<<"per_page">>, <<"timeout">>], <<"properties">> => [ {<<"per_page">>, #{ description => <<"good per page desc">>, diff --git a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl index 314154633..4d1501dae 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl @@ -59,7 +59,7 @@ t_object(_config) -> <<"application/json">> => #{ <<"schema">> => #{ - required => [<<"timeout">>, <<"per_page">>], + required => [<<"per_page">>, <<"timeout">>], <<"properties">> => [ {<<"per_page">>, #{ description => <<"good per page desc">>, From 5a19b6f73e986250960d135a18a628e36455575d Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 27 Mar 2023 16:09:15 +0200 Subject: [PATCH 108/196] chore: pin hocon 0.38.0 (with config importance level annotation) --- apps/emqx/rebar.config | 2 +- mix.exs | 2 +- rebar.config | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 3ec5b6c30..9079322eb 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -29,7 +29,7 @@ {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}}, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.14.6"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}, - {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.37.2"}}}, + {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.38.0"}}}, {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}, diff --git a/mix.exs b/mix.exs index 3c3ce0766..ffa2ce056 100644 --- a/mix.exs +++ b/mix.exs @@ -72,7 +72,7 @@ defmodule EMQXUmbrella.MixProject do # in conflict by emqtt and hocon {:getopt, "1.0.2", override: true}, {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.7", override: true}, - {:hocon, github: "emqx/hocon", tag: "0.37.2", override: true}, + {:hocon, github: "emqx/hocon", tag: "0.38.0", override: true}, {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.2", override: true}, {:esasl, github: "emqx/esasl", tag: "0.2.0"}, {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"}, diff --git a/rebar.config b/rebar.config index 04470030a..b641077ea 100644 --- a/rebar.config +++ b/rebar.config @@ -75,7 +75,7 @@ , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}} , {getopt, "1.0.2"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.7"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.37.2"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.38.0"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}} , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}} From d07987288afe562932891a3b9ced1a13c68638e8 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 27 Mar 2023 16:10:36 +0200 Subject: [PATCH 109/196] chore: add some example annotations for config importance level --- apps/emqx/src/emqx_schema.erl | 8 ++--- apps/emqx_conf/src/emqx_conf.app.src | 2 +- apps/emqx_conf/src/emqx_conf.erl | 13 +++++--- apps/emqx_conf/src/emqx_conf_schema.erl | 33 ++++++++++++++++--- .../src/emqx_dashboard_swagger.erl | 4 +-- .../src/emqx_prometheus.app.src | 2 +- .../src/emqx_prometheus_schema.erl | 12 +++---- .../src/schema/emqx_resource_schema.erl | 2 +- .../src/emqx_ee_bridge_kafka.erl | 2 +- 9 files changed, 54 insertions(+), 24 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index bf509b4e5..94e73cecb 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -230,7 +230,7 @@ roots(low) -> {"crl_cache", sc( ref("crl_cache"), - #{hidden => true} + #{importance => ?IMPORTANCE_HIDDEN} )} ]. @@ -1492,7 +1492,7 @@ fields("broker") -> {"perf", sc( ref("broker_perf"), - #{hidden => true} + #{importance => ?IMPORTANCE_HIDDEN} )}, {"shared_subscription_group", sc( @@ -2997,7 +2997,7 @@ quic_feature_toggle(Desc) -> typerefl:alias("boolean", typerefl:union([true, false, 0, 1])), #{ desc => Desc, - hidden => true, + importance => ?IMPORTANCE_HIDDEN, required => false, converter => fun (true) -> 1; @@ -3012,7 +3012,7 @@ quic_lowlevel_settings_uint(Low, High, Desc) -> range(Low, High), #{ required => false, - hidden => true, + importance => ?IMPORTANCE_HIDDEN, desc => Desc } ). diff --git a/apps/emqx_conf/src/emqx_conf.app.src b/apps/emqx_conf/src/emqx_conf.app.src index fbbffba1f..37707431a 100644 --- a/apps/emqx_conf/src/emqx_conf.app.src +++ b/apps/emqx_conf/src/emqx_conf.app.src @@ -1,6 +1,6 @@ {application, emqx_conf, [ {description, "EMQX configuration management"}, - {vsn, "0.1.14"}, + {vsn, "0.1.15"}, {registered, []}, {mod, {emqx_conf_app, []}}, {applications, [kernel, stdlib, emqx_ctl]}, diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl index a840c3f1b..d03cf9c27 100644 --- a/apps/emqx_conf/src/emqx_conf.erl +++ b/apps/emqx_conf/src/emqx_conf.erl @@ -158,9 +158,13 @@ gen_schema_json(Dir, I18nFile, SchemaModule, Lang) -> io:format(user, "===< Generating: ~s~n", [SchemaJsonFile]), %% EMQX_SCHEMA_FULL_DUMP is quite a hidden API %% it is used to dump the full schema for EMQX developers and supporters - IncludeHidden = (os:getenv("EMQX_SCHEMA_FULL_DUMP") =:= "1"), - IncludeHidden andalso io:format(user, "===< Including hidden fields~n", []), - Opts = #{desc_file => I18nFile, lang => Lang, include_hidden_fields => IncludeHidden}, + IncludeImportance = + case os:getenv("EMQX_SCHEMA_FULL_DUMP") =:= "1" of + true -> ?IMPORTANCE_HIDDEN; + false -> ?IMPORTANCE_LOW + end, + io:format(user, "===< Including fields from importance level: ~p~n", [IncludeImportance]), + Opts = #{desc_file => I18nFile, lang => Lang, include_importance_up_from => IncludeImportance}, JsonMap = hocon_schema_json:gen(SchemaModule, Opts), IoData = jsx:encode(JsonMap, [space, {indent, 4}]), ok = file:write_file(SchemaJsonFile, IoData). @@ -224,7 +228,8 @@ gen_example(File, SchemaModule, I18nFile, Lang) -> title => <<"EMQX Configuration Example">>, body => <<"">>, desc_file => I18nFile, - lang => Lang + lang => Lang, + include_importance_up_from => ?IMPORTANCE_MEDIUM }, Example = hocon_schema_example:gen(SchemaModule, Opts), file:write_file(File, Example). diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 4862be5fe..58bcf9700 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -397,6 +397,7 @@ fields("node") -> #{ default => <<"emqx@127.0.0.1">>, 'readOnly' => true, + importance => ?IMPORTANCE_HIGH, desc => ?DESC(node_name) } )}, @@ -409,6 +410,7 @@ fields("node") -> 'readOnly' => true, sensitive => true, desc => ?DESC(node_cookie), + importance => ?IMPORTANCE_HIGH, converter => fun emqx_schema:password_converter/2 } )}, @@ -419,6 +421,7 @@ fields("node") -> mapping => "vm_args.+P", desc => ?DESC(process_limit), default => 2097152, + importance => ?IMPORTANCE_MEDIUM, 'readOnly' => true } )}, @@ -429,6 +432,7 @@ fields("node") -> mapping => "vm_args.+Q", desc => ?DESC(max_ports), default => 1048576, + importance => ?IMPORTANCE_HIGH, 'readOnly' => true } )}, @@ -439,6 +443,7 @@ fields("node") -> mapping => "vm_args.+zdbbl", desc => ?DESC(dist_buffer_size), default => 8192, + importance => ?IMPORTANCE_LOW, 'readOnly' => true } )}, @@ -449,6 +454,7 @@ fields("node") -> mapping => "vm_args.+e", desc => ?DESC(max_ets_tables), default => 262144, + importance => ?IMPORTANCE_LOW, 'readOnly' => true } )}, @@ -459,6 +465,10 @@ fields("node") -> required => true, 'readOnly' => true, mapping => "emqx.data_dir", + %% for now, it's tricky to use a different data_dir + %% otherwise data paths in cluster config may differ + %% TODO: change configurable data file paths to relative + importance => ?IMPORTANCE_HIDDEN, desc => ?DESC(node_data_dir) } )}, @@ -467,7 +477,7 @@ fields("node") -> hoconsc:array(string()), #{ mapping => "emqx.config_files", - hidden => true, + importance => ?IMPORTANCE_HIDDEN, required => false, 'readOnly' => true } @@ -479,6 +489,7 @@ fields("node") -> mapping => "emqx_machine.global_gc_interval", default => <<"15m">>, desc => ?DESC(node_global_gc_interval), + importance => ?IMPORTANCE_LOW, 'readOnly' => true } )}, @@ -489,6 +500,7 @@ fields("node") -> mapping => "vm_args.-env ERL_CRASH_DUMP", desc => ?DESC(node_crash_dump_file), default => crash_dump_file_default(), + importance => ?IMPORTANCE_LOW, 'readOnly' => true } )}, @@ -499,6 +511,7 @@ fields("node") -> mapping => "vm_args.-env ERL_CRASH_DUMP_SECONDS", default => <<"30s">>, desc => ?DESC(node_crash_dump_seconds), + importance => ?IMPORTANCE_LOW, 'readOnly' => true } )}, @@ -509,6 +522,7 @@ fields("node") -> mapping => "vm_args.-env ERL_CRASH_DUMP_BYTES", default => <<"100MB">>, desc => ?DESC(node_crash_dump_bytes), + importance => ?IMPORTANCE_LOW, 'readOnly' => true } )}, @@ -519,6 +533,7 @@ fields("node") -> mapping => "vm_args.-kernel net_ticktime", default => <<"2m">>, 'readOnly' => true, + importance => ?IMPORTANCE_LOW, desc => ?DESC(node_dist_net_ticktime) } )}, @@ -529,6 +544,7 @@ fields("node") -> mapping => "emqx_machine.backtrace_depth", default => 23, 'readOnly' => true, + importance => ?IMPORTANCE_LOW, desc => ?DESC(node_backtrace_depth) } )}, @@ -539,6 +555,7 @@ fields("node") -> mapping => "emqx_machine.applications", default => [], 'readOnly' => true, + importance => ?IMPORTANCE_LOW, desc => ?DESC(node_applications) } )}, @@ -548,13 +565,17 @@ fields("node") -> #{ desc => ?DESC(node_etc_dir), 'readOnly' => true, + importance => ?IMPORTANCE_LOW, deprecated => {since, "5.0.8"} } )}, {"cluster_call", sc( ?R_REF("cluster_call"), - #{'readOnly' => true} + #{ + 'readOnly' => true, + importance => ?IMPORTANCE_LOW + } )}, {"db_backend", sc( @@ -563,6 +584,7 @@ fields("node") -> mapping => "mria.db_backend", default => rlog, 'readOnly' => true, + importance => ?IMPORTANCE_HIDDEN, desc => ?DESC(db_backend) } )}, @@ -573,6 +595,7 @@ fields("node") -> mapping => "mria.node_role", default => core, 'readOnly' => true, + importance => ?IMPORTANCE_HIGH, desc => ?DESC(db_role) } )}, @@ -583,6 +606,7 @@ fields("node") -> mapping => "mria.rlog_rpc_module", default => gen_rpc, 'readOnly' => true, + importance => ?IMPORTANCE_HIDDEN, desc => ?DESC(db_rpc_module) } )}, @@ -593,6 +617,7 @@ fields("node") -> mapping => "mria.tlog_push_mode", default => async, 'readOnly' => true, + importance => ?IMPORTANCE_LOW, desc => ?DESC(db_tlog_push_mode) } )}, @@ -601,7 +626,7 @@ fields("node") -> hoconsc:enum([gen_rpc, distr]), #{ mapping => "mria.shard_transport", - hidden => true, + importance => ?IMPORTANCE_HIDDEN, default => gen_rpc, desc => ?DESC(db_default_shard_transport) } @@ -611,7 +636,7 @@ fields("node") -> map(shard, hoconsc:enum([gen_rpc, distr])), #{ desc => ?DESC(db_shard_transports), - hidden => true, + importance => ?IMPORTANCE_HIDDEN, mapping => "emqx_machine.custom_shard_transports", default => #{} } diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 18196425f..e2872c0d7 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -891,9 +891,9 @@ parse_object_loop([{Name, Hocon} | Rest], Module, Options, Props, Required, Refs ) end. +%% return true if the field has 'importance' set to 'hidden' is_hidden(Hocon) -> - hocon_schema:is_hidden(Hocon). - + hocon_schema:is_hidden(Hocon, #{include_importance_up_from => ?IMPORTANCE_LOW}). is_required(Hocon) -> hocon_schema:field_schema(Hocon, required) =:= true. diff --git a/apps/emqx_prometheus/src/emqx_prometheus.app.src b/apps/emqx_prometheus/src/emqx_prometheus.app.src index 07ae38d75..1e7e59f7a 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.app.src +++ b/apps/emqx_prometheus/src/emqx_prometheus.app.src @@ -2,7 +2,7 @@ {application, emqx_prometheus, [ {description, "Prometheus for EMQX"}, % strict semver, bump manually! - {vsn, "5.0.7"}, + {vsn, "5.0.8"}, {modules, []}, {registered, [emqx_prometheus_sup]}, {applications, [kernel, stdlib, prometheus, emqx, emqx_management]}, diff --git a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl index 6ced0bf42..f8005f06b 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl @@ -90,7 +90,7 @@ fields("prometheus") -> #{ default => enabled, required => true, - hidden => true, + importance => ?IMPORTANCE_HIDDEN, desc => ?DESC(vm_dist_collector) } )}, @@ -100,7 +100,7 @@ fields("prometheus") -> #{ default => enabled, required => true, - hidden => true, + importance => ?IMPORTANCE_HIDDEN, desc => ?DESC(mnesia_collector) } )}, @@ -110,7 +110,7 @@ fields("prometheus") -> #{ default => enabled, required => true, - hidden => true, + importance => ?IMPORTANCE_HIDDEN, desc => ?DESC(vm_statistics_collector) } )}, @@ -120,7 +120,7 @@ fields("prometheus") -> #{ default => enabled, required => true, - hidden => true, + importance => ?IMPORTANCE_HIDDEN, desc => ?DESC(vm_system_info_collector) } )}, @@ -130,7 +130,7 @@ fields("prometheus") -> #{ default => enabled, required => true, - hidden => true, + importance => ?IMPORTANCE_HIDDEN, desc => ?DESC(vm_memory_collector) } )}, @@ -140,7 +140,7 @@ fields("prometheus") -> #{ default => enabled, required => true, - hidden => true, + importance => ?IMPORTANCE_HIDDEN, desc => ?DESC(vm_msacc_collector) } )} diff --git a/apps/emqx_resource/src/schema/emqx_resource_schema.erl b/apps/emqx_resource/src/schema/emqx_resource_schema.erl index b9ed176fe..04a0ffd72 100644 --- a/apps/emqx_resource/src/schema/emqx_resource_schema.erl +++ b/apps/emqx_resource/src/schema/emqx_resource_schema.erl @@ -83,7 +83,7 @@ worker_pool_size(required) -> false; worker_pool_size(_) -> undefined. resume_interval(type) -> emqx_schema:duration_ms(); -resume_interval(hidden) -> true; +resume_interval(importance) -> hidden; resume_interval(desc) -> ?DESC("resume_interval"); resume_interval(required) -> false; resume_interval(_) -> undefined. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl index 3db8dd5f1..d9401b7fd 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl @@ -364,7 +364,7 @@ fields(consumer_kafka_opts) -> })}, {max_rejoin_attempts, mk(non_neg_integer(), #{ - hidden => true, + importance => ?IMPORTANCE_HIDDEN, default => 5, desc => ?DESC(consumer_max_rejoin_attempts) })}, From 3a88e7739da941cf2d5bf2271e5eee5580b83ff9 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 23 Mar 2023 21:22:38 +0100 Subject: [PATCH 110/196] test: add swagger check to smoke tests --- scripts/test/emqx-smoke-test.sh | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/scripts/test/emqx-smoke-test.sh b/scripts/test/emqx-smoke-test.sh index 361137bc0..82de296ba 100755 --- a/scripts/test/emqx-smoke-test.sh +++ b/scripts/test/emqx-smoke-test.sh @@ -8,6 +8,7 @@ IP=$1 PORT=$2 URL="http://$IP:$PORT/status" +## Check if EMQX is responding ATTEMPTS=10 while ! curl "$URL" >/dev/null 2>&1; do if [ $ATTEMPTS -eq 0 ]; then @@ -17,3 +18,23 @@ while ! curl "$URL" >/dev/null 2>&1; do sleep 5 ATTEMPTS=$((ATTEMPTS-1)) done + +## Check if the API docs are available +API_DOCS_URL="http://$IP:$PORT/api-docs/index.html" +API_DOCS_STATUS="$(curl -s -o /dev/null -w "%{http_code}" "$API_DOCS_URL")" +if [ "$API_DOCS_STATUS" != "200" ]; then + echo "emqx is not responding on $API_DOCS_URL" + exit 1 +fi + +## Check if the swagger.json contains hidden fields +## fail if it does +SWAGGER_JSON_URL="http://$IP:$PORT/api-docs/swagger.json" +## assert swagger.json is valid json +JSON="$(curl -s "$SWAGGER_JSON_URL")" +echo "$JSON" | jq . >/dev/null +## assert swagger.json does not contain trie_compaction (which is a hidden field) +if echo "$JSON" | grep -q trie_compaction; then + echo "swagger.json contains hidden fields" + exit 1 +fi From d35c5e0516c74c553fcb0228b1f7cfce3e96181e Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 23 Mar 2023 21:35:10 +0100 Subject: [PATCH 111/196] docs: add changelogs --- changes/ce/fix-10211.en.md | 3 +++ changes/ce/fix-10211.zh.md | 3 +++ 2 files changed, 6 insertions(+) create mode 100644 changes/ce/fix-10211.en.md create mode 100644 changes/ce/fix-10211.zh.md diff --git a/changes/ce/fix-10211.en.md b/changes/ce/fix-10211.en.md new file mode 100644 index 000000000..9474f2027 --- /dev/null +++ b/changes/ce/fix-10211.en.md @@ -0,0 +1,3 @@ +Hide `broker.broker_perf` config and API documents. +The two configs `route_lock_type` and `trie_compaction` are rarely used and requires a full cluster restart to take effect. They are not suitable for being exposed to users. +Detailed changes can be found here: https://gist.github.com/zmstone/01ad5754b9beaeaf3f5b86d14d49a0b7/revisions diff --git a/changes/ce/fix-10211.zh.md b/changes/ce/fix-10211.zh.md new file mode 100644 index 000000000..e8db64f86 --- /dev/null +++ b/changes/ce/fix-10211.zh.md @@ -0,0 +1,3 @@ +隐藏 `broker.broker_perf` 配置项，不再在 配置和 API 的文档中展示。 +`route_lock_type` 和 `trie_compaction` 这两个配置项很少使用，且需要全集群重启才能生效，不适合暴露给用户。 +详细对比： https://gist.github.com/zmstone/01ad5754b9beaeaf3f5b86d14d49a0b7/revisions From 3f9ed0830810f66be86f5bd1440d6afd5bceecec Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 28 Mar 2023 14:18:50 +0200 Subject: [PATCH 112/196] fix(emqx_schema): ocsp field use 'importance' to replace 'hidden' --- apps/emqx/src/emqx_schema.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 94e73cecb..3177294ee 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -2299,7 +2299,7 @@ server_ssl_opts_schema(Defaults, IsRanchListener) -> #{ required => false, %% TODO: remove after e5.0.2 - hidden => true, + importance => ?IMPORTANCE_HIDDEN, validator => fun ocsp_inner_validator/1 } )}, From f5e9d3eab282b98b5605fd295267d87c6ea0add5 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 28 Mar 2023 14:28:49 +0200 Subject: [PATCH 113/196] test: use valid config for jq function timeout --- .../test/emqx_rule_funcs_SUITE.erl | 20 +++++-------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl index 5d78f5e4a..209332fe7 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl @@ -28,7 +28,7 @@ init_per_suite(Config) -> application:load(emqx_conf), - ConfigConf = <<"rule_engine {jq_function_default_timeout {}}">>, + ConfigConf = <<"rule_engine {jq_function_default_timeout=10s}">>, ok = emqx_common_test_helpers:load_config(emqx_rule_engine_schema, ConfigConf), Config. @@ -691,20 +691,10 @@ t_jq(_) -> ConfigRootKey, jq_function_default_timeout ]), - case DefaultTimeOut =< 15000 of - true -> - got_timeout = - try - apply_func(jq, [TOProgram, <<"-2">>]) - catch - throw:{jq_exception, {timeout, _}} -> - %% Got timeout as expected - got_timeout - end; - false -> - %% Skip test as we don't want it to take to long time to run - ok - end. + ?assertThrow( + {jq_exception, {timeout, _}}, + apply_func(jq, [TOProgram, <<"-2">>]) + ). ascii_string() -> list(range(0, 127)). From 0aab24d9173cf77b417c6ef99bdd69099278349e Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 28 Mar 2023 16:03:55 +0200 Subject: [PATCH 114/196] ci: do not check hidden fields if running older version --- .github/workflows/build_slim_packages.yaml | 3 +++ scripts/test/emqx-smoke-test.sh | 11 +++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 7f7f35e16..d6e4fc961 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -194,12 +194,15 @@ jobs: run: | CID=$(docker run -d --rm -P $EMQX_IMAGE_TAG) HTTP_PORT=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "18083/tcp") 0).HostPort}}' $CID) + export EMQX_SMOKE_TEST_CHECK_HIDDEN_FIELDS='yes' ./scripts/test/emqx-smoke-test.sh localhost $HTTP_PORT docker stop $CID - name: test two nodes cluster with proto_dist=inet_tls in docker run: | ./scripts/test/start-two-nodes-in-docker.sh -P $EMQX_IMAGE_TAG $EMQX_IMAGE_OLD_VERSION_TAG HTTP_PORT=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "18083/tcp") 0).HostPort}}' haproxy) + # versions before 5.0.22 have hidden fields included in the API spec + export EMQX_SMOKE_TEST_CHECK_HIDDEN_FIELDS='no' ./scripts/test/emqx-smoke-test.sh localhost $HTTP_PORT # cleanup ./scripts/test/start-two-nodes-in-docker.sh -c diff --git a/scripts/test/emqx-smoke-test.sh b/scripts/test/emqx-smoke-test.sh index 82de296ba..ce8116b39 100755 --- a/scripts/test/emqx-smoke-test.sh +++ b/scripts/test/emqx-smoke-test.sh @@ -33,8 +33,11 @@ SWAGGER_JSON_URL="http://$IP:$PORT/api-docs/swagger.json" ## assert swagger.json is valid json JSON="$(curl -s "$SWAGGER_JSON_URL")" echo "$JSON" | jq . >/dev/null -## assert swagger.json does not contain trie_compaction (which is a hidden field) -if echo "$JSON" | grep -q trie_compaction; then - echo "swagger.json contains hidden fields" - exit 1 + +if [ "${EMQX_SMOKE_TEST_CHECK_HIDDEN_FIELDS:-yes}" = 'yes' ]; then + ## assert swagger.json does not contain trie_compaction (which is a hidden field) + if echo "$JSON" | grep -q trie_compaction; then + echo "swagger.json contains hidden fields" + exit 1 + fi fi From ecc2cd1a949e7ed84484d7f2e1f1ddddfaa5c29f Mon Sep 17 00:00:00 2001 From: William Yang Date: Sat, 25 Mar 2023 10:34:00 +0100 Subject: [PATCH 115/196] test: password cert for SSL listener --- apps/emqx/test/emqx_listeners_SUITE.erl | 33 +++++++++++++++++++++---- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/apps/emqx/test/emqx_listeners_SUITE.erl b/apps/emqx/test/emqx_listeners_SUITE.erl index ac4bf6c76..107f3d4e7 100644 --- a/apps/emqx/test/emqx_listeners_SUITE.erl +++ b/apps/emqx/test/emqx_listeners_SUITE.erl @@ -35,6 +35,7 @@ init_per_suite(Config) -> application:ensure_all_started(esockd), application:ensure_all_started(quicer), application:ensure_all_started(cowboy), + generate_tls_certs(Config), lists:foreach(fun set_app_env/1, NewConfig), Config. @@ -183,26 +184,48 @@ t_wss_conn(_) -> ok = ssl:close(Socket). t_quic_conn(Config) -> + Port = 24568, DataDir = ?config(data_dir, Config), - generate_quic_tls_certs(Config), SSLOpts = #{ password => ?SERVER_KEY_PASSWORD, certfile => filename:join(DataDir, "server-password.pem"), cacertfile => filename:join(DataDir, "ca.pem"), keyfile => filename:join(DataDir, "server-password.key") }, - emqx_common_test_helpers:ensure_quic_listener(?FUNCTION_NAME, 24568, #{ssl_options => SSLOpts}), + emqx_common_test_helpers:ensure_quic_listener(?FUNCTION_NAME, Port, #{ssl_options => SSLOpts}), ct:pal("~p", [emqx_listeners:list()]), {ok, Conn} = quicer:connect( {127, 0, 0, 1}, - 24568, + Port, [ {verify, verify_none}, {alpn, ["mqtt"]} ], 1000 ), - ok = quicer:close_connection(Conn). + ok = quicer:close_connection(Conn), + emqx_listeners:stop_listener(quic, ?FUNCTION_NAME, #{bind => Port}). + +t_ssl_password_cert(Config) -> + Port = 24568, + DataDir = ?config(data_dir, Config), + SSLOptsPWD = #{ + password => ?SERVER_KEY_PASSWORD, + certfile => filename:join(DataDir, "server-password.pem"), + cacertfile => filename:join(DataDir, "ca.pem"), + keyfile => filename:join(DataDir, "server-password.key") + }, + LConf = #{ + enabled => true, + bind => {{127, 0, 0, 1}, Port}, + mountpoint => <<>>, + zone => default, + ssl_options => SSLOptsPWD + }, + ok = emqx_listeners:start_listener(ssl, ?FUNCTION_NAME, LConf), + {ok, SSLSocket} = ssl:connect("127.0.0.1", Port, [{verify, verify_none}]), + ssl:close(SSLSocket), + emqx_listeners:stop_listener(ssl, ?FUNCTION_NAME, LConf). t_format_bind(_) -> ?assertEqual( @@ -289,7 +312,7 @@ remove_default_limiter(Listeners) -> Listeners ). -generate_quic_tls_certs(Config) -> +generate_tls_certs(Config) -> DataDir = ?config(data_dir, Config), emqx_common_test_helpers:gen_ca(DataDir, "ca"), emqx_common_test_helpers:gen_host_cert("server-password", "ca", DataDir, #{ From d9fc8fbc87575152415406e74c024fa9fae3f96f Mon Sep 17 00:00:00 2001 From: Paulo Zulato Date: Tue, 28 Mar 2023 16:42:05 -0300 Subject: [PATCH 116/196] feat: add new command for Elixir expression evaluation Currently, Elixir expressions are evaluated on an EMQX node using the 'eval' command, which works for both Erlang and Elixir expressions. This commit adds the new command 'eval-ex' exclusively for evaluating Elixir expressions on Elixir nodes, similar to 'eval-erl' for Erlang. --- bin/emqx | 25 ++++++++++++++++++++++++- changes/ce/feat-10263.en.md | 1 + 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 changes/ce/feat-10263.en.md diff --git a/bin/emqx b/bin/emqx index 741aa3718..c0601ec85 100755 --- a/bin/emqx +++ b/bin/emqx @@ -164,6 +164,9 @@ usage() { eval-erl) echo "Evaluate an Erlang expression in the EMQX node, even on Elixir node" ;; + eval-ex) + echo "Evaluate an Elixir expression in the EMQX node. Only applies to Elixir node" + ;; versions) echo "List installed EMQX release versions and their status" ;; @@ -228,7 +231,7 @@ usage() { echo " Install Info: ertspath | root_dir" echo " Runtime Status: pid | ping" echo " Validate Config: check_config" - echo " Advanced: console_clean | escript | rpc | rpcterms | eval | eval-erl" + echo " Advanced: console_clean | escript | rpc | rpcterms | eval | eval-erl | eval-ex" echo '' echo "Execute '$REL_NAME COMMAND help' for more information" ;; @@ -1280,7 +1283,27 @@ case "${COMMAND}" in shift relx_nodetool "eval" "$@" ;; + eval-ex) + assert_node_alive + shift + if [ "$IS_ELIXIR" = "yes" ] + then + "$REL_DIR/elixir" \ + --hidden \ + --name "rand-$(relx_gen_id)-$NAME" \ + --cookie "$COOKIE" \ + --boot "$REL_DIR/start_clean" \ + --boot-var RELEASE_LIB "$ERTS_LIB_DIR" \ + --vm-args "$REL_DIR/remote.vm.args" \ + --erl "-start_epmd false -epmd_module ekka_epmd" \ + --rpc-eval "$NAME" "$@" + else + echo "EMQX node is not an Elixir node" + usage "$COMMAND" + exit 1 + fi + ;; check_config) check_config ;; diff --git a/changes/ce/feat-10263.en.md b/changes/ce/feat-10263.en.md new file mode 100644 index 000000000..e069fc17f --- /dev/null +++ b/changes/ce/feat-10263.en.md @@ -0,0 +1 @@ +Add command 'eval-ex' for Elixir expression evaluation. From 27aa3f49aa64a3691218d67529c9e0f8e938657e Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 29 Mar 2023 15:02:01 +0800 Subject: [PATCH 117/196] docs: improve some mqtt options description --- apps/emqx/i18n/emqx_schema_i18n.conf | 106 ++++++++++++++------------- 1 file changed, 54 insertions(+), 52 deletions(-) diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/apps/emqx/i18n/emqx_schema_i18n.conf index 6f926ec39..c14ae3f61 100644 --- a/apps/emqx/i18n/emqx_schema_i18n.conf +++ b/apps/emqx/i18n/emqx_schema_i18n.conf @@ -666,15 +666,15 @@ mqtt 下所有的配置作为全局的默认值存在，它可以被 zone< mqtt_idle_timeout { desc { - en: """After the TCP connection is established, if the MQTT CONNECT packet from the client is -not received within the time specified by idle_timeout, the connection will be disconnected. -After the CONNECT packet has been accepted by EMQX, if the connection idles for this long time, -then the Erlang process is put to hibernation to save OS resources. Note: long idle_timeout -interval may impose risk at the system if large number of malicious clients only establish connections -but do not send any data.""" - zh: """TCP 连接建立后，如果在 idle_timeout 指定的时间内未收到客户端的 MQTT CONNECT 报文，则连接将被断开。 -如果连接在 CONNECT 报文被 EMQX 接受之后空闲超过该时长，那么服务这个连接的 Erlang 进程会进入休眠以节省系统资源。 -注意，该配置值如果设置过大的情况下，如果大量恶意客户端只连接，但不发任何数据，可能会导致系统资源被恶意消耗。""" + en: """Configure the duration of time that a connection can remain idle (i.e., without any data transfer) before being: + - Automatically disconnected if no CONNECT package is received from the client yet. + - Put into hibernation mode to save resources if some CONNECT packages are already received. +Note: Please set the parameter with caution as long idle time will lead to resouce waste.""" + zh: """设置连接被断开或进入休眠状态前的等待时间，空闲超时后， + - 如暂未收到客户端的 CONNECT 报文，连接将断开； + - 如已收到客户端的 CONNECT 报文，连接将进入休眠模式以节省系统资源。 + +注意：请合理设置该参数值，如等待时间设置过长，可能造成系统资源的浪费。""" } label: { en: """Idle Timeout""" @@ -783,8 +783,8 @@ but do not send any data.""" mqtt_ignore_loop_deliver { desc { - en: """Ignore loop delivery of messages for MQTT v3.1.1/v3.1.0, similar to No Local subscription option in MQTT 5.0.""" - zh: """是否为 MQTT v3.1.1/v3.1.0 客户端忽略投递自己发布的消息，类似于 MQTT 5.0 中的 No Local 订阅选项。""" + en: """Whether the messages sent by the MQTT v3.1.1/v3.1.0 client will be forwarded to the client itself, similar to No Local in MQTT 5.0.""" + zh: """设置由 MQTT v3.1.1/v3.1.0 客户端发布的消息是否将转发给其本身；类似 MQTT 5.0 协议中的 No Local 选项。""" } label: { en: """Ignore Loop Deliver""" @@ -794,10 +794,10 @@ but do not send any data.""" mqtt_strict_mode { desc { - en: """Parse MQTT messages in strict mode. -When set to true, invalid utf8 strings in for example client ID, topic name, etc. will cause the client to be disconnected""" + en: """Whether to parse MQTT messages in strict mode. +In strict mode, invalid utf8 strings in for example client ID, topic name, etc. will cause the client to be disconnected.""" zh: """是否以严格模式解析 MQTT 消息。 -当设置为 true 时，例如客户端 ID、主题名称等中的无效 utf8 字符串将导致客户端断开连接。""" +严格模式下，如客户端 ID、主题名称等中包含无效 utf8 字符串，连接将被断开。""" } label: { en: """Strict Mode""" @@ -807,8 +807,10 @@ When set to true, invalid utf8 strings in for example client ID, topic name, etc mqtt_response_information { desc { - en: """Specify the response information returned to the client. This feature is disabled if is set to \"\". Applies only to clients using MQTT 5.0.""" - zh: """指定返回给客户端的响应信息。如果设置为 \"\"，则禁用此功能。仅适用于使用 MQTT 5.0 协议的客户端。""" + en: """UTF-8 string, for creating the response topic, for example, if set to reqrsp/, the publisher/subscriber will communite under the topic reqrsp/. +To disable this feature, input \"\" in the text box below.""" + zh: """UTF-8 字符串，用于指定返回给客户端的响应主题，如 reqrsp/，此时请求和应答客户端都需要使用 reqrsp/ 前缀的主题来完成通讯。 +如希望禁用此功能，请在下方的文字框中输入\"\"；仅适用于 MQTT 5.0 客户端。""" } label: { en: """Response Information""" @@ -818,23 +820,23 @@ When set to true, invalid utf8 strings in for example client ID, topic name, etc mqtt_server_keepalive { desc { - en: """The keep alive that EMQX requires the client to use. If configured as disabled, it means that the keep alive specified by the client will be used. Requires Server Keep Alive in MQTT 5.0, so it is only applicable to clients using MQTT 5.0 protocol.""" - zh: """EMQX 要求客户端使用的保活时间，配置为 disabled 表示将使用客户端指定的保活时间。需要用到 MQTT 5.0 中的 Server Keep Alive，因此仅适用于使用 MQTT 5.0 协议的客户端。""" + en: """The keep alive duration required by EMQX. To use the setting from the client side, choose disabled from the drop-down list. Only applicable to MQTT 5.0 clients.""" + zh: """EMQX 要求的保活时间，如设为 disabled，则将使用客户端指定的保持连接时间；仅适用于 MQTT 5.0 客户端。""" } label: { en: """Server Keep Alive""" - zh: """服务端保持连接""" + zh: """服务端保活时间""" } } mqtt_keepalive_backoff { desc { - en: """The backoff multiplier used by the broker to determine the client keep alive timeout. If EMQX doesn't receive any packet in Keep Alive * Backoff * 2 seconds, EMQX will close the current connection.""" - zh: """Broker 判定客户端保活超时使用的退避乘数。如果 EMQX 在 Keep Alive * Backoff * 2 秒内未收到任何报文，EMQX 将关闭当前连接。""" + en: """The coeffient EMQX uses to confirm whether the keep alive duration of the client expires. Formula: Keep Alive * Backoff * 2""" + zh: """EMQX 判定客户端保活超时使用的阈值系数。计算公式为：Keep Alive * Backoff * 2""" } label: { en: """Keep Alive Backoff""" - zh: """保持连接退避乘数""" + zh: """保活超时阈值系数""" } } @@ -978,14 +980,14 @@ To configure \"topic/1\" > \"topic/2\": mqtt_use_username_as_clientid { desc { - en: """Whether to user Client ID as Username. -This setting takes effect later than Use Peer Certificate as Username (peer_cert_as_username) and Use peer certificate as Client ID (peer_cert_as_clientid).""" + en: """Whether to use Username as Client ID. +This setting takes effect later than Use Peer Certificate as Username and Use peer certificate as Client ID.""" zh: """是否使用用户名作为客户端 ID。 -此设置的作用时间晚于 使用对端证书作为用户名（peer_cert_as_username） 和 使用对端证书作为客户端 ID（peer_cert_as_clientid）。""" +此设置的作用时间晚于 对端证书作为用户名 和 对端证书作为客户端 ID。""" } label: { en: """Use Username as Client ID""" - zh: """使用用户名作为客户端 ID""" + zh: """用户名作为客户端 ID""" } } @@ -993,22 +995,22 @@ This setting takes effect later than Use Peer Certificate as Usernamecn: Take the CN field of the certificate as Username -- dn: Take the DN field of the certificate as Username -- crt: Take the content of the DER or PEM certificate as Username -- pem: Convert DER certificate content to PEM format as Username -- md5: Take the MD5 value of the content of the DER or PEM certificate as Username""" - zh: """使用对端证书中的 CN、DN 字段或整个证书内容来作为用户名。仅适用于 TLS 连接。 -目前支持配置为以下内容： -- cn: 取证书的 CN 字段作为 Username -- dn: 取证书的 DN 字段作为 Username -- crt: 取 DER 或 PEM 证书的内容作为 Username -- pem: 将 DER 证书内容转换为 PEM 格式后作为 Username -- md5: 取 DER 或 PEM 证书的内容的 MD5 值作为 Username""" +- cn: CN field of the certificate +- dn: DN field of the certificate +- crt: Content of the DER or PEM certificate +- pem: Convert DER certificate content to PEM format and use as Username +- md5: MD5 value of the DER or PEM certificate""" + zh: """使用对端证书中的 CN、DN 字段或整个证书内容来作为用户名；仅适用于 TLS 连接。 +目前支持： +- cn: 取证书的 CN 字段 +- dn: 取证书的 DN 字段 +- crt: 取 DER 或 PEM 的证书内容 +- pem: 将 DER 证书转换为 PEM 格式作为用户名 +- md5: 取 DER 或 PEM 证书内容的 MD5 值""" } label: { en: """Use Peer Certificate as Username""" - zh: """使用对端证书作为用户名""" + zh: """对端证书作为用户名""" } } @@ -1016,22 +1018,22 @@ Supported configurations are the following: desc { en: """Use the CN, DN field in the peer certificate or the entire certificate content as Client ID. Only works for the TLS connection. Supported configurations are the following: -- cn: Take the CN field of the certificate as Client ID -- dn: Take the DN field of the certificate as Client ID -- crt: Take the content of the DER or PEM certificate as Client ID -- pem: Convert DER certificate content to PEM format as Client ID -- md5: Take the MD5 value of the content of the DER or PEM certificate as Client ID""" - zh: """使用对端证书中的 CN、DN 字段或整个证书内容来作为客户端 ID。仅适用于 TLS 连接。 -目前支持配置为以下内容： -- cn: 取证书的 CN 字段作为 Client ID -- dn: 取证书的 DN 字段作为 Client ID -- crt: 取 DER 或 PEM 证书的内容作为 Client ID -- pem: 将 DER 证书内容转换为 PEM 格式后作为 Client ID -- md5: 取 DER 或 PEM 证书的内容的 MD5 值作为 Client ID""" +- cn: CN field of the certificate +- dn: DN field of the certificate +- crt: DER or PEM certificate +- pem: Convert DER certificate content to PEM format and use as Client ID +- md5: MD5 value of the DER or PEM certificate""" + zh: """使用对端证书中的 CN、DN 字段或整个证书内容来作为客户端 ID。仅适用于 TLS 连接； +目前支持： +- cn: 取证书的 CN 字段 +- dn: 取证书的 DN 字段 +- crt: 取 DER 或 PEM 证书的内容 +- pem: 将 DER 证书内容转换为 PEM 格式作为客户端 ID +- md5: 取 DER 或 PEM 证书内容的 MD5 值""" } label: { en: """Use Peer Certificate as Client ID""" - zh: """使用对端证书作为客户端 ID""" + zh: """对端证书作为客户端 ID""" } } From c693851740201ecb4f31b9db4c74c45e420fd4c1 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Wed, 29 Mar 2023 11:17:04 +0200 Subject: [PATCH 118/196] fix: on_get_status clickhouse just reporting connecting The on_get_status callback for clickhouse just returned `connecting` without error information when the status check was unsuccessful. This is fixed by letting the callback return error information similarly to how the HTTP connector does it. Fixes: https://emqx.atlassian.net/browse/EMQX-9374 https://emqx.atlassian.net/browse/EMQX-9278 --- .../i18n/emqx_ee_connector_clickhouse.conf | 11 +++ lib-ee/emqx_ee_connector/rebar.config | 2 +- .../src/emqx_ee_connector_clickhouse.erl | 71 +++++++++++++++---- .../test/ee_connector_clickhouse_SUITE.erl | 3 +- 4 files changed, 72 insertions(+), 15 deletions(-) diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_clickhouse.conf b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_clickhouse.conf index 1e07c29b4..533d100bf 100644 --- a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_clickhouse.conf +++ b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_clickhouse.conf @@ -12,4 +12,15 @@ emqx_ee_connector_clickhouse { } } + connect_timeout { + desc { + en: "The timeout when connecting to the Clickhouse server." + zh: "连接HTTP服务器的超时时间。" + } + label: { + en: "Clickhouse Timeout" + zh: "连接超时" + } + } + } diff --git a/lib-ee/emqx_ee_connector/rebar.config b/lib-ee/emqx_ee_connector/rebar.config index 96b3df6a3..d49ce59c0 100644 --- a/lib-ee/emqx_ee_connector/rebar.config +++ b/lib-ee/emqx_ee_connector/rebar.config @@ -3,7 +3,7 @@ {hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.2.5"}}}, {influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.9"}}}, {tdengine, {git, "https://github.com/emqx/tdengine-client-erl", {tag, "0.1.5"}}}, - {clickhouse, {git, "https://github.com/emqx/clickhouse-client-erl", {tag, "0.2"}}}, + {clickhouse, {git, "https://github.com/emqx/clickhouse-client-erl", {tag, "0.3"}}}, {erlcloud, {git, "https://github.com/emqx/erlcloud.git", {tag,"3.5.16-emqx-1"}}}, {rocketmq, {git, "https://github.com/emqx/rocketmq-client-erl.git", {tag, "v0.5.1"}}}, {emqx, {path, "../../apps/emqx"}} diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_clickhouse.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_clickhouse.erl index b1ad6c787..8f2fdc042 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_clickhouse.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_clickhouse.erl @@ -53,7 +53,6 @@ %% Internal exports used to execute code with ecpool worker -export([ - check_database_status/1, execute_sql_in_clickhouse_server_using_connection/2 ]). @@ -102,6 +101,14 @@ fields(config) -> end, desc => ?DESC("base_url") } + )}, + {connect_timeout, + hoconsc:mk( + emqx_schema:duration_ms(), + #{ + default => <<"15s">>, + desc => ?DESC("connect_timeout") + } )} ] ++ emqx_connector_schema_lib:relational_db_fields(). @@ -137,7 +144,8 @@ on_start( #{ url := URL, database := DB, - pool_size := PoolSize + pool_size := PoolSize, + connect_timeout := ConnectTimeout } = Config ) -> ?SLOG(info, #{ @@ -155,7 +163,10 @@ on_start( {pool_size, PoolSize}, {pool, PoolName} ], - InitState = #{poolname => PoolName}, + InitState = #{ + poolname => PoolName, + connect_timeout => ConnectTimeout + }, try Templates = prepare_sql_templates(Config), State = maps:merge(InitState, #{templates => Templates}), @@ -282,18 +293,52 @@ on_stop(ResourceID, #{poolname := PoolName}) -> %% on_get_status emqx_resouce callback and related functions %% ------------------------------------------------------------------- -on_get_status(_ResourceID, #{poolname := Pool} = _State) -> - case - emqx_plugin_libs_pool:health_check_ecpool_workers(Pool, fun ?MODULE:check_database_status/1) - of - true -> - connected; - false -> - connecting +on_get_status( + _InstId, + #{ + poolname := PoolName, + connect_timeout := Timeout + } = State +) -> + case do_get_status(PoolName, Timeout) of + ok -> + {connected, State}; + {error, Reason} -> + {disconnected, State, Reason} end. -check_database_status(Connection) -> - clickhouse:status(Connection). +do_get_status(PoolName, Timeout) -> + Workers = [Worker || {_WorkerName, Worker} <- ecpool:workers(PoolName)], + DoPerWorker = + fun(Worker) -> + case ecpool_worker:exec(Worker, fun clickhouse:detailed_status/1, Timeout) of + ok -> + ok; + {error, Reason} = Error -> + ?SLOG(error, #{ + msg => "clickhouse_connector_get_status_failed", + reason => Reason, + worker => Worker + }), + Error + end + end, + try emqx_misc:pmap(DoPerWorker, Workers, Timeout) of + Results -> + case [E || {error, _} = E <- Results] of + [] -> + ok; + Errors -> + hd(Errors) + end + catch + exit:timeout -> + ?SLOG(error, #{ + msg => "clickhouse_connector_pmap_failed", + reason => timeout + }), + {error, timeout} + end. %% ------------------------------------------------------------------- %% on_query emqx_resouce callback and related functions diff --git a/lib-ee/emqx_ee_connector/test/ee_connector_clickhouse_SUITE.erl b/lib-ee/emqx_ee_connector/test/ee_connector_clickhouse_SUITE.erl index eab1aa054..111deba06 100644 --- a/lib-ee/emqx_ee_connector/test/ee_connector_clickhouse_SUITE.erl +++ b/lib-ee/emqx_ee_connector/test/ee_connector_clickhouse_SUITE.erl @@ -190,7 +190,8 @@ clickhouse_config() -> ?CLICKHOUSE_DEFAULT_PORT ] ) - ) + ), + connect_timeout => 10000 }, #{<<"config">> => Config}. From 0a4a976e23fd0f5be95180ce8fc903cafd739a67 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Wed, 29 Mar 2023 14:43:13 +0200 Subject: [PATCH 119/196] docs: add changelog entry for clickhouse fix --- changes/ee/fix-10270.en.md | 1 + changes/ee/fix-10270.zh.md | 1 + 2 files changed, 2 insertions(+) create mode 100644 changes/ee/fix-10270.en.md create mode 100644 changes/ee/fix-10270.zh.md diff --git a/changes/ee/fix-10270.en.md b/changes/ee/fix-10270.en.md new file mode 100644 index 000000000..65eed7b5d --- /dev/null +++ b/changes/ee/fix-10270.en.md @@ -0,0 +1 @@ +Clickhouse has got a fix that makes the error message better when users click the test button in the settings dialog. diff --git a/changes/ee/fix-10270.zh.md b/changes/ee/fix-10270.zh.md new file mode 100644 index 000000000..d47278c16 --- /dev/null +++ b/changes/ee/fix-10270.zh.md @@ -0,0 +1 @@ +Clickhouse 已经修复了一个问题，当用户在设置对话框中点击测试按钮时，错误信息会更清晰。 From 916ce99fd5eb63523dfeeb0871dbe7208b38a0ae Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 29 Mar 2023 14:55:21 -0300 Subject: [PATCH 120/196] fix: handle subscription list api when there are `emqx:subscribe` subscriptions --- .../src/emqx_mgmt_api_subscriptions.erl | 2 +- .../test/emqx_mgmt_api_subscription_SUITE.erl | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl index bf84d03d5..409af4d95 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl @@ -177,7 +177,7 @@ format(WhichNode, {{Topic, _Subscriber}, Options}) -> maps:merge( #{ topic => get_topic(Topic, Options), - clientid => maps:get(subid, Options), + clientid => maps:get(subid, Options, null), node => WhichNode }, maps:with([qos, nl, rap, rh], Options) diff --git a/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl index ccfa30037..b9e9fffd8 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl @@ -19,6 +19,7 @@ -compile(nowarn_export_all). -include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). -define(CLIENTID, <<"api_clientid">>). -define(USERNAME, <<"api_username">>). @@ -142,6 +143,18 @@ t_subscription_fuzzy_search(Config) -> ?assertEqual(#{<<"page">> => 2, <<"limit">> => 3, <<"hasnext">> => false}, MatchMeta2P2), ?assertEqual(1, length(maps:get(<<"data">>, MatchData2P2))). +%% checks that we can list when there are subscriptions made by +%% `emqx:subscribe'. +t_list_with_internal_subscription(_Config) -> + emqx:subscribe(<<"some/topic">>), + QS = [], + Headers = emqx_mgmt_api_test_util:auth_header_(), + ?assertMatch( + #{<<"data">> := [#{<<"clientid">> := null}]}, + request_json(get, QS, Headers) + ), + ok. + request_json(Method, Query, Headers) when is_list(Query) -> Qs = uri_string:compose_query(Query), {ok, MatchRes} = emqx_mgmt_api_test_util:request_api(Method, path(), Qs, Headers), From 758e61005996fdde7c4463507164d5f46d77fd84 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 30 Mar 2023 08:42:08 +0800 Subject: [PATCH 121/196] chore: apply suggestions from code review Co-authored-by: Zaiming (Stone) Shi --- apps/emqx/i18n/emqx_schema_i18n.conf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/apps/emqx/i18n/emqx_schema_i18n.conf index c14ae3f61..1d8c100d0 100644 --- a/apps/emqx/i18n/emqx_schema_i18n.conf +++ b/apps/emqx/i18n/emqx_schema_i18n.conf @@ -783,8 +783,8 @@ Note: Please set the parameter with caution as long idle time will lead to resou mqtt_ignore_loop_deliver { desc { - en: """Whether the messages sent by the MQTT v3.1.1/v3.1.0 client will be forwarded to the client itself, similar to No Local in MQTT 5.0.""" - zh: """设置由 MQTT v3.1.1/v3.1.0 客户端发布的消息是否将转发给其本身；类似 MQTT 5.0 协议中的 No Local 选项。""" + en: """Whether the messages sent by the MQTT v3.1.1/v3.1.0 client will be looped back to the publisher itself, similar to No Local in MQTT 5.0.""" + zh: """设置由 MQTT v3.1.1/v3.1.0 客户端发布的消息是否将转发给其本身；类似 MQTT 5.0 协议中的 No Local 选项。""" } label: { en: """Ignore Loop Deliver""" @@ -807,7 +807,7 @@ In strict mode, invalid utf8 strings in for example client ID, topic name, etc. mqtt_response_information { desc { - en: """UTF-8 string, for creating the response topic, for example, if set to reqrsp/, the publisher/subscriber will communite under the topic reqrsp/. + en: """UTF-8 string, for creating the response topic, for example, if set to reqrsp/, the publisher/subscriber will communicate using the topic prefix reqrsp/. To disable this feature, input \"\" in the text box below.""" zh: """UTF-8 字符串，用于指定返回给客户端的响应主题，如 reqrsp/，此时请求和应答客户端都需要使用 reqrsp/ 前缀的主题来完成通讯。 如希望禁用此功能，请在下方的文字框中输入\"\"；仅适用于 MQTT 5.0 客户端。""" From 3c495f8fd2f57af25d54ba5c93f6d26f92af43cb Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 30 Mar 2023 08:44:28 +0800 Subject: [PATCH 122/196] chore: update apps/emqx/i18n/emqx_schema_i18n.conf --- apps/emqx/i18n/emqx_schema_i18n.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/apps/emqx/i18n/emqx_schema_i18n.conf index 1d8c100d0..5992f2c5d 100644 --- a/apps/emqx/i18n/emqx_schema_i18n.conf +++ b/apps/emqx/i18n/emqx_schema_i18n.conf @@ -808,7 +808,7 @@ In strict mode, invalid utf8 strings in for example client ID, topic name, etc. mqtt_response_information { desc { en: """UTF-8 string, for creating the response topic, for example, if set to reqrsp/, the publisher/subscriber will communicate using the topic prefix reqrsp/. -To disable this feature, input \"\" in the text box below.""" +To disable this feature, input \"\" in the text box below. Only applicable to MQTT 5.0 clients.""" zh: """UTF-8 字符串，用于指定返回给客户端的响应主题，如 reqrsp/，此时请求和应答客户端都需要使用 reqrsp/ 前缀的主题来完成通讯。 如希望禁用此功能，请在下方的文字框中输入\"\"；仅适用于 MQTT 5.0 客户端。""" } From ee5b9d7e0a893f7c401e98f6494a147482bba55c Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 22 Mar 2023 13:56:47 +0800 Subject: [PATCH 123/196] fix(tdengine): remove the redundant table name in SQL template --- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_tdengine.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_tdengine.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_tdengine.erl index b72d79955..f031cbfbf 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_tdengine.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_tdengine.erl @@ -22,7 +22,7 @@ ]). -define(DEFAULT_SQL, << - "insert into mqtt.t_mqtt_msg(ts, msgid, mqtt_topic, qos, payload, arrived) " + "insert into t_mqtt_msg(ts, msgid, mqtt_topic, qos, payload, arrived) " "values (${ts}, ${id}, ${topic}, ${qos}, ${payload}, ${timestamp})" >>). From 1ff8ecf60455f9ba4107f3ad994af98e504ddb95 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 30 Mar 2023 15:10:10 +0800 Subject: [PATCH 124/196] chore: fix typos --- apps/emqx/i18n/emqx_schema_i18n.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/apps/emqx/i18n/emqx_schema_i18n.conf index 5992f2c5d..dd53be19f 100644 --- a/apps/emqx/i18n/emqx_schema_i18n.conf +++ b/apps/emqx/i18n/emqx_schema_i18n.conf @@ -669,7 +669,7 @@ mqtt 下所有的配置作为全局的默认值存在，它可以被 zone< en: """Configure the duration of time that a connection can remain idle (i.e., without any data transfer) before being: - Automatically disconnected if no CONNECT package is received from the client yet. - Put into hibernation mode to save resources if some CONNECT packages are already received. -Note: Please set the parameter with caution as long idle time will lead to resouce waste.""" +Note: Please set the parameter with caution as long idle time will lead to resource waste.""" zh: """设置连接被断开或进入休眠状态前的等待时间，空闲超时后， - 如暂未收到客户端的 CONNECT 报文，连接将断开； - 如已收到客户端的 CONNECT 报文，连接将进入休眠模式以节省系统资源。 @@ -831,7 +831,7 @@ To disable this feature, input \"\" in the text box below. Only app mqtt_keepalive_backoff { desc { - en: """The coeffient EMQX uses to confirm whether the keep alive duration of the client expires. Formula: Keep Alive * Backoff * 2""" + en: """The coefficient EMQX uses to confirm whether the keep alive duration of the client expires. Formula: Keep Alive * Backoff * 2""" zh: """EMQX 判定客户端保活超时使用的阈值系数。计算公式为：Keep Alive * Backoff * 2""" } label: { From f794666c7f6e4c912e375633495ebd1dad877a5b Mon Sep 17 00:00:00 2001 From: firest Date: Thu, 30 Mar 2023 11:08:09 +0800 Subject: [PATCH 125/196] fix(i18n): fix the description error about the default value of `until` --- apps/emqx_management/i18n/emqx_mgmt_api_banned_i18n.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_management/i18n/emqx_mgmt_api_banned_i18n.conf b/apps/emqx_management/i18n/emqx_mgmt_api_banned_i18n.conf index 3045cb293..b45a40ba6 100644 --- a/apps/emqx_management/i18n/emqx_mgmt_api_banned_i18n.conf +++ b/apps/emqx_management/i18n/emqx_mgmt_api_banned_i18n.conf @@ -87,8 +87,8 @@ emqx_mgmt_api_banned { } until { desc { - en: """The end time of the ban, the format is rfc3339, the default is the time when the operation was initiated + 5 minutes.""" - zh: """封禁的结束时间，式为 rfc3339，默认为发起操作的时间 + 5 分钟。""" + en: """The end time of the ban, the format is rfc3339, the default is the time when the operation was initiated + 1 year.""" + zh: """封禁的结束时间，格式为 rfc3339，默认值为发起操作的时间 + 1 年。""" } label { en: """Ban End Time""" From 6b8652108f271867817b10c10908d85247c1bcb4 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Thu, 30 Mar 2023 11:12:58 +0200 Subject: [PATCH 126/196] chore: show ulimit warning on start only --- bin/emqx | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/bin/emqx b/bin/emqx index c0601ec85..d8940ad45 100755 --- a/bin/emqx +++ b/bin/emqx @@ -375,12 +375,6 @@ maybe_use_portable_dynlibs() { fi } -# Warn the user if ulimit -n is less than 1024 -ULIMIT_F=$(ulimit -n) -if [ "$ULIMIT_F" -lt 1024 ]; then - logwarn "ulimit -n is ${ULIMIT_F}; 1024 is the recommended minimum." -fi - SED_REPLACE="sed -i " case $(sed --help 2>&1) in *GNU*) SED_REPLACE="sed -i ";; @@ -912,6 +906,14 @@ maybe_log_to_console() { fi } +# Warn the user if ulimit -n is less than 1024 +maybe_warn_ulimit() { + ULIMIT_F=$(ulimit -n) + if [ "$ULIMIT_F" -lt 1024 ]; then + logwarn "ulimit -n is ${ULIMIT_F}; 1024 is the recommended minimum." + fi +} + ## Possible ways to configure emqx node name: ## 1. configure node.name in emqx.conf ## 2. override with environment variable EMQX_NODE__NAME @@ -1033,6 +1035,7 @@ cd "$RUNNER_ROOT_DIR" case "${COMMAND}" in start) + maybe_warn_ulimit maybe_warn_default_cookie # this flag passes down to console mode @@ -1184,6 +1187,7 @@ case "${COMMAND}" in tr_log_to_env else maybe_log_to_console + maybe_warn_ulimit maybe_warn_default_cookie fi From 28021c466a3756ab938bc37a127323bdc197b746 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 30 Mar 2023 12:31:34 +0200 Subject: [PATCH 127/196] chore(bin/emqx): do not use -r option in rm command when deleting .siz --- bin/emqx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/emqx b/bin/emqx index c0601ec85..0bfe94d4c 100755 --- a/bin/emqx +++ b/bin/emqx @@ -766,7 +766,7 @@ generate_config() { local node_name="$2" ## Delete the *.siz files first or it can't start after ## changing the config 'log.rotation.size' - rm -rf "${RUNNER_LOG_DIR}"/*.siz + rm -f "${RUNNER_LOG_DIR}"/*.siz ## timestamp for each generation local NOW_TIME From 494e4b639a82168f7b6bc4e9bffe77f0c78d50ac Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 30 Mar 2023 12:52:51 +0200 Subject: [PATCH 128/196] chore: exit with non-zero status if config initialization failed --- apps/emqx/src/emqx_app.erl | 4 ++++ apps/emqx_conf/src/emqx_cluster_rpc.erl | 6 ++++++ apps/emqx_conf/src/emqx_conf_app.erl | 2 +- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_app.erl b/apps/emqx/src/emqx_app.erl index 6188d8030..77ece1c60 100644 --- a/apps/emqx/src/emqx_app.erl +++ b/apps/emqx/src/emqx_app.erl @@ -72,9 +72,13 @@ set_init_config_load_done() -> get_init_config_load_done() -> application:get_env(emqx, init_config_load_done, false). +%% @doc Set the transaction id from which this node should start applying after boot. +%% The transaction ID is received from the core node which we just copied the latest +%% config from. set_init_tnx_id(TnxId) -> application:set_env(emqx, cluster_rpc_init_tnx_id, TnxId). +%% @doc Get the transaction id from which this node should start applying after boot. get_init_tnx_id() -> application:get_env(emqx, cluster_rpc_init_tnx_id, -1). diff --git a/apps/emqx_conf/src/emqx_cluster_rpc.erl b/apps/emqx_conf/src/emqx_cluster_rpc.erl index 89f678554..f7c34031c 100644 --- a/apps/emqx_conf/src/emqx_cluster_rpc.erl +++ b/apps/emqx_conf/src/emqx_cluster_rpc.erl @@ -275,8 +275,13 @@ init([Node, RetryMs]) -> _ = mria:wait_for_tables([?CLUSTER_MFA, ?CLUSTER_COMMIT]), {ok, _} = mnesia:subscribe({table, ?CLUSTER_MFA, simple}), State = #{node => Node, retry_interval => RetryMs}, + %% The init transaction ID is set in emqx_conf_app after + %% it has fetched the latest config from one of the core nodes TnxId = emqx_app:get_init_tnx_id(), ok = maybe_init_tnx_id(Node, TnxId), + %% Now continue with the normal catch-up process + %% That is: apply the missing transactions after the config + %% was copied until now. {ok, State, {continue, ?CATCH_UP}}. %% @private @@ -396,6 +401,7 @@ get_cluster_tnx_id() -> Id -> Id end. +%% The entry point of a config change transaction. init_mfa(Node, MFA) -> mnesia:write_lock_table(?CLUSTER_MFA), LatestId = get_cluster_tnx_id(), diff --git a/apps/emqx_conf/src/emqx_conf_app.erl b/apps/emqx_conf/src/emqx_conf_app.erl index 09478e304..0896eb718 100644 --- a/apps/emqx_conf/src/emqx_conf_app.erl +++ b/apps/emqx_conf/src/emqx_conf_app.erl @@ -38,7 +38,7 @@ start(_StartType, _StartArgs) -> reason => E, stacktrace => St }), - init:stop() + init:stop(1) end, ok = emqx_config_logger:refresh_config(), emqx_conf_sup:start_link(). From 3c52faa55e8f69c7032458cb135f442594c9b18f Mon Sep 17 00:00:00 2001 From: William Yang Date: Thu, 30 Mar 2023 13:32:45 +0200 Subject: [PATCH 129/196] chore(docs): update shared_subscription_strategy --- apps/emqx/i18n/emqx_schema_i18n.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/apps/emqx/i18n/emqx_schema_i18n.conf index 6f926ec39..71a008ca3 100644 --- a/apps/emqx/i18n/emqx_schema_i18n.conf +++ b/apps/emqx/i18n/emqx_schema_i18n.conf @@ -1082,8 +1082,8 @@ Supported configurations are the following: - `round_robin_per_group`：在共享组内循环选择下一个成员； - `local`：选择随机的本地成员，否则选择随机的集群范围内成员; - `sticky`：总是使用上次选中的订阅者派发，直到它断开连接； - - `hash_clientid`：使用发送者的 Client ID 进行 Hash 来选择订阅者; - - `hash_topic`：使用源主题进行 Hash 来选择订阅者。""" + - `hash_clientid`：通过对发送者的客户端 ID 进行 Hash 处理来选择订阅者; + - `hash_topic`：通过对源主题进行 Hash 处理来选择订阅者。""" } } From 0e2c700f9ed4fd1bab5b0caeff9469f82754976c Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Thu, 30 Mar 2023 13:54:44 +0200 Subject: [PATCH 130/196] docs(readme): update heading and add slack link to "Get Involved" --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9a3387902..280371a41 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ For more information, please visit [EMQX homepage](https://www.emqx.io/). ## Get Started -#### EMQX Cloud +#### Run EMQX in the Cloud The simplest way to set up EMQX is to create a managed deployment with EMQX Cloud. You can [try EMQX Cloud for free](https://www.emqx.com/en/signup?utm_source=github.com&utm_medium=referral&utm_campaign=emqx-readme-to-cloud&continue=https://cloud-intl.emqx.com/console/deployments/0?oper=new), no credit card required. @@ -59,6 +59,7 @@ For more organised improvement proposals, you can send pull requests to [EIP](ht ## Get Involved - Follow [@EMQTech on Twitter](https://twitter.com/EMQTech). +- Join our [Slack](https://slack-invite.emqx.io/). - If you have a specific question, check out our [discussion forums](https://github.com/emqx/emqx/discussions). - For general discussions, join us on the [official Discord](https://discord.gg/xYGf3fQnES) team. - Keep updated on [EMQX YouTube](https://www.youtube.com/channel/UC5FjR77ErAxvZENEWzQaO5Q) by subscribing. From 75817e23bd68babbba8b3d2ecbb10e23530cae11 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 30 Mar 2023 14:03:23 +0200 Subject: [PATCH 131/196] test: add some tests for EMQX boot failures --- scripts/test/emqx-boot.bats | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 scripts/test/emqx-boot.bats diff --git a/scripts/test/emqx-boot.bats b/scripts/test/emqx-boot.bats new file mode 100644 index 000000000..96f9a4457 --- /dev/null +++ b/scripts/test/emqx-boot.bats @@ -0,0 +1,21 @@ +#!/usr/bin/env bats + +# https://github.com/bats-core/bats-core +# env PROFILE=emqx bats -t -p --verbose-run scripts/test/emqx-boot.bats + +@test "PROFILE must be set" { + [[ -n "$PROFILE" ]] +} + +@test "emqx boot with invalid node name" { + output="$(env EMQX_NODE_NAME="invliadename#" ./_build/$PROFILE/rel/emqx/bin/emqx console 2>&1|| true)" + [[ "$output" =~ "ERROR: Invalid node name,".+ ]] +} + +@test "corrupted cluster config file" { + conffile="./_build/$PROFILE/rel/emqx/data/configs/cluster-override.conf" + echo "{" > $conffile + run ./_build/$PROFILE/rel/emqx/bin/emqx console + [[ $status -ne 0 ]] + rm -f $conffile +} From da8794ede03762aaf70f71f0373393324037d70e Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 30 Mar 2023 14:17:08 +0200 Subject: [PATCH 132/196] docs: Add change logs --- changes/ce/fix-10286.en.md | 2 ++ changes/ce/fix-10286.zh.md | 2 ++ 2 files changed, 4 insertions(+) create mode 100644 changes/ce/fix-10286.en.md create mode 100644 changes/ce/fix-10286.zh.md diff --git a/changes/ce/fix-10286.en.md b/changes/ce/fix-10286.en.md new file mode 100644 index 000000000..1bf721e55 --- /dev/null +++ b/changes/ce/fix-10286.en.md @@ -0,0 +1,2 @@ +Enhance logging behaviour during boot failure. +When EMQX fails to start due to a corrupted configuration files, excessive logging is eliminated and no crash dump file is generated. diff --git a/changes/ce/fix-10286.zh.md b/changes/ce/fix-10286.zh.md new file mode 100644 index 000000000..83455b8fd --- /dev/null +++ b/changes/ce/fix-10286.zh.md @@ -0,0 +1,2 @@ +优化启动失败的错误日志。 +如果 EMQX 因为损坏的配置文件无法启动时，不会再打印过多的错误日志，也不再生成 crash.dump 文件。 From d1b500b22d6662d10e82414c67c8d195cd485e7e Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 30 Mar 2023 14:33:17 +0200 Subject: [PATCH 133/196] chore: update PR template --- .github/pull_request_template.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 9b96db554..a27ee1cb5 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,11 +1,13 @@ Fixes +Make sure to target release-50 branch if this PR is intended to fix the issues for the release candidate. + ## PR Checklist Please convert it to a draft if any of the following conditions are not met. Reviewers may skip over until all the items are checked: - [ ] Added tests for the changes - [ ] Changed lines covered in coverage report -- [ ] Change log has been added to `changes/{ce,ee}/(feat|perf|fix)-.en.md` and `.zh.md` files +- [ ] Change log has been added to `changes/{ce,ee}/(feat|perf|fix)-.en.md` files - [ ] For internal contributor: there is a jira ticket to track this change - [ ] If there should be document changes, a PR to emqx-docs.git is sent, or a jira ticket is created to follow up - [ ] Schema changes are backward compatible From 81a104690df1e3429048aea9b063737c413e26d1 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 30 Mar 2023 16:19:22 +0200 Subject: [PATCH 134/196] test: fix flaky influxdb test --- .../test/emqx_ee_bridge_influxdb_SUITE.erl | 10 ++-------- .../src/emqx_ee_connector_influxdb.erl | 9 +++------ 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl index e8dd970f3..1b4b4aeb2 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl @@ -990,9 +990,7 @@ t_write_failure(Config) -> ?assertMatch([_ | _], Trace), [#{result := Result} | _] = Trace, ?assert( - {error, {error, {closed, "The connection was lost."}}} =:= Result orelse - {error, {error, closed}} =:= Result orelse - {error, {recoverable_error, econnrefused}} =:= Result, + not emqx_ee_connector_influxdb:is_unrecoverable_error(Result), #{got => Result} ); async -> @@ -1000,11 +998,7 @@ t_write_failure(Config) -> ?assertMatch([#{action := nack} | _], Trace), [#{result := Result} | _] = Trace, ?assert( - {error, {recoverable_error, {closed, "The connection was lost."}}} =:= - Result orelse - {error, {error, closed}} =:= Result orelse - {error, {recoverable_error, econnrefused}} =:= Result orelse - {error, {recoverable_error, noproc}} =:= Result, + not emqx_ee_connector_influxdb:is_unrecoverable_error(Result), #{got => Result} ) end, diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl index 5c99a23a8..afe17cae6 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl @@ -35,6 +35,9 @@ desc/1 ]). +%% only for test +-export([is_unrecoverable_error/1]). + -type ts_precision() :: ns | us | ms | s. %% influxdb servers don't need parse @@ -655,12 +658,6 @@ str(S) when is_list(S) -> is_unrecoverable_error({error, {unrecoverable_error, _}}) -> true; -is_unrecoverable_error({error, {recoverable_error, _}}) -> - false; -is_unrecoverable_error({error, {error, econnrefused}}) -> - false; -is_unrecoverable_error({error, econnrefused}) -> - false; is_unrecoverable_error(_) -> false. From 87a262edff6d847fa71342ad76ee5c694608b3ff Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Thu, 30 Mar 2023 16:53:27 +0200 Subject: [PATCH 135/196] docs: add README for EMQX Dashboard --- apps/emqx_dashboard/README.md | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/apps/emqx_dashboard/README.md b/apps/emqx_dashboard/README.md index 7466b5afe..88c714aca 100644 --- a/apps/emqx_dashboard/README.md +++ b/apps/emqx_dashboard/README.md @@ -1 +1,17 @@ -# TODO: Doc +# EMQX Dashboard + +This application provides access to the EMQX Dashboard as well as the actual, +underlying REST API itself and provides authorization to protect against +unauthorized access. Furthermore it connects middleware adding CORS headers. +Last but not least it exposes the `/status` endpoint needed for healtcheck +monitoring. + +## Implementation details + +This implementation is based on `minirest`, and relies on `hoconsc` to provide an +OpenAPI spec for `swagger`. + +Note, at this point EMQX Dashboard itself is an independent frontend project and +is integrated through a static file handler. This code here is responsible to +provide an HTTP(S) server to give access to it and its underlying API calls. +This includes user management and login for the frontend. From 7c597bfaa91092fa5050f0ad167b26de9371896e Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Thu, 30 Mar 2023 16:05:32 +0200 Subject: [PATCH 136/196] docs: give a better idea of what this application is supposed to do --- apps/emqx_management/README.md | 46 ++++++++++++++++++++++++++++------ 1 file changed, 39 insertions(+), 7 deletions(-) diff --git a/apps/emqx_management/README.md b/apps/emqx_management/README.md index fa37d0f0f..2edeae91a 100644 --- a/apps/emqx_management/README.md +++ b/apps/emqx_management/README.md @@ -1,12 +1,44 @@ -# emqx-management +# EMQX Management -EMQX Management API +EMQX Management offers various interfaces for administrators to interact with +the system, either by a remote console attached to a running node, a CLI (i.e. +`./emqx ctl`), or through its rich CRUD-style REST API (mostly used by EMQX' +dashboard). The system enables administrators to modify both cluster and +individual node configurations, and provides the ability to view and reset +different statistics and metrics. -## How to Design RESTful API? +## Functionality -http://restful-api-design.readthedocs.io/en/latest/scope.html +Amongst others it allows to manage -default application see: -header: -authorization: Basic YWRtaW46cHVibGlj +* alarms, +* API Keys, +* banned clients, users or hosts, +* connected clients including their topic subscriptions, +* cluster configurations, +* configuration of MQTT listeners, +* node configuration, +* custom plugins, +* fixed subscriptions, +* and topics. + +Moreover it lets you + +* modify hot and non-hot updatable configuration values, +* publish messages, as well as bulk messages, +* create trace files, +* and last but not least monitor system status. + +## Implementation Notes + +API endpoints are implemented using the `minirest` framework in combination with +`hoconsc` and `emqx_dashboard_swagger`. + +## TODO/FIXME + +At its current state there are some reverse dependencies from other applications +that do calls directly into `emqx_mgmt`. + +Also, and somewhat related, its bpapi proto modules do calls directly into +other applications. From 14039e393a7f2d41400bdfce638b53880d6d68d7 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 30 Mar 2023 17:11:10 +0200 Subject: [PATCH 137/196] docs: Update changes/ce/fix-10286.en.md Co-authored-by: Thales Macedo Garitezi --- changes/ce/fix-10286.en.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changes/ce/fix-10286.en.md b/changes/ce/fix-10286.en.md index 1bf721e55..3a51fefe2 100644 --- a/changes/ce/fix-10286.en.md +++ b/changes/ce/fix-10286.en.md @@ -1,2 +1,2 @@ Enhance logging behaviour during boot failure. -When EMQX fails to start due to a corrupted configuration files, excessive logging is eliminated and no crash dump file is generated. +When EMQX fails to start due to corrupted configuration files, excessive logging is eliminated and no crash dump file is generated. From e808fef1e47d6ae9f2aae7cc8d039d6c5fa4003d Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Thu, 30 Mar 2023 17:49:56 +0200 Subject: [PATCH 138/196] feat: (MongoDB bridge) use ${var} syntax for MongoDB collection This commit makes it possible to use the ${var} syntax to refer to variables in the payload of the message in the collection field. This makes it possible to select which collection to insert into dynamically. Fixes: https://emqx.atlassian.net/browse/EMQX-9246 --- .../test/emqx_ee_bridge_mongodb_SUITE.erl | 24 ++++++++++++++++++- .../src/emqx_ee_connector_mongodb.erl | 9 ++++++- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl index f81571223..9850c9529 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl @@ -26,7 +26,8 @@ group_tests() -> [ t_setup_via_config_and_publish, t_setup_via_http_api_and_publish, - t_payload_template + t_payload_template, + t_collection_template ]. groups() -> @@ -302,3 +303,24 @@ t_payload_template(Config) -> find_all(Config) ), ok. + +t_collection_template(Config) -> + {ok, _} = create_bridge( + Config, + #{ + <<"payload_template">> => <<"{\"foo\": \"${clientid}\"}">>, + <<"collection">> => <<"${mycollectionvar}">> + } + ), + Val = erlang:unique_integer(), + ClientId = emqx_guid:to_hexstr(emqx_guid:gen()), + ok = send_message(Config, #{ + key => Val, + clientid => ClientId, + mycollectionvar => <<"mycol">> + }), + ?assertMatch( + {ok, [#{<<"foo">> := ClientId}]}, + find_all(Config) + ), + ok. diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl index b1327fef6..8df77fbe0 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl @@ -35,8 +35,11 @@ on_start(InstanceId, Config) -> {ok, ConnectorState} -> PayloadTemplate0 = maps:get(payload_template, Config, undefined), PayloadTemplate = preprocess_template(PayloadTemplate0), + CollectionTemplateSource = maps:get(collection, Config), + CollectionTemplate = preprocess_template(CollectionTemplateSource), State = #{ payload_template => PayloadTemplate, + collection_template => CollectionTemplate, connector_state => ConnectorState }, {ok, State}; @@ -50,10 +53,14 @@ on_stop(InstanceId, _State = #{connector_state := ConnectorState}) -> on_query(InstanceId, {send_message, Message0}, State) -> #{ payload_template := PayloadTemplate, + collection_template := CollectionTemplate, connector_state := ConnectorState } = State, + NewConnectorState = ConnectorState#{ + collection => emqx_plugin_libs_rule:proc_tmpl(CollectionTemplate, Message0) + }, Message = render_message(PayloadTemplate, Message0), - emqx_connector_mongo:on_query(InstanceId, {send_message, Message}, ConnectorState); + emqx_connector_mongo:on_query(InstanceId, {send_message, Message}, NewConnectorState); on_query(InstanceId, Request, _State = #{connector_state := ConnectorState}) -> emqx_connector_mongo:on_query(InstanceId, Request, ConnectorState). From 161435d29debb8487dd599d532c871fdb6bcd6ca Mon Sep 17 00:00:00 2001 From: Paulo Zulato Date: Thu, 30 Mar 2023 13:55:02 -0300 Subject: [PATCH 139/196] fix: keep eval command backward compatible with v4 Keeps "eval" command evaluating only Erlang expressions, even on Elixir node. Fixes: https://emqx.atlassian.net/browse/EMQX-8947 --- .ci/docker-compose-file/scripts/run-emqx.sh | 2 +- bin/emqx | 24 +-------------------- bin/node_dump | 4 ++-- changes/ce/fix-10297.en.md | 1 + 4 files changed, 5 insertions(+), 26 deletions(-) create mode 100644 changes/ce/fix-10297.en.md diff --git a/.ci/docker-compose-file/scripts/run-emqx.sh b/.ci/docker-compose-file/scripts/run-emqx.sh index e8cdfdf4f..8f124aa63 100755 --- a/.ci/docker-compose-file/scripts/run-emqx.sh +++ b/.ci/docker-compose-file/scripts/run-emqx.sh @@ -29,7 +29,7 @@ esac is_node_up() { local node="$1" docker exec -i "$node" \ - bash -c "emqx eval-erl \"['emqx@node1.emqx.io','emqx@node2.emqx.io'] = maps:get(running_nodes, ekka_cluster:info()).\"" > /dev/null 2>&1 + bash -c "emqx eval \"['emqx@node1.emqx.io','emqx@node2.emqx.io'] = maps:get(running_nodes, ekka_cluster:info()).\"" > /dev/null 2>&1 } is_node_listening() { diff --git a/bin/emqx b/bin/emqx index d8940ad45..442f3ce6e 100755 --- a/bin/emqx +++ b/bin/emqx @@ -159,9 +159,6 @@ usage() { echo "Print EMQX installation root dir" ;; eval) - echo "Evaluate an Erlang or Elixir expression in the EMQX node" - ;; - eval-erl) echo "Evaluate an Erlang expression in the EMQX node, even on Elixir node" ;; eval-ex) @@ -231,7 +228,7 @@ usage() { echo " Install Info: ertspath | root_dir" echo " Runtime Status: pid | ping" echo " Validate Config: check_config" - echo " Advanced: console_clean | escript | rpc | rpcterms | eval | eval-erl | eval-ex" + echo " Advanced: console_clean | escript | rpc | rpcterms | eval | eval-ex" echo '' echo "Execute '$REL_NAME COMMAND help' for more information" ;; @@ -1265,25 +1262,6 @@ case "${COMMAND}" in eval) assert_node_alive - shift - if [ "$IS_ELIXIR" = "yes" ] - then - "$REL_DIR/elixir" \ - --hidden \ - --name "rand-$(relx_gen_id)-$NAME" \ - --cookie "$COOKIE" \ - --boot "$REL_DIR/start_clean" \ - --boot-var RELEASE_LIB "$ERTS_LIB_DIR" \ - --vm-args "$REL_DIR/remote.vm.args" \ - --erl "-start_epmd false -epmd_module ekka_epmd" \ - --rpc-eval "$NAME" "$@" - else - relx_nodetool "eval" "$@" - fi - ;; - eval-erl) - assert_node_alive - shift relx_nodetool "eval" "$@" ;; diff --git a/bin/node_dump b/bin/node_dump index 09baf04fd..1c4df08b5 100755 --- a/bin/node_dump +++ b/bin/node_dump @@ -49,7 +49,7 @@ done # Collect system info: { collect "$RUNNER_BIN_DIR"/emqx_ctl broker - collect "$RUNNER_BIN_DIR"/emqx eval-erl "'emqx_node_dump:sys_info()'" + collect "$RUNNER_BIN_DIR"/emqx eval "'emqx_node_dump:sys_info()'" collect uname -a collect uptime @@ -64,7 +64,7 @@ done # Collect information about the configuration: { - collect "$RUNNER_BIN_DIR"/emqx eval-erl "'emqx_node_dump:app_env_dump()'" + collect "$RUNNER_BIN_DIR"/emqx eval "'emqx_node_dump:app_env_dump()'" } > "${CONF_DUMP}" # Collect license info: diff --git a/changes/ce/fix-10297.en.md b/changes/ce/fix-10297.en.md new file mode 100644 index 000000000..305473b22 --- /dev/null +++ b/changes/ce/fix-10297.en.md @@ -0,0 +1 @@ +Keeps `eval` command backward compatible with v4 by evaluating only Erlang expressions, even on Elixir node. For Elixir expressions, use `eval-ex` command. From 634d00302552dd6850c283cfc66f014e4b46728a Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 30 Mar 2023 19:47:02 +0200 Subject: [PATCH 140/196] chore: add copilot markers to pr template --- .github/pull_request_template.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index a27ee1cb5..7cb91f0d4 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,6 +1,9 @@ Fixes -Make sure to target release-50 branch if this PR is intended to fix the issues for the release candidate. + + +## Summary +copilot:summary ## PR Checklist Please convert it to a draft if any of the following conditions are not met. Reviewers may skip over until all the items are checked: From a2b82dae69322f4faa5f0c09eff83f25f68aa263 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 30 Mar 2023 17:40:53 -0300 Subject: [PATCH 141/196] fix(plugins): create directory before uploading Fixes https://emqx.atlassian.net/browse/EMQX-9434 Fixes https://github.com/emqx/emqx-elixir-plugin/issues/23 --- apps/emqx_management/src/emqx_mgmt_api_plugins.erl | 1 + changes/ce/fix-10300.en.md | 1 + mix.exs | 3 +++ 3 files changed, 5 insertions(+) create mode 100644 changes/ce/fix-10300.en.md diff --git a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl index a46584f7f..92814d112 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl @@ -428,6 +428,7 @@ update_boot_order(post, #{bindings := #{name := Name}, body := Body}) -> %% For RPC upload_install/2 install_package(FileName, Bin) -> File = filename:join(emqx_plugins:install_dir(), FileName), + ok = filelib:ensure_dir(File), ok = file:write_file(File, Bin), PackageName = string:trim(FileName, trailing, ".tar.gz"), case emqx_plugins:ensure_installed(PackageName) of diff --git a/changes/ce/fix-10300.en.md b/changes/ce/fix-10300.en.md new file mode 100644 index 000000000..a60f9dfcd --- /dev/null +++ b/changes/ce/fix-10300.en.md @@ -0,0 +1 @@ +Fixed an issue where a build made with Elixir could not receive uploaded plugins until the `plugins` folder was created manually to receive the uploaded files. diff --git a/mix.exs b/mix.exs index ffa2ce056..514c9139d 100644 --- a/mix.exs +++ b/mix.exs @@ -394,10 +394,12 @@ defmodule EMQXUmbrella.MixProject do bin = Path.join(release.path, "bin") etc = Path.join(release.path, "etc") log = Path.join(release.path, "log") + plugins = Path.join(release.path, "plugins") Mix.Generator.create_directory(bin) Mix.Generator.create_directory(etc) Mix.Generator.create_directory(log) + Mix.Generator.create_directory(plugins) Mix.Generator.create_directory(Path.join(etc, "certs")) Enum.each( @@ -610,6 +612,7 @@ defmodule EMQXUmbrella.MixProject do &[ "etc", "data", + "plugins", "bin/node_dump" | &1 ] From fec0e7ab0f591602d500e9cacb049dffc1387ac8 Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 22 Mar 2023 14:17:45 +0800 Subject: [PATCH 142/196] chore: update changes --- changes/ee/fix-10201.en.md | 1 + changes/ee/fix-10201.zh.md | 1 + 2 files changed, 2 insertions(+) create mode 100644 changes/ee/fix-10201.en.md create mode 100644 changes/ee/fix-10201.zh.md diff --git a/changes/ee/fix-10201.en.md b/changes/ee/fix-10201.en.md new file mode 100644 index 000000000..b3dd53150 --- /dev/null +++ b/changes/ee/fix-10201.en.md @@ -0,0 +1 @@ +In TDengine, removed the redundant database name from the SQL template. diff --git a/changes/ee/fix-10201.zh.md b/changes/ee/fix-10201.zh.md new file mode 100644 index 000000000..53b175551 --- /dev/null +++ b/changes/ee/fix-10201.zh.md @@ -0,0 +1 @@ +在 TDengine 桥接的 SQL 模板中，删除了多余的数据库表名。 From 91df54028786d5ed0b9c0cc3bcba1b883f077aae Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Fri, 31 Mar 2023 06:36:21 +0200 Subject: [PATCH 143/196] docs: add changelog entry for MongoDB ${var} syntax for collection --- changes/ee/feat-10294.en.md | 1 + changes/ee/feat-10294.zh.md | 1 + 2 files changed, 2 insertions(+) create mode 100644 changes/ee/feat-10294.en.md create mode 100644 changes/ee/feat-10294.zh.md diff --git a/changes/ee/feat-10294.en.md b/changes/ee/feat-10294.en.md new file mode 100644 index 000000000..cac3a7587 --- /dev/null +++ b/changes/ee/feat-10294.en.md @@ -0,0 +1 @@ +When configuring a MongoDB bridge, you can now use the ${var} syntax to reference fields in the message payload within the collection field. This enables you to select the collection to insert data into dynamically. diff --git a/changes/ee/feat-10294.zh.md b/changes/ee/feat-10294.zh.md new file mode 100644 index 000000000..ca1727012 --- /dev/null +++ b/changes/ee/feat-10294.zh.md @@ -0,0 +1 @@ +在配置 MongoDB 桥时，现在可以使用 ${var} 语法来引用消息负载中的字段，以便动态选择要插入的集合。 From 1ff96f5314dfc83051e361c27444cadeaf02fc00 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 31 Mar 2023 09:18:39 +0200 Subject: [PATCH 144/196] style: fix wording Co-authored-by: Zaiming (Stone) Shi --- apps/emqx_management/README.md | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/apps/emqx_management/README.md b/apps/emqx_management/README.md index 2edeae91a..9378d18b1 100644 --- a/apps/emqx_management/README.md +++ b/apps/emqx_management/README.md @@ -12,16 +12,13 @@ different statistics and metrics. Amongst others it allows to manage -* alarms, -* API Keys, -* banned clients, users or hosts, -* connected clients including their topic subscriptions, -* cluster configurations, -* configuration of MQTT listeners, -* node configuration, -* custom plugins, -* fixed subscriptions, -* and topics. +* Alarms +* API Keys +* Banned clients, users or hosts +* Clients (and sessions) including their topic subscriptions +* Configurations +* Manage plugins +* Fixed subscriptions Moreover it lets you @@ -33,7 +30,7 @@ Moreover it lets you ## Implementation Notes API endpoints are implemented using the `minirest` framework in combination with -`hoconsc` and `emqx_dashboard_swagger`. +HOCON schema and OpenAPI 3.0 specifications. ## TODO/FIXME From 7e31e60e90f358fbbc1d46fdfceb6270cb655f1b Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 31 Mar 2023 09:19:48 +0200 Subject: [PATCH 145/196] style: fix wording --- apps/emqx_management/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/emqx_management/README.md b/apps/emqx_management/README.md index 9378d18b1..aa5d0c606 100644 --- a/apps/emqx_management/README.md +++ b/apps/emqx_management/README.md @@ -19,6 +19,7 @@ Amongst others it allows to manage * Configurations * Manage plugins * Fixed subscriptions +* Topics Moreover it lets you From 2dd10df417279a28bb8f3721e0a67bf15376cdb0 Mon Sep 17 00:00:00 2001 From: Paulo Zulato Date: Fri, 31 Mar 2023 09:24:38 -0300 Subject: [PATCH 146/196] fix: remove redundant text for eval command Co-authored-by: Zaiming (Stone) Shi --- bin/emqx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/emqx b/bin/emqx index 442f3ce6e..884895332 100755 --- a/bin/emqx +++ b/bin/emqx @@ -159,7 +159,7 @@ usage() { echo "Print EMQX installation root dir" ;; eval) - echo "Evaluate an Erlang expression in the EMQX node, even on Elixir node" + echo "Evaluate an Erlang expression in the EMQX node." ;; eval-ex) echo "Evaluate an Elixir expression in the EMQX node. Only applies to Elixir node" From 159bcf329cecfcae9ea8764f5a15ad8eb45b8fef Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 29 Mar 2023 20:31:58 +0800 Subject: [PATCH 147/196] refactor: make Stomp and MQTT-SN gateway as an independent apps --- .../i18n/emqx_gateway_schema_i18n.conf | 92 --------- apps/emqx_gateway/src/emqx_gateway_api.erl | 4 +- apps/emqx_gateway/src/emqx_gateway_app.erl | 66 +++--- apps/emqx_gateway/src/emqx_gateway_schema.erl | 155 +++----------- apps/emqx_gateway/src/emqx_gateway_utils.erl | 81 +++++++- apps/emqx_mqttsn/.gitignore | 19 ++ apps/emqx_mqttsn/LICENSE | 191 ++++++++++++++++++ .../src/mqttsn => emqx_mqttsn}/README.md | 0 apps/emqx_mqttsn/i18n/emqx_mqttsn_schema.conf | 64 ++++++ .../include/emqx_mqttsn.hrl} | 0 apps/emqx_mqttsn/rebar.config | 2 + apps/emqx_mqttsn/src/emqx_mqttsn.app.src | 10 + .../src/emqx_mqttsn.erl} | 64 +++--- .../src/emqx_mqttsn_broadcast.erl} | 14 +- .../src/emqx_mqttsn_channel.erl} | 26 +-- .../src/emqx_mqttsn_frame.erl} | 6 +- .../src/emqx_mqttsn_registry.erl} | 30 ++- apps/emqx_mqttsn/src/emqx_mqttsn_schema.erl | 107 ++++++++++ apps/emqx_stomp/.gitignore | 19 ++ apps/emqx_stomp/LICENSE | 191 ++++++++++++++++++ .../src/stomp => emqx_stomp}/README.md | 0 apps/emqx_stomp/i18n/emqx_stomp_schema.conf | 32 +++ .../include/emqx_stomp.hrl | 0 apps/emqx_stomp/rebar.config | 2 + apps/emqx_stomp/src/emqx_stomp.app.src | 10 + .../src/emqx_stomp.erl} | 51 ++--- .../src}/emqx_stomp_channel.erl | 2 +- .../src}/emqx_stomp_frame.erl | 2 +- .../src}/emqx_stomp_heartbeat.erl | 2 +- apps/emqx_stomp/src/emqx_stomp_schema.erl | 80 ++++++++ rebar.config.erl | 2 + 31 files changed, 968 insertions(+), 356 deletions(-) create mode 100644 apps/emqx_mqttsn/.gitignore create mode 100644 apps/emqx_mqttsn/LICENSE rename apps/{emqx_gateway/src/mqttsn => emqx_mqttsn}/README.md (100%) create mode 100644 apps/emqx_mqttsn/i18n/emqx_mqttsn_schema.conf rename apps/{emqx_gateway/src/mqttsn/include/emqx_sn.hrl => emqx_mqttsn/include/emqx_mqttsn.hrl} (100%) create mode 100644 apps/emqx_mqttsn/rebar.config create mode 100644 apps/emqx_mqttsn/src/emqx_mqttsn.app.src rename apps/{emqx_gateway/src/mqttsn/emqx_sn_impl.erl => emqx_mqttsn/src/emqx_mqttsn.erl} (76%) rename apps/{emqx_gateway/src/mqttsn/emqx_sn_broadcast.erl => emqx_mqttsn/src/emqx_mqttsn_broadcast.erl} (89%) rename apps/{emqx_gateway/src/mqttsn/emqx_sn_channel.erl => emqx_mqttsn/src/emqx_mqttsn_channel.erl} (98%) rename apps/{emqx_gateway/src/mqttsn/emqx_sn_frame.erl => emqx_mqttsn/src/emqx_mqttsn_frame.erl} (99%) rename apps/{emqx_gateway/src/mqttsn/emqx_sn_registry.erl => emqx_mqttsn/src/emqx_mqttsn_registry.erl} (91%) create mode 100644 apps/emqx_mqttsn/src/emqx_mqttsn_schema.erl create mode 100644 apps/emqx_stomp/.gitignore create mode 100644 apps/emqx_stomp/LICENSE rename apps/{emqx_gateway/src/stomp => emqx_stomp}/README.md (100%) create mode 100644 apps/emqx_stomp/i18n/emqx_stomp_schema.conf rename apps/{emqx_gateway/src/stomp => emqx_stomp}/include/emqx_stomp.hrl (100%) create mode 100644 apps/emqx_stomp/rebar.config create mode 100644 apps/emqx_stomp/src/emqx_stomp.app.src rename apps/{emqx_gateway/src/stomp/emqx_stomp_impl.erl => emqx_stomp/src/emqx_stomp.erl} (83%) rename apps/{emqx_gateway/src/stomp => emqx_stomp/src}/emqx_stomp_channel.erl (99%) rename apps/{emqx_gateway/src/stomp => emqx_stomp/src}/emqx_stomp_frame.erl (99%) rename apps/{emqx_gateway/src/stomp => emqx_stomp/src}/emqx_stomp_heartbeat.erl (98%) create mode 100644 apps/emqx_stomp/src/emqx_stomp_schema.erl diff --git a/apps/emqx_gateway/i18n/emqx_gateway_schema_i18n.conf b/apps/emqx_gateway/i18n/emqx_gateway_schema_i18n.conf index 74a70eb73..aaa5007ee 100644 --- a/apps/emqx_gateway/i18n/emqx_gateway_schema_i18n.conf +++ b/apps/emqx_gateway/i18n/emqx_gateway_schema_i18n.conf @@ -1,97 +1,5 @@ emqx_gateway_schema { - stomp { - desc { - en: """The Stomp Gateway configuration. -This gateway supports v1.2/1.1/1.0""" - zh: """Stomp 网关配置。当前实现支持 v1.2/1.1/1.0 协议版本""" - } - } - - stom_frame_max_headers { - desc { - en: """The maximum number of Header""" - zh: """允许的 Header 最大数量""" - } - } - - stomp_frame_max_headers_length { - desc { - en: """The maximum string length of the Header Value""" - zh: """允许的 Header 字符串的最大长度""" - } - } - - stom_frame_max_body_length { - desc { - en: """Maximum number of bytes of Body allowed per Stomp packet""" - zh: """允许的 Stomp 报文 Body 的最大字节数""" - } - } - - mqttsn { - desc { - en: """The MQTT-SN Gateway configuration. -This gateway only supports the v1.2 protocol""" - zh: """MQTT-SN 网关配置。当前实现仅支持 v1.2 版本""" - } - } - - mqttsn_gateway_id { - desc { - en: """MQTT-SN Gateway ID. -When the broadcast option is enabled, the gateway will broadcast ADVERTISE message with this value""" - zh: """MQTT-SN 网关 ID。 -当 broadcast 打开时，MQTT-SN 网关会使用该 ID 来广播 ADVERTISE 消息""" - } - } - - mqttsn_broadcast { - desc { - en: """Whether to periodically broadcast ADVERTISE messages""" - zh: """是否周期性广播 ADVERTISE 消息""" - } - } - - mqttsn_enable_qos3 { - desc { - en: """Allows connectionless clients to publish messages with a Qos of -1. -This feature is defined for very simple client implementations which do not support any other features except this one. There is no connection setup nor tear down, no registration nor subscription. The client just sends its 'PUBLISH' messages to a GW""" - zh: """是否允许无连接的客户端发送 QoS 等于 -1 的消息。 -该功能主要用于支持轻量的 MQTT-SN 客户端实现，它不会向网关建立连接，注册主题，也不会发起订阅；它只使用 QoS 为 -1 来发布消息""" - } - } - - mqttsn_subs_resume { - desc { - en: """Whether to initiate all subscribed topic name registration messages to the client after the Session has been taken over by a new channel""" - zh: """在会话被重用后，网关是否主动向客户端注册对已订阅主题名称""" - } - } - - mqttsn_predefined { - desc { - en: """The pre-defined topic IDs and topic names. -A 'pre-defined' topic ID is a topic ID whose mapping to a topic name is known in advance by both the client's application and the gateway""" - zh: """预定义主题列表。 -预定义的主题列表，是一组 主题 ID 和 主题名称 的映射关系。使用预先定义的主题列表，可以减少 MQTT-SN 客户端和网关对于固定主题的注册请求""" - } - } - - mqttsn_predefined_id { - desc { - en: """Topic ID. Range: 1-65535""" - zh: """主题 ID。范围：1-65535""" - } - } - - mqttsn_predefined_topic { - desc { - en: """Topic Name""" - zh: """主题名称。注：不支持通配符""" - } - } - coap { desc { en: """The CoAP Gateway configuration. diff --git a/apps/emqx_gateway/src/emqx_gateway_api.erl b/apps/emqx_gateway/src/emqx_gateway_api.erl index 62f723d59..bc44daca8 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api.erl @@ -395,7 +395,7 @@ fields(Gw) when Gw == exproto -> [{name, mk(Gw, #{desc => ?DESC(gateway_name)})}] ++ - convert_listener_struct(emqx_gateway_schema:fields(Gw)); + convert_listener_struct(emqx_gateway_schema:gateway_schema(Gw)); fields(Gw) when Gw == update_stomp; Gw == update_mqttsn; @@ -405,7 +405,7 @@ fields(Gw) when -> "update_" ++ GwStr = atom_to_list(Gw), Gw1 = list_to_existing_atom(GwStr), - remove_listener_and_authn(emqx_gateway_schema:fields(Gw1)); + remove_listener_and_authn(emqx_gateway_schema:gateway_schema(Gw1)); fields(Listener) when Listener == tcp_listener; Listener == ssl_listener; diff --git a/apps/emqx_gateway/src/emqx_gateway_app.erl b/apps/emqx_gateway/src/emqx_gateway_app.erl index cb5a16fde..a805a0ceb 100644 --- a/apps/emqx_gateway/src/emqx_gateway_app.erl +++ b/apps/emqx_gateway/src/emqx_gateway_app.erl @@ -41,33 +41,49 @@ stop(_State) -> %% Internal funcs load_default_gateway_applications() -> - Apps = gateway_type_searching(), - lists:foreach(fun reg/1, Apps). + BuiltInGateways = [ + #{ + name => lwm2m, + callback_module => emqx_lwm2m_impl, + config_schema_module => emqx_lwm2m_schema + }, + #{ + name => coap, + callback_module => emqx_coap_impl, + config_schema_module => emqx_gateway_schema + }, + #{ + name => exproto, + callback_module => emqx_exproto_impl, + config_schema_module => emqx_gateway_schema + } + ], + lists:foreach( + fun(Def) -> + load_gateway_application(Def) + end, + emqx_gateway_utils:find_gateway_definations() ++ BuiltInGateways + ). -gateway_type_searching() -> - %% FIXME: Hardcoded apps - [ - emqx_stomp_impl, - emqx_sn_impl, - emqx_exproto_impl, - emqx_coap_impl, - emqx_lwm2m_impl - ]. - -reg(Mod) -> - try - Mod:reg(), - ?SLOG(debug, #{ - msg => "register_gateway_succeed", - callback_module => Mod - }) - catch - Class:Reason:Stk -> +load_gateway_application( + #{ + name := Name, + callback_module := CbMod, + config_schema_module := SchemaMod + } +) -> + RegistryOptions = [{cbkmod, CbMod}, {schema, SchemaMod}], + case emqx_gateway_registry:reg(Name, RegistryOptions) of + ok -> + ?SLOG(debug, #{ + msg => "register_gateway_succeed", + callback_module => CbMod + }); + {error, already_registered} -> ?SLOG(error, #{ - msg => "failed_to_register_gateway", - callback_module => Mod, - reason => {Class, Reason}, - stacktrace => Stk + msg => "gateway_already_registered", + name => Name, + callback_module => CbMod }) end. diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl index 2034a40eb..f7ce3d05c 100644 --- a/apps/emqx_gateway/src/emqx_gateway_schema.erl +++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl @@ -53,6 +53,8 @@ -export([proxy_protocol_opts/0]). +-export([mountpoint/0, mountpoint/1, gateway_common_options/0, gateway_schema/1]). + namespace() -> gateway. tags() -> @@ -62,22 +64,6 @@ roots() -> [gateway]. fields(gateway) -> [ - {stomp, - sc( - ref(stomp), - #{ - required => {false, recursively}, - desc => ?DESC(stomp) - } - )}, - {mqttsn, - sc( - ref(mqttsn), - #{ - required => {false, recursively}, - desc => ?DESC(mqttsn) - } - )}, {coap, sc( ref(coap), @@ -102,102 +88,7 @@ fields(gateway) -> desc => ?DESC(exproto) } )} - ]; -fields(stomp) -> - [ - {frame, sc(ref(stomp_frame))}, - {mountpoint, mountpoint()}, - {listeners, sc(ref(tcp_listeners), #{desc => ?DESC(tcp_listeners)})} - ] ++ gateway_common_options(); -fields(stomp_frame) -> - [ - {max_headers, - sc( - non_neg_integer(), - #{ - default => 10, - desc => ?DESC(stom_frame_max_headers) - } - )}, - {max_headers_length, - sc( - non_neg_integer(), - #{ - default => 1024, - desc => ?DESC(stomp_frame_max_headers_length) - } - )}, - {max_body_length, - sc( - integer(), - #{ - default => 65536, - desc => ?DESC(stom_frame_max_body_length) - } - )} - ]; -fields(mqttsn) -> - [ - {gateway_id, - sc( - integer(), - #{ - default => 1, - required => true, - desc => ?DESC(mqttsn_gateway_id) - } - )}, - {broadcast, - sc( - boolean(), - #{ - default => false, - desc => ?DESC(mqttsn_broadcast) - } - )}, - %% TODO: rename - {enable_qos3, - sc( - boolean(), - #{ - default => true, - desc => ?DESC(mqttsn_enable_qos3) - } - )}, - {subs_resume, - sc( - boolean(), - #{ - default => false, - desc => ?DESC(mqttsn_subs_resume) - } - )}, - {predefined, - sc( - hoconsc:array(ref(mqttsn_predefined)), - #{ - default => [], - required => {false, recursively}, - desc => ?DESC(mqttsn_predefined) - } - )}, - {mountpoint, mountpoint()}, - {listeners, sc(ref(udp_listeners), #{desc => ?DESC(udp_listeners)})} - ] ++ gateway_common_options(); -fields(mqttsn_predefined) -> - [ - {id, - sc(integer(), #{ - required => true, - desc => ?DESC(mqttsn_predefined_id) - })}, - - {topic, - sc(binary(), #{ - required => true, - desc => ?DESC(mqttsn_predefined_topic) - })} - ]; + ] ++ gateway_schemas(); fields(coap) -> [ {heartbeat, @@ -522,17 +413,6 @@ fields(dtls_opts) -> desc(gateway) -> "EMQX Gateway configuration root."; -desc(stomp) -> - "The STOMP protocol gateway provides EMQX with the ability to access STOMP\n" - "(Simple (or Streaming) Text Orientated Messaging Protocol) protocol."; -desc(stomp_frame) -> - "Size limits for the STOMP frames."; -desc(mqttsn) -> - "The MQTT-SN (MQTT for Sensor Networks) protocol gateway."; -desc(mqttsn_predefined) -> - "The pre-defined topic name corresponding to the pre-defined topic\n" - "ID of N.\n\n" - "Note: the pre-defined topic ID of 0 is reserved."; desc(coap) -> "The CoAP protocol gateway provides EMQX with the access capability of the CoAP protocol.\n" "It allows publishing, subscribing, and receiving messages to EMQX in accordance\n" @@ -713,8 +593,33 @@ proxy_protocol_opts() -> )} ]. -sc(Type) -> - sc(Type, #{}). +%%-------------------------------------------------------------------- +%% dynamic schemas + +%% FIXME: don't hardcode the gateway names +gateway_schema(coap) -> fields(coap); +gateway_schema(lwm2m) -> fields(lwm2m); +gateway_schema(exproto) -> fields(exproto); +gateway_schema(stomp) -> emqx_stomp_schema:fields(stomp); +gateway_schema(mqttsn) -> emqx_mqttsn_schema:fields(mqttsn). + +gateway_schemas() -> + lists:map( + fun(#{name := Name, config_schema_module := Mod}) -> + {Name, + sc( + ref(Mod, Name), + #{ + required => {false, recursively}, + desc => ?DESC(Name) + } + )} + end, + emqx_gateway_utils:find_gateway_definations() + ). + +%%-------------------------------------------------------------------- +%% helpers sc(Type, Meta) -> hoconsc:mk(Type, Meta). diff --git a/apps/emqx_gateway/src/emqx_gateway_utils.erl b/apps/emqx_gateway/src/emqx_gateway_utils.erl index cee5baaa8..94c7490cc 100644 --- a/apps/emqx_gateway/src/emqx_gateway_utils.erl +++ b/apps/emqx_gateway/src/emqx_gateway_utils.erl @@ -46,7 +46,8 @@ global_chain/1, listener_chain/3, make_deprecated_paths/1, - make_compatible_schema/2 + make_compatible_schema/2, + find_gateway_definations/0 ]). -export([stringfy/1]). @@ -562,3 +563,81 @@ make_compatible_schema2(Path, SchemaFun) -> end, Schema ). + +find_gateway_definations() -> + lists:flatten( + lists:map( + fun(App) -> + gateways(find_attrs(App, gateway)) + end, + ignore_lib_apps(application:loaded_applications()) + ) + ). + +gateways([]) -> + []; +gateways([ + {_App, _Mod, + Defination = + #{ + name := Name, + callback_module := CbMod, + config_schema_module := SchemaMod + }} + | More +]) when is_atom(Name), is_atom(CbMod), is_atom(SchemaMod) -> + [Defination | gateways(More)]. + +find_attrs(App, Def) -> + [ + {App, Mod, Attr} + || {ok, Modules} <- [application:get_key(App, modules)], + Mod <- Modules, + {Name, Attrs} <- module_attributes(Mod), + Name =:= Def, + Attr <- Attrs + ]. + +module_attributes(Module) -> + try + Module:module_info(attributes) + catch + error:undef -> [] + end. + +ignore_lib_apps(Apps) -> + LibApps = [ + kernel, + stdlib, + sasl, + appmon, + eldap, + erts, + syntax_tools, + ssl, + crypto, + mnesia, + os_mon, + inets, + goldrush, + gproc, + runtime_tools, + snmp, + otp_mibs, + public_key, + asn1, + ssh, + hipe, + common_test, + observer, + webtool, + xmerl, + tools, + test_server, + compiler, + debugger, + eunit, + et, + wx + ], + [AppName || {AppName, _, _} <- Apps, not lists:member(AppName, LibApps)]. diff --git a/apps/emqx_mqttsn/.gitignore b/apps/emqx_mqttsn/.gitignore new file mode 100644 index 000000000..f1c455451 --- /dev/null +++ b/apps/emqx_mqttsn/.gitignore @@ -0,0 +1,19 @@ +.rebar3 +_* +.eunit +*.o +*.beam +*.plt +*.swp +*.swo +.erlang.cookie +ebin +log +erl_crash.dump +.rebar +logs +_build +.idea +*.iml +rebar3.crashdump +*~ diff --git a/apps/emqx_mqttsn/LICENSE b/apps/emqx_mqttsn/LICENSE new file mode 100644 index 000000000..5a5418f0f --- /dev/null +++ b/apps/emqx_mqttsn/LICENSE @@ -0,0 +1,191 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2023, JianBo He . + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/apps/emqx_gateway/src/mqttsn/README.md b/apps/emqx_mqttsn/README.md similarity index 100% rename from apps/emqx_gateway/src/mqttsn/README.md rename to apps/emqx_mqttsn/README.md diff --git a/apps/emqx_mqttsn/i18n/emqx_mqttsn_schema.conf b/apps/emqx_mqttsn/i18n/emqx_mqttsn_schema.conf new file mode 100644 index 000000000..20c160b11 --- /dev/null +++ b/apps/emqx_mqttsn/i18n/emqx_mqttsn_schema.conf @@ -0,0 +1,64 @@ +emqx_mqttsn_schema { + mqttsn { + desc { + en: """The MQTT-SN Gateway configuration. +This gateway only supports the v1.2 protocol""" + zh: """MQTT-SN 网关配置。当前实现仅支持 v1.2 版本""" + } + } + + mqttsn_gateway_id { + desc { + en: """MQTT-SN Gateway ID. +When the broadcast option is enabled, the gateway will broadcast ADVERTISE message with this value""" + zh: """MQTT-SN 网关 ID。 +当 broadcast 打开时，MQTT-SN 网关会使用该 ID 来广播 ADVERTISE 消息""" + } + } + + mqttsn_broadcast { + desc { + en: """Whether to periodically broadcast ADVERTISE messages""" + zh: """是否周期性广播 ADVERTISE 消息""" + } + } + + mqttsn_enable_qos3 { + desc { + en: """Allows connectionless clients to publish messages with a Qos of -1. +This feature is defined for very simple client implementations which do not support any other features except this one. There is no connection setup nor tear down, no registration nor subscription. The client just sends its 'PUBLISH' messages to a GW""" + zh: """是否允许无连接的客户端发送 QoS 等于 -1 的消息。 +该功能主要用于支持轻量的 MQTT-SN 客户端实现，它不会向网关建立连接，注册主题，也不会发起订阅；它只使用 QoS 为 -1 来发布消息""" + } + } + + mqttsn_subs_resume { + desc { + en: """Whether to initiate all subscribed topic name registration messages to the client after the Session has been taken over by a new channel""" + zh: """在会话被重用后，网关是否主动向客户端注册对已订阅主题名称""" + } + } + + mqttsn_predefined { + desc { + en: """The pre-defined topic IDs and topic names. +A 'pre-defined' topic ID is a topic ID whose mapping to a topic name is known in advance by both the client's application and the gateway""" + zh: """预定义主题列表。 +预定义的主题列表，是一组 主题 ID 和 主题名称 的映射关系。使用预先定义的主题列表，可以减少 MQTT-SN 客户端和网关对于固定主题的注册请求""" + } + } + + mqttsn_predefined_id { + desc { + en: """Topic ID. Range: 1-65535""" + zh: """主题 ID。范围：1-65535""" + } + } + + mqttsn_predefined_topic { + desc { + en: """Topic Name""" + zh: """主题名称。注：不支持通配符""" + } + } +} diff --git a/apps/emqx_gateway/src/mqttsn/include/emqx_sn.hrl b/apps/emqx_mqttsn/include/emqx_mqttsn.hrl similarity index 100% rename from apps/emqx_gateway/src/mqttsn/include/emqx_sn.hrl rename to apps/emqx_mqttsn/include/emqx_mqttsn.hrl diff --git a/apps/emqx_mqttsn/rebar.config b/apps/emqx_mqttsn/rebar.config new file mode 100644 index 000000000..2656fd554 --- /dev/null +++ b/apps/emqx_mqttsn/rebar.config @@ -0,0 +1,2 @@ +{erl_opts, [debug_info]}. +{deps, []}. diff --git a/apps/emqx_mqttsn/src/emqx_mqttsn.app.src b/apps/emqx_mqttsn/src/emqx_mqttsn.app.src new file mode 100644 index 000000000..e58cf5147 --- /dev/null +++ b/apps/emqx_mqttsn/src/emqx_mqttsn.app.src @@ -0,0 +1,10 @@ +{application, emqx_mqttsn, + [{description, "MQTT-SN Gateway"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [kernel, stdlib]}, + {env,[]}, + {modules, []}, + {licenses, ["Apache 2.0"]}, + {links, []} + ]}. diff --git a/apps/emqx_gateway/src/mqttsn/emqx_sn_impl.erl b/apps/emqx_mqttsn/src/emqx_mqttsn.erl similarity index 76% rename from apps/emqx_gateway/src/mqttsn/emqx_sn_impl.erl rename to apps/emqx_mqttsn/src/emqx_mqttsn.erl index db730aee1..5d6a94df4 100644 --- a/apps/emqx_gateway/src/mqttsn/emqx_sn_impl.erl +++ b/apps/emqx_mqttsn/src/emqx_mqttsn.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -14,13 +14,28 @@ %% limitations under the License. %%-------------------------------------------------------------------- -%% @doc The MQTT-SN Gateway Implement interface --module(emqx_sn_impl). - --behaviour(emqx_gateway_impl). +%% @doc The MQTT-SN Gateway implement interface +-module(emqx_mqttsn). -include_lib("emqx/include/logger.hrl"). +%% define a gateway named stomp +-gateway(#{ + name => mqttsn, + callback_module => ?MODULE, + config_schema_module => emqx_mqttsn_schema +}). + +%% callback_module must implement the emqx_gateway_impl behaviour +-behaviour(emqx_gateway_impl). + +%% callback for emqx_gateway_impl +-export([ + on_gateway_load/2, + on_gateway_update/3, + on_gateway_unload/2 +]). + -import( emqx_gateway_utils, [ @@ -30,31 +45,8 @@ ] ). -%% APIs --export([ - reg/0, - unreg/0 -]). - --export([ - on_gateway_load/2, - on_gateway_update/3, - on_gateway_unload/2 -]). - %%-------------------------------------------------------------------- -%% APIs -%%-------------------------------------------------------------------- - -reg() -> - RegistryOptions = [{cbkmod, ?MODULE}], - emqx_gateway_registry:reg(mqttsn, RegistryOptions). - -unreg() -> - emqx_gateway_registry:unreg(mqttsn). - -%%-------------------------------------------------------------------- -%% emqx_gateway_registry callbacks +%% emqx_gateway_impl callbacks %%-------------------------------------------------------------------- on_gateway_load( @@ -64,8 +56,8 @@ on_gateway_load( }, Ctx ) -> - %% We Also need to start `emqx_sn_broadcast` & - %% `emqx_sn_registry` process + %% We Also need to start `emqx_mqttsn_broadcast` & + %% `emqx_mqttsn_registry` process case maps:get(broadcast, Config, false) of false -> ok; @@ -73,23 +65,23 @@ on_gateway_load( %% FIXME: Port = 1884, SnGwId = maps:get(gateway_id, Config, undefined), - _ = emqx_sn_broadcast:start_link(SnGwId, Port), + _ = emqx_mqttsn_broadcast:start_link(SnGwId, Port), ok end, PredefTopics = maps:get(predefined, Config, []), - {ok, RegistrySvr} = emqx_sn_registry:start_link(GwName, PredefTopics), + {ok, RegistrySvr} = emqx_mqttsn_registry:start_link(GwName, PredefTopics), NConfig = maps:without( [broadcast, predefined], - Config#{registry => emqx_sn_registry:lookup_name(RegistrySvr)} + Config#{registry => emqx_mqttsn_registry:lookup_name(RegistrySvr)} ), Listeners = emqx_gateway_utils:normalize_config(NConfig), ModCfg = #{ - frame_mod => emqx_sn_frame, - chann_mod => emqx_sn_channel + frame_mod => emqx_mqttsn_frame, + chann_mod => emqx_mqttsn_channel }, case diff --git a/apps/emqx_gateway/src/mqttsn/emqx_sn_broadcast.erl b/apps/emqx_mqttsn/src/emqx_mqttsn_broadcast.erl similarity index 89% rename from apps/emqx_gateway/src/mqttsn/emqx_sn_broadcast.erl rename to apps/emqx_mqttsn/src/emqx_mqttsn_broadcast.erl index 5fc08ad7f..be0122e0e 100644 --- a/apps/emqx_gateway/src/mqttsn/emqx_sn_broadcast.erl +++ b/apps/emqx_mqttsn/src/emqx_mqttsn_broadcast.erl @@ -14,17 +14,11 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_sn_broadcast). +-module(emqx_mqttsn_broadcast). -behaviour(gen_server). --ifdef(TEST). -%% make rebar3 ct happy when testing with --suite path/to/module_SUITE.erl --include_lib("emqx_gateway/src/mqttsn/include/emqx_sn.hrl"). --else. -%% make mix happy --include("src/mqttsn/include/emqx_sn.hrl"). --endif. +-include("emqx_mqttsn.hrl"). -include_lib("emqx/include/logger.hrl"). -export([ @@ -65,7 +59,7 @@ stop() -> init([GwId, Port]) -> %% FIXME: - Duration = application:get_env(emqx_sn, advertise_duration, ?DEFAULT_DURATION), + Duration = application:get_env(emqx_mqttsn, advertise_duration, ?DEFAULT_DURATION), {ok, Sock} = gen_udp:open(0, [binary, {broadcast, true}]), {ok, ensure_advertise(#state{ @@ -121,7 +115,7 @@ send_advertise(#state{ addrs = Addrs, duration = Duration }) -> - Data = emqx_sn_frame:serialize_pkt(?SN_ADVERTISE_MSG(GwId, Duration), #{}), + Data = emqx_mqttsn_frame:serialize_pkt(?SN_ADVERTISE_MSG(GwId, Duration), #{}), lists:foreach( fun(Addr) -> ?SLOG(debug, #{ diff --git a/apps/emqx_gateway/src/mqttsn/emqx_sn_channel.erl b/apps/emqx_mqttsn/src/emqx_mqttsn_channel.erl similarity index 98% rename from apps/emqx_gateway/src/mqttsn/emqx_sn_channel.erl rename to apps/emqx_mqttsn/src/emqx_mqttsn_channel.erl index 23d07113c..c27c0ba3f 100644 --- a/apps/emqx_gateway/src/mqttsn/emqx_sn_channel.erl +++ b/apps/emqx_mqttsn/src/emqx_mqttsn_channel.erl @@ -14,11 +14,11 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_sn_channel). +-module(emqx_mqttsn_channel). -behaviour(emqx_gateway_channel). --include("src/mqttsn/include/emqx_sn.hrl"). +-include("emqx_mqttsn.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/types.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). @@ -51,7 +51,7 @@ %% Context ctx :: emqx_gateway_ctx:context(), %% Registry - registry :: emqx_sn_registry:registry(), + registry :: emqx_mqttsn_registry:registry(), %% Gateway Id gateway_id :: integer(), %% Enable QoS3 @@ -478,7 +478,7 @@ handle_in( true -> <>; false -> - emqx_sn_registry:lookup_topic( + emqx_mqttsn_registry:lookup_topic( Registry, ?NEG_QOS_CLIENT_ID, TopicId @@ -624,7 +624,7 @@ handle_in( clientinfo = #{clientid := ClientId} } ) -> - case emqx_sn_registry:register_topic(Registry, ClientId, TopicName) of + case emqx_mqttsn_registry:register_topic(Registry, ClientId, TopicName) of TopicId when is_integer(TopicId) -> ?SLOG(debug, #{ msg => "registered_topic_name", @@ -778,7 +778,7 @@ handle_in( {ok, Channel} end; ?SN_RC_INVALID_TOPIC_ID -> - case emqx_sn_registry:lookup_topic(Registry, ClientId, TopicId) of + case emqx_mqttsn_registry:lookup_topic(Registry, ClientId, TopicId) of undefined -> {ok, Channel}; TopicName -> @@ -1093,7 +1093,7 @@ convert_topic_id_to_name( clientinfo = #{clientid := ClientId} } ) -> - case emqx_sn_registry:lookup_topic(Registry, ClientId, TopicId) of + case emqx_mqttsn_registry:lookup_topic(Registry, ClientId, TopicId) of undefined -> {error, ?SN_RC_INVALID_TOPIC_ID}; TopicName -> @@ -1202,7 +1202,7 @@ preproc_subs_type( %% If the gateway is able accept the subscription, %% it assigns a topic id to the received topic name %% and returns it within a SUBACK message - case emqx_sn_registry:register_topic(Registry, ClientId, TopicName) of + case emqx_mqttsn_registry:register_topic(Registry, ClientId, TopicName) of {error, too_large} -> {error, ?SN_RC2_EXCEED_LIMITATION}; {error, wildcard_topic} -> @@ -1228,7 +1228,7 @@ preproc_subs_type( } ) -> case - emqx_sn_registry:lookup_topic( + emqx_mqttsn_registry:lookup_topic( Registry, ClientId, TopicId @@ -1344,7 +1344,7 @@ preproc_unsub_type( } ) -> case - emqx_sn_registry:lookup_topic( + emqx_mqttsn_registry:lookup_topic( Registry, ClientId, TopicId @@ -1765,7 +1765,7 @@ message_to_packet( ?QOS_0 -> 0; _ -> MsgId end, - case emqx_sn_registry:lookup_topic_id(Registry, ClientId, Topic) of + case emqx_mqttsn_registry:lookup_topic_id(Registry, ClientId, Topic) of {predef, PredefTopicId} -> Flags = #mqtt_sn_flags{qos = QoS, topic_id_type = ?SN_PREDEFINED_TOPIC}, ?SN_PUBLISH_MSG(Flags, PredefTopicId, NMsgId, Payload); @@ -1932,9 +1932,9 @@ ensure_registered_topic_name( Channel = #channel{registry = Registry} ) -> ClientId = clientid(Channel), - case emqx_sn_registry:lookup_topic_id(Registry, ClientId, TopicName) of + case emqx_mqttsn_registry:lookup_topic_id(Registry, ClientId, TopicName) of undefined -> - case emqx_sn_registry:register_topic(Registry, ClientId, TopicName) of + case emqx_mqttsn_registry:register_topic(Registry, ClientId, TopicName) of {error, Reason} -> {error, Reason}; TopicId -> {ok, TopicId} end; diff --git a/apps/emqx_gateway/src/mqttsn/emqx_sn_frame.erl b/apps/emqx_mqttsn/src/emqx_mqttsn_frame.erl similarity index 99% rename from apps/emqx_gateway/src/mqttsn/emqx_sn_frame.erl rename to apps/emqx_mqttsn/src/emqx_mqttsn_frame.erl index 39bd9e889..bf0fc52a4 100644 --- a/apps/emqx_gateway/src/mqttsn/emqx_sn_frame.erl +++ b/apps/emqx_mqttsn/src/emqx_mqttsn_frame.erl @@ -16,11 +16,11 @@ %%-------------------------------------------------------------------- %% @doc The frame parser for MQTT-SN protocol --module(emqx_sn_frame). +-module(emqx_mqttsn_frame). -behaviour(emqx_gateway_frame). --include("src/mqttsn/include/emqx_sn.hrl"). +-include("emqx_mqttsn.hrl"). -export([ initial_parse_state/1, @@ -438,7 +438,7 @@ format(?SN_DISCONNECT_MSG(Duration)) -> format(#mqtt_sn_message{type = Type, variable = Var}) -> io_lib:format( "mqtt_sn_message(type=~s, Var=~w)", - [emqx_sn_frame:message_type(Type), Var] + [emqx_mqttsn_frame:message_type(Type), Var] ). is_message(#mqtt_sn_message{type = Type}) when diff --git a/apps/emqx_gateway/src/mqttsn/emqx_sn_registry.erl b/apps/emqx_mqttsn/src/emqx_mqttsn_registry.erl similarity index 91% rename from apps/emqx_gateway/src/mqttsn/emqx_sn_registry.erl rename to apps/emqx_mqttsn/src/emqx_mqttsn_registry.erl index 689aab8ce..07da8c351 100644 --- a/apps/emqx_gateway/src/mqttsn/emqx_sn_registry.erl +++ b/apps/emqx_mqttsn/src/emqx_mqttsn_registry.erl @@ -15,13 +15,11 @@ %%-------------------------------------------------------------------- %% @doc The MQTT-SN Topic Registry -%% -%% XXX: --module(emqx_sn_registry). +-module(emqx_mqttsn_registry). -behaviour(gen_server). --include("src/mqttsn/include/emqx_sn.hrl"). +-include("emqx_mqttsn.hrl"). -include_lib("emqx/include/logger.hrl"). -export([start_link/2]). @@ -53,11 +51,11 @@ -export([lookup_name/1]). --define(SN_SHARD, emqx_sn_shard). +-define(SN_SHARD, emqx_mqttsn_shard). -record(state, {tabname, max_predef_topic_id = 0}). --record(emqx_sn_registry, {key, value}). +-record(emqx_mqttsn_registry, {key, value}). -type registry() :: {Tab :: atom(), RegistryPid :: pid()}. @@ -126,7 +124,7 @@ lookup_name(Pid) -> %%----------------------------------------------------------------------------- name(InstaId) -> - list_to_atom(lists:concat([emqx_sn_, InstaId, '_registry'])). + list_to_atom(lists:concat([emqx_mqttsn_, InstaId, '_registry'])). init([InstaId, PredefTopics]) -> %% {predef, TopicId} -> TopicName @@ -136,8 +134,8 @@ init([InstaId, PredefTopics]) -> Tab = name(InstaId), ok = mria:create_table(Tab, [ {storage, ram_copies}, - {record_name, emqx_sn_registry}, - {attributes, record_info(fields, emqx_sn_registry)}, + {record_name, emqx_mqttsn_registry}, + {attributes, record_info(fields, emqx_mqttsn_registry)}, {storage_properties, [{ets, [{read_concurrency, true}]}]}, {rlog_shard, ?SN_SHARD} ]), @@ -145,11 +143,11 @@ init([InstaId, PredefTopics]) -> MaxPredefId = lists:foldl( fun(#{id := TopicId, topic := TopicName0}, AccId) -> TopicName = iolist_to_binary(TopicName0), - mria:dirty_write(Tab, #emqx_sn_registry{ + mria:dirty_write(Tab, #emqx_mqttsn_registry{ key = {predef, TopicId}, value = TopicName }), - mria:dirty_write(Tab, #emqx_sn_registry{ + mria:dirty_write(Tab, #emqx_mqttsn_registry{ key = {predef, TopicName}, value = TopicId }), @@ -193,7 +191,7 @@ handle_call( handle_call({unregister, ClientId}, _From, State = #state{tabname = Tab}) -> Registry = mnesia:dirty_match_object( Tab, - {emqx_sn_registry, {ClientId, '_'}, '_'} + {emqx_mqttsn_registry, {ClientId, '_'}, '_'} ), lists:foreach( fun(R) -> @@ -234,7 +232,7 @@ code_change(_OldVsn, State, _Extra) -> do_register(Tab, ClientId, TopicId, TopicName) -> mnesia:write( Tab, - #emqx_sn_registry{ + #emqx_mqttsn_registry{ key = {ClientId, next_topic_id}, value = TopicId + 1 }, @@ -242,7 +240,7 @@ do_register(Tab, ClientId, TopicId, TopicName) -> ), mnesia:write( Tab, - #emqx_sn_registry{ + #emqx_mqttsn_registry{ key = {ClientId, TopicName}, value = TopicId }, @@ -250,7 +248,7 @@ do_register(Tab, ClientId, TopicId, TopicName) -> ), mnesia:write( Tab, - #emqx_sn_registry{ + #emqx_mqttsn_registry{ key = {ClientId, TopicId}, value = TopicName }, @@ -261,6 +259,6 @@ do_register(Tab, ClientId, TopicId, TopicName) -> next_topic_id(Tab, PredefId, ClientId) -> case mnesia:dirty_read(Tab, {ClientId, next_topic_id}) of - [#emqx_sn_registry{value = Id}] -> Id; + [#emqx_mqttsn_registry{value = Id}] -> Id; [] -> PredefId + 1 end. diff --git a/apps/emqx_mqttsn/src/emqx_mqttsn_schema.erl b/apps/emqx_mqttsn/src/emqx_mqttsn_schema.erl new file mode 100644 index 000000000..cb33cbe95 --- /dev/null +++ b/apps/emqx_mqttsn/src/emqx_mqttsn_schema.erl @@ -0,0 +1,107 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_mqttsn_schema). + +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("typerefl/include/types.hrl"). + +%% config schema provides +-export([fields/1, desc/1]). + +fields(mqttsn) -> + [ + {gateway_id, + sc( + integer(), + #{ + default => 1, + required => true, + desc => ?DESC(mqttsn_gateway_id) + } + )}, + {broadcast, + sc( + boolean(), + #{ + default => false, + desc => ?DESC(mqttsn_broadcast) + } + )}, + %% TODO: rename + {enable_qos3, + sc( + boolean(), + #{ + default => true, + desc => ?DESC(mqttsn_enable_qos3) + } + )}, + {subs_resume, + sc( + boolean(), + #{ + default => false, + desc => ?DESC(mqttsn_subs_resume) + } + )}, + {predefined, + sc( + hoconsc:array(ref(mqttsn_predefined)), + #{ + default => [], + required => {false, recursively}, + desc => ?DESC(mqttsn_predefined) + } + )}, + {mountpoint, emqx_gateway_schema:mountpoint()}, + {listeners, sc(ref(emqx_gateway_schema, udp_listeners), #{desc => ?DESC(udp_listeners)})} + ] ++ emqx_gateway_schema:gateway_common_options(); +fields(mqttsn_predefined) -> + [ + {id, + sc(integer(), #{ + required => true, + desc => ?DESC(mqttsn_predefined_id) + })}, + + {topic, + sc(binary(), #{ + required => true, + desc => ?DESC(mqttsn_predefined_topic) + })} + ]. + +desc(mqttsn) -> + "The MQTT-SN (MQTT for Sensor Networks) protocol gateway."; +desc(mqttsn_predefined) -> + "The pre-defined topic name corresponding to the pre-defined topic\n" + "ID of N.\n\n" + "Note: the pre-defined topic ID of 0 is reserved."; +desc(_) -> + undefined. + +%%-------------------------------------------------------------------- +%% internal functions + +sc(Type, Meta) -> + hoconsc:mk(Type, Meta). + +ref(StructName) -> + ref(?MODULE, StructName). + +ref(Mod, Field) -> + hoconsc:ref(Mod, Field). diff --git a/apps/emqx_stomp/.gitignore b/apps/emqx_stomp/.gitignore new file mode 100644 index 000000000..f1c455451 --- /dev/null +++ b/apps/emqx_stomp/.gitignore @@ -0,0 +1,19 @@ +.rebar3 +_* +.eunit +*.o +*.beam +*.plt +*.swp +*.swo +.erlang.cookie +ebin +log +erl_crash.dump +.rebar +logs +_build +.idea +*.iml +rebar3.crashdump +*~ diff --git a/apps/emqx_stomp/LICENSE b/apps/emqx_stomp/LICENSE new file mode 100644 index 000000000..5a5418f0f --- /dev/null +++ b/apps/emqx_stomp/LICENSE @@ -0,0 +1,191 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2023, JianBo He . + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/apps/emqx_gateway/src/stomp/README.md b/apps/emqx_stomp/README.md similarity index 100% rename from apps/emqx_gateway/src/stomp/README.md rename to apps/emqx_stomp/README.md diff --git a/apps/emqx_stomp/i18n/emqx_stomp_schema.conf b/apps/emqx_stomp/i18n/emqx_stomp_schema.conf new file mode 100644 index 000000000..3d166abb5 --- /dev/null +++ b/apps/emqx_stomp/i18n/emqx_stomp_schema.conf @@ -0,0 +1,32 @@ +emqx_stomp_schema { + stomp { + desc { + en: """The Stomp Gateway configuration. +This gateway supports v1.2/1.1/1.0""" + zh: """Stomp 网关配置。当前实现支持 v1.2/1.1/1.0 协议版本""" + } + } + + stom_frame_max_headers { + desc { + en: """The maximum number of Header""" + zh: """允许的 Header 最大数量""" + } + } + + stomp_frame_max_headers_length { + desc { + en: """The maximum string length of the Header Value""" + zh: """允许的 Header 字符串的最大长度""" + } + } + + stom_frame_max_body_length { + desc { + en: """Maximum number of bytes of Body allowed per Stomp packet""" + zh: """允许的 Stomp 报文 Body 的最大字节数""" + } + } + + +} diff --git a/apps/emqx_gateway/src/stomp/include/emqx_stomp.hrl b/apps/emqx_stomp/include/emqx_stomp.hrl similarity index 100% rename from apps/emqx_gateway/src/stomp/include/emqx_stomp.hrl rename to apps/emqx_stomp/include/emqx_stomp.hrl diff --git a/apps/emqx_stomp/rebar.config b/apps/emqx_stomp/rebar.config new file mode 100644 index 000000000..2656fd554 --- /dev/null +++ b/apps/emqx_stomp/rebar.config @@ -0,0 +1,2 @@ +{erl_opts, [debug_info]}. +{deps, []}. diff --git a/apps/emqx_stomp/src/emqx_stomp.app.src b/apps/emqx_stomp/src/emqx_stomp.app.src new file mode 100644 index 000000000..dffb9b7a4 --- /dev/null +++ b/apps/emqx_stomp/src/emqx_stomp.app.src @@ -0,0 +1,10 @@ +{application, emqx_stomp, [ + {description, "Stomp gateway"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [kernel, stdlib]}, + {env, []}, + {modules, []}, + {licenses, ["Apache 2.0"]}, + {links, []} +]}. diff --git a/apps/emqx_gateway/src/stomp/emqx_stomp_impl.erl b/apps/emqx_stomp/src/emqx_stomp.erl similarity index 83% rename from apps/emqx_gateway/src/stomp/emqx_stomp_impl.erl rename to apps/emqx_stomp/src/emqx_stomp.erl index c2907c262..6c14e222c 100644 --- a/apps/emqx_gateway/src/stomp/emqx_stomp_impl.erl +++ b/apps/emqx_stomp/src/emqx_stomp.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -14,13 +14,29 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_stomp_impl). - --behaviour(emqx_gateway_impl). +%% @doc The Stomp Gateway implement interface +-module(emqx_stomp). -include_lib("emqx/include/logger.hrl"). -include_lib("emqx_gateway/include/emqx_gateway.hrl"). +%% define a gateway named stomp +-gateway(#{ + name => stomp, + callback_module => ?MODULE, + config_schema_module => emqx_stomp_schema +}). + +%% callback_module must implement the emqx_gateway_impl behaviour +-behaviour(emqx_gateway_impl). + +%% callback for emqx_gateway_impl +-export([ + on_gateway_load/2, + on_gateway_update/3, + on_gateway_unload/2 +]). + -import( emqx_gateway_utils, [ @@ -30,33 +46,8 @@ ] ). -%% APIs --export([ - reg/0, - unreg/0 -]). - --export([ - on_gateway_load/2, - on_gateway_update/3, - on_gateway_unload/2 -]). - %%-------------------------------------------------------------------- -%% APIs -%%-------------------------------------------------------------------- - --spec reg() -> ok | {error, any()}. -reg() -> - RegistryOptions = [{cbkmod, ?MODULE}], - emqx_gateway_registry:reg(stomp, RegistryOptions). - --spec unreg() -> ok | {error, any()}. -unreg() -> - emqx_gateway_registry:unreg(stomp). - -%%-------------------------------------------------------------------- -%% emqx_gateway_registry callbacks +%% emqx_gateway_impl callbacks %%-------------------------------------------------------------------- on_gateway_load( diff --git a/apps/emqx_gateway/src/stomp/emqx_stomp_channel.erl b/apps/emqx_stomp/src/emqx_stomp_channel.erl similarity index 99% rename from apps/emqx_gateway/src/stomp/emqx_stomp_channel.erl rename to apps/emqx_stomp/src/emqx_stomp_channel.erl index b95bb827c..13b70348a 100644 --- a/apps/emqx_gateway/src/stomp/emqx_stomp_channel.erl +++ b/apps/emqx_stomp/src/emqx_stomp_channel.erl @@ -18,7 +18,7 @@ -behaviour(emqx_gateway_channel). --include("src/stomp/include/emqx_stomp.hrl"). +-include("emqx_stomp.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/logger.hrl"). diff --git a/apps/emqx_gateway/src/stomp/emqx_stomp_frame.erl b/apps/emqx_stomp/src/emqx_stomp_frame.erl similarity index 99% rename from apps/emqx_gateway/src/stomp/emqx_stomp_frame.erl rename to apps/emqx_stomp/src/emqx_stomp_frame.erl index 47e045412..4913d6b2a 100644 --- a/apps/emqx_gateway/src/stomp/emqx_stomp_frame.erl +++ b/apps/emqx_stomp/src/emqx_stomp_frame.erl @@ -70,7 +70,7 @@ -behaviour(emqx_gateway_frame). --include("src/stomp/include/emqx_stomp.hrl"). +-include("emqx_stomp.hrl"). -export([ initial_parse_state/1, diff --git a/apps/emqx_gateway/src/stomp/emqx_stomp_heartbeat.erl b/apps/emqx_stomp/src/emqx_stomp_heartbeat.erl similarity index 98% rename from apps/emqx_gateway/src/stomp/emqx_stomp_heartbeat.erl rename to apps/emqx_stomp/src/emqx_stomp_heartbeat.erl index 88720c513..f5ed99623 100644 --- a/apps/emqx_gateway/src/stomp/emqx_stomp_heartbeat.erl +++ b/apps/emqx_stomp/src/emqx_stomp_heartbeat.erl @@ -17,7 +17,7 @@ %% @doc Stomp heartbeat. -module(emqx_stomp_heartbeat). --include("src/stomp/include/emqx_stomp.hrl"). +-include("emqx_stomp.hrl"). -export([ init/1, diff --git a/apps/emqx_stomp/src/emqx_stomp_schema.erl b/apps/emqx_stomp/src/emqx_stomp_schema.erl new file mode 100644 index 000000000..b1c6a92e2 --- /dev/null +++ b/apps/emqx_stomp/src/emqx_stomp_schema.erl @@ -0,0 +1,80 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_stomp_schema). + +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("typerefl/include/types.hrl"). + +%% config schema provides +-export([fields/1, desc/1]). + +fields(stomp) -> + [ + {frame, sc(ref(stomp_frame))}, + {mountpoint, emqx_gateway_schema:mountpoint()}, + {listeners, sc(ref(emqx_gateway_schema, tcp_listeners), #{desc => ?DESC(tcp_listeners)})} + ] ++ emqx_gateway_schema:gateway_common_options(); +fields(stomp_frame) -> + [ + {max_headers, + sc( + non_neg_integer(), + #{ + default => 10, + desc => ?DESC(stom_frame_max_headers) + } + )}, + {max_headers_length, + sc( + non_neg_integer(), + #{ + default => 1024, + desc => ?DESC(stomp_frame_max_headers_length) + } + )}, + {max_body_length, + sc( + integer(), + #{ + default => 65536, + desc => ?DESC(stom_frame_max_body_length) + } + )} + ]. + +desc(stomp) -> + "The STOMP protocol gateway provides EMQX with the ability to access STOMP\n" + "(Simple (or Streaming) Text Orientated Messaging Protocol) protocol."; +desc(stomp_frame) -> + "Size limits for the STOMP frames."; +desc(_) -> + undefined. + +%%-------------------------------------------------------------------- +%% internal functions + +sc(Type) -> + sc(Type, #{}). + +sc(Type, Meta) -> + hoconsc:mk(Type, Meta). + +ref(StructName) -> + ref(?MODULE, StructName). + +ref(Mod, Field) -> + hoconsc:ref(Mod, Field). diff --git a/rebar.config.erl b/rebar.config.erl index 98cd30570..47bf925d4 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -389,6 +389,8 @@ relx_apps(ReleaseType, Edition) -> emqx_authz, emqx_auto_subscribe, emqx_gateway, + emqx_stomp, + emqx_mqttsn, emqx_exhook, emqx_bridge, emqx_rule_engine, From 786f03095820a606198ac30cb4c60268e6d20c71 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 30 Mar 2023 11:50:41 +0800 Subject: [PATCH 148/196] test: move tests into splited gateway dirs --- apps/{emqx_gateway => emqx_mqttsn}/test/broadcast_test.py | 0 .../test/emqx_sn_frame_SUITE.erl | 6 +++--- .../test/emqx_sn_protocol_SUITE.erl | 5 +++-- .../test/emqx_sn_registry_SUITE.erl | 4 ++-- .../test/intergration_test/Makefile | 0 .../test/intergration_test/README.md | 0 .../test/intergration_test/add_emqx_sn_to_project.py | 0 .../test/intergration_test/client/case1_qos0pub.c | 0 .../test/intergration_test/client/case1_qos0sub.c | 0 .../test/intergration_test/client/case2_qos0pub.c | 0 .../test/intergration_test/client/case2_qos0sub.c | 0 .../test/intergration_test/client/case3_qos0pub.c | 0 .../test/intergration_test/client/case3_qos0sub.c | 0 .../test/intergration_test/client/case4_qos3pub.c | 0 .../test/intergration_test/client/case4_qos3sub.c | 0 .../test/intergration_test/client/case5_qos3pub.c | 0 .../test/intergration_test/client/case5_qos3sub.c | 0 .../test/intergration_test/client/case6_sleep.c | 0 .../test/intergration_test/client/case7_double_connect.c | 0 .../test/intergration_test/client/int_test_result.c | 0 .../test/intergration_test/client/int_test_result.h | 0 .../test/intergration_test/disable_qos3.py | 0 .../test/intergration_test/enable_qos3.py | 0 .../test/props/emqx_sn_proper_types.erl | 2 +- .../test/props/prop_emqx_sn_frame.erl | 6 +++--- apps/{emqx_gateway => emqx_stomp}/test/emqx_stomp_SUITE.erl | 3 ++- .../test/emqx_stomp_heartbeat_SUITE.erl | 0 27 files changed, 14 insertions(+), 12 deletions(-) rename apps/{emqx_gateway => emqx_mqttsn}/test/broadcast_test.py (100%) rename apps/{emqx_gateway => emqx_mqttsn}/test/emqx_sn_frame_SUITE.erl (97%) rename apps/{emqx_gateway => emqx_mqttsn}/test/emqx_sn_protocol_SUITE.erl (99%) rename apps/{emqx_gateway => emqx_mqttsn}/test/emqx_sn_registry_SUITE.erl (98%) rename apps/{emqx_gateway => emqx_mqttsn}/test/intergration_test/Makefile (100%) rename apps/{emqx_gateway => emqx_mqttsn}/test/intergration_test/README.md (100%) rename apps/{emqx_gateway => emqx_mqttsn}/test/intergration_test/add_emqx_sn_to_project.py (100%) rename apps/{emqx_gateway => emqx_mqttsn}/test/intergration_test/client/case1_qos0pub.c (100%) rename apps/{emqx_gateway => emqx_mqttsn}/test/intergration_test/client/case1_qos0sub.c (100%) rename apps/{emqx_gateway => emqx_mqttsn}/test/intergration_test/client/case2_qos0pub.c (100%) rename apps/{emqx_gateway => emqx_mqttsn}/test/intergration_test/client/case2_qos0sub.c (100%) rename apps/{emqx_gateway => emqx_mqttsn}/test/intergration_test/client/case3_qos0pub.c (100%) rename apps/{emqx_gateway => emqx_mqttsn}/test/intergration_test/client/case3_qos0sub.c (100%) rename apps/{emqx_gateway => emqx_mqttsn}/test/intergration_test/client/case4_qos3pub.c (100%) rename apps/{emqx_gateway => emqx_mqttsn}/test/intergration_test/client/case4_qos3sub.c (100%) rename apps/{emqx_gateway => emqx_mqttsn}/test/intergration_test/client/case5_qos3pub.c (100%) rename apps/{emqx_gateway => emqx_mqttsn}/test/intergration_test/client/case5_qos3sub.c (100%) rename apps/{emqx_gateway => emqx_mqttsn}/test/intergration_test/client/case6_sleep.c (100%) rename apps/{emqx_gateway => emqx_mqttsn}/test/intergration_test/client/case7_double_connect.c (100%) rename apps/{emqx_gateway => emqx_mqttsn}/test/intergration_test/client/int_test_result.c (100%) rename apps/{emqx_gateway => emqx_mqttsn}/test/intergration_test/client/int_test_result.h (100%) rename apps/{emqx_gateway => emqx_mqttsn}/test/intergration_test/disable_qos3.py (100%) rename apps/{emqx_gateway => emqx_mqttsn}/test/intergration_test/enable_qos3.py (100%) rename apps/{emqx_gateway => emqx_mqttsn}/test/props/emqx_sn_proper_types.erl (99%) rename apps/{emqx_gateway => emqx_mqttsn}/test/props/prop_emqx_sn_frame.erl (94%) rename apps/{emqx_gateway => emqx_stomp}/test/emqx_stomp_SUITE.erl (99%) rename apps/{emqx_gateway => emqx_stomp}/test/emqx_stomp_heartbeat_SUITE.erl (100%) diff --git a/apps/emqx_gateway/test/broadcast_test.py b/apps/emqx_mqttsn/test/broadcast_test.py similarity index 100% rename from apps/emqx_gateway/test/broadcast_test.py rename to apps/emqx_mqttsn/test/broadcast_test.py diff --git a/apps/emqx_gateway/test/emqx_sn_frame_SUITE.erl b/apps/emqx_mqttsn/test/emqx_sn_frame_SUITE.erl similarity index 97% rename from apps/emqx_gateway/test/emqx_sn_frame_SUITE.erl rename to apps/emqx_mqttsn/test/emqx_sn_frame_SUITE.erl index aa3fed707..86cc0cf7e 100644 --- a/apps/emqx_gateway/test/emqx_sn_frame_SUITE.erl +++ b/apps/emqx_mqttsn/test/emqx_sn_frame_SUITE.erl @@ -19,7 +19,7 @@ -compile(export_all). -compile(nowarn_export_all). --include("src/mqttsn/include/emqx_sn.hrl"). +-include("emqx_mqttsn.hrl"). -include_lib("eunit/include/eunit.hrl"). %%-------------------------------------------------------------------- @@ -30,11 +30,11 @@ all() -> emqx_common_test_helpers:all(?MODULE). parse(D) -> - {ok, P, _Rest, _State} = emqx_sn_frame:parse(D, #{}), + {ok, P, _Rest, _State} = emqx_mqttsn_frame:parse(D, #{}), P. serialize_pkt(P) -> - emqx_sn_frame:serialize_pkt(P, #{}). + emqx_mqttsn_frame:serialize_pkt(P, #{}). %%-------------------------------------------------------------------- %% Test cases diff --git a/apps/emqx_gateway/test/emqx_sn_protocol_SUITE.erl b/apps/emqx_mqttsn/test/emqx_sn_protocol_SUITE.erl similarity index 99% rename from apps/emqx_gateway/test/emqx_sn_protocol_SUITE.erl rename to apps/emqx_mqttsn/test/emqx_sn_protocol_SUITE.erl index adc1e7382..0e04ec67a 100644 --- a/apps/emqx_gateway/test/emqx_sn_protocol_SUITE.erl +++ b/apps/emqx_mqttsn/test/emqx_sn_protocol_SUITE.erl @@ -27,7 +27,7 @@ ] ). --include("src/mqttsn/include/emqx_sn.hrl"). +-include("emqx_mqttsn.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). @@ -97,6 +97,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> + application:load(emqx_mqttsn), ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_authn, emqx_gateway]), Config. @@ -270,7 +271,7 @@ t_subscribe_case03(_) -> %% In this case We use predefined topic name to register and subscribe, %% and expect to receive the corresponding predefined topic id but not a new %% generated topic id from broker. We design this case to illustrate -%% emqx_sn_gateway's compatibility of dealing with predefined and normal +%% MQTT-SN Gateway's compatibility of dealing with predefined and normal %% topics. %% %% Once we give more restrictions to different topic id type, this case diff --git a/apps/emqx_gateway/test/emqx_sn_registry_SUITE.erl b/apps/emqx_mqttsn/test/emqx_sn_registry_SUITE.erl similarity index 98% rename from apps/emqx_gateway/test/emqx_sn_registry_SUITE.erl rename to apps/emqx_mqttsn/test/emqx_sn_registry_SUITE.erl index 739255e71..4d89a802d 100644 --- a/apps/emqx_gateway/test/emqx_sn_registry_SUITE.erl +++ b/apps/emqx_mqttsn/test/emqx_sn_registry_SUITE.erl @@ -21,7 +21,7 @@ -include_lib("eunit/include/eunit.hrl"). --define(REGISTRY, emqx_sn_registry). +-define(REGISTRY, emqx_mqttsn_registry). -define(MAX_PREDEF_ID, 2). -define(PREDEF_TOPICS, [ #{id => 1, topic => <<"/predefined/topic/name/hello">>}, @@ -66,7 +66,7 @@ t_register(Config) -> ?assertEqual(<<"Topic2">>, ?REGISTRY:lookup_topic(Reg, <<"ClientId">>, ?MAX_PREDEF_ID + 2)), ?assertEqual(?MAX_PREDEF_ID + 1, ?REGISTRY:lookup_topic_id(Reg, <<"ClientId">>, <<"Topic1">>)), ?assertEqual(?MAX_PREDEF_ID + 2, ?REGISTRY:lookup_topic_id(Reg, <<"ClientId">>, <<"Topic2">>)), - emqx_sn_registry:unregister_topic(Reg, <<"ClientId">>), + emqx_mqttsn_registry:unregister_topic(Reg, <<"ClientId">>), ?assertEqual(undefined, ?REGISTRY:lookup_topic(Reg, <<"ClientId">>, ?MAX_PREDEF_ID + 1)), ?assertEqual(undefined, ?REGISTRY:lookup_topic(Reg, <<"ClientId">>, ?MAX_PREDEF_ID + 2)), ?assertEqual(undefined, ?REGISTRY:lookup_topic_id(Reg, <<"ClientId">>, <<"Topic1">>)), diff --git a/apps/emqx_gateway/test/intergration_test/Makefile b/apps/emqx_mqttsn/test/intergration_test/Makefile similarity index 100% rename from apps/emqx_gateway/test/intergration_test/Makefile rename to apps/emqx_mqttsn/test/intergration_test/Makefile diff --git a/apps/emqx_gateway/test/intergration_test/README.md b/apps/emqx_mqttsn/test/intergration_test/README.md similarity index 100% rename from apps/emqx_gateway/test/intergration_test/README.md rename to apps/emqx_mqttsn/test/intergration_test/README.md diff --git a/apps/emqx_gateway/test/intergration_test/add_emqx_sn_to_project.py b/apps/emqx_mqttsn/test/intergration_test/add_emqx_sn_to_project.py similarity index 100% rename from apps/emqx_gateway/test/intergration_test/add_emqx_sn_to_project.py rename to apps/emqx_mqttsn/test/intergration_test/add_emqx_sn_to_project.py diff --git a/apps/emqx_gateway/test/intergration_test/client/case1_qos0pub.c b/apps/emqx_mqttsn/test/intergration_test/client/case1_qos0pub.c similarity index 100% rename from apps/emqx_gateway/test/intergration_test/client/case1_qos0pub.c rename to apps/emqx_mqttsn/test/intergration_test/client/case1_qos0pub.c diff --git a/apps/emqx_gateway/test/intergration_test/client/case1_qos0sub.c b/apps/emqx_mqttsn/test/intergration_test/client/case1_qos0sub.c similarity index 100% rename from apps/emqx_gateway/test/intergration_test/client/case1_qos0sub.c rename to apps/emqx_mqttsn/test/intergration_test/client/case1_qos0sub.c diff --git a/apps/emqx_gateway/test/intergration_test/client/case2_qos0pub.c b/apps/emqx_mqttsn/test/intergration_test/client/case2_qos0pub.c similarity index 100% rename from apps/emqx_gateway/test/intergration_test/client/case2_qos0pub.c rename to apps/emqx_mqttsn/test/intergration_test/client/case2_qos0pub.c diff --git a/apps/emqx_gateway/test/intergration_test/client/case2_qos0sub.c b/apps/emqx_mqttsn/test/intergration_test/client/case2_qos0sub.c similarity index 100% rename from apps/emqx_gateway/test/intergration_test/client/case2_qos0sub.c rename to apps/emqx_mqttsn/test/intergration_test/client/case2_qos0sub.c diff --git a/apps/emqx_gateway/test/intergration_test/client/case3_qos0pub.c b/apps/emqx_mqttsn/test/intergration_test/client/case3_qos0pub.c similarity index 100% rename from apps/emqx_gateway/test/intergration_test/client/case3_qos0pub.c rename to apps/emqx_mqttsn/test/intergration_test/client/case3_qos0pub.c diff --git a/apps/emqx_gateway/test/intergration_test/client/case3_qos0sub.c b/apps/emqx_mqttsn/test/intergration_test/client/case3_qos0sub.c similarity index 100% rename from apps/emqx_gateway/test/intergration_test/client/case3_qos0sub.c rename to apps/emqx_mqttsn/test/intergration_test/client/case3_qos0sub.c diff --git a/apps/emqx_gateway/test/intergration_test/client/case4_qos3pub.c b/apps/emqx_mqttsn/test/intergration_test/client/case4_qos3pub.c similarity index 100% rename from apps/emqx_gateway/test/intergration_test/client/case4_qos3pub.c rename to apps/emqx_mqttsn/test/intergration_test/client/case4_qos3pub.c diff --git a/apps/emqx_gateway/test/intergration_test/client/case4_qos3sub.c b/apps/emqx_mqttsn/test/intergration_test/client/case4_qos3sub.c similarity index 100% rename from apps/emqx_gateway/test/intergration_test/client/case4_qos3sub.c rename to apps/emqx_mqttsn/test/intergration_test/client/case4_qos3sub.c diff --git a/apps/emqx_gateway/test/intergration_test/client/case5_qos3pub.c b/apps/emqx_mqttsn/test/intergration_test/client/case5_qos3pub.c similarity index 100% rename from apps/emqx_gateway/test/intergration_test/client/case5_qos3pub.c rename to apps/emqx_mqttsn/test/intergration_test/client/case5_qos3pub.c diff --git a/apps/emqx_gateway/test/intergration_test/client/case5_qos3sub.c b/apps/emqx_mqttsn/test/intergration_test/client/case5_qos3sub.c similarity index 100% rename from apps/emqx_gateway/test/intergration_test/client/case5_qos3sub.c rename to apps/emqx_mqttsn/test/intergration_test/client/case5_qos3sub.c diff --git a/apps/emqx_gateway/test/intergration_test/client/case6_sleep.c b/apps/emqx_mqttsn/test/intergration_test/client/case6_sleep.c similarity index 100% rename from apps/emqx_gateway/test/intergration_test/client/case6_sleep.c rename to apps/emqx_mqttsn/test/intergration_test/client/case6_sleep.c diff --git a/apps/emqx_gateway/test/intergration_test/client/case7_double_connect.c b/apps/emqx_mqttsn/test/intergration_test/client/case7_double_connect.c similarity index 100% rename from apps/emqx_gateway/test/intergration_test/client/case7_double_connect.c rename to apps/emqx_mqttsn/test/intergration_test/client/case7_double_connect.c diff --git a/apps/emqx_gateway/test/intergration_test/client/int_test_result.c b/apps/emqx_mqttsn/test/intergration_test/client/int_test_result.c similarity index 100% rename from apps/emqx_gateway/test/intergration_test/client/int_test_result.c rename to apps/emqx_mqttsn/test/intergration_test/client/int_test_result.c diff --git a/apps/emqx_gateway/test/intergration_test/client/int_test_result.h b/apps/emqx_mqttsn/test/intergration_test/client/int_test_result.h similarity index 100% rename from apps/emqx_gateway/test/intergration_test/client/int_test_result.h rename to apps/emqx_mqttsn/test/intergration_test/client/int_test_result.h diff --git a/apps/emqx_gateway/test/intergration_test/disable_qos3.py b/apps/emqx_mqttsn/test/intergration_test/disable_qos3.py similarity index 100% rename from apps/emqx_gateway/test/intergration_test/disable_qos3.py rename to apps/emqx_mqttsn/test/intergration_test/disable_qos3.py diff --git a/apps/emqx_gateway/test/intergration_test/enable_qos3.py b/apps/emqx_mqttsn/test/intergration_test/enable_qos3.py similarity index 100% rename from apps/emqx_gateway/test/intergration_test/enable_qos3.py rename to apps/emqx_mqttsn/test/intergration_test/enable_qos3.py diff --git a/apps/emqx_gateway/test/props/emqx_sn_proper_types.erl b/apps/emqx_mqttsn/test/props/emqx_sn_proper_types.erl similarity index 99% rename from apps/emqx_gateway/test/props/emqx_sn_proper_types.erl rename to apps/emqx_mqttsn/test/props/emqx_sn_proper_types.erl index 2869a8958..70b13ef8f 100644 --- a/apps/emqx_gateway/test/props/emqx_sn_proper_types.erl +++ b/apps/emqx_mqttsn/test/props/emqx_sn_proper_types.erl @@ -16,7 +16,7 @@ -module(emqx_sn_proper_types). --include("src/mqttsn/include/emqx_sn.hrl"). +-include("emqx_mqttsn.hrl"). -include_lib("proper/include/proper.hrl"). -compile({no_auto_import, [register/1]}). diff --git a/apps/emqx_gateway/test/props/prop_emqx_sn_frame.erl b/apps/emqx_mqttsn/test/props/prop_emqx_sn_frame.erl similarity index 94% rename from apps/emqx_gateway/test/props/prop_emqx_sn_frame.erl rename to apps/emqx_mqttsn/test/props/prop_emqx_sn_frame.erl index f2dfbb8e9..0abe2485c 100644 --- a/apps/emqx_gateway/test/props/prop_emqx_sn_frame.erl +++ b/apps/emqx_mqttsn/test/props/prop_emqx_sn_frame.erl @@ -16,7 +16,7 @@ -module(prop_emqx_sn_frame). --include("src/mqttsn/include/emqx_sn.hrl"). +-include("emqx_mqttsn.hrl"). -include_lib("proper/include/proper.hrl"). -compile({no_auto_import, [register/1]}). @@ -32,11 +32,11 @@ ). parse(D) -> - {ok, P, _Rest, _State} = emqx_sn_frame:parse(D, #{}), + {ok, P, _Rest, _State} = emqx_mqttsn_frame:parse(D, #{}), P. serialize(P) -> - emqx_sn_frame:serialize_pkt(P, #{}). + emqx_mqttsn_frame:serialize_pkt(P, #{}). %%-------------------------------------------------------------------- %% Properties diff --git a/apps/emqx_gateway/test/emqx_stomp_SUITE.erl b/apps/emqx_stomp/test/emqx_stomp_SUITE.erl similarity index 99% rename from apps/emqx_gateway/test/emqx_stomp_SUITE.erl rename to apps/emqx_stomp/test/emqx_stomp_SUITE.erl index 2cf245ce2..fed7f5163 100644 --- a/apps/emqx_gateway/test/emqx_stomp_SUITE.erl +++ b/apps/emqx_stomp/test/emqx_stomp_SUITE.erl @@ -17,7 +17,7 @@ -module(emqx_stomp_SUITE). -include_lib("eunit/include/eunit.hrl"). --include("src/stomp/include/emqx_stomp.hrl"). +-include("emqx_stomp.hrl"). -compile(export_all). -compile(nowarn_export_all). @@ -53,6 +53,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). %%-------------------------------------------------------------------- init_per_suite(Cfg) -> + application:load(emqx_stomp), ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), emqx_mgmt_api_test_util:init_suite([emqx_authn, emqx_gateway]), Cfg. diff --git a/apps/emqx_gateway/test/emqx_stomp_heartbeat_SUITE.erl b/apps/emqx_stomp/test/emqx_stomp_heartbeat_SUITE.erl similarity index 100% rename from apps/emqx_gateway/test/emqx_stomp_heartbeat_SUITE.erl rename to apps/emqx_stomp/test/emqx_stomp_heartbeat_SUITE.erl From 0b6c5c4c91b49b8bb7af5e2e8809219b8527114f Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 30 Mar 2023 15:21:42 +0800 Subject: [PATCH 149/196] refactor: split out emqx_coap application --- apps/emqx_coap/.gitignore | 19 ++ apps/emqx_coap/LICENSE | 191 ++++++++++++++++++ .../src/coap => emqx_coap}/README.md | 0 .../src/coap => emqx_coap}/doc/flow.png | Bin .../coap => emqx_coap}/doc/shared_state.png | Bin .../src/coap => emqx_coap}/doc/transport.png | Bin .../i18n/emqx_coap_api_i18n.conf | 0 apps/emqx_coap/i18n/emqx_coap_schema.conf | 80 ++++++++ .../coap => emqx_coap}/include/emqx_coap.hrl | 0 apps/emqx_coap/rebar.config | 2 + apps/emqx_coap/src/emqx_coap.app.src | 10 + .../src/emqx_coap.erl} | 47 ++--- .../coap => emqx_coap/src}/emqx_coap_api.erl | 2 +- .../src}/emqx_coap_channel.erl | 2 +- .../src}/emqx_coap_frame.erl | 2 +- .../src}/emqx_coap_medium.erl | 2 +- .../src}/emqx_coap_message.erl | 2 +- .../src}/emqx_coap_mqtt_handler.erl | 2 +- .../src}/emqx_coap_observe_res.erl | 0 .../src}/emqx_coap_pubsub_handler.erl | 2 +- apps/emqx_coap/src/emqx_coap_schema.erl | 95 +++++++++ .../src}/emqx_coap_session.erl | 2 +- .../coap => emqx_coap/src}/emqx_coap_tm.erl | 2 +- .../src}/emqx_coap_transport.erl | 2 +- .../test/emqx_coap_SUITE.erl | 1 + .../test/emqx_coap_api_SUITE.erl | 3 +- .../i18n/emqx_gateway_schema_i18n.conf | 77 ------- apps/emqx_gateway/src/emqx_gateway_app.erl | 5 - apps/emqx_gateway/src/emqx_gateway_schema.erl | 65 +----- apps/emqx_mqttsn/src/emqx_mqttsn.app.src | 20 +- apps/emqx_stomp/src/emqx_stomp.app.src | 2 +- apps/emqx_stomp/src/emqx_stomp.erl | 2 +- mix.exs | 3 + rebar.config.erl | 1 + 34 files changed, 448 insertions(+), 195 deletions(-) create mode 100644 apps/emqx_coap/.gitignore create mode 100644 apps/emqx_coap/LICENSE rename apps/{emqx_gateway/src/coap => emqx_coap}/README.md (100%) rename apps/{emqx_gateway/src/coap => emqx_coap}/doc/flow.png (100%) rename apps/{emqx_gateway/src/coap => emqx_coap}/doc/shared_state.png (100%) rename apps/{emqx_gateway/src/coap => emqx_coap}/doc/transport.png (100%) rename apps/{emqx_gateway => emqx_coap}/i18n/emqx_coap_api_i18n.conf (100%) create mode 100644 apps/emqx_coap/i18n/emqx_coap_schema.conf rename apps/{emqx_gateway/src/coap => emqx_coap}/include/emqx_coap.hrl (100%) create mode 100644 apps/emqx_coap/rebar.config create mode 100644 apps/emqx_coap/src/emqx_coap.app.src rename apps/{emqx_gateway/src/coap/emqx_coap_impl.erl => emqx_coap/src/emqx_coap.erl} (86%) rename apps/{emqx_gateway/src/coap => emqx_coap/src}/emqx_coap_api.erl (99%) rename apps/{emqx_gateway/src/coap => emqx_coap/src}/emqx_coap_channel.erl (99%) rename apps/{emqx_gateway/src/coap => emqx_coap/src}/emqx_coap_frame.erl (99%) rename apps/{emqx_gateway/src/coap => emqx_coap/src}/emqx_coap_medium.erl (98%) rename apps/{emqx_gateway/src/coap => emqx_coap/src}/emqx_coap_message.erl (99%) rename apps/{emqx_gateway/src/coap/handler => emqx_coap/src}/emqx_coap_mqtt_handler.erl (96%) rename apps/{emqx_gateway/src/coap => emqx_coap/src}/emqx_coap_observe_res.erl (100%) rename apps/{emqx_gateway/src/coap/handler => emqx_coap/src}/emqx_coap_pubsub_handler.erl (99%) create mode 100644 apps/emqx_coap/src/emqx_coap_schema.erl rename apps/{emqx_gateway/src/coap => emqx_coap/src}/emqx_coap_session.erl (99%) rename apps/{emqx_gateway/src/coap => emqx_coap/src}/emqx_coap_tm.erl (99%) rename apps/{emqx_gateway/src/coap => emqx_coap/src}/emqx_coap_transport.erl (99%) rename apps/{emqx_gateway => emqx_coap}/test/emqx_coap_SUITE.erl (99%) rename apps/{emqx_gateway => emqx_coap}/test/emqx_coap_api_SUITE.erl (99%) diff --git a/apps/emqx_coap/.gitignore b/apps/emqx_coap/.gitignore new file mode 100644 index 000000000..f1c455451 --- /dev/null +++ b/apps/emqx_coap/.gitignore @@ -0,0 +1,19 @@ +.rebar3 +_* +.eunit +*.o +*.beam +*.plt +*.swp +*.swo +.erlang.cookie +ebin +log +erl_crash.dump +.rebar +logs +_build +.idea +*.iml +rebar3.crashdump +*~ diff --git a/apps/emqx_coap/LICENSE b/apps/emqx_coap/LICENSE new file mode 100644 index 000000000..5a5418f0f --- /dev/null +++ b/apps/emqx_coap/LICENSE @@ -0,0 +1,191 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2023, JianBo He . + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/apps/emqx_gateway/src/coap/README.md b/apps/emqx_coap/README.md similarity index 100% rename from apps/emqx_gateway/src/coap/README.md rename to apps/emqx_coap/README.md diff --git a/apps/emqx_gateway/src/coap/doc/flow.png b/apps/emqx_coap/doc/flow.png similarity index 100% rename from apps/emqx_gateway/src/coap/doc/flow.png rename to apps/emqx_coap/doc/flow.png diff --git a/apps/emqx_gateway/src/coap/doc/shared_state.png b/apps/emqx_coap/doc/shared_state.png similarity index 100% rename from apps/emqx_gateway/src/coap/doc/shared_state.png rename to apps/emqx_coap/doc/shared_state.png diff --git a/apps/emqx_gateway/src/coap/doc/transport.png b/apps/emqx_coap/doc/transport.png similarity index 100% rename from apps/emqx_gateway/src/coap/doc/transport.png rename to apps/emqx_coap/doc/transport.png diff --git a/apps/emqx_gateway/i18n/emqx_coap_api_i18n.conf b/apps/emqx_coap/i18n/emqx_coap_api_i18n.conf similarity index 100% rename from apps/emqx_gateway/i18n/emqx_coap_api_i18n.conf rename to apps/emqx_coap/i18n/emqx_coap_api_i18n.conf diff --git a/apps/emqx_coap/i18n/emqx_coap_schema.conf b/apps/emqx_coap/i18n/emqx_coap_schema.conf new file mode 100644 index 000000000..1e6452e49 --- /dev/null +++ b/apps/emqx_coap/i18n/emqx_coap_schema.conf @@ -0,0 +1,80 @@ +emqx_coap_schema { + coap { + desc { + en: """The CoAP Gateway configuration. +This gateway is implemented based on RFC-7252 and https://core-wg.github.io/coap-pubsub/draft-ietf-core-pubsub.html""" + zh: """CoAP 网关配置。 +该网关的实现基于 RFC-7252 和 https://core-wg.github.io/coap-pubsub/draft-ietf-core-pubsub.html""" + } + } + + coap_heartbeat { + desc { + en: """The gateway server required minimum heartbeat interval. +When connection mode is enabled, this parameter is used to set the minimum heartbeat interval for the connection to be alive""" + zh: """CoAP 网关要求客户端的最小心跳间隔时间。 +当 connection_required 开启后，该参数用于检查客户端连接是否存活""" + } + } + + coap_connection_required { + desc { + en: """Enable or disable connection mode. +Connection mode is a feature of non-standard protocols. When connection mode is enabled, it is necessary to maintain the creation, authentication and alive of connection resources""" + zh: """是否开启连接模式。 +连接模式是非标准协议的功能。它维护 CoAP 客户端上线、认证、和连接状态的保持""" + } + } + + coap_notify_type { + desc { + en: """The Notification Message will be delivered to the CoAP client if a new message received on an observed topic. +The type of delivered coap message can be set to:
+ - non: Non-confirmable;
+ - con: Confirmable;
+ - qos: Mapping from QoS type of received message, QoS0 -> non, QoS1,2 -> con""" + zh: """投递给 CoAP 客户端的通知消息类型。当客户端 Observe 一个资源（或订阅某个主题）时，网关会向客户端推送新产生的消息。其消息类型可设置为：
+ - non: 不需要客户端返回确认消息;
+ - con: 需要客户端返回一个确认消息;
+ - qos: 取决于消息的 QoS 等级; QoS 0 会以 `non` 类型下发，QoS 1/2 会以 `con` 类型下发""" + } + } + + coap_subscribe_qos { + desc { + en: """The Default QoS Level indicator for subscribe request. +This option specifies the QoS level for the CoAP Client when establishing a subscription membership, if the subscribe request is not carried `qos` option. The indicator can be set to:
+ - qos0, qos1, qos2: Fixed default QoS level
+ - coap: Dynamic QoS level by the message type of subscribe request
+ * qos0: If the subscribe request is non-confirmable
+ * qos1: If the subscribe request is confirmable""" + + zh: """客户端订阅请求的默认 QoS 等级。 +当 CoAP 客户端发起订阅请求时，如果未携带 `qos` 参数则会使用该默认值。默认值可设置为：
+ - qos0、 qos1、qos2: 设置为固定的 QoS 等级
+ - coap: 依据订阅操作的 CoAP 报文类型来动态决定
+ * 当订阅请求为 `non-confirmable` 类型时，取值为 qos0
+ * 当订阅请求为 `confirmable` 类型时，取值为 qos1""" + } + } + + coap_publish_qos { + desc { + en: """The Default QoS Level indicator for publish request. +This option specifies the QoS level for the CoAP Client when publishing a message to EMQX PUB/SUB system, if the publish request is not carried `qos` option. The indicator can be set to:
+ - qos0, qos1, qos2: Fixed default QoS level
+ - coap: Dynamic QoS level by the message type of publish request
+ * qos0: If the publish request is non-confirmable
+ * qos1: If the publish request is confirmable""" + + zh: """客户端发布请求的默认 QoS 等级。 +当 CoAP 客户端发起发布请求时，如果未携带 `qos` 参数则会使用该默认值。默认值可设置为：
+ - qos0、qos1、qos2: 设置为固定的 QoS 等级
+ - coap: 依据发布操作的 CoAP 报文类型来动态决定
+ * 当发布请求为 `non-confirmable` 类型时，取值为 qos0
+ * 当发布请求为 `confirmable` 类型时，取值为 qos1""" + } + } + + +} diff --git a/apps/emqx_gateway/src/coap/include/emqx_coap.hrl b/apps/emqx_coap/include/emqx_coap.hrl similarity index 100% rename from apps/emqx_gateway/src/coap/include/emqx_coap.hrl rename to apps/emqx_coap/include/emqx_coap.hrl diff --git a/apps/emqx_coap/rebar.config b/apps/emqx_coap/rebar.config new file mode 100644 index 000000000..2656fd554 --- /dev/null +++ b/apps/emqx_coap/rebar.config @@ -0,0 +1,2 @@ +{erl_opts, [debug_info]}. +{deps, []}. diff --git a/apps/emqx_coap/src/emqx_coap.app.src b/apps/emqx_coap/src/emqx_coap.app.src new file mode 100644 index 000000000..55c9de59d --- /dev/null +++ b/apps/emqx_coap/src/emqx_coap.app.src @@ -0,0 +1,10 @@ +{application, emqx_coap, [ + {description, "CoAP Gateway"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [kernel, stdlib]}, + {env, []}, + {modules, []}, + {licenses, ["Apache 2.0"]}, + {links, []} +]}. diff --git a/apps/emqx_gateway/src/coap/emqx_coap_impl.erl b/apps/emqx_coap/src/emqx_coap.erl similarity index 86% rename from apps/emqx_gateway/src/coap/emqx_coap_impl.erl rename to apps/emqx_coap/src/emqx_coap.erl index bebcef237..d553349a4 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_impl.erl +++ b/apps/emqx_coap/src/emqx_coap.erl @@ -14,13 +14,29 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_coap_impl). - --behaviour(emqx_gateway_impl). +%% @doc The CoAP Gateway implement +-module(emqx_coap). -include_lib("emqx/include/logger.hrl"). -include_lib("emqx_gateway/include/emqx_gateway.hrl"). +%% define a gateway named stomp +-gateway(#{ + name => coap, + callback_module => ?MODULE, + config_schema_module => emqx_coap_schema +}). + +%% callback_module must implement the emqx_gateway_impl behaviour +-behaviour(emqx_gateway_impl). + +%% callback for emqx_gateway_impl +-export([ + on_gateway_load/2, + on_gateway_update/3, + on_gateway_unload/2 +]). + -import( emqx_gateway_utils, [ @@ -30,31 +46,8 @@ ] ). -%% APIs --export([ - reg/0, - unreg/0 -]). - --export([ - on_gateway_load/2, - on_gateway_update/3, - on_gateway_unload/2 -]). - %%-------------------------------------------------------------------- -%% APIs -%%-------------------------------------------------------------------- - -reg() -> - RegistryOptions = [{cbkmod, ?MODULE}], - emqx_gateway_registry:reg(coap, RegistryOptions). - -unreg() -> - emqx_gateway_registry:unreg(coap). - -%%-------------------------------------------------------------------- -%% emqx_gateway_registry callbacks +%% emqx_gateway_impl callbacks %%-------------------------------------------------------------------- on_gateway_load( diff --git a/apps/emqx_gateway/src/coap/emqx_coap_api.erl b/apps/emqx_coap/src/emqx_coap_api.erl similarity index 99% rename from apps/emqx_gateway/src/coap/emqx_coap_api.erl rename to apps/emqx_coap/src/emqx_coap_api.erl index 0f4c7a053..50ea9829a 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_api.erl +++ b/apps/emqx_coap/src/emqx_coap_api.erl @@ -18,10 +18,10 @@ -behaviour(minirest_api). +-include("emqx_coap.hrl"). -include_lib("hocon/include/hoconsc.hrl"). -include_lib("typerefl/include/types.hrl"). -include_lib("emqx/include/logger.hrl"). --include("src/coap/include/emqx_coap.hrl"). %% API -export([api_spec/0, paths/0, schema/1, namespace/0]). diff --git a/apps/emqx_gateway/src/coap/emqx_coap_channel.erl b/apps/emqx_coap/src/emqx_coap_channel.erl similarity index 99% rename from apps/emqx_gateway/src/coap/emqx_coap_channel.erl rename to apps/emqx_coap/src/emqx_coap_channel.erl index d6b8594b1..4cf362d9d 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_channel.erl +++ b/apps/emqx_coap/src/emqx_coap_channel.erl @@ -45,8 +45,8 @@ -export_type([channel/0]). +-include("emqx_coap.hrl"). -include_lib("emqx/include/logger.hrl"). --include("src/coap/include/emqx_coap.hrl"). -include_lib("emqx/include/emqx_authentication.hrl"). -define(AUTHN, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM). diff --git a/apps/emqx_gateway/src/coap/emqx_coap_frame.erl b/apps/emqx_coap/src/emqx_coap_frame.erl similarity index 99% rename from apps/emqx_gateway/src/coap/emqx_coap_frame.erl rename to apps/emqx_coap/src/emqx_coap_frame.erl index 4d2479d75..a05116b14 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_frame.erl +++ b/apps/emqx_coap/src/emqx_coap_frame.erl @@ -29,7 +29,7 @@ is_message/1 ]). --include("src/coap/include/emqx_coap.hrl"). +-include("emqx_coap.hrl"). -include_lib("emqx/include/types.hrl"). -define(VERSION, 1). diff --git a/apps/emqx_gateway/src/coap/emqx_coap_medium.erl b/apps/emqx_coap/src/emqx_coap_medium.erl similarity index 98% rename from apps/emqx_gateway/src/coap/emqx_coap_medium.erl rename to apps/emqx_coap/src/emqx_coap_medium.erl index 8f5028f25..b6bd8e764 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_medium.erl +++ b/apps/emqx_coap/src/emqx_coap_medium.erl @@ -20,7 +20,7 @@ -module(emqx_coap_medium). --include("src/coap/include/emqx_coap.hrl"). +-include("emqx_coap.hrl"). %% API -export([ diff --git a/apps/emqx_gateway/src/coap/emqx_coap_message.erl b/apps/emqx_coap/src/emqx_coap_message.erl similarity index 99% rename from apps/emqx_gateway/src/coap/emqx_coap_message.erl rename to apps/emqx_coap/src/emqx_coap_message.erl index 99c9e0840..ee17231a7 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_message.erl +++ b/apps/emqx_coap/src/emqx_coap_message.erl @@ -43,7 +43,7 @@ set_payload_block/3, set_payload_block/4 ]). --include("src/coap/include/emqx_coap.hrl"). +-include("emqx_coap.hrl"). request(Type, Method) -> request(Type, Method, <<>>, []). diff --git a/apps/emqx_gateway/src/coap/handler/emqx_coap_mqtt_handler.erl b/apps/emqx_coap/src/emqx_coap_mqtt_handler.erl similarity index 96% rename from apps/emqx_gateway/src/coap/handler/emqx_coap_mqtt_handler.erl rename to apps/emqx_coap/src/emqx_coap_mqtt_handler.erl index 59825a745..4bcf71b1a 100644 --- a/apps/emqx_gateway/src/coap/handler/emqx_coap_mqtt_handler.erl +++ b/apps/emqx_coap/src/emqx_coap_mqtt_handler.erl @@ -16,7 +16,7 @@ -module(emqx_coap_mqtt_handler). --include("src/coap/include/emqx_coap.hrl"). +-include("emqx_coap.hrl"). -export([handle_request/4]). -import(emqx_coap_message, [response/2, response/3]). diff --git a/apps/emqx_gateway/src/coap/emqx_coap_observe_res.erl b/apps/emqx_coap/src/emqx_coap_observe_res.erl similarity index 100% rename from apps/emqx_gateway/src/coap/emqx_coap_observe_res.erl rename to apps/emqx_coap/src/emqx_coap_observe_res.erl diff --git a/apps/emqx_gateway/src/coap/handler/emqx_coap_pubsub_handler.erl b/apps/emqx_coap/src/emqx_coap_pubsub_handler.erl similarity index 99% rename from apps/emqx_gateway/src/coap/handler/emqx_coap_pubsub_handler.erl rename to apps/emqx_coap/src/emqx_coap_pubsub_handler.erl index 5e14ba9e4..da1f5e0ef 100644 --- a/apps/emqx_gateway/src/coap/handler/emqx_coap_pubsub_handler.erl +++ b/apps/emqx_coap/src/emqx_coap_pubsub_handler.erl @@ -18,7 +18,7 @@ -module(emqx_coap_pubsub_handler). -include_lib("emqx/include/emqx_mqtt.hrl"). --include("src/coap/include/emqx_coap.hrl"). +-include("emqx_coap.hrl"). -export([handle_request/4]). diff --git a/apps/emqx_coap/src/emqx_coap_schema.erl b/apps/emqx_coap/src/emqx_coap_schema.erl new file mode 100644 index 000000000..b7ce88451 --- /dev/null +++ b/apps/emqx_coap/src/emqx_coap_schema.erl @@ -0,0 +1,95 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_coap_schema). + +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("typerefl/include/types.hrl"). + +-type duration() :: non_neg_integer(). + +-typerefl_from_string({duration/0, emqx_schema, to_duration}). + +-reflect_type([duration/0]). + +%% config schema provides +-export([fields/1, desc/1]). + +fields(coap) -> + [ + {heartbeat, + sc( + duration(), + #{ + default => <<"30s">>, + desc => ?DESC(coap_heartbeat) + } + )}, + {connection_required, + sc( + boolean(), + #{ + default => false, + desc => ?DESC(coap_connection_required) + } + )}, + {notify_type, + sc( + hoconsc:enum([non, con, qos]), + #{ + default => qos, + desc => ?DESC(coap_notify_type) + } + )}, + {subscribe_qos, + sc( + hoconsc:enum([qos0, qos1, qos2, coap]), + #{ + default => coap, + desc => ?DESC(coap_subscribe_qos) + } + )}, + {publish_qos, + sc( + hoconsc:enum([qos0, qos1, qos2, coap]), + #{ + default => coap, + desc => ?DESC(coap_publish_qos) + } + )}, + {mountpoint, emqx_gateway_schema:mountpoint()}, + {listeners, + sc( + ref(emqx_gateway_schema, udp_listeners), + #{desc => ?DESC(udp_listeners)} + )} + ] ++ emqx_gateway_schema:gateway_common_options(). + +desc(coap) -> + "The CoAP protocol gateway provides EMQX with the access capability of the CoAP protocol.\n" + "It allows publishing, subscribing, and receiving messages to EMQX in accordance\n" + "with a certain defined CoAP message format."; +desc(_) -> + undefined. + +%%-------------------------------------------------------------------- +%% helpers + +sc(Type, Meta) -> + hoconsc:mk(Type, Meta). + +ref(Mod, Field) -> + hoconsc:ref(Mod, Field). diff --git a/apps/emqx_gateway/src/coap/emqx_coap_session.erl b/apps/emqx_coap/src/emqx_coap_session.erl similarity index 99% rename from apps/emqx_gateway/src/coap/emqx_coap_session.erl rename to apps/emqx_coap/src/emqx_coap_session.erl index 253f34d4d..688defcbb 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_session.erl +++ b/apps/emqx_coap/src/emqx_coap_session.erl @@ -15,10 +15,10 @@ %%-------------------------------------------------------------------- -module(emqx_coap_session). +-include("emqx_coap.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/logger.hrl"). --include("src/coap/include/emqx_coap.hrl"). %% API -export([ diff --git a/apps/emqx_gateway/src/coap/emqx_coap_tm.erl b/apps/emqx_coap/src/emqx_coap_tm.erl similarity index 99% rename from apps/emqx_gateway/src/coap/emqx_coap_tm.erl rename to apps/emqx_coap/src/emqx_coap_tm.erl index 1a0004f8c..297cbca6b 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_tm.erl +++ b/apps/emqx_coap/src/emqx_coap_tm.erl @@ -29,8 +29,8 @@ -export_type([manager/0, event_result/1]). +-include("emqx_coap.hrl"). -include_lib("emqx/include/logger.hrl"). --include("src/coap/include/emqx_coap.hrl"). -type direction() :: in | out. diff --git a/apps/emqx_gateway/src/coap/emqx_coap_transport.erl b/apps/emqx_coap/src/emqx_coap_transport.erl similarity index 99% rename from apps/emqx_gateway/src/coap/emqx_coap_transport.erl rename to apps/emqx_coap/src/emqx_coap_transport.erl index 1e6c5238a..c58a8abbd 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_transport.erl +++ b/apps/emqx_coap/src/emqx_coap_transport.erl @@ -16,8 +16,8 @@ -module(emqx_coap_transport). +-include("emqx_coap.hrl"). -include_lib("emqx/include/logger.hrl"). --include("src/coap/include/emqx_coap.hrl"). -define(ACK_TIMEOUT, 2000). -define(ACK_RANDOM_FACTOR, 1000). diff --git a/apps/emqx_gateway/test/emqx_coap_SUITE.erl b/apps/emqx_coap/test/emqx_coap_SUITE.erl similarity index 99% rename from apps/emqx_gateway/test/emqx_coap_SUITE.erl rename to apps/emqx_coap/test/emqx_coap_SUITE.erl index db99c3df1..1d33e042a 100644 --- a/apps/emqx_gateway/test/emqx_coap_SUITE.erl +++ b/apps/emqx_coap/test/emqx_coap_SUITE.erl @@ -56,6 +56,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> + application:load(emqx_coap), ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), emqx_mgmt_api_test_util:init_suite([emqx_authn, emqx_gateway]), ok = meck:new(emqx_access_control, [passthrough, no_history, no_link]), diff --git a/apps/emqx_gateway/test/emqx_coap_api_SUITE.erl b/apps/emqx_coap/test/emqx_coap_api_SUITE.erl similarity index 99% rename from apps/emqx_gateway/test/emqx_coap_api_SUITE.erl rename to apps/emqx_coap/test/emqx_coap_api_SUITE.erl index 6c1354bc0..9c418ab57 100644 --- a/apps/emqx_gateway/test/emqx_coap_api_SUITE.erl +++ b/apps/emqx_coap/test/emqx_coap_api_SUITE.erl @@ -19,7 +19,7 @@ -compile(export_all). -compile(nowarn_export_all). --include("src/coap/include/emqx_coap.hrl"). +-include("emqx_coap.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). @@ -56,6 +56,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> + application:load(emqx_coap), ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), emqx_mgmt_api_test_util:init_suite([emqx_authn, emqx_gateway]), Config. diff --git a/apps/emqx_gateway/i18n/emqx_gateway_schema_i18n.conf b/apps/emqx_gateway/i18n/emqx_gateway_schema_i18n.conf index aaa5007ee..9ef5f3d5d 100644 --- a/apps/emqx_gateway/i18n/emqx_gateway_schema_i18n.conf +++ b/apps/emqx_gateway/i18n/emqx_gateway_schema_i18n.conf @@ -1,82 +1,5 @@ emqx_gateway_schema { - coap { - desc { - en: """The CoAP Gateway configuration. -This gateway is implemented based on RFC-7252 and https://core-wg.github.io/coap-pubsub/draft-ietf-core-pubsub.html""" - zh: """CoAP 网关配置。 -该网关的实现基于 RFC-7252 和 https://core-wg.github.io/coap-pubsub/draft-ietf-core-pubsub.html""" - } - } - - coap_heartbeat { - desc { - en: """The gateway server required minimum heartbeat interval. -When connection mode is enabled, this parameter is used to set the minimum heartbeat interval for the connection to be alive""" - zh: """CoAP 网关要求客户端的最小心跳间隔时间。 -当 connection_required 开启后，该参数用于检查客户端连接是否存活""" - } - } - - coap_connection_required { - desc { - en: """Enable or disable connection mode. -Connection mode is a feature of non-standard protocols. When connection mode is enabled, it is necessary to maintain the creation, authentication and alive of connection resources""" - zh: """是否开启连接模式。 -连接模式是非标准协议的功能。它维护 CoAP 客户端上线、认证、和连接状态的保持""" - } - } - - coap_notify_type { - desc { - en: """The Notification Message will be delivered to the CoAP client if a new message received on an observed topic. -The type of delivered coap message can be set to:
- - non: Non-confirmable;
- - con: Confirmable;
- - qos: Mapping from QoS type of received message, QoS0 -> non, QoS1,2 -> con""" - zh: """投递给 CoAP 客户端的通知消息类型。当客户端 Observe 一个资源（或订阅某个主题）时，网关会向客户端推送新产生的消息。其消息类型可设置为：
- - non: 不需要客户端返回确认消息;
- - con: 需要客户端返回一个确认消息;
- - qos: 取决于消息的 QoS 等级; QoS 0 会以 `non` 类型下发，QoS 1/2 会以 `con` 类型下发""" - } - } - - coap_subscribe_qos { - desc { - en: """The Default QoS Level indicator for subscribe request. -This option specifies the QoS level for the CoAP Client when establishing a subscription membership, if the subscribe request is not carried `qos` option. The indicator can be set to:
- - qos0, qos1, qos2: Fixed default QoS level
- - coap: Dynamic QoS level by the message type of subscribe request
- * qos0: If the subscribe request is non-confirmable
- * qos1: If the subscribe request is confirmable""" - - zh: """客户端订阅请求的默认 QoS 等级。 -当 CoAP 客户端发起订阅请求时，如果未携带 `qos` 参数则会使用该默认值。默认值可设置为：
- - qos0、 qos1、qos2: 设置为固定的 QoS 等级
- - coap: 依据订阅操作的 CoAP 报文类型来动态决定
- * 当订阅请求为 `non-confirmable` 类型时，取值为 qos0
- * 当订阅请求为 `confirmable` 类型时，取值为 qos1""" - } - } - - coap_publish_qos { - desc { - en: """The Default QoS Level indicator for publish request. -This option specifies the QoS level for the CoAP Client when publishing a message to EMQX PUB/SUB system, if the publish request is not carried `qos` option. The indicator can be set to:
- - qos0, qos1, qos2: Fixed default QoS level
- - coap: Dynamic QoS level by the message type of publish request
- * qos0: If the publish request is non-confirmable
- * qos1: If the publish request is confirmable""" - - zh: """客户端发布请求的默认 QoS 等级。 -当 CoAP 客户端发起发布请求时，如果未携带 `qos` 参数则会使用该默认值。默认值可设置为：
- - qos0、qos1、qos2: 设置为固定的 QoS 等级
- - coap: 依据发布操作的 CoAP 报文类型来动态决定
- * 当发布请求为 `non-confirmable` 类型时，取值为 qos0
- * 当发布请求为 `confirmable` 类型时，取值为 qos1""" - } - } - lwm2m { desc { en: """The LwM2M Gateway configuration. This gateway only supports the v1.0.1 protocol.""" diff --git a/apps/emqx_gateway/src/emqx_gateway_app.erl b/apps/emqx_gateway/src/emqx_gateway_app.erl index a805a0ceb..0c78341e1 100644 --- a/apps/emqx_gateway/src/emqx_gateway_app.erl +++ b/apps/emqx_gateway/src/emqx_gateway_app.erl @@ -47,11 +47,6 @@ load_default_gateway_applications() -> callback_module => emqx_lwm2m_impl, config_schema_module => emqx_lwm2m_schema }, - #{ - name => coap, - callback_module => emqx_coap_impl, - config_schema_module => emqx_gateway_schema - }, #{ name => exproto, callback_module => emqx_exproto_impl, diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl index f7ce3d05c..f9cdcfe26 100644 --- a/apps/emqx_gateway/src/emqx_gateway_schema.erl +++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl @@ -64,14 +64,6 @@ roots() -> [gateway]. fields(gateway) -> [ - {coap, - sc( - ref(coap), - #{ - required => {false, recursively}, - desc => ?DESC(coap) - } - )}, {lwm2m, sc( ref(lwm2m), @@ -89,55 +81,6 @@ fields(gateway) -> } )} ] ++ gateway_schemas(); -fields(coap) -> - [ - {heartbeat, - sc( - duration(), - #{ - default => <<"30s">>, - desc => ?DESC(coap_heartbeat) - } - )}, - {connection_required, - sc( - boolean(), - #{ - default => false, - desc => ?DESC(coap_connection_required) - } - )}, - {notify_type, - sc( - hoconsc:enum([non, con, qos]), - #{ - default => qos, - desc => ?DESC(coap_notify_type) - } - )}, - {subscribe_qos, - sc( - hoconsc:enum([qos0, qos1, qos2, coap]), - #{ - default => coap, - desc => ?DESC(coap_subscribe_qos) - } - )}, - {publish_qos, - sc( - hoconsc:enum([qos0, qos1, qos2, coap]), - #{ - default => coap, - desc => ?DESC(coap_publish_qos) - } - )}, - {mountpoint, mountpoint()}, - {listeners, - sc( - ref(udp_listeners), - #{desc => ?DESC(udp_listeners)} - )} - ] ++ gateway_common_options(); fields(lwm2m) -> [ {xml_dir, @@ -413,10 +356,6 @@ fields(dtls_opts) -> desc(gateway) -> "EMQX Gateway configuration root."; -desc(coap) -> - "The CoAP protocol gateway provides EMQX with the access capability of the CoAP protocol.\n" - "It allows publishing, subscribing, and receiving messages to EMQX in accordance\n" - "with a certain defined CoAP message format."; desc(lwm2m) -> "The LwM2M protocol gateway."; desc(exproto) -> @@ -597,11 +536,11 @@ proxy_protocol_opts() -> %% dynamic schemas %% FIXME: don't hardcode the gateway names -gateway_schema(coap) -> fields(coap); gateway_schema(lwm2m) -> fields(lwm2m); gateway_schema(exproto) -> fields(exproto); gateway_schema(stomp) -> emqx_stomp_schema:fields(stomp); -gateway_schema(mqttsn) -> emqx_mqttsn_schema:fields(mqttsn). +gateway_schema(mqttsn) -> emqx_mqttsn_schema:fields(mqttsn); +gateway_schema(coap) -> emqx_coap_schema:fields(coap). gateway_schemas() -> lists:map( diff --git a/apps/emqx_mqttsn/src/emqx_mqttsn.app.src b/apps/emqx_mqttsn/src/emqx_mqttsn.app.src index e58cf5147..36e4342d1 100644 --- a/apps/emqx_mqttsn/src/emqx_mqttsn.app.src +++ b/apps/emqx_mqttsn/src/emqx_mqttsn.app.src @@ -1,10 +1,10 @@ -{application, emqx_mqttsn, - [{description, "MQTT-SN Gateway"}, - {vsn, "0.1.0"}, - {registered, []}, - {applications, [kernel, stdlib]}, - {env,[]}, - {modules, []}, - {licenses, ["Apache 2.0"]}, - {links, []} - ]}. +{application, emqx_mqttsn, [ + {description, "MQTT-SN Gateway"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [kernel, stdlib]}, + {env, []}, + {modules, []}, + {licenses, ["Apache 2.0"]}, + {links, []} +]}. diff --git a/apps/emqx_stomp/src/emqx_stomp.app.src b/apps/emqx_stomp/src/emqx_stomp.app.src index dffb9b7a4..e2d1f997b 100644 --- a/apps/emqx_stomp/src/emqx_stomp.app.src +++ b/apps/emqx_stomp/src/emqx_stomp.app.src @@ -1,5 +1,5 @@ {application, emqx_stomp, [ - {description, "Stomp gateway"}, + {description, "Stomp Gateway"}, {vsn, "0.1.0"}, {registered, []}, {applications, [kernel, stdlib]}, diff --git a/apps/emqx_stomp/src/emqx_stomp.erl b/apps/emqx_stomp/src/emqx_stomp.erl index 6c14e222c..dbfdfdce5 100644 --- a/apps/emqx_stomp/src/emqx_stomp.erl +++ b/apps/emqx_stomp/src/emqx_stomp.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- -%% @doc The Stomp Gateway implement interface +%% @doc The Stomp Gateway implement -module(emqx_stomp). -include_lib("emqx/include/logger.hrl"). diff --git a/mix.exs b/mix.exs index 514c9139d..229e40824 100644 --- a/mix.exs +++ b/mix.exs @@ -281,6 +281,9 @@ defmodule EMQXUmbrella.MixProject do emqx_authz: :permanent, emqx_auto_subscribe: :permanent, emqx_gateway: :permanent, + emqx_stomp: :permanent, + emqx_mqttsn: :permanent, + emqx_coap: :permanent, emqx_exhook: :permanent, emqx_bridge: :permanent, emqx_rule_engine: :permanent, diff --git a/rebar.config.erl b/rebar.config.erl index 47bf925d4..ee6532de9 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -391,6 +391,7 @@ relx_apps(ReleaseType, Edition) -> emqx_gateway, emqx_stomp, emqx_mqttsn, + emqx_coap, emqx_exhook, emqx_bridge, emqx_rule_engine, From 40c413ac055ed8cb81a33cf197d678b5a91868a1 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 30 Mar 2023 15:48:42 +0800 Subject: [PATCH 150/196] chore: fix dialyzer warnings --- apps/emqx_coap/src/emqx_coap.app.src | 2 +- apps/emqx_gateway/include/emqx_gateway.hrl | 7 +++++++ apps/emqx_gateway/src/emqx_gateway_app.erl | 6 +++++- apps/emqx_gateway/src/emqx_gateway_utils.erl | 1 + apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl | 2 +- apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl | 2 +- apps/emqx_gateway/src/lwm2m/emqx_lwm2m_session.erl | 2 +- apps/emqx_mqttsn/src/emqx_mqttsn.app.src | 2 +- apps/emqx_stomp/src/emqx_stomp.app.src | 2 +- 9 files changed, 19 insertions(+), 7 deletions(-) diff --git a/apps/emqx_coap/src/emqx_coap.app.src b/apps/emqx_coap/src/emqx_coap.app.src index 55c9de59d..50b593ac7 100644 --- a/apps/emqx_coap/src/emqx_coap.app.src +++ b/apps/emqx_coap/src/emqx_coap.app.src @@ -2,7 +2,7 @@ {description, "CoAP Gateway"}, {vsn, "0.1.0"}, {registered, []}, - {applications, [kernel, stdlib]}, + {applications, [kernel, stdlib, emqx_gateway]}, {env, []}, {modules, []}, {licenses, ["Apache 2.0"]}, diff --git a/apps/emqx_gateway/include/emqx_gateway.hrl b/apps/emqx_gateway/include/emqx_gateway.hrl index 3466ecd98..51a519589 100644 --- a/apps/emqx_gateway/include/emqx_gateway.hrl +++ b/apps/emqx_gateway/include/emqx_gateway.hrl @@ -37,4 +37,11 @@ config => emqx_config:config() }. +-type gateway_def() :: + #{ + name := gateway_name(), + callback_module := module(), + config_schema_module := module() + }. + -endif. diff --git a/apps/emqx_gateway/src/emqx_gateway_app.erl b/apps/emqx_gateway/src/emqx_gateway_app.erl index 0c78341e1..0f9ef87e1 100644 --- a/apps/emqx_gateway/src/emqx_gateway_app.erl +++ b/apps/emqx_gateway/src/emqx_gateway_app.erl @@ -80,7 +80,11 @@ load_gateway_application( name => Name, callback_module => CbMod }) - end. + end; +load_gateway_application(_) -> + ?SLOG(error, #{ + msg => "invalid_gateway_defination" + }). load_gateway_by_default() -> load_gateway_by_default(confs()). diff --git a/apps/emqx_gateway/src/emqx_gateway_utils.erl b/apps/emqx_gateway/src/emqx_gateway_utils.erl index 94c7490cc..9d71263f8 100644 --- a/apps/emqx_gateway/src/emqx_gateway_utils.erl +++ b/apps/emqx_gateway/src/emqx_gateway_utils.erl @@ -564,6 +564,7 @@ make_compatible_schema2(Path, SchemaFun) -> Schema ). +-spec find_gateway_definations() -> list(gateway_def()). find_gateway_definations() -> lists:flatten( lists:map( diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl index 16d0f9630..12fd07d93 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl @@ -17,7 +17,7 @@ -module(emqx_lwm2m_channel). -include_lib("emqx/include/logger.hrl"). --include("src/coap/include/emqx_coap.hrl"). +-include_lib("emqx_coap/include/emqx_coap.hrl"). -include("src/lwm2m/include/emqx_lwm2m.hrl"). %% API diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl index 090af3e87..470cab8b7 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl @@ -17,7 +17,7 @@ -module(emqx_lwm2m_cmd). -include_lib("emqx/include/logger.hrl"). --include("src/coap/include/emqx_coap.hrl"). +-include_lib("emqx_coap/include/emqx_coap.hrl"). -include("src/lwm2m/include/emqx_lwm2m.hrl"). -export([ diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_session.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_session.erl index 8634280e3..36244847a 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_session.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_session.erl @@ -15,12 +15,12 @@ %%-------------------------------------------------------------------- -module(emqx_lwm2m_session). --include("src/coap/include/emqx_coap.hrl"). -include("src/lwm2m/include/emqx_lwm2m.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). +-include_lib("emqx_coap/include/emqx_coap.hrl"). %% API -export([ diff --git a/apps/emqx_mqttsn/src/emqx_mqttsn.app.src b/apps/emqx_mqttsn/src/emqx_mqttsn.app.src index 36e4342d1..76acc648e 100644 --- a/apps/emqx_mqttsn/src/emqx_mqttsn.app.src +++ b/apps/emqx_mqttsn/src/emqx_mqttsn.app.src @@ -2,7 +2,7 @@ {description, "MQTT-SN Gateway"}, {vsn, "0.1.0"}, {registered, []}, - {applications, [kernel, stdlib]}, + {applications, [kernel, stdlib, emqx_gateway]}, {env, []}, {modules, []}, {licenses, ["Apache 2.0"]}, diff --git a/apps/emqx_stomp/src/emqx_stomp.app.src b/apps/emqx_stomp/src/emqx_stomp.app.src index e2d1f997b..cd9670056 100644 --- a/apps/emqx_stomp/src/emqx_stomp.app.src +++ b/apps/emqx_stomp/src/emqx_stomp.app.src @@ -2,7 +2,7 @@ {description, "Stomp Gateway"}, {vsn, "0.1.0"}, {registered, []}, - {applications, [kernel, stdlib]}, + {applications, [kernel, stdlib, emqx_gateway]}, {env, []}, {modules, []}, {licenses, ["Apache 2.0"]}, From b58ce0965877d3ece5c6b39ac3f66bc61c3ee4a6 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 30 Mar 2023 16:32:22 +0800 Subject: [PATCH 151/196] chore: split out lwm2m --- .../i18n/emqx_gateway_schema_i18n.conf | 125 ------------ apps/emqx_gateway/include/emqx_gateway.hrl | 2 +- apps/emqx_gateway/src/emqx_gateway_app.erl | 5 - apps/emqx_gateway/src/emqx_gateway_schema.erl | 149 +------------- apps/emqx_gateway/src/lwm2m/.gitignore | 25 --- apps/emqx_lwm2m/.gitignore | 19 ++ apps/emqx_lwm2m/LICENSE | 191 ++++++++++++++++++ .../src/lwm2m => emqx_lwm2m}/README.md | 1 - apps/emqx_lwm2m/i18n/emqx_lwm2m_schema.conf | 127 ++++++++++++ .../include/emqx_lwm2m.hrl | 0 .../lwm2m_xml/LWM2M_Access_Control-v1_0_1.xml | 0 .../LWM2M_Connectivity_Statistics-v1_0_1.xml | 0 .../lwm2m_xml/LWM2M_Device-v1_0_1.xml | 0 .../LWM2M_Firmware_Update-v1_0_1.xml | 0 .../lwm2m_xml/LWM2M_Location-v1_0.xml | 0 .../lwm2m_xml/LWM2M_Security-v1_0.xml | 0 .../lwm2m_xml/LWM2M_Server-v1_0.xml | 0 apps/emqx_lwm2m/rebar.config | 2 + .../lwm2m => emqx_lwm2m/src}/binary_util.erl | 0 apps/emqx_lwm2m/src/emqx_lwm2m.app.src | 10 + .../src/emqx_lwm2m.erl} | 40 ++-- .../src}/emqx_lwm2m_api.erl | 0 .../src}/emqx_lwm2m_channel.erl | 2 +- .../src}/emqx_lwm2m_cmd.erl | 2 +- .../src}/emqx_lwm2m_message.erl | 2 +- apps/emqx_lwm2m/src/emqx_lwm2m_schema.erl | 184 +++++++++++++++++ .../src}/emqx_lwm2m_session.erl | 2 +- .../src}/emqx_lwm2m_tlv.erl | 2 +- .../src}/emqx_lwm2m_xml_object.erl | 2 +- .../src}/emqx_lwm2m_xml_object_db.erl | 2 +- .../test/emqx_lwm2m_SUITE.erl | 5 +- .../test/emqx_lwm2m_api_SUITE.erl | 7 +- .../test/emqx_tlv_SUITE.erl | 4 +- mix.exs | 1 + rebar.config.erl | 3 +- 35 files changed, 576 insertions(+), 338 deletions(-) delete mode 100644 apps/emqx_gateway/src/lwm2m/.gitignore create mode 100644 apps/emqx_lwm2m/.gitignore create mode 100644 apps/emqx_lwm2m/LICENSE rename apps/{emqx_gateway/src/lwm2m => emqx_lwm2m}/README.md (99%) create mode 100644 apps/emqx_lwm2m/i18n/emqx_lwm2m_schema.conf rename apps/{emqx_gateway/src/lwm2m => emqx_lwm2m}/include/emqx_lwm2m.hrl (100%) rename apps/{emqx_gateway/src/lwm2m => emqx_lwm2m}/lwm2m_xml/LWM2M_Access_Control-v1_0_1.xml (100%) rename apps/{emqx_gateway/src/lwm2m => emqx_lwm2m}/lwm2m_xml/LWM2M_Connectivity_Statistics-v1_0_1.xml (100%) rename apps/{emqx_gateway/src/lwm2m => emqx_lwm2m}/lwm2m_xml/LWM2M_Device-v1_0_1.xml (100%) rename apps/{emqx_gateway/src/lwm2m => emqx_lwm2m}/lwm2m_xml/LWM2M_Firmware_Update-v1_0_1.xml (100%) rename apps/{emqx_gateway/src/lwm2m => emqx_lwm2m}/lwm2m_xml/LWM2M_Location-v1_0.xml (100%) rename apps/{emqx_gateway/src/lwm2m => emqx_lwm2m}/lwm2m_xml/LWM2M_Security-v1_0.xml (100%) rename apps/{emqx_gateway/src/lwm2m => emqx_lwm2m}/lwm2m_xml/LWM2M_Server-v1_0.xml (100%) create mode 100644 apps/emqx_lwm2m/rebar.config rename apps/{emqx_gateway/src/lwm2m => emqx_lwm2m/src}/binary_util.erl (100%) create mode 100644 apps/emqx_lwm2m/src/emqx_lwm2m.app.src rename apps/{emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl => emqx_lwm2m/src/emqx_lwm2m.erl} (87%) rename apps/{emqx_gateway/src/lwm2m => emqx_lwm2m/src}/emqx_lwm2m_api.erl (100%) rename apps/{emqx_gateway/src/lwm2m => emqx_lwm2m/src}/emqx_lwm2m_channel.erl (99%) rename apps/{emqx_gateway/src/lwm2m => emqx_lwm2m/src}/emqx_lwm2m_cmd.erl (99%) rename apps/{emqx_gateway/src/lwm2m => emqx_lwm2m/src}/emqx_lwm2m_message.erl (99%) create mode 100644 apps/emqx_lwm2m/src/emqx_lwm2m_schema.erl rename apps/{emqx_gateway/src/lwm2m => emqx_lwm2m/src}/emqx_lwm2m_session.erl (99%) rename apps/{emqx_gateway/src/lwm2m => emqx_lwm2m/src}/emqx_lwm2m_tlv.erl (99%) rename apps/{emqx_gateway/src/lwm2m => emqx_lwm2m/src}/emqx_lwm2m_xml_object.erl (98%) rename apps/{emqx_gateway/src/lwm2m => emqx_lwm2m/src}/emqx_lwm2m_xml_object_db.erl (99%) rename apps/{emqx_gateway => emqx_lwm2m}/test/emqx_lwm2m_SUITE.erl (99%) rename apps/{emqx_gateway => emqx_lwm2m}/test/emqx_lwm2m_api_SUITE.erl (99%) rename apps/{emqx_gateway => emqx_lwm2m}/test/emqx_tlv_SUITE.erl (99%) diff --git a/apps/emqx_gateway/i18n/emqx_gateway_schema_i18n.conf b/apps/emqx_gateway/i18n/emqx_gateway_schema_i18n.conf index 9ef5f3d5d..561627241 100644 --- a/apps/emqx_gateway/i18n/emqx_gateway_schema_i18n.conf +++ b/apps/emqx_gateway/i18n/emqx_gateway_schema_i18n.conf @@ -1,130 +1,5 @@ emqx_gateway_schema { - lwm2m { - desc { - en: """The LwM2M Gateway configuration. This gateway only supports the v1.0.1 protocol.""" - zh: """LwM2M 网关配置。仅支持 v1.0.1 协议。""" - } - } - - lwm2m_xml_dir { - desc { - en: """The Directory for LwM2M Resource definition.""" - zh: """LwM2M Resource 定义的 XML 文件目录路径。""" - } - } - - lwm2m_lifetime_min { - desc { - en: """Minimum value of lifetime allowed to be set by the LwM2M client.""" - zh: """允许 LwM2M 客户端允许设置的心跳最小值。""" - } - } - - lwm2m_lifetime_max { - desc { - en: """Maximum value of lifetime allowed to be set by the LwM2M client.""" - zh: """允许 LwM2M 客户端允许设置的心跳最大值。""" - } - } - - lwm2m_qmode_time_window { - desc { - en: """The value of the time window during which the network link is considered valid by the LwM2M Gateway in QMode mode. -For example, after receiving an update message from a client, any messages within this time window are sent directly to the LwM2M client, and all messages beyond this time window are temporarily stored in memory.""" - - zh: """在QMode模式下，LwM2M网关认为网络链接有效的时间窗口的值。 -例如，在收到客户端的更新信息后，在这个时间窗口内的任何信息都会直接发送到LwM2M客户端，而超过这个时间窗口的所有信息都会暂时储存在内存中。""" - } - } - - lwm2m_auto_observe { - desc { - en: """Automatically observe the object list of REGISTER packet.""" - zh: """自动 Observe REGISTER 数据包的 Object 列表。""" - } - } - - lwm2m_update_msg_publish_condition { - desc { - en: """Policy for publishing UPDATE event message.
- - always: send update events as long as the UPDATE request is received.
- - contains_object_list: send update events only if the UPDATE request carries any Object List""" - zh: """发布UPDATE事件消息的策略。
- - always: 只要收到 UPDATE 请求，就发送更新事件。
- - contains_object_list: 仅当 UPDATE 请求携带 Object 列表时才发送更新事件。""" - } - } - - lwm2m_translators { - desc { - en: """Topic configuration for LwM2M's gateway publishing and subscription.""" - zh: """LwM2M 网关订阅/发布消息的主题映射配置。""" - } - } - - lwm2m_translators_command { - desc { - en: """The topic for receiving downstream commands. -For each new LwM2M client that succeeds in going online, the gateway creates a subscription relationship to receive downstream commands and send it to the LwM2M client""" - - zh: """下行命令主题。 -对于每个成功上线的新 LwM2M 客户端，网关会创建一个订阅关系来接收下行消息并将其发送给客户端。""" - } - } - - lwm2m_translators_response { - desc { - en: """The topic for gateway to publish the acknowledge events from LwM2M client""" - zh: """用于网关发布来自 LwM2M 客户端的确认事件的主题。""" - } - } - - lwm2m_translators_notify { - desc { - en: """The topic for gateway to publish the notify events from LwM2M client. -After succeed observe a resource of LwM2M client, Gateway will send the notify events via this topic, if the client reports any resource changes""" - - zh: """用于发布来自 LwM2M 客户端的通知事件的主题。 -在成功 Observe 到 LwM2M 客户端的资源后，如果客户端报告任何资源状态的变化，网关将通过该主题发送通知事件。""" - } - } - - lwm2m_translators_register { - desc { - en: """The topic for gateway to publish the register events from LwM2M client.""" - zh: """用于发布来自 LwM2M 客户端的注册事件的主题。""" - } - } - - lwm2m_translators_update { - desc { - en: """The topic for gateway to publish the update events from LwM2M client""" - zh: """用于发布来自LwM2M客户端的更新事件的主题。""" - } - } - - translator { - desc { - en: """MQTT topic that corresponds to a particular type of event.""" - zh: """配置某网关客户端对于发布消息或订阅的主题和 QoS 等级。""" - } - } - - translator_topic { - desc { - en: """Topic Name""" - zh: """主题名称""" - } - } - - translator_qos { - desc { - en: """QoS Level""" - zh: """QoS 等级""" - } - } - exproto { desc { en: """The Extension Protocol configuration""" diff --git a/apps/emqx_gateway/include/emqx_gateway.hrl b/apps/emqx_gateway/include/emqx_gateway.hrl index 51a519589..c880aca26 100644 --- a/apps/emqx_gateway/include/emqx_gateway.hrl +++ b/apps/emqx_gateway/include/emqx_gateway.hrl @@ -42,6 +42,6 @@ name := gateway_name(), callback_module := module(), config_schema_module := module() - }. + }. -endif. diff --git a/apps/emqx_gateway/src/emqx_gateway_app.erl b/apps/emqx_gateway/src/emqx_gateway_app.erl index 0f9ef87e1..5999e85c9 100644 --- a/apps/emqx_gateway/src/emqx_gateway_app.erl +++ b/apps/emqx_gateway/src/emqx_gateway_app.erl @@ -42,11 +42,6 @@ stop(_State) -> load_default_gateway_applications() -> BuiltInGateways = [ - #{ - name => lwm2m, - callback_module => emqx_lwm2m_impl, - config_schema_module => emqx_lwm2m_schema - }, #{ name => exproto, callback_module => emqx_exproto_impl, diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl index f9cdcfe26..6a4811b94 100644 --- a/apps/emqx_gateway/src/emqx_gateway_schema.erl +++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl @@ -64,14 +64,6 @@ roots() -> [gateway]. fields(gateway) -> [ - {lwm2m, - sc( - ref(lwm2m), - #{ - required => {false, recursively}, - desc => ?DESC(lwm2m) - } - )}, {exproto, sc( ref(exproto), @@ -81,75 +73,6 @@ fields(gateway) -> } )} ] ++ gateway_schemas(); -fields(lwm2m) -> - [ - {xml_dir, - sc( - binary(), - #{ - %% since this is not packaged with emqx, nor - %% present in the packages, we must let the user - %% specify it rather than creating a dynamic - %% default (especially difficult to handle when - %% generating docs). - example => <<"/etc/emqx/lwm2m_xml">>, - required => true, - desc => ?DESC(lwm2m_xml_dir) - } - )}, - {lifetime_min, - sc( - duration(), - #{ - default => <<"15s">>, - desc => ?DESC(lwm2m_lifetime_min) - } - )}, - {lifetime_max, - sc( - duration(), - #{ - default => <<"86400s">>, - desc => ?DESC(lwm2m_lifetime_max) - } - )}, - {qmode_time_window, - sc( - duration_s(), - #{ - default => <<"22s">>, - desc => ?DESC(lwm2m_qmode_time_window) - } - )}, - %% TODO: Support config resource path - {auto_observe, - sc( - boolean(), - #{ - default => false, - desc => ?DESC(lwm2m_auto_observe) - } - )}, - %% FIXME: not working now - {update_msg_publish_condition, - sc( - hoconsc:enum([always, contains_object_list]), - #{ - default => contains_object_list, - desc => ?DESC(lwm2m_update_msg_publish_condition) - } - )}, - {translators, - sc( - ref(lwm2m_translators), - #{ - required => true, - desc => ?DESC(lwm2m_translators) - } - )}, - {mountpoint, mountpoint("lwm2m/${endpoint_name}/")}, - {listeners, sc(ref(udp_listeners), #{desc => ?DESC(udp_listeners)})} - ] ++ gateway_common_options(); fields(exproto) -> [ {server, @@ -223,68 +146,6 @@ fields(clientinfo_override) -> })}, {clientid, sc(binary(), #{desc => ?DESC(gateway_common_clientinfo_override_clientid)})} ]; -fields(lwm2m_translators) -> - [ - {command, - sc( - ref(translator), - #{ - desc => ?DESC(lwm2m_translators_command), - required => true - } - )}, - {response, - sc( - ref(translator), - #{ - desc => ?DESC(lwm2m_translators_response), - required => true - } - )}, - {notify, - sc( - ref(translator), - #{ - desc => ?DESC(lwm2m_translators_notify), - required => true - } - )}, - {register, - sc( - ref(translator), - #{ - desc => ?DESC(lwm2m_translators_register), - required => true - } - )}, - {update, - sc( - ref(translator), - #{ - desc => ?DESC(lwm2m_translators_update), - required => true - } - )} - ]; -fields(translator) -> - [ - {topic, - sc( - binary(), - #{ - required => true, - desc => ?DESC(translator_topic) - } - )}, - {qos, - sc( - emqx_schema:qos(), - #{ - default => 0, - desc => ?DESC(translator_qos) - } - )} - ]; fields(udp_listeners) -> [ {udp, sc(map(name, ref(udp_listener)), #{desc => ?DESC(udp_listener)})}, @@ -356,8 +217,6 @@ fields(dtls_opts) -> desc(gateway) -> "EMQX Gateway configuration root."; -desc(lwm2m) -> - "The LwM2M protocol gateway."; desc(exproto) -> "Settings for EMQX extension protocol (exproto)."; desc(exproto_grpc_server) -> @@ -368,10 +227,6 @@ desc(ssl_server_opts) -> "SSL configuration for the server."; desc(clientinfo_override) -> "ClientInfo override."; -desc(lwm2m_translators) -> - "MQTT topics that correspond to LwM2M events."; -desc(translator) -> - "MQTT topic that corresponds to a particular type of event."; desc(udp_listeners) -> "Settings for the UDP listeners."; desc(tcp_listeners) -> @@ -536,11 +391,11 @@ proxy_protocol_opts() -> %% dynamic schemas %% FIXME: don't hardcode the gateway names -gateway_schema(lwm2m) -> fields(lwm2m); gateway_schema(exproto) -> fields(exproto); gateway_schema(stomp) -> emqx_stomp_schema:fields(stomp); gateway_schema(mqttsn) -> emqx_mqttsn_schema:fields(mqttsn); -gateway_schema(coap) -> emqx_coap_schema:fields(coap). +gateway_schema(coap) -> emqx_coap_schema:fields(coap); +gateway_schema(lwm2m) -> emqx_lwm2m_schema:fields(lwm2m). gateway_schemas() -> lists:map( diff --git a/apps/emqx_gateway/src/lwm2m/.gitignore b/apps/emqx_gateway/src/lwm2m/.gitignore deleted file mode 100644 index be6914be3..000000000 --- a/apps/emqx_gateway/src/lwm2m/.gitignore +++ /dev/null @@ -1,25 +0,0 @@ -deps/ -ebin/ -_rel/ -.erlang.mk/ -*.d -*.o -*.exe -data/ -*.iml -.idea/ -logs/ -*.beam -emqx_coap.d -erlang.mk -integration_test/emqx-rel/ -integration_test/build_wakaama/ -integration_test/case*.txt -integration_test/paho/ -integration_test/wakaama/ -_build/ -rebar.lock -rebar3.crashdump -*.conf.rendered -.rebar3/ -*.swp diff --git a/apps/emqx_lwm2m/.gitignore b/apps/emqx_lwm2m/.gitignore new file mode 100644 index 000000000..f1c455451 --- /dev/null +++ b/apps/emqx_lwm2m/.gitignore @@ -0,0 +1,19 @@ +.rebar3 +_* +.eunit +*.o +*.beam +*.plt +*.swp +*.swo +.erlang.cookie +ebin +log +erl_crash.dump +.rebar +logs +_build +.idea +*.iml +rebar3.crashdump +*~ diff --git a/apps/emqx_lwm2m/LICENSE b/apps/emqx_lwm2m/LICENSE new file mode 100644 index 000000000..5a5418f0f --- /dev/null +++ b/apps/emqx_lwm2m/LICENSE @@ -0,0 +1,191 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2023, JianBo He . + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/apps/emqx_gateway/src/lwm2m/README.md b/apps/emqx_lwm2m/README.md similarity index 99% rename from apps/emqx_gateway/src/lwm2m/README.md rename to apps/emqx_lwm2m/README.md index bf7626c6f..faca6dad3 100644 --- a/apps/emqx_gateway/src/lwm2m/README.md +++ b/apps/emqx_lwm2m/README.md @@ -1,4 +1,3 @@ - # LwM2M Gateway [The LwM2M Specifications](http://www.openmobilealliance.org/release/LightweightM2M) is a Lightweight Machine to Machine protocol. diff --git a/apps/emqx_lwm2m/i18n/emqx_lwm2m_schema.conf b/apps/emqx_lwm2m/i18n/emqx_lwm2m_schema.conf new file mode 100644 index 000000000..822570f1d --- /dev/null +++ b/apps/emqx_lwm2m/i18n/emqx_lwm2m_schema.conf @@ -0,0 +1,127 @@ +emqx_lwm2m_schema { + + lwm2m { + desc { + en: """The LwM2M Gateway configuration. This gateway only supports the v1.0.1 protocol.""" + zh: """LwM2M 网关配置。仅支持 v1.0.1 协议。""" + } + } + + lwm2m_xml_dir { + desc { + en: """The Directory for LwM2M Resource definition.""" + zh: """LwM2M Resource 定义的 XML 文件目录路径。""" + } + } + + lwm2m_lifetime_min { + desc { + en: """Minimum value of lifetime allowed to be set by the LwM2M client.""" + zh: """允许 LwM2M 客户端允许设置的心跳最小值。""" + } + } + + lwm2m_lifetime_max { + desc { + en: """Maximum value of lifetime allowed to be set by the LwM2M client.""" + zh: """允许 LwM2M 客户端允许设置的心跳最大值。""" + } + } + + lwm2m_qmode_time_window { + desc { + en: """The value of the time window during which the network link is considered valid by the LwM2M Gateway in QMode mode. +For example, after receiving an update message from a client, any messages within this time window are sent directly to the LwM2M client, and all messages beyond this time window are temporarily stored in memory.""" + + zh: """在QMode模式下，LwM2M网关认为网络链接有效的时间窗口的值。 +例如，在收到客户端的更新信息后，在这个时间窗口内的任何信息都会直接发送到LwM2M客户端，而超过这个时间窗口的所有信息都会暂时储存在内存中。""" + } + } + + lwm2m_auto_observe { + desc { + en: """Automatically observe the object list of REGISTER packet.""" + zh: """自动 Observe REGISTER 数据包的 Object 列表。""" + } + } + + lwm2m_update_msg_publish_condition { + desc { + en: """Policy for publishing UPDATE event message.
+ - always: send update events as long as the UPDATE request is received.
+ - contains_object_list: send update events only if the UPDATE request carries any Object List""" + zh: """发布UPDATE事件消息的策略。
+ - always: 只要收到 UPDATE 请求，就发送更新事件。
+ - contains_object_list: 仅当 UPDATE 请求携带 Object 列表时才发送更新事件。""" + } + } + + lwm2m_translators { + desc { + en: """Topic configuration for LwM2M's gateway publishing and subscription.""" + zh: """LwM2M 网关订阅/发布消息的主题映射配置。""" + } + } + + lwm2m_translators_command { + desc { + en: """The topic for receiving downstream commands. +For each new LwM2M client that succeeds in going online, the gateway creates a subscription relationship to receive downstream commands and send it to the LwM2M client""" + + zh: """下行命令主题。 +对于每个成功上线的新 LwM2M 客户端，网关会创建一个订阅关系来接收下行消息并将其发送给客户端。""" + } + } + + lwm2m_translators_response { + desc { + en: """The topic for gateway to publish the acknowledge events from LwM2M client""" + zh: """用于网关发布来自 LwM2M 客户端的确认事件的主题。""" + } + } + + lwm2m_translators_notify { + desc { + en: """The topic for gateway to publish the notify events from LwM2M client. +After succeed observe a resource of LwM2M client, Gateway will send the notify events via this topic, if the client reports any resource changes""" + + zh: """用于发布来自 LwM2M 客户端的通知事件的主题。 +在成功 Observe 到 LwM2M 客户端的资源后，如果客户端报告任何资源状态的变化，网关将通过该主题发送通知事件。""" + } + } + + lwm2m_translators_register { + desc { + en: """The topic for gateway to publish the register events from LwM2M client.""" + zh: """用于发布来自 LwM2M 客户端的注册事件的主题。""" + } + } + + lwm2m_translators_update { + desc { + en: """The topic for gateway to publish the update events from LwM2M client""" + zh: """用于发布来自LwM2M客户端的更新事件的主题。""" + } + } + + translator { + desc { + en: """MQTT topic that corresponds to a particular type of event.""" + zh: """配置某网关客户端对于发布消息或订阅的主题和 QoS 等级。""" + } + } + + translator_topic { + desc { + en: """Topic Name""" + zh: """主题名称""" + } + } + + translator_qos { + desc { + en: """QoS Level""" + zh: """QoS 等级""" + } + } +} diff --git a/apps/emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl b/apps/emqx_lwm2m/include/emqx_lwm2m.hrl similarity index 100% rename from apps/emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl rename to apps/emqx_lwm2m/include/emqx_lwm2m.hrl diff --git a/apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Access_Control-v1_0_1.xml b/apps/emqx_lwm2m/lwm2m_xml/LWM2M_Access_Control-v1_0_1.xml similarity index 100% rename from apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Access_Control-v1_0_1.xml rename to apps/emqx_lwm2m/lwm2m_xml/LWM2M_Access_Control-v1_0_1.xml diff --git a/apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Connectivity_Statistics-v1_0_1.xml b/apps/emqx_lwm2m/lwm2m_xml/LWM2M_Connectivity_Statistics-v1_0_1.xml similarity index 100% rename from apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Connectivity_Statistics-v1_0_1.xml rename to apps/emqx_lwm2m/lwm2m_xml/LWM2M_Connectivity_Statistics-v1_0_1.xml diff --git a/apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Device-v1_0_1.xml b/apps/emqx_lwm2m/lwm2m_xml/LWM2M_Device-v1_0_1.xml similarity index 100% rename from apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Device-v1_0_1.xml rename to apps/emqx_lwm2m/lwm2m_xml/LWM2M_Device-v1_0_1.xml diff --git a/apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Firmware_Update-v1_0_1.xml b/apps/emqx_lwm2m/lwm2m_xml/LWM2M_Firmware_Update-v1_0_1.xml similarity index 100% rename from apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Firmware_Update-v1_0_1.xml rename to apps/emqx_lwm2m/lwm2m_xml/LWM2M_Firmware_Update-v1_0_1.xml diff --git a/apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Location-v1_0.xml b/apps/emqx_lwm2m/lwm2m_xml/LWM2M_Location-v1_0.xml similarity index 100% rename from apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Location-v1_0.xml rename to apps/emqx_lwm2m/lwm2m_xml/LWM2M_Location-v1_0.xml diff --git a/apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Security-v1_0.xml b/apps/emqx_lwm2m/lwm2m_xml/LWM2M_Security-v1_0.xml similarity index 100% rename from apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Security-v1_0.xml rename to apps/emqx_lwm2m/lwm2m_xml/LWM2M_Security-v1_0.xml diff --git a/apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Server-v1_0.xml b/apps/emqx_lwm2m/lwm2m_xml/LWM2M_Server-v1_0.xml similarity index 100% rename from apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Server-v1_0.xml rename to apps/emqx_lwm2m/lwm2m_xml/LWM2M_Server-v1_0.xml diff --git a/apps/emqx_lwm2m/rebar.config b/apps/emqx_lwm2m/rebar.config new file mode 100644 index 000000000..2656fd554 --- /dev/null +++ b/apps/emqx_lwm2m/rebar.config @@ -0,0 +1,2 @@ +{erl_opts, [debug_info]}. +{deps, []}. diff --git a/apps/emqx_gateway/src/lwm2m/binary_util.erl b/apps/emqx_lwm2m/src/binary_util.erl similarity index 100% rename from apps/emqx_gateway/src/lwm2m/binary_util.erl rename to apps/emqx_lwm2m/src/binary_util.erl diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m.app.src b/apps/emqx_lwm2m/src/emqx_lwm2m.app.src new file mode 100644 index 000000000..08c3dbe3f --- /dev/null +++ b/apps/emqx_lwm2m/src/emqx_lwm2m.app.src @@ -0,0 +1,10 @@ +{application, emqx_lwm2m, [ + {description, "LwM2M Gateway"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [kernel, stdlib, emqx_gateway, emqx_coap]}, + {env, []}, + {modules, []}, + {licenses, ["Apache 2.0"]}, + {links, []} +]}. diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl b/apps/emqx_lwm2m/src/emqx_lwm2m.erl similarity index 87% rename from apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl rename to apps/emqx_lwm2m/src/emqx_lwm2m.erl index fa4537315..222d1076e 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m.erl @@ -14,35 +14,37 @@ %% limitations under the License. %%-------------------------------------------------------------------- -%% @doc The LwM2M Gateway Implement interface --module(emqx_lwm2m_impl). - --behaviour(emqx_gateway_impl). +%% @doc The LwM2M Gateway implement +-module(emqx_lwm2m). -include_lib("emqx/include/logger.hrl"). +-include_lib("emqx_gateway/include/emqx_gateway.hrl"). -%% APIs --export([ - reg/0, - unreg/0 -]). +%% define a gateway named stomp +-gateway(#{ + name => lwm2m, + callback_module => ?MODULE, + config_schema_module => emqx_lwm2m_schema +}). +%% callback_module must implement the emqx_gateway_impl behaviour +-behaviour(emqx_gateway_impl). + +%% callback for emqx_gateway_impl -export([ on_gateway_load/2, on_gateway_update/3, on_gateway_unload/2 ]). -%%-------------------------------------------------------------------- -%% APIs -%%-------------------------------------------------------------------- - -reg() -> - RegistryOptions = [{cbkmod, ?MODULE}], - emqx_gateway_registry:reg(lwm2m, RegistryOptions). - -unreg() -> - emqx_gateway_registry:unreg(lwm2m). +-import( + emqx_gateway_utils, + [ + normalize_config/1, + start_listeners/4, + stop_listeners/2 + ] +). %%-------------------------------------------------------------------- %% emqx_gateway_registry callbacks diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_api.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_api.erl similarity index 100% rename from apps/emqx_gateway/src/lwm2m/emqx_lwm2m_api.erl rename to apps/emqx_lwm2m/src/emqx_lwm2m_api.erl diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_channel.erl similarity index 99% rename from apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl rename to apps/emqx_lwm2m/src/emqx_lwm2m_channel.erl index 12fd07d93..54e5723cd 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_channel.erl @@ -16,9 +16,9 @@ -module(emqx_lwm2m_channel). +-include("emqx_lwm2m.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("emqx_coap/include/emqx_coap.hrl"). --include("src/lwm2m/include/emqx_lwm2m.hrl"). %% API -export([ diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_cmd.erl similarity index 99% rename from apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl rename to apps/emqx_lwm2m/src/emqx_lwm2m_cmd.erl index 470cab8b7..9b1f6b65d 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_cmd.erl @@ -16,9 +16,9 @@ -module(emqx_lwm2m_cmd). +-include("emqx_lwm2m.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("emqx_coap/include/emqx_coap.hrl"). --include("src/lwm2m/include/emqx_lwm2m.hrl"). -export([ mqtt_to_coap/2, diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_message.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_message.erl similarity index 99% rename from apps/emqx_gateway/src/lwm2m/emqx_lwm2m_message.erl rename to apps/emqx_lwm2m/src/emqx_lwm2m_message.erl index f09a8ea3d..e541e83f1 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_message.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_message.erl @@ -24,7 +24,7 @@ translate_json/1 ]). --include("src/lwm2m/include/emqx_lwm2m.hrl"). +-include("emqx_lwm2m.hrl"). tlv_to_json(BaseName, TlvData) -> DecodedTlv = emqx_lwm2m_tlv:parse(TlvData), diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_schema.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_schema.erl new file mode 100644 index 000000000..b674c3260 --- /dev/null +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_schema.erl @@ -0,0 +1,184 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_lwm2m_schema). + +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("typerefl/include/types.hrl"). + +-type duration() :: non_neg_integer(). +-type duration_s() :: non_neg_integer(). + +-typerefl_from_string({duration/0, emqx_schema, to_duration}). +-typerefl_from_string({duration_s/0, emqx_schema, to_duration_s}). + +-reflect_type([duration/0, duration_s/0]). + +%% config schema provides +-export([fields/1, desc/1]). + +fields(lwm2m) -> + [ + {xml_dir, + sc( + binary(), + #{ + %% since this is not packaged with emqx, nor + %% present in the packages, we must let the user + %% specify it rather than creating a dynamic + %% default (especially difficult to handle when + %% generating docs). + example => <<"/etc/emqx/lwm2m_xml">>, + required => true, + desc => ?DESC(lwm2m_xml_dir) + } + )}, + {lifetime_min, + sc( + duration(), + #{ + default => <<"15s">>, + desc => ?DESC(lwm2m_lifetime_min) + } + )}, + {lifetime_max, + sc( + duration(), + #{ + default => <<"86400s">>, + desc => ?DESC(lwm2m_lifetime_max) + } + )}, + {qmode_time_window, + sc( + duration_s(), + #{ + default => <<"22s">>, + desc => ?DESC(lwm2m_qmode_time_window) + } + )}, + %% TODO: Support config resource path + {auto_observe, + sc( + boolean(), + #{ + default => false, + desc => ?DESC(lwm2m_auto_observe) + } + )}, + %% FIXME: not working now + {update_msg_publish_condition, + sc( + hoconsc:enum([always, contains_object_list]), + #{ + default => contains_object_list, + desc => ?DESC(lwm2m_update_msg_publish_condition) + } + )}, + {translators, + sc( + ref(lwm2m_translators), + #{ + required => true, + desc => ?DESC(lwm2m_translators) + } + )}, + {mountpoint, emqx_gateway_schema:mountpoint("lwm2m/${endpoint_name}/")}, + {listeners, sc(ref(emqx_gateway_schema, udp_listeners), #{desc => ?DESC(udp_listeners)})} + ] ++ emqx_gateway_schema:gateway_common_options(); +fields(lwm2m_translators) -> + [ + {command, + sc( + ref(translator), + #{ + desc => ?DESC(lwm2m_translators_command), + required => true + } + )}, + {response, + sc( + ref(translator), + #{ + desc => ?DESC(lwm2m_translators_response), + required => true + } + )}, + {notify, + sc( + ref(translator), + #{ + desc => ?DESC(lwm2m_translators_notify), + required => true + } + )}, + {register, + sc( + ref(translator), + #{ + desc => ?DESC(lwm2m_translators_register), + required => true + } + )}, + {update, + sc( + ref(translator), + #{ + desc => ?DESC(lwm2m_translators_update), + required => true + } + )} + ]; +fields(translator) -> + [ + {topic, + sc( + binary(), + #{ + required => true, + desc => ?DESC(translator_topic) + } + )}, + {qos, + sc( + emqx_schema:qos(), + #{ + default => 0, + desc => ?DESC(translator_qos) + } + )} + ]. + +desc(lwm2m) -> + "The LwM2M protocol gateway."; +desc(lwm2m_translators) -> + "MQTT topics that correspond to LwM2M events."; +desc(translator) -> + "MQTT topic that corresponds to a particular type of event."; +desc(_) -> + undefined. + +%%-------------------------------------------------------------------- +%% helpers + +sc(Type, Meta) -> + hoconsc:mk(Type, Meta). + +ref(StructName) -> + ref(?MODULE, StructName). + +ref(Mod, Field) -> + hoconsc:ref(Mod, Field). diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_session.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_session.erl similarity index 99% rename from apps/emqx_gateway/src/lwm2m/emqx_lwm2m_session.erl rename to apps/emqx_lwm2m/src/emqx_lwm2m_session.erl index 36244847a..67543a910 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_session.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_session.erl @@ -15,7 +15,7 @@ %%-------------------------------------------------------------------- -module(emqx_lwm2m_session). --include("src/lwm2m/include/emqx_lwm2m.hrl"). +-include("emqx_lwm2m.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_tlv.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_tlv.erl similarity index 99% rename from apps/emqx_gateway/src/lwm2m/emqx_lwm2m_tlv.erl rename to apps/emqx_lwm2m/src/emqx_lwm2m_tlv.erl index 782bbec5e..2f53573c4 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_tlv.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_tlv.erl @@ -25,7 +25,7 @@ -export([binary_to_hex_string/1]). -endif. --include("src/lwm2m/include/emqx_lwm2m.hrl"). +-include("emqx_lwm2m.hrl"). -define(TLV_TYPE_OBJECT_INSTANCE, 0). -define(TLV_TYPE_RESOURCE_INSTANCE, 1). diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object.erl similarity index 98% rename from apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object.erl rename to apps/emqx_lwm2m/src/emqx_lwm2m_xml_object.erl index a4dc44f2c..3525f72aa 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object.erl @@ -16,7 +16,7 @@ -module(emqx_lwm2m_xml_object). --include("src/lwm2m/include/emqx_lwm2m.hrl"). +-include("emqx_lwm2m.hrl"). -include_lib("xmerl/include/xmerl.hrl"). -export([ diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object_db.erl similarity index 99% rename from apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl rename to apps/emqx_lwm2m/src/emqx_lwm2m_xml_object_db.erl index 58373e114..04c4c1af2 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object_db.erl @@ -16,7 +16,7 @@ -module(emqx_lwm2m_xml_object_db). --include("src/lwm2m/include/emqx_lwm2m.hrl"). +-include("emqx_lwm2m.hrl"). -include_lib("xmerl/include/xmerl.hrl"). -include_lib("emqx/include/logger.hrl"). diff --git a/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl b/apps/emqx_lwm2m/test/emqx_lwm2m_SUITE.erl similarity index 99% rename from apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl rename to apps/emqx_lwm2m/test/emqx_lwm2m_SUITE.erl index fc852709c..9abe16a35 100644 --- a/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl +++ b/apps/emqx_lwm2m/test/emqx_lwm2m_SUITE.erl @@ -31,8 +31,8 @@ -define(LOGT(Format, Args), ct:pal("TEST_SUITE: " ++ Format, Args)). --include("src/lwm2m/include/emqx_lwm2m.hrl"). --include("src/coap/include/emqx_coap.hrl"). +-include("emqx_lwm2m.hrl"). +-include_lib("emqx_coap/include/emqx_coap.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). @@ -134,6 +134,7 @@ groups() -> init_per_suite(Config) -> %% load application first for minirest api searching application:load(emqx_gateway), + application:load(emqx_lwm2m), emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_authn]), Config. diff --git a/apps/emqx_gateway/test/emqx_lwm2m_api_SUITE.erl b/apps/emqx_lwm2m/test/emqx_lwm2m_api_SUITE.erl similarity index 99% rename from apps/emqx_gateway/test/emqx_lwm2m_api_SUITE.erl rename to apps/emqx_lwm2m/test/emqx_lwm2m_api_SUITE.erl index c40d1af55..bfeeb2c9b 100644 --- a/apps/emqx_gateway/test/emqx_lwm2m_api_SUITE.erl +++ b/apps/emqx_lwm2m/test/emqx_lwm2m_api_SUITE.erl @@ -23,8 +23,8 @@ -define(LOGT(Format, Args), ct:pal("TEST_SUITE: " ++ Format, Args)). --include("src/lwm2m/include/emqx_lwm2m.hrl"). --include("src/coap/include/emqx_coap.hrl"). +-include("emqx_lwm2m.hrl"). +-include("emqx_coap/include/emqx_coap.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). @@ -81,8 +81,9 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), application:load(emqx_gateway), + application:load(emqx_lwm2m), + ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_authn]), Config. diff --git a/apps/emqx_gateway/test/emqx_tlv_SUITE.erl b/apps/emqx_lwm2m/test/emqx_tlv_SUITE.erl similarity index 99% rename from apps/emqx_gateway/test/emqx_tlv_SUITE.erl rename to apps/emqx_lwm2m/test/emqx_tlv_SUITE.erl index 5dcef7e72..da1e3a9c4 100644 --- a/apps/emqx_gateway/test/emqx_tlv_SUITE.erl +++ b/apps/emqx_lwm2m/test/emqx_tlv_SUITE.erl @@ -21,8 +21,8 @@ -define(LOGT(Format, Args), logger:debug("TEST_SUITE: " ++ Format, Args)). --include("src/lwm2m/include/emqx_lwm2m.hrl"). --include("src/coap/include/emqx_coap.hrl"). +-include("emqx_lwm2m.hrl"). +-include("emqx_coap/include/emqx_coap.hrl"). -include_lib("eunit/include/eunit.hrl"). %%-------------------------------------------------------------------- diff --git a/mix.exs b/mix.exs index 229e40824..264d4f87e 100644 --- a/mix.exs +++ b/mix.exs @@ -284,6 +284,7 @@ defmodule EMQXUmbrella.MixProject do emqx_stomp: :permanent, emqx_mqttsn: :permanent, emqx_coap: :permanent, + emqx_lwm2m: :permanent, emqx_exhook: :permanent, emqx_bridge: :permanent, emqx_rule_engine: :permanent, diff --git a/rebar.config.erl b/rebar.config.erl index ee6532de9..978f5ec87 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -392,6 +392,7 @@ relx_apps(ReleaseType, Edition) -> emqx_stomp, emqx_mqttsn, emqx_coap, + emqx_lwm2m, emqx_exhook, emqx_bridge, emqx_rule_engine, @@ -452,7 +453,7 @@ relx_overlay(ReleaseType, Edition) -> {copy, "bin/emqx_ctl", "bin/emqx_ctl-{{release_version}}"}, %% for relup {copy, "bin/install_upgrade.escript", "bin/install_upgrade.escript-{{release_version}}"}, - {copy, "apps/emqx_gateway/src/lwm2m/lwm2m_xml", "etc/lwm2m_xml"}, + {copy, "apps/emqx_lwm2m/lwm2m_xml", "etc/lwm2m_xml"}, {copy, "apps/emqx_authz/etc/acl.conf", "etc/acl.conf"}, {template, "bin/emqx.cmd", "bin/emqx.cmd"}, {template, "bin/emqx_ctl.cmd", "bin/emqx_ctl.cmd"}, From a70545b64a1e497fbcd6badef66bdcb83815c5bb Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 30 Mar 2023 18:06:43 +0800 Subject: [PATCH 152/196] chore: split out exproto gateway --- apps/emqx_exproto/.gitignore | 24 +++ apps/emqx_exproto/LICENSE | 191 ++++++++++++++++++ .../src/exproto => emqx_exproto}/README.md | 0 .../i18n/emqx_exproto_schema.conf | 52 +++++ .../include/emqx_exproto.hrl | 0 .../priv}/protos/exproto.proto | 0 apps/emqx_exproto/rebar.config | 31 +++ apps/emqx_exproto/src/emqx_exproto.app.src | 10 + .../src/emqx_exproto.erl} | 49 ++--- .../src}/emqx_exproto_channel.erl | 3 +- .../src}/emqx_exproto_frame.erl | 0 .../src}/emqx_exproto_gcli.erl | 0 .../src}/emqx_exproto_gsvr.erl | 2 +- apps/emqx_exproto/src/emqx_exproto_schema.erl | 117 +++++++++++ .../test/emqx_exproto_SUITE.erl | 1 + .../test/emqx_exproto_echo_svr.erl | 0 apps/emqx_gateway/.gitignore | 6 +- apps/emqx_gateway/Makefile | 28 --- .../i18n/emqx_gateway_schema_i18n.conf | 51 ----- apps/emqx_gateway/rebar.config | 33 --- apps/emqx_gateway/src/emqx_gateway.app.src | 2 +- apps/emqx_gateway/src/emqx_gateway_app.erl | 9 +- apps/emqx_gateway/src/emqx_gateway_schema.erl | 112 ++-------- mix.exs | 1 + rebar.config.erl | 1 + 25 files changed, 471 insertions(+), 252 deletions(-) create mode 100644 apps/emqx_exproto/.gitignore create mode 100644 apps/emqx_exproto/LICENSE rename apps/{emqx_gateway/src/exproto => emqx_exproto}/README.md (100%) create mode 100644 apps/emqx_exproto/i18n/emqx_exproto_schema.conf rename apps/{emqx_gateway/src/exproto => emqx_exproto}/include/emqx_exproto.hrl (100%) rename apps/{emqx_gateway/src/exproto => emqx_exproto/priv}/protos/exproto.proto (100%) create mode 100644 apps/emqx_exproto/rebar.config create mode 100644 apps/emqx_exproto/src/emqx_exproto.app.src rename apps/{emqx_gateway/src/exproto/emqx_exproto_impl.erl => emqx_exproto/src/emqx_exproto.erl} (93%) rename apps/{emqx_gateway/src/exproto => emqx_exproto/src}/emqx_exproto_channel.erl (99%) rename apps/{emqx_gateway/src/exproto => emqx_exproto/src}/emqx_exproto_frame.erl (100%) rename apps/{emqx_gateway/src/exproto => emqx_exproto/src}/emqx_exproto_gcli.erl (100%) rename apps/{emqx_gateway/src/exproto => emqx_exproto/src}/emqx_exproto_gsvr.erl (99%) create mode 100644 apps/emqx_exproto/src/emqx_exproto_schema.erl rename apps/{emqx_gateway => emqx_exproto}/test/emqx_exproto_SUITE.erl (99%) rename apps/{emqx_gateway => emqx_exproto}/test/emqx_exproto_echo_svr.erl (100%) delete mode 100644 apps/emqx_gateway/Makefile diff --git a/apps/emqx_exproto/.gitignore b/apps/emqx_exproto/.gitignore new file mode 100644 index 000000000..922b0f989 --- /dev/null +++ b/apps/emqx_exproto/.gitignore @@ -0,0 +1,24 @@ +.rebar3 +_* +.eunit +*.o +*.beam +*.plt +*.swp +*.swo +.erlang.cookie +ebin +log +erl_crash.dump +.rebar +logs +_build +.idea +*.iml +rebar3.crashdump +*~ +src/emqx_exproto_pb.erl +src/emqx_exproto_v_1_connection_adapter_bhvr.erl +src/emqx_exproto_v_1_connection_adapter_client.erl +src/emqx_exproto_v_1_connection_handler_bhvr.erl +src/emqx_exproto_v_1_connection_handler_client.erl diff --git a/apps/emqx_exproto/LICENSE b/apps/emqx_exproto/LICENSE new file mode 100644 index 000000000..5a5418f0f --- /dev/null +++ b/apps/emqx_exproto/LICENSE @@ -0,0 +1,191 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2023, JianBo He . + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/apps/emqx_gateway/src/exproto/README.md b/apps/emqx_exproto/README.md similarity index 100% rename from apps/emqx_gateway/src/exproto/README.md rename to apps/emqx_exproto/README.md diff --git a/apps/emqx_exproto/i18n/emqx_exproto_schema.conf b/apps/emqx_exproto/i18n/emqx_exproto_schema.conf new file mode 100644 index 000000000..0c6fd2286 --- /dev/null +++ b/apps/emqx_exproto/i18n/emqx_exproto_schema.conf @@ -0,0 +1,52 @@ +emqx_exproto_schema { + exproto { + desc { + en: """The Extension Protocol configuration""" + zh: """ExProto 网关""" + } + } + + exproto_server { + desc { + en: """Configurations for starting the ConnectionAdapter service""" + zh: """配置 ExProto 网关需要启动的 ConnectionAdapter 服务。 +该服务用于提供客户端的认证、发布、订阅和数据下行等功能。""" + } + } + + exproto_grpc_server_bind { + desc { + en: """Listening address and port for the gRPC server.""" + zh: """服务监听地址和端口。""" + } + } + + exproto_grpc_server_ssl { + desc { + en: """SSL configuration for the gRPC server.""" + zh: """服务 SSL 配置。""" + } + } + + exproto_handler { + desc { + en: """Configurations for request to ConnectionHandler service""" + zh: """配置 ExProto 网关需要请求的 ConnectionHandler 服务地址。 +该服务用于给 ExProto 提供客户端的 Socket 事件处理、字节解码、订阅消息接收等功能。""" + } + } + + exproto_grpc_handler_address { + desc { + en: """gRPC server address.""" + zh: """对端 gRPC 服务器地址。""" + } + } + + exproto_grpc_handler_ssl { + desc { + en: """SSL configuration for the gRPC client.""" + zh: """gRPC 客户端的 SSL 配置。""" + } + } +} diff --git a/apps/emqx_gateway/src/exproto/include/emqx_exproto.hrl b/apps/emqx_exproto/include/emqx_exproto.hrl similarity index 100% rename from apps/emqx_gateway/src/exproto/include/emqx_exproto.hrl rename to apps/emqx_exproto/include/emqx_exproto.hrl diff --git a/apps/emqx_gateway/src/exproto/protos/exproto.proto b/apps/emqx_exproto/priv/protos/exproto.proto similarity index 100% rename from apps/emqx_gateway/src/exproto/protos/exproto.proto rename to apps/emqx_exproto/priv/protos/exproto.proto diff --git a/apps/emqx_exproto/rebar.config b/apps/emqx_exproto/rebar.config new file mode 100644 index 000000000..556404166 --- /dev/null +++ b/apps/emqx_exproto/rebar.config @@ -0,0 +1,31 @@ +{erl_opts, [debug_info]}. + +{plugins, [ + {grpc_plugin, {git, "https://github.com/HJianBo/grpc_plugin", {tag, "v0.10.2"}}} +]}. + +{grpc, [ + {protos, ["priv/protos"]}, + {out_dir, "src"}, + {gpb_opts, [ + {module_name_prefix, "emqx_"}, + {module_name_suffix, "_pb"} + ]} +]}. + +{provider_hooks, [ + {pre, [ + {compile, {grpc, gen}}, + {clean, {grpc, clean}} + ]} +]}. + +{xref_ignores, [emqx_exproto_pb]}. + +{cover_excl_mods, [ + emqx_exproto_pb, + emqx_exproto_v_1_connection_adapter_client, + emqx_exproto_v_1_connection_adapter_bhvr, + emqx_exproto_v_1_connection_handler_client, + emqx_exproto_v_1_connection_handler_bhvr +]}. diff --git a/apps/emqx_exproto/src/emqx_exproto.app.src b/apps/emqx_exproto/src/emqx_exproto.app.src new file mode 100644 index 000000000..0b4ac3966 --- /dev/null +++ b/apps/emqx_exproto/src/emqx_exproto.app.src @@ -0,0 +1,10 @@ +{application, emqx_exproto, [ + {description, "ExProto Gateway"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [kernel, stdlib, emqx_gateway, grpc]}, + {env, []}, + {modules, []}, + {licenses, ["Apache 2.0"]}, + {links, []} +]}. diff --git a/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl b/apps/emqx_exproto/src/emqx_exproto.erl similarity index 93% rename from apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl rename to apps/emqx_exproto/src/emqx_exproto.erl index 0c25e5e08..1e6e0e6de 100644 --- a/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl +++ b/apps/emqx_exproto/src/emqx_exproto.erl @@ -14,12 +14,28 @@ %% limitations under the License. %%-------------------------------------------------------------------- -%% @doc The ExProto Gateway Implement interface --module(emqx_exproto_impl). - --behaviour(emqx_gateway_impl). +%% @doc The ExProto Gateway implement +-module(emqx_exproto). -include_lib("emqx/include/logger.hrl"). +-include_lib("emqx_gateway/include/emqx_gateway.hrl"). + +%% define a gateway named stomp +-gateway(#{ + name => exproto, + callback_module => ?MODULE, + config_schema_module => emqx_exproto_schema +}). + +%% callback_module must implement the emqx_gateway_impl behaviour +-behaviour(emqx_gateway_impl). + +%% callback for emqx_gateway_impl +-export([ + on_gateway_load/2, + on_gateway_update/3, + on_gateway_unload/2 +]). -import( emqx_gateway_utils, @@ -30,31 +46,8 @@ ] ). -%% APIs --export([ - reg/0, - unreg/0 -]). - --export([ - on_gateway_load/2, - on_gateway_update/3, - on_gateway_unload/2 -]). - %%-------------------------------------------------------------------- -%% APIs -%%-------------------------------------------------------------------- - -reg() -> - RegistryOptions = [{cbkmod, ?MODULE}], - emqx_gateway_registry:reg(exproto, RegistryOptions). - -unreg() -> - emqx_gateway_registry:unreg(exproto). - -%%-------------------------------------------------------------------- -%% emqx_gateway_registry callbacks +%% emqx_gateway_impl callbacks %%-------------------------------------------------------------------- on_gateway_load( diff --git a/apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl b/apps/emqx_exproto/src/emqx_exproto_channel.erl similarity index 99% rename from apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl rename to apps/emqx_exproto/src/emqx_exproto_channel.erl index 301154df0..7234e7a2f 100644 --- a/apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl +++ b/apps/emqx_exproto/src/emqx_exproto_channel.erl @@ -15,7 +15,8 @@ %%-------------------------------------------------------------------- -module(emqx_exproto_channel). --include("src/exproto/include/emqx_exproto.hrl"). + +-include("emqx_exproto.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/types.hrl"). diff --git a/apps/emqx_gateway/src/exproto/emqx_exproto_frame.erl b/apps/emqx_exproto/src/emqx_exproto_frame.erl similarity index 100% rename from apps/emqx_gateway/src/exproto/emqx_exproto_frame.erl rename to apps/emqx_exproto/src/emqx_exproto_frame.erl diff --git a/apps/emqx_gateway/src/exproto/emqx_exproto_gcli.erl b/apps/emqx_exproto/src/emqx_exproto_gcli.erl similarity index 100% rename from apps/emqx_gateway/src/exproto/emqx_exproto_gcli.erl rename to apps/emqx_exproto/src/emqx_exproto_gcli.erl diff --git a/apps/emqx_gateway/src/exproto/emqx_exproto_gsvr.erl b/apps/emqx_exproto/src/emqx_exproto_gsvr.erl similarity index 99% rename from apps/emqx_gateway/src/exproto/emqx_exproto_gsvr.erl rename to apps/emqx_exproto/src/emqx_exproto_gsvr.erl index 13bd49e55..5bbe7bf37 100644 --- a/apps/emqx_gateway/src/exproto/emqx_exproto_gsvr.erl +++ b/apps/emqx_exproto/src/emqx_exproto_gsvr.erl @@ -19,7 +19,7 @@ % -behaviour(emqx_exproto_v_1_connection_adapter_bhvr). --include("src/exproto/include/emqx_exproto.hrl"). +-include("emqx_exproto.hrl"). -include_lib("emqx/include/logger.hrl"). -define(IS_QOS(X), (X =:= 0 orelse X =:= 1 orelse X =:= 2)). diff --git a/apps/emqx_exproto/src/emqx_exproto_schema.erl b/apps/emqx_exproto/src/emqx_exproto_schema.erl new file mode 100644 index 000000000..eb44c030b --- /dev/null +++ b/apps/emqx_exproto/src/emqx_exproto_schema.erl @@ -0,0 +1,117 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_exproto_schema). + +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("typerefl/include/types.hrl"). + +-type ip_port() :: tuple() | integer(). + +-typerefl_from_string({ip_port/0, emqx_schema, to_ip_port}). + +-reflect_type([ + ip_port/0 +]). + +%% config schema provides +-export([fields/1, desc/1]). + +fields(exproto) -> + [ + {server, + sc( + ref(exproto_grpc_server), + #{ + required => true, + desc => ?DESC(exproto_server) + } + )}, + {handler, + sc( + ref(exproto_grpc_handler), + #{ + required => true, + desc => ?DESC(exproto_handler) + } + )}, + {mountpoint, emqx_gateway_schema:mountpoint()}, + {listeners, + sc(ref(emqx_gateway_schema, tcp_udp_listeners), #{desc => ?DESC(tcp_udp_listeners)})} + ] ++ emqx_gateway_schema:gateway_common_options(); +fields(exproto_grpc_server) -> + [ + {bind, + sc( + hoconsc:union([ip_port(), integer()]), + #{ + required => true, + desc => ?DESC(exproto_grpc_server_bind) + } + )}, + {ssl_options, + sc( + ref(ssl_server_opts), + #{ + required => {false, recursively}, + desc => ?DESC(exproto_grpc_server_ssl) + } + )} + ]; +fields(exproto_grpc_handler) -> + [ + {address, sc(binary(), #{required => true, desc => ?DESC(exproto_grpc_handler_address)})}, + {ssl_options, + sc( + ref(emqx_schema, "ssl_client_opts"), + #{ + required => {false, recursively}, + desc => ?DESC(exproto_grpc_handler_ssl) + } + )} + ]; +fields(ssl_server_opts) -> + emqx_schema:server_ssl_opts_schema( + #{ + depth => 10, + reuse_sessions => true, + versions => tls_all_available + }, + true + ). + +desc(exproto) -> + "Settings for EMQX extension protocol (exproto)."; +desc(exproto_grpc_server) -> + "Settings for the exproto gRPC server."; +desc(exproto_grpc_handler) -> + "Settings for the exproto gRPC connection handler."; +desc(ssl_server_opts) -> + "SSL configuration for the server."; +desc(_) -> + undefined. + +%%-------------------------------------------------------------------- +%% helpers + +sc(Type, Meta) -> + hoconsc:mk(Type, Meta). + +ref(StructName) -> + ref(?MODULE, StructName). + +ref(Mod, Field) -> + hoconsc:ref(Mod, Field). diff --git a/apps/emqx_gateway/test/emqx_exproto_SUITE.erl b/apps/emqx_exproto/test/emqx_exproto_SUITE.erl similarity index 99% rename from apps/emqx_gateway/test/emqx_exproto_SUITE.erl rename to apps/emqx_exproto/test/emqx_exproto_SUITE.erl index b476a40cb..a8ce41f44 100644 --- a/apps/emqx_gateway/test/emqx_exproto_SUITE.erl +++ b/apps/emqx_exproto/test/emqx_exproto_SUITE.erl @@ -76,6 +76,7 @@ metrics() -> [tcp, ssl, udp, dtls]. init_per_group(GrpName, Cfg) -> + application:load(emqx_exproto), put(grpname, GrpName), Svrs = emqx_exproto_echo_svr:start(), emqx_common_test_helpers:start_apps([emqx_authn, emqx_gateway], fun set_special_cfg/1), diff --git a/apps/emqx_gateway/test/emqx_exproto_echo_svr.erl b/apps/emqx_exproto/test/emqx_exproto_echo_svr.erl similarity index 100% rename from apps/emqx_gateway/test/emqx_exproto_echo_svr.erl rename to apps/emqx_exproto/test/emqx_exproto_echo_svr.erl diff --git a/apps/emqx_gateway/.gitignore b/apps/emqx_gateway/.gitignore index 5bff8a84d..a81bb07da 100644 --- a/apps/emqx_gateway/.gitignore +++ b/apps/emqx_gateway/.gitignore @@ -18,8 +18,4 @@ _build rebar3.crashdump *~ rebar.lock -src/exproto/emqx_exproto_pb.erl -src/exproto/emqx_exproto_v_1_connection_adapter_bhvr.erl -src/exproto/emqx_exproto_v_1_connection_adapter_client.erl -src/exproto/emqx_exproto_v_1_connection_handler_bhvr.erl -src/exproto/emqx_exproto_v_1_connection_handler_client.erl + diff --git a/apps/emqx_gateway/Makefile b/apps/emqx_gateway/Makefile deleted file mode 100644 index b2a54f7dd..000000000 --- a/apps/emqx_gateway/Makefile +++ /dev/null @@ -1,28 +0,0 @@ -## shallow clone for speed - -REBAR_GIT_CLONE_OPTIONS += --depth 1 -export REBAR_GIT_CLONE_OPTIONS - -REBAR = rebar3 -all: compile - -compile: - $(REBAR) compile - -clean: distclean - -ct: - $(REBAR) as test ct -v - -eunit: - $(REBAR) as test eunit - -xref: - $(REBAR) xref - -cover: - $(REBAR) cover - -distclean: - @rm -rf _build - @rm -f data/app.*.config data/vm.*.args rebar.lock diff --git a/apps/emqx_gateway/i18n/emqx_gateway_schema_i18n.conf b/apps/emqx_gateway/i18n/emqx_gateway_schema_i18n.conf index 561627241..1ffc5c6c1 100644 --- a/apps/emqx_gateway/i18n/emqx_gateway_schema_i18n.conf +++ b/apps/emqx_gateway/i18n/emqx_gateway_schema_i18n.conf @@ -1,56 +1,5 @@ emqx_gateway_schema { - exproto { - desc { - en: """The Extension Protocol configuration""" - zh: """ExProto 网关""" - } - } - - exproto_server { - desc { - en: """Configurations for starting the ConnectionAdapter service""" - zh: """配置 ExProto 网关需要启动的 ConnectionAdapter 服务。 -该服务用于提供客户端的认证、发布、订阅和数据下行等功能。""" - } - } - - exproto_grpc_server_bind { - desc { - en: """Listening address and port for the gRPC server.""" - zh: """服务监听地址和端口。""" - } - } - - exproto_grpc_server_ssl { - desc { - en: """SSL configuration for the gRPC server.""" - zh: """服务 SSL 配置。""" - } - } - - exproto_handler { - desc { - en: """Configurations for request to ConnectionHandler service""" - zh: """配置 ExProto 网关需要请求的 ConnectionHandler 服务地址。 -该服务用于给 ExProto 提供客户端的 Socket 事件处理、字节解码、订阅消息接收等功能。""" - } - } - - exproto_grpc_handler_address { - desc { - en: """gRPC server address.""" - zh: """对端 gRPC 服务器地址。""" - } - } - - exproto_grpc_handler_ssl { - desc { - en: """SSL configuration for the gRPC client.""" - zh: """gRPC 客户端的 SSL 配置。""" - } - } - gateway_common_enable { desc { en: """Whether to enable this gateway""" diff --git a/apps/emqx_gateway/rebar.config b/apps/emqx_gateway/rebar.config index 272783758..7e5228a9e 100644 --- a/apps/emqx_gateway/rebar.config +++ b/apps/emqx_gateway/rebar.config @@ -1,38 +1,5 @@ %% -*- mode: erlang -*- - {erl_opts, [debug_info]}. {deps, [ {emqx, {path, "../emqx"}} ]}. - -{plugins, [ - {grpc_plugin, {git, "https://github.com/HJianBo/grpc_plugin", {tag, "v0.10.2"}}} -]}. - -{grpc, [ - {protos, ["src/exproto/protos"]}, - {out_dir, "src/exproto/"}, - {gpb_opts, [ - {module_name_prefix, "emqx_"}, - {module_name_suffix, "_pb"} - ]} -]}. - -{provider_hooks, [ - {pre, [ - {compile, {grpc, gen}}, - {clean, {grpc, clean}} - ]} -]}. - -{xref_ignores, [emqx_exproto_pb]}. - -{cover_excl_mods, [ - emqx_exproto_pb, - emqx_exproto_v_1_connection_adapter_client, - emqx_exproto_v_1_connection_adapter_bhvr, - emqx_exproto_v_1_connection_handler_client, - emqx_exproto_v_1_connection_handler_bhvr -]}. - -{project_plugins, [erlfmt]}. diff --git a/apps/emqx_gateway/src/emqx_gateway.app.src b/apps/emqx_gateway/src/emqx_gateway.app.src index ced013497..850d38cdd 100644 --- a/apps/emqx_gateway/src/emqx_gateway.app.src +++ b/apps/emqx_gateway/src/emqx_gateway.app.src @@ -4,7 +4,7 @@ {vsn, "0.1.14"}, {registered, []}, {mod, {emqx_gateway_app, []}}, - {applications, [kernel, stdlib, grpc, emqx, emqx_authn, emqx_ctl]}, + {applications, [kernel, stdlib, emqx, emqx_authn, emqx_ctl]}, {env, []}, {modules, []}, {licenses, ["Apache 2.0"]}, diff --git a/apps/emqx_gateway/src/emqx_gateway_app.erl b/apps/emqx_gateway/src/emqx_gateway_app.erl index 5999e85c9..01a1aaddd 100644 --- a/apps/emqx_gateway/src/emqx_gateway_app.erl +++ b/apps/emqx_gateway/src/emqx_gateway_app.erl @@ -41,18 +41,11 @@ stop(_State) -> %% Internal funcs load_default_gateway_applications() -> - BuiltInGateways = [ - #{ - name => exproto, - callback_module => emqx_exproto_impl, - config_schema_module => emqx_gateway_schema - } - ], lists:foreach( fun(Def) -> load_gateway_application(Def) end, - emqx_gateway_utils:find_gateway_definations() ++ BuiltInGateways + emqx_gateway_utils:find_gateway_definations() ). load_gateway_application( diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl index 6a4811b94..84e9ee7e4 100644 --- a/apps/emqx_gateway/src/emqx_gateway_schema.erl +++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl @@ -60,79 +60,22 @@ namespace() -> gateway. tags() -> [<<"Gateway">>]. -roots() -> [gateway]. +roots() -> + [{gateway, sc(ref(?MODULE, gateway), #{importance => ?IMPORTANCE_HIDDEN})}]. fields(gateway) -> - [ - {exproto, - sc( - ref(exproto), - #{ - required => {false, recursively}, - desc => ?DESC(exproto) - } - )} - ] ++ gateway_schemas(); -fields(exproto) -> - [ - {server, - sc( - ref(exproto_grpc_server), - #{ - required => true, - desc => ?DESC(exproto_server) - } - )}, - {handler, - sc( - ref(exproto_grpc_handler), - #{ - required => true, - desc => ?DESC(exproto_handler) - } - )}, - {mountpoint, mountpoint()}, - {listeners, sc(ref(tcp_udp_listeners), #{desc => ?DESC(tcp_udp_listeners)})} - ] ++ gateway_common_options(); -fields(exproto_grpc_server) -> - [ - {bind, - sc( - hoconsc:union([ip_port(), integer()]), - #{ - required => true, - desc => ?DESC(exproto_grpc_server_bind) - } - )}, - {ssl_options, - sc( - ref(ssl_server_opts), - #{ - required => {false, recursively}, - desc => ?DESC(exproto_grpc_server_ssl) - } - )} - ]; -fields(exproto_grpc_handler) -> - [ - {address, sc(binary(), #{required => true, desc => ?DESC(exproto_grpc_handler_address)})}, - {ssl_options, - sc( - ref(emqx_schema, "ssl_client_opts"), - #{ - required => {false, recursively}, - desc => ?DESC(exproto_grpc_handler_ssl) - } - )} - ]; -fields(ssl_server_opts) -> - emqx_schema:server_ssl_opts_schema( - #{ - depth => 10, - reuse_sessions => true, - versions => tls_all_available - }, - true + lists:map( + fun(#{name := Name, config_schema_module := Mod}) -> + {Name, + sc( + ref(Mod, Name), + #{ + required => {false, recursively}, + desc => ?DESC(Name) + } + )} + end, + emqx_gateway_utils:find_gateway_definations() ); fields(clientinfo_override) -> [ @@ -217,14 +160,6 @@ fields(dtls_opts) -> desc(gateway) -> "EMQX Gateway configuration root."; -desc(exproto) -> - "Settings for EMQX extension protocol (exproto)."; -desc(exproto_grpc_server) -> - "Settings for the exproto gRPC server."; -desc(exproto_grpc_handler) -> - "Settings for the exproto gRPC connection handler."; -desc(ssl_server_opts) -> - "SSL configuration for the server."; desc(clientinfo_override) -> "ClientInfo override."; desc(udp_listeners) -> @@ -391,26 +326,11 @@ proxy_protocol_opts() -> %% dynamic schemas %% FIXME: don't hardcode the gateway names -gateway_schema(exproto) -> fields(exproto); gateway_schema(stomp) -> emqx_stomp_schema:fields(stomp); gateway_schema(mqttsn) -> emqx_mqttsn_schema:fields(mqttsn); gateway_schema(coap) -> emqx_coap_schema:fields(coap); -gateway_schema(lwm2m) -> emqx_lwm2m_schema:fields(lwm2m). - -gateway_schemas() -> - lists:map( - fun(#{name := Name, config_schema_module := Mod}) -> - {Name, - sc( - ref(Mod, Name), - #{ - required => {false, recursively}, - desc => ?DESC(Name) - } - )} - end, - emqx_gateway_utils:find_gateway_definations() - ). +gateway_schema(lwm2m) -> emqx_lwm2m_schema:fields(lwm2m); +gateway_schema(exproto) -> emqx_exproto_schema:fields(exproto). %%-------------------------------------------------------------------- %% helpers diff --git a/mix.exs b/mix.exs index 264d4f87e..15b6b997b 100644 --- a/mix.exs +++ b/mix.exs @@ -285,6 +285,7 @@ defmodule EMQXUmbrella.MixProject do emqx_mqttsn: :permanent, emqx_coap: :permanent, emqx_lwm2m: :permanent, + emqx_exproto: :permanent, emqx_exhook: :permanent, emqx_bridge: :permanent, emqx_rule_engine: :permanent, diff --git a/rebar.config.erl b/rebar.config.erl index 978f5ec87..d1f26fda2 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -393,6 +393,7 @@ relx_apps(ReleaseType, Edition) -> emqx_mqttsn, emqx_coap, emqx_lwm2m, + emqx_exproto, emqx_exhook, emqx_bridge, emqx_rule_engine, From b24ff9bc6e76bdbbe4518aae39b81c956515ae57 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Sat, 1 Apr 2023 10:30:37 +0800 Subject: [PATCH 153/196] test(gateway): refine all test cases --- apps/emqx_coap/src/emqx_coap_api.erl | 3 ++ apps/emqx_coap/src/emqx_coap_frame.erl | 2 ++ apps/emqx_coap/src/emqx_coap_tm.erl | 8 +++-- apps/emqx_coap/src/emqx_coap_transport.erl | 6 ++++ apps/emqx_gateway/src/emqx_gateway_utils.erl | 2 +- apps/emqx_gateway/test/emqx_gateway_SUITE.erl | 20 +++++++++---- .../test/emqx_gateway_api_SUITE.erl | 10 ++++++- .../test/emqx_gateway_authz_SUITE.erl | 3 +- .../test/emqx_gateway_cli_SUITE.erl | 11 +++---- .../test/emqx_gateway_conf_SUITE.erl | 1 + .../test/emqx_gateway_test_utils.erl | 6 ++++ apps/emqx_lwm2m/src/emqx_lwm2m_api.erl | 2 ++ apps/emqx_lwm2m/src/emqx_lwm2m_channel.erl | 6 ++-- apps/emqx_lwm2m/src/emqx_lwm2m_cmd.erl | 4 +-- apps/emqx_lwm2m/src/emqx_lwm2m_message.erl | 8 +++-- apps/emqx_lwm2m/src/emqx_lwm2m_session.erl | 10 +++---- apps/emqx_lwm2m/src/emqx_lwm2m_tlv.erl | 18 ++++++++---- .../src/emqx_lwm2m_xml_object_db.erl | 8 +++-- apps/emqx_lwm2m/test/emqx_lwm2m_SUITE.erl | 11 ++++++- apps/emqx_lwm2m/test/emqx_lwm2m_api_SUITE.erl | 29 +++---------------- apps/emqx_mqttsn/src/emqx_mqttsn_frame.erl | 16 +++++----- apps/emqx_mqttsn/src/emqx_mqttsn_registry.erl | 6 ++-- apps/emqx_stomp/src/emqx_stomp_heartbeat.erl | 2 ++ 23 files changed, 117 insertions(+), 75 deletions(-) diff --git a/apps/emqx_coap/src/emqx_coap_api.erl b/apps/emqx_coap/src/emqx_coap_api.erl index 50ea9829a..b4fce5473 100644 --- a/apps/emqx_coap/src/emqx_coap_api.erl +++ b/apps/emqx_coap/src/emqx_coap_api.erl @@ -34,9 +34,12 @@ -import(hoconsc, [mk/2, enum/1]). -import(emqx_dashboard_swagger, [error_codes/2]). +-elvis([{elvis_style, atom_naming_convention, disable}]). + %%-------------------------------------------------------------------- %% API %%-------------------------------------------------------------------- + namespace() -> "gateway_coap". api_spec() -> diff --git a/apps/emqx_coap/src/emqx_coap_frame.erl b/apps/emqx_coap/src/emqx_coap_frame.erl index a05116b14..535d07a94 100644 --- a/apps/emqx_coap/src/emqx_coap_frame.erl +++ b/apps/emqx_coap/src/emqx_coap_frame.erl @@ -55,6 +55,8 @@ -define(OPTION_PROXY_SCHEME, 39). -define(OPTION_SIZE1, 60). +-elvis([{elvis_style, no_if_expression, disable}]). + %%-------------------------------------------------------------------- %% API %%-------------------------------------------------------------------- diff --git a/apps/emqx_coap/src/emqx_coap_tm.erl b/apps/emqx_coap/src/emqx_coap_tm.erl index 297cbca6b..82f616b25 100644 --- a/apps/emqx_coap/src/emqx_coap_tm.erl +++ b/apps/emqx_coap/src/emqx_coap_tm.erl @@ -80,6 +80,8 @@ -import(emqx_coap_medium, [empty/0, iter/4, reset/1, proto_out/2]). +-elvis([{elvis_style, no_if_expression, disable}]). + %%-------------------------------------------------------------------- %% API %%-------------------------------------------------------------------- @@ -401,9 +403,9 @@ alloc_message_id(MsgId, TM) -> next_message_id(MsgId) -> Next = MsgId + 1, - if - Next >= ?MAX_MESSAGE_ID -> - 1; + case Next >= ?MAX_MESSAGE_ID of true -> + 1; + false -> Next end. diff --git a/apps/emqx_coap/src/emqx_coap_transport.erl b/apps/emqx_coap/src/emqx_coap_transport.erl index c58a8abbd..1948c969d 100644 --- a/apps/emqx_coap/src/emqx_coap_transport.erl +++ b/apps/emqx_coap/src/emqx_coap_transport.erl @@ -60,6 +60,12 @@ reply/2 ]). +-elvis([{elvis_style, atom_naming_convention, disable}]). +-elvis([{elvis_style, no_if_expression, disable}]). + +%%-------------------------------------------------------------------- +%% APIs + -spec new() -> transport(). new() -> new(undefined). diff --git a/apps/emqx_gateway/src/emqx_gateway_utils.erl b/apps/emqx_gateway/src/emqx_gateway_utils.erl index 9d71263f8..9d80de00e 100644 --- a/apps/emqx_gateway/src/emqx_gateway_utils.erl +++ b/apps/emqx_gateway/src/emqx_gateway_utils.erl @@ -601,7 +601,7 @@ find_attrs(App, Def) -> module_attributes(Module) -> try - Module:module_info(attributes) + apply(Module, module_info, [attributes]) catch error:undef -> [] end. diff --git a/apps/emqx_gateway/test/emqx_gateway_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_SUITE.erl index f611988a0..5120e096e 100644 --- a/apps/emqx_gateway/test/emqx_gateway_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_SUITE.erl @@ -33,6 +33,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Conf) -> emqx_config:erase(gateway), + emqx_gateway_test_utils:load_all_gateway_apps(), emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), emqx_common_test_helpers:start_apps([emqx_authn, emqx_gateway]), Conf. @@ -67,11 +68,11 @@ end_per_testcase(_TestCase, _Config) -> t_registered_gateway(_) -> [ - {coap, #{cbkmod := emqx_coap_impl}}, - {exproto, #{cbkmod := emqx_exproto_impl}}, - {lwm2m, #{cbkmod := emqx_lwm2m_impl}}, - {mqttsn, #{cbkmod := emqx_sn_impl}}, - {stomp, #{cbkmod := emqx_stomp_impl}} + {coap, #{cbkmod := emqx_coap}}, + {exproto, #{cbkmod := emqx_exproto}}, + {lwm2m, #{cbkmod := emqx_lwm2m}}, + {mqttsn, #{cbkmod := emqx_mqttsn}}, + {stomp, #{cbkmod := emqx_stomp}} ] = emqx_gateway:registered_gateway(). t_load_unload_list_lookup(_) -> @@ -187,7 +188,14 @@ read_lwm2m_conf(DataDir) -> Conf. setup_fake_usage_data(Lwm2mDataDir) -> - XmlDir = emqx_common_test_helpers:deps_path(emqx_gateway, "src/lwm2m/lwm2m_xml"), + XmlDir = filename:join( + [ + emqx_common_test_helpers:proj_root(), + "apps", + "emqx_lwm2m", + "lwm2m_xml" + ] + ), Lwm2mConf = read_lwm2m_conf(Lwm2mDataDir), ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, Lwm2mConf), emqx_config:put([gateway, lwm2m, xml_dir], XmlDir), diff --git a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl index 7aac45d61..bfcebd772 100644 --- a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl @@ -214,9 +214,17 @@ t_gateway_coap(_) -> t_gateway_lwm2m(_) -> {200, Gw} = request(get, "/gateways/lwm2m"), assert_gw_unloaded(Gw), + XmlDir = filename:join( + [ + emqx_common_test_helpers:proj_root(), + "apps", + "emqx_lwm2m", + "lwm2m_xml" + ] + ), GwConf = #{ name => <<"lwm2m">>, - xml_dir => <<"../../lib/emqx_gateway/src/lwm2m/lwm2m_xml">>, + xml_dir => list_to_binary(XmlDir), lifetime_min => <<"1s">>, lifetime_max => <<"1000s">>, qmode_time_window => <<"30s">>, diff --git a/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl index 2e44415aa..9bbcf2711 100644 --- a/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl @@ -66,6 +66,7 @@ end_per_group(AuthName, Conf) -> init_per_suite(Config) -> emqx_config:erase(gateway), + emqx_gateway_test_utils:load_all_gateway_apps(), init_gateway_conf(), meck:new(emqx_authz_file, [non_strict, passthrough, no_history, no_link]), meck:expect(emqx_authz_file, create, fun(S) -> S end), @@ -225,7 +226,7 @@ t_case_sn_subscribe(_) -> ) end, Sub(<<"/subscribe">>, fun(Data) -> - {ok, Msg, _, _} = emqx_sn_frame:parse(Data, undefined), + {ok, Msg, _, _} = emqx_mqttsn_frame:parse(Data, undefined), ?assertMatch({mqtt_sn_message, _, {_, 3, 0, Payload}}, Msg) end), Sub(<<"/badsubscribe">>, fun(Data) -> diff --git a/apps/emqx_gateway/test/emqx_gateway_cli_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_cli_SUITE.erl index c66785e00..a234dd126 100644 --- a/apps/emqx_gateway/test/emqx_gateway_cli_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_cli_SUITE.erl @@ -62,6 +62,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Conf) -> emqx_config:erase(gateway), + emqx_gateway_test_utils:load_all_gateway_apps(), emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_authn, emqx_gateway]), Conf. @@ -116,11 +117,11 @@ t_gateway_registry_usage(_) -> t_gateway_registry_list(_) -> emqx_gateway_cli:'gateway-registry'(["list"]), ?assertEqual( - "Registered Name: coap, Callback Module: emqx_coap_impl\n" - "Registered Name: exproto, Callback Module: emqx_exproto_impl\n" - "Registered Name: lwm2m, Callback Module: emqx_lwm2m_impl\n" - "Registered Name: mqttsn, Callback Module: emqx_sn_impl\n" - "Registered Name: stomp, Callback Module: emqx_stomp_impl\n", + "Registered Name: coap, Callback Module: emqx_coap\n" + "Registered Name: exproto, Callback Module: emqx_exproto\n" + "Registered Name: lwm2m, Callback Module: emqx_lwm2m\n" + "Registered Name: mqttsn, Callback Module: emqx_mqttsn\n" + "Registered Name: stomp, Callback Module: emqx_stomp\n", acc_print() ). diff --git a/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl index 6f6c2c45a..33c307770 100644 --- a/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl @@ -37,6 +37,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Conf) -> + emqx_gateway_test_utils:load_all_gateway_apps(), emqx_common_test_helpers:load_config(emqx_gateway_schema, <<"gateway {}">>), emqx_common_test_helpers:start_apps([emqx_conf, emqx_authn, emqx_gateway]), Conf. diff --git a/apps/emqx_gateway/test/emqx_gateway_test_utils.erl b/apps/emqx_gateway/test/emqx_gateway_test_utils.erl index deb602bc7..56a2fe7f9 100644 --- a/apps/emqx_gateway/test/emqx_gateway_test_utils.erl +++ b/apps/emqx_gateway/test/emqx_gateway_test_utils.erl @@ -101,6 +101,12 @@ assert_fields_exist(Ks, Map) -> end, Ks ). +load_all_gateway_apps() -> + application:load(emqx_stomp), + application:load(emqx_mqttsn), + application:load(emqx_coap), + application:load(emqx_lwm2m), + application:load(emqx_exproto). %%-------------------------------------------------------------------- %% http diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_api.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_api.erl index 2cd53d6eb..80afadb8e 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_api.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_api.erl @@ -32,6 +32,8 @@ -import(hoconsc, [mk/2, ref/1, ref/2]). -import(emqx_dashboard_swagger, [error_codes/2]). +-elvis([{elvis_style, atom_naming_convention, disable}]). + namespace() -> "lwm2m". api_spec() -> diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_channel.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_channel.erl index 54e5723cd..276b4f19d 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_channel.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_channel.erl @@ -464,14 +464,14 @@ check_lwm2m_version( _ -> false end, - if - IsValid -> + case IsValid of + true -> NConnInfo = ConnInfo#{ connected_at => erlang:system_time(millisecond), proto_ver => Ver }, {ok, Channel#channel{conninfo = NConnInfo}}; - true -> + _ -> ?SLOG(error, #{ msg => "reject_REGISTRE_request", reason => {unsupported_version, Ver} diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_cmd.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_cmd.erl index 9b1f6b65d..9ef3fb10d 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_cmd.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_cmd.erl @@ -292,9 +292,9 @@ make_response(Code, Ref = #{}) -> BaseRsp = make_base_response(Ref), make_data_response(BaseRsp, Code). -make_response(Code, Ref = #{}, _Format, Result) -> +make_response(Code, Ref = #{}, Format, Result) -> BaseRsp = make_base_response(Ref), - make_data_response(BaseRsp, Code, _Format, Result). + make_data_response(BaseRsp, Code, Format, Result). %% The base response format is what included in the request: %% diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_message.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_message.erl index e541e83f1..90a0306b7 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_message.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_message.erl @@ -412,9 +412,11 @@ byte_size_of_signed(UInt) -> byte_size_of_signed(UInt, N) -> BitSize = (8 * N - 1), Max = (1 bsl BitSize), - if - UInt =< Max -> N; - UInt > Max -> byte_size_of_signed(UInt, N + 1) + case UInt =< Max of + true -> + N; + false -> + byte_size_of_signed(UInt, N + 1) end. binary_to_number(NumStr) -> diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_session.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_session.erl index 67543a910..6c8b419ee 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_session.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_session.erl @@ -379,8 +379,8 @@ is_alternate_path(LinkAttrs) -> true; [AttrKey, _] when AttrKey =/= <<>> -> false; - _BadAttr -> - throw({bad_attr, _BadAttr}) + BadAttr -> + throw({bad_attr, BadAttr}) end end, LinkAttrs @@ -679,10 +679,10 @@ send_to_coap(#session{queue = Queue} = Session) -> case queue:out(Queue) of {{value, {Timestamp, Ctx, Req}}, Q2} -> Now = ?NOW, - if - Timestamp =:= 0 orelse Timestamp > Now -> - send_to_coap(Ctx, Req, Session#session{queue = Q2}); + case Timestamp =:= 0 orelse Timestamp > Now of true -> + send_to_coap(Ctx, Req, Session#session{queue = Q2}); + false -> send_to_coap(Session#session{queue = Q2}) end; {empty, _} -> diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_tlv.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_tlv.erl index 2f53573c4..314666638 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_tlv.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_tlv.erl @@ -37,13 +37,18 @@ -define(TLV_LEGNTH_16_BIT, 2). -define(TLV_LEGNTH_24_BIT, 3). -%---------------------------------------------------------------------------------------------------------------------------------------- -% [#{tlv_object_instance := Id11, value := Value11}, #{tlv_object_instance := Id12, value := Value12}, ...] +-elvis([{elvis_style, no_if_expression, disable}]). + +%%-------------------------------------------------------------------- +% [#{tlv_object_instance := Id11, value := Value11}, +% #{tlv_object_instance := Id12, value := Value12}, ...] % where Value11 and Value12 is a list: -% [#{tlv_resource_with_value => Id21, value => Value21}, #{tlv_multiple_resource => Id22, value = Value22}, ...] +% [#{tlv_resource_with_value => Id21, value => Value21}, +% #{tlv_multiple_resource => Id22, value = Value22}, ...] % where Value21 is a binary % Value22 is a list: -% [#{tlv_resource_instance => Id31, value => Value31}, #{tlv_resource_instance => Id32, value => Value32}, ...] +% [#{tlv_resource_instance => Id31, value => Value31}, +% #{tlv_resource_instance => Id32, value => Value32}, ...] % where Value31 and Value32 is a binary % % correspond to three levels: @@ -51,8 +56,9 @@ % 2) Resource Level % 3) Resource Instance Level % -% NOTE: TLV does not has object level, only has object instance level. It implies TLV can not represent multiple objects -%---------------------------------------------------------------------------------------------------------------------------------------- +% NOTE: TLV does not has object level, only has object instance level. +% It implies TLV can not represent multiple objects +%%-------------------------------------------------------------------- parse(Data) -> parse_loop(Data, []). diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object_db.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object_db.erl index 04c4c1af2..2908a65e0 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object_db.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object_db.erl @@ -45,6 +45,8 @@ -record(state, {}). +-elvis([{elvis_style, atom_naming_convention, disable}]). + %% ------------------------------------------------------------------ %% API Function Definitions %% ------------------------------------------------------------------ @@ -124,10 +126,10 @@ code_change(_OldVsn, State, _Extra) -> load(BaseDir) -> Wild = filename:join(BaseDir, "*.xml"), Wild2 = - if - is_binary(Wild) -> - erlang:binary_to_list(Wild); + case is_binary(Wild) of true -> + erlang:binary_to_list(Wild); + false -> Wild end, case filelib:wildcard(Wild2) of diff --git a/apps/emqx_lwm2m/test/emqx_lwm2m_SUITE.erl b/apps/emqx_lwm2m/test/emqx_lwm2m_SUITE.erl index 9abe16a35..dd2e3bbfd 100644 --- a/apps/emqx_lwm2m/test/emqx_lwm2m_SUITE.erl +++ b/apps/emqx_lwm2m/test/emqx_lwm2m_SUITE.erl @@ -177,11 +177,19 @@ default_config() -> default_config(#{}). default_config(Overrides) -> + XmlDir = filename:join( + [ + emqx_common_test_helpers:proj_root(), + "apps", + "emqx_lwm2m", + "lwm2m_xml" + ] + ), iolist_to_binary( io_lib:format( "\n" "gateway.lwm2m {\n" - " xml_dir = \"../../lib/emqx_gateway/src/lwm2m/lwm2m_xml\"\n" + " xml_dir = \"~s\"\n" " lifetime_min = 1s\n" " lifetime_max = 86400s\n" " qmode_time_window = 22\n" @@ -200,6 +208,7 @@ default_config(Overrides) -> " }\n" "}\n", [ + XmlDir, maps:get(auto_observe, Overrides, false), maps:get(bind, Overrides, ?PORT) ] diff --git a/apps/emqx_lwm2m/test/emqx_lwm2m_api_SUITE.erl b/apps/emqx_lwm2m/test/emqx_lwm2m_api_SUITE.erl index bfeeb2c9b..a1d048d76 100644 --- a/apps/emqx_lwm2m/test/emqx_lwm2m_api_SUITE.erl +++ b/apps/emqx_lwm2m/test/emqx_lwm2m_api_SUITE.erl @@ -28,29 +28,6 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). --define(CONF_DEFAULT, << - "\n" - "gateway.lwm2m {\n" - " xml_dir = \"../../lib/emqx_gateway/src/lwm2m/lwm2m_xml\"\n" - " lifetime_min = 100s\n" - " lifetime_max = 86400s\n" - " qmode_time_window = 200\n" - " auto_observe = false\n" - " mountpoint = \"lwm2m/${username}\"\n" - " update_msg_publish_condition = contains_object_list\n" - " translators {\n" - " command = {topic = \"/dn/#\", qos = 0}\n" - " response = {topic = \"/up/resp\", qos = 0}\n" - " notify = {topic = \"/up/notify\", qos = 0}\n" - " register = {topic = \"/up/resp\", qos = 0}\n" - " update = {topic = \"/up/resp\", qos = 0}\n" - " }\n" - " listeners.udp.default {\n" - " bind = 5783\n" - " }\n" - "}\n" ->>). - -define(assertExists(Map, Key), ?assertNotEqual(maps:get(Key, Map, undefined), undefined) ). @@ -83,7 +60,8 @@ all() -> init_per_suite(Config) -> application:load(emqx_gateway), application:load(emqx_lwm2m), - ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), + DefaultConfig = emqx_lwm2m_SUITE:default_config(), + ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, DefaultConfig), emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_authn]), Config. @@ -94,7 +72,8 @@ end_per_suite(Config) -> Config. init_per_testcase(_AllTestCase, Config) -> - ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), + DefaultConfig = emqx_lwm2m_SUITE:default_config(), + ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, DefaultConfig), {ok, _} = application:ensure_all_started(emqx_gateway), {ok, ClientUdpSock} = gen_udp:open(0, [binary, {active, false}]), diff --git a/apps/emqx_mqttsn/src/emqx_mqttsn_frame.erl b/apps/emqx_mqttsn/src/emqx_mqttsn_frame.erl index bf0fc52a4..3be2f1dc2 100644 --- a/apps/emqx_mqttsn/src/emqx_mqttsn_frame.erl +++ b/apps/emqx_mqttsn/src/emqx_mqttsn_frame.erl @@ -58,10 +58,10 @@ serialize_opts() -> %% Parse MQTT-SN Message %%-------------------------------------------------------------------- -parse(<<16#01:?byte, Len:?short, Type:?byte, Var/binary>>, _State) -> - {ok, parse(Type, Len - 4, Var), <<>>, _State}; -parse(<>, _State) -> - {ok, parse(Type, Len - 2, Var), <<>>, _State}. +parse(<<16#01:?byte, Len:?short, Type:?byte, Var/binary>>, State) -> + {ok, parse(Type, Len - 4, Var), <<>>, State}; +parse(<>, State) -> + {ok, parse(Type, Len - 2, Var), <<>>, State}. parse(Type, Len, Var) when Len =:= size(Var) -> #mqtt_sn_message{type = Type, variable = parse_var(Type, Var)}; @@ -160,9 +160,11 @@ parse_topic(2#11, Topic) -> Topic. serialize_pkt(#mqtt_sn_message{type = Type, variable = Var}, Opts) -> VarBin = serialize(Type, Var, Opts), VarLen = size(VarBin), - if - VarLen < 254 -> <<(VarLen + 2), Type, VarBin/binary>>; - true -> <<16#01, (VarLen + 4):?short, Type, VarBin/binary>> + case VarLen < 254 of + true -> + <<(VarLen + 2), Type, VarBin/binary>>; + false -> + <<16#01, (VarLen + 4):?short, Type, VarBin/binary>> end. serialize(?SN_ADVERTISE, {GwId, Duration}, _Opts) -> diff --git a/apps/emqx_mqttsn/src/emqx_mqttsn_registry.erl b/apps/emqx_mqttsn/src/emqx_mqttsn_registry.erl index 07da8c351..9db355a9b 100644 --- a/apps/emqx_mqttsn/src/emqx_mqttsn_registry.erl +++ b/apps/emqx_mqttsn/src/emqx_mqttsn_registry.erl @@ -151,9 +151,9 @@ init([InstaId, PredefTopics]) -> key = {predef, TopicName}, value = TopicId }), - if - TopicId > AccId -> TopicId; - true -> AccId + case TopicId > AccId of + true -> TopicId; + false -> AccId end end, 0, diff --git a/apps/emqx_stomp/src/emqx_stomp_heartbeat.erl b/apps/emqx_stomp/src/emqx_stomp_heartbeat.erl index f5ed99623..2e4239bdc 100644 --- a/apps/emqx_stomp/src/emqx_stomp_heartbeat.erl +++ b/apps/emqx_stomp/src/emqx_stomp_heartbeat.erl @@ -36,6 +36,8 @@ outgoing => #heartbeater{} }. +-elvis([{elvis_style, no_if_expression, disable}]). + %%-------------------------------------------------------------------- %% APIs %%-------------------------------------------------------------------- From 9577beaa4eeabef347119bdd7a55d1e42c72c619 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Sat, 1 Apr 2023 10:34:09 +0800 Subject: [PATCH 154/196] chore: update rebar.conf in emqx_exproto --- apps/emqx_exproto/rebar.config | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/emqx_exproto/rebar.config b/apps/emqx_exproto/rebar.config index 556404166..d21a7ece3 100644 --- a/apps/emqx_exproto/rebar.config +++ b/apps/emqx_exproto/rebar.config @@ -1,4 +1,5 @@ {erl_opts, [debug_info]}. +{deps, []}. {plugins, [ {grpc_plugin, {git, "https://github.com/HJianBo/grpc_plugin", {tag, "v0.10.2"}}} From 3a3879f99f43b2dfada7512876382a3231f554a5 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Sat, 1 Apr 2023 11:22:01 +0800 Subject: [PATCH 155/196] chore: update gateways deps --- apps/emqx_coap/rebar.config | 4 +++- apps/emqx_coap/src/emqx_coap.app.src | 2 +- apps/emqx_exproto/rebar.config | 4 +++- apps/emqx_exproto/src/emqx_exproto.app.src | 2 +- apps/emqx_lwm2m/rebar.config | 4 +++- apps/emqx_lwm2m/src/emqx_lwm2m.app.src | 2 +- apps/emqx_mqttsn/rebar.config | 4 +++- apps/emqx_mqttsn/src/emqx_mqttsn.app.src | 2 +- apps/emqx_stomp/rebar.config | 4 +++- apps/emqx_stomp/src/emqx_stomp.app.src | 2 +- mix.exs | 5 +++++ 11 files changed, 25 insertions(+), 10 deletions(-) diff --git a/apps/emqx_coap/rebar.config b/apps/emqx_coap/rebar.config index 2656fd554..c8675c3ba 100644 --- a/apps/emqx_coap/rebar.config +++ b/apps/emqx_coap/rebar.config @@ -1,2 +1,4 @@ {erl_opts, [debug_info]}. -{deps, []}. +{deps, [ {emqx, {path, "../../apps/emqx"}}, + {emqx_gateway, {path, "../../apps/emqx_gateway"}} + ]}. diff --git a/apps/emqx_coap/src/emqx_coap.app.src b/apps/emqx_coap/src/emqx_coap.app.src index 50b593ac7..c0f3f23da 100644 --- a/apps/emqx_coap/src/emqx_coap.app.src +++ b/apps/emqx_coap/src/emqx_coap.app.src @@ -2,7 +2,7 @@ {description, "CoAP Gateway"}, {vsn, "0.1.0"}, {registered, []}, - {applications, [kernel, stdlib, emqx_gateway]}, + {applications, [kernel, stdlib, emqx, emqx_gateway]}, {env, []}, {modules, []}, {licenses, ["Apache 2.0"]}, diff --git a/apps/emqx_exproto/rebar.config b/apps/emqx_exproto/rebar.config index d21a7ece3..928949c69 100644 --- a/apps/emqx_exproto/rebar.config +++ b/apps/emqx_exproto/rebar.config @@ -1,5 +1,7 @@ {erl_opts, [debug_info]}. -{deps, []}. +{deps, [ {emqx, {path, "../../apps/emqx"}}, + {emqx_gateway, {path, "../../apps/emqx_gateway"}} + ]}. {plugins, [ {grpc_plugin, {git, "https://github.com/HJianBo/grpc_plugin", {tag, "v0.10.2"}}} diff --git a/apps/emqx_exproto/src/emqx_exproto.app.src b/apps/emqx_exproto/src/emqx_exproto.app.src index 0b4ac3966..aa586a4fd 100644 --- a/apps/emqx_exproto/src/emqx_exproto.app.src +++ b/apps/emqx_exproto/src/emqx_exproto.app.src @@ -2,7 +2,7 @@ {description, "ExProto Gateway"}, {vsn, "0.1.0"}, {registered, []}, - {applications, [kernel, stdlib, emqx_gateway, grpc]}, + {applications, [kernel, stdlib, grpc, emqx, emqx_gateway]}, {env, []}, {modules, []}, {licenses, ["Apache 2.0"]}, diff --git a/apps/emqx_lwm2m/rebar.config b/apps/emqx_lwm2m/rebar.config index 2656fd554..c8675c3ba 100644 --- a/apps/emqx_lwm2m/rebar.config +++ b/apps/emqx_lwm2m/rebar.config @@ -1,2 +1,4 @@ {erl_opts, [debug_info]}. -{deps, []}. +{deps, [ {emqx, {path, "../../apps/emqx"}}, + {emqx_gateway, {path, "../../apps/emqx_gateway"}} + ]}. diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m.app.src b/apps/emqx_lwm2m/src/emqx_lwm2m.app.src index 08c3dbe3f..6338fa9d3 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m.app.src +++ b/apps/emqx_lwm2m/src/emqx_lwm2m.app.src @@ -2,7 +2,7 @@ {description, "LwM2M Gateway"}, {vsn, "0.1.0"}, {registered, []}, - {applications, [kernel, stdlib, emqx_gateway, emqx_coap]}, + {applications, [kernel, stdlib, emqx, emqx_gateway, emqx_coap]}, {env, []}, {modules, []}, {licenses, ["Apache 2.0"]}, diff --git a/apps/emqx_mqttsn/rebar.config b/apps/emqx_mqttsn/rebar.config index 2656fd554..c8675c3ba 100644 --- a/apps/emqx_mqttsn/rebar.config +++ b/apps/emqx_mqttsn/rebar.config @@ -1,2 +1,4 @@ {erl_opts, [debug_info]}. -{deps, []}. +{deps, [ {emqx, {path, "../../apps/emqx"}}, + {emqx_gateway, {path, "../../apps/emqx_gateway"}} + ]}. diff --git a/apps/emqx_mqttsn/src/emqx_mqttsn.app.src b/apps/emqx_mqttsn/src/emqx_mqttsn.app.src index 76acc648e..55e18e800 100644 --- a/apps/emqx_mqttsn/src/emqx_mqttsn.app.src +++ b/apps/emqx_mqttsn/src/emqx_mqttsn.app.src @@ -2,7 +2,7 @@ {description, "MQTT-SN Gateway"}, {vsn, "0.1.0"}, {registered, []}, - {applications, [kernel, stdlib, emqx_gateway]}, + {applications, [kernel, stdlib, emqx, emqx_gateway]}, {env, []}, {modules, []}, {licenses, ["Apache 2.0"]}, diff --git a/apps/emqx_stomp/rebar.config b/apps/emqx_stomp/rebar.config index 2656fd554..c8675c3ba 100644 --- a/apps/emqx_stomp/rebar.config +++ b/apps/emqx_stomp/rebar.config @@ -1,2 +1,4 @@ {erl_opts, [debug_info]}. -{deps, []}. +{deps, [ {emqx, {path, "../../apps/emqx"}}, + {emqx_gateway, {path, "../../apps/emqx_gateway"}} + ]}. diff --git a/apps/emqx_stomp/src/emqx_stomp.app.src b/apps/emqx_stomp/src/emqx_stomp.app.src index cd9670056..e118f8370 100644 --- a/apps/emqx_stomp/src/emqx_stomp.app.src +++ b/apps/emqx_stomp/src/emqx_stomp.app.src @@ -2,7 +2,7 @@ {description, "Stomp Gateway"}, {vsn, "0.1.0"}, {registered, []}, - {applications, [kernel, stdlib, emqx_gateway]}, + {applications, [kernel, stdlib, emqx, emqx_gateway]}, {env, []}, {modules, []}, {licenses, ["Apache 2.0"]}, diff --git a/mix.exs b/mix.exs index 15b6b997b..981755b01 100644 --- a/mix.exs +++ b/mix.exs @@ -221,6 +221,11 @@ defmodule EMQXUmbrella.MixProject do applications: applications(edition_type), skip_mode_validation_for: [ :emqx_gateway, + :emqx_stomp, + :emqx_mqttsn, + :emqx_coap, + :emqx_lwm2m, + :emqx_exproto, :emqx_dashboard, :emqx_resource, :emqx_connector, From b2d018f2490125b959459111e026a57f8e90afda Mon Sep 17 00:00:00 2001 From: JianBo He Date: Sat, 1 Apr 2023 18:51:39 +0800 Subject: [PATCH 156/196] chore: fix test cases --- apps/emqx_modules/test/emqx_telemetry_SUITE.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/emqx_modules/test/emqx_telemetry_SUITE.erl b/apps/emqx_modules/test/emqx_telemetry_SUITE.erl index cee255e77..a61781e13 100644 --- a/apps/emqx_modules/test/emqx_telemetry_SUITE.erl +++ b/apps/emqx_modules/test/emqx_telemetry_SUITE.erl @@ -45,6 +45,7 @@ init_per_suite(Config) -> ok = emqx_common_test_helpers:load_config(emqx_modules_schema, ?BASE_CONF, #{ raw_with_default => true }), + emqx_gateway_test_utils:load_all_gateway_apps(), emqx_common_test_helpers:start_apps( [emqx_conf, emqx_authn, emqx_authz, emqx_modules], fun set_special_configs/1 From 5138e6371c2979c6d79a9eb46d358367b24e3b3e Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 3 Apr 2023 09:47:05 +0800 Subject: [PATCH 157/196] chore: update changes --- changes/ce/feat-10278.en.md | 1 + changes/ce/feat-10278.zh.md | 1 + 2 files changed, 2 insertions(+) create mode 100644 changes/ce/feat-10278.en.md create mode 100644 changes/ce/feat-10278.zh.md diff --git a/changes/ce/feat-10278.en.md b/changes/ce/feat-10278.en.md new file mode 100644 index 000000000..d029c1420 --- /dev/null +++ b/changes/ce/feat-10278.en.md @@ -0,0 +1 @@ +Refactor the directory structure of all gateways. diff --git a/changes/ce/feat-10278.zh.md b/changes/ce/feat-10278.zh.md new file mode 100644 index 000000000..d2e738ec1 --- /dev/null +++ b/changes/ce/feat-10278.zh.md @@ -0,0 +1 @@ +重构所有网关的源码目录结构。 From 205e97fdca07b0d9446b1cd9342425c4f6e0aa78 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 3 Apr 2023 14:30:41 +0800 Subject: [PATCH 158/196] chore(gw): update README files --- apps/emqx_coap/README.md | 452 ++---------------------------------- apps/emqx_gateway/README.md | 346 +++------------------------ apps/emqx_mqttsn/README.md | 118 ++-------- apps/emqx_stomp/README.md | 84 ++----- 4 files changed, 98 insertions(+), 902 deletions(-) diff --git a/apps/emqx_coap/README.md b/apps/emqx_coap/README.md index 045db529d..405366e89 100644 --- a/apps/emqx_coap/README.md +++ b/apps/emqx_coap/README.md @@ -1,443 +1,31 @@ +# emqx_coap -# Table of Contents +The CoAP gateway implements publish, subscribe, and receive messages as standard +with [Publish-Subscribe Broker for the CoAP](https://datatracker.ietf.org/doc/html/draft-ietf-core-coap-pubsub-09). -1. [EMQX 5.0 CoAP Gateway](#org61e5bb8) - 1. [Features](#orgeddbc94) - 1. [PubSub Handler](#orgfc7be2d) - 2. [MQTT Handler](#org55be508) - 3. [Heartbeat](#org3d1a32e) - 4. [Query String](#org9a6b996) - 2. [Implementation](#org9985dfe) - 1. [Request/Response flow](#orge94210c) - 3. [Example](#ref_example) +## Quick Start +In EMQX 5.0, CoAP gateways can be configured and enabled through the Dashboard. +It can also be enabled via the HTTP API or emqx.conf, e.g. In emqx.conf: - +```properties +gateway.coap { -# EMQX 5.0 CoAP Gateway + mountpoint = "coap/" -emqx-coap is a CoAP Gateway for EMQX. It translates CoAP messages into MQTT messages and make it possible to communiate between CoAP clients and MQTT clients. + connection_required = false - - - -## Features - -- Partially achieves [Publish-Subscribe Broker for the Constrained Application Protocol (CoAP)](https://datatracker.ietf.org/doc/html/draft-ietf-core-coap-pubsub-09) - we called this as ps handler, include following functions: - - Publish - - Subscribe - - UnSubscribe -- Long connection and authorization verification called as MQTT handler - - - - -### PubSub Handler - -1. Publish - - Method: POST\ - URI Schema: ps/{+topic}{?q\*}\ - q\*: [Shared Options](#orgc50043b)\ - Response: - - - 2.04 "Changed" when success - - 4.00 "Bad Request" when error - - 4.01 "Unauthorized" when with wrong auth uri query - -2. Subscribe - - Method: GET - Options: - - - Observer = 0 - - URI Schema: ps/{+topic}{?q\*}\ - q\*: see [Shared Options](#orgc50043b)\ - Response: - - - 2.05 "Content" when success - - 4.00 "Bad Request" when error - - 4.01 "Unauthorized" when with wrong auth uri query - -``` - Client1 Client2 Broker - | | Subscribe | - | | ----- GET /ps/topic1 Observe:0 Token:XX ----> | - | | | - | | <---------- 2.05 Content Observe:10---------- | - | | | - | | | - | | Publish | - | ---------|----------- PUT /ps/topic1 "1033.3" --------> | - | | Notify | - | | <---------- 2.05 Content Observe:11 --------- | - | | | + listeners.udp.default { + bind = "5683" + max_connections = 1024000 + max_conn_rate = 1000 + } +} ``` -3. UnSubscribe - - Method : GET - Options: - - - Observe = 1 - - URI Schema: ps/{+topic}{?q\*}\ - q\*: see [Shared Options](#orgc50043b)\ - Response: - - - 2.07 "No Content" when success - - 4.00 "Bad Request" when error - - 4.01 "Unauthorized" when with wrong auth uri query - - - - -### MQTT Handler - - Establishing a connection is optional. If the CoAP client needs to use connection-based operations, it must first establish a connection. -At the same time, the connectionless mode and the connected mode cannot be mixed. -In connection mode, the Publish/Subscribe/UnSubscribe sent by the client must be has Token and ClientId in query string. -If the Token and Clientid is wrong/miss, EMQX will reset the request. -The communication token is the data carried in the response payload after the client successfully establishes a connection. -After obtaining the token, the client's subsequent request must attach "token=Token" to the Query String -ClientId is necessary when there is a connection, and is a unique identifier defined by the client. -The server manages the client through the ClientId. If the ClientId is wrong, EMQX will reset the request. - -1. Create a Connection - - Method: POST - URI Schema: mqtt/connection{?q\*} - q\*: - - - clientid := client uid - - username - - password - - Response: - - - 2.01 "Created" when success - - 4.00 "Bad Request" when error - - 4.01 "Unauthorized" wrong username or password - - Payload: Token if success - -2. Close a Connection - - Method : DELETE - URI Schema: mqtt/connection{?q\*} - q\*: - - - clientid := client uid - - token - - Response: - - - 2.01 "Deleted" when success - - 4.00 "Bad Request" when error - - 4.01 "Unauthorized" wrong clientid or token - - - - -### Heartbeat - -The Coap client can maintain the "connection" with the server through the heartbeat, -regardless of whether it is authenticated or not, -so that the server will not release related resources -Method : PUT -URI Schema: mqtt/connection{?q\*} -q\*: - -- clientid if authenticated -- token if authenticated - -Response: - -- 2.01 "Changed" when success -- 4.00 "Bad Request" when error -- 4.01 "Unauthorized" wrong clientid or token - - - - -### Query String - -CoAP gateway uses some options in query string to conversion between MQTT CoAP. - -1. Shared Options - - - clientid - - token - -2. Connect Options - - - username - - password - -3. Publish - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
OptionTypeDefault
retainbooleanfalse
qosMQTT QosSee here
expiryMessage Expiry Interval0(Never expiry)
- -4. Subscribe - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
OptionTypeDefault
qosMQTT QosSee here
nlMQTT Subscribe No Local0
rhMQTT Subscribe Retain Handing0
- -5. MQTT Qos <=> CoAP non/con - - 1.notif_type - Control the type of notify messages when the observed object has changed.Can be: - - - non - - con - - qos - in this value, MQTT Qos0 -> non, Qos1/Qos2 -> con - - 2.subscribe_qos - Control the qos of subscribe.Can be: - - - qos0 - - qos1 - - qos2 - - coap - in this value, CoAP non -> qos0, con -> qos1 - - 3.publish_qos - like subscribe_qos, but control the qos of the publish MQTT message - - - - -## Implementation - - - - -### Request/Response flow - -![img](./doc/flow.png) - -1. Authorization check - - Check whether the clientid and token in the query string match the current connection - -2. Session - - Manager the "Transport Manager" "Observe Resources Manager" and next message id - -3. Transport Mnager - - Manager "Transport" create/close/dispatch - -4. Observe resources Mnager - - Mnager observe topic and token - -5. Transport - - ![img](./doc/transport.png) - - 1. Shared State - - ![img](./doc/shared_state.png) - -6. Handler - - 1. pubsub - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
MethodObserveAction
GET0subscribe and reply result
GET1unsubscribe and reply result
POSTXpublish and reply result
- - 2. mqtt - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
MethodAction
PUTreply result
POSTreturn create connection action
DELETEreturn close connection action
- - - -## Example -1. Create Connection -``` -coap-client -m post -e "" "coap://127.0.0.1/mqtt/connection?clientid=123&username=admin&password=public" -``` -Server will return token **X** in payload - -2. Update Connection -``` -coap-client -m put -e "" "coap://127.0.0.1/mqtt/connection?clientid=123&token=X" -``` - -3. Publish -``` -coap-client -m post -e "Hellow" "obstoken" "coap://127.0.0.1/ps/coap/test?clientid=123&username=admin&password=public" -``` -if you want to publish with auth, you must first establish a connection, and then post publish request on the same socket, so libcoap client can't simulation publish with a token - -``` -coap-client -m post -e "Hellow" "coap://127.0.0.1/ps/coap/test?clientid=123&token=X" -``` - -4. Subscribe -``` -coap-client -m get -s 60 -O 6,0x00 -o - -T "obstoken" "coap://127.0.0.1/ps/coap/test?clientid=123&username=admin&password=public" -``` -**Or** - -``` -coap-client -m get -s 60 -O 6,0x00 -o - -T "obstoken" "coap://127.0.0.1/ps/coap/test?clientid=123&token=X" -``` -5. Close Connection -``` -coap-client -m delete -e "" "coap://127.0.0.1/mqtt/connection?clientid=123&token=X -``` +> Note: +> Configuring the gateway via emqx.conf requires changes on a per-node basis, +> but configuring it via Dashboard or the HTTP API will take effect across the cluster. +More documentations: [CoAP Gateway](https://www.emqx.io/docs/en/v5.0/gateway/coap.html) diff --git a/apps/emqx_gateway/README.md b/apps/emqx_gateway/README.md index be8f6cb35..57e8febab 100644 --- a/apps/emqx_gateway/README.md +++ b/apps/emqx_gateway/README.md @@ -1,332 +1,58 @@ # emqx_gateway -EMQX Gateway +EMQX Gateway is an application that managing all gateways in EMQX. -## Concept +It provides a set of standards to define how to implement a certain type of +protocol access on EMQX. For example: - EMQX Gateway Management - - Gateway-Registry (or Gateway Type) - - *Load - - *UnLoad - - *List +- Frame parsing +- Access authentication +- Publish and subscribe +- Configuration & Schema +- HTTP/CLI management interfaces - - Gateway - - *Create - - *Delete - - *Update - - *Stop-And-Start - - *Hot-Upgrade - - *Satrt/Enable - - *Stop/Disable - - Listener +There are some standard implementations available, such as [Stomp](../emqx_stomp/README.md), +[MQTT-SN](../emqx_mqttsn/README.md), [CoAP](../emqx_coap/README.md), +and [LwM2M](../emqx_lwm2m/README.md) gateway. -## ROADMAP +The emqx_gateway application depends on `emqx`, `emqx_authn`, `emqx_ctl` that +provide the foundation for protocol access. -Gateway v0.1: "Basic Functionals" - - Management support - - Conn/Frame/Protocol Template - - Support Stomp/MQTT-SN/CoAP/LwM2M/ExProto +## Three ways to create your gateway -Gateway v0.2: "Integration & Friendly Management" - - Hooks & Metrics & Statistic - - HTTP APIs - - Management in the cluster - - Integrate with AuthN - - Integrate with `emqx_config` - - Improve hocon config - - Mountpoint & ClientInfo's Metadata - - The Concept Review +## Raw Erlang Application -Gateway v0.3: "Fault tolerance and high availability" - - A common session modoule for message delivery policy - - The restart mechanism for gateway-instance - - Consistency of cluster state - - Configuration hot update +This approach is the same as in EMQX 4.x. You need to implement an Erlang application, +which is packaged in EMQX as a [Plugin](todo) or as a source code dependency. +In this approach, you do not need to respect any specifications of emqx_gateway, +and you can freely implement the features you need. -Gateway v1.0: "Best practices for each type of protocol" - - CoAP - - Stomp - - MQTT-SN - - LwM2M -### Compatible with EMQX +Steps guide: [Implement Gateway via Raw Application](doc/implement_gateway_via_raw_appliction.md) -> Why we need to compatible +## Respect emqx_gateway framework -1. Authentication -2. Hooks/Event system -3. Messages Mode & Rule Engine -4. Cluster registration -5. Metrics & Statistic +Similar to the first approach, you still need to implement an application using Erlang +and package it into EMQX. +The only difference is that you need to follow the standard behaviors(callbacks) provided +by emqx_gateway. -> How to do it +This is the approach we recommend. In this approach, your implementation can be managed +by the emqx_gateway framework, even if it may require you to understand more details about it. -> -### User Interface +Steps guide: [Implement Gateway via Gateway framework](doc/implement_gateway_via_gateway_framekwork.md) -#### Configurations +## Use ExProto Gateway (Non-Erlang developers) -```hocon -gateway { +If you want to implement your gateway using other programming languages such as +Java, Python, Go, etc. - ## ... some confs for top scope - .. - ## End. +You need to implement a gRPC service in the other programming language to parse +your device protocol and integrate it with EMQX. - ## Gateway Instances +Refer to: [ExProto Gateway](../emqx_exproto/README.md) - lwm2m[.name] { +## Cookbook for emqx_gateway framework - ## variable support - mountpoint: lwm2m/%e/ - - lifetime_min: 1s - lifetime_max: 86400s - #qmode_time_window: 22 - #auto_observe: off - - #update_msg_publish_condition: contains_object_list - - xml_dir: {{ platform_etc_dir }}/lwm2m_xml - - clientinfo_override: { - username: ${register.opts.uname} - password: ${register.opts.passwd} - clientid: ${epn} - } - - #authenticator: allow_anonymous - authenticator: [ - { - type: auth-http - method: post - //?? how to generate clientinfo ?? - params: $client.credential - } - ] - - translator: { - downlink: "dn/#" - uplink: { - notify: "up/notify" - response: "up/resp" - register: "up/resp" - update: "up/reps" - } - } - - %% ?? listener.$type.name ?? - listener.udp[.name] { - listen_on: 0.0.0.0:5683 - max_connections: 1024000 - max_conn_rate: 1000 - ## ?? udp keepalive in socket level ??? - #keepalive: - ## ?? udp proxy-protocol in socket level ??? - #proxy_protocol: on - #proxy_timeout: 30s - recbuf: 2KB - sndbuf: 2KB - buffer: 2KB - tune_buffer: off - #access: allow all - read_packets: 20 - } - - listener.dtls[.name] { - listen_on: 0.0.0.0:5684 - ... - } - } - - ## The CoAP Gateway - coap[.name] { - - #enable_stats: on - - authenticator: [ - ... - ] - - listener.udp[.name] { - ... - } - - listener.dtls[.name] { - ... - } -} - - ## The Stomp Gateway - stomp[.name] { - - allow_anonymous: true - - default_user.login: guest - default_user.passcode: guest - - frame.max_headers: 10 - frame.max_header_length: 1024 - frame.max_body_length: 8192 - - listener.tcp[.name] { - ... - } - - listener.ssl[.name] { - ... - } - } - - exproto[.name] { - - proto_name: DL-648 - - authenticators: [...] - - adapter: { - type: grpc - options: { - listen_on: 9100 - } - } - - handler: { - type: grpc - options: { - url: - } - } - - listener.tcp[.name] { - ... - } - } - - ## ============================ Enterpise gateways - - ## The JT/T 808 Gateway - jtt808[.name] { - - idle_timeout: 30s - enable_stats: on - max_packet_size: 8192 - - clientinfo_override: { - clientid: $phone - username: xxx - password: xxx - } - - authenticator: [ - { - type: auth-http - method: post - params: $clientinfo.credential - } - ] - - translator: { - subscribe: [jt808/%c/dn] - publish: [jt808/%c/up] - } - - listener.tcp[.name] { - ... - } - - listener.ssl[.name] { - ... - } - } - - gbt32960[.name] { - - frame.max_length: 8192 - retx_interval: 8s - retx_max_times: 3 - message_queue_len: 10 - - authenticators: [...] - - translator: { - ## upstream - login: gbt32960/${vin}/upstream/vlogin - logout: gbt32960/${vin}/upstream/vlogout - informing: gbt32960/${vin}/upstream/info - reinforming: gbt32960/${vin}/upstream/reinfo - ## downstream - downstream: gbt32960/${vin}/dnstream - response: gbt32960/${vin}/upstream/response - } - - listener.tcp[.name] { - ... - } - - listener.ssl[.name] { - ... - } - } - - privtcp[.name] { - - max_packet_size: 65535 - idle_timeout: 15s - - enable_stats: on - - force_gc_policy: 1000|1MB - force_shutdown_policy: 8000|800MB - - translator: { - up_topic: tcp/%c/up - dn_topic: tcp/%c/dn - } - - listener.tcp[.name]: { - ... - } - } -} -``` - -#### CLI - -##### Gateway - -```bash -## List all started gateway and gateway-instance -emqx_ctl gateway list -emqx_ctl gateway lookup -emqx_ctl gateway stop -emqx_ctl gateway start - -emqx_ctl gateway-registry re-searching -emqx_ctl gateway-registry list - -emqx_ctl gateway-clients list -emqx_ctl gateway-clients show -emqx_ctl gateway-clients kick - -## Banned ?? -emqx_ctl gateway-banned - -## Metrics -emqx_ctl gateway-metrics [] -``` - -#### Management by HTTP-API/Dashboard/ - -#### How to integrate a protocol to your platform - -### Develop your protocol gateway - -There are 3 way to create your protocol gateway for EMQX 5.0: - -1. Use Erlang to create a new emqx plugin to handle all of protocol packets (same as v5.0 before) - -2. Based on the emqx-gateway-impl-bhvr and emqx-gateway - -3. Use the gRPC Gateway +*WIP* diff --git a/apps/emqx_mqttsn/README.md b/apps/emqx_mqttsn/README.md index 67938b748..dd72a86a5 100644 --- a/apps/emqx_mqttsn/README.md +++ b/apps/emqx_mqttsn/README.md @@ -1,110 +1,34 @@ -# MQTT-SN Gateway +# emqx_mqttsn -EMQX MQTT-SN Gateway. +The MQTT-SN gateway is based on the +[MQTT-SN v1.2](https://www.oasis-open.org/committees/download.php/66091/MQTT-SN_spec_v1.2.pdf). -## Configure Plugin +## Quick Start +In EMQX 5.0, MQTT-SN gateway can be configured and enabled through the Dashboard. -File: etc/emqx_sn.conf +It can also be enabled via the HTTP API or emqx.conf, e.g. In emqx.conf: -``` -## The UDP port which emq-sn is listening on. -## -## Value: IP:Port | Port -## -## Examples: 1884, 127.0.0.1:1884, ::1:1884 -mqtt.sn.port = 1884 +```properties +gateway.mqttsn { -## The duration(seconds) that emq-sn broadcast ADVERTISE message through. -## -## Value: Second -mqtt.sn.advertise_duration = 900 + mountpoint = "mqtt/sn" -## The MQTT-SN Gateway id in ADVERTISE message. -## -## Value: Number -mqtt.sn.gateway_id = 1 + gateway_id = 1 -## To control whether write statistics data into ETS table for dashboard to read. -## -## Value: on | off -mqtt.sn.enable_stats = off + broadcast = true -## To control whether accept and process the received publish message with qos=-1. -## -## Value: on | off -mqtt.sn.enable_qos3 = off + enable_qos3 = true -## The pre-defined topic name corresponding to the pre-defined topic id of N. -## Note that the pre-defined topic id of 0 is reserved. -mqtt.sn.predefined.topic.0 = reserved -mqtt.sn.predefined.topic.1 = /predefined/topic/name/hello -mqtt.sn.predefined.topic.2 = /predefined/topic/name/nice - -## Default username for MQTT-SN. This parameter is optional. If specified, -## emq-sn will connect EMQ core with this username. It is useful if any auth -## plug-in is enabled. -## -## Value: String -mqtt.sn.username = mqtt_sn_user - -## This parameter is optional. Pair with username above. -## -## Value: String -mqtt.sn.password = abc + listeners.udp.default { + bind = 1884 + max_connections = 10240000 max_conn_rate = 1000 + } +} ``` -- mqtt.sn.port - * The UDP port which emqx-sn is listening on. -- mqtt.sn.advertise_duration - * The duration(seconds) that emqx-sn broadcast ADVERTISE message through. -- mqtt.sn.gateway_id - * Gateway id in ADVERTISE message. -- mqtt.sn.enable_stats - * To control whether write statistics data into ETS table for dashboard to read. -- mqtt.sn.enable_qos3 - * To control whether accept and process the received publish message with qos=-1. -- mqtt.sn.predefined.topic.N - * The pre-defined topic name corresponding to the pre-defined topic id of N. Note that the pre-defined topic id of 0 is reserved. -- mqtt.sn.username - * This parameter is optional. If specified, emqx-sn will connect EMQX core with this username. It is useful if any auth plug-in is enabled. -- mqtt.sn.password - * This parameter is optional. Pair with username above. +> Note: +> Configuring the gateway via emqx.conf requires changes on a per-node basis, +> but configuring it via Dashboard or the HTTP API will take effect across the cluster. -## Load Plugin - -``` -./bin/emqx_ctl plugins load emqx_sn -``` - -## Client - -### NOTE -- Topic ID is per-client, and will be cleared if client disconnected with broker or keepalive failure is detected in broker. -- Please register your topics again each time connected with broker. -- If your udp socket(mqtt-sn client) has successfully connected to broker, don't try to send another CONNECT on this socket again, which will lead to confusing behaviour. If you want to start from beging, please do as following: - + destroy your present socket and create a new socket to connect again - + or send DISCONNECT on the same socket and connect again. - -### Library - -- https://github.com/eclipse/paho.mqtt-sn.embedded-c/ -- https://github.com/ty4tw/MQTT-SN -- https://github.com/njh/mqtt-sn-tools -- https://github.com/arobenko/mqtt-sn - -### sleeping device - -PINGREQ must have a ClientId which is identical to the one in CONNECT message. Without ClientId, emqx-sn will ignore such PINGREQ. - -### pre-defined topics - -The mapping of a pre-defined topic id and topic name should be known inadvance by both client's application and gateway. We define this mapping info in emqx_sn.conf file, and which shall be kept equivalent in all client's side. - -## License - -Apache License Version 2.0 - -## Author - -EMQX Team. +More documentations: [MQTT-SN Gateway](https://www.emqx.io/docs/en/v5.0/gateway/mqttsn.html) diff --git a/apps/emqx_stomp/README.md b/apps/emqx_stomp/README.md index d96999aec..0c41ff520 100644 --- a/apps/emqx_stomp/README.md +++ b/apps/emqx_stomp/README.md @@ -1,73 +1,31 @@ +# emqx_stomp -# emqx-stomp +The Stomp Gateway is based on the +[Stomp v1.2](https://stomp.github.io/stomp-specification-1.2.html) and is +compatible with the Stomp v1.0 and v1.1 specification. +## Quick Start -The plugin adds STOMP 1.0/1.1/1.2 protocol supports to the EMQX broker. +In EMQX 5.0, Stomp gateway can be configured and enabled through the Dashboard. -The STOMP clients could PubSub to the MQTT clients. +It can also be enabled via the HTTP API or emqx.conf, e.g. In emqx.conf: -## Configuration +```properties +gateway.stomp { -etc/emqx_stomp.conf + mountpoint = "stomp/" -``` -## The Port that stomp listener will bind. -## -## Value: Port -stomp.listener = 61613 - -## The acceptor pool for stomp listener. -## -## Value: Number -stomp.listener.acceptors = 4 - -## Maximum number of concurrent stomp connections. -## -## Value: Number -stomp.listener.max_connections = 512 - -## Default login user -## -## Value: String -stomp.default_user.login = guest - -## Default login password -## -## Value: String -stomp.default_user.passcode = guest - -## Allow anonymous authentication. -## -## Value: true | false -stomp.allow_anonymous = true - -## Maximum numbers of frame headers. -## -## Value: Number -stomp.frame.max_headers = 10 - -## Maximum length of frame header. -## -## Value: Number -stomp.frame.max_header_length = 1024 - -## Maximum body length of frame. -## -## Value: Number -stomp.frame.max_body_length = 8192 + listeners.tcp.default { + bind = 61613 + acceptors = 16 + max_connections = 1024000 + max_conn_rate = 1000 + } +} ``` -## Load the Plugin - -``` -./bin/emqx_ctl plugins load emqx_stomp -``` - -## License - -Apache License Version 2.0 - -## Author - -EMQX Team. +> Note: +> Configuring the gateway via emqx.conf requires changes on a per-node basis, +> but configuring it via Dashboard or the HTTP API will take effect across the cluster. +More documentations: [Stomp Gateway](https://www.emqx.io/docs/en/v5.0/gateway/stomp.html) From c8dca74b188f4f60f3e99a1a1e89b75eb67db8ad Mon Sep 17 00:00:00 2001 From: firest Date: Thu, 30 Mar 2023 17:19:55 +0800 Subject: [PATCH 159/196] chore: bump hackney version --- lib-ee/emqx_ee_connector/rebar.config | 2 +- mix.exs | 2 +- rebar.config | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib-ee/emqx_ee_connector/rebar.config b/lib-ee/emqx_ee_connector/rebar.config index d49ce59c0..e754bd573 100644 --- a/lib-ee/emqx_ee_connector/rebar.config +++ b/lib-ee/emqx_ee_connector/rebar.config @@ -2,7 +2,7 @@ {deps, [ {hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.2.5"}}}, {influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.9"}}}, - {tdengine, {git, "https://github.com/emqx/tdengine-client-erl", {tag, "0.1.5"}}}, + {tdengine, {git, "https://github.com/emqx/tdengine-client-erl", {tag, "0.1.6"}}}, {clickhouse, {git, "https://github.com/emqx/clickhouse-client-erl", {tag, "0.3"}}}, {erlcloud, {git, "https://github.com/emqx/erlcloud.git", {tag,"3.5.16-emqx-1"}}}, {rocketmq, {git, "https://github.com/emqx/rocketmq-client-erl.git", {tag, "v0.5.1"}}}, diff --git a/mix.exs b/mix.exs index 514c9139d..600218e52 100644 --- a/mix.exs +++ b/mix.exs @@ -93,7 +93,7 @@ defmodule EMQXUmbrella.MixProject do github: "ninenines/ranch", ref: "a692f44567034dacf5efcaa24a24183788594eb7", override: true}, # in conflict by grpc and eetcd {:gpb, "4.19.5", override: true, runtime: false}, - {:hackney, github: "benoitc/hackney", tag: "1.18.1", override: true} + {:hackney, github: "emqx/hackney", tag: "1.18.1-1", override: true} ] ++ emqx_apps(profile_info, version) ++ enterprise_deps(profile_info) ++ bcrypt_dep() ++ jq_dep() ++ quicer_dep() diff --git a/rebar.config b/rebar.config index b641077ea..50a8124be 100644 --- a/rebar.config +++ b/rebar.config @@ -80,7 +80,7 @@ , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}} , {telemetry, "1.1.0"} - , {hackney, {git, "https://github.com/benoitc/hackney", {tag, "1.18.1"}}} + , {hackney, {git, "https://github.com/emqx/hackney.git", {tag, "1.18.1-1"}}} ]}. {xref_ignores, From 36000abf51b284e5b6271d516878604d0ec956fb Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 28 Mar 2023 16:50:21 +0200 Subject: [PATCH 160/196] refactor: relocate i18n files for apps/emqx --- .../emqx_authn_api_i18n.conf => rel/i18n/emqx_authn_api.hocon | 0 .../emqx_authn_http_i18n.conf => rel/i18n/emqx_authn_http.hocon | 0 .../emqx_authn_jwt_i18n.conf => rel/i18n/emqx_authn_jwt.hocon | 0 .../i18n/emqx_authn_mnesia.hocon | 0 .../i18n/emqx_authn_mongodb.hocon | 0 .../i18n/emqx_authn_mysql.hocon | 0 .../i18n/emqx_authn_pgsql.hocon | 0 .../i18n/emqx_authn_redis.hocon | 0 .../i18n/emqx_authn_schema.hocon | 0 .../i18n/emqx_authn_user_import_api.hocon | 0 .../i18n/emqx_authz_api_cache.hocon | 0 .../i18n/emqx_authz_api_mnesia.hocon | 0 .../i18n/emqx_authz_api_schema.hocon | 0 .../i18n/emqx_authz_api_settings.hocon | 0 .../i18n/emqx_authz_api_sources.hocon | 0 .../i18n/emqx_authz_schema.hocon | 0 .../i18n/emqx_auto_subscribe_api.hocon | 0 .../i18n/emqx_auto_subscribe_schema.hocon | 0 .../i18n/emqx_bridge_api.conf => rel/i18n/emqx_bridge_api.hocon | 0 .../i18n/emqx_bridge_mqtt_schema.hocon | 0 .../emqx_bridge_schema.conf => rel/i18n/emqx_bridge_schema.hocon | 0 .../i18n/emqx_bridge_webhook_schema.hocon | 0 .../i18n/emqx_coap_api_i18n.conf => rel/i18n/emqx_coap_api.hocon | 0 .../emqx_conf_schema.conf => rel/i18n/emqx_conf_schema.hocon | 0 .../emqx_connector_api.conf => rel/i18n/emqx_connector_api.hocon | 0 .../i18n/emqx_connector_http.hocon | 0 .../i18n/emqx_connector_ldap.hocon | 0 .../i18n/emqx_connector_mongo.hocon | 0 .../i18n/emqx_connector_mqtt.hocon | 0 .../i18n/emqx_connector_mqtt_schema.hocon | 0 .../i18n/emqx_connector_mysql.hocon | 0 .../i18n/emqx_connector_pgsql.hocon | 0 .../i18n/emqx_connector_redis.hocon | 0 .../i18n/emqx_connector_schema_lib.hocon | 0 .../i18n/emqx_dashboard_api.hocon | 0 .../i18n/emqx_dashboard_schema.hocon | 0 .../i18n/emqx_delayed_api.hocon | 0 .../i18n/emqx_ee_bridge_cassa.hocon | 0 .../i18n/emqx_ee_bridge_clickhouse.hocon | 0 .../i18n/emqx_ee_bridge_dynamo.hocon | 0 .../i18n/emqx_ee_bridge_gcp_pubsub.hocon | 0 .../i18n/emqx_ee_bridge_hstreamdb.hocon | 0 .../i18n/emqx_ee_bridge_influxdb.hocon | 0 .../i18n/emqx_ee_bridge_kafka.hocon | 0 .../i18n/emqx_ee_bridge_mongodb.hocon | 0 .../i18n/emqx_ee_bridge_mysql.hocon | 0 .../i18n/emqx_ee_bridge_pgsql.hocon | 0 .../i18n/emqx_ee_bridge_redis.hocon | 0 .../i18n/emqx_ee_bridge_rocketmq.hocon | 0 .../i18n/emqx_ee_bridge_tdengine.hocon | 0 .../i18n/emqx_ee_connector_cassa.hocon | 0 .../i18n/emqx_ee_connector_clickhouse.hocon | 1 - .../i18n/emqx_ee_connector_dynamo.hocon | 0 .../i18n/emqx_ee_connector_hstreamdb.hocon | 0 .../i18n/emqx_ee_connector_influxdb.hocon | 0 .../i18n/emqx_ee_connector_rocketmq.hocon | 0 .../i18n/emqx_ee_connector_tdengine.hocon | 0 .../emqx_exhook_api_i18n.conf => rel/i18n/emqx_exhook_api.hocon | 0 .../emqx_exhook_i18n.conf => rel/i18n/emqx_exhook_schema.hocon | 0 .../i18n/emqx_gateway_api.hocon | 0 .../i18n/emqx_gateway_api_authn.hocon | 0 .../i18n/emqx_gateway_api_clients.hocon | 0 .../i18n/emqx_gateway_api_listeners.hocon | 0 .../i18n/emqx_gateway_schema.hocon | 0 .../i18n/emqx_license_http_api.hocon | 0 .../i18n/emqx_license_schema.hocon | 0 .../emqx_limiter_i18n.conf => rel/i18n/emqx_limiter_schema.hocon | 0 .../emqx_lwm2m_api_i18n.conf => rel/i18n/emqx_lwm2m_api.hocon | 0 .../i18n/emqx_mgmt_api_alarms.hocon | 0 .../i18n/emqx_mgmt_api_banned.hocon | 0 .../i18n/emqx_mgmt_api_key_schema.hocon | 0 .../i18n/emqx_mgmt_api_publish.hocon | 1 - .../i18n/emqx_mgmt_api_status.hocon | 0 .../i18n/emqx_modules_schema.hocon | 0 .../i18n/emqx_plugins_schema.hocon | 0 .../i18n/emqx_prometheus_schema.hocon | 0 .../i18n/emqx_psk_i18n.conf => rel/i18n/emqx_psk_schema.hocon | 0 .../i18n/emqx_resource_schema.hocon | 0 .../i18n/emqx_retainer_api.hocon | 0 .../i18n/emqx_retainer_schema.hocon | 0 .../i18n/emqx_rewrite_api.hocon | 0 .../i18n/emqx_rule_api_schema.hocon | 0 .../i18n/emqx_rule_engine_api.hocon | 0 .../i18n/emqx_rule_engine_schema.hocon | 0 .../i18n/emqx_schema_i18n.conf => rel/i18n/emqx_schema.hocon | 0 .../i18n/emqx_slow_subs_api.hocon | 0 .../i18n/emqx_slow_subs_schema.hocon | 0 .../emqx_statsd_api_i18n.conf => rel/i18n/emqx_statsd_api.hocon | 0 .../i18n/emqx_statsd_schema.hocon | 0 .../i18n/emqx_telemetry_api.hocon | 0 .../i18n/emqx_topic_metrics_api.hocon | 0 91 files changed, 2 deletions(-) rename apps/emqx_authn/i18n/emqx_authn_api_i18n.conf => rel/i18n/emqx_authn_api.hocon (100%) rename apps/emqx_authn/i18n/emqx_authn_http_i18n.conf => rel/i18n/emqx_authn_http.hocon (100%) rename apps/emqx_authn/i18n/emqx_authn_jwt_i18n.conf => rel/i18n/emqx_authn_jwt.hocon (100%) rename apps/emqx_authn/i18n/emqx_authn_mnesia_i18n.conf => rel/i18n/emqx_authn_mnesia.hocon (100%) rename apps/emqx_authn/i18n/emqx_authn_mongodb_i18n.conf => rel/i18n/emqx_authn_mongodb.hocon (100%) rename apps/emqx_authn/i18n/emqx_authn_mysql_i18n.conf => rel/i18n/emqx_authn_mysql.hocon (100%) rename apps/emqx_authn/i18n/emqx_authn_pgsql_i18n.conf => rel/i18n/emqx_authn_pgsql.hocon (100%) rename apps/emqx_authn/i18n/emqx_authn_redis_i18n.conf => rel/i18n/emqx_authn_redis.hocon (100%) rename apps/emqx_authn/i18n/emqx_authn_schema_i18n.conf => rel/i18n/emqx_authn_schema.hocon (100%) rename apps/emqx_authn/i18n/emqx_authn_user_import_api_i18n.conf => rel/i18n/emqx_authn_user_import_api.hocon (100%) rename apps/emqx_authz/i18n/emqx_authz_api_cache_i18n.conf => rel/i18n/emqx_authz_api_cache.hocon (100%) rename apps/emqx_authz/i18n/emqx_authz_api_mnesia_i18n.conf => rel/i18n/emqx_authz_api_mnesia.hocon (100%) rename apps/emqx_authz/i18n/emqx_authz_api_schema_i18n.conf => rel/i18n/emqx_authz_api_schema.hocon (100%) rename apps/emqx_authz/i18n/emqx_authz_api_settings_i18n.conf => rel/i18n/emqx_authz_api_settings.hocon (100%) rename apps/emqx_authz/i18n/emqx_authz_api_sources_i18n.conf => rel/i18n/emqx_authz_api_sources.hocon (100%) rename apps/emqx_authz/i18n/emqx_authz_schema_i18n.conf => rel/i18n/emqx_authz_schema.hocon (100%) rename apps/emqx_auto_subscribe/i18n/emqx_auto_subscribe_api_i18n.conf => rel/i18n/emqx_auto_subscribe_api.hocon (100%) rename apps/emqx_auto_subscribe/i18n/emqx_auto_subscribe_i18n.conf => rel/i18n/emqx_auto_subscribe_schema.hocon (100%) rename apps/emqx_bridge/i18n/emqx_bridge_api.conf => rel/i18n/emqx_bridge_api.hocon (100%) rename apps/emqx_bridge/i18n/emqx_bridge_mqtt_schema.conf => rel/i18n/emqx_bridge_mqtt_schema.hocon (100%) rename apps/emqx_bridge/i18n/emqx_bridge_schema.conf => rel/i18n/emqx_bridge_schema.hocon (100%) rename apps/emqx_bridge/i18n/emqx_bridge_webhook_schema.conf => rel/i18n/emqx_bridge_webhook_schema.hocon (100%) rename apps/emqx_gateway/i18n/emqx_coap_api_i18n.conf => rel/i18n/emqx_coap_api.hocon (100%) rename apps/emqx_conf/i18n/emqx_conf_schema.conf => rel/i18n/emqx_conf_schema.hocon (100%) rename apps/emqx_connector/i18n/emqx_connector_api.conf => rel/i18n/emqx_connector_api.hocon (100%) rename apps/emqx_connector/i18n/emqx_connector_http.conf => rel/i18n/emqx_connector_http.hocon (100%) rename apps/emqx_connector/i18n/emqx_connector_ldap.conf => rel/i18n/emqx_connector_ldap.hocon (100%) rename apps/emqx_connector/i18n/emqx_connector_mongo.conf => rel/i18n/emqx_connector_mongo.hocon (100%) rename apps/emqx_connector/i18n/emqx_connector_mqtt.conf => rel/i18n/emqx_connector_mqtt.hocon (100%) rename apps/emqx_connector/i18n/emqx_connector_mqtt_schema.conf => rel/i18n/emqx_connector_mqtt_schema.hocon (100%) rename apps/emqx_connector/i18n/emqx_connector_mysql.conf => rel/i18n/emqx_connector_mysql.hocon (100%) rename apps/emqx_connector/i18n/emqx_connector_pgsql.conf => rel/i18n/emqx_connector_pgsql.hocon (100%) rename apps/emqx_connector/i18n/emqx_connector_redis.conf => rel/i18n/emqx_connector_redis.hocon (100%) rename apps/emqx_connector/i18n/emqx_connector_schema_lib.conf => rel/i18n/emqx_connector_schema_lib.hocon (100%) rename apps/emqx_dashboard/i18n/emqx_dashboard_api_i18n.conf => rel/i18n/emqx_dashboard_api.hocon (100%) rename apps/emqx_dashboard/i18n/emqx_dashboard_i18n.conf => rel/i18n/emqx_dashboard_schema.hocon (100%) rename apps/emqx_modules/i18n/emqx_delayed_api_i18n.conf => rel/i18n/emqx_delayed_api.hocon (100%) rename lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_cassa.conf => rel/i18n/emqx_ee_bridge_cassa.hocon (100%) rename lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_clickhouse.conf => rel/i18n/emqx_ee_bridge_clickhouse.hocon (100%) rename lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_dynamo.conf => rel/i18n/emqx_ee_bridge_dynamo.hocon (100%) rename lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_gcp_pubsub.conf => rel/i18n/emqx_ee_bridge_gcp_pubsub.hocon (100%) rename lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstreamdb.conf => rel/i18n/emqx_ee_bridge_hstreamdb.hocon (100%) rename lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf => rel/i18n/emqx_ee_bridge_influxdb.hocon (100%) rename lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf => rel/i18n/emqx_ee_bridge_kafka.hocon (100%) rename lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mongodb.conf => rel/i18n/emqx_ee_bridge_mongodb.hocon (100%) rename lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mysql.conf => rel/i18n/emqx_ee_bridge_mysql.hocon (100%) rename lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_pgsql.conf => rel/i18n/emqx_ee_bridge_pgsql.hocon (100%) rename lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_redis.conf => rel/i18n/emqx_ee_bridge_redis.hocon (100%) rename lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_rocketmq.conf => rel/i18n/emqx_ee_bridge_rocketmq.hocon (100%) rename lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_tdengine.conf => rel/i18n/emqx_ee_bridge_tdengine.hocon (100%) rename lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_cassa.conf => rel/i18n/emqx_ee_connector_cassa.hocon (100%) rename lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_clickhouse.conf => rel/i18n/emqx_ee_connector_clickhouse.hocon (99%) rename lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_dynamo.conf => rel/i18n/emqx_ee_connector_dynamo.hocon (100%) rename lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstreamdb.conf => rel/i18n/emqx_ee_connector_hstreamdb.hocon (100%) rename lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_influxdb.conf => rel/i18n/emqx_ee_connector_influxdb.hocon (100%) rename lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_rocketmq.conf => rel/i18n/emqx_ee_connector_rocketmq.hocon (100%) rename lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_tdengine.conf => rel/i18n/emqx_ee_connector_tdengine.hocon (100%) rename apps/emqx_exhook/i18n/emqx_exhook_api_i18n.conf => rel/i18n/emqx_exhook_api.hocon (100%) rename apps/emqx_exhook/i18n/emqx_exhook_i18n.conf => rel/i18n/emqx_exhook_schema.hocon (100%) rename apps/emqx_gateway/i18n/emqx_gateway_api_i18n.conf => rel/i18n/emqx_gateway_api.hocon (100%) rename apps/emqx_gateway/i18n/emqx_gateway_api_authn_i18n.conf => rel/i18n/emqx_gateway_api_authn.hocon (100%) rename apps/emqx_gateway/i18n/emqx_gateway_api_clients_i18n.conf => rel/i18n/emqx_gateway_api_clients.hocon (100%) rename apps/emqx_gateway/i18n/emqx_gateway_api_listeners_i18n.conf => rel/i18n/emqx_gateway_api_listeners.hocon (100%) rename apps/emqx_gateway/i18n/emqx_gateway_schema_i18n.conf => rel/i18n/emqx_gateway_schema.hocon (100%) rename lib-ee/emqx_license/i18n/emqx_license_http_api.conf => rel/i18n/emqx_license_http_api.hocon (100%) rename lib-ee/emqx_license/i18n/emqx_license_schema_i18n.conf => rel/i18n/emqx_license_schema.hocon (100%) rename apps/emqx/i18n/emqx_limiter_i18n.conf => rel/i18n/emqx_limiter_schema.hocon (100%) rename apps/emqx_gateway/i18n/emqx_lwm2m_api_i18n.conf => rel/i18n/emqx_lwm2m_api.hocon (100%) rename apps/emqx_management/i18n/emqx_mgmt_api_alarms_i18n.conf => rel/i18n/emqx_mgmt_api_alarms.hocon (100%) rename apps/emqx_management/i18n/emqx_mgmt_api_banned_i18n.conf => rel/i18n/emqx_mgmt_api_banned.hocon (100%) rename apps/emqx_management/i18n/emqx_mgmt_api_key_i18n.conf => rel/i18n/emqx_mgmt_api_key_schema.hocon (100%) rename apps/emqx_management/i18n/emqx_mgmt_api_publish_i18n.conf => rel/i18n/emqx_mgmt_api_publish.hocon (99%) rename apps/emqx_management/i18n/emqx_mgmt_api_status_i18n.conf => rel/i18n/emqx_mgmt_api_status.hocon (100%) rename apps/emqx_modules/i18n/emqx_modules_schema_i18n.conf => rel/i18n/emqx_modules_schema.hocon (100%) rename apps/emqx_plugins/i18n/emqx_plugins_schema.conf => rel/i18n/emqx_plugins_schema.hocon (100%) rename apps/emqx_prometheus/i18n/emqx_prometheus_schema_i18n.conf => rel/i18n/emqx_prometheus_schema.hocon (100%) rename apps/emqx_psk/i18n/emqx_psk_i18n.conf => rel/i18n/emqx_psk_schema.hocon (100%) rename apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf => rel/i18n/emqx_resource_schema.hocon (100%) rename apps/emqx_retainer/i18n/emqx_retainer_api_i18n.conf => rel/i18n/emqx_retainer_api.hocon (100%) rename apps/emqx_retainer/i18n/emqx_retainer_i18n.conf => rel/i18n/emqx_retainer_schema.hocon (100%) rename apps/emqx_modules/i18n/emqx_rewrite_api_i18n.conf => rel/i18n/emqx_rewrite_api.hocon (100%) rename apps/emqx_rule_engine/i18n/emqx_rule_api_schema.conf => rel/i18n/emqx_rule_api_schema.hocon (100%) rename apps/emqx_rule_engine/i18n/emqx_rule_engine_api.conf => rel/i18n/emqx_rule_engine_api.hocon (100%) rename apps/emqx_rule_engine/i18n/emqx_rule_engine_schema.conf => rel/i18n/emqx_rule_engine_schema.hocon (100%) rename apps/emqx/i18n/emqx_schema_i18n.conf => rel/i18n/emqx_schema.hocon (100%) rename apps/emqx_slow_subs/i18n/emqx_slow_subs_api_i18n.conf => rel/i18n/emqx_slow_subs_api.hocon (100%) rename apps/emqx_slow_subs/i18n/emqx_slow_subs_i18n.conf => rel/i18n/emqx_slow_subs_schema.hocon (100%) rename apps/emqx_statsd/i18n/emqx_statsd_api_i18n.conf => rel/i18n/emqx_statsd_api.hocon (100%) rename apps/emqx_statsd/i18n/emqx_statsd_schema_i18n.conf => rel/i18n/emqx_statsd_schema.hocon (100%) rename apps/emqx_modules/i18n/emqx_telemetry_api_i18n.conf => rel/i18n/emqx_telemetry_api.hocon (100%) rename apps/emqx_modules/i18n/emqx_topic_metrics_api_i18n.conf => rel/i18n/emqx_topic_metrics_api.hocon (100%) diff --git a/apps/emqx_authn/i18n/emqx_authn_api_i18n.conf b/rel/i18n/emqx_authn_api.hocon similarity index 100% rename from apps/emqx_authn/i18n/emqx_authn_api_i18n.conf rename to rel/i18n/emqx_authn_api.hocon diff --git a/apps/emqx_authn/i18n/emqx_authn_http_i18n.conf b/rel/i18n/emqx_authn_http.hocon similarity index 100% rename from apps/emqx_authn/i18n/emqx_authn_http_i18n.conf rename to rel/i18n/emqx_authn_http.hocon diff --git a/apps/emqx_authn/i18n/emqx_authn_jwt_i18n.conf b/rel/i18n/emqx_authn_jwt.hocon similarity index 100% rename from apps/emqx_authn/i18n/emqx_authn_jwt_i18n.conf rename to rel/i18n/emqx_authn_jwt.hocon diff --git a/apps/emqx_authn/i18n/emqx_authn_mnesia_i18n.conf b/rel/i18n/emqx_authn_mnesia.hocon similarity index 100% rename from apps/emqx_authn/i18n/emqx_authn_mnesia_i18n.conf rename to rel/i18n/emqx_authn_mnesia.hocon diff --git a/apps/emqx_authn/i18n/emqx_authn_mongodb_i18n.conf b/rel/i18n/emqx_authn_mongodb.hocon similarity index 100% rename from apps/emqx_authn/i18n/emqx_authn_mongodb_i18n.conf rename to rel/i18n/emqx_authn_mongodb.hocon diff --git a/apps/emqx_authn/i18n/emqx_authn_mysql_i18n.conf b/rel/i18n/emqx_authn_mysql.hocon similarity index 100% rename from apps/emqx_authn/i18n/emqx_authn_mysql_i18n.conf rename to rel/i18n/emqx_authn_mysql.hocon diff --git a/apps/emqx_authn/i18n/emqx_authn_pgsql_i18n.conf b/rel/i18n/emqx_authn_pgsql.hocon similarity index 100% rename from apps/emqx_authn/i18n/emqx_authn_pgsql_i18n.conf rename to rel/i18n/emqx_authn_pgsql.hocon diff --git a/apps/emqx_authn/i18n/emqx_authn_redis_i18n.conf b/rel/i18n/emqx_authn_redis.hocon similarity index 100% rename from apps/emqx_authn/i18n/emqx_authn_redis_i18n.conf rename to rel/i18n/emqx_authn_redis.hocon diff --git a/apps/emqx_authn/i18n/emqx_authn_schema_i18n.conf b/rel/i18n/emqx_authn_schema.hocon similarity index 100% rename from apps/emqx_authn/i18n/emqx_authn_schema_i18n.conf rename to rel/i18n/emqx_authn_schema.hocon diff --git a/apps/emqx_authn/i18n/emqx_authn_user_import_api_i18n.conf b/rel/i18n/emqx_authn_user_import_api.hocon similarity index 100% rename from apps/emqx_authn/i18n/emqx_authn_user_import_api_i18n.conf rename to rel/i18n/emqx_authn_user_import_api.hocon diff --git a/apps/emqx_authz/i18n/emqx_authz_api_cache_i18n.conf b/rel/i18n/emqx_authz_api_cache.hocon similarity index 100% rename from apps/emqx_authz/i18n/emqx_authz_api_cache_i18n.conf rename to rel/i18n/emqx_authz_api_cache.hocon diff --git a/apps/emqx_authz/i18n/emqx_authz_api_mnesia_i18n.conf b/rel/i18n/emqx_authz_api_mnesia.hocon similarity index 100% rename from apps/emqx_authz/i18n/emqx_authz_api_mnesia_i18n.conf rename to rel/i18n/emqx_authz_api_mnesia.hocon diff --git a/apps/emqx_authz/i18n/emqx_authz_api_schema_i18n.conf b/rel/i18n/emqx_authz_api_schema.hocon similarity index 100% rename from apps/emqx_authz/i18n/emqx_authz_api_schema_i18n.conf rename to rel/i18n/emqx_authz_api_schema.hocon diff --git a/apps/emqx_authz/i18n/emqx_authz_api_settings_i18n.conf b/rel/i18n/emqx_authz_api_settings.hocon similarity index 100% rename from apps/emqx_authz/i18n/emqx_authz_api_settings_i18n.conf rename to rel/i18n/emqx_authz_api_settings.hocon diff --git a/apps/emqx_authz/i18n/emqx_authz_api_sources_i18n.conf b/rel/i18n/emqx_authz_api_sources.hocon similarity index 100% rename from apps/emqx_authz/i18n/emqx_authz_api_sources_i18n.conf rename to rel/i18n/emqx_authz_api_sources.hocon diff --git a/apps/emqx_authz/i18n/emqx_authz_schema_i18n.conf b/rel/i18n/emqx_authz_schema.hocon similarity index 100% rename from apps/emqx_authz/i18n/emqx_authz_schema_i18n.conf rename to rel/i18n/emqx_authz_schema.hocon diff --git a/apps/emqx_auto_subscribe/i18n/emqx_auto_subscribe_api_i18n.conf b/rel/i18n/emqx_auto_subscribe_api.hocon similarity index 100% rename from apps/emqx_auto_subscribe/i18n/emqx_auto_subscribe_api_i18n.conf rename to rel/i18n/emqx_auto_subscribe_api.hocon diff --git a/apps/emqx_auto_subscribe/i18n/emqx_auto_subscribe_i18n.conf b/rel/i18n/emqx_auto_subscribe_schema.hocon similarity index 100% rename from apps/emqx_auto_subscribe/i18n/emqx_auto_subscribe_i18n.conf rename to rel/i18n/emqx_auto_subscribe_schema.hocon diff --git a/apps/emqx_bridge/i18n/emqx_bridge_api.conf b/rel/i18n/emqx_bridge_api.hocon similarity index 100% rename from apps/emqx_bridge/i18n/emqx_bridge_api.conf rename to rel/i18n/emqx_bridge_api.hocon diff --git a/apps/emqx_bridge/i18n/emqx_bridge_mqtt_schema.conf b/rel/i18n/emqx_bridge_mqtt_schema.hocon similarity index 100% rename from apps/emqx_bridge/i18n/emqx_bridge_mqtt_schema.conf rename to rel/i18n/emqx_bridge_mqtt_schema.hocon diff --git a/apps/emqx_bridge/i18n/emqx_bridge_schema.conf b/rel/i18n/emqx_bridge_schema.hocon similarity index 100% rename from apps/emqx_bridge/i18n/emqx_bridge_schema.conf rename to rel/i18n/emqx_bridge_schema.hocon diff --git a/apps/emqx_bridge/i18n/emqx_bridge_webhook_schema.conf b/rel/i18n/emqx_bridge_webhook_schema.hocon similarity index 100% rename from apps/emqx_bridge/i18n/emqx_bridge_webhook_schema.conf rename to rel/i18n/emqx_bridge_webhook_schema.hocon diff --git a/apps/emqx_gateway/i18n/emqx_coap_api_i18n.conf b/rel/i18n/emqx_coap_api.hocon similarity index 100% rename from apps/emqx_gateway/i18n/emqx_coap_api_i18n.conf rename to rel/i18n/emqx_coap_api.hocon diff --git a/apps/emqx_conf/i18n/emqx_conf_schema.conf b/rel/i18n/emqx_conf_schema.hocon similarity index 100% rename from apps/emqx_conf/i18n/emqx_conf_schema.conf rename to rel/i18n/emqx_conf_schema.hocon diff --git a/apps/emqx_connector/i18n/emqx_connector_api.conf b/rel/i18n/emqx_connector_api.hocon similarity index 100% rename from apps/emqx_connector/i18n/emqx_connector_api.conf rename to rel/i18n/emqx_connector_api.hocon diff --git a/apps/emqx_connector/i18n/emqx_connector_http.conf b/rel/i18n/emqx_connector_http.hocon similarity index 100% rename from apps/emqx_connector/i18n/emqx_connector_http.conf rename to rel/i18n/emqx_connector_http.hocon diff --git a/apps/emqx_connector/i18n/emqx_connector_ldap.conf b/rel/i18n/emqx_connector_ldap.hocon similarity index 100% rename from apps/emqx_connector/i18n/emqx_connector_ldap.conf rename to rel/i18n/emqx_connector_ldap.hocon diff --git a/apps/emqx_connector/i18n/emqx_connector_mongo.conf b/rel/i18n/emqx_connector_mongo.hocon similarity index 100% rename from apps/emqx_connector/i18n/emqx_connector_mongo.conf rename to rel/i18n/emqx_connector_mongo.hocon diff --git a/apps/emqx_connector/i18n/emqx_connector_mqtt.conf b/rel/i18n/emqx_connector_mqtt.hocon similarity index 100% rename from apps/emqx_connector/i18n/emqx_connector_mqtt.conf rename to rel/i18n/emqx_connector_mqtt.hocon diff --git a/apps/emqx_connector/i18n/emqx_connector_mqtt_schema.conf b/rel/i18n/emqx_connector_mqtt_schema.hocon similarity index 100% rename from apps/emqx_connector/i18n/emqx_connector_mqtt_schema.conf rename to rel/i18n/emqx_connector_mqtt_schema.hocon diff --git a/apps/emqx_connector/i18n/emqx_connector_mysql.conf b/rel/i18n/emqx_connector_mysql.hocon similarity index 100% rename from apps/emqx_connector/i18n/emqx_connector_mysql.conf rename to rel/i18n/emqx_connector_mysql.hocon diff --git a/apps/emqx_connector/i18n/emqx_connector_pgsql.conf b/rel/i18n/emqx_connector_pgsql.hocon similarity index 100% rename from apps/emqx_connector/i18n/emqx_connector_pgsql.conf rename to rel/i18n/emqx_connector_pgsql.hocon diff --git a/apps/emqx_connector/i18n/emqx_connector_redis.conf b/rel/i18n/emqx_connector_redis.hocon similarity index 100% rename from apps/emqx_connector/i18n/emqx_connector_redis.conf rename to rel/i18n/emqx_connector_redis.hocon diff --git a/apps/emqx_connector/i18n/emqx_connector_schema_lib.conf b/rel/i18n/emqx_connector_schema_lib.hocon similarity index 100% rename from apps/emqx_connector/i18n/emqx_connector_schema_lib.conf rename to rel/i18n/emqx_connector_schema_lib.hocon diff --git a/apps/emqx_dashboard/i18n/emqx_dashboard_api_i18n.conf b/rel/i18n/emqx_dashboard_api.hocon similarity index 100% rename from apps/emqx_dashboard/i18n/emqx_dashboard_api_i18n.conf rename to rel/i18n/emqx_dashboard_api.hocon diff --git a/apps/emqx_dashboard/i18n/emqx_dashboard_i18n.conf b/rel/i18n/emqx_dashboard_schema.hocon similarity index 100% rename from apps/emqx_dashboard/i18n/emqx_dashboard_i18n.conf rename to rel/i18n/emqx_dashboard_schema.hocon diff --git a/apps/emqx_modules/i18n/emqx_delayed_api_i18n.conf b/rel/i18n/emqx_delayed_api.hocon similarity index 100% rename from apps/emqx_modules/i18n/emqx_delayed_api_i18n.conf rename to rel/i18n/emqx_delayed_api.hocon diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_cassa.conf b/rel/i18n/emqx_ee_bridge_cassa.hocon similarity index 100% rename from lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_cassa.conf rename to rel/i18n/emqx_ee_bridge_cassa.hocon diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_clickhouse.conf b/rel/i18n/emqx_ee_bridge_clickhouse.hocon similarity index 100% rename from lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_clickhouse.conf rename to rel/i18n/emqx_ee_bridge_clickhouse.hocon diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_dynamo.conf b/rel/i18n/emqx_ee_bridge_dynamo.hocon similarity index 100% rename from lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_dynamo.conf rename to rel/i18n/emqx_ee_bridge_dynamo.hocon diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_gcp_pubsub.conf b/rel/i18n/emqx_ee_bridge_gcp_pubsub.hocon similarity index 100% rename from lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_gcp_pubsub.conf rename to rel/i18n/emqx_ee_bridge_gcp_pubsub.hocon diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstreamdb.conf b/rel/i18n/emqx_ee_bridge_hstreamdb.hocon similarity index 100% rename from lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_hstreamdb.conf rename to rel/i18n/emqx_ee_bridge_hstreamdb.hocon diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf b/rel/i18n/emqx_ee_bridge_influxdb.hocon similarity index 100% rename from lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_influxdb.conf rename to rel/i18n/emqx_ee_bridge_influxdb.hocon diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf b/rel/i18n/emqx_ee_bridge_kafka.hocon similarity index 100% rename from lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf rename to rel/i18n/emqx_ee_bridge_kafka.hocon diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mongodb.conf b/rel/i18n/emqx_ee_bridge_mongodb.hocon similarity index 100% rename from lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mongodb.conf rename to rel/i18n/emqx_ee_bridge_mongodb.hocon diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mysql.conf b/rel/i18n/emqx_ee_bridge_mysql.hocon similarity index 100% rename from lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_mysql.conf rename to rel/i18n/emqx_ee_bridge_mysql.hocon diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_pgsql.conf b/rel/i18n/emqx_ee_bridge_pgsql.hocon similarity index 100% rename from lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_pgsql.conf rename to rel/i18n/emqx_ee_bridge_pgsql.hocon diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_redis.conf b/rel/i18n/emqx_ee_bridge_redis.hocon similarity index 100% rename from lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_redis.conf rename to rel/i18n/emqx_ee_bridge_redis.hocon diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_rocketmq.conf b/rel/i18n/emqx_ee_bridge_rocketmq.hocon similarity index 100% rename from lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_rocketmq.conf rename to rel/i18n/emqx_ee_bridge_rocketmq.hocon diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_tdengine.conf b/rel/i18n/emqx_ee_bridge_tdengine.hocon similarity index 100% rename from lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_tdengine.conf rename to rel/i18n/emqx_ee_bridge_tdengine.hocon diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_cassa.conf b/rel/i18n/emqx_ee_connector_cassa.hocon similarity index 100% rename from lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_cassa.conf rename to rel/i18n/emqx_ee_connector_cassa.hocon diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_clickhouse.conf b/rel/i18n/emqx_ee_connector_clickhouse.hocon similarity index 99% rename from lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_clickhouse.conf rename to rel/i18n/emqx_ee_connector_clickhouse.hocon index 069505a69..4d30e1715 100644 --- a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_clickhouse.conf +++ b/rel/i18n/emqx_ee_connector_clickhouse.hocon @@ -1,4 +1,3 @@ - emqx_ee_connector_clickhouse { base_url { diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_dynamo.conf b/rel/i18n/emqx_ee_connector_dynamo.hocon similarity index 100% rename from lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_dynamo.conf rename to rel/i18n/emqx_ee_connector_dynamo.hocon diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstreamdb.conf b/rel/i18n/emqx_ee_connector_hstreamdb.hocon similarity index 100% rename from lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_hstreamdb.conf rename to rel/i18n/emqx_ee_connector_hstreamdb.hocon diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_influxdb.conf b/rel/i18n/emqx_ee_connector_influxdb.hocon similarity index 100% rename from lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_influxdb.conf rename to rel/i18n/emqx_ee_connector_influxdb.hocon diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_rocketmq.conf b/rel/i18n/emqx_ee_connector_rocketmq.hocon similarity index 100% rename from lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_rocketmq.conf rename to rel/i18n/emqx_ee_connector_rocketmq.hocon diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_tdengine.conf b/rel/i18n/emqx_ee_connector_tdengine.hocon similarity index 100% rename from lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_tdengine.conf rename to rel/i18n/emqx_ee_connector_tdengine.hocon diff --git a/apps/emqx_exhook/i18n/emqx_exhook_api_i18n.conf b/rel/i18n/emqx_exhook_api.hocon similarity index 100% rename from apps/emqx_exhook/i18n/emqx_exhook_api_i18n.conf rename to rel/i18n/emqx_exhook_api.hocon diff --git a/apps/emqx_exhook/i18n/emqx_exhook_i18n.conf b/rel/i18n/emqx_exhook_schema.hocon similarity index 100% rename from apps/emqx_exhook/i18n/emqx_exhook_i18n.conf rename to rel/i18n/emqx_exhook_schema.hocon diff --git a/apps/emqx_gateway/i18n/emqx_gateway_api_i18n.conf b/rel/i18n/emqx_gateway_api.hocon similarity index 100% rename from apps/emqx_gateway/i18n/emqx_gateway_api_i18n.conf rename to rel/i18n/emqx_gateway_api.hocon diff --git a/apps/emqx_gateway/i18n/emqx_gateway_api_authn_i18n.conf b/rel/i18n/emqx_gateway_api_authn.hocon similarity index 100% rename from apps/emqx_gateway/i18n/emqx_gateway_api_authn_i18n.conf rename to rel/i18n/emqx_gateway_api_authn.hocon diff --git a/apps/emqx_gateway/i18n/emqx_gateway_api_clients_i18n.conf b/rel/i18n/emqx_gateway_api_clients.hocon similarity index 100% rename from apps/emqx_gateway/i18n/emqx_gateway_api_clients_i18n.conf rename to rel/i18n/emqx_gateway_api_clients.hocon diff --git a/apps/emqx_gateway/i18n/emqx_gateway_api_listeners_i18n.conf b/rel/i18n/emqx_gateway_api_listeners.hocon similarity index 100% rename from apps/emqx_gateway/i18n/emqx_gateway_api_listeners_i18n.conf rename to rel/i18n/emqx_gateway_api_listeners.hocon diff --git a/apps/emqx_gateway/i18n/emqx_gateway_schema_i18n.conf b/rel/i18n/emqx_gateway_schema.hocon similarity index 100% rename from apps/emqx_gateway/i18n/emqx_gateway_schema_i18n.conf rename to rel/i18n/emqx_gateway_schema.hocon diff --git a/lib-ee/emqx_license/i18n/emqx_license_http_api.conf b/rel/i18n/emqx_license_http_api.hocon similarity index 100% rename from lib-ee/emqx_license/i18n/emqx_license_http_api.conf rename to rel/i18n/emqx_license_http_api.hocon diff --git a/lib-ee/emqx_license/i18n/emqx_license_schema_i18n.conf b/rel/i18n/emqx_license_schema.hocon similarity index 100% rename from lib-ee/emqx_license/i18n/emqx_license_schema_i18n.conf rename to rel/i18n/emqx_license_schema.hocon diff --git a/apps/emqx/i18n/emqx_limiter_i18n.conf b/rel/i18n/emqx_limiter_schema.hocon similarity index 100% rename from apps/emqx/i18n/emqx_limiter_i18n.conf rename to rel/i18n/emqx_limiter_schema.hocon diff --git a/apps/emqx_gateway/i18n/emqx_lwm2m_api_i18n.conf b/rel/i18n/emqx_lwm2m_api.hocon similarity index 100% rename from apps/emqx_gateway/i18n/emqx_lwm2m_api_i18n.conf rename to rel/i18n/emqx_lwm2m_api.hocon diff --git a/apps/emqx_management/i18n/emqx_mgmt_api_alarms_i18n.conf b/rel/i18n/emqx_mgmt_api_alarms.hocon similarity index 100% rename from apps/emqx_management/i18n/emqx_mgmt_api_alarms_i18n.conf rename to rel/i18n/emqx_mgmt_api_alarms.hocon diff --git a/apps/emqx_management/i18n/emqx_mgmt_api_banned_i18n.conf b/rel/i18n/emqx_mgmt_api_banned.hocon similarity index 100% rename from apps/emqx_management/i18n/emqx_mgmt_api_banned_i18n.conf rename to rel/i18n/emqx_mgmt_api_banned.hocon diff --git a/apps/emqx_management/i18n/emqx_mgmt_api_key_i18n.conf b/rel/i18n/emqx_mgmt_api_key_schema.hocon similarity index 100% rename from apps/emqx_management/i18n/emqx_mgmt_api_key_i18n.conf rename to rel/i18n/emqx_mgmt_api_key_schema.hocon diff --git a/apps/emqx_management/i18n/emqx_mgmt_api_publish_i18n.conf b/rel/i18n/emqx_mgmt_api_publish.hocon similarity index 99% rename from apps/emqx_management/i18n/emqx_mgmt_api_publish_i18n.conf rename to rel/i18n/emqx_mgmt_api_publish.hocon index f91115df5..a09732cfc 100644 --- a/apps/emqx_management/i18n/emqx_mgmt_api_publish_i18n.conf +++ b/rel/i18n/emqx_mgmt_api_publish.hocon @@ -1,4 +1,3 @@ - emqx_mgmt_api_publish { publish_api { desc { diff --git a/apps/emqx_management/i18n/emqx_mgmt_api_status_i18n.conf b/rel/i18n/emqx_mgmt_api_status.hocon similarity index 100% rename from apps/emqx_management/i18n/emqx_mgmt_api_status_i18n.conf rename to rel/i18n/emqx_mgmt_api_status.hocon diff --git a/apps/emqx_modules/i18n/emqx_modules_schema_i18n.conf b/rel/i18n/emqx_modules_schema.hocon similarity index 100% rename from apps/emqx_modules/i18n/emqx_modules_schema_i18n.conf rename to rel/i18n/emqx_modules_schema.hocon diff --git a/apps/emqx_plugins/i18n/emqx_plugins_schema.conf b/rel/i18n/emqx_plugins_schema.hocon similarity index 100% rename from apps/emqx_plugins/i18n/emqx_plugins_schema.conf rename to rel/i18n/emqx_plugins_schema.hocon diff --git a/apps/emqx_prometheus/i18n/emqx_prometheus_schema_i18n.conf b/rel/i18n/emqx_prometheus_schema.hocon similarity index 100% rename from apps/emqx_prometheus/i18n/emqx_prometheus_schema_i18n.conf rename to rel/i18n/emqx_prometheus_schema.hocon diff --git a/apps/emqx_psk/i18n/emqx_psk_i18n.conf b/rel/i18n/emqx_psk_schema.hocon similarity index 100% rename from apps/emqx_psk/i18n/emqx_psk_i18n.conf rename to rel/i18n/emqx_psk_schema.hocon diff --git a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf b/rel/i18n/emqx_resource_schema.hocon similarity index 100% rename from apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf rename to rel/i18n/emqx_resource_schema.hocon diff --git a/apps/emqx_retainer/i18n/emqx_retainer_api_i18n.conf b/rel/i18n/emqx_retainer_api.hocon similarity index 100% rename from apps/emqx_retainer/i18n/emqx_retainer_api_i18n.conf rename to rel/i18n/emqx_retainer_api.hocon diff --git a/apps/emqx_retainer/i18n/emqx_retainer_i18n.conf b/rel/i18n/emqx_retainer_schema.hocon similarity index 100% rename from apps/emqx_retainer/i18n/emqx_retainer_i18n.conf rename to rel/i18n/emqx_retainer_schema.hocon diff --git a/apps/emqx_modules/i18n/emqx_rewrite_api_i18n.conf b/rel/i18n/emqx_rewrite_api.hocon similarity index 100% rename from apps/emqx_modules/i18n/emqx_rewrite_api_i18n.conf rename to rel/i18n/emqx_rewrite_api.hocon diff --git a/apps/emqx_rule_engine/i18n/emqx_rule_api_schema.conf b/rel/i18n/emqx_rule_api_schema.hocon similarity index 100% rename from apps/emqx_rule_engine/i18n/emqx_rule_api_schema.conf rename to rel/i18n/emqx_rule_api_schema.hocon diff --git a/apps/emqx_rule_engine/i18n/emqx_rule_engine_api.conf b/rel/i18n/emqx_rule_engine_api.hocon similarity index 100% rename from apps/emqx_rule_engine/i18n/emqx_rule_engine_api.conf rename to rel/i18n/emqx_rule_engine_api.hocon diff --git a/apps/emqx_rule_engine/i18n/emqx_rule_engine_schema.conf b/rel/i18n/emqx_rule_engine_schema.hocon similarity index 100% rename from apps/emqx_rule_engine/i18n/emqx_rule_engine_schema.conf rename to rel/i18n/emqx_rule_engine_schema.hocon diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/rel/i18n/emqx_schema.hocon similarity index 100% rename from apps/emqx/i18n/emqx_schema_i18n.conf rename to rel/i18n/emqx_schema.hocon diff --git a/apps/emqx_slow_subs/i18n/emqx_slow_subs_api_i18n.conf b/rel/i18n/emqx_slow_subs_api.hocon similarity index 100% rename from apps/emqx_slow_subs/i18n/emqx_slow_subs_api_i18n.conf rename to rel/i18n/emqx_slow_subs_api.hocon diff --git a/apps/emqx_slow_subs/i18n/emqx_slow_subs_i18n.conf b/rel/i18n/emqx_slow_subs_schema.hocon similarity index 100% rename from apps/emqx_slow_subs/i18n/emqx_slow_subs_i18n.conf rename to rel/i18n/emqx_slow_subs_schema.hocon diff --git a/apps/emqx_statsd/i18n/emqx_statsd_api_i18n.conf b/rel/i18n/emqx_statsd_api.hocon similarity index 100% rename from apps/emqx_statsd/i18n/emqx_statsd_api_i18n.conf rename to rel/i18n/emqx_statsd_api.hocon diff --git a/apps/emqx_statsd/i18n/emqx_statsd_schema_i18n.conf b/rel/i18n/emqx_statsd_schema.hocon similarity index 100% rename from apps/emqx_statsd/i18n/emqx_statsd_schema_i18n.conf rename to rel/i18n/emqx_statsd_schema.hocon diff --git a/apps/emqx_modules/i18n/emqx_telemetry_api_i18n.conf b/rel/i18n/emqx_telemetry_api.hocon similarity index 100% rename from apps/emqx_modules/i18n/emqx_telemetry_api_i18n.conf rename to rel/i18n/emqx_telemetry_api.hocon diff --git a/apps/emqx_modules/i18n/emqx_topic_metrics_api_i18n.conf b/rel/i18n/emqx_topic_metrics_api.hocon similarity index 100% rename from apps/emqx_modules/i18n/emqx_topic_metrics_api_i18n.conf rename to rel/i18n/emqx_topic_metrics_api.hocon From 7ec9b9a40888f3a7f2cb61c867684b21745b0b07 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 28 Mar 2023 18:20:02 +0200 Subject: [PATCH 161/196] refactor(merge-i18n.escript): merge files in rel/i18n --- scripts/merge-i18n.escript | 48 +++++--------------------------------- 1 file changed, 6 insertions(+), 42 deletions(-) diff --git a/scripts/merge-i18n.escript b/scripts/merge-i18n.escript index 816cbe182..b2501d10a 100755 --- a/scripts/merge-i18n.escript +++ b/scripts/merge-i18n.escript @@ -4,12 +4,8 @@ main(_) -> BaseConf = <<"">>, - Cfgs0 = get_all_cfgs("apps/"), - Cfgs1 = get_all_cfgs("lib-ee/"), - Conf0 = merge(BaseConf, Cfgs0), - Conf = [merge(Conf0, Cfgs1), - io_lib:nl() - ], + Cfgs0 = get_all_files(), + Conf = merge(BaseConf, Cfgs0), OutputFile = "apps/emqx_dashboard/priv/i18n.conf", ok = filelib:ensure_dir(OutputFile), ok = file:write_file(OutputFile, Conf). @@ -25,39 +21,7 @@ merge(BaseConf, Cfgs) -> end end, BaseConf, Cfgs). -get_all_cfgs(Root) -> - Apps = filelib:wildcard("*", Root) -- ["emqx_machine"], - Dirs = [filename:join([Root, App]) || App <- Apps], - lists:foldl(fun get_cfgs/2, [], Dirs). - -get_all_cfgs(Dir, Cfgs) -> - Fun = fun(E, Acc) -> - Path = filename:join([Dir, E]), - get_cfgs(Path, Acc) - end, - lists:foldl(Fun, Cfgs, filelib:wildcard("*", Dir)). - -get_cfgs(Dir, Cfgs) -> - case filelib:is_dir(Dir) of - false -> - Cfgs; - _ -> - Files = filelib:wildcard("*", Dir), - case lists:member("i18n", Files) of - false -> - try_enter_child(Dir, Files, Cfgs); - true -> - EtcDir = filename:join([Dir, "i18n"]), - Confs = filelib:wildcard("*.conf", EtcDir), - NewCfgs = [filename:join([EtcDir, Name]) || Name <- Confs], - try_enter_child(Dir, Files, NewCfgs ++ Cfgs) - end - end. - -try_enter_child(Dir, Files, Cfgs) -> - case lists:member("src", Files) of - false -> - Cfgs; - true -> - get_all_cfgs(filename:join([Dir, "src"]), Cfgs) - end. +get_all_files() -> + Dir = filename:join(["rel","i18n"]), + Files = filelib:wildcard("*.hocon", Dir), + lists:map(fun(Name) -> filename:join([Dir, Name]) end, Files). From c17de6c415215234cdc98c58b3986b91827a7269 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 28 Mar 2023 21:42:03 +0200 Subject: [PATCH 162/196] docs: fix i18n desc style --- rel/i18n/emqx_ee_bridge_clickhouse.hocon | 6 ++---- rel/i18n/emqx_ee_bridge_kafka.hocon | 4 ++-- rel/i18n/emqx_ee_bridge_tdengine.hocon | 6 ++---- rel/i18n/emqx_ee_connector_dynamo.hocon | 4 ++-- rel/i18n/emqx_ee_connector_rocketmq.hocon | 12 ++++-------- rel/i18n/emqx_ee_connector_tdengine.hocon | 12 ++++-------- rel/i18n/emqx_schema.hocon | 4 ++-- scripts/check-i18n-style.escript | 2 ++ scripts/check-i18n-style.sh | 2 +- 9 files changed, 21 insertions(+), 31 deletions(-) diff --git a/rel/i18n/emqx_ee_bridge_clickhouse.hocon b/rel/i18n/emqx_ee_bridge_clickhouse.hocon index 5096f8590..5bc6f12b2 100644 --- a/rel/i18n/emqx_ee_bridge_clickhouse.hocon +++ b/rel/i18n/emqx_ee_bridge_clickhouse.hocon @@ -6,11 +6,9 @@ emqx_ee_bridge_clickhouse { matching the local_topic will be forwarded.
NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is configured, then both the data got from the rule and the MQTT messages that match local_topic -will be forwarded. -""" +will be forwarded.""" zh: """发送到 'local_topic' 的消息都会转发到 Clickhouse。
-注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。 -""" +注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。""" } label { en: "Local Topic" diff --git a/rel/i18n/emqx_ee_bridge_kafka.hocon b/rel/i18n/emqx_ee_bridge_kafka.hocon index df32c1cae..d1a017416 100644 --- a/rel/i18n/emqx_ee_bridge_kafka.hocon +++ b/rel/i18n/emqx_ee_bridge_kafka.hocon @@ -547,7 +547,7 @@ emqx_ee_bridge_kafka { "ts: message timestamp.\n" "ts_type: message timestamp type, which is one of" " create, append or undefined.\n" - "value: Kafka message value (uses the chosen value encoding).\n" + "value: Kafka message value (uses the chosen value encoding)." zh: "用于转换收到的 Kafka 消息的模板。 " "默认情况下，它将使用 JSON 格式来序列化来自 Kafka 的所有字段。 " "这些字段包括：" @@ -558,7 +558,7 @@ emqx_ee_bridge_kafka { "ts: 消息的时间戳。\n" "ts_type：消息的时间戳类型，值可能是：" " create， append 或 undefined。\n" - "value: Kafka 消息值（使用选择的编码方式编码）。\n" + "value: Kafka 消息值（使用选择的编码方式编码）。" } label { diff --git a/rel/i18n/emqx_ee_bridge_tdengine.hocon b/rel/i18n/emqx_ee_bridge_tdengine.hocon index 2d5af9f16..21fc013df 100644 --- a/rel/i18n/emqx_ee_bridge_tdengine.hocon +++ b/rel/i18n/emqx_ee_bridge_tdengine.hocon @@ -6,11 +6,9 @@ emqx_ee_bridge_tdengine { matching the local_topic will be forwarded.
NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is configured, then both the data got from the rule and the MQTT messages that match local_topic -will be forwarded. -""" +will be forwarded.""" zh: """发送到 'local_topic' 的消息都会转发到 TDengine。
-注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。 -""" +注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。""" } label { en: "Local Topic" diff --git a/rel/i18n/emqx_ee_connector_dynamo.hocon b/rel/i18n/emqx_ee_connector_dynamo.hocon index e1fc11e03..295929a72 100644 --- a/rel/i18n/emqx_ee_connector_dynamo.hocon +++ b/rel/i18n/emqx_ee_connector_dynamo.hocon @@ -2,8 +2,8 @@ emqx_ee_connector_dynamo { url { desc { - en: """The url of DynamoDB endpoint.
""" - zh: """DynamoDB 的地址。
""" + en: """The url of DynamoDB endpoint.""" + zh: """DynamoDB 的地址。""" } label: { en: "DynamoDB Endpoint" diff --git a/rel/i18n/emqx_ee_connector_rocketmq.hocon b/rel/i18n/emqx_ee_connector_rocketmq.hocon index d4a610212..44dda7931 100644 --- a/rel/i18n/emqx_ee_connector_rocketmq.hocon +++ b/rel/i18n/emqx_ee_connector_rocketmq.hocon @@ -2,16 +2,12 @@ emqx_ee_connector_rocketmq { server { desc { - en: """ -The IPv4 or IPv6 address or the hostname to connect to.
+ en: """The IPv4 or IPv6 address or the hostname to connect to.
A host entry has the following form: `Host[:Port]`.
-The RocketMQ default port 9876 is used if `[:Port]` is not specified. -""" - zh: """ -将要连接的 IPv4 或 IPv6 地址，或者主机名。
+The RocketMQ default port 9876 is used if `[:Port]` is not specified.""" + zh: """将要连接的 IPv4 或 IPv6 地址，或者主机名。
主机名具有以下形式：`Host[:Port]`。
-如果未指定 `[:Port]`，则使用 RocketMQ 默认端口 9876。 -""" +如果未指定 `[:Port]`，则使用 RocketMQ 默认端口 9876。""" } label: { en: "Server Host" diff --git a/rel/i18n/emqx_ee_connector_tdengine.hocon b/rel/i18n/emqx_ee_connector_tdengine.hocon index c6c58d82d..02254124c 100644 --- a/rel/i18n/emqx_ee_connector_tdengine.hocon +++ b/rel/i18n/emqx_ee_connector_tdengine.hocon @@ -2,16 +2,12 @@ emqx_ee_connector_tdengine { server { desc { - en: """ -The IPv4 or IPv6 address or the hostname to connect to.
+ en: """The IPv4 or IPv6 address or the hostname to connect to.
A host entry has the following form: `Host[:Port]`.
-The TDengine default port 6041 is used if `[:Port]` is not specified. -""" - zh: """ -将要连接的 IPv4 或 IPv6 地址，或者主机名。
+The TDengine default port 6041 is used if `[:Port]` is not specified.""" + zh: """将要连接的 IPv4 或 IPv6 地址，或者主机名。
主机名具有以下形式：`Host[:Port]`。
-如果未指定 `[:Port]`，则使用 TDengine 默认端口 6041。 -""" +如果未指定 `[:Port]`，则使用 TDengine 默认端口 6041。""" } label: { en: "Server Host" diff --git a/rel/i18n/emqx_schema.hocon b/rel/i18n/emqx_schema.hocon index 28c58713d..d36809c3b 100644 --- a/rel/i18n/emqx_schema.hocon +++ b/rel/i18n/emqx_schema.hocon @@ -1505,8 +1505,8 @@ In case PSK cipher suites are intended, make sure to configure common_ssl_opts_schema_hibernate_after { desc { - en: """ Hibernate the SSL process after idling for amount of time reducing its memory footprint. """ - zh: """ 在闲置一定时间后休眠 SSL 进程，减少其内存占用。""" + en: """Hibernate the SSL process after idling for amount of time reducing its memory footprint.""" + zh: """在闲置一定时间后休眠 SSL 进程，减少其内存占用。""" } label: { en: "hibernate after" diff --git a/scripts/check-i18n-style.escript b/scripts/check-i18n-style.escript index 6ad6c1770..7e90f0807 100755 --- a/scripts/check-i18n-style.escript +++ b/scripts/check-i18n-style.escript @@ -1,5 +1,7 @@ #!/usr/bin/env escript +%% called from check-i18n-style.sh + -mode(compile). -define(YELLOW, "\e[33m"). diff --git a/scripts/check-i18n-style.sh b/scripts/check-i18n-style.sh index 0be565f30..d21f43a72 100755 --- a/scripts/check-i18n-style.sh +++ b/scripts/check-i18n-style.sh @@ -3,6 +3,6 @@ set -euo pipefail cd -P -- "$(dirname -- "$0")/.." -all_files="$(git ls-files '*i18n*.conf')" +all_files="$(git ls-files 'rel/i18n/*.hocon')" ./scripts/check-i18n-style.escript "$all_files" From bdd3960e178319698a8a45b6da404e405eada777 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 28 Mar 2023 21:48:27 +0200 Subject: [PATCH 163/196] build: check i18n style earlier --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 13e3a6d43..a5adf0e0a 100644 --- a/Makefile +++ b/Makefile @@ -82,7 +82,7 @@ ct: $(REBAR) merge-config static_checks: @$(REBAR) as check do xref, dialyzer @if [ "$${PROFILE}" = 'emqx-enterprise' ]; then $(REBAR) ct --suite apps/emqx/test/emqx_static_checks --readable $(CT_READABLE); fi - @if [ "$${PROFILE}" = 'emqx-enterprise' ]; then ./scripts/check-i18n-style.sh; fi + ./scripts/check-i18n-style.sh APPS=$(shell $(SCRIPTS)/find-apps.sh) From 5f6d318cf0cadeb6048dec407adffac32b2d8419 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 28 Mar 2023 23:14:01 +0200 Subject: [PATCH 164/196] fix(i18n): fix missing docs for gateway configs --- apps/emqx_gateway/src/emqx_gateway_schema.erl | 34 +++++++++--------- rel/i18n/emqx_gateway_schema.hocon | 36 +++---------------- rel/i18n/emqx_rule_api_schema.hocon | 4 +-- scripts/check-i18n-style.escript | 7 ++-- 4 files changed, 27 insertions(+), 54 deletions(-) diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl index 2034a40eb..741fb98ae 100644 --- a/apps/emqx_gateway/src/emqx_gateway_schema.erl +++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl @@ -453,20 +453,20 @@ fields(translator) -> ]; fields(udp_listeners) -> [ - {udp, sc(map(name, ref(udp_listener)), #{desc => ?DESC(udp_listener)})}, - {dtls, sc(map(name, ref(dtls_listener)), #{desc => ?DESC(dtls_listener)})} + {udp, sc(map(name, ref(udp_listener)), #{desc => ?DESC(listener_name_to_settings_map)})}, + {dtls, sc(map(name, ref(dtls_listener)), #{desc => ?DESC(listener_name_to_settings_map)})} ]; fields(tcp_listeners) -> [ - {tcp, sc(map(name, ref(tcp_listener)), #{desc => ?DESC(tcp_listener)})}, - {ssl, sc(map(name, ref(ssl_listener)), #{desc => ?DESC(ssl_listener)})} + {tcp, sc(map(name, ref(tcp_listener)), #{desc => ?DESC(listener_name_to_settings_map)})}, + {ssl, sc(map(name, ref(ssl_listener)), #{desc => ?DESC(listener_name_to_settings_map)})} ]; fields(tcp_udp_listeners) -> [ - {tcp, sc(map(name, ref(tcp_listener)), #{desc => ?DESC(tcp_listener)})}, - {ssl, sc(map(name, ref(ssl_listener)), #{desc => ?DESC(ssl_listener)})}, - {udp, sc(map(name, ref(udp_listener)), #{desc => ?DESC(udp_listener)})}, - {dtls, sc(map(name, ref(dtls_listener)), #{desc => ?DESC(dtls_listener)})} + {tcp, sc(map(name, ref(tcp_listener)), #{desc => ?DESC(listener_name_to_settings_map)})}, + {ssl, sc(map(name, ref(ssl_listener)), #{desc => ?DESC(listener_name_to_settings_map)})}, + {udp, sc(map(name, ref(udp_listener)), #{desc => ?DESC(listener_name_to_settings_map)})}, + {dtls, sc(map(name, ref(dtls_listener)), #{desc => ?DESC(listener_name_to_settings_map)})} ]; fields(tcp_listener) -> %% some special configs for tcp listener @@ -558,19 +558,19 @@ desc(udp_listeners) -> desc(tcp_listeners) -> "Settings for the TCP listeners."; desc(tcp_udp_listeners) -> - "Settings for the listeners."; + "Settings for TCP and UDP listeners."; desc(tcp_listener) -> - "Settings for the TCP listener."; + "Settings for TCP listener."; desc(ssl_listener) -> - "Settings for the SSL listener."; + "Settings for SSL listener."; desc(udp_listener) -> - "Settings for the UDP listener."; + "Settings for UDP listener."; desc(dtls_listener) -> - "Settings for the DTLS listener."; + "Settings for DTLS listener."; desc(udp_opts) -> - "Settings for the UDP sockets."; + "Settings for UDP sockets."; desc(dtls_opts) -> - "Settings for the DTLS protocol."; + "Settings for DTLS protocol."; desc(_) -> undefined. @@ -625,7 +625,7 @@ mountpoint(Default) -> binary(), #{ default => iolist_to_binary(Default), - desc => ?DESC(gateway_common_mountpoint) + desc => ?DESC(gateway_mountpoint) } ). @@ -674,7 +674,7 @@ common_listener_opts() -> binary(), #{ default => undefined, - desc => ?DESC(gateway_common_listener_mountpoint) + desc => ?DESC(gateway_mountpoint) } )}, {access_rules, diff --git a/rel/i18n/emqx_gateway_schema.hocon b/rel/i18n/emqx_gateway_schema.hocon index 74a70eb73..ebc955557 100644 --- a/rel/i18n/emqx_gateway_schema.hocon +++ b/rel/i18n/emqx_gateway_schema.hocon @@ -370,13 +370,6 @@ After succeed observe a resource of LwM2M client, Gateway will send the notify e } } - gateway_common_mountpoint { - desc { - en: """""" - zh: """""" - } - } - gateway_common_clientinfo_override { desc { en: """ClientInfo override.""" @@ -431,10 +424,10 @@ After succeed observe a resource of LwM2M client, Gateway will send the notify e } } - tcp_listener { + listener_name_to_settings_map{ desc { - en: """""" - zh: """""" + en: """A map from listener names to listener settings.""" + zh: """从监听器名称到配置参数的映射。""" } } @@ -468,13 +461,6 @@ EMQX will close the TCP connection if proxy protocol packet is not received with } } - ssl_listener { - desc { - en: """""" - zh: """""" - } - } - ssl_listener_options { desc { en: """SSL Socket options.""" @@ -482,13 +468,6 @@ EMQX will close the TCP connection if proxy protocol packet is not received with } } - udp_listener { - desc { - en: """""" - zh: """""" - } - } - udp_listener_udp_opts { desc { en: """Settings for the UDP sockets.""" @@ -533,13 +512,6 @@ See: https://erlang.org/doc/man/inet.html#setopts-2""" } } - dtls_listener { - desc { - en: """""" - zh: """""" - } - } - dtls_listener_acceptors { desc { en: """Size of the acceptor pool.""" @@ -592,7 +564,7 @@ When set to false clients will be allowed to connect without authen } } - gateway_common_listener_mountpoint { + gateway_mountpoint { desc { en: """When publishing or subscribing, prefix all topics with a mountpoint string. The prefixed string will be removed from the topic name when the message is delivered to the subscriber. diff --git a/rel/i18n/emqx_rule_api_schema.hocon b/rel/i18n/emqx_rule_api_schema.hocon index e4c2314de..f9b344666 100644 --- a/rel/i18n/emqx_rule_api_schema.hocon +++ b/rel/i18n/emqx_rule_api_schema.hocon @@ -35,8 +35,8 @@ emqx_rule_api_schema { event_username { desc { - en: "The User Name" - zh: "" + en: "Username" + zh: "用户名" } label: { en: "Username" diff --git a/scripts/check-i18n-style.escript b/scripts/check-i18n-style.escript index 7e90f0807..cbe79c82e 100755 --- a/scripts/check-i18n-style.escript +++ b/scripts/check-i18n-style.escript @@ -4,11 +4,12 @@ -mode(compile). --define(YELLOW, "\e[33m"). +% -define(YELLOW, "\e[33m"). % not used -define(RED, "\e[31m"). -define(RESET, "\e[39m"). main([Files0]) -> + io:format(user, "checking i18n file styles", []), _ = put(errors, 0), Files = string:tokens(Files0, "\n"), ok = load_hocon(), @@ -48,7 +49,7 @@ logerr(Fmt, Args) -> check(File) -> - io:format(user, "checking: ~s~n", [File]), + io:format(user, ".", []), {ok, C} = hocon:load(File), maps:foreach(fun check_one_field/2, C), ok. @@ -86,7 +87,7 @@ do_check_desc(Name, _) -> die("~s: missing 'zh' or 'en'~n", [Name]). check_desc_string(Name, Tr, <<>>) -> - io:format(standard_error, ?YELLOW ++ "WARNING: ~s.~s: empty string~n" ++ ?RESET, [Name, Tr]); + logerr("~s.~s: empty string~n", [Name, Tr]); check_desc_string(Name, Tr, BinStr) -> Str = unicode:characters_to_list(BinStr, utf8), Err = fun(Reason) -> From 677b76afce1ac7826f5d2e4a3aab9ca332eaae00 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 3 Apr 2023 13:13:43 +0200 Subject: [PATCH 165/196] refactor: replace hidden => true with importance => hidden --- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl index e11ef1c93..529e28dea 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl @@ -233,7 +233,7 @@ fields(socket_opts) -> boolean(), #{ default => true, - hidden => true, + importance => ?IMPORTANCE_HIDDEN, desc => ?DESC(socket_nodelay) } )} From ec1871ffdee91d117dfecfecd0ba30ddacbe7c86 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 31 Mar 2023 16:18:03 -0300 Subject: [PATCH 166/196] test(janitor): catch each callback invocation --- apps/emqx/test/emqx_test_janitor.erl | 2 +- .../test/emqx_bridge_impl_kafka_consumer_SUITE.erl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx/test/emqx_test_janitor.erl b/apps/emqx/test/emqx_test_janitor.erl index c9b297dc7..c3f82a3e1 100644 --- a/apps/emqx/test/emqx_test_janitor.erl +++ b/apps/emqx/test/emqx_test_janitor.erl @@ -65,7 +65,7 @@ terminate(_Reason, #{callbacks := Callbacks}) -> handle_call({push, Callback}, _From, State = #{callbacks := Callbacks}) -> {reply, ok, State#{callbacks := [Callback | Callbacks]}}; handle_call(terminate, _From, State = #{callbacks := Callbacks}) -> - lists:foreach(fun(Fun) -> Fun() end, Callbacks), + lists:foreach(fun(Fun) -> catch Fun() end, Callbacks), {stop, normal, ok, State}; handle_call(_Req, _From, State) -> {reply, error, State}. diff --git a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl index 02a4c3c3b..8664f9745 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl @@ -388,7 +388,7 @@ end_per_testcase(_Testcase, Config) -> maps:values(ProducersMapping) ), ok = wolff:stop_and_delete_supervised_client(KafkaProducerClientId), - emqx_common_test_helpers:call_janitor(), + emqx_common_test_helpers:call_janitor(30_000), ok = snabbkaffe:stop(), ok end. From 0e5a22f500911080cebbcc011c48c0b1f2804e49 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 31 Mar 2023 16:27:06 -0300 Subject: [PATCH 167/196] test(cassandra): cap test container memory usage Default memory usage is excessive. --- .ci/docker-compose-file/cassandra/cassandra.yaml | 4 ++-- .ci/docker-compose-file/docker-compose-cassandra.yaml | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.ci/docker-compose-file/cassandra/cassandra.yaml b/.ci/docker-compose-file/cassandra/cassandra.yaml index 968efe5f6..bc1bf3357 100644 --- a/.ci/docker-compose-file/cassandra/cassandra.yaml +++ b/.ci/docker-compose-file/cassandra/cassandra.yaml @@ -469,8 +469,8 @@ concurrent_materialized_view_writes: 32 # accepting writes when the limit is exceeded until a flush completes, # and will trigger a flush based on memtable_cleanup_threshold # If omitted, Cassandra will set both to 1/4 the size of the heap. -# memtable_heap_space_in_mb: 2048 -# memtable_offheap_space_in_mb: 2048 +memtable_heap_space_in_mb: 2048 +memtable_offheap_space_in_mb: 2048 # memtable_cleanup_threshold is deprecated. The default calculation # is the only reasonable choice. See the comments on memtable_flush_writers diff --git a/.ci/docker-compose-file/docker-compose-cassandra.yaml b/.ci/docker-compose-file/docker-compose-cassandra.yaml index a54f621c1..f7143f471 100644 --- a/.ci/docker-compose-file/docker-compose-cassandra.yaml +++ b/.ci/docker-compose-file/docker-compose-cassandra.yaml @@ -12,6 +12,8 @@ services: environment: CASSANDRA_BROADCAST_ADDRESS: "1.2.3.4" CASSANDRA_RPC_ADDRESS: "0.0.0.0" + HEAP_NEWSIZE: "128M" + MAX_HEAP_SIZE: "2048M" volumes: - ./certs:/certs #ports: From c1cb5357e1df4b4d3c6cd8ca19813c46bd9bb9b9 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Mon, 3 Apr 2023 15:48:33 +0200 Subject: [PATCH 168/196] fix: enable schema check --- apps/emqx_modules/src/emqx_delayed_api.erl | 2 +- apps/emqx_modules/src/emqx_modules.app.src | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_modules/src/emqx_delayed_api.erl b/apps/emqx_modules/src/emqx_delayed_api.erl index d4e7e5b90..3499d83ca 100644 --- a/apps/emqx_modules/src/emqx_delayed_api.erl +++ b/apps/emqx_modules/src/emqx_delayed_api.erl @@ -52,7 +52,7 @@ -define(INVALID_NODE, 'INVALID_NODE'). api_spec() -> - emqx_dashboard_swagger:spec(?MODULE). + emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). paths() -> [ diff --git a/apps/emqx_modules/src/emqx_modules.app.src b/apps/emqx_modules/src/emqx_modules.app.src index 4a9cb6723..fdc13f354 100644 --- a/apps/emqx_modules/src/emqx_modules.app.src +++ b/apps/emqx_modules/src/emqx_modules.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_modules, [ {description, "EMQX Modules"}, - {vsn, "5.0.11"}, + {vsn, "5.0.12"}, {modules, []}, {applications, [kernel, stdlib, emqx, emqx_ctl]}, {mod, {emqx_modules_app, []}}, From 0efa9c7a110ba2fede4b3abeea784bf058e1fb35 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Mon, 3 Apr 2023 15:48:52 +0200 Subject: [PATCH 169/196] fix: pretty format error responses --- apps/emqx_modules/src/emqx_delayed_api.erl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/emqx_modules/src/emqx_delayed_api.erl b/apps/emqx_modules/src/emqx_delayed_api.erl index 3499d83ca..766d23d6b 100644 --- a/apps/emqx_modules/src/emqx_delayed_api.erl +++ b/apps/emqx_modules/src/emqx_delayed_api.erl @@ -202,9 +202,9 @@ delayed_message(get, #{bindings := #{node := NodeBin, msgid := HexId}}) -> {200, Message#{payload => base64:encode(Payload)}} end; {error, not_found} -> - {404, generate_http_code_map(not_found, Id)}; + {404, generate_http_code_map(not_found, HexId)}; {badrpc, _} -> - {400, generate_http_code_map(invalid_node, Id)} + {400, generate_http_code_map(invalid_node, NodeBin)} end end ); @@ -271,19 +271,19 @@ generate_http_code_map(id_schema_error, Id) -> #{ code => ?MESSAGE_ID_SCHEMA_ERROR, message => - iolist_to_binary(io_lib:format("Message ID ~p schema error", [Id])) + iolist_to_binary(io_lib:format("Message ID ~s schema error", [Id])) }; generate_http_code_map(not_found, Id) -> #{ code => ?MESSAGE_ID_NOT_FOUND, message => - iolist_to_binary(io_lib:format("Message ID ~p not found", [Id])) + iolist_to_binary(io_lib:format("Message ID ~s not found", [Id])) }; generate_http_code_map(invalid_node, Node) -> #{ code => ?INVALID_NODE, message => - iolist_to_binary(io_lib:format("The node name ~p is invalid", [Node])) + iolist_to_binary(io_lib:format("The node name ~s is invalid", [Node])) }. make_maybe(X, Error, Fun) -> From 5d722f8d465178a6fdcc60816458868cfa0da529 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Mon, 3 Apr 2023 15:55:25 +0200 Subject: [PATCH 170/196] style: add changelog --- changes/ce/fix-10315.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ce/fix-10315.en.md diff --git a/changes/ce/fix-10315.en.md b/changes/ce/fix-10315.en.md new file mode 100644 index 000000000..67445252d --- /dev/null +++ b/changes/ce/fix-10315.en.md @@ -0,0 +1 @@ +Fix crash checking `limit` and `page` parameters in `/mqtt/delayed/messages` API call. From ed25ee6fecfdd49304e276b57d5763f91711b4f6 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 3 Apr 2023 10:58:31 -0300 Subject: [PATCH 171/196] test(crl): fix flaky test (v5.0) --- apps/emqx/test/emqx_crl_cache_SUITE.erl | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/apps/emqx/test/emqx_crl_cache_SUITE.erl b/apps/emqx/test/emqx_crl_cache_SUITE.erl index 7a61f7835..01f9c7172 100644 --- a/apps/emqx/test/emqx_crl_cache_SUITE.erl +++ b/apps/emqx/test/emqx_crl_cache_SUITE.erl @@ -884,7 +884,20 @@ t_revoked(Config) -> {port, 8883} ]), process_flag(trap_exit, true), - ?assertMatch({error, {{shutdown, {tls_alert, {certificate_revoked, _}}}, _}}, emqtt:connect(C)), + Res = emqtt:connect(C), + %% apparently, sometimes there's some race condition in + %% `emqtt_sock:ssl_upgrade' when it calls + %% `ssl:conetrolling_process' and a bad match happens at that + %% point. + case Res of + {error, {{shutdown, {tls_alert, {certificate_revoked, _}}}, _}} -> + ok; + {error, closed} -> + %% race condition? + ok; + _ -> + ct:fail("unexpected result: ~p", [Res]) + end, ok. t_revoke_then_refresh(Config) -> From e978d86c866d96fd68d38001fe9fc412a4f208d2 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 3 Apr 2023 13:41:13 +0200 Subject: [PATCH 172/196] chore: add doc_lift for authorization.sources doc_lift is to make the doc render application to lift this field to the root level and force the field's doc to refernec it instead of expanding the structs in a nested way --- apps/emqx_authz/src/emqx_authz.app.src | 2 +- apps/emqx_authz/src/emqx_authz_schema.erl | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/emqx_authz/src/emqx_authz.app.src b/apps/emqx_authz/src/emqx_authz.app.src index 943978519..2f8b26894 100644 --- a/apps/emqx_authz/src/emqx_authz.app.src +++ b/apps/emqx_authz/src/emqx_authz.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_authz, [ {description, "An OTP application"}, - {vsn, "0.1.15"}, + {vsn, "0.1.16"}, {registered, []}, {mod, {emqx_authz_app, []}}, {applications, [ diff --git a/apps/emqx_authz/src/emqx_authz_schema.erl b/apps/emqx_authz/src/emqx_authz_schema.erl index b15d4abd4..6630ed526 100644 --- a/apps/emqx_authz/src/emqx_authz_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_schema.erl @@ -492,7 +492,9 @@ authz_fields() -> ?ARRAY(?UNION(UnionMemberSelector)), #{ default => [], - desc => ?DESC(sources) + desc => ?DESC(sources), + %% doc_lift is force a root level reference instead of nesting sub-structs + extra => #{doc_lift => true} } )} ]. From 2d6ca69ffb2724463d4fd284a37521ce9f243515 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 3 Apr 2023 16:37:18 +0200 Subject: [PATCH 173/196] refactor: no support for listener level authentication for now --- apps/emqx/src/emqx_schema.erl | 4 +++- apps/emqx_gateway/src/emqx_gateway_schema.erl | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 0f90677bd..6bfff38d3 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -1880,7 +1880,9 @@ mqtt_listener(Bind) -> default => <<"3s">> } )}, - {?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME, authentication(listener)} + {?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME, (authentication(listener))#{ + importance => ?IMPORTANCE_HIDDEN + }} ]. base_listener(Bind) -> diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl index 741fb98ae..28c1e6f89 100644 --- a/apps/emqx_gateway/src/emqx_gateway_schema.erl +++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl @@ -580,6 +580,8 @@ authentication_schema() -> #{ required => {false, recursively}, desc => ?DESC(gateway_common_authentication), + %% we do not expose this to the user for now + importance => ?IMPORTANCE_HIDDEN, examples => emqx_authn_api:authenticator_examples() } ). From a4e27e56a855389f60d42076353b2f4516f83bed Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 3 Apr 2023 16:52:49 +0200 Subject: [PATCH 174/196] docs: add change logs --- changes/ce/fix-10317.en.md | 1 + changes/ce/fix-10317.zh.md | 1 + 2 files changed, 2 insertions(+) create mode 100644 changes/ce/fix-10317.en.md create mode 100644 changes/ce/fix-10317.zh.md diff --git a/changes/ce/fix-10317.en.md b/changes/ce/fix-10317.en.md new file mode 100644 index 000000000..7a83dcaca --- /dev/null +++ b/changes/ce/fix-10317.en.md @@ -0,0 +1 @@ +Do not expose listener level authentications before extensive verification. diff --git a/changes/ce/fix-10317.zh.md b/changes/ce/fix-10317.zh.md new file mode 100644 index 000000000..69cf09901 --- /dev/null +++ b/changes/ce/fix-10317.zh.md @@ -0,0 +1 @@ +在大量验证完成前不暴露监听器级的认证功能。 From 8b5a717a1f7617a25e2d1e6a132063441f7850a5 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 31 Mar 2023 11:02:15 -0300 Subject: [PATCH 175/196] test(peer): increase init and startup timeout for peer nodes Attempt to stabilize tests that use cluster nodes. --- apps/emqx/test/emqx_common_test_helpers.erl | 6 +++--- .../emqx_bridge_impl_kafka_consumer_SUITE.erl | 20 ++++++++++++------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index 9a4461fac..077ebe138 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -660,6 +660,7 @@ start_slave(Name, Opts) when is_list(Opts) -> start_slave(Name, Opts) when is_map(Opts) -> SlaveMod = maps:get(peer_mod, Opts, ct_slave), Node = node_name(Name), + put_peer_mod(Node, SlaveMod), DoStart = fun() -> case SlaveMod of @@ -669,8 +670,8 @@ start_slave(Name, Opts) when is_map(Opts) -> [ {kill_if_fail, true}, {monitor_master, true}, - {init_timeout, 10000}, - {startup_timeout, 10000}, + {init_timeout, 20_000}, + {startup_timeout, 20_000}, {erl_flags, erl_flags()} ] ); @@ -687,7 +688,6 @@ start_slave(Name, Opts) when is_map(Opts) -> throw(Other) end, pong = net_adm:ping(Node), - put_peer_mod(Node, SlaveMod), setup_node(Node, Opts), ok = snabbkaffe:forward_trace(Node), Node. diff --git a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl index 8664f9745..4019a9c42 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl @@ -388,7 +388,9 @@ end_per_testcase(_Testcase, Config) -> maps:values(ProducersMapping) ), ok = wolff:stop_and_delete_supervised_client(KafkaProducerClientId), - emqx_common_test_helpers:call_janitor(30_000), + %% in CI, apparently this needs more time since the + %% machines struggle with all the containers running... + emqx_common_test_helpers:call_janitor(60_000), ok = snabbkaffe:stop(), ok end. @@ -1664,7 +1666,7 @@ t_cluster_group(Config) -> || {Name, Opts} <- Cluster ], on_exit(fun() -> - lists:foreach( + emqx_misc:pmap( fun(N) -> ct:pal("stopping ~p", [N]), ok = emqx_common_test_helpers:stop_slave(N) @@ -1875,7 +1877,7 @@ t_cluster_node_down(Config) -> Cluster ), on_exit(fun() -> - lists:foreach( + emqx_misc:pmap( fun(N) -> ct:pal("stopping ~p", [N]), ok = emqx_common_test_helpers:stop_slave(N) @@ -1894,10 +1896,14 @@ t_cluster_node_down(Config) -> {ok, _} = snabbkaffe:receive_events(SRef0), lists:foreach( fun(N) -> - ?assertMatch( - {ok, _}, - erpc:call(N, emqx_bridge, lookup, [BridgeId]), - #{node => N} + ?retry( + _Sleep1 = 100, + _Attempts1 = 50, + ?assertMatch( + {ok, _}, + erpc:call(N, emqx_bridge, lookup, [BridgeId]), + #{node => N} + ) ) end, Nodes From f3ffc02bff34a89f173b272ebea2abe8acd4bc0e Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 31 Mar 2023 09:41:25 -0300 Subject: [PATCH 176/196] feat(bridges): enable async query mode for all bridges with buffer workers Fixes https://emqx.atlassian.net/browse/EMQX-9130 Since buffer workers always support async calls ("outer calls"), we should decouple those two call modes (inner and outer), and avoid exposing the inner call configuration to user to avoid complexity. For bridges that currently only allow sync query modes, we should allow them to be configured with async. That means basically all bridge types except Kafka Producer. --- .../src/emqx_connector_mysql.erl | 19 +- .../src/emqx_connector_pgsql.erl | 3 +- .../src/schema/emqx_resource_schema.erl | 23 +-- .../test/emqx_rule_funcs_SUITE.erl | 4 - changes/ce/feat-10306.en.md | 3 + .../src/emqx_ee_bridge_cassa.erl | 15 +- .../src/emqx_ee_bridge_mongodb.erl | 2 +- .../src/emqx_ee_bridge_mysql.erl | 15 +- .../src/emqx_ee_bridge_pgsql.erl | 15 +- .../src/emqx_ee_bridge_redis.erl | 4 +- .../src/emqx_ee_bridge_rocketmq.erl | 15 +- .../src/emqx_ee_bridge_tdengine.erl | 15 +- .../test/emqx_ee_bridge_cassa_SUITE.erl | 120 +++++++++-- .../test/emqx_ee_bridge_mongodb_SUITE.erl | 94 +++++++-- .../test/emqx_ee_bridge_mysql_SUITE.erl | 190 ++++++++++++++---- .../test/emqx_ee_bridge_pgsql_SUITE.erl | 189 +++++++++++++---- .../test/emqx_ee_bridge_redis_SUITE.erl | 22 +- .../test/emqx_ee_bridge_rocketmq_SUITE.erl | 12 +- .../test/emqx_ee_bridge_tdengine_SUITE.erl | 156 +++++++++++--- .../src/emqx_ee_connector_mongodb.erl | 4 +- rel/i18n/emqx_resource_schema.hocon | 11 - 21 files changed, 639 insertions(+), 292 deletions(-) create mode 100644 changes/ce/feat-10306.en.md diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl index 68ec59894..fe495252a 100644 --- a/apps/emqx_connector/src/emqx_connector_mysql.erl +++ b/apps/emqx_connector/src/emqx_connector_mysql.erl @@ -172,10 +172,15 @@ on_query( %% not return result, next loop will try again on_query(InstId, {TypeOrKey, SQLOrKey, Params, Timeout}, State); {error, Reason} -> - LogMeta = #{connector => InstId, sql => SQLOrKey, state => State}, - ?SLOG( + ?tp( error, - LogMeta#{msg => "mysql_connector_do_prepare_failed", reason => Reason} + "mysql_connector_do_prepare_failed", + #{ + connector => InstId, + sql => SQLOrKey, + state => State, + reason => Reason + } ), {error, Reason} end; @@ -417,12 +422,10 @@ on_sql_query( ), do_sql_query(SQLFunc, Conn, SQLOrKey, Params, Timeout, LogMeta); {error, disconnected} -> - ?SLOG( + ?tp( error, - LogMeta#{ - msg => "mysql_connector_do_sql_query_failed", - reason => worker_is_disconnected - } + "mysql_connector_do_sql_query_failed", + LogMeta#{reason => worker_is_disconnected} ), {error, {recoverable_error, disconnected}} end. diff --git a/apps/emqx_connector/src/emqx_connector_pgsql.erl b/apps/emqx_connector/src/emqx_connector_pgsql.erl index 1fc994275..14cbbc80f 100644 --- a/apps/emqx_connector/src/emqx_connector_pgsql.erl +++ b/apps/emqx_connector/src/emqx_connector_pgsql.erl @@ -44,7 +44,8 @@ execute_batch/3 ]). --export([do_get_status/1]). +%% for ecpool workers usage +-export([do_get_status/1, prepare_sql_to_conn/2]). -define(PGSQL_HOST_OPTIONS, #{ default_port => ?PGSQL_DEFAULT_PORT diff --git a/apps/emqx_resource/src/schema/emqx_resource_schema.erl b/apps/emqx_resource/src/schema/emqx_resource_schema.erl index e89278e8c..7db886542 100644 --- a/apps/emqx_resource/src/schema/emqx_resource_schema.erl +++ b/apps/emqx_resource/src/schema/emqx_resource_schema.erl @@ -30,18 +30,6 @@ namespace() -> "resource_schema". roots() -> []. -fields("resource_opts_sync_only") -> - [ - {resource_opts, - mk( - ref(?MODULE, "creation_opts_sync_only"), - resource_opts_meta() - )} - ]; -fields("creation_opts_sync_only") -> - Fields = fields("creation_opts"), - QueryMod = {query_mode, fun query_mode_sync_only/1}, - lists:keyreplace(query_mode, 1, Fields, QueryMod); fields("resource_opts") -> [ {resource_opts, @@ -117,12 +105,6 @@ query_mode(default) -> async; query_mode(required) -> false; query_mode(_) -> undefined. -query_mode_sync_only(type) -> enum([sync]); -query_mode_sync_only(desc) -> ?DESC("query_mode_sync_only"); -query_mode_sync_only(default) -> sync; -query_mode_sync_only(required) -> false; -query_mode_sync_only(_) -> undefined. - request_timeout(type) -> hoconsc:union([infinity, emqx_schema:duration_ms()]); request_timeout(desc) -> ?DESC("request_timeout"); request_timeout(default) -> <<"15s">>; @@ -167,7 +149,4 @@ max_queue_bytes(default) -> ?DEFAULT_QUEUE_SIZE_RAW; max_queue_bytes(required) -> false; max_queue_bytes(_) -> undefined. -desc("creation_opts") -> - ?DESC("creation_opts"); -desc("creation_opts_sync_only") -> - ?DESC("creation_opts"). +desc("creation_opts") -> ?DESC("creation_opts"). diff --git a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl index 209332fe7..94adb3506 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl @@ -687,10 +687,6 @@ t_jq(_) -> got_timeout end, ConfigRootKey = emqx_rule_engine_schema:namespace(), - DefaultTimeOut = emqx_config:get([ - ConfigRootKey, - jq_function_default_timeout - ]), ?assertThrow( {jq_exception, {timeout, _}}, apply_func(jq, [TOProgram, <<"-2">>]) diff --git a/changes/ce/feat-10306.en.md b/changes/ce/feat-10306.en.md new file mode 100644 index 000000000..11754c5c0 --- /dev/null +++ b/changes/ce/feat-10306.en.md @@ -0,0 +1,3 @@ +Add support for `async` query mode for most bridges. + +Before this change, some bridges (Cassandra, MongoDB, MySQL, Postgres, Redis, RocketMQ, TDengine) were only allowed to be created with a `sync` query mode. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl index 12f86fcf7..78db8352a 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl @@ -86,21 +86,10 @@ fields("config") -> mk( binary(), #{desc => ?DESC("local_topic"), default => undefined} - )}, - {resource_opts, - mk( - ref(?MODULE, "creation_opts"), - #{ - required => false, - default => #{}, - desc => ?DESC(emqx_resource_schema, <<"resource_opts">>) - } )} - ] ++ + ] ++ emqx_resource_schema:fields("resource_opts") ++ (emqx_ee_connector_cassa:fields(config) -- emqx_connector_schema_lib:prepare_statement_fields()); -fields("creation_opts") -> - emqx_resource_schema:fields("creation_opts_sync_only"); fields("post") -> fields("post", cassandra); fields("put") -> @@ -115,8 +104,6 @@ desc("config") -> ?DESC("desc_config"); desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> ["Configuration for Cassandra using `", string:to_upper(Method), "` method."]; -desc("creation_opts" = Name) -> - emqx_resource_schema:desc(Name); desc(_) -> undefined. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mongodb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mongodb.erl index bc450f39b..5dd3ef121 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mongodb.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mongodb.erl @@ -38,7 +38,7 @@ fields("config") -> {enable, mk(boolean(), #{desc => ?DESC("enable"), default => true})}, {collection, mk(binary(), #{desc => ?DESC("collection"), default => <<"mqtt">>})}, {payload_template, mk(binary(), #{required => false, desc => ?DESC("payload_template")})} - ] ++ emqx_resource_schema:fields("resource_opts_sync_only"); + ] ++ emqx_resource_schema:fields("resource_opts"); fields(mongodb_rs) -> emqx_connector_mongo:fields(rs) ++ fields("config"); fields(mongodb_sharded) -> diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl index eed4172ab..e5c9a5aab 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl @@ -79,21 +79,10 @@ fields("config") -> mk( binary(), #{desc => ?DESC("local_topic"), default => undefined} - )}, - {resource_opts, - mk( - ref(?MODULE, "creation_opts"), - #{ - required => false, - default => #{}, - desc => ?DESC(emqx_resource_schema, <<"resource_opts">>) - } )} - ] ++ + ] ++ emqx_resource_schema:fields("resource_opts") ++ (emqx_connector_mysql:fields(config) -- emqx_connector_schema_lib:prepare_statement_fields()); -fields("creation_opts") -> - emqx_resource_schema:fields("creation_opts_sync_only"); fields("post") -> [type_field(), name_field() | fields("config")]; fields("put") -> @@ -105,8 +94,6 @@ desc("config") -> ?DESC("desc_config"); desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> ["Configuration for MySQL using `", string:to_upper(Method), "` method."]; -desc("creation_opts" = Name) -> - emqx_resource_schema:desc(Name); desc(_) -> undefined. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl index 46132bd99..b5f0c3e62 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl @@ -81,21 +81,10 @@ fields("config") -> mk( binary(), #{desc => ?DESC("local_topic"), default => undefined} - )}, - {resource_opts, - mk( - ref(?MODULE, "creation_opts"), - #{ - required => false, - default => #{}, - desc => ?DESC(emqx_resource_schema, <<"resource_opts">>) - } )} - ] ++ + ] ++ emqx_resource_schema:fields("resource_opts") ++ (emqx_connector_pgsql:fields(config) -- emqx_connector_schema_lib:prepare_statement_fields()); -fields("creation_opts") -> - emqx_resource_schema:fields("creation_opts_sync_only"); fields("post") -> fields("post", pgsql); fields("put") -> @@ -110,8 +99,6 @@ desc("config") -> ?DESC("desc_config"); desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> ["Configuration for PostgreSQL using `", string:to_upper(Method), "` method."]; -desc("creation_opts" = Name) -> - emqx_resource_schema:desc(Name); desc(_) -> undefined. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_redis.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_redis.erl index fa6958b6d..1861b56ec 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_redis.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_redis.erl @@ -180,10 +180,10 @@ resource_fields(Type) -> resource_creation_fields("redis_cluster") -> % TODO % Cluster bridge is currently incompatible with batching. - Fields = emqx_resource_schema:fields("creation_opts_sync_only"), + Fields = emqx_resource_schema:fields("creation_opts"), lists:foldl(fun proplists:delete/2, Fields, [batch_size, batch_time, enable_batch]); resource_creation_fields(_) -> - emqx_resource_schema:fields("creation_opts_sync_only"). + emqx_resource_schema:fields("creation_opts"). desc("config") -> ?DESC("desc_config"); diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_rocketmq.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_rocketmq.erl index 124e18069..78fd527d3 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_rocketmq.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_rocketmq.erl @@ -80,21 +80,10 @@ fields("config") -> mk( binary(), #{desc => ?DESC("local_topic"), required => false} - )}, - {resource_opts, - mk( - ref(?MODULE, "creation_opts"), - #{ - required => false, - default => #{<<"request_timeout">> => ?DEFFAULT_REQ_TIMEOUT}, - desc => ?DESC(emqx_resource_schema, <<"resource_opts">>) - } )} - ] ++ + ] ++ emqx_resource_schema:fields("resource_opts") ++ (emqx_ee_connector_rocketmq:fields(config) -- emqx_connector_schema_lib:prepare_statement_fields()); -fields("creation_opts") -> - emqx_resource_schema:fields("creation_opts_sync_only"); fields("post") -> [type_field(), name_field() | fields("config")]; fields("put") -> @@ -106,8 +95,6 @@ desc("config") -> ?DESC("desc_config"); desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> ["Configuration for RocketMQ using `", string:to_upper(Method), "` method."]; -desc("creation_opts" = Name) -> - emqx_resource_schema:desc(Name); desc(_) -> undefined. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_tdengine.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_tdengine.erl index f031cbfbf..7a958d45f 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_tdengine.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_tdengine.erl @@ -80,19 +80,8 @@ fields("config") -> mk( binary(), #{desc => ?DESC("local_topic"), default => undefined} - )}, - {resource_opts, - mk( - ref(?MODULE, "creation_opts"), - #{ - required => false, - default => #{}, - desc => ?DESC(emqx_resource_schema, <<"resource_opts">>) - } )} - ] ++ emqx_ee_connector_tdengine:fields(config); -fields("creation_opts") -> - emqx_resource_schema:fields("creation_opts_sync_only"); + ] ++ emqx_resource_schema:fields("resource_opts") ++ emqx_ee_connector_tdengine:fields(config); fields("post") -> [type_field(), name_field() | fields("config")]; fields("put") -> @@ -104,8 +93,6 @@ desc("config") -> ?DESC("desc_config"); desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> ["Configuration for TDengine using `", string:to_upper(Method), "` method."]; -desc("creation_opts" = Name) -> - emqx_resource_schema:desc(Name); desc(_) -> undefined. diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl index d040000e2..f1ea6e930 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl @@ -73,15 +73,16 @@ all() -> groups() -> TCs = emqx_common_test_helpers:all(?MODULE), NonBatchCases = [t_write_timeout], + QueryModeGroups = [{group, async}, {group, sync}], + BatchingGroups = [ + %{group, with_batch}, + {group, without_batch} + ], [ - {tcp, [ - %{group, with_batch}, - {group, without_batch} - ]}, - {tls, [ - %{group, with_batch}, - {group, without_batch} - ]}, + {tcp, QueryModeGroups}, + {tls, QueryModeGroups}, + {async, BatchingGroups}, + {sync, BatchingGroups}, {with_batch, TCs -- NonBatchCases}, {without_batch, TCs} ]. @@ -93,7 +94,6 @@ init_per_group(tcp, Config) -> {cassa_host, Host}, {cassa_port, Port}, {enable_tls, false}, - {query_mode, sync}, {proxy_name, "cassa_tcp"} | Config ]; @@ -104,10 +104,13 @@ init_per_group(tls, Config) -> {cassa_host, Host}, {cassa_port, Port}, {enable_tls, true}, - {query_mode, sync}, {proxy_name, "cassa_tls"} | Config ]; +init_per_group(async, Config) -> + [{query_mode, async} | Config]; +init_per_group(sync, Config) -> + [{query_mode, sync} | Config]; init_per_group(with_batch, Config0) -> Config = [{enable_batch, true} | Config0], common_init(Config); @@ -139,14 +142,15 @@ end_per_suite(_Config) -> init_per_testcase(_Testcase, Config) -> connect_and_clear_table(Config), delete_bridge(Config), + snabbkaffe:start_trace(), Config. end_per_testcase(_Testcase, Config) -> ProxyHost = ?config(proxy_host, Config), ProxyPort = ?config(proxy_port, Config), emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), - connect_and_clear_table(Config), ok = snabbkaffe:stop(), + connect_and_clear_table(Config), delete_bridge(Config), ok. @@ -171,6 +175,7 @@ common_init(Config0) -> ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]), emqx_mgmt_api_test_util:init_suite(), % Connect to cassnadra directly and create the table + catch connect_and_drop_table(Config0), connect_and_create_table(Config0), {Name, CassaConf} = cassa_config(BridgeType, Config0), Config = @@ -250,9 +255,13 @@ parse_and_check(ConfigString, BridgeType, Name) -> Config. create_bridge(Config) -> + create_bridge(Config, _Overrides = #{}). + +create_bridge(Config, Overrides) -> BridgeType = ?config(cassa_bridge_type, Config), Name = ?config(cassa_name, Config), - BridgeConfig = ?config(cassa_config, Config), + BridgeConfig0 = ?config(cassa_config, Config), + BridgeConfig = emqx_map_lib:deep_merge(BridgeConfig0, Overrides), emqx_bridge:create(BridgeType, Name, BridgeConfig). delete_bridge(Config) -> @@ -288,6 +297,27 @@ query_resource(Config, Request) -> ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), emqx_resource:query(ResourceID, Request, #{timeout => 1_000}). +query_resource_async(Config, Request) -> + Name = ?config(cassa_name, Config), + BridgeType = ?config(cassa_bridge_type, Config), + Ref = alias([reply]), + AsyncReplyFun = fun(Result) -> Ref ! {result, Ref, Result} end, + ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), + Return = emqx_resource:query(ResourceID, Request, #{ + timeout => 500, async_reply_fun => {AsyncReplyFun, []} + }), + {Return, Ref}. + +receive_result(Ref, Timeout) when is_reference(Ref) -> + receive + {result, Ref, Result} -> + {ok, Result}; + {Ref, Result} -> + {ok, Result} + after Timeout -> + timeout + end. + connect_direct_cassa(Config) -> Opts = #{ nodes => [{?config(cassa_host, Config), ?config(cassa_port, Config)}], @@ -546,15 +576,27 @@ t_write_failure(Config) -> % ok. t_simple_sql_query(Config) -> + EnableBatch = ?config(enable_batch, Config), + QueryMode = ?config(query_mode, Config), ?assertMatch( {ok, _}, create_bridge(Config) ), Request = {query, <<"SELECT count(1) AS T FROM system.local">>}, - Result = query_resource(Config, Request), - case ?config(enable_batch, Config) of - true -> ?assertEqual({error, {unrecoverable_error, batch_prepare_not_implemented}}, Result); - false -> ?assertMatch({ok, {<<"system.local">>, _, [[1]]}}, Result) + Result = + case QueryMode of + sync -> + query_resource(Config, Request); + async -> + {_, Ref} = query_resource_async(Config, Request), + {ok, Res} = receive_result(Ref, 2_000), + Res + end, + case EnableBatch of + true -> + ?assertEqual({error, {unrecoverable_error, batch_prepare_not_implemented}}, Result); + false -> + ?assertMatch({ok, {<<"system.local">>, _, [[1]]}}, Result) end, ok. @@ -565,22 +607,56 @@ t_missing_data(Config) -> ), %% emqx_ee_connector_cassa will send missed data as a `null` atom %% to ecql driver - Result = send_message(Config, #{}), + {_, {ok, Event}} = + ?wait_async_action( + send_message(Config, #{}), + #{?snk_kind := buffer_worker_flush_ack}, + 2_000 + ), ?assertMatch( %% TODO: match error msgs - {error, {unrecoverable_error, {8704, <<"Expected 8 or 0 byte long for date (4)">>}}}, - Result + #{ + result := + {error, {unrecoverable_error, {8704, <<"Expected 8 or 0 byte long for date (4)">>}}} + }, + Event ), ok. t_bad_sql_parameter(Config) -> + QueryMode = ?config(query_mode, Config), + EnableBatch = ?config(enable_batch, Config), + Name = ?config(cassa_name, Config), + ResourceId = emqx_bridge_resource:resource_id(cassandra, Name), ?assertMatch( {ok, _}, - create_bridge(Config) + create_bridge( + Config, + #{ + <<"resource_opts">> => #{ + <<"request_timeout">> => 500, + <<"resume_interval">> => 100, + <<"health_check_interval">> => 100 + } + } + ) ), Request = {query, <<"">>, [bad_parameter]}, - Result = query_resource(Config, Request), - case ?config(enable_batch, Config) of + Result = + case QueryMode of + sync -> + query_resource(Config, Request); + async -> + {_, Ref} = query_resource_async(Config, Request), + case receive_result(Ref, 5_000) of + {ok, Res} -> + Res; + timeout -> + ct:pal("mailbox:\n ~p", [process_info(self(), messages)]), + ct:fail("no response received") + end + end, + case EnableBatch of true -> ?assertEqual({error, {unrecoverable_error, invalid_request}}, Result); false -> diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl index 9850c9529..116dcc729 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl @@ -9,6 +9,7 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). %%------------------------------------------------------------------------------ %% CT boilerplate @@ -16,9 +17,8 @@ all() -> [ - {group, rs}, - {group, sharded}, - {group, single} + {group, async}, + {group, sync} | (emqx_common_test_helpers:all(?MODULE) -- group_tests()) ]. @@ -31,12 +31,23 @@ group_tests() -> ]. groups() -> + TypeGroups = [ + {group, rs}, + {group, sharded}, + {group, single} + ], [ + {async, TypeGroups}, + {sync, TypeGroups}, {rs, group_tests()}, {sharded, group_tests()}, {single, group_tests()} ]. +init_per_group(async, Config) -> + [{query_mode, async} | Config]; +init_per_group(sync, Config) -> + [{query_mode, sync} | Config]; init_per_group(Type = rs, Config) -> MongoHost = os:getenv("MONGO_RS_HOST", "mongo1"), MongoPort = list_to_integer(os:getenv("MONGO_RS_PORT", "27017")), @@ -44,7 +55,7 @@ init_per_group(Type = rs, Config) -> true -> ok = start_apps(), emqx_mgmt_api_test_util:init_suite(), - {Name, MongoConfig} = mongo_config(MongoHost, MongoPort, Type), + {Name, MongoConfig} = mongo_config(MongoHost, MongoPort, Type, Config), [ {mongo_host, MongoHost}, {mongo_port, MongoPort}, @@ -63,7 +74,7 @@ init_per_group(Type = sharded, Config) -> true -> ok = start_apps(), emqx_mgmt_api_test_util:init_suite(), - {Name, MongoConfig} = mongo_config(MongoHost, MongoPort, Type), + {Name, MongoConfig} = mongo_config(MongoHost, MongoPort, Type, Config), [ {mongo_host, MongoHost}, {mongo_port, MongoPort}, @@ -82,7 +93,7 @@ init_per_group(Type = single, Config) -> true -> ok = start_apps(), emqx_mgmt_api_test_util:init_suite(), - {Name, MongoConfig} = mongo_config(MongoHost, MongoPort, Type), + {Name, MongoConfig} = mongo_config(MongoHost, MongoPort, Type, Config), [ {mongo_host, MongoHost}, {mongo_port, MongoPort}, @@ -99,6 +110,7 @@ end_per_group(_Type, _Config) -> ok. init_per_suite(Config) -> + emqx_common_test_helpers:clear_screen(), Config. end_per_suite(_Config) -> @@ -109,11 +121,13 @@ end_per_suite(_Config) -> init_per_testcase(_Testcase, Config) -> catch clear_db(Config), delete_bridge(Config), + snabbkaffe:start_trace(), Config. end_per_testcase(_Testcase, Config) -> catch clear_db(Config), delete_bridge(Config), + snabbkaffe:stop(), ok. %%------------------------------------------------------------------------------ @@ -140,7 +154,8 @@ mongo_type_bin(sharded) -> mongo_type_bin(single) -> <<"mongodb_single">>. -mongo_config(MongoHost, MongoPort0, rs = Type) -> +mongo_config(MongoHost, MongoPort0, rs = Type, Config) -> + QueryMode = ?config(query_mode, Config), MongoPort = integer_to_list(MongoPort0), Servers = MongoHost ++ ":" ++ MongoPort, Name = atom_to_binary(?MODULE), @@ -154,13 +169,19 @@ mongo_config(MongoHost, MongoPort0, rs = Type) -> " w_mode = safe\n" " database = mqtt\n" " resource_opts = {\n" + " query_mode = ~s\n" " worker_pool_size = 1\n" " }\n" "}", - [Name, Servers] + [ + Name, + Servers, + QueryMode + ] ), {Name, parse_and_check(ConfigString, Type, Name)}; -mongo_config(MongoHost, MongoPort0, sharded = Type) -> +mongo_config(MongoHost, MongoPort0, sharded = Type, Config) -> + QueryMode = ?config(query_mode, Config), MongoPort = integer_to_list(MongoPort0), Servers = MongoHost ++ ":" ++ MongoPort, Name = atom_to_binary(?MODULE), @@ -173,13 +194,19 @@ mongo_config(MongoHost, MongoPort0, sharded = Type) -> " w_mode = safe\n" " database = mqtt\n" " resource_opts = {\n" + " query_mode = ~s\n" " worker_pool_size = 1\n" " }\n" "}", - [Name, Servers] + [ + Name, + Servers, + QueryMode + ] ), {Name, parse_and_check(ConfigString, Type, Name)}; -mongo_config(MongoHost, MongoPort0, single = Type) -> +mongo_config(MongoHost, MongoPort0, single = Type, Config) -> + QueryMode = ?config(query_mode, Config), MongoPort = integer_to_list(MongoPort0), Server = MongoHost ++ ":" ++ MongoPort, Name = atom_to_binary(?MODULE), @@ -192,10 +219,15 @@ mongo_config(MongoHost, MongoPort0, single = Type) -> " w_mode = safe\n" " database = mqtt\n" " resource_opts = {\n" + " query_mode = ~s\n" " worker_pool_size = 1\n" " }\n" "}", - [Name, Server] + [ + Name, + Server, + QueryMode + ] ), {Name, parse_and_check(ConfigString, Type, Name)}. @@ -248,7 +280,7 @@ find_all(Config) -> Name = ?config(mongo_name, Config), #{<<"collection">> := Collection} = ?config(mongo_config, Config), ResourceID = emqx_bridge_resource:resource_id(Type, Name), - emqx_resource:query(ResourceID, {find, Collection, #{}, #{}}). + emqx_resource:simple_sync_query(ResourceID, {find, Collection, #{}, #{}}). send_message(Config, Payload) -> Name = ?config(mongo_name, Config), @@ -266,7 +298,12 @@ t_setup_via_config_and_publish(Config) -> create_bridge(Config) ), Val = erlang:unique_integer(), - ok = send_message(Config, #{key => Val}), + {ok, {ok, _}} = + ?wait_async_action( + send_message(Config, #{key => Val}), + #{?snk_kind := mongo_ee_connector_on_query_return}, + 5_000 + ), ?assertMatch( {ok, [#{<<"key">> := Val}]}, find_all(Config) @@ -286,7 +323,12 @@ t_setup_via_http_api_and_publish(Config) -> create_bridge_http(MongoConfig) ), Val = erlang:unique_integer(), - ok = send_message(Config, #{key => Val}), + {ok, {ok, _}} = + ?wait_async_action( + send_message(Config, #{key => Val}), + #{?snk_kind := mongo_ee_connector_on_query_return}, + 5_000 + ), ?assertMatch( {ok, [#{<<"key">> := Val}]}, find_all(Config) @@ -297,7 +339,12 @@ t_payload_template(Config) -> {ok, _} = create_bridge(Config, #{<<"payload_template">> => <<"{\"foo\": \"${clientid}\"}">>}), Val = erlang:unique_integer(), ClientId = emqx_guid:to_hexstr(emqx_guid:gen()), - ok = send_message(Config, #{key => Val, clientid => ClientId}), + {ok, {ok, _}} = + ?wait_async_action( + send_message(Config, #{key => Val, clientid => ClientId}), + #{?snk_kind := mongo_ee_connector_on_query_return}, + 5_000 + ), ?assertMatch( {ok, [#{<<"foo">> := ClientId}]}, find_all(Config) @@ -314,11 +361,16 @@ t_collection_template(Config) -> ), Val = erlang:unique_integer(), ClientId = emqx_guid:to_hexstr(emqx_guid:gen()), - ok = send_message(Config, #{ - key => Val, - clientid => ClientId, - mycollectionvar => <<"mycol">> - }), + {ok, {ok, _}} = + ?wait_async_action( + send_message(Config, #{ + key => Val, + clientid => ClientId, + mycollectionvar => <<"mycol">> + }), + #{?snk_kind := mongo_ee_connector_on_query_return}, + 5_000 + ), ?assertMatch( {ok, [#{<<"foo">> := ClientId}]}, find_all(Config) diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl index 93e9e6fee..38e31c7ae 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl @@ -45,15 +45,16 @@ all() -> groups() -> TCs = emqx_common_test_helpers:all(?MODULE), NonBatchCases = [t_write_timeout, t_uninitialized_prepared_statement], + BatchingGroups = [ + {group, with_batch}, + {group, without_batch} + ], + QueryModeGroups = [{group, async}, {group, sync}], [ - {tcp, [ - {group, with_batch}, - {group, without_batch} - ]}, - {tls, [ - {group, with_batch}, - {group, without_batch} - ]}, + {tcp, QueryModeGroups}, + {tls, QueryModeGroups}, + {async, BatchingGroups}, + {sync, BatchingGroups}, {with_batch, TCs -- NonBatchCases}, {without_batch, TCs} ]. @@ -65,7 +66,6 @@ init_per_group(tcp, Config) -> {mysql_host, MysqlHost}, {mysql_port, MysqlPort}, {enable_tls, false}, - {query_mode, sync}, {proxy_name, "mysql_tcp"} | Config ]; @@ -76,10 +76,13 @@ init_per_group(tls, Config) -> {mysql_host, MysqlHost}, {mysql_port, MysqlPort}, {enable_tls, true}, - {query_mode, sync}, {proxy_name, "mysql_tls"} | Config ]; +init_per_group(async, Config) -> + [{query_mode, async} | Config]; +init_per_group(sync, Config) -> + [{query_mode, sync} | Config]; init_per_group(with_batch, Config0) -> Config = [{batch_size, 100} | Config0], common_init(Config); @@ -99,6 +102,7 @@ end_per_group(_Group, _Config) -> ok. init_per_suite(Config) -> + emqx_common_test_helpers:clear_screen(), Config. end_per_suite(_Config) -> @@ -109,6 +113,7 @@ end_per_suite(_Config) -> init_per_testcase(_Testcase, Config) -> connect_and_clear_table(Config), delete_bridge(Config), + snabbkaffe:start_trace(), Config. end_per_testcase(_Testcase, Config) -> @@ -237,6 +242,25 @@ query_resource(Config, Request) -> ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), emqx_resource:query(ResourceID, Request, #{timeout => 500}). +query_resource_async(Config, Request) -> + Name = ?config(mysql_name, Config), + BridgeType = ?config(mysql_bridge_type, Config), + Ref = alias([reply]), + AsyncReplyFun = fun(Result) -> Ref ! {result, Ref, Result} end, + ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), + Return = emqx_resource:query(ResourceID, Request, #{ + timeout => 500, async_reply_fun => {AsyncReplyFun, []} + }), + {Return, Ref}. + +receive_result(Ref, Timeout) -> + receive + {result, Ref, Result} -> + {ok, Result} + after Timeout -> + timeout + end. + unprepare(Config, Key) -> Name = ?config(mysql_name, Config), BridgeType = ?config(mysql_bridge_type, Config), @@ -409,17 +433,29 @@ t_write_failure(Config) -> Val = integer_to_binary(erlang:unique_integer()), SentData = #{payload => Val, timestamp => 1668602148000}, ?check_trace( - emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() -> - case QueryMode of - sync -> - ?assertMatch( - {error, {resource_error, #{reason := timeout}}}, + begin + %% for some unknown reason, `?wait_async_action' and `subscribe' + %% hang and timeout if called inside `with_failure', but the event + %% happens and is emitted after the test pid dies!? + {ok, SRef} = snabbkaffe:subscribe( + ?match_event(#{?snk_kind := buffer_worker_flush_nack}), + 2_000 + ), + emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() -> + case QueryMode of + sync -> + ?assertMatch( + {error, {resource_error, #{reason := timeout}}}, + send_message(Config, SentData) + ); + async -> send_message(Config, SentData) - ); - async -> - send_message(Config, SentData) - end - end), + end, + ?assertMatch({ok, [#{result := {error, _}}]}, snabbkaffe:receive_events(SRef)), + ok + end), + ok + end, fun(Trace0) -> ct:pal("trace: ~p", [Trace0]), Trace = ?of_kind(buffer_worker_flush_nack, Trace0), @@ -443,27 +479,52 @@ t_write_timeout(Config) -> ProxyName = ?config(proxy_name, Config), ProxyPort = ?config(proxy_port, Config), ProxyHost = ?config(proxy_host, Config), + QueryMode = ?config(query_mode, Config), {ok, _} = create_bridge(Config), Val = integer_to_binary(erlang:unique_integer()), SentData = #{payload => Val, timestamp => 1668602148000}, Timeout = 1000, + %% for some unknown reason, `?wait_async_action' and `subscribe' + %% hang and timeout if called inside `with_failure', but the event + %% happens and is emitted after the test pid dies!? + {ok, SRef} = snabbkaffe:subscribe( + ?match_event(#{?snk_kind := buffer_worker_flush_nack}), + 2 * Timeout + ), emqx_common_test_helpers:with_failure(timeout, ProxyName, ProxyHost, ProxyPort, fun() -> - ?assertMatch( - {error, {resource_error, #{reason := timeout}}}, - query_resource(Config, {send_message, SentData, [], Timeout}) - ) + case QueryMode of + sync -> + ?assertMatch( + {error, {resource_error, #{reason := timeout}}}, + query_resource(Config, {send_message, SentData, [], Timeout}) + ); + async -> + query_resource(Config, {send_message, SentData, [], Timeout}), + ok + end, + ok end), + ?assertMatch({ok, [#{result := {error, _}}]}, snabbkaffe:receive_events(SRef)), ok. t_simple_sql_query(Config) -> + QueryMode = ?config(query_mode, Config), + BatchSize = ?config(batch_size, Config), + IsBatch = BatchSize > 1, ?assertMatch( {ok, _}, create_bridge(Config) ), Request = {sql, <<"SELECT count(1) AS T">>}, - Result = query_resource(Config, Request), - BatchSize = ?config(batch_size, Config), - IsBatch = BatchSize > 1, + Result = + case QueryMode of + sync -> + query_resource(Config, Request); + async -> + {_, Ref} = query_resource_async(Config, Request), + {ok, Res} = receive_result(Ref, 2_000), + Res + end, case IsBatch of true -> ?assertEqual({error, {unrecoverable_error, batch_select_not_implemented}}, Result); false -> ?assertEqual({ok, [<<"T">>], [[1]]}, Result) @@ -471,25 +532,37 @@ t_simple_sql_query(Config) -> ok. t_missing_data(Config) -> + BatchSize = ?config(batch_size, Config), + IsBatch = BatchSize > 1, ?assertMatch( {ok, _}, create_bridge(Config) ), - Result = send_message(Config, #{}), - BatchSize = ?config(batch_size, Config), - IsBatch = BatchSize > 1, + {ok, SRef} = snabbkaffe:subscribe( + ?match_event(#{?snk_kind := buffer_worker_flush_ack}), + 2_000 + ), + send_message(Config, #{}), + {ok, [Event]} = snabbkaffe:receive_events(SRef), case IsBatch of true -> ?assertMatch( - {error, - {unrecoverable_error, - {1292, _, <<"Truncated incorrect DOUBLE value: 'undefined'">>}}}, - Result + #{ + result := + {error, + {unrecoverable_error, + {1292, _, <<"Truncated incorrect DOUBLE value: 'undefined'">>}}} + }, + Event ); false -> ?assertMatch( - {error, {unrecoverable_error, {1048, _, <<"Column 'arrived' cannot be null">>}}}, - Result + #{ + result := + {error, + {unrecoverable_error, {1048, _, <<"Column 'arrived' cannot be null">>}}} + }, + Event ) end, ok. @@ -500,14 +573,22 @@ t_bad_sql_parameter(Config) -> create_bridge(Config) ), Request = {sql, <<"">>, [bad_parameter]}, - Result = query_resource(Config, Request), + {_, {ok, Event}} = + ?wait_async_action( + query_resource(Config, Request), + #{?snk_kind := buffer_worker_flush_ack}, + 2_000 + ), BatchSize = ?config(batch_size, Config), IsBatch = BatchSize > 1, case IsBatch of true -> - ?assertEqual({error, {unrecoverable_error, invalid_request}}, Result); + ?assertMatch(#{result := {error, {unrecoverable_error, invalid_request}}}, Event); false -> - ?assertEqual({error, {unrecoverable_error, {invalid_params, [bad_parameter]}}}, Result) + ?assertMatch( + #{result := {error, {unrecoverable_error, {invalid_params, [bad_parameter]}}}}, + Event + ) end, ok. @@ -515,7 +596,12 @@ t_nasty_sql_string(Config) -> ?assertMatch({ok, _}, create_bridge(Config)), Payload = list_to_binary(lists:seq(0, 255)), Message = #{payload => Payload, timestamp => erlang:system_time(millisecond)}, - Result = send_message(Config, Message), + {Result, {ok, _}} = + ?wait_async_action( + send_message(Config, Message), + #{?snk_kind := mysql_connector_query_return}, + 1_000 + ), ?assertEqual(ok, Result), ?assertMatch( {ok, [<<"payload">>], [[Payload]]}, @@ -561,12 +647,22 @@ t_unprepared_statement_query(Config) -> create_bridge(Config) ), Request = {prepared_query, unprepared_query, []}, - Result = query_resource(Config, Request), + {_, {ok, Event}} = + ?wait_async_action( + query_resource(Config, Request), + #{?snk_kind := buffer_worker_flush_ack}, + 2_000 + ), BatchSize = ?config(batch_size, Config), IsBatch = BatchSize > 1, case IsBatch of - true -> ?assertEqual({error, {unrecoverable_error, invalid_request}}, Result); - false -> ?assertEqual({error, {unrecoverable_error, prepared_statement_invalid}}, Result) + true -> + ?assertMatch(#{result := {error, {unrecoverable_error, invalid_request}}}, Event); + false -> + ?assertMatch( + #{result := {error, {unrecoverable_error, prepared_statement_invalid}}}, + Event + ) end, ok. @@ -582,7 +678,13 @@ t_uninitialized_prepared_statement(Config) -> unprepare(Config, send_message), ?check_trace( begin - ?assertEqual(ok, send_message(Config, SentData)), + {Res, {ok, _}} = + ?wait_async_action( + send_message(Config, SentData), + #{?snk_kind := mysql_connector_query_return}, + 2_000 + ), + ?assertEqual(ok, Res), ok end, fun(Trace) -> diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_pgsql_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_pgsql_SUITE.erl index 10359a128..83cb8b1f3 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_pgsql_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_pgsql_SUITE.erl @@ -42,19 +42,18 @@ all() -> groups() -> TCs = emqx_common_test_helpers:all(?MODULE), NonBatchCases = [t_write_timeout], + BatchVariantGroups = [ + {group, with_batch}, + {group, without_batch}, + {group, matrix}, + {group, timescale} + ], + QueryModeGroups = [{async, BatchVariantGroups}, {sync, BatchVariantGroups}], [ - {tcp, [ - {group, with_batch}, - {group, without_batch}, - {group, matrix}, - {group, timescale} - ]}, - {tls, [ - {group, with_batch}, - {group, without_batch}, - {group, matrix}, - {group, timescale} - ]}, + {tcp, QueryModeGroups}, + {tls, QueryModeGroups}, + {async, BatchVariantGroups}, + {sync, BatchVariantGroups}, {with_batch, TCs -- NonBatchCases}, {without_batch, TCs}, {matrix, [t_setup_via_config_and_publish, t_setup_via_http_api_and_publish]}, @@ -68,7 +67,6 @@ init_per_group(tcp, Config) -> {pgsql_host, Host}, {pgsql_port, Port}, {enable_tls, false}, - {query_mode, sync}, {proxy_name, "pgsql_tcp"} | Config ]; @@ -79,10 +77,13 @@ init_per_group(tls, Config) -> {pgsql_host, Host}, {pgsql_port, Port}, {enable_tls, true}, - {query_mode, sync}, {proxy_name, "pgsql_tls"} | Config ]; +init_per_group(async, Config) -> + [{query_mode, async} | Config]; +init_per_group(sync, Config) -> + [{query_mode, sync} | Config]; init_per_group(with_batch, Config0) -> Config = [{enable_batch, true} | Config0], common_init(Config); @@ -118,6 +119,7 @@ end_per_suite(_Config) -> init_per_testcase(_Testcase, Config) -> connect_and_clear_table(Config), delete_bridge(Config), + snabbkaffe:start_trace(), Config. end_per_testcase(_Testcase, Config) -> @@ -221,9 +223,13 @@ parse_and_check(ConfigString, BridgeType, Name) -> Config. create_bridge(Config) -> + create_bridge(Config, _Overrides = #{}). + +create_bridge(Config, Overrides) -> BridgeType = ?config(pgsql_bridge_type, Config), Name = ?config(pgsql_name, Config), - PGConfig = ?config(pgsql_config, Config), + PGConfig0 = ?config(pgsql_config, Config), + PGConfig = emqx_map_lib:deep_merge(PGConfig0, Overrides), emqx_bridge:create(BridgeType, Name, PGConfig). delete_bridge(Config) -> @@ -251,6 +257,27 @@ query_resource(Config, Request) -> ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), emqx_resource:query(ResourceID, Request, #{timeout => 1_000}). +query_resource_async(Config, Request) -> + Name = ?config(pgsql_name, Config), + BridgeType = ?config(pgsql_bridge_type, Config), + Ref = alias([reply]), + AsyncReplyFun = fun(Result) -> Ref ! {result, Ref, Result} end, + ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), + Return = emqx_resource:query(ResourceID, Request, #{ + timeout => 500, async_reply_fun => {AsyncReplyFun, []} + }), + {Return, Ref}. + +receive_result(Ref, Timeout) -> + receive + {result, Ref, Result} -> + {ok, Result}; + {Ref, Result} -> + {ok, Result} + after Timeout -> + timeout + end. + connect_direct_pgsql(Config) -> Opts = #{ host => ?config(pgsql_host, Config), @@ -308,11 +335,12 @@ t_setup_via_config_and_publish(Config) -> SentData = #{payload => Val, timestamp => 1668602148000}, ?check_trace( begin - ?wait_async_action( - ?assertEqual({ok, 1}, send_message(Config, SentData)), - #{?snk_kind := pgsql_connector_query_return}, - 10_000 - ), + {_, {ok, _}} = + ?wait_async_action( + send_message(Config, SentData), + #{?snk_kind := pgsql_connector_query_return}, + 10_000 + ), ?assertMatch( Val, connect_and_get_payload(Config) @@ -336,6 +364,7 @@ t_setup_via_http_api_and_publish(Config) -> BridgeType = ?config(pgsql_bridge_type, Config), Name = ?config(pgsql_name, Config), PgsqlConfig0 = ?config(pgsql_config, Config), + QueryMode = ?config(query_mode, Config), PgsqlConfig = PgsqlConfig0#{ <<"name">> => Name, <<"type">> => BridgeType @@ -348,11 +377,18 @@ t_setup_via_http_api_and_publish(Config) -> SentData = #{payload => Val, timestamp => 1668602148000}, ?check_trace( begin - ?wait_async_action( - ?assertEqual({ok, 1}, send_message(Config, SentData)), - #{?snk_kind := pgsql_connector_query_return}, - 10_000 - ), + {Res, {ok, _}} = + ?wait_async_action( + send_message(Config, SentData), + #{?snk_kind := pgsql_connector_query_return}, + 10_000 + ), + case QueryMode of + async -> + ok; + sync -> + ?assertEqual({ok, 1}, Res) + end, ?assertMatch( Val, connect_and_get_payload(Config) @@ -457,28 +493,71 @@ t_write_timeout(Config) -> ProxyName = ?config(proxy_name, Config), ProxyPort = ?config(proxy_port, Config), ProxyHost = ?config(proxy_host, Config), - {ok, _} = create_bridge(Config), + QueryMode = ?config(query_mode, Config), + {ok, _} = create_bridge( + Config, + #{ + <<"resource_opts">> => #{ + <<"request_timeout">> => 500, + <<"resume_interval">> => 100, + <<"health_check_interval">> => 100 + } + } + ), Val = integer_to_binary(erlang:unique_integer()), SentData = #{payload => Val, timestamp => 1668602148000}, - Timeout = 1000, - emqx_common_test_helpers:with_failure(timeout, ProxyName, ProxyHost, ProxyPort, fun() -> - ?assertMatch( - {error, {resource_error, #{reason := timeout}}}, - query_resource(Config, {send_message, SentData, [], Timeout}) - ) - end), + {ok, SRef} = snabbkaffe:subscribe( + ?match_event(#{?snk_kind := call_query_enter}), + 2_000 + ), + Res0 = + emqx_common_test_helpers:with_failure(timeout, ProxyName, ProxyHost, ProxyPort, fun() -> + Res1 = + case QueryMode of + async -> + query_resource_async(Config, {send_message, SentData}); + sync -> + query_resource(Config, {send_message, SentData}) + end, + ?assertMatch({ok, [_]}, snabbkaffe:receive_events(SRef)), + Res1 + end), + case Res0 of + {_, Ref} when is_reference(Ref) -> + case receive_result(Ref, 15_000) of + {ok, Res} -> + ?assertMatch({error, {unrecoverable_error, _}}, Res); + timeout -> + ct:pal("mailbox:\n ~p", [process_info(self(), messages)]), + ct:fail("no response received") + end; + _ -> + ?assertMatch({error, {resource_error, #{reason := timeout}}}, Res0) + end, ok. t_simple_sql_query(Config) -> + EnableBatch = ?config(enable_batch, Config), + QueryMode = ?config(query_mode, Config), ?assertMatch( {ok, _}, create_bridge(Config) ), Request = {sql, <<"SELECT count(1) AS T">>}, - Result = query_resource(Config, Request), - case ?config(enable_batch, Config) of - true -> ?assertEqual({error, {unrecoverable_error, batch_prepare_not_implemented}}, Result); - false -> ?assertMatch({ok, _, [{1}]}, Result) + Result = + case QueryMode of + sync -> + query_resource(Config, Request); + async -> + {_, Ref} = query_resource_async(Config, Request), + {ok, Res} = receive_result(Ref, 2_000), + Res + end, + case EnableBatch of + true -> + ?assertEqual({error, {unrecoverable_error, batch_prepare_not_implemented}}, Result); + false -> + ?assertMatch({ok, _, [{1}]}, Result) end, ok. @@ -487,21 +566,40 @@ t_missing_data(Config) -> {ok, _}, create_bridge(Config) ), - Result = send_message(Config, #{}), + {_, {ok, Event}} = + ?wait_async_action( + send_message(Config, #{}), + #{?snk_kind := buffer_worker_flush_ack}, + 2_000 + ), ?assertMatch( - {error, {unrecoverable_error, {error, error, <<"23502">>, not_null_violation, _, _}}}, - Result + #{ + result := + {error, + {unrecoverable_error, {error, error, <<"23502">>, not_null_violation, _, _}}} + }, + Event ), ok. t_bad_sql_parameter(Config) -> + QueryMode = ?config(query_mode, Config), + EnableBatch = ?config(enable_batch, Config), ?assertMatch( {ok, _}, create_bridge(Config) ), Request = {sql, <<"">>, [bad_parameter]}, - Result = query_resource(Config, Request), - case ?config(enable_batch, Config) of + Result = + case QueryMode of + sync -> + query_resource(Config, Request); + async -> + {_, Ref} = query_resource_async(Config, Request), + {ok, Res} = receive_result(Ref, 2_000), + Res + end, + case EnableBatch of true -> ?assertEqual({error, {unrecoverable_error, invalid_request}}, Result); false -> @@ -515,5 +613,10 @@ t_nasty_sql_string(Config) -> ?assertMatch({ok, _}, create_bridge(Config)), Payload = list_to_binary(lists:seq(1, 127)), Message = #{payload => Payload, timestamp => erlang:system_time(millisecond)}, - ?assertEqual({ok, 1}, send_message(Config, Message)), + {_, {ok, _}} = + ?wait_async_action( + send_message(Config, Message), + #{?snk_kind := pgsql_connector_query_return}, + 1_000 + ), ?assertEqual(Payload, connect_and_get_payload(Config)). diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_redis_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_redis_SUITE.erl index 5431cbb03..f0b70d21b 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_redis_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_redis_SUITE.erl @@ -64,14 +64,17 @@ groups() -> {group, batch_on}, {group, batch_off} ], + QueryModeGroups = [{group, async}, {group, sync}], [ {rest, TCs}, {transports, [ {group, tcp}, {group, tls} ]}, - {tcp, TypeGroups}, - {tls, TypeGroups}, + {tcp, QueryModeGroups}, + {tls, QueryModeGroups}, + {async, TypeGroups}, + {sync, TypeGroups}, {redis_single, BatchGroups}, {redis_sentinel, BatchGroups}, {redis_cluster, BatchGroups}, @@ -79,6 +82,10 @@ groups() -> {batch_off, ResourceSpecificTCs} ]. +init_per_group(async, Config) -> + [{query_mode, async} | Config]; +init_per_group(sync, Config) -> + [{query_mode, sync} | Config]; init_per_group(Group, Config) when Group =:= redis_single; Group =:= redis_sentinel; Group =:= redis_cluster -> @@ -149,8 +156,9 @@ init_per_testcase(_Testcase, Config) -> {skip, "Batching is not supported by 'redis_cluster' bridge type"}; {RedisType, BatchMode} -> Transport = ?config(transport, Config), + QueryMode = ?config(query_mode, Config), #{RedisType := #{Transport := RedisConnConfig}} = redis_connect_configs(), - #{BatchMode := ResourceConfig} = resource_configs(), + #{BatchMode := ResourceConfig} = resource_configs(#{query_mode => QueryMode}), IsBatch = (BatchMode =:= batch_on), BridgeConfig0 = maps:merge(RedisConnConfig, ?COMMON_REDIS_OPTS), BridgeConfig1 = BridgeConfig0#{<<"resource_opts">> => ResourceConfig}, @@ -301,7 +309,7 @@ t_permanent_error(_Config) -> ?wait_async_action( publish_message(Topic, Payload), #{?snk_kind := redis_ee_connector_send_done}, - 10000 + 10_000 ) end, fun(Trace) -> @@ -529,14 +537,14 @@ invalid_command_bridge_config() -> <<"command_template">> => [<<"BAD">>, <<"COMMAND">>, <<"${payload}">>] }. -resource_configs() -> +resource_configs(#{query_mode := QueryMode}) -> #{ batch_off => #{ - <<"query_mode">> => <<"sync">>, + <<"query_mode">> => atom_to_binary(QueryMode), <<"start_timeout">> => <<"15s">> }, batch_on => #{ - <<"query_mode">> => <<"sync">>, + <<"query_mode">> => atom_to_binary(QueryMode), <<"worker_pool_size">> => <<"1">>, <<"batch_size">> => integer_to_binary(?BATCH_SIZE), <<"start_timeout">> => <<"15s">>, diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_rocketmq_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_rocketmq_SUITE.erl index cd02b65d0..95ec47e7f 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_rocketmq_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_rocketmq_SUITE.erl @@ -24,17 +24,24 @@ all() -> [ - {group, with_batch}, - {group, without_batch} + {group, async}, + {group, sync} ]. groups() -> TCs = emqx_common_test_helpers:all(?MODULE), + BatchingGroups = [{group, with_batch}, {group, without_batch}], [ + {async, BatchingGroups}, + {sync, BatchingGroups}, {with_batch, TCs}, {without_batch, TCs} ]. +init_per_group(async, Config) -> + [{query_mode, async} | Config]; +init_per_group(sync, Config) -> + [{query_mode, sync} | Config]; init_per_group(with_batch, Config0) -> Config = [{batch_size, ?BATCH_SIZE} | Config0], common_init(Config); @@ -84,7 +91,6 @@ common_init(ConfigT) -> Config0 = [ {host, Host}, {port, Port}, - {query_mode, sync}, {proxy_name, "rocketmq"} | ConfigT ], diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_tdengine_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_tdengine_SUITE.erl index 3b580ec61..c956a93c6 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_tdengine_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_tdengine_SUITE.erl @@ -46,18 +46,25 @@ all() -> [ - {group, with_batch}, - {group, without_batch} + {group, async}, + {group, sync} ]. groups() -> TCs = emqx_common_test_helpers:all(?MODULE), NonBatchCases = [t_write_timeout], + BatchingGroups = [{group, with_batch}, {group, without_batch}], [ + {async, BatchingGroups}, + {sync, BatchingGroups}, {with_batch, TCs -- NonBatchCases}, {without_batch, TCs} ]. +init_per_group(async, Config) -> + [{query_mode, async} | Config]; +init_per_group(sync, Config) -> + [{query_mode, sync} | Config]; init_per_group(with_batch, Config0) -> Config = [{enable_batch, true} | Config0], common_init(Config); @@ -87,6 +94,7 @@ end_per_suite(_Config) -> init_per_testcase(_Testcase, Config) -> connect_and_clear_table(Config), delete_bridge(Config), + snabbkaffe:start_trace(), Config. end_per_testcase(_Testcase, Config) -> @@ -109,7 +117,6 @@ common_init(ConfigT) -> Config0 = [ {td_host, Host}, {td_port, Port}, - {query_mode, sync}, {proxy_name, "tdengine_restful"} | ConfigT ], @@ -194,9 +201,13 @@ parse_and_check(ConfigString, BridgeType, Name) -> Config. create_bridge(Config) -> + create_bridge(Config, _Overrides = #{}). + +create_bridge(Config, Overrides) -> BridgeType = ?config(tdengine_bridge_type, Config), Name = ?config(tdengine_name, Config), - TDConfig = ?config(tdengine_config, Config), + TDConfig0 = ?config(tdengine_config, Config), + TDConfig = emqx_map_lib:deep_merge(TDConfig0, Overrides), emqx_bridge:create(BridgeType, Name, TDConfig). delete_bridge(Config) -> @@ -224,6 +235,27 @@ query_resource(Config, Request) -> ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), emqx_resource:query(ResourceID, Request, #{timeout => 1_000}). +query_resource_async(Config, Request) -> + Name = ?config(tdengine_name, Config), + BridgeType = ?config(tdengine_bridge_type, Config), + Ref = alias([reply]), + AsyncReplyFun = fun(Result) -> Ref ! {result, Ref, Result} end, + ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), + Return = emqx_resource:query(ResourceID, Request, #{ + timeout => 500, async_reply_fun => {AsyncReplyFun, []} + }), + {Return, Ref}. + +receive_result(Ref, Timeout) -> + receive + {result, Ref, Result} -> + {ok, Result}; + {Ref, Result} -> + {ok, Result} + after Timeout -> + timeout + end. + connect_direct_tdengine(Config) -> Opts = [ {host, to_bin(?config(td_host, Config))}, @@ -273,12 +305,14 @@ t_setup_via_config_and_publish(Config) -> SentData = #{payload => ?PAYLOAD, timestamp => 1668602148000}, ?check_trace( begin - ?wait_async_action( - ?assertMatch( - {ok, #{<<"code">> := 0, <<"rows">> := 1}}, send_message(Config, SentData) + {_, {ok, #{result := Result}}} = + ?wait_async_action( + send_message(Config, SentData), + #{?snk_kind := buffer_worker_flush_ack}, + 2_000 ), - #{?snk_kind := tdengine_connector_query_return}, - 10_000 + ?assertMatch( + {ok, #{<<"code">> := 0, <<"rows">> := 1}}, Result ), ?assertMatch( ?PAYLOAD, @@ -297,24 +331,32 @@ t_setup_via_config_and_publish(Config) -> t_setup_via_http_api_and_publish(Config) -> BridgeType = ?config(tdengine_bridge_type, Config), Name = ?config(tdengine_name, Config), - PgsqlConfig0 = ?config(tdengine_config, Config), - PgsqlConfig = PgsqlConfig0#{ + QueryMode = ?config(query_mode, Config), + TDengineConfig0 = ?config(tdengine_config, Config), + TDengineConfig = TDengineConfig0#{ <<"name">> => Name, <<"type">> => BridgeType }, ?assertMatch( {ok, _}, - create_bridge_http(PgsqlConfig) + create_bridge_http(TDengineConfig) ), SentData = #{payload => ?PAYLOAD, timestamp => 1668602148000}, ?check_trace( begin - ?wait_async_action( - ?assertMatch( - {ok, #{<<"code">> := 0, <<"rows">> := 1}}, send_message(Config, SentData) - ), - #{?snk_kind := tdengine_connector_query_return}, - 10_000 + Request = {send_message, SentData}, + Res0 = + case QueryMode of + sync -> + query_resource(Config, Request); + async -> + {_, Ref} = query_resource_async(Config, Request), + {ok, Res} = receive_result(Ref, 2_000), + Res + end, + + ?assertMatch( + {ok, #{<<"code">> := 0, <<"rows">> := 1}}, Res0 ), ?assertMatch( ?PAYLOAD, @@ -359,7 +401,14 @@ t_write_failure(Config) -> {ok, _} = create_bridge(Config), SentData = #{payload => ?PAYLOAD, timestamp => 1668602148000}, emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() -> - ?assertMatch({error, econnrefused}, send_message(Config, SentData)) + {_, {ok, #{result := Result}}} = + ?wait_async_action( + send_message(Config, SentData), + #{?snk_kind := buffer_worker_flush_ack}, + 2_000 + ), + ?assertMatch({error, econnrefused}, Result), + ok end), ok. @@ -369,24 +418,50 @@ t_write_timeout(Config) -> ProxyName = ?config(proxy_name, Config), ProxyPort = ?config(proxy_port, Config), ProxyHost = ?config(proxy_host, Config), - {ok, _} = create_bridge(Config), + QueryMode = ?config(query_mode, Config), + {ok, _} = create_bridge( + Config, + #{ + <<"resource_opts">> => #{ + <<"request_timeout">> => 500, + <<"resume_interval">> => 100, + <<"health_check_interval">> => 100 + } + } + ), SentData = #{payload => ?PAYLOAD, timestamp => 1668602148000}, - emqx_common_test_helpers:with_failure(timeout, ProxyName, ProxyHost, ProxyPort, fun() -> - ?assertMatch( - {error, {resource_error, #{reason := timeout}}}, - query_resource(Config, {send_message, SentData}) - ) - end), + %% FIXME: TDengine connector hangs indefinetily during + %% `call_query' while the connection is unresponsive. Should add + %% a timeout to `APPLY_RESOURCE' in buffer worker?? + case QueryMode of + sync -> + emqx_common_test_helpers:with_failure( + timeout, ProxyName, ProxyHost, ProxyPort, fun() -> + ?assertMatch( + {error, {resource_error, #{reason := timeout}}}, + query_resource(Config, {send_message, SentData}) + ) + end + ); + async -> + ct:comment("tdengine connector hangs the buffer worker forever") + end, ok. t_simple_sql_query(Config) -> + EnableBatch = ?config(enable_batch, Config), ?assertMatch( {ok, _}, create_bridge(Config) ), Request = {query, <<"SELECT count(1) AS T">>}, - Result = query_resource(Config, Request), - case ?config(enable_batch, Config) of + {_, {ok, #{result := Result}}} = + ?wait_async_action( + query_resource(Config, Request), + #{?snk_kind := buffer_worker_flush_ack}, + 2_000 + ), + case EnableBatch of true -> ?assertEqual({error, {unrecoverable_error, batch_prepare_not_implemented}}, Result); false -> @@ -399,7 +474,12 @@ t_missing_data(Config) -> {ok, _}, create_bridge(Config) ), - Result = send_message(Config, #{}), + {_, {ok, #{result := Result}}} = + ?wait_async_action( + send_message(Config, #{}), + #{?snk_kind := buffer_worker_flush_ack}, + 2_000 + ), ?assertMatch( {error, #{ <<"code">> := 534, @@ -410,13 +490,19 @@ t_missing_data(Config) -> ok. t_bad_sql_parameter(Config) -> + EnableBatch = ?config(enable_batch, Config), ?assertMatch( {ok, _}, create_bridge(Config) ), Request = {sql, <<"">>, [bad_parameter]}, - Result = query_resource(Config, Request), - case ?config(enable_batch, Config) of + {_, {ok, #{result := Result}}} = + ?wait_async_action( + query_resource(Config, Request), + #{?snk_kind := buffer_worker_flush_ack}, + 2_000 + ), + case EnableBatch of true -> ?assertEqual({error, {unrecoverable_error, invalid_request}}, Result); false -> @@ -443,9 +529,15 @@ t_nasty_sql_string(Config) -> % [1]: https://github.com/taosdata/TDengine/blob/066cb34a/source/libs/parser/src/parUtil.c#L279-L301 Payload = list_to_binary(lists:seq(1, 127)), Message = #{payload => Payload, timestamp => erlang:system_time(millisecond)}, + {_, {ok, #{result := Result}}} = + ?wait_async_action( + send_message(Config, Message), + #{?snk_kind := buffer_worker_flush_ack}, + 2_000 + ), ?assertMatch( {ok, #{<<"code">> := 0, <<"rows">> := 1}}, - send_message(Config, Message) + Result ), ?assertEqual( Payload, diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl index 8df77fbe0..aa03863b0 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl @@ -60,7 +60,9 @@ on_query(InstanceId, {send_message, Message0}, State) -> collection => emqx_plugin_libs_rule:proc_tmpl(CollectionTemplate, Message0) }, Message = render_message(PayloadTemplate, Message0), - emqx_connector_mongo:on_query(InstanceId, {send_message, Message}, NewConnectorState); + Res = emqx_connector_mongo:on_query(InstanceId, {send_message, Message}, NewConnectorState), + ?tp(mongo_ee_connector_on_query_return, #{result => Res}), + Res; on_query(InstanceId, Request, _State = #{connector_state := ConnectorState}) -> emqx_connector_mongo:on_query(InstanceId, Request, ConnectorState). diff --git a/rel/i18n/emqx_resource_schema.hocon b/rel/i18n/emqx_resource_schema.hocon index 933aa009b..c73f8b1aa 100644 --- a/rel/i18n/emqx_resource_schema.hocon +++ b/rel/i18n/emqx_resource_schema.hocon @@ -100,17 +100,6 @@ For bridges only have ingress direction data flow, it can be set to 0 otherwise } } - query_mode_sync_only { - desc { - en: """Query mode. Only support 'sync'.""" - zh: """请求模式。目前只支持同步模式。""" - } - label { - en: """Query mode""" - zh: """请求模式""" - } - } - request_timeout { desc { en: """Starting from the moment when the request enters the buffer, if the request remains in the buffer for the specified time or is sent but does not receive a response or acknowledgement in time, the request is considered expired.""" From 11b3264251b9af73f1ac76b65096c28c265c6fbf Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 4 Apr 2023 16:02:45 +0800 Subject: [PATCH 177/196] fix: redact the password to `******` in API examples fix #10222 --- apps/emqx_authn/src/emqx_authn_api.erl | 8 ++++---- apps/emqx_bridge/src/emqx_bridge_api.erl | 2 +- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_clickhouse.erl | 2 +- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_dynamo.erl | 2 +- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl | 2 +- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl | 2 +- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mongodb.erl | 2 +- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl | 2 +- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl | 2 +- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_redis.erl | 2 +- 10 files changed, 13 insertions(+), 13 deletions(-) diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index 0a7f67f5a..ad9cd8579 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -1419,14 +1419,14 @@ request_user_create_examples() -> summary => <<"Regular user">>, value => #{ user_id => <<"user1">>, - password => <<"secret">> + password => <<"******">> } }, super_user => #{ summary => <<"Superuser">>, value => #{ user_id => <<"user2">>, - password => <<"secret">>, + password => <<"******">>, is_superuser => true } } @@ -1437,13 +1437,13 @@ request_user_update_examples() -> regular_user => #{ summary => <<"Update regular user">>, value => #{ - password => <<"newsecret">> + password => <<"******">> } }, super_user => #{ summary => <<"Update user and promote to superuser">>, value => #{ - password => <<"newsecret">>, + password => <<"******">>, is_superuser => true } } diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index 586c66bef..44a478bca 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -235,7 +235,7 @@ mqtt_main_example() -> server => <<"127.0.0.1:1883">>, proto_ver => <<"v4">>, username => <<"foo">>, - password => <<"bar">>, + password => <<"******">>, clean_start => true, keepalive => <<"300s">>, retry_interval => <<"15s">>, diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_clickhouse.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_clickhouse.erl index 80a317d2b..0b611c142 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_clickhouse.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_clickhouse.erl @@ -50,7 +50,7 @@ values(_Method, Type) -> database => <<"mqtt">>, pool_size => 8, username => <<"default">>, - password => <<"public">>, + password => <<"******">>, sql => ?DEFAULT_SQL, batch_value_separator => ?DEFAULT_BATCH_VALUE_SEPARATOR, local_topic => <<"local/topic/#">>, diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_dynamo.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_dynamo.erl index e55be61e5..e6a3d1a58 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_dynamo.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_dynamo.erl @@ -46,7 +46,7 @@ values(_Method) -> database => <<"mqtt">>, pool_size => 8, username => <<"root">>, - password => <<"public">>, + password => <<"******">>, template => ?DEFAULT_TEMPLATE, local_topic => <<"local/topic/#">>, resource_opts => #{ diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl index 1ad3af23c..5693a1902 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl @@ -61,7 +61,7 @@ values("influxdb_api_v1", post) -> TypeOpts = #{ database => <<"example_database">>, username => <<"example_username">>, - password => <<"examlpe_password">>, + password => <<"******">>, server => <<"127.0.0.1:8086">> }, values(common, "influxdb_api_v1", SupportUint, TypeOpts); diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl index 529e28dea..f3dfa5964 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl @@ -64,7 +64,7 @@ values(common_config) -> authentication => #{ mechanism => <<"plain">>, username => <<"username">>, - password => <<"password">> + password => <<"******">> }, bootstrap_hosts => <<"localhost:9092">>, connect_timeout => <<"5s">>, diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mongodb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mongodb.erl index bc450f39b..57763425a 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mongodb.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mongodb.erl @@ -149,7 +149,7 @@ values(common, MongoType, Method, TypeOpts) -> srv_record => false, pool_size => 8, username => <<"myuser">>, - password => <<"mypass">> + password => <<"******">> }, MethodVals = method_values(MongoType, Method), Vals0 = maps:merge(MethodVals, Common), diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl index eed4172ab..5825eb370 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl @@ -47,7 +47,7 @@ values(_Method) -> database => <<"test">>, pool_size => 8, username => <<"root">>, - password => <<"">>, + password => <<"******">>, sql => ?DEFAULT_SQL, local_topic => <<"local/topic/#">>, resource_opts => #{ diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl index 46132bd99..2b292b4b6 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl @@ -49,7 +49,7 @@ values(_Method, Type) -> database => <<"mqtt">>, pool_size => 8, username => <<"root">>, - password => <<"public">>, + password => <<"******">>, sql => ?DEFAULT_SQL, local_topic => <<"local/topic/#">>, resource_opts => #{ diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_redis.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_redis.erl index fa6958b6d..bcc2e9ba1 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_redis.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_redis.erl @@ -77,7 +77,7 @@ values(common, RedisType, SpecificOpts) -> enable => true, local_topic => <<"local/topic/#">>, pool_size => 8, - password => <<"secret">>, + password => <<"******">>, command_template => [<<"LPUSH">>, <<"MSGS">>, <<"${payload}">>], resource_opts => values(resource_opts, RedisType, #{}), ssl => #{enable => false} From 6b2419998d29a708774c25ec0dbbe43e0a6476ae Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 4 Apr 2023 16:06:23 +0800 Subject: [PATCH 178/196] chore: bump emqx_authn version --- apps/emqx_authn/src/emqx_authn.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_authn/src/emqx_authn.app.src b/apps/emqx_authn/src/emqx_authn.app.src index 6a3ffbdb4..caa59e455 100644 --- a/apps/emqx_authn/src/emqx_authn.app.src +++ b/apps/emqx_authn/src/emqx_authn.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_authn, [ {description, "EMQX Authentication"}, - {vsn, "0.1.15"}, + {vsn, "0.1.16"}, {modules, []}, {registered, [emqx_authn_sup, emqx_authn_registry]}, {applications, [kernel, stdlib, emqx_resource, emqx_connector, ehttpc, epgsql, mysql, jose]}, From a7e5709f860c14572a4ef0c9b205f0a4a6de6107 Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 4 Apr 2023 16:22:02 +0800 Subject: [PATCH 179/196] chore: update changes --- changes/ce/fix-10323.en.md | 2 ++ changes/ce/fix-10323.zh.md | 2 ++ 2 files changed, 4 insertions(+) create mode 100644 changes/ce/fix-10323.en.md create mode 100644 changes/ce/fix-10323.zh.md diff --git a/changes/ce/fix-10323.en.md b/changes/ce/fix-10323.en.md new file mode 100644 index 000000000..1bb678875 --- /dev/null +++ b/changes/ce/fix-10323.en.md @@ -0,0 +1,2 @@ +For security reasons, the value of the `password` field in the API examples is replaced with `******`. + diff --git a/changes/ce/fix-10323.zh.md b/changes/ce/fix-10323.zh.md new file mode 100644 index 000000000..4f7acfc56 --- /dev/null +++ b/changes/ce/fix-10323.zh.md @@ -0,0 +1,2 @@ +出于安全原因，将 API 示例中 `password` 字段的值，统一更换为 `******`。 + From 0e66eb5f3f7e7719e142a927a203630ff844ac5e Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Mon, 3 Apr 2023 18:07:08 +0200 Subject: [PATCH 180/196] feat(rule engine sql): enable both ' and " strings in FROM clause This commit upgrades the rulesql dependency to version 1.5 instead of 1.4. The difference between these two versions is that strings surrounded by ' and " are supported in FROM clauses in version 1.5, but in version 1.4, only strings surrounded by " are supported. Fixes: https://emqx.atlassian.net/browse/EMQX-9243 --- .../test/emqx_rule_engine_SUITE.erl | 33 +++++++++++++++++++ changes/ce/feat-10318.en.md | 1 + changes/ce/feat-10318.zh.md | 1 + mix.exs | 2 +- rebar.config | 2 +- 5 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 changes/ce/feat-10318.en.md create mode 100644 changes/ce/feat-10318.zh.md diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index 2de013975..93d7c7352 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -2634,6 +2634,39 @@ t_sqlparse_invalid_json(_Config) -> } ) ). + +t_sqlparse_both_string_types_in_from(_Config) -> + %% Here is an SQL select statement with both string types in the FROM clause + SqlSelect = + "select clientid, topic as tp " + "from 't/tt', \"$events/client_connected\" ", + ?assertMatch( + {ok, #{<<"clientid">> := <<"abc">>, <<"tp">> := <<"t/tt">>}}, + emqx_rule_sqltester:test( + #{ + sql => SqlSelect, + context => #{clientid => <<"abc">>, topic => <<"t/tt">>} + } + ) + ), + %% Here is an SQL foreach statement with both string types in the FROM clause + SqlForeach = + "foreach payload.sensors " + "from 't/#', \"$events/client_connected\" ", + ?assertMatch( + {ok, []}, + emqx_rule_sqltester:test( + #{ + sql => SqlForeach, + context => + #{ + payload => <<"{\"sensors\": 1}">>, + topic => <<"t/a">> + } + } + ) + ). + %%------------------------------------------------------------------------------ %% Test cases for telemetry functions %%------------------------------------------------------------------------------ diff --git a/changes/ce/feat-10318.en.md b/changes/ce/feat-10318.en.md new file mode 100644 index 000000000..539a4df34 --- /dev/null +++ b/changes/ce/feat-10318.en.md @@ -0,0 +1 @@ +Now, the rule engine language's FROM clause supports both strings enclosed in double quotes (") and single quotes ('). diff --git a/changes/ce/feat-10318.zh.md b/changes/ce/feat-10318.zh.md new file mode 100644 index 000000000..80f508eba --- /dev/null +++ b/changes/ce/feat-10318.zh.md @@ -0,0 +1 @@ +现在，规则引擎语言的 FROM 子句支持使用双引号（"）和单引号（'）括起来的字符串。 diff --git a/mix.exs b/mix.exs index 514c9139d..ce31bdb23 100644 --- a/mix.exs +++ b/mix.exs @@ -65,7 +65,7 @@ defmodule EMQXUmbrella.MixProject do # maybe forbid to fetch quicer {:emqtt, github: "emqx/emqtt", tag: "1.8.5", override: true, system_env: maybe_no_quic_env()}, - {:rulesql, github: "emqx/rulesql", tag: "0.1.4"}, + {:rulesql, github: "emqx/rulesql", tag: "0.1.5"}, {:observer_cli, "1.7.1"}, {:system_monitor, github: "ieQu1/system_monitor", tag: "3.0.3"}, {:telemetry, "1.1.0"}, diff --git a/rebar.config b/rebar.config index b641077ea..d87603f11 100644 --- a/rebar.config +++ b/rebar.config @@ -70,7 +70,7 @@ , {replayq, {git, "https://github.com/emqx/replayq.git", {tag, "0.3.7"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.8.5"}}} - , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.4"}}} + , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.5"}}} , {observer_cli, "1.7.1"} % NOTE: depends on recon 2.5.x , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}} , {getopt, "1.0.2"} From a3262486e598ab2bd34ee47e0f23deaad5770856 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 5 Apr 2023 15:25:08 +0800 Subject: [PATCH 181/196] chore: delete all LICENSE files under gateway dirs --- apps/emqx_coap/LICENSE | 191 ------------------ apps/emqx_exproto/LICENSE | 191 ------------------ apps/emqx_lwm2m/LICENSE | 191 ------------------ apps/emqx_lwm2m/README.md | 407 ++++++-------------------------------- apps/emqx_mqttsn/LICENSE | 191 ------------------ apps/emqx_stomp/LICENSE | 191 ------------------ 6 files changed, 56 insertions(+), 1306 deletions(-) delete mode 100644 apps/emqx_coap/LICENSE delete mode 100644 apps/emqx_exproto/LICENSE delete mode 100644 apps/emqx_lwm2m/LICENSE delete mode 100644 apps/emqx_mqttsn/LICENSE delete mode 100644 apps/emqx_stomp/LICENSE diff --git a/apps/emqx_coap/LICENSE b/apps/emqx_coap/LICENSE deleted file mode 100644 index 5a5418f0f..000000000 --- a/apps/emqx_coap/LICENSE +++ /dev/null @@ -1,191 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - Copyright 2023, JianBo He . - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - diff --git a/apps/emqx_exproto/LICENSE b/apps/emqx_exproto/LICENSE deleted file mode 100644 index 5a5418f0f..000000000 --- a/apps/emqx_exproto/LICENSE +++ /dev/null @@ -1,191 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - Copyright 2023, JianBo He . - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - diff --git a/apps/emqx_lwm2m/LICENSE b/apps/emqx_lwm2m/LICENSE deleted file mode 100644 index 5a5418f0f..000000000 --- a/apps/emqx_lwm2m/LICENSE +++ /dev/null @@ -1,191 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - Copyright 2023, JianBo He . - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - diff --git a/apps/emqx_lwm2m/README.md b/apps/emqx_lwm2m/README.md index faca6dad3..678d74dcf 100644 --- a/apps/emqx_lwm2m/README.md +++ b/apps/emqx_lwm2m/README.md @@ -1,356 +1,61 @@ -# LwM2M Gateway +# emqx_lwm2m -[The LwM2M Specifications](http://www.openmobilealliance.org/release/LightweightM2M) is a Lightweight Machine to Machine protocol. +[LwM2M (Lightweight Machine-to-Machine)](https://lwm2m.openmobilealliance.org/) +is a protocol designed for IoT devices and machine-to-machine communication. +It is a lightweight protocol that supports devices with limited processing power and memory. -With `emqx_lwm2m`, user is able to send LwM2M commands(READ/WRITE/EXECUTE/...) and get LwM2M response in MQTT way. `emqx_lwm2m` transforms data between MQTT and LwM2M protocol. + +The **LwM2M Gateway** in EMQX can accept LwM2M clients and translate theirevents +and messages into MQTT Publish messages. + +In the current implementation, it has the following limitations: +- Based UDP/DTLS transport. +- Only supports v1.0.2. The v1.1.x and v1.2.x is not supported yet. +- Not included LwM2M Bootstrap services. + +## Quick Start + +In EMQX 5.0, LwM2M gateways can be configured and enabled through the Dashboard. + +It can also be enabled via the HTTP API, and emqx.conf e.g, In emqx.conf: + +```properties +gateway.lwm2m { + xml_dir = "etc/lwm2m_xml/" + auto_observe = true + enable_stats = true + idle_timeout = "30s" + lifetime_max = "86400s" + lifetime_min = "1s" + mountpoint = "lwm2m/${endpoint_namea}/" + qmode_time_window = "22s" + update_msg_publish_condition = "contains_object_list" + translators { + command {qos = 0, topic = "dn/#"} + notify {qos = 0, topic = "up/notify"} + register {qos = 0, topic = "up/resp"} + response {qos = 0, topic = "up/resp"} + update {qos = 0, topic = "up/update"} + } + listeners { + udp { + default { + bind = "5783" + max_conn_rate = 1000 + max_connections = 1024000 + } + } + } +} +``` + +> Note: +> Configuring the gateway via emqx.conf requires changes on a per-node basis, +> but configuring it via Dashboard or the HTTP API will take effect across the cluster. +::: + +## Object definations emqx_lwm2m needs object definitions to parse data from lwm2m devices. Object definitions are declared by organizations in XML format, you could find those XMLs from [LwM2MRegistry](http://www.openmobilealliance.org/wp/OMNA/LwM2M/LwM2MRegistry.html), download and put them into the directory specified by `lwm2m.xml_dir`. If no associated object definition is found, response from device will be discarded and report an error message in log. -## Load emqx_lwm2m - -``` -./bin/emqx_ctl plugins load emqx_lwm2m -``` - -## Test emqx-lwm2m using *wakaama* - -[wakaama](https://github.com/eclipse/wakaama) is an easy-to-use lwm2m client command line tool. - -Start *lwm2mclient* using an endpoint name `ep1`: -``` -./lwm2mclient -n ep1 -h 127.0.0.1 -p 5683 -4 -``` - -To send an LwM2M DISCOVER command to *lwm2mclient*, publish an MQTT message to topic `lwm2m//dn` (where `` is the endpoint name of the client), with following payload: - -```json -{ - "reqID": "2", - "msgType": "discover", - "data": { - "path": "/3/0" - } -} -``` - -The MQTT message will be translated to an LwM2M DISCOVER command and sent to the *lwm2mclient*. Then the response of *lwm2mclient* will be in turn translated to an MQTT message, with topic `lwm2m//up/resp`, with following payload: - -```json -{ - "reqID": "2", - "msgType": "discover", - "data": { - "code":"2.05", - "codeMsg": "content", - "content": [ - ";dim=8", - "", - "", - "", - "" - ] - } -} -``` - -## LwM2M <--> MQTT Mapping - -### Register/Update (LwM2M Client Registration Interface) - -- **LwM2M Register and Update message will be converted to following MQTT message:** - - - **Method:** PUBLISH - - **Topic:** `lwm2m/{?EndpointName}/up/resp` (configurable) - - **Payload**: - - MsgType **register** and **update**: - ```json - { - "msgType": {?MsgType}, - "data": { - "ep": {?EndpointName}, - "lt": {?LifeTime}, - "sms": {?MSISDN}, - "lwm2m": {?Lwm2mVersion}, - "b": {?Binding}, - "alternatePath": {?AlternatePath}, - "objectList": {?ObjectList} - } - } - ``` - - {?EndpointName}: String, the endpoint name of the LwM2M client - - {?MsgType}: String, could be: - - "register": LwM2M Register - - "update": LwM2M Update - - "data" contains the query options and the object-list of the register message - - The *update* message is only published if the object-list changed. - -### Downlink Command and Uplink Response (LwM2M Device Management & Service Enablement Interface) - -- **To send a downlink command to device, publish following MQTT message:** - - **Method:** PUBLISH - - **Topic:** `lwm2m/{?EndpointName}/dn` - - **Request Payload**: - ```json - { - "reqID": {?ReqID}, - "msgType": {?MsgType}, - "data": {?Data} - } - ``` - - {?ReqID}: Integer, request-id, used for matching the response to the request - - {?MsgType}: String, can be one of the following: - - "read": LwM2M Read - - "discover": LwM2M Discover - - "write": LwM2M Write - - "write-attr": LwM2M Write Attributes - - "execute": LwM2M Execute - - "create": LwM2M Create - - "delete": LwM2M Delete - - {?Data}: JSON Object, its value depends on the {?MsgType}: - - **If {?MsgType} = "read" or "discover"**: - ```json - { - "path": {?ResourcePath} - } - ``` - - {?ResourcePath}: String, LwM2M full resource path. e.g. "3/0", "/3/0/0", "/3/0/6/0" - - **If {?MsgType} = "write" (single write)**: - ```json - { - "path": {?ResourcePath}, - "type": {?ValueType}, - "value": {?Value} - } - ``` - - {?ValueType}: String, can be: "Time", "String", "Integer", "Float", "Boolean", "Opaque", "Objlnk" - - {?Value}: Value of the resource, depends on "type". - - **If {?MsgType} = "write" (batch write)**: - ```json - { - "basePath": {?BasePath}, - "content": [ - { - "path": {?ResourcePath}, - "type": {?ValueType}, - "value": {?Value} - } - ] - } - ``` - - The full path is concatenation of "basePath" and "path". - - **If {?MsgType} = "write-attr"**: - ```json - { - "path": {?ResourcePath}, - "pmin": {?PeriodMin}, - "pmax": {?PeriodMax}, - "gt": {?GreaterThan}, - "lt": {?LessThan}, - "st": {?Step} - } - ``` - - {?PeriodMin}: Number, LwM2M Notification Class Attribute - Minimum Period. - - {?PeriodMax}: Number, LwM2M Notification Class Attribute - Maximum Period. - - {?GreaterThan}: Number, LwM2M Notification Class Attribute - Greater Than. - - {?LessThan}: Number, LwM2M Notification Class Attribute - Less Than. - - {?Step}: Number, LwM2M Notification Class Attribute - Step. - - - **If {?MsgType} = "execute"**: - ```json - { - "path": {?ResourcePath}, - "args": {?Arguments} - } - ``` - - {?Arguments}: String, LwM2M Execute Arguments. - - - **If {?MsgType} = "create"**: - ```json - { - "basePath": "/{?ObjectID}", - "content": [ - { - "path": {?ResourcePath}, - "type": {?ValueType}, - "value": {?Value} - } - ] - } - ``` - - {?ObjectID}: Integer, LwM2M Object ID - - - **If {?MsgType} = "delete"**: - ```json - { - "path": "{?ObjectID}/{?ObjectInstanceID}" - } - ``` - - {?ObjectInstanceID}: Integer, LwM2M Object Instance ID - -- **The response of LwM2M will be converted to following MQTT message:** - - **Method:** PUBLISH - - **Topic:** `"lwm2m/{?EndpointName}/up/resp"` - - **Response Payload:** - - ```json - { - "reqID": {?ReqID}, - "imei": {?IMEI}, - "imsi": {?IMSI}, - "msgType": {?MsgType}, - "data": {?Data} - } - ``` - - - {?MsgType}: String, can be: - - "read": LwM2M Read - - "discover": LwM2M Discover - - "write": LwM2M Write - - "write-attr": LwM2M Write Attributes - - "execute": LwM2M Execute - - "create": LwM2M Create - - "delete": LwM2M Delete - - **"ack"**: [CoAP Empty ACK](https://tools.ietf.org/html/rfc7252#section-5.2.2) - - {?Data}: JSON Object, its value depends on {?MsgType}: - - **If {?MsgType} = "write", "write-attr", "execute", "create", "delete", or "read"(when response without content)**: - ```json - { - "code": {?StatusCode}, - "codeMsg": {?CodeMsg}, - "reqPath": {?RequestPath} - } - ``` - - {?StatusCode}: String, LwM2M status code, e.g. "2.01", "4.00", etc. - - {?CodeMsg}: String, LwM2M response message, e.g. "content", "bad_request" - - {?RequestPath}: String, the requested "path" or "basePath" - - - **If {?MsgType} = "discover"**: - ```json - { - "code": {?StatusCode}, - "codeMsg": {?CodeMsg}, - "reqPath": {?RequestPath}, - "content": [ - {?Link}, - ... - ] - } - ``` - - {?Link}: String(LwM2M link format) e.g. `""`, `"<3/0/1>;dim=8"` - - - **If {?MsgType} = "read"(when response with content)**: - ```json - { - "code": {?StatusCode}, - "codeMsg": {?CodeMsg}, - "content": {?Content} - } - ``` - - {?Content} - ```json - [ - { - "path": {?ResourcePath}, - "value": {?Value} - } - ] - ``` - - - **If {?MsgType} = "ack", "data" does not exists** - -### Observe (Information Reporting Interface - Observe/Cancel-Observe) - -- **To observe/cancel-observe LwM2M client, send following MQTT PUBLISH:** - - **Method:** PUBLISH - - **Topic:** `lwm2m/{?EndpointName}/dn` - - **Request Payload**: - ```json - { - "reqID": {?ReqID}, - "msgType": {?MsgType}, - "data": { - "path": {?ResourcePath} - } - } - ``` - - {?ResourcePath}: String, the LwM2M resource to be observed/cancel-observed. - - {?MsgType}: String, can be: - - "observe": LwM2M Observe - - "cancel-observe": LwM2M Cancel Observe - - {?ReqID}: Integer, request-id, is the {?ReqID} in the request - -- **Responses will be converted to following MQTT message:** - - **Method:** PUBLISH - - **Topic:** `lwm2m/{?EndpointName}/up/resp` - - **Response Payload**: - ```json - { - "reqID": {?ReqID}, - "msgType": {?MsgType}, - "data": { - "code": {?StatusCode}, - "codeMsg": {?CodeMsg}, - "reqPath": {?RequestPath}, - "content": [ - { - "path": {?ResourcePath}, - "value": {?Value} - } - ] - } - } - ``` - - {?MsgType}: String, can be: - - "observe": LwM2M Observe - - "cancel-observe": LwM2M Cancel Observe - - **"ack"**: [CoAP Empty ACK](https://tools.ietf.org/html/rfc7252#section-5.2.2) - -### Notification (Information Reporting Interface - Notify) - -- **The notifications from LwM2M clients will be converted to MQTT PUBLISH:** - - **Method:** PUBLISH - - **Topic:** `lwm2m/{?EndpiontName}/up/notify` - - **Notification Payload**: - ```json - { - "reqID": {?ReqID}, - "msgType": {?MsgType}, - "seqNum": {?ObserveSeqNum}, - "data": { - "code": {?StatusCode}, - "codeMsg": {?CodeMsg}, - "reqPath": {?RequestPath}, - "content": [ - { - "path": {?ResourcePath}, - "value": {?Value} - } - ] - } - } - ``` - - {?MsgType}: String, must be "notify" - - {?ObserveSeqNum}: Number, value of "Observe" option in CoAP message - - "content": same to the "content" field contains in the response of "read" command - -## Feature limitations - -- emqx_lwm2m implements LwM2M gateway to EMQX, not a full-featured and independent LwM2M server. -- emqx_lwm2m does not include LwM2M bootstrap server. -- emqx_lwm2m supports UDP binding, no SMS binding yet. -- Firmware object is not fully supported now since mqtt to coap block-wise transfer is not available. -- Object Versioning is not supported now. - -## DTLS - -emqx-lwm2m support DTLS to secure UDP data. - -Please config lwm2m.certfile and lwm2m.keyfile in emqx_lwm2m.conf. If certfile or keyfile are invalid, DTLS will be turned off and you could read a error message in the log. - -## License - -Apache License Version 2.0 - -## Author - -EMQX-Men Team. +More documentations: [LwM2M Gateway](https://www.emqx.io/docs/en/v5.0/gateway/lwm2m.html) diff --git a/apps/emqx_mqttsn/LICENSE b/apps/emqx_mqttsn/LICENSE deleted file mode 100644 index 5a5418f0f..000000000 --- a/apps/emqx_mqttsn/LICENSE +++ /dev/null @@ -1,191 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - Copyright 2023, JianBo He . - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - diff --git a/apps/emqx_stomp/LICENSE b/apps/emqx_stomp/LICENSE deleted file mode 100644 index 5a5418f0f..000000000 --- a/apps/emqx_stomp/LICENSE +++ /dev/null @@ -1,191 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - Copyright 2023, JianBo He . - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - From aea870f319b2f07cbcddb0f21e5f351bb158e25c Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Wed, 5 Apr 2023 14:40:18 +0200 Subject: [PATCH 182/196] feat: add `/rule_engine` API endpoint --- .../src/emqx_rule_api_schema.erl | 3 + .../src/emqx_rule_engine_api.erl | 81 ++++++++++++++++--- .../test/emqx_rule_engine_api_SUITE.erl | 13 +++ changes/ce/feat-10336.en.md | 1 + rel/i18n/emqx_rule_api_schema.hocon | 11 +++ rel/i18n/emqx_rule_engine_api.hocon | 37 ++++++--- 6 files changed, 125 insertions(+), 21 deletions(-) create mode 100644 changes/ce/feat-10336.en.md diff --git a/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl index 23c2aab50..188dd10f9 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl @@ -48,12 +48,15 @@ check_params(Params, Tag) -> roots() -> [ + {"rule_engine", sc(ref("rule_engine"), #{desc => ?DESC("root_rule_engine")})}, {"rule_creation", sc(ref("rule_creation"), #{desc => ?DESC("root_rule_creation")})}, {"rule_info", sc(ref("rule_info"), #{desc => ?DESC("root_rule_info")})}, {"rule_events", sc(ref("rule_events"), #{desc => ?DESC("root_rule_events")})}, {"rule_test", sc(ref("rule_test"), #{desc => ?DESC("root_rule_test")})} ]. +fields("rule_engine") -> + emqx_rule_engine_schema:fields("rule_engine"); fields("rule_creation") -> emqx_rule_engine_schema:fields("rules"); fields("rule_info") -> diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index 106693a0a..251ba053d 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -32,6 +32,7 @@ %% API callbacks -export([ + '/rule_engine'/2, '/rule_events'/2, '/rule_test'/2, '/rules'/2, @@ -41,7 +42,7 @@ ]). %% query callback --export([qs2ms/2, run_fuzzy_match/2, format_rule_resp/1]). +-export([qs2ms/2, run_fuzzy_match/2, format_rule_info_resp/1]). -define(ERR_BADARGS(REASON), begin R0 = err_msg(REASON), @@ -134,6 +135,7 @@ api_spec() -> paths() -> [ + "/rule_engine", "/rule_events", "/rule_test", "/rules", @@ -145,6 +147,9 @@ paths() -> error_schema(Code, Message) when is_atom(Code) -> emqx_dashboard_swagger:error_codes([Code], list_to_binary(Message)). +rule_engine_schema() -> + ref(emqx_rule_api_schema, "rule_engine"). + rule_creation_schema() -> ref(emqx_rule_api_schema, "rule_creation"). @@ -184,7 +189,7 @@ schema("/rules") -> responses => #{ 200 => [ - {data, mk(array(rule_info_schema()), #{desc => ?DESC("desc9")})}, + {data, mk(array(rule_info_schema()), #{desc => ?DESC("api1_resp")})}, {meta, mk(ref(emqx_dashboard_swagger, meta), #{})} ], 400 => error_schema('BAD_REQUEST', "Invalid Parameters") @@ -289,6 +294,26 @@ schema("/rule_test") -> 200 => <<"Rule Test Pass">> } } + }; +schema("/rule_engine") -> + #{ + 'operationId' => '/rule_engine', + get => #{ + tags => [<<"rules">>], + description => ?DESC("api9"), + responses => #{ + 200 => rule_engine_schema() + } + }, + put => #{ + tags => [<<"rules">>], + description => ?DESC("api10"), + 'requestBody' => rule_engine_schema(), + responses => #{ + 200 => rule_engine_schema(), + 400 => error_schema('BAD_REQUEST', "Invalid request") + } + } }. param_path_id() -> @@ -309,7 +334,7 @@ param_path_id() -> QueryString, ?RULE_QS_SCHEMA, fun ?MODULE:qs2ms/2, - fun ?MODULE:format_rule_resp/1 + fun ?MODULE:format_rule_info_resp/1 ) of {error, page_limit_invalid} -> @@ -331,7 +356,7 @@ param_path_id() -> case emqx_conf:update(ConfPath, Params, #{override_to => cluster}) of {ok, #{post_config_update := #{emqx_rule_engine := AllRules}}} -> [Rule] = get_one_rule(AllRules, Id), - {201, format_rule_resp(Rule)}; + {201, format_rule_info_resp(Rule)}; {error, Reason} -> ?SLOG(error, #{ msg => "create_rule_failed", @@ -362,7 +387,7 @@ param_path_id() -> '/rules/:id'(get, #{bindings := #{id := Id}}) -> case emqx_rule_engine:get_rule(Id) of {ok, Rule} -> - {200, format_rule_resp(Rule)}; + {200, format_rule_info_resp(Rule)}; not_found -> {404, #{code => 'NOT_FOUND', message => <<"Rule Id Not Found">>}} end; @@ -372,7 +397,7 @@ param_path_id() -> case emqx_conf:update(ConfPath, Params, #{override_to => cluster}) of {ok, #{post_config_update := #{emqx_rule_engine := AllRules}}} -> [Rule] = get_one_rule(AllRules, Id), - {200, format_rule_resp(Rule)}; + {200, format_rule_info_resp(Rule)}; {error, Reason} -> ?SLOG(error, #{ msg => "update_rule_failed", @@ -419,6 +444,20 @@ param_path_id() -> {404, #{code => 'NOT_FOUND', message => <<"Rule Id Not Found">>}} end. +'/rule_engine'(get, _Params) -> + {200, format_rule_engine_resp(emqx_conf:get([rule_engine]))}; +'/rule_engine'(put, #{body := Params}) -> + case emqx_conf:update([rule_engine], Params, #{override_to => cluster}) of + {ok, #{config := Config}} -> + {200, format_rule_engine_resp(Config)}; + {error, Reason} -> + ?SLOG(error, #{ + msg => "update_rule_engine_failed", + reason => Reason + }), + {400, #{code => 'BAD_REQUEST', message => ?ERR_BADARGS(Reason)}} + end. + %%------------------------------------------------------------------------------ %% Internal functions %%------------------------------------------------------------------------------ @@ -440,11 +479,11 @@ encode_nested_error(RuleError, Reason) -> {RuleError, Reason} end. -format_rule_resp(Rules) when is_list(Rules) -> - [format_rule_resp(R) || R <- Rules]; -format_rule_resp({Id, Rule}) -> - format_rule_resp(Rule#{id => Id}); -format_rule_resp(#{ +format_rule_info_resp(Rules) when is_list(Rules) -> + [format_rule_info_resp(R) || R <- Rules]; +format_rule_info_resp({Id, Rule}) -> + format_rule_info_resp(Rule#{id => Id}); +format_rule_info_resp(#{ id := Id, name := Name, created_at := CreatedAt, @@ -465,6 +504,26 @@ format_rule_resp(#{ description => Descr }. +format_rule_engine_resp(#{rules := Rules} = Config) -> + Config#{rules => maps:map(fun format_rule_resp/2, Rules)}. + +format_rule_resp(_Id, #{ + name := Name, + metadata := MetaData = #{created_at := CreatedAt}, + actions := Action, + sql := SQL, + enable := Enable, + description := Descr +}) -> + #{ + name => Name, + actions => format_action(Action), + sql => SQL, + enable => Enable, + metadata => MetaData#{created_at => format_datetime(CreatedAt, millisecond)}, + description => Descr + }. + format_datetime(Timestamp, Unit) -> list_to_binary(calendar:system_time_to_rfc3339(Timestamp, [{unit, Unit}])). diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl index d89bc2651..47dce98b1 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl @@ -281,3 +281,16 @@ test_rule_params(Sql, Payload) -> <<"sql">> => Sql } }. + +t_rule_engine(_) -> + {200, _} = emqx_rule_engine_api:'/rule_engine'(get, foo), + {200, #{ + jq_function_default_timeout := 12000, + jq_implementation_module := jq_port + }} = emqx_rule_engine_api:'/rule_engine'(put, #{ + body => #{ + <<"jq_function_default_timeout">> => <<"12s">>, + <<"jq_implementation_module">> => <<"jq_port">> + } + }), + {400, _} = emqx_rule_engine_api:'/rule_engine'(put, #{body => #{<<"something">> => <<"weird">>}}). diff --git a/changes/ce/feat-10336.en.md b/changes/ce/feat-10336.en.md new file mode 100644 index 000000000..5e6039f9b --- /dev/null +++ b/changes/ce/feat-10336.en.md @@ -0,0 +1 @@ +Add `/rule_engine` API endpoint to manage configuration of rule engine. diff --git a/rel/i18n/emqx_rule_api_schema.hocon b/rel/i18n/emqx_rule_api_schema.hocon index f9b344666..e2c8532e7 100644 --- a/rel/i18n/emqx_rule_api_schema.hocon +++ b/rel/i18n/emqx_rule_api_schema.hocon @@ -638,6 +638,17 @@ emqx_rule_api_schema { } } + root_rule_engine { + desc { + en: "Rule engine configuration schema" + zh: "规则引擎配置模式" + } + label: { + en: "Configuration Schema" + zh: "配置模式" + } + } + root_rule_creation { desc { en: "Schema for creating rules" diff --git a/rel/i18n/emqx_rule_engine_api.hocon b/rel/i18n/emqx_rule_engine_api.hocon index 39fc3186c..181c1ba40 100644 --- a/rel/i18n/emqx_rule_engine_api.hocon +++ b/rel/i18n/emqx_rule_engine_api.hocon @@ -50,7 +50,16 @@ emqx_rule_engine_api { zh: "根据规则来源 Topic 过滤, 使用 MQTT Topic 匹配" } } - + api1_resp { + desc { + en: "List of rules" + zh: "列出所有规则" + } + label: { + en: "List Rules" + zh: "列出所有规则" + } + } api2 { desc { en: "Create a new rule using given Id" @@ -116,7 +125,6 @@ emqx_rule_engine_api { zh: "删除集群规则" } } - api7 { desc { en: "Reset a rule metrics" @@ -127,7 +135,6 @@ emqx_rule_engine_api { zh: "重置规则计数" } } - api8 { desc { en: "Test a rule" @@ -138,14 +145,24 @@ emqx_rule_engine_api { zh: "测试规则" } } - desc9 { + api9 { desc { - en: "List of rules" - zh: "列出所有规则" + en: "Get rule engine configuration" + zh: "获取规则引擎配置" } - label: { - en: "List Rules" - zh: "列出所有规则" + label { + en: "Get configuration" + zh: "获取配置" + } + } + api10 { + desc { + en: "Update rule engine configuration" + zh: "更新规则引擎配置" + } + label { + en: "Update configuration" + zh: "更新配置" } - } + } } From 5cd8865a9335ead6beaf9950092be15c7482d286 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Wed, 5 Apr 2023 15:34:42 +0200 Subject: [PATCH 183/196] fix: deprecate and hide jq_implementation_module --- apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl index 2281eea53..33dedae9a 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl @@ -61,7 +61,9 @@ fields("rule_engine") -> #{ default => jq_nif, mapping => "jq.jq_implementation_module", - desc => ?DESC("rule_engine_jq_implementation_module") + desc => ?DESC("rule_engine_jq_implementation_module"), + deprecated => {since, "v5.0.22"}, + importance => ?IMPORTANCE_HIDDEN } )} ]; From 9810c9f7e3f2d92c84eeafc535f91fbe39434922 Mon Sep 17 00:00:00 2001 From: Serge Tupchii Date: Tue, 4 Apr 2023 17:36:51 +0300 Subject: [PATCH 184/196] refactor(rule_engine): test 'ok' results in one dedicated function --- apps/emqx_rule_engine/src/emqx_rule_runtime.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl index 153832246..e8d807d38 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl @@ -508,8 +508,6 @@ nested_put(Alias, Val, Columns0) -> emqx_rule_maps:nested_put(Alias, Val, Columns). -define(IS_RES_DOWN(R), R == stopped; R == not_connected; R == not_found). -inc_action_metrics(ok, RuleId) -> - emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.success'); inc_action_metrics({error, {recoverable_error, _}}, RuleId) -> emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.failed.out_of_service'); inc_action_metrics(?RESOURCE_ERROR_M(R, _), RuleId) when ?IS_RES_DOWN(R) -> @@ -525,6 +523,8 @@ inc_action_metrics(R, RuleId) -> emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.success') end. +is_ok_result(ok) -> + true; is_ok_result(R) when is_tuple(R) -> ok == erlang:element(1, R); is_ok_result(_) -> From 9cfe9cc709731b97df9ccb38dcee5b127110f491 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Wed, 5 Apr 2023 16:28:54 +0200 Subject: [PATCH 185/196] fix: don't allow `rules` to be set from `/rule_engine` --- .../src/emqx_rule_api_schema.erl | 2 +- .../src/emqx_rule_engine_api.erl | 51 +++++++++++-------- .../src/emqx_rule_engine_schema.erl | 12 +++-- .../test/emqx_rule_engine_api_SUITE.erl | 9 +++- 4 files changed, 47 insertions(+), 27 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl index 188dd10f9..72fd085a3 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl @@ -56,7 +56,7 @@ roots() -> ]. fields("rule_engine") -> - emqx_rule_engine_schema:fields("rule_engine"); + emqx_rule_engine_schema:fields("rule_engine_api"); fields("rule_creation") -> emqx_rule_engine_schema:fields("rules"); fields("rule_info") -> diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index 251ba053d..b138d992e 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -447,16 +447,20 @@ param_path_id() -> '/rule_engine'(get, _Params) -> {200, format_rule_engine_resp(emqx_conf:get([rule_engine]))}; '/rule_engine'(put, #{body := Params}) -> - case emqx_conf:update([rule_engine], Params, #{override_to => cluster}) of - {ok, #{config := Config}} -> - {200, format_rule_engine_resp(Config)}; - {error, Reason} -> - ?SLOG(error, #{ - msg => "update_rule_engine_failed", - reason => Reason - }), - {400, #{code => 'BAD_REQUEST', message => ?ERR_BADARGS(Reason)}} - end. + ?CHECK_PARAMS( + Params, + rule_engine, + case emqx_conf:update([rule_engine], Params, #{override_to => cluster}) of + {ok, #{config := Config}} -> + {200, format_rule_engine_resp(Config)}; + {error, Reason} -> + ?SLOG(error, #{ + msg => "update_rule_engine_failed", + reason => Reason + }), + {400, #{code => 'BAD_REQUEST', message => ?ERR_BADARGS(Reason)}} + end + ). %%------------------------------------------------------------------------------ %% Internal functions @@ -507,22 +511,29 @@ format_rule_info_resp(#{ format_rule_engine_resp(#{rules := Rules} = Config) -> Config#{rules => maps:map(fun format_rule_resp/2, Rules)}. -format_rule_resp(_Id, #{ - name := Name, - metadata := MetaData = #{created_at := CreatedAt}, - actions := Action, - sql := SQL, - enable := Enable, - description := Descr -}) -> +format_rule_resp( + _Id, #{ + name := Name, + actions := Action, + sql := SQL, + enable := Enable, + description := Descr + } = Rule +) -> + Format = #{ name => Name, actions => format_action(Action), sql => SQL, enable => Enable, - metadata => MetaData#{created_at => format_datetime(CreatedAt, millisecond)}, description => Descr - }. + }, + case Rule of + #{metadata := MetaData = #{created_at := CreatedAt}} -> + Format#{metadata => MetaData#{created_at => format_datetime(CreatedAt, millisecond)}}; + _ -> + Format + end. format_datetime(Timestamp, Unit) -> list_to_binary(calendar:system_time_to_rfc3339(Timestamp, [{unit, Unit}])). diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl index 33dedae9a..57b64cb49 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl @@ -40,13 +40,17 @@ tags() -> roots() -> ["rule_engine"]. fields("rule_engine") -> + fields("rule_engine_api") ++ + [ + {rules, + ?HOCON(hoconsc:map("id", ?R_REF("rules")), #{ + desc => ?DESC("rule_engine_rules"), default => #{} + })} + ]; +fields("rule_engine_api") -> [ {ignore_sys_message, ?HOCON(boolean(), #{default => true, desc => ?DESC("rule_engine_ignore_sys_message")})}, - {rules, - ?HOCON(hoconsc:map("id", ?R_REF("rules")), #{ - desc => ?DESC("rule_engine_rules"), default => #{} - })}, {jq_function_default_timeout, ?HOCON( emqx_schema:duration_ms(), diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl index 47dce98b1..e59b5c6df 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl @@ -285,12 +285,17 @@ test_rule_params(Sql, Payload) -> t_rule_engine(_) -> {200, _} = emqx_rule_engine_api:'/rule_engine'(get, foo), {200, #{ - jq_function_default_timeout := 12000, - jq_implementation_module := jq_port + %, + jq_function_default_timeout := 12000 + % hidden! jq_implementation_module := jq_port }} = emqx_rule_engine_api:'/rule_engine'(put, #{ body => #{ <<"jq_function_default_timeout">> => <<"12s">>, <<"jq_implementation_module">> => <<"jq_port">> } }), + SomeRule = #{<<"sql">> => <<"SELECT * FROM \"t/#\"">>}, + {400, _} = emqx_rule_engine_api:'/rule_engine'(put, #{ + body => #{<<"rules">> => #{<<"some_rule">> => SomeRule}} + }), {400, _} = emqx_rule_engine_api:'/rule_engine'(put, #{body => #{<<"something">> => <<"weird">>}}). From 576d1524bb411c1987b80ad58a1c2bfb51f96c93 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Wed, 5 Apr 2023 16:29:17 +0200 Subject: [PATCH 186/196] style: fix wording of i18n files --- rel/i18n/emqx_rule_api_schema.hocon | 8 ++++---- rel/i18n/emqx_rule_engine_api.hocon | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/rel/i18n/emqx_rule_api_schema.hocon b/rel/i18n/emqx_rule_api_schema.hocon index e2c8532e7..3c2b3e7e4 100644 --- a/rel/i18n/emqx_rule_api_schema.hocon +++ b/rel/i18n/emqx_rule_api_schema.hocon @@ -640,12 +640,12 @@ emqx_rule_api_schema { root_rule_engine { desc { - en: "Rule engine configuration schema" - zh: "规则引擎配置模式" + en: "Rule engine configurations. This API can be used to change EMQX rule engine settings. But not for the rules. To list, create, or update rules, call the '/rules' API instead." + zh: "规则引擎配置。该 API 可用于查看和修改规则引擎相关的一些设置。但不可用于规则，请调用 '/rules' API 来对规则进行操作。" } label: { - en: "Configuration Schema" - zh: "配置模式" + en: "Rule engine configuration" + zh: "规则引擎配置" } } diff --git a/rel/i18n/emqx_rule_engine_api.hocon b/rel/i18n/emqx_rule_engine_api.hocon index 181c1ba40..7be0588c9 100644 --- a/rel/i18n/emqx_rule_engine_api.hocon +++ b/rel/i18n/emqx_rule_engine_api.hocon @@ -147,8 +147,8 @@ emqx_rule_engine_api { } api9 { desc { - en: "Get rule engine configuration" - zh: "获取规则引擎配置" + en: "Get rule engine configuration." + zh: "获取规则引擎配置。" } label { en: "Get configuration" @@ -157,8 +157,8 @@ emqx_rule_engine_api { } api10 { desc { - en: "Update rule engine configuration" - zh: "更新规则引擎配置" + en: "Update rule engine configuration." + zh: "更新规则引擎配置。" } label { en: "Update configuration" From b799af1f716f50e906d11f938dd2177144821d77 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Wed, 5 Apr 2023 16:46:19 +0200 Subject: [PATCH 187/196] fix: don't create virtual type "rule_engine_api" --- .../src/emqx_rule_api_schema.erl | 2 +- .../src/emqx_rule_engine_schema.erl | 54 ++++++++++--------- 2 files changed, 29 insertions(+), 27 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl index 72fd085a3..8a8822044 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl @@ -56,7 +56,7 @@ roots() -> ]. fields("rule_engine") -> - emqx_rule_engine_schema:fields("rule_engine_api"); + emqx_rule_engine_schema:rule_engine_settings(); fields("rule_creation") -> emqx_rule_engine_schema:fields("rules"); fields("rule_info") -> diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl index 57b64cb49..5b205f355 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl @@ -27,7 +27,8 @@ roots/0, fields/1, desc/1, - post_config_update/5 + post_config_update/5, + rule_engine_settings/0 ]). -export([validate_sql/1]). @@ -40,37 +41,13 @@ tags() -> roots() -> ["rule_engine"]. fields("rule_engine") -> - fields("rule_engine_api") ++ + rule_engine_settings() ++ [ {rules, ?HOCON(hoconsc:map("id", ?R_REF("rules")), #{ desc => ?DESC("rule_engine_rules"), default => #{} })} ]; -fields("rule_engine_api") -> - [ - {ignore_sys_message, - ?HOCON(boolean(), #{default => true, desc => ?DESC("rule_engine_ignore_sys_message")})}, - {jq_function_default_timeout, - ?HOCON( - emqx_schema:duration_ms(), - #{ - default => <<"10s">>, - desc => ?DESC("rule_engine_jq_function_default_timeout") - } - )}, - {jq_implementation_module, - ?HOCON( - hoconsc:enum([jq_nif, jq_port]), - #{ - default => jq_nif, - mapping => "jq.jq_implementation_module", - desc => ?DESC("rule_engine_jq_implementation_module"), - deprecated => {since, "v5.0.22"}, - importance => ?IMPORTANCE_HIDDEN - } - )} - ]; fields("rules") -> [ rule_name(), @@ -233,6 +210,31 @@ actions() -> qos() -> ?UNION([emqx_schema:qos(), binary()]). +rule_engine_settings() -> + [ + {ignore_sys_message, + ?HOCON(boolean(), #{default => true, desc => ?DESC("rule_engine_ignore_sys_message")})}, + {jq_function_default_timeout, + ?HOCON( + emqx_schema:duration_ms(), + #{ + default => <<"10s">>, + desc => ?DESC("rule_engine_jq_function_default_timeout") + } + )}, + {jq_implementation_module, + ?HOCON( + hoconsc:enum([jq_nif, jq_port]), + #{ + default => jq_nif, + mapping => "jq.jq_implementation_module", + desc => ?DESC("rule_engine_jq_implementation_module"), + deprecated => {since, "v5.0.22"}, + importance => ?IMPORTANCE_HIDDEN + } + )} + ]. + validate_sql(Sql) -> case emqx_rule_sqlparser:parse(Sql) of {ok, _Result} -> ok; From 33100ecca68a9f16c669aae6d3367faa619b28bb Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 4 Apr 2023 08:55:43 -0300 Subject: [PATCH 188/196] feat: implement schema registry for 5.0 (avro) Part of https://emqx.atlassian.net/browse/EMQX-9251 This ports part of the Schema Registry app from 4.x to 5.0. Here, only support for Avro is added. Subsequent PRs will follow to add support for other formats. --- apps/emqx/src/emqx_schema.erl | 16 +- apps/emqx/test/emqx_common_test_helpers.erl | 43 +- .../src/emqx_dashboard_swagger.erl | 2 + apps/emqx_rule_engine/src/emqx_rule_funcs.erl | 33 +- changes/ee/feat-10337.en.md | 3 + lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src | 2 +- .../emqx_ee_conf/src/emqx_ee_conf_schema.erl | 2 +- lib-ee/emqx_ee_schema_registry/.gitignore | 19 + lib-ee/emqx_ee_schema_registry/README.md | 69 +++ .../etc/emqx_ee_schema_registry.conf | 0 .../include/emqx_ee_schema_registry.hrl | 39 ++ lib-ee/emqx_ee_schema_registry/rebar.config | 12 + .../src/emqx_ee_schema_registry.app.src | 15 + .../src/emqx_ee_schema_registry.erl | 242 ++++++++++ .../src/emqx_ee_schema_registry_app.erl | 19 + .../src/emqx_ee_schema_registry_http_api.erl | 251 ++++++++++ .../src/emqx_ee_schema_registry_schema.erl | 127 +++++ .../src/emqx_ee_schema_registry_serde.erl | 70 +++ .../src/emqx_ee_schema_registry_sup.erl | 43 ++ .../test/emqx_ee_schema_registry_SUITE.erl | 433 ++++++++++++++++++ ...emqx_ee_schema_registry_http_api_SUITE.erl | 250 ++++++++++ .../emqx_ee_schema_registry_serde_SUITE.erl | 121 +++++ mix.exs | 5 +- rebar.config | 2 + rebar.config.erl | 3 +- rel/i18n/emqx_bridge_api.hocon | 2 +- .../emqx_ee_schema_registry_http_api.hocon | 69 +++ rel/i18n/emqx_ee_schema_registry_schema.hocon | 78 ++++ scripts/spellcheck/dicts/emqx.txt | 1 + 29 files changed, 1941 insertions(+), 30 deletions(-) create mode 100644 changes/ee/feat-10337.en.md create mode 100644 lib-ee/emqx_ee_schema_registry/.gitignore create mode 100644 lib-ee/emqx_ee_schema_registry/README.md create mode 100644 lib-ee/emqx_ee_schema_registry/etc/emqx_ee_schema_registry.conf create mode 100644 lib-ee/emqx_ee_schema_registry/include/emqx_ee_schema_registry.hrl create mode 100644 lib-ee/emqx_ee_schema_registry/rebar.config create mode 100644 lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.app.src create mode 100644 lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.erl create mode 100644 lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_app.erl create mode 100644 lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_http_api.erl create mode 100644 lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_schema.erl create mode 100644 lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_serde.erl create mode 100644 lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_sup.erl create mode 100644 lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl create mode 100644 lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_http_api_SUITE.erl create mode 100644 lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_serde_SUITE.erl create mode 100644 rel/i18n/emqx_ee_schema_registry_http_api.hocon create mode 100644 rel/i18n/emqx_ee_schema_registry_schema.hocon diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 6bfff38d3..20018b2d5 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -44,6 +44,7 @@ -type port_number() :: 1..65536. -type server_parse_option() :: #{default_port => port_number(), no_port => boolean()}. -type url() :: binary(). +-type json_binary() :: binary(). -typerefl_from_string({duration/0, emqx_schema, to_duration}). -typerefl_from_string({duration_s/0, emqx_schema, to_duration_s}). @@ -58,6 +59,7 @@ -typerefl_from_string({cipher/0, emqx_schema, to_erl_cipher_suite}). -typerefl_from_string({comma_separated_atoms/0, emqx_schema, to_comma_separated_atoms}). -typerefl_from_string({url/0, emqx_schema, to_url}). +-typerefl_from_string({json_binary/0, emqx_schema, to_json_binary}). -export([ validate_heap_size/1, @@ -84,7 +86,8 @@ to_ip_port/1, to_erl_cipher_suite/1, to_comma_separated_atoms/1, - to_url/1 + to_url/1, + to_json_binary/1 ]). -export([ @@ -112,7 +115,8 @@ ip_port/0, cipher/0, comma_separated_atoms/0, - url/0 + url/0, + json_binary/0 ]). -export([namespace/0, roots/0, roots/1, fields/1, desc/1, tags/0]). @@ -2576,6 +2580,14 @@ to_url(Str) -> Error end. +to_json_binary(Str) -> + case emqx_json:safe_decode(Str) of + {ok, _} -> + {ok, iolist_to_binary(Str)}; + Error -> + Error + end. + to_bar_separated_list(Str) -> {ok, string:tokens(Str, "| ")}. diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index 077ebe138..406183094 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -676,7 +676,8 @@ start_slave(Name, Opts) when is_map(Opts) -> ] ); slave -> - slave:start_link(host(), Name, ebin_path()) + Env = " -env HOCON_ENV_OVERRIDE_PREFIX EMQX_", + slave:start_link(host(), Name, ebin_path() ++ Env) end end, case DoStart() of @@ -749,6 +750,20 @@ setup_node(Node, Opts) when is_map(Opts) -> %% `emqx_conf' app and correctly catch up the config. StartAutocluster = maps:get(start_autocluster, Opts, false), + ct:pal( + "setting up node ~p:\n ~p", + [ + Node, + #{ + start_autocluster => StartAutocluster, + load_apps => LoadApps, + apps => Apps, + env => Env, + start_apps => StartApps + } + ] + ), + %% Load env before doing anything to avoid overriding [ok = erpc:call(Node, ?MODULE, load, [App]) || App <- [gen_rpc, ekka, mria, emqx | LoadApps]], @@ -773,10 +788,7 @@ setup_node(Node, Opts) when is_map(Opts) -> end, %% Setting env before starting any applications - [ - ok = rpc:call(Node, application, set_env, [Application, Key, Value]) - || {Application, Key, Value} <- Env - ], + set_envs(Node, Env), %% Here we start the apps EnvHandlerForRpc = @@ -794,8 +806,9 @@ setup_node(Node, Opts) when is_map(Opts) -> node(), integer_to_list(erlang:unique_integer()) ]), + Cookie = atom_to_list(erlang:get_cookie()), os:putenv("EMQX_NODE__DATA_DIR", NodeDataDir), - os:putenv("EMQX_NODE__COOKIE", atom_to_list(erlang:get_cookie())), + os:putenv("EMQX_NODE__COOKIE", Cookie), emqx_config:init_load(SchemaMod), os:unsetenv("EMQX_NODE__DATA_DIR"), os:unsetenv("EMQX_NODE__COOKIE"), @@ -826,7 +839,15 @@ setup_node(Node, Opts) when is_map(Opts) -> ok; _ -> StartAutocluster andalso - (ok = rpc:call(Node, emqx_machine_boot, start_autocluster, [])), + begin + %% Note: we need to re-set the env because + %% starting the apps apparently make some of them + %% to be lost... This is particularly useful for + %% setting extra apps to be restarted after + %% joining. + set_envs(Node, Env), + ok = erpc:call(Node, emqx_machine_boot, start_autocluster, []) + end, case rpc:call(Node, ekka, join, [JoinTo]) of ok -> ok; @@ -883,6 +904,14 @@ merge_opts(Opts1, Opts2) -> Opts2 ). +set_envs(Node, Env) -> + lists:foreach( + fun({Application, Key, Value}) -> + ok = rpc:call(Node, application, set_env, [Application, Key, Value]) + end, + Env + ). + erl_flags() -> %% One core and redirecting logs to master "+S 1:1 -master " ++ atom_to_list(node()) ++ " " ++ ebin_path(). diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index e2872c0d7..eb7f6c741 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -755,6 +755,8 @@ typename_to_spec("initial()", _Mod) -> #{type => string, example => <<"0MB">>}; typename_to_spec("bucket_name()", _Mod) -> #{type => string, example => <<"retainer">>}; +typename_to_spec("json_binary()", _Mod) -> + #{type => string, example => <<"{\"a\": [1,true]}">>}; typename_to_spec(Name, Mod) -> Spec = range(Name), Spec1 = remote_module_type(Spec, Name, Mod), diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index b8bfeb84c..79e0406c1 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -1097,26 +1097,27 @@ date_to_unix_ts(TimeUnit, Offset, FormatString, InputString) -> %% Here the emqx_rule_funcs module acts as a proxy, forwarding %% the function handling to the worker module. %% @end -% '$handle_undefined_function'(schema_decode, [SchemaId, Data|MoreArgs]) -> -% emqx_schema_parser:decode(SchemaId, Data, MoreArgs); -% '$handle_undefined_function'(schema_decode, Args) -> -% error({args_count_error, {schema_decode, Args}}); - -% '$handle_undefined_function'(schema_encode, [SchemaId, Term|MoreArgs]) -> -% emqx_schema_parser:encode(SchemaId, Term, MoreArgs); -% '$handle_undefined_function'(schema_encode, Args) -> -% error({args_count_error, {schema_encode, Args}}); - -% '$handle_undefined_function'(sprintf, [Format|Args]) -> -% erlang:apply(fun sprintf_s/2, [Format, Args]); - -% '$handle_undefined_function'(Fun, Args) -> -% error({sql_function_not_supported, function_literal(Fun, Args)}). - +-if(?EMQX_RELEASE_EDITION == ee). +%% EE +'$handle_undefined_function'(schema_decode, [SchemaId, Data | MoreArgs]) -> + emqx_ee_schema_registry_serde:decode(SchemaId, Data, MoreArgs); +'$handle_undefined_function'(schema_decode, Args) -> + error({args_count_error, {schema_decode, Args}}); +'$handle_undefined_function'(schema_encode, [SchemaId, Term | MoreArgs]) -> + emqx_ee_schema_registry_serde:encode(SchemaId, Term, MoreArgs); +'$handle_undefined_function'(schema_encode, Args) -> + error({args_count_error, {schema_encode, Args}}); '$handle_undefined_function'(sprintf, [Format | Args]) -> erlang:apply(fun sprintf_s/2, [Format, Args]); '$handle_undefined_function'(Fun, Args) -> error({sql_function_not_supported, function_literal(Fun, Args)}). +-else. +%% CE +'$handle_undefined_function'(sprintf, [Format | Args]) -> + erlang:apply(fun sprintf_s/2, [Format, Args]); +'$handle_undefined_function'(Fun, Args) -> + error({sql_function_not_supported, function_literal(Fun, Args)}). +-endif. map_path(Key) -> {path, [{key, P} || P <- string:split(Key, ".", all)]}. diff --git a/changes/ee/feat-10337.en.md b/changes/ee/feat-10337.en.md new file mode 100644 index 000000000..299933351 --- /dev/null +++ b/changes/ee/feat-10337.en.md @@ -0,0 +1,3 @@ +Add schema registry feature. + +With schema registry, one can encode and decode special serialization formats in payloads when transforming messages in Rule Engine. Currently, only [Apache Avro](https://avro.apache.org/) is supported. diff --git a/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src b/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src index 324e7e308..771fdcb27 100644 --- a/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src +++ b/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src @@ -1,6 +1,6 @@ {application, emqx_ee_conf, [ {description, "EMQX Enterprise Edition configuration schema"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {registered, []}, {applications, [ kernel, diff --git a/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl b/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl index 5137574e3..7bf41deb5 100644 --- a/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl +++ b/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl @@ -8,7 +8,7 @@ -export([namespace/0, roots/0, fields/1, translations/0, translation/1]). --define(EE_SCHEMA_MODULES, [emqx_license_schema]). +-define(EE_SCHEMA_MODULES, [emqx_license_schema, emqx_ee_schema_registry_schema]). namespace() -> emqx_conf_schema:namespace(). diff --git a/lib-ee/emqx_ee_schema_registry/.gitignore b/lib-ee/emqx_ee_schema_registry/.gitignore new file mode 100644 index 000000000..f1c455451 --- /dev/null +++ b/lib-ee/emqx_ee_schema_registry/.gitignore @@ -0,0 +1,19 @@ +.rebar3 +_* +.eunit +*.o +*.beam +*.plt +*.swp +*.swo +.erlang.cookie +ebin +log +erl_crash.dump +.rebar +logs +_build +.idea +*.iml +rebar3.crashdump +*~ diff --git a/lib-ee/emqx_ee_schema_registry/README.md b/lib-ee/emqx_ee_schema_registry/README.md new file mode 100644 index 000000000..9f477208c --- /dev/null +++ b/lib-ee/emqx_ee_schema_registry/README.md @@ -0,0 +1,69 @@ +# EMQX Schema Registry + +EMQX Schema Registry for managing various of schemas for +decoding/encoding messages. + +To use schema in rule engine, a schema name should be passed to the +SQL functions that decode/encode data, like: + +```sql +SELECT + schema_decode('sensor_notify', payload) as payload +FROM + "message.publish" +WHERE + topic = 't/1' +``` + +## Using schema registry with rule engine + +``` + +---------------------------+ + | | + Events/Msgs | | Events/Msgs + --------------------> EMQX |------------------> + | | + | | + +-------------|-------------+ + | + HOOK | + | + +-------------v-------------+ +----------+ + | | Data | | + | Rule Engine ------------- Backends | + | | | | + +------|-------------|------+ +----------+ + |^ |^ + Decode|| ||Encode + || || + +------v|------------v|-----+ + | | + | Schema Registry | + | | + +---------------------------+ +``` + +## Architecture + +``` + | | + Decode | [APIs] | Encode + | | + | | [Registry] + +------v--------------v------+ + REGISTER SCHEMA | | + -------------------> | +--------+ + | | | | +[Management APIs] | Schema Registry ------ Schema | + | | | | + -------------------> | +--------+ + LOAD PARSERS | | + +----------------------------+ + / | \ + +---/---+ +---|----+ +---\---+ + | | | | | | + [Decoders] | Avro | |ProtoBuf| |Others | + | | | | | | + +-------+ +--------+ +-------+ + +``` diff --git a/lib-ee/emqx_ee_schema_registry/etc/emqx_ee_schema_registry.conf b/lib-ee/emqx_ee_schema_registry/etc/emqx_ee_schema_registry.conf new file mode 100644 index 000000000..e69de29bb diff --git a/lib-ee/emqx_ee_schema_registry/include/emqx_ee_schema_registry.hrl b/lib-ee/emqx_ee_schema_registry/include/emqx_ee_schema_registry.hrl new file mode 100644 index 000000000..af49db6dd --- /dev/null +++ b/lib-ee/emqx_ee_schema_registry/include/emqx_ee_schema_registry.hrl @@ -0,0 +1,39 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-ifndef(EMQX_EE_SCHEMA_REGISTRY_HRL). +-define(EMQX_EE_SCHEMA_REGISTRY_HRL, true). + +-define(CONF_KEY_ROOT, schema_registry). +-define(CONF_KEY_PATH, [?CONF_KEY_ROOT]). + +-define(SCHEMA_REGISTRY_SHARD, emqx_ee_schema_registry_shard). +-define(SERDE_TAB, emqx_ee_schema_registry_serde_tab). + +-type schema_name() :: binary(). +-type schema_source() :: binary(). + +-type encoded_data() :: iodata(). +-type decoded_data() :: map(). +-type serializer() :: fun((decoded_data()) -> encoded_data()). +-type deserializer() :: fun((encoded_data()) -> decoded_data()). +-type destructor() :: fun(() -> ok). +-type serde_type() :: avro. +-type serde_opts() :: map(). + +-record(serde, { + name :: schema_name(), + serializer :: serializer(), + deserializer :: deserializer(), + destructor :: destructor() +}). +-type serde() :: #serde{}. +-type serde_map() :: #{ + name := schema_name(), + serializer := serializer(), + deserializer := deserializer(), + destructor := destructor() +}. + +-endif. diff --git a/lib-ee/emqx_ee_schema_registry/rebar.config b/lib-ee/emqx_ee_schema_registry/rebar.config new file mode 100644 index 000000000..b19fb05ae --- /dev/null +++ b/lib-ee/emqx_ee_schema_registry/rebar.config @@ -0,0 +1,12 @@ +%% -*- mode: erlang -*- + +{erl_opts, [debug_info]}. +{deps, [ + {emqx, {path, "../../apps/emqx"}}, + {erlavro, {git, "https://github.com/klarna/erlavro.git", {tag, "2.9.8"}}} +]}. + +{shell, [ + % {config, "config/sys.config"}, + {apps, [emqx_ee_schema_registry]} +]}. diff --git a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.app.src b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.app.src new file mode 100644 index 000000000..c40fb808a --- /dev/null +++ b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.app.src @@ -0,0 +1,15 @@ +{application, emqx_ee_schema_registry, [ + {description, "EMQX Schema Registry"}, + {vsn, "0.1.0"}, + {registered, [emqx_ee_schema_registry_sup]}, + {mod, {emqx_ee_schema_registry_app, []}}, + {applications, [ + kernel, + stdlib, + erlavro + ]}, + {env, []}, + {modules, []}, + + {links, []} +]}. diff --git a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.erl b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.erl new file mode 100644 index 000000000..436777e9f --- /dev/null +++ b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.erl @@ -0,0 +1,242 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_ee_schema_registry). + +-behaviour(gen_server). +-behaviour(emqx_config_handler). + +-include("emqx_ee_schema_registry.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +%% API +-export([ + start_link/0, + + get_serde/1, + + add_schema/2, + delete_schema/1, + list_schemas/0 +]). + +%% `gen_server' API +-export([ + init/1, + handle_call/3, + handle_cast/2, + terminate/2 +]). + +%% `emqx_config_handler' API +-export([post_config_update/5]). + +-type schema() :: #{ + type := serde_type(), + source := binary(), + description => binary() +}. + +%%------------------------------------------------------------------------------------------------- +%% API +%%------------------------------------------------------------------------------------------------- + +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +-spec get_serde(schema_name()) -> {ok, serde_map()} | {error, not_found}. +get_serde(SchemaName) -> + case ets:lookup(?SERDE_TAB, to_bin(SchemaName)) of + [] -> + {error, not_found}; + [Serde] -> + {ok, serde_to_map(Serde)} + end. + +-spec add_schema(schema_name(), schema()) -> ok | {error, term()}. +add_schema(Name, Schema) -> + RawSchema = emqx_map_lib:binary_key_map(Schema), + Res = emqx_conf:update( + [?CONF_KEY_ROOT, schemas, Name], + RawSchema, + #{override_to => cluster} + ), + case Res of + {ok, _} -> + ok; + Error -> + Error + end. + +-spec delete_schema(schema_name()) -> ok | {error, term()}. +delete_schema(Name) -> + Res = emqx_conf:remove( + [?CONF_KEY_ROOT, schemas, Name], + #{override_to => cluster} + ), + case Res of + {ok, _} -> + ok; + Error -> + Error + end. + +-spec list_schemas() -> #{schema_name() => schema()}. +list_schemas() -> + emqx_config:get([?CONF_KEY_ROOT, schemas], #{}). + +%%------------------------------------------------------------------------------------------------- +%% `emqx_config_handler' API +%%------------------------------------------------------------------------------------------------- + +post_config_update( + [?CONF_KEY_ROOT, schemas] = _Path, + _Cmd, + NewConf = #{schemas := NewSchemas}, + OldConf = #{}, + _AppEnvs +) -> + OldSchemas = maps:get(schemas, OldConf, #{}), + #{ + added := Added, + changed := Changed0, + removed := Removed + } = emqx_map_lib:diff_maps(NewSchemas, OldSchemas), + Changed = maps:map(fun(_N, {_Old, New}) -> New end, Changed0), + RemovedNames = maps:keys(Removed), + case RemovedNames of + [] -> + ok; + _ -> + async_delete_serdes(RemovedNames) + end, + SchemasToBuild = maps:to_list(maps:merge(Changed, Added)), + case build_serdes(SchemasToBuild) of + ok -> + {ok, NewConf}; + {error, Reason, SerdesToRollback} -> + lists:foreach(fun ensure_serde_absent/1, SerdesToRollback), + {error, Reason} + end; +post_config_update(_Path, _Cmd, NewConf, _OldConf, _AppEnvs) -> + {ok, NewConf}. + +%%------------------------------------------------------------------------------------------------- +%% `gen_server' API +%%------------------------------------------------------------------------------------------------- + +init(_) -> + process_flag(trap_exit, true), + create_tables(), + Schemas = emqx_conf:get([?CONF_KEY_ROOT, schemas], #{}), + async_build_serdes(Schemas), + State = #{}, + {ok, State}. + +handle_call(_Call, _From, State) -> + {reply, {error, unknown_call}, State}. + +handle_cast({delete_serdes, Names}, State) -> + lists:foreach(fun ensure_serde_absent/1, Names), + ?tp(schema_registry_serdes_deleted, #{}), + {noreply, State}; +handle_cast({build_serdes, Schemas}, State) -> + do_build_serdes(Schemas), + {noreply, State}; +handle_cast(_Cast, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +%%------------------------------------------------------------------------------------------------- +%% Internal fns +%%------------------------------------------------------------------------------------------------- + +create_tables() -> + ok = mria:create_table(?SERDE_TAB, [ + {type, ordered_set}, + {rlog_shard, ?SCHEMA_REGISTRY_SHARD}, + {storage, ram_copies}, + {record_name, serde}, + {attributes, record_info(fields, serde)} + ]), + ok = mria:wait_for_tables([?SERDE_TAB]), + ok. + +do_build_serdes(Schemas) -> + %% TODO: use some kind of mutex to make each core build a + %% different serde to avoid duplicate work. Maybe ekka_locker? + maps:foreach(fun do_build_serde/2, Schemas), + ?tp(schema_registry_serdes_built, #{}). + +build_serdes(Serdes) -> + build_serdes(Serdes, []). + +build_serdes([{Name, Params} | Rest], Acc0) -> + Acc = [Name | Acc0], + case do_build_serde(Name, Params) of + ok -> + build_serdes(Rest, Acc); + {error, Error} -> + {error, Error, Acc} + end; +build_serdes([], _Acc) -> + ok. + +do_build_serde(Name0, #{type := Type, source := Source}) -> + try + Name = to_bin(Name0), + {Serializer, Deserializer, Destructor} = + emqx_ee_schema_registry_serde:make_serde(Type, Name, Source), + Serde = #serde{ + name = Name, + serializer = Serializer, + deserializer = Deserializer, + destructor = Destructor + }, + ok = mria:dirty_write(?SERDE_TAB, Serde), + ok + catch + Kind:Error:Stacktrace -> + ?SLOG( + error, + #{ + msg => "error_building_serde", + name => Name0, + type => Type, + kind => Kind, + error => Error, + stacktrace => Stacktrace + } + ), + {error, Error} + end. + +ensure_serde_absent(Name) -> + case get_serde(Name) of + {ok, #{destructor := Destructor}} -> + Destructor(), + ok = mria:dirty_delete(?SERDE_TAB, to_bin(Name)); + {error, not_found} -> + ok + end. + +async_build_serdes(Schemas) -> + gen_server:cast(?MODULE, {build_serdes, Schemas}). + +async_delete_serdes(Names) -> + gen_server:cast(?MODULE, {delete_serdes, Names}). + +to_bin(A) when is_atom(A) -> atom_to_binary(A); +to_bin(B) when is_binary(B) -> B. + +-spec serde_to_map(serde()) -> serde_map(). +serde_to_map(#serde{} = Serde) -> + #{ + name => Serde#serde.name, + serializer => Serde#serde.serializer, + deserializer => Serde#serde.deserializer, + destructor => Serde#serde.destructor + }. diff --git a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_app.erl b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_app.erl new file mode 100644 index 000000000..e82ed95bd --- /dev/null +++ b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_app.erl @@ -0,0 +1,19 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_ee_schema_registry_app). + +-behaviour(application). + +-include("emqx_ee_schema_registry.hrl"). + +-export([start/2, stop/1]). + +start(_StartType, _StartArgs) -> + ok = mria_rlog:wait_for_shards([?SCHEMA_REGISTRY_SHARD], infinity), + emqx_conf:add_handler(?CONF_KEY_PATH, emqx_ee_schema_registry), + emqx_ee_schema_registry_sup:start_link(). + +stop(_State) -> + emqx_conf:remove_handler(?CONF_KEY_PATH), + ok. diff --git a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_http_api.erl b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_http_api.erl new file mode 100644 index 000000000..fca66a0b1 --- /dev/null +++ b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_http_api.erl @@ -0,0 +1,251 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_ee_schema_registry_http_api). + +-behaviour(minirest_api). + +-include("emqx_ee_schema_registry.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/emqx_api_lib.hrl"). + +-export([ + namespace/0, + api_spec/0, + paths/0, + schema/1 +]). + +-export([ + '/schema_registry'/2, + '/schema_registry/:name'/2 +]). + +%%------------------------------------------------------------------------------------------------- +%% `minirest' and `minirest_trails' API +%%------------------------------------------------------------------------------------------------- + +namespace() -> "schema_registry_http_api". + +api_spec() -> + emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). + +paths() -> + [ + "/schema_registry", + "/schema_registry/:name" + ]. + +schema("/schema_registry") -> + #{ + 'operationId' => '/schema_registry', + get => #{ + tags => [<<"schema_registry">>], + summary => <<"List registered schemas">>, + description => ?DESC("desc_schema_registry_api_list"), + responses => + #{ + 200 => + emqx_dashboard_swagger:schema_with_examples( + hoconsc:array(emqx_ee_schema_registry_schema:api_schema("get")), + #{ + sample => + #{value => sample_list_schemas_response()} + } + ) + } + }, + post => #{ + tags => [<<"schema_registry">>], + summary => <<"Register a new schema">>, + description => ?DESC("desc_schema_registry_api_post"), + 'requestBody' => emqx_dashboard_swagger:schema_with_examples( + emqx_ee_schema_registry_schema:api_schema("post"), + post_examples() + ), + responses => + #{ + 201 => + emqx_dashboard_swagger:schema_with_examples( + emqx_ee_schema_registry_schema:api_schema("post"), + post_examples() + ), + 400 => error_schema('ALREADY_EXISTS', "Schema already exists") + } + } + }; +schema("/schema_registry/:name") -> + #{ + 'operationId' => '/schema_registry/:name', + get => #{ + tags => [<<"schema_registry">>], + summary => <<"Get registered schema">>, + description => ?DESC("desc_schema_registry_api_get"), + parameters => [param_path_schema_name()], + responses => + #{ + 200 => + emqx_dashboard_swagger:schema_with_examples( + emqx_ee_schema_registry_schema:api_schema("get"), + get_examples() + ), + 404 => error_schema('NOT_FOUND', "Schema not found") + } + }, + put => #{ + tags => [<<"schema_registry">>], + summary => <<"Update a schema">>, + description => ?DESC("desc_schema_registry_api_put"), + parameters => [param_path_schema_name()], + 'requestBody' => emqx_dashboard_swagger:schema_with_examples( + emqx_ee_schema_registry_schema:api_schema("put"), + post_examples() + ), + responses => + #{ + 200 => + emqx_dashboard_swagger:schema_with_examples( + emqx_ee_schema_registry_schema:api_schema("put"), + put_examples() + ), + 404 => error_schema('NOT_FOUND', "Schema not found") + } + }, + delete => #{ + tags => [<<"schema_registry">>], + summary => <<"Delete registered schema">>, + description => ?DESC("desc_schema_registry_api_delete"), + parameters => [param_path_schema_name()], + responses => + #{ + 204 => <<"Schema deleted">>, + 404 => error_schema('NOT_FOUND', "Schema not found") + } + } + }. + +%%------------------------------------------------------------------------------------------------- +%% API +%%------------------------------------------------------------------------------------------------- + +'/schema_registry'(get, _Params) -> + Schemas = emqx_ee_schema_registry:list_schemas(), + Response = + lists:map( + fun({Name, Params}) -> + Params#{name => Name} + end, + maps:to_list(Schemas) + ), + ?OK(Response); +'/schema_registry'(post, #{body := Params0 = #{<<"name">> := Name}}) -> + Params = maps:without([<<"name">>], Params0), + case emqx_config:get([?CONF_KEY_ROOT, schemas, Name], undefined) of + undefined -> + case emqx_ee_schema_registry:add_schema(Name, Params) of + ok -> + Res = emqx_config:get([?CONF_KEY_ROOT, schemas, Name]), + {201, Res#{name => Name}}; + {error, Error} -> + ?BAD_REQUEST(Error) + end; + _ -> + ?BAD_REQUEST('ALREADY_EXISTS', <<"Schema already exists">>) + end. + +'/schema_registry/:name'(get, #{bindings := #{name := Name}}) -> + case emqx_config:get([?CONF_KEY_ROOT, schemas, Name], undefined) of + undefined -> + ?NOT_FOUND(<<"Schema not found">>); + Res -> + ?OK(Res#{name => Name}) + end; +'/schema_registry/:name'(put, #{bindings := #{name := Name}, body := Params}) -> + case emqx_config:get([?CONF_KEY_ROOT, schemas, Name], undefined) of + undefined -> + ?NOT_FOUND(<<"Schema not found">>); + _ -> + case emqx_ee_schema_registry:add_schema(Name, Params) of + ok -> + Res = emqx_config:get([?CONF_KEY_ROOT, schemas, Name]), + ?OK(Res#{name => Name}); + {error, Error} -> + ?BAD_REQUEST(Error) + end + end; +'/schema_registry/:name'(delete, #{bindings := #{name := Name}}) -> + case emqx_config:get([?CONF_KEY_ROOT, schemas, Name], undefined) of + undefined -> + ?NOT_FOUND(<<"Schema not found">>); + _ -> + case emqx_ee_schema_registry:delete_schema(Name) of + ok -> + ?NO_CONTENT; + {error, Error} -> + ?BAD_REQUEST(Error) + end + end. + +%%------------------------------------------------------------------------------------------------- +%% Examples +%%------------------------------------------------------------------------------------------------- + +sample_list_schemas_response() -> + [sample_get_schema_response(avro)]. + +sample_get_schema_response(avro) -> + #{ + type => <<"avro">>, + name => <<"my_avro_schema">>, + description => <<"My Avro Schema">>, + source => << + "{\"type\":\"record\"," + "\"fields\":[{\"type\":\"int\",\"name\":\"i\"}," + "{\"type\":\"string\",\"name\":\"s\"}]}" + >> + }. + +put_examples() -> + post_examples(). + +post_examples() -> + get_examples(). + +get_examples() -> + #{ + <<"avro_schema">> => + #{ + summary => <<"Avro">>, + value => sample_get_schema_response(avro) + } + }. + +%%------------------------------------------------------------------------------------------------- +%% Schemas and hocon types +%%------------------------------------------------------------------------------------------------- + +param_path_schema_name() -> + {name, + mk( + binary(), + #{ + in => path, + required => true, + example => <<"my_schema">>, + desc => ?DESC("desc_param_path_schema_name") + } + )}. + +%%------------------------------------------------------------------------------------------------- +%% Internal fns +%%------------------------------------------------------------------------------------------------- + +mk(Type, Meta) -> hoconsc:mk(Type, Meta). + +error_schema(Code, Message) when is_atom(Code) -> + error_schema([Code], Message); +error_schema(Codes, Message) when is_list(Message) -> + error_schema(Codes, list_to_binary(Message)); +error_schema(Codes, Message) when is_list(Codes) andalso is_binary(Message) -> + emqx_dashboard_swagger:error_codes(Codes, Message). diff --git a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_schema.erl b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_schema.erl new file mode 100644 index 000000000..01177345a --- /dev/null +++ b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_schema.erl @@ -0,0 +1,127 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_ee_schema_registry_schema). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). +-include("emqx_ee_schema_registry.hrl"). + +%% `hocon_schema' API +-export([ + roots/0, + fields/1, + desc/1, + tags/0, + union_member_selector/1 +]). + +%% `minirest_trails' API +-export([ + api_schema/1 +]). + +%%------------------------------------------------------------------------------ +%% `hocon_schema' APIs +%%------------------------------------------------------------------------------ + +roots() -> + [{?CONF_KEY_ROOT, mk(ref(?CONF_KEY_ROOT), #{required => false})}]. + +tags() -> + [<<"Schema Registry">>]. + +fields(?CONF_KEY_ROOT) -> + [ + {schemas, + mk( + hoconsc:map( + name, + hoconsc:union(fun union_member_selector/1) + ), + #{ + default => #{}, + desc => ?DESC("schema_registry_schemas") + } + )} + ]; +fields(avro) -> + [ + {type, mk(hoconsc:enum([avro]), #{required => true, desc => ?DESC("schema_type")})}, + {source, + mk(emqx_schema:json_binary(), #{required => true, desc => ?DESC("schema_source")})}, + {description, mk(binary(), #{default => <<>>, desc => ?DESC("schema_description")})} + ]; +fields("get_avro") -> + [{name, mk(binary(), #{required => true, desc => ?DESC("schema_name")})} | fields(avro)]; +fields("put_avro") -> + fields(avro); +fields("post_" ++ Type) -> + fields("get_" ++ Type). + +desc(?CONF_KEY_ROOT) -> + ?DESC("schema_registry_root"); +desc(avro) -> + ?DESC("avro_type"); +desc(_) -> + undefined. + +union_member_selector(all_union_members) -> + refs(); +union_member_selector({value, V}) -> + refs(V). + +union_member_selector_get_api(all_union_members) -> + refs_get_api(); +union_member_selector_get_api({value, V}) -> + refs_get_api(V). + +%%------------------------------------------------------------------------------ +%% `minirest_trails' "APIs" +%%------------------------------------------------------------------------------ + +api_schema("get") -> + hoconsc:union(fun union_member_selector_get_api/1); +api_schema("post") -> + api_schema("get"); +api_schema("put") -> + hoconsc:union(fun union_member_selector/1). + +%%------------------------------------------------------------------------------ +%% Internal fns +%%------------------------------------------------------------------------------ + +mk(Type, Meta) -> hoconsc:mk(Type, Meta). +ref(Name) -> hoconsc:ref(?MODULE, Name). + +supported_serde_types() -> + [avro]. + +refs() -> + [ref(Type) || Type <- supported_serde_types()]. + +refs(#{<<"type">> := TypeAtom} = Value) when is_atom(TypeAtom) -> + refs(Value#{<<"type">> := atom_to_binary(TypeAtom)}); +refs(#{<<"type">> := <<"avro">>}) -> + [ref(avro)]; +refs(_) -> + Expected = lists:join(" | ", [atom_to_list(T) || T <- supported_serde_types()]), + throw(#{ + field_name => type, + expected => Expected + }). + +refs_get_api() -> + [ref("get_avro")]. + +refs_get_api(#{<<"type">> := TypeAtom} = Value) when is_atom(TypeAtom) -> + refs(Value#{<<"type">> := atom_to_binary(TypeAtom)}); +refs_get_api(#{<<"type">> := <<"avro">>}) -> + [ref("get_avro")]; +refs_get_api(_) -> + Expected = lists:join(" | ", [atom_to_list(T) || T <- supported_serde_types()]), + throw(#{ + field_name => type, + expected => Expected + }). diff --git a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_serde.erl b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_serde.erl new file mode 100644 index 000000000..43145fb16 --- /dev/null +++ b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_serde.erl @@ -0,0 +1,70 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_ee_schema_registry_serde). + +-include("emqx_ee_schema_registry.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +%% API +-export([ + decode/2, + decode/3, + encode/2, + encode/3, + make_serde/3 +]). + +%%------------------------------------------------------------------------------ +%% API +%%------------------------------------------------------------------------------ + +-spec decode(schema_name(), encoded_data()) -> decoded_data(). +decode(SerdeName, RawData) -> + decode(SerdeName, RawData, []). + +-spec decode(schema_name(), encoded_data(), [term()]) -> decoded_data(). +decode(SerdeName, RawData, VarArgs) when is_list(VarArgs) -> + case emqx_ee_schema_registry:get_serde(SerdeName) of + {error, not_found} -> + error({serde_not_found, SerdeName}); + {ok, #{deserializer := Deserializer}} -> + apply(Deserializer, [RawData | VarArgs]) + end. + +-spec encode(schema_name(), decoded_data()) -> encoded_data(). +encode(SerdeName, RawData) -> + encode(SerdeName, RawData, []). + +-spec encode(schema_name(), decoded_data(), [term()]) -> encoded_data(). +encode(SerdeName, EncodedData, VarArgs) when is_list(VarArgs) -> + case emqx_ee_schema_registry:get_serde(SerdeName) of + {error, not_found} -> + error({serde_not_found, SerdeName}); + {ok, #{serializer := Serializer}} -> + apply(Serializer, [EncodedData | VarArgs]) + end. + +-spec make_serde(serde_type(), schema_name(), schema_source()) -> + {serializer(), deserializer(), destructor()}. +make_serde(avro, Name, Source0) -> + Source = inject_avro_name(Name, Source0), + Serializer = avro:make_simple_encoder(Source, _Opts = []), + Deserializer = avro:make_simple_decoder(Source, [{map_type, map}, {record_type, map}]), + Destructor = fun() -> + ?tp(serde_destroyed, #{type => avro, name => Name}), + ok + end, + {Serializer, Deserializer, Destructor}. + +%%------------------------------------------------------------------------------ +%% Internal fns +%%------------------------------------------------------------------------------ + +-spec inject_avro_name(schema_name(), schema_source()) -> schema_source(). +inject_avro_name(Name, Source0) -> + %% The schema checks that the source is a valid JSON when + %% typechecking, so we shouldn't need to validate here. + Schema0 = emqx_json:decode(Source0, [return_maps]), + Schema = Schema0#{<<"name">> => Name}, + emqx_json:encode(Schema). diff --git a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_sup.erl b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_sup.erl new file mode 100644 index 000000000..0dfc601d3 --- /dev/null +++ b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_sup.erl @@ -0,0 +1,43 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_ee_schema_registry_sup). + +-behaviour(supervisor). + +-export([start_link/0]). + +-export([init/1]). + +-define(SERVER, ?MODULE). + +start_link() -> + supervisor:start_link({local, ?SERVER}, ?MODULE, []). + +%% sup_flags() = #{strategy => strategy(), % optional +%% intensity => non_neg_integer(), % optional +%% period => pos_integer()} % optional +%% child_spec() = #{id => child_id(), % mandatory +%% start => mfargs(), % mandatory +%% restart => restart(), % optional +%% shutdown => shutdown(), % optional +%% type => worker(), % optional +%% modules => modules()} % optional +init([]) -> + SupFlags = #{ + strategy => one_for_one, + intensity => 10, + period => 100 + }, + ChildSpecs = [child_spec(emqx_ee_schema_registry)], + {ok, {SupFlags, ChildSpecs}}. + +child_spec(Mod) -> + #{ + id => Mod, + start => {Mod, start_link, []}, + restart => permanent, + shutdown => 5_000, + type => worker, + modules => [Mod] + }. diff --git a/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl new file mode 100644 index 000000000..9b2f64c03 --- /dev/null +++ b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl @@ -0,0 +1,433 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_ee_schema_registry_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-include("emqx_ee_schema_registry.hrl"). + +-import(emqx_common_test_helpers, [on_exit/1]). + +-define(APPS, [emqx_conf, emqx_rule_engine, emqx_ee_schema_registry]). + +%%------------------------------------------------------------------------------ +%% CT boilerplate +%%------------------------------------------------------------------------------ + +all() -> + [{group, avro}]. + +groups() -> + TCs = emqx_common_test_helpers:all(?MODULE), + [{avro, TCs}]. + +init_per_suite(Config) -> + emqx_config:save_schema_mod_and_names(emqx_ee_schema_registry_schema), + emqx_mgmt_api_test_util:init_suite(?APPS), + Config. + +end_per_suite(_Config) -> + emqx_mgmt_api_test_util:end_suite(lists:reverse(?APPS)), + ok. + +init_per_group(avro, Config) -> + [{serde_type, avro} | Config]; +init_per_group(_Group, Config) -> + Config. + +end_per_group(_Group, _Config) -> + ok. + +init_per_testcase(_TestCase, Config) -> + ok = snabbkaffe:start_trace(), + Config. + +end_per_testcase(_TestCase, _Config) -> + ok = snabbkaffe:stop(), + emqx_common_test_helpers:call_janitor(), + clear_schemas(), + ok. + +%%------------------------------------------------------------------------------ +%% Helper fns +%%------------------------------------------------------------------------------ + +trace_rule(Data, Envs, _Args) -> + Now = erlang:monotonic_time(), + ets:insert(recorded_actions, {Now, #{data => Data, envs => Envs}}), + TestPid = persistent_term:get({?MODULE, test_pid}), + TestPid ! {action, #{data => Data, envs => Envs}}, + ok. + +make_trace_fn_action() -> + persistent_term:put({?MODULE, test_pid}, self()), + Fn = <<(atom_to_binary(?MODULE))/binary, ":trace_rule">>, + emqx_tables:new(recorded_actions, [named_table, public, ordered_set]), + #{function => Fn, args => #{}}. + +create_rule_http(RuleParams) -> + RepublishTopic = <<"republish/schema_registry">>, + emqx:subscribe(RepublishTopic), + DefaultParams = #{ + enable => true, + actions => [ + make_trace_fn_action(), + #{ + <<"function">> => <<"republish">>, + <<"args">> => + #{ + <<"topic">> => RepublishTopic, + <<"payload">> => <<>>, + <<"qos">> => 0, + <<"retain">> => false, + <<"user_properties">> => <<>> + } + } + ] + }, + Params = maps:merge(DefaultParams, RuleParams), + Path = emqx_mgmt_api_test_util:api_path(["rules"]), + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of + {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + Error -> Error + end. + +schema_params(avro) -> + Source = #{ + type => record, + fields => [ + #{name => <<"i">>, type => <<"int">>}, + #{name => <<"s">>, type => <<"string">>} + ] + }, + SourceBin = emqx_json:encode(Source), + #{type => avro, source => SourceBin}. + +create_serde(SerdeType, SerdeName) -> + Schema = schema_params(SerdeType), + ok = emqx_ee_schema_registry:add_schema(SerdeName, Schema), + ok. + +sql_for(avro, encode_decode1) -> + << + "select\n" + " schema_decode('my_serde',\n" + " schema_encode('my_serde', json_decode(payload))) as decoded,\n" + " decoded.i as decoded_int,\n" + " decoded.s as decoded_string\n" + " from t" + >>; +sql_for(avro, encode1) -> + << + "select\n" + " schema_encode('my_serde', json_decode(payload)) as encoded\n" + " from t" + >>; +sql_for(avro, decode1) -> + << + "select\n" + " schema_decode('my_serde', payload) as decoded\n" + " from t" + >>; +sql_for(Type, Name) -> + ct:fail("unimplemented: ~p", [{Type, Name}]). + +clear_schemas() -> + maps:foreach( + fun(Name, _Schema) -> + ok = emqx_ee_schema_registry:delete_schema(Name) + end, + emqx_ee_schema_registry:list_schemas() + ). + +receive_action_results() -> + receive + {action, #{data := _} = Res} -> + Res + after 1_000 -> + ct:fail("action didn't run") + end. + +receive_published(Line) -> + receive + {deliver, _Topic, Msg} -> + MsgMap = emqx_message:to_map(Msg), + maps:update_with( + payload, + fun(Raw) -> + case emqx_json:safe_decode(Raw, [return_maps]) of + {ok, Decoded} -> Decoded; + {error, _} -> Raw + end + end, + MsgMap + ) + after 1_000 -> + ct:pal("mailbox: ~p", [process_info(self(), messages)]), + ct:fail("publish not received, line ~b", [Line]) + end. + +cluster(Config) -> + PrivDataDir = ?config(priv_dir, Config), + PeerModule = + case os:getenv("IS_CI") of + false -> + slave; + _ -> + ct_slave + end, + Cluster = emqx_common_test_helpers:emqx_cluster( + [core, core], + [ + {apps, ?APPS}, + {listener_ports, []}, + {peer_mod, PeerModule}, + {priv_data_dir, PrivDataDir}, + {load_schema, true}, + {start_autocluster, true}, + {schema_mod, emqx_ee_conf_schema}, + %% need to restart schema registry app in the tests so + %% that it re-registers the config handler that is lost + %% when emqx_conf restarts during join. + {env, [{emqx_machine, applications, [emqx_ee_schema_registry]}]}, + {load_apps, [emqx_machine | ?APPS]}, + {env_handler, fun + (emqx) -> + application:set_env(emqx, boot_modules, [broker, router]), + ok; + (emqx_conf) -> + ok; + (_) -> + ok + end} + ] + ), + ct:pal("cluster:\n ~p", [Cluster]), + Cluster. + +start_cluster(Cluster) -> + Nodes = [ + emqx_common_test_helpers:start_slave(Name, Opts) + || {Name, Opts} <- Cluster + ], + on_exit(fun() -> + emqx_misc:pmap( + fun(N) -> + ct:pal("stopping ~p", [N]), + ok = emqx_common_test_helpers:stop_slave(N) + end, + Nodes + ) + end), + erpc:multicall(Nodes, mria_rlog, wait_for_shards, [[?SCHEMA_REGISTRY_SHARD], 30_000]), + Nodes. + +wait_for_cluster_rpc(Node) -> + %% need to wait until the config handler is ready after + %% restarting during the cluster join. + ?retry( + _Sleep0 = 100, + _Attempts0 = 50, + true = is_pid(erpc:call(Node, erlang, whereis, [emqx_config_handler])) + ). + +%%------------------------------------------------------------------------------ +%% Testcases +%%------------------------------------------------------------------------------ + +t_unknown_calls(_Config) -> + Ref = monitor(process, emqx_ee_schema_registry), + %% for coverage + emqx_ee_schema_registry ! unknown, + gen_server:cast(emqx_ee_schema_registry, unknown), + ?assertEqual({error, unknown_call}, gen_server:call(emqx_ee_schema_registry, unknown)), + receive + {'DOWN', Ref, process, _, _} -> + ct:fail("registry shouldn't have died") + after 500 -> + ok + end. + +t_encode_decode(Config) -> + SerdeType = ?config(serde_type, Config), + SerdeName = my_serde, + ok = create_serde(SerdeType, SerdeName), + {ok, #{<<"id">> := RuleId}} = create_rule_http(#{sql => sql_for(SerdeType, encode_decode1)}), + on_exit(fun() -> ok = emqx_rule_engine:delete_rule(RuleId) end), + Payload = #{<<"i">> => 10, <<"s">> => <<"text">>}, + PayloadBin = emqx_json:encode(Payload), + emqx:publish(emqx_message:make(<<"t">>, PayloadBin)), + Res = receive_action_results(), + ?assertMatch( + #{ + data := + #{ + <<"decoded">> := + #{ + <<"i">> := 10, + <<"s">> := <<"text">> + }, + <<"decoded_int">> := 10, + <<"decoded_string">> := <<"text">> + } + }, + Res + ), + ok. + +t_delete_serde(Config) -> + SerdeType = ?config(serde_type, Config), + SerdeName = my_serde, + ?check_trace( + begin + ok = create_serde(SerdeType, SerdeName), + {ok, {ok, _}} = + ?wait_async_action( + emqx_ee_schema_registry:delete_schema(SerdeName), + #{?snk_kind := schema_registry_serdes_deleted}, + 1_000 + ), + ok + end, + fun(Trace) -> + ?assertMatch([_], ?of_kind(schema_registry_serdes_deleted, Trace)), + ?assertMatch([#{type := SerdeType}], ?of_kind(serde_destroyed, Trace)), + ok + end + ), + ok. + +t_encode(Config) -> + SerdeType = ?config(serde_type, Config), + SerdeName = my_serde, + ok = create_serde(SerdeType, SerdeName), + {ok, #{<<"id">> := RuleId}} = create_rule_http(#{sql => sql_for(SerdeType, encode1)}), + on_exit(fun() -> ok = emqx_rule_engine:delete_rule(RuleId) end), + Payload = #{<<"i">> => 10, <<"s">> => <<"text">>}, + PayloadBin = emqx_json:encode(Payload), + emqx:publish(emqx_message:make(<<"t">>, PayloadBin)), + Published = receive_published(?LINE), + ?assertMatch( + #{payload := #{<<"encoded">> := _}}, + Published + ), + #{payload := #{<<"encoded">> := Encoded}} = Published, + {ok, #{deserializer := Deserializer}} = emqx_ee_schema_registry:get_serde(SerdeName), + ?assertEqual(Payload, Deserializer(Encoded)), + ok. + +t_decode(Config) -> + SerdeType = ?config(serde_type, Config), + SerdeName = my_serde, + ok = create_serde(SerdeType, SerdeName), + {ok, #{<<"id">> := RuleId}} = create_rule_http(#{sql => sql_for(SerdeType, decode1)}), + on_exit(fun() -> ok = emqx_rule_engine:delete_rule(RuleId) end), + Payload = #{<<"i">> => 10, <<"s">> => <<"text">>}, + {ok, #{serializer := Serializer}} = emqx_ee_schema_registry:get_serde(SerdeName), + EncodedBin = Serializer(Payload), + emqx:publish(emqx_message:make(<<"t">>, EncodedBin)), + Published = receive_published(?LINE), + ?assertMatch( + #{payload := #{<<"decoded">> := _}}, + Published + ), + #{payload := #{<<"decoded">> := Decoded}} = Published, + ?assertEqual(Payload, Decoded), + ok. + +t_fail_rollback(Config) -> + SerdeType = ?config(serde_type, Config), + OkSchema = emqx_map_lib:binary_key_map(schema_params(SerdeType)), + BrokenSchema = OkSchema#{<<"source">> := <<"{}">>}, + %% hopefully, for this small map, the key order is used. + Serdes = #{ + <<"a">> => OkSchema, + <<"z">> => BrokenSchema + }, + ?assertMatch( + {error, _}, + emqx_conf:update( + [?CONF_KEY_ROOT, schemas], + Serdes, + #{} + ) + ), + %% no serdes should be in the table + ?assertEqual({error, not_found}, emqx_ee_schema_registry:get_serde(<<"a">>)), + ?assertEqual({error, not_found}, emqx_ee_schema_registry:get_serde(<<"z">>)), + ok. + +t_cluster_serde_build(Config) -> + SerdeType = ?config(serde_type, Config), + Cluster = cluster(Config), + SerdeName = my_serde, + Schema = schema_params(SerdeType), + ?check_trace( + begin + Nodes = [N1, N2 | _] = start_cluster(Cluster), + NumNodes = length(Nodes), + wait_for_cluster_rpc(N2), + ?assertEqual( + ok, + erpc:call(N2, emqx_ee_schema_registry, add_schema, [SerdeName, Schema]) + ), + %% check that we can serialize/deserialize in all nodes + lists:foreach( + fun(N) -> + erpc:call(N, fun() -> + Res0 = emqx_ee_schema_registry:get_serde(SerdeName), + ?assertMatch({ok, #{}}, Res0, #{node => N}), + {ok, #{serializer := Serializer, deserializer := Deserializer}} = Res0, + Payload = #{<<"i">> => 10, <<"s">> => <<"text">>}, + ?assertEqual(Payload, Deserializer(Serializer(Payload)), #{node => N}), + ok + end) + end, + Nodes + ), + %% now we delete and check it's removed from the table + ?tp(will_delete_schema, #{}), + {ok, SRef1} = snabbkaffe:subscribe( + ?match_event(#{?snk_kind := schema_registry_serdes_deleted}), + NumNodes, + 5_000 + ), + ?assertEqual( + ok, + erpc:call(N1, emqx_ee_schema_registry, delete_schema, [SerdeName]) + ), + {ok, _} = snabbkaffe:receive_events(SRef1), + lists:foreach( + fun(N) -> + erpc:call(N, fun() -> + ?assertMatch( + {error, not_found}, + emqx_ee_schema_registry:get_serde(SerdeName), + #{node => N} + ), + ok + end) + end, + Nodes + ), + ok + end, + fun(Trace) -> + ?assert( + ?strict_causality( + #{?snk_kind := will_delete_schema}, + #{?snk_kind := serde_destroyed, type := SerdeType}, + Trace + ) + ), + ok + end + ), + ok. diff --git a/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_http_api_SUITE.erl b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_http_api_SUITE.erl new file mode 100644 index 000000000..bbb6d5ef0 --- /dev/null +++ b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_http_api_SUITE.erl @@ -0,0 +1,250 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_ee_schema_registry_http_api_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-import(emqx_mgmt_api_test_util, [uri/1]). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-define(APPS, [emqx_conf, emqx_ee_schema_registry]). + +%%------------------------------------------------------------------------------ +%% CT boilerplate +%%------------------------------------------------------------------------------ + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + emqx_config:save_schema_mod_and_names(emqx_ee_schema_registry_schema), + emqx_mgmt_api_test_util:init_suite(?APPS), + Config. + +end_per_suite(_Config) -> + emqx_mgmt_api_test_util:end_suite(lists:reverse(?APPS)), + ok. + +init_per_testcase(_TestCase, Config) -> + clear_schemas(), + ok = snabbkaffe:start_trace(), + Config. + +end_per_testcase(_TestCase, _Config) -> + clear_schemas(), + ok = snabbkaffe:stop(), + ok. + +%%------------------------------------------------------------------------------ +%% Helper fns +%%------------------------------------------------------------------------------ + +request(get) -> + do_request(get, _Parts = [], _Body = []); +request({get, Name}) -> + do_request(get, _Parts = [Name], _Body = []); +request({delete, Name}) -> + do_request(delete, _Parts = [Name], _Body = []); +request({put, Name, Params}) -> + do_request(put, _Parts = [Name], Params); +request({post, Params}) -> + do_request(post, _Parts = [], Params). + +do_request(Method, PathParts, Body) -> + Header = emqx_common_test_http:default_auth_header(), + URI = uri(["schema_registry" | PathParts]), + Opts = #{compatible_mode => true, httpc_req_opts => [{body_format, binary}]}, + Res0 = emqx_mgmt_api_test_util:request_api(Method, URI, [], Header, Body, Opts), + case Res0 of + {ok, Code, <<>>} -> + {ok, Code, <<>>}; + {ok, Code, Res1} -> + Res2 = emqx_json:decode(Res1, [return_maps]), + Res3 = try_decode_error_message(Res2), + {ok, Code, Res3}; + Error -> + Error + end. + +try_decode_error_message(#{<<"message">> := Msg0} = Res0) -> + case emqx_json:safe_decode(Msg0, [return_maps]) of + {ok, Msg} -> + Res0#{<<"message">> := Msg}; + {error, _} -> + Res0 + end; +try_decode_error_message(Res) -> + Res. + +clear_schemas() -> + maps:foreach( + fun(Name, _Schema) -> + ok = emqx_ee_schema_registry:delete_schema(Name) + end, + emqx_ee_schema_registry:list_schemas() + ). + +%%------------------------------------------------------------------------------ +%% Testcases +%%------------------------------------------------------------------------------ + +t_crud(_Config) -> + SchemaName = <<"my_avro_schema">>, + Source = #{ + type => record, + fields => [ + #{name => <<"i">>, type => <<"int">>}, + #{name => <<"s">>, type => <<"string">>} + ] + }, + SourceBin = emqx_json:encode(Source), + Params = #{ + <<"type">> => <<"avro">>, + <<"source">> => SourceBin, + <<"name">> => SchemaName, + <<"description">> => <<"My schema">> + }, + UpdateParams = maps:without([<<"name">>], Params), + + %% no schemas at first + ?assertMatch({ok, 200, []}, request(get)), + ?assertMatch( + {ok, 404, #{ + <<"code">> := <<"NOT_FOUND">>, + <<"message">> := <<"Schema not found">> + }}, + request({get, SchemaName}) + ), + ?assertMatch( + {ok, 404, #{ + <<"code">> := <<"NOT_FOUND">>, + <<"message">> := <<"Schema not found">> + }}, + request({put, SchemaName, UpdateParams}) + ), + ?assertMatch( + {ok, 404, #{ + <<"code">> := <<"NOT_FOUND">>, + <<"message">> := <<"Schema not found">> + }}, + request({delete, SchemaName}) + ), + + %% create a schema + ?assertMatch( + {ok, 201, #{ + <<"type">> := <<"avro">>, + <<"source">> := SourceBin, + <<"name">> := SchemaName, + <<"description">> := <<"My schema">> + }}, + request({post, Params}) + ), + ?assertMatch( + {ok, 200, #{ + <<"type">> := <<"avro">>, + <<"source">> := SourceBin, + <<"name">> := SchemaName, + <<"description">> := <<"My schema">> + }}, + request({get, SchemaName}) + ), + ?assertMatch( + {ok, 200, [ + #{ + <<"type">> := <<"avro">>, + <<"source">> := SourceBin, + <<"name">> := SchemaName, + <<"description">> := <<"My schema">> + } + ]}, + request(get) + ), + UpdateParams1 = UpdateParams#{<<"description">> := <<"My new schema">>}, + ?assertMatch( + {ok, 200, #{ + <<"type">> := <<"avro">>, + <<"source">> := SourceBin, + <<"name">> := SchemaName, + <<"description">> := <<"My new schema">> + }}, + request({put, SchemaName, UpdateParams1}) + ), + + ?assertMatch( + {ok, 400, #{ + <<"code">> := <<"ALREADY_EXISTS">>, + <<"message">> := <<"Schema already exists">> + }}, + request({post, Params}) + ), + %% typechecks, but is invalid + ?assertMatch( + {ok, 400, #{ + <<"code">> := <<"BAD_REQUEST">>, + <<"message">> := + <<"{post_config_update,emqx_ee_schema_registry,{not_found,<<\"type\">>}}">> + }}, + request({put, SchemaName, UpdateParams#{<<"source">> := <<"{}">>}}) + ), + + ?assertMatch( + {ok, 204, <<>>}, + request({delete, SchemaName}) + ), + + %% doesn't typecheck + lists:foreach( + fun(Field) -> + ?assertMatch( + {ok, 400, #{ + <<"code">> := <<"BAD_REQUEST">>, + <<"message">> := #{<<"reason">> := <<"required_field">>} + }}, + request({post, maps:without([Field], Params)}), + #{field => Field} + ) + end, + [<<"name">>, <<"source">>] + ), + ?assertMatch( + {ok, 400, #{ + <<"code">> := <<"BAD_REQUEST">>, + <<"message">> := + #{ + <<"expected">> := [_ | _], + <<"field_name">> := <<"type">> + } + }}, + request({post, maps:without([<<"type">>], Params)}), + #{field => <<"type">>} + ), + %% typechecks, but is invalid + ?assertMatch( + {ok, 400, #{ + <<"code">> := <<"BAD_REQUEST">>, + <<"message">> := + <<"{post_config_update,emqx_ee_schema_registry,{not_found,<<\"type\">>}}">> + }}, + request({post, Params#{<<"source">> := <<"{}">>}}) + ), + + %% unknown serde type + ?assertMatch( + {ok, 400, #{ + <<"code">> := <<"BAD_REQUEST">>, + <<"message">> := + #{ + <<"expected">> := [_ | _], + <<"field_name">> := <<"type">> + } + }}, + request({post, Params#{<<"type">> := <<"foo">>}}) + ), + + ok. diff --git a/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_serde_SUITE.erl b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_serde_SUITE.erl new file mode 100644 index 000000000..be62717d3 --- /dev/null +++ b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_serde_SUITE.erl @@ -0,0 +1,121 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_ee_schema_registry_serde_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-include("emqx_ee_schema_registry.hrl"). + +-import(emqx_common_test_helpers, [on_exit/1]). + +-define(APPS, [emqx_conf, emqx_rule_engine, emqx_ee_schema_registry]). + +%%------------------------------------------------------------------------------ +%% CT boilerplate +%%------------------------------------------------------------------------------ + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + emqx_config:save_schema_mod_and_names(emqx_ee_schema_registry_schema), + emqx_mgmt_api_test_util:init_suite(?APPS), + Config. + +end_per_suite(_Config) -> + emqx_mgmt_api_test_util:end_suite(lists:reverse(?APPS)), + ok. +init_per_testcase(_TestCase, Config) -> + Config. + +end_per_testcase(_TestCase, _Config) -> + emqx_common_test_helpers:call_janitor(), + clear_schemas(), + ok. + +%%------------------------------------------------------------------------------ +%% Helper fns +%%------------------------------------------------------------------------------ + +clear_schemas() -> + maps:foreach( + fun(Name, _Schema) -> + ok = emqx_ee_schema_registry:delete_schema(Name) + end, + emqx_ee_schema_registry:list_schemas() + ). + +schema_params(avro) -> + Source = #{ + type => record, + fields => [ + #{name => <<"i">>, type => <<"int">>}, + #{name => <<"s">>, type => <<"string">>} + ] + }, + SourceBin = emqx_json:encode(Source), + #{type => avro, source => SourceBin}. + +assert_roundtrip(SerdeName, Original) -> + Encoded = emqx_ee_schema_registry_serde:encode(SerdeName, Original), + Decoded = emqx_ee_schema_registry_serde:decode(SerdeName, Encoded), + ?assertEqual(Original, Decoded, #{original => Original}). + +assert_roundtrip(SerdeName, Original, ArgsSerialize, ArgsDeserialize) -> + Encoded = emqx_ee_schema_registry_serde:encode(SerdeName, Original, ArgsSerialize), + Decoded = emqx_ee_schema_registry_serde:decode(SerdeName, Encoded, ArgsDeserialize), + ?assertEqual(Original, Decoded, #{original => Original}). + +%%------------------------------------------------------------------------------ +%% Testcases +%%------------------------------------------------------------------------------ + +t_roundtrip_avro(_Config) -> + SerdeName = my_serde, + Params = schema_params(avro), + ok = emqx_ee_schema_registry:add_schema(SerdeName, Params), + Original = #{<<"i">> => 10, <<"s">> => <<"hi">>}, + %% for coverage + assert_roundtrip(SerdeName, Original, _ArgsSerialize = [], _ArgsDeserialize = []), + assert_roundtrip(SerdeName, Original), + ok. + +t_avro_invalid_json_schema(_Config) -> + SerdeName = my_serde, + Params = schema_params(avro), + WrongParams = Params#{source := <<"{">>}, + ?assertMatch( + {error, #{reason := #{expected_type := _}}}, + emqx_ee_schema_registry:add_schema(SerdeName, WrongParams) + ), + ok. + +t_avro_invalid_schema(_Config) -> + SerdeName = my_serde, + Params = schema_params(avro), + WrongParams = Params#{source := <<"{}">>}, + ?assertMatch( + {error, {post_config_update, _, {not_found, <<"type">>}}}, + emqx_ee_schema_registry:add_schema(SerdeName, WrongParams) + ), + ok. + +t_serde_not_found(_Config) -> + %% for coverage + NonexistentSerde = <<"nonexistent">>, + Original = #{}, + ?assertError( + {serde_not_found, NonexistentSerde}, + emqx_ee_schema_registry_serde:encode(NonexistentSerde, Original) + ), + ?assertError( + {serde_not_found, NonexistentSerde}, + emqx_ee_schema_registry_serde:decode(NonexistentSerde, Original) + ), + ok. diff --git a/mix.exs b/mix.exs index 600218e52..712e1f4e2 100644 --- a/mix.exs +++ b/mix.exs @@ -83,6 +83,8 @@ defmodule EMQXUmbrella.MixProject do # in conflict by emqx and observer_cli {:recon, github: "ferd/recon", tag: "2.5.1", override: true}, {:jsx, github: "talentdeficit/jsx", tag: "v3.1.0", override: true}, + # in conflict by erlavro and rocketmq + {:jsone, github: "emqx/jsone", tag: "1.7.1", override: true}, # dependencies of dependencies; we choose specific refs to match # what rebar3 chooses. # in conflict by gun and emqtt @@ -307,7 +309,8 @@ defmodule EMQXUmbrella.MixProject do emqx_license: :permanent, emqx_ee_conf: :load, emqx_ee_connector: :permanent, - emqx_ee_bridge: :permanent + emqx_ee_bridge: :permanent, + emqx_ee_schema_registry: :permanent ], else: [] ) diff --git a/rebar.config b/rebar.config index 50a8124be..8ad8133fd 100644 --- a/rebar.config +++ b/rebar.config @@ -81,6 +81,8 @@ , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}} , {telemetry, "1.1.0"} , {hackney, {git, "https://github.com/emqx/hackney.git", {tag, "1.18.1-1"}}} + %% in conflict by erlavro and rocketmq + , {jsone, {git, "https://github.com/emqx/jsone.git", {tag, "1.7.1"}}} ]}. {xref_ignores, diff --git a/rebar.config.erl b/rebar.config.erl index 98cd30570..e00fe730d 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -422,7 +422,8 @@ relx_apps_per_edition(ee) -> emqx_license, {emqx_ee_conf, load}, emqx_ee_connector, - emqx_ee_bridge + emqx_ee_bridge, + emqx_ee_schema_registry ]; relx_apps_per_edition(ce) -> []. diff --git a/rel/i18n/emqx_bridge_api.hocon b/rel/i18n/emqx_bridge_api.hocon index f5d372128..66960619a 100644 --- a/rel/i18n/emqx_bridge_api.hocon +++ b/rel/i18n/emqx_bridge_api.hocon @@ -57,7 +57,7 @@ emqx_bridge_api { desc_api1 { desc { en: """List all created bridges""" - zh: """列出所有 Birdge""" + zh: """列出所有 Bridge""" } label: { en: "List All Bridges" diff --git a/rel/i18n/emqx_ee_schema_registry_http_api.hocon b/rel/i18n/emqx_ee_schema_registry_http_api.hocon new file mode 100644 index 000000000..058796a66 --- /dev/null +++ b/rel/i18n/emqx_ee_schema_registry_http_api.hocon @@ -0,0 +1,69 @@ +emqx_ee_schema_registry_http_api { + # apis + desc_schema_registry_api_list { + desc { + en: "List all registered schemas" + zh: "列出所有注册的模式" + } + label { + en: "List schemas" + zh: "列表模式" + } + } + + desc_schema_registry_api_get { + desc { + en: "Get a schema by its name" + zh: "通过名称获取模式" + } + label { + en: "Get schema" + zh: "获取模式" + } + } + + desc_schema_registry_api_post { + desc { + en: "Register a new schema" + zh: "注册一个新的模式" + } + label { + en: "Register schema" + zh: "注册模式" + } + } + + desc_schema_registry_api_put { + desc { + en: "Update an existing schema" + zh: "更新一个现有的模式" + } + label { + en: "Update schema" + zh: "更新模式" + } + } + + desc_schema_registry_api_delete { + desc { + en: "Delete a schema" + zh: "删除一个模式" + } + label { + en: "Delete schema" + zh: "删除模式" + } + } + + # params + desc_param_path_schema_name { + desc { + en: "The schema name" + zh: "模式名称" + } + label { + en: "Schema name" + zh: "模式名称" + } + } +} diff --git a/rel/i18n/emqx_ee_schema_registry_schema.hocon b/rel/i18n/emqx_ee_schema_registry_schema.hocon new file mode 100644 index 000000000..1538fe5f9 --- /dev/null +++ b/rel/i18n/emqx_ee_schema_registry_schema.hocon @@ -0,0 +1,78 @@ +emqx_ee_schema_registry_schema { + schema_registry_root { + desc { + en: "Schema registry configurations." + zh: "模式注册表的配置。" + } + label { + en: "Schema registry" + zh: "模式注册表" + } + } + + schema_registry_schemas { + desc { + en: "Registered schemas." + zh: "注册的模式。" + } + label { + en: "Registered schemas" + zh: "注册的模式" + } + } + + schema_name { + desc { + en: "A name for the schema that will serve as its identifier." + zh: "模式的一个名称，将作为其标识符。" + } + label { + en: "Schema name" + zh: "模式名称" + } + } + + schema_type { + desc { + en: "Schema type." + zh: "模式类型。" + } + label { + en: "Schema type" + zh: "模式类型" + } + } + + schema_source { + desc { + en: "Source text for the schema." + zh: "模式的源文本。" + } + label { + en: "Schema source" + zh: "模式来源" + } + } + + schema_description { + desc { + en: "A description for this schema." + zh: "对该模式的描述。" + } + label { + en: "Schema description" + zh: "模式描述" + } + } + + avro_type { + desc { + en: "[Apache Avro](https://avro.apache.org/) serialization format." + zh: "[阿帕奇-阿夫罗](https://avro.apache.org/) 序列化格式。" + } + label { + en: "Apache Avro" + zh: "阿帕奇-阿夫罗" + } + } +} diff --git a/scripts/spellcheck/dicts/emqx.txt b/scripts/spellcheck/dicts/emqx.txt index 79c8b7e3a..168275e1e 100644 --- a/scripts/spellcheck/dicts/emqx.txt +++ b/scripts/spellcheck/dicts/emqx.txt @@ -1,6 +1,7 @@ ACL AES APIs +Avro BPAPI BSON Backplane From 502cc2b8b8ea1bdf42ef34cefe6965357f64103d Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 7 Apr 2023 15:34:16 +0800 Subject: [PATCH 189/196] chore: fix common tests --- apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl | 1 + apps/emqx_gateway/test/emqx_gateway_cm_SUITE.erl | 1 + apps/emqx_gateway/test/emqx_gateway_cm_registry_SUITE.erl | 1 + apps/emqx_gateway/test/emqx_gateway_ctx_SUITE.erl | 1 + apps/emqx_gateway/test/emqx_gateway_metrics_SUITE.erl | 1 + apps/emqx_gateway/test/emqx_gateway_registry_SUITE.erl | 1 + 6 files changed, 6 insertions(+) diff --git a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl index bfcebd772..c5fabf2fd 100644 --- a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl @@ -46,6 +46,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Conf) -> application:load(emqx), + emqx_gateway_test_utils:load_all_gateway_apps(), emqx_config:delete_override_conf_files(), emqx_config:erase(gateway), emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), diff --git a/apps/emqx_gateway/test/emqx_gateway_cm_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_cm_SUITE.erl index c5e8d9a92..8b0dacd75 100644 --- a/apps/emqx_gateway/test/emqx_gateway_cm_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_cm_SUITE.erl @@ -34,6 +34,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Conf) -> emqx_config:erase(gateway), + emqx_gateway_test_utils:load_all_gateway_apps(), emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), emqx_common_test_helpers:start_apps([]), diff --git a/apps/emqx_gateway/test/emqx_gateway_cm_registry_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_cm_registry_SUITE.erl index 77f4058e7..35e32d3da 100644 --- a/apps/emqx_gateway/test/emqx_gateway_cm_registry_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_cm_registry_SUITE.erl @@ -34,6 +34,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Conf) -> emqx_config:erase(gateway), + emqx_gateway_test_utils:load_all_gateway_apps(), emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), emqx_common_test_helpers:start_apps([]), Conf. diff --git a/apps/emqx_gateway/test/emqx_gateway_ctx_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_ctx_SUITE.erl index 0aa3172f1..35ce5fb31 100644 --- a/apps/emqx_gateway/test/emqx_gateway_ctx_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_ctx_SUITE.erl @@ -28,6 +28,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Conf) -> + emqx_gateway_test_utils:load_all_gateway_apps(), ok = meck:new(emqx_access_control, [passthrough, no_history, no_link]), ok = meck:expect( emqx_access_control, diff --git a/apps/emqx_gateway/test/emqx_gateway_metrics_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_metrics_SUITE.erl index 211315e6c..b82e049d3 100644 --- a/apps/emqx_gateway/test/emqx_gateway_metrics_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_metrics_SUITE.erl @@ -33,6 +33,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Conf) -> emqx_config:erase(gateway), + emqx_gateway_test_utils:load_all_gateway_apps(), emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), emqx_common_test_helpers:start_apps([]), Conf. diff --git a/apps/emqx_gateway/test/emqx_gateway_registry_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_registry_SUITE.erl index cc5f7bf37..a51621688 100644 --- a/apps/emqx_gateway/test/emqx_gateway_registry_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_registry_SUITE.erl @@ -37,6 +37,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). %%-------------------------------------------------------------------- init_per_suite(Cfg) -> + emqx_gateway_test_utils:load_all_gateway_apps(), ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), emqx_common_test_helpers:start_apps([emqx_authn, emqx_gateway]), Cfg. From c2ca9089caf4bcec046461dc38e648b3ab21f452 Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Thu, 6 Apr 2023 18:29:44 +0200 Subject: [PATCH 190/196] docs(emqx_machine): Add readme --- apps/emqx_machine/README.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 apps/emqx_machine/README.md diff --git a/apps/emqx_machine/README.md b/apps/emqx_machine/README.md new file mode 100644 index 000000000..9ff33a5e5 --- /dev/null +++ b/apps/emqx_machine/README.md @@ -0,0 +1,5 @@ +# EMQX Machine + +This application manages other OTP applications in EMQX and serves as the entry point when BEAM VM starts up. +It prepares the node before starting mnesia/mria, as well as EMQX business logic. +It keeps track of the business applications storing data in Mnesia, which need to be restarted when the node joins the cluster by registering `ekka` callbacks. From 5a58dfc3a44d485010fd447cb6213340ba8f0f44 Mon Sep 17 00:00:00 2001 From: firest Date: Mon, 10 Apr 2023 14:39:13 +0800 Subject: [PATCH 191/196] chore: update README for auto subscribe --- apps/emqx_auto_subscribe/README.md | 57 ++++++++++++++++++++++++++---- 1 file changed, 51 insertions(+), 6 deletions(-) diff --git a/apps/emqx_auto_subscribe/README.md b/apps/emqx_auto_subscribe/README.md index 96d368715..981e4cb1f 100644 --- a/apps/emqx_auto_subscribe/README.md +++ b/apps/emqx_auto_subscribe/README.md @@ -1,9 +1,54 @@ -emqx_auto_subscribe -===== +# Auto Subscribe -An OTP application +This application can help clients automatically subscribe to topics compiled from user definitions when they connect, and the clients no longer need to send the MQTT `SUBSCRIBE ` request. -Build ------ +# How To Use - $ rebar3 compile +Add the following configuration items to the `emqx.conf` file + +```yaml +auto_subscribe { + topics = [ + { + topic = "c/${clientid}" + }, + { + topic = "client/${clientid}/username/${username}/host/${host}/port/${port}" + qos = 1 + rh = 0 + rap = 0 + nl = 0 + } + ] +} +``` + +This example defines two templates, all of which will be compiled into the final topic by replacing placeholders like `${clientid}` `${port}` with the actual values when the client connects. + +# Configuration + +## Configuration Definition + +| Field | Definition | Value Range | Default | +| -------------- | ----------------------------- | ----------------------------------------------------------- | ------- | +| auto_subscribe | Auto subscribe configuration | topics | topics | +| topics | Subscription Options | Subscription configurations list. See `Subscription Option` | [] | + +## Subscription Option + +| Field | Definition | Value Range | Default | +|-------|---------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------|------------------| +| topic | Required. Topic template. | String, placeholders supported | No default value | +| qos | Optional. Subscription QoS | 0 or 1 or 2. Refer to the MQTT QoS definition | 0 | +| rh | Optional. MQTT version 5.0. Whether to send retain message when a subscription is created. | 0: Not send the retain message
1: Send the retain message | 0 | +| rap | Optional. MQTT version 5.0. When forwarding messages, Whether to send with retain flag | 0: Set retain 0
1: Keep retain flag | 0 | +| nl | Optional. MQTT version 5.0. Whether the message can be forwarded to the client when published by itself | 0: Forwarded to self
1: Not forwarded to self | 0 | + +## Subscription Placeholders + +| Placeholder | Definition | +| ----------- | -------------------------------------- | +| ${clientid} | Client ID | +| ${username} | Client Username | +| ${ip} | Client TCP connection local IP address | +| ${port} | Client TCP connection local Port | From e282d361f8501685871cd5b417b0606d653554f3 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Tue, 11 Apr 2023 09:34:51 +0200 Subject: [PATCH 192/196] style: fix Chinese translations Co-authored-by: LenaLenaPan <120552185+LenaLenaPan@users.noreply.github.com> --- rel/i18n/emqx_rule_api_schema.hocon | 2 +- rel/i18n/emqx_rule_engine_api.hocon | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rel/i18n/emqx_rule_api_schema.hocon b/rel/i18n/emqx_rule_api_schema.hocon index 3c2b3e7e4..0d8253223 100644 --- a/rel/i18n/emqx_rule_api_schema.hocon +++ b/rel/i18n/emqx_rule_api_schema.hocon @@ -641,7 +641,7 @@ emqx_rule_api_schema { root_rule_engine { desc { en: "Rule engine configurations. This API can be used to change EMQX rule engine settings. But not for the rules. To list, create, or update rules, call the '/rules' API instead." - zh: "规则引擎配置。该 API 可用于查看和修改规则引擎相关的一些设置。但不可用于规则，请调用 '/rules' API 来对规则进行操作。" + zh: "规则引擎配置。该 API 可用于查看和修改规则引擎相关的一些设置。但不可用于规则，如需查看或修改规则，请调用 '/rules' API 进行操作。" } label: { en: "Rule engine configuration" diff --git a/rel/i18n/emqx_rule_engine_api.hocon b/rel/i18n/emqx_rule_engine_api.hocon index 7be0588c9..8a57f8e31 100644 --- a/rel/i18n/emqx_rule_engine_api.hocon +++ b/rel/i18n/emqx_rule_engine_api.hocon @@ -53,7 +53,7 @@ emqx_rule_engine_api { api1_resp { desc { en: "List of rules" - zh: "列出所有规则" + zh: "规则列表" } label: { en: "List Rules" @@ -122,7 +122,7 @@ emqx_rule_engine_api { } label: { en: "Delete Cluster Rule" - zh: "删除集群规则" + zh: "基于给定 ID 新建一条规则" } } api7 { From b48fb17f4a0dc2281b4e6904932e01137e6bd9e1 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Tue, 11 Apr 2023 10:20:25 +0200 Subject: [PATCH 193/196] fix: CHECK_PARAMS macro defines unused var --- .../src/emqx_rule_engine_api.erl | 37 ++++++++++++------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index b138d992e..b19816542 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -447,20 +447,12 @@ param_path_id() -> '/rule_engine'(get, _Params) -> {200, format_rule_engine_resp(emqx_conf:get([rule_engine]))}; '/rule_engine'(put, #{body := Params}) -> - ?CHECK_PARAMS( - Params, - rule_engine, - case emqx_conf:update([rule_engine], Params, #{override_to => cluster}) of - {ok, #{config := Config}} -> - {200, format_rule_engine_resp(Config)}; - {error, Reason} -> - ?SLOG(error, #{ - msg => "update_rule_engine_failed", - reason => Reason - }), - {400, #{code => 'BAD_REQUEST', message => ?ERR_BADARGS(Reason)}} - end - ). + case rule_engine_update(Params) of + {ok, Config} -> + {200, format_rule_engine_resp(Config)}; + {error, Reason} -> + {400, #{code => 'BAD_REQUEST', message => ?ERR_BADARGS(Reason)}} + end. %%------------------------------------------------------------------------------ %% Internal functions @@ -731,3 +723,20 @@ run_fuzzy_match(E = {_Id, #{from := Topics}}, [{from, like, Pattern} | Fuzzy]) - run_fuzzy_match(E, Fuzzy); run_fuzzy_match(E, [_ | Fuzzy]) -> run_fuzzy_match(E, Fuzzy). + +rule_engine_update(Params) -> + case emqx_rule_api_schema:check_params(Params, rule_engine) of + {ok, _CheckedParams} -> + case emqx_conf:update([rule_engine], Params, #{override_to => cluster}) of + {ok, #{config := Config}} -> + {ok, Config}; + {error, Reason} -> + ?SLOG(error, #{ + msg => "update_rule_engine_failed", + reason => Reason + }), + {error, Reason} + end; + {error, Reason} -> + {error, Reason} + end. From 8ef36f29ce252490b6e05188893f2dbd92ebaf48 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Tue, 11 Apr 2023 10:40:11 +0200 Subject: [PATCH 194/196] fix: add 'rule_engine' as possible tag() value for spec --- apps/emqx_rule_engine/src/emqx_rule_api_schema.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl index 8a8822044..c9926f56f 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl @@ -26,7 +26,7 @@ -export([roots/0, fields/1]). --type tag() :: rule_creation | rule_test. +-type tag() :: rule_creation | rule_test | rule_engine. -spec check_params(map(), tag()) -> {ok, map()} | {error, term()}. check_params(Params, Tag) -> From e6f8682c47535e2101379495e2ceb9961946b12b Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Tue, 11 Apr 2023 15:42:56 +0200 Subject: [PATCH 195/196] fix: ensure we don't return 'rules' in rule_engine --- .../src/emqx_rule_engine_api.erl | 44 +------ .../test/emqx_rule_engine_api_SUITE.erl | 120 +++++++++--------- 2 files changed, 69 insertions(+), 95 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index b19816542..f640f8303 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -475,8 +475,6 @@ encode_nested_error(RuleError, Reason) -> {RuleError, Reason} end. -format_rule_info_resp(Rules) when is_list(Rules) -> - [format_rule_info_resp(R) || R <- Rules]; format_rule_info_resp({Id, Rule}) -> format_rule_info_resp(Rule#{id => Id}); format_rule_info_resp(#{ @@ -500,32 +498,8 @@ format_rule_info_resp(#{ description => Descr }. -format_rule_engine_resp(#{rules := Rules} = Config) -> - Config#{rules => maps:map(fun format_rule_resp/2, Rules)}. - -format_rule_resp( - _Id, - #{ - name := Name, - actions := Action, - sql := SQL, - enable := Enable, - description := Descr - } = Rule -) -> - Format = #{ - name => Name, - actions => format_action(Action), - sql => SQL, - enable => Enable, - description => Descr - }, - case Rule of - #{metadata := MetaData = #{created_at := CreatedAt}} -> - Format#{metadata => MetaData#{created_at => format_datetime(CreatedAt, millisecond)}}; - _ -> - Format - end. +format_rule_engine_resp(Config) -> + maps:remove(rules, Config). format_datetime(Timestamp, Unit) -> list_to_binary(calendar:system_time_to_rfc3339(Timestamp, [{unit, Unit}])). @@ -727,16 +701,10 @@ run_fuzzy_match(E, [_ | Fuzzy]) -> rule_engine_update(Params) -> case emqx_rule_api_schema:check_params(Params, rule_engine) of {ok, _CheckedParams} -> - case emqx_conf:update([rule_engine], Params, #{override_to => cluster}) of - {ok, #{config := Config}} -> - {ok, Config}; - {error, Reason} -> - ?SLOG(error, #{ - msg => "update_rule_engine_failed", - reason => Reason - }), - {error, Reason} - end; + {ok, #{config := Config}} = emqx_conf:update([rule_engine], Params, #{ + override_to => cluster + }), + {ok, Config}; {error, Reason} -> {error, Reason} end. diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl index e59b5c6df..e94806a7b 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl @@ -23,6 +23,14 @@ -include_lib("common_test/include/ct.hrl"). -define(CONF_DEFAULT, <<"rule_engine {rules {}}">>). +-define(SIMPLE_RULE(NAME_SUFFIX), #{ + <<"description">> => <<"A simple rule">>, + <<"enable">> => true, + <<"actions">> => [#{<<"function">> => <<"console">>}], + <<"sql">> => <<"SELECT * from \"t/1\"">>, + <<"name">> => <<"test_rule", NAME_SUFFIX/binary>> +}). +-define(SIMPLE_RULE(ID, NAME_SUFFIX), ?SIMPLE_RULE(NAME_SUFFIX)#{<<"id">> => ID}). all() -> emqx_common_test_helpers:all(?MODULE). @@ -37,6 +45,9 @@ end_per_suite(_Config) -> emqx_common_test_helpers:stop_apps([emqx_conf, emqx_rule_engine]), ok. +init_per_testcase(t_crud_rule_api, Config) -> + meck:new(emqx_json, [passthrough]), + init_per_testcase(common, Config); init_per_testcase(_, Config) -> Config. @@ -48,7 +59,7 @@ end_per_testcase(_, _Config) -> emqx_rule_engine_api:'/rules'(get, #{query_string => #{}}), lists:foreach( fun(#{id := Id}) -> - emqx_rule_engine_api:'/rules/:id'( + {204} = emqx_rule_engine_api:'/rules/:id'( delete, #{bindings => #{id => Id}} ) @@ -57,45 +68,38 @@ end_per_testcase(_, _Config) -> ). t_crud_rule_api(_Config) -> - RuleID = <<"my_rule">>, - Params0 = #{ - <<"description">> => <<"A simple rule">>, - <<"enable">> => true, - <<"id">> => RuleID, - <<"actions">> => [#{<<"function">> => <<"console">>}], - <<"sql">> => <<"SELECT * from \"t/1\"">>, - <<"name">> => <<"test_rule">> - }, - {201, Rule} = emqx_rule_engine_api:'/rules'(post, #{body => Params0}), - %% if we post again with the same params, it return with 400 "rule id already exists" - ?assertMatch( - {400, #{code := _, message := _Message}}, - emqx_rule_engine_api:'/rules'(post, #{body => Params0}) - ), + RuleId = <<"my_rule">>, + Rule = simple_rule_fixture(RuleId, <<>>), + ?assertEqual(RuleId, maps:get(id, Rule)), - ?assertEqual(RuleID, maps:get(id, Rule)), {200, #{data := Rules}} = emqx_rule_engine_api:'/rules'(get, #{query_string => #{}}), ct:pal("RList : ~p", [Rules]), ?assert(length(Rules) > 0), + %% if we post again with the same id, it return with 400 "rule id already exists" + ?assertMatch( + {400, #{code := _, message := _Message}}, + emqx_rule_engine_api:'/rules'(post, #{body => ?SIMPLE_RULE(RuleId, <<"some_other">>)}) + ), + {204} = emqx_rule_engine_api:'/rules/:id/metrics/reset'(put, #{ - bindings => #{id => RuleID} + bindings => #{id => RuleId} }), - {200, Rule1} = emqx_rule_engine_api:'/rules/:id'(get, #{bindings => #{id => RuleID}}), + {200, Rule1} = emqx_rule_engine_api:'/rules/:id'(get, #{bindings => #{id => RuleId}}), ct:pal("RShow : ~p", [Rule1]), ?assertEqual(Rule, Rule1), - {200, Metrics} = emqx_rule_engine_api:'/rules/:id/metrics'(get, #{bindings => #{id => RuleID}}), + {200, Metrics} = emqx_rule_engine_api:'/rules/:id/metrics'(get, #{bindings => #{id => RuleId}}), ct:pal("RMetrics : ~p", [Metrics]), - ?assertMatch(#{id := RuleID, metrics := _, node_metrics := _}, Metrics), + ?assertMatch(#{id := RuleId, metrics := _, node_metrics := _}, Metrics), {200, Rule2} = emqx_rule_engine_api:'/rules/:id'(put, #{ - bindings => #{id => RuleID}, - body => Params0#{<<"sql">> => <<"select * from \"t/b\"">>} + bindings => #{id => RuleId}, + body => ?SIMPLE_RULE(RuleId)#{<<"sql">> => <<"select * from \"t/b\"">>} }), - {200, Rule3} = emqx_rule_engine_api:'/rules/:id'(get, #{bindings => #{id => RuleID}}), + {200, Rule3} = emqx_rule_engine_api:'/rules/:id'(get, #{bindings => #{id => RuleId}}), %ct:pal("RShow : ~p", [Rule3]), ?assertEqual(Rule3, Rule2), ?assertEqual(<<"select * from \"t/b\"">>, maps:get(sql, Rule3)), @@ -112,14 +116,14 @@ t_crud_rule_api(_Config) -> {204}, emqx_rule_engine_api:'/rules/:id'( delete, - #{bindings => #{id => RuleID}} + #{bindings => #{id => RuleId}} ) ), %ct:pal("Show After Deleted: ~p", [NotFound]), ?assertMatch( {404, #{code := _, message := _Message}}, - emqx_rule_engine_api:'/rules/:id'(get, #{bindings => #{id => RuleID}}) + emqx_rule_engine_api:'/rules/:id'(get, #{bindings => #{id => RuleId}}) ), {400, #{ @@ -174,30 +178,15 @@ t_crud_rule_api(_Config) -> ok. t_list_rule_api(_Config) -> - AddIds = - lists:map( - fun(Seq0) -> - Seq = integer_to_binary(Seq0), - Params = #{ - <<"description">> => <<"A simple rule">>, - <<"enable">> => true, - <<"actions">> => [#{<<"function">> => <<"console">>}], - <<"sql">> => <<"SELECT * from \"t/1\"">>, - <<"name">> => <<"test_rule", Seq/binary>> - }, - {201, #{id := Id}} = emqx_rule_engine_api:'/rules'(post, #{body => Params}), - Id - end, - lists:seq(1, 20) - ), - + AddIds = rules_fixture(20), + ct:pal("rule ids: ~p", [AddIds]), {200, #{data := Rules, meta := #{count := Count}}} = emqx_rule_engine_api:'/rules'(get, #{query_string => #{}}), ?assertEqual(20, length(AddIds)), ?assertEqual(20, length(Rules)), ?assertEqual(20, Count), - [RuleID | _] = AddIds, + [RuleId | _] = AddIds, UpdateParams = #{ <<"description">> => <<"中文的描述也能搜索"/utf8>>, <<"enable">> => false, @@ -206,7 +195,7 @@ t_list_rule_api(_Config) -> <<"name">> => <<"test_rule_update1">> }, {200, _Rule2} = emqx_rule_engine_api:'/rules/:id'(put, #{ - bindings => #{id => RuleID}, + bindings => #{id => RuleId}, body => UpdateParams }), QueryStr1 = #{query_string => #{<<"enable">> => false}}, @@ -229,20 +218,13 @@ t_list_rule_api(_Config) -> {200, Result5} = emqx_rule_engine_api:'/rules'(get, QueryStr5), ?assertEqual(maps:get(data, Result1), maps:get(data, Result5)), - QueryStr6 = #{query_string => #{<<"like_id">> => RuleID}}, + QueryStr6 = #{query_string => #{<<"like_id">> => RuleId}}, {200, Result6} = emqx_rule_engine_api:'/rules'(get, QueryStr6), ?assertEqual(maps:get(data, Result1), maps:get(data, Result6)), ok. t_reset_metrics_on_disable(_Config) -> - Params = #{ - <<"description">> => <<"A simple rule">>, - <<"enable">> => true, - <<"actions">> => [#{<<"function">> => <<"console">>}], - <<"sql">> => <<"SELECT * from \"t/1\"">>, - <<"name">> => atom_to_binary(?FUNCTION_NAME) - }, - {201, #{id := RuleId}} = emqx_rule_engine_api:'/rules'(post, #{body => Params}), + #{id := RuleId} = simple_rule_fixture(), %% generate some fake metrics emqx_metrics_worker:inc(rule_metrics, RuleId, 'matched', 10), @@ -256,7 +238,7 @@ t_reset_metrics_on_disable(_Config) -> %% disable the rule; metrics should be reset {200, _Rule2} = emqx_rule_engine_api:'/rules/:id'(put, #{ bindings => #{id => RuleId}, - body => Params#{<<"enable">> := false} + body => #{<<"enable">> => false} }), {200, #{metrics := Metrics1}} = emqx_rule_engine_api:'/rules/:id/metrics'( @@ -283,9 +265,10 @@ test_rule_params(Sql, Payload) -> }. t_rule_engine(_) -> - {200, _} = emqx_rule_engine_api:'/rule_engine'(get, foo), + _ = simple_rule_fixture(), + {200, Config} = emqx_rule_engine_api:'/rule_engine'(get, #{}), + ?assert(not maps:is_key(rules, Config)), {200, #{ - %, jq_function_default_timeout := 12000 % hidden! jq_implementation_module := jq_port }} = emqx_rule_engine_api:'/rule_engine'(put, #{ @@ -299,3 +282,26 @@ t_rule_engine(_) -> body => #{<<"rules">> => #{<<"some_rule">> => SomeRule}} }), {400, _} = emqx_rule_engine_api:'/rule_engine'(put, #{body => #{<<"something">> => <<"weird">>}}). + +rules_fixture(N) -> + lists:map( + fun(Seq0) -> + Seq = integer_to_binary(Seq0), + #{id := Id} = simple_rule_fixture(Seq), + Id + end, + lists:seq(1, N) + ). + +simple_rule_fixture() -> + simple_rule_fixture(<<>>). + +simple_rule_fixture(NameSuffix) -> + create_rule(?SIMPLE_RULE(NameSuffix)). + +simple_rule_fixture(Id, NameSuffix) -> + create_rule(?SIMPLE_RULE(Id, NameSuffix)). + +create_rule(Params) -> + {201, Rule} = emqx_rule_engine_api:'/rules'(post, #{body => Params}), + Rule. From 97b94b88825a0b1f7b4eda0dc7baa58df8cdfa15 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 4 Apr 2023 08:55:43 -0300 Subject: [PATCH 196/196] feat: implement schema registry for 5.0 (avro) Part of https://emqx.atlassian.net/browse/EMQX-9251 This ports part of the Schema Registry app from 4.x to 5.0. Here, only support for Avro is added. Subsequent PRs will follow to add support for other formats. --- lib-ee/emqx_ee_schema_registry/README.md | 12 +++++--- .../src/emqx_ee_schema_registry.erl | 21 +++++++++---- .../src/emqx_ee_schema_registry_http_api.erl | 30 +++++++++---------- .../src/emqx_ee_schema_registry_schema.erl | 2 +- 4 files changed, 40 insertions(+), 25 deletions(-) diff --git a/lib-ee/emqx_ee_schema_registry/README.md b/lib-ee/emqx_ee_schema_registry/README.md index 9f477208c..c1c409c7d 100644 --- a/lib-ee/emqx_ee_schema_registry/README.md +++ b/lib-ee/emqx_ee_schema_registry/README.md @@ -52,12 +52,12 @@ WHERE | | [Registry] +------v--------------v------+ REGISTER SCHEMA | | - -------------------> | +--------+ - | | | | + INSTANCE | | +--------+ + -------------------> | | | [Management APIs] | Schema Registry ------ Schema | | | | | - -------------------> | +--------+ - LOAD PARSERS | | + | | +--------+ + | | +----------------------------+ / | \ +---/---+ +---|----+ +---\---+ @@ -67,3 +67,7 @@ WHERE +-------+ +--------+ +-------+ ``` + +- Register schema instance: adds a new instance of a schema of a + certain type. For example, when the user may have several Avro or + Protobuf schemas that they wish to use with different data flows. diff --git a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.erl b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.erl index 436777e9f..3569b246e 100644 --- a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.erl +++ b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.erl @@ -17,6 +17,7 @@ get_serde/1, add_schema/2, + get_schema/1, delete_schema/1, list_schemas/0 ]). @@ -26,6 +27,7 @@ init/1, handle_call/3, handle_cast/2, + handle_continue/2, terminate/2 ]). @@ -54,6 +56,15 @@ get_serde(SchemaName) -> {ok, serde_to_map(Serde)} end. +-spec get_schema(schema_name()) -> {ok, map()} | {error, not_found}. +get_schema(SchemaName) -> + case emqx_config:get([?CONF_KEY_ROOT, schemas, SchemaName], undefined) of + undefined -> + {error, not_found}; + Config -> + {ok, Config} + end. + -spec add_schema(schema_name(), schema()) -> ok | {error, term()}. add_schema(Name, Schema) -> RawSchema = emqx_map_lib:binary_key_map(Schema), @@ -130,9 +141,12 @@ init(_) -> process_flag(trap_exit, true), create_tables(), Schemas = emqx_conf:get([?CONF_KEY_ROOT, schemas], #{}), - async_build_serdes(Schemas), State = #{}, - {ok, State}. + {ok, State, {continue, {build_serdes, Schemas}}}. + +handle_continue({build_serdes, Schemas}, State) -> + do_build_serdes(Schemas), + {noreply, State}. handle_call(_Call, _From, State) -> {reply, {error, unknown_call}, State}. @@ -223,9 +237,6 @@ ensure_serde_absent(Name) -> ok end. -async_build_serdes(Schemas) -> - gen_server:cast(?MODULE, {build_serdes, Schemas}). - async_delete_serdes(Names) -> gen_server:cast(?MODULE, {delete_serdes, Names}). diff --git a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_http_api.erl b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_http_api.erl index fca66a0b1..897d29e07 100644 --- a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_http_api.erl +++ b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_http_api.erl @@ -141,44 +141,44 @@ schema("/schema_registry/:name") -> ?OK(Response); '/schema_registry'(post, #{body := Params0 = #{<<"name">> := Name}}) -> Params = maps:without([<<"name">>], Params0), - case emqx_config:get([?CONF_KEY_ROOT, schemas, Name], undefined) of - undefined -> + case emqx_ee_schema_registry:get_schema(Name) of + {error, not_found} -> case emqx_ee_schema_registry:add_schema(Name, Params) of ok -> - Res = emqx_config:get([?CONF_KEY_ROOT, schemas, Name]), + {ok, Res} = emqx_ee_schema_registry:get_schema(Name), {201, Res#{name => Name}}; {error, Error} -> ?BAD_REQUEST(Error) end; - _ -> + {ok, _} -> ?BAD_REQUEST('ALREADY_EXISTS', <<"Schema already exists">>) end. '/schema_registry/:name'(get, #{bindings := #{name := Name}}) -> - case emqx_config:get([?CONF_KEY_ROOT, schemas, Name], undefined) of - undefined -> + case emqx_ee_schema_registry:get_schema(Name) of + {error, not_found} -> ?NOT_FOUND(<<"Schema not found">>); - Res -> - ?OK(Res#{name => Name}) + {ok, Schema} -> + ?OK(Schema#{name => Name}) end; '/schema_registry/:name'(put, #{bindings := #{name := Name}, body := Params}) -> - case emqx_config:get([?CONF_KEY_ROOT, schemas, Name], undefined) of - undefined -> + case emqx_ee_schema_registry:get_schema(Name) of + {error, not_found} -> ?NOT_FOUND(<<"Schema not found">>); - _ -> + {ok, _} -> case emqx_ee_schema_registry:add_schema(Name, Params) of ok -> - Res = emqx_config:get([?CONF_KEY_ROOT, schemas, Name]), + {ok, Res} = emqx_ee_schema_registry:get_schema(Name), ?OK(Res#{name => Name}); {error, Error} -> ?BAD_REQUEST(Error) end end; '/schema_registry/:name'(delete, #{bindings := #{name := Name}}) -> - case emqx_config:get([?CONF_KEY_ROOT, schemas, Name], undefined) of - undefined -> + case emqx_ee_schema_registry:get_schema(Name) of + {error, not_found} -> ?NOT_FOUND(<<"Schema not found">>); - _ -> + {ok, _} -> case emqx_ee_schema_registry:delete_schema(Name) of ok -> ?NO_CONTENT; diff --git a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_schema.erl b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_schema.erl index 01177345a..bcdc63166 100644 --- a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_schema.erl +++ b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_schema.erl @@ -48,7 +48,7 @@ fields(?CONF_KEY_ROOT) -> ]; fields(avro) -> [ - {type, mk(hoconsc:enum([avro]), #{required => true, desc => ?DESC("schema_type")})}, + {type, mk(avro, #{required => true, desc => ?DESC("schema_type")})}, {source, mk(emqx_schema:json_binary(), #{required => true, desc => ?DESC("schema_source")})}, {description, mk(binary(), #{default => <<>>, desc => ?DESC("schema_description")})}