diff --git a/.ci/docker-compose-file/.env b/.ci/docker-compose-file/.env index 9675937d8..1d602e5bd 100644 --- a/.ci/docker-compose-file/.env +++ b/.ci/docker-compose-file/.env @@ -9,4 +9,7 @@ DYNAMO_TAG=1.21.0 CASSANDRA_TAG=3.11.6 MINIO_TAG=RELEASE.2023-03-20T20-16-18Z +MS_IMAGE_ADDR=mcr.microsoft.com/mssql/server +SQLSERVER_TAG=2019-CU19-ubuntu-20.04 + TARGET=emqx/emqx diff --git a/.ci/docker-compose-file/docker-compose-sqlserver.yaml b/.ci/docker-compose-file/docker-compose-sqlserver.yaml new file mode 100644 index 000000000..63fcfeecd --- /dev/null +++ b/.ci/docker-compose-file/docker-compose-sqlserver.yaml @@ -0,0 +1,19 @@ +version: '3.9' + +services: + sql_server: + container_name: sqlserver + # See also: + # https://mcr.microsoft.com/en-us/product/mssql/server/about + # https://hub.docker.com/_/microsoft-mssql-server + image: ${MS_IMAGE_ADDR}:${SQLSERVER_TAG} + environment: + # See also: + # https://learn.microsoft.com/en-us/sql/linux/sql-server-linux-configure-environment-variables + ACCEPT_EULA: "Y" + MSSQL_SA_PASSWORD: "mqtt_public1" + restart: always + # ports: + # - "1433:1433" + networks: + - emqx_bridge diff --git a/.ci/docker-compose-file/docker-compose-toxiproxy.yaml b/.ci/docker-compose-file/docker-compose-toxiproxy.yaml index 0cf689921..ea5c099dc 100644 --- a/.ci/docker-compose-file/docker-compose-toxiproxy.yaml +++ b/.ci/docker-compose-file/docker-compose-toxiproxy.yaml @@ -19,6 +19,8 @@ services: - 8086:8086 # InfluxDB TLS - 8087:8087 + # SQL Server + - 11433:1433 # MySQL - 13306:3306 # MySQL TLS diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml index 48d900400..6f4b7c04b 100644 --- a/.ci/docker-compose-file/docker-compose.yaml +++ b/.ci/docker-compose-file/docker-compose.yaml @@ -24,6 +24,7 @@ services: - /tmp/emqx-ci/emqx-shared-secret:/var/lib/secret - ./kerberos/krb5.conf:/etc/kdc/krb5.conf - ./kerberos/krb5.conf:/etc/krb5.conf + # - ./odbc/odbcinst.ini:/etc/odbcinst.ini working_dir: /emqx tty: true user: "${DOCKER_USER:-root}" diff --git a/.ci/docker-compose-file/odbc/odbcinst.ini b/.ci/docker-compose-file/odbc/odbcinst.ini new file mode 100644 index 000000000..dd0241543 --- /dev/null +++ b/.ci/docker-compose-file/odbc/odbcinst.ini @@ -0,0 +1,9 @@ +[ms-sql] +Description=Microsoft ODBC Driver 17 for SQL Server +Driver=/opt/microsoft/msodbcsql17/lib64/libmsodbcsql-17.10.so.2.1 +UsageCount=1 + +[ODBC Driver 17 for SQL Server] +Description=Microsoft ODBC Driver 17 for SQL Server +Driver=/opt/microsoft/msodbcsql17/lib64/libmsodbcsql-17.10.so.2.1 +UsageCount=1 diff --git a/.ci/docker-compose-file/toxiproxy.json b/.ci/docker-compose-file/toxiproxy.json index d8dd8a166..3934df0df 100644 --- a/.ci/docker-compose-file/toxiproxy.json +++ b/.ci/docker-compose-file/toxiproxy.json @@ -96,6 +96,12 @@ "upstream": "cassandra:9142", "enabled": true }, + { + "name": "sqlserver", + "listen": "0.0.0.0:1433", + "upstream": "sqlserver:1433", + "enabled": true + }, { "name": "minio_tcp", "listen": "0.0.0.0:19000", diff --git a/.github/workflows/build_and_push_docker_images.yaml b/.github/workflows/build_and_push_docker_images.yaml index 7391adb5c..7c0a5dc87 100644 --- a/.github/workflows/build_and_push_docker_images.yaml +++ b/.github/workflows/build_and_push_docker_images.yaml @@ -25,7 +25,7 @@ jobs: prepare: runs-on: ubuntu-22.04 # prepare source with any OTP version, no need for a matrix - container: "ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-24.3.4.2-3-ubuntu22.04" + container: "ghcr.io/emqx/emqx-builder/5.0-34:1.13.4-24.3.4.2-3-ubuntu22.04" outputs: PROFILE: ${{ steps.get_profile.outputs.PROFILE }} @@ -121,7 +121,7 @@ jobs: # NOTE: 'otp' and 'elixir' are to configure emqx-builder image # only support latest otp and elixir, not a matrix builder: - - 5.0-33 # update to latest + - 5.0-34 # update to latest otp: - 24.3.4.2-3 # switch to 25 once ready to release 5.1 elixir: diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 2afe23f67..60b4c1b84 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -24,7 +24,7 @@ jobs: prepare: runs-on: ubuntu-22.04 if: (github.repository_owner == 'emqx' && github.event_name == 'schedule') || github.event_name != 'schedule' - container: ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-24.3.4.2-3-ubuntu22.04 + container: ghcr.io/emqx/emqx-builder/5.0-34:1.13.4-24.3.4.2-3-ubuntu22.04 outputs: BUILD_PROFILE: ${{ steps.get_profile.outputs.BUILD_PROFILE }} IS_EXACT_TAG: ${{ steps.get_profile.outputs.IS_EXACT_TAG }} @@ -221,7 +221,7 @@ jobs: - aws-arm64 - ubuntu-22.04 builder: - - 5.0-33 + - 5.0-34 elixir: - 1.13.4 exclude: @@ -235,7 +235,7 @@ jobs: arch: amd64 os: ubuntu22.04 build_machine: ubuntu-22.04 - builder: 5.0-33 + builder: 5.0-34 elixir: 1.13.4 release_with: elixir - profile: emqx @@ -243,7 +243,7 @@ jobs: arch: amd64 os: amzn2 build_machine: ubuntu-22.04 - builder: 5.0-33 + builder: 5.0-34 elixir: 1.13.4 release_with: elixir diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index d6e4fc961..9ae5ba944 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -35,7 +35,7 @@ jobs: - ["emqx-enterprise", "24.3.4.2-3", "amzn2", "erlang"] - ["emqx-enterprise", "25.1.2-3", "ubuntu20.04", "erlang"] builder: - - 5.0-33 + - 5.0-34 elixir: - '1.13.4' diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml index 62dfa24ef..52ebf9efc 100644 --- a/.github/workflows/check_deps_integrity.yaml +++ b/.github/workflows/check_deps_integrity.yaml @@ -6,7 +6,7 @@ on: jobs: check_deps_integrity: runs-on: ubuntu-22.04 - container: ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-25.1.2-3-ubuntu22.04 + container: ghcr.io/emqx/emqx-builder/5.0-34:1.13.4-25.1.2-3-ubuntu22.04 steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/code_style_check.yaml b/.github/workflows/code_style_check.yaml index 97c6b0c88..1508cdd6e 100644 --- a/.github/workflows/code_style_check.yaml +++ b/.github/workflows/code_style_check.yaml @@ -5,7 +5,7 @@ on: [pull_request] jobs: code_style_check: runs-on: ubuntu-22.04 - container: "ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-25.1.2-3-ubuntu22.04" + container: "ghcr.io/emqx/emqx-builder/5.0-34:1.13.4-25.1.2-3-ubuntu22.04" steps: - uses: actions/checkout@v3 with: diff --git a/.github/workflows/elixir_apps_check.yaml b/.github/workflows/elixir_apps_check.yaml index 247f67a8f..7e942f3f3 100644 --- a/.github/workflows/elixir_apps_check.yaml +++ b/.github/workflows/elixir_apps_check.yaml @@ -9,7 +9,7 @@ jobs: elixir_apps_check: runs-on: ubuntu-22.04 # just use the latest builder - container: "ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-25.1.2-3-ubuntu22.04" + container: "ghcr.io/emqx/emqx-builder/5.0-34:1.13.4-25.1.2-3-ubuntu22.04" strategy: fail-fast: false diff --git a/.github/workflows/elixir_deps_check.yaml b/.github/workflows/elixir_deps_check.yaml index 511639a3c..e967c186b 100644 --- a/.github/workflows/elixir_deps_check.yaml +++ b/.github/workflows/elixir_deps_check.yaml @@ -8,7 +8,7 @@ on: jobs: elixir_deps_check: runs-on: ubuntu-22.04 - container: ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-25.1.2-3-ubuntu22.04 + container: ghcr.io/emqx/emqx-builder/5.0-34:1.13.4-25.1.2-3-ubuntu22.04 steps: - name: Checkout @@ -23,7 +23,18 @@ jobs: mix local.hex --force mix local.rebar --force mix deps.get + # we check only enterprise because `rebar3 tree`, even if an + # enterprise app is excluded from `project_app_dirs` in + # `rebar.config.erl`, will still list dependencies from it. + # Since the enterprise profile is a superset of the + # community one and thus more complete, we use the former. + env: + MIX_ENV: emqx-enterprise + PROFILE: emqx-enterprise - name: check elixir deps run: ./scripts/check-elixir-deps-discrepancies.exs + env: + MIX_ENV: emqx-enterprise + PROFILE: emqx-enterprise ... diff --git a/.github/workflows/elixir_release.yml b/.github/workflows/elixir_release.yml index 7bd6102ff..9a916d332 100644 --- a/.github/workflows/elixir_release.yml +++ b/.github/workflows/elixir_release.yml @@ -17,7 +17,7 @@ jobs: profile: - emqx - emqx-enterprise - container: ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-25.1.2-3-ubuntu22.04 + container: ghcr.io/emqx/emqx-builder/5.0-34:1.13.4-25.1.2-3-ubuntu22.04 steps: - name: Checkout uses: actions/checkout@v3 diff --git a/.github/workflows/run_emqx_app_tests.yaml b/.github/workflows/run_emqx_app_tests.yaml index 0a15f6c0b..551d0d9e6 100644 --- a/.github/workflows/run_emqx_app_tests.yaml +++ b/.github/workflows/run_emqx_app_tests.yaml @@ -12,7 +12,7 @@ jobs: strategy: matrix: builder: - - 5.0-33 + - 5.0-34 otp: - 24.3.4.2-3 - 25.1.2-3 diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index bb5aa4a1a..1bb0486f1 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -17,7 +17,7 @@ jobs: prepare: runs-on: ubuntu-22.04 # prepare source with any OTP version, no need for a matrix - container: ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-24.3.4.2-3-debian11 + container: ghcr.io/emqx/emqx-builder/5.0-34:1.13.4-24.3.4.2-3-debian11 steps: - uses: actions/checkout@v3 @@ -50,7 +50,7 @@ jobs: os: - ["debian11", "debian:11-slim"] builder: - - 5.0-33 + - 5.0-34 otp: - 24.3.4.2-3 elixir: @@ -123,7 +123,7 @@ jobs: os: - ["debian11", "debian:11-slim"] builder: - - 5.0-33 + - 5.0-34 otp: - 24.3.4.2-3 elixir: diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml index 8727f4d9d..2ad7c3345 100644 --- a/.github/workflows/run_relup_tests.yaml +++ b/.github/workflows/run_relup_tests.yaml @@ -15,7 +15,7 @@ concurrency: jobs: relup_test_plan: runs-on: ubuntu-22.04 - container: "ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-24.3.4.2-3-ubuntu22.04" + container: "ghcr.io/emqx/emqx-builder/5.0-34:1.13.4-24.3.4.2-3-ubuntu22.04" outputs: CUR_EE_VSN: ${{ steps.find-versions.outputs.CUR_EE_VSN }} OLD_VERSIONS: ${{ steps.find-versions.outputs.OLD_VERSIONS }} diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 3360bec6c..39c755053 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -31,12 +31,12 @@ jobs: MATRIX="$(echo "${APPS}" | jq -c ' [ (.[] | select(.profile == "emqx") | . + { - builder: "5.0-33", + builder: "5.0-34", otp: "25.1.2-3", elixir: "1.13.4" }), (.[] | select(.profile == "emqx-enterprise") | . + { - builder: "5.0-33", + builder: "5.0-34", otp: ["24.3.4.2-3", "25.1.2-3"][], elixir: "1.13.4" }) @@ -231,7 +231,7 @@ jobs: - ct - ct_docker runs-on: ubuntu-22.04 - container: "ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-24.3.4.2-3-ubuntu22.04" + container: "ghcr.io/emqx/emqx-builder/5.0-34:1.13.4-24.3.4.2-3-ubuntu22.04" steps: - uses: AutoModality/action-clean@v1 - uses: actions/download-artifact@v3 diff --git a/Makefile b/Makefile index a5adf0e0a..10e6d1424 100644 --- a/Makefile +++ b/Makefile @@ -6,8 +6,8 @@ export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2 export EMQX_DEFAULT_RUNNER = debian:11-slim export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh) -export EMQX_DASHBOARD_VERSION ?= v1.2.0 -export EMQX_EE_DASHBOARD_VERSION ?= e1.0.5 +export EMQX_DASHBOARD_VERSION ?= v1.2.3 +export EMQX_EE_DASHBOARD_VERSION ?= e1.0.6-beta.1 export EMQX_REL_FORM ?= tgz export QUICER_DOWNLOAD_FROM_RELEASE = 1 ifeq ($(OS),Windows_NT) @@ -89,12 +89,17 @@ APPS=$(shell $(SCRIPTS)/find-apps.sh) .PHONY: $(APPS:%=%-ct) define gen-app-ct-target $1-ct: $(REBAR) - @$(SCRIPTS)/pre-compile.sh $(PROFILE) - @ENABLE_COVER_COMPILE=1 $(REBAR) ct -c -v \ - --readable=$(CT_READABLE) \ - --name $(CT_NODE_NAME) \ - --cover_export_name $(CT_COVER_EXPORT_PREFIX)-$(subst /,-,$1) \ - --suite $(shell $(SCRIPTS)/find-suites.sh $1) + $(eval SUITES := $(shell $(SCRIPTS)/find-suites.sh $1)) +ifneq ($(SUITES),) + @$(SCRIPTS)/pre-compile.sh $(PROFILE) + @ENABLE_COVER_COMPILE=1 $(REBAR) ct -c -v \ + --readable=$(CT_READABLE) \ + --name $(CT_NODE_NAME) \ + --cover_export_name $(CT_COVER_EXPORT_PREFIX)-$(subst /,-,$1) \ + --suite $(SUITES) +else + @echo 'No suites found for $1' +endif endef $(foreach app,$(APPS),$(eval $(call gen-app-ct-target,$(app)))) @@ -239,7 +244,6 @@ $(foreach zt,$(ALL_DOCKERS),$(eval $(call gen-docker-target,$(zt)))) .PHONY: merge-config: @$(SCRIPTS)/merge-config.escript - @$(SCRIPTS)/merge-i18n.escript ## elixir target is to create release packages using Elixir's Mix .PHONY: $(REL_PROFILES:%=%-elixir) $(PKG_PROFILES:%=%-elixir) diff --git a/apps/emqx/etc/ssl_dist.conf b/apps/emqx/etc/ssl_dist.conf index af1c7506d..b4c16e2cc 100644 --- a/apps/emqx/etc/ssl_dist.conf +++ b/apps/emqx/etc/ssl_dist.conf @@ -1,4 +1,4 @@ -%% This additional config file is used when the config 'cluster.proto_dis' in emqx.conf is set to 'inet_tls'. +%% This additional config file is used when the config 'cluster.proto_dist' in emqx.conf is set to 'inet_tls'. %% Which means the EMQX nodes will connect to each other over TLS. %% For more information about inter-broker security, see: https://docs.emqx.com/en/enterprise/v5.0/deploy/cluster/security.html diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index 76920928b..ec3883c77 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -32,10 +32,10 @@ %% `apps/emqx/src/bpapi/README.md' %% Community edition --define(EMQX_RELEASE_CE, "5.0.21"). +-define(EMQX_RELEASE_CE, "5.0.23"). %% Enterprise edition --define(EMQX_RELEASE_EE, "5.0.2-rc.4"). +-define(EMQX_RELEASE_EE, "5.0.3-alpha.1"). %% the HTTP API version -define(EMQX_API_VERSION, "5.0"). diff --git a/apps/emqx/src/emqx_trace/emqx_trace.hrl b/apps/emqx/include/emqx_trace.hrl similarity index 93% rename from apps/emqx/src/emqx_trace/emqx_trace.hrl rename to apps/emqx/include/emqx_trace.hrl index 096e786dd..62028bcc0 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace.hrl +++ b/apps/emqx/include/emqx_trace.hrl @@ -24,6 +24,8 @@ filter :: emqx_types:topic() | emqx_types:clientid() | emqx_trace:ip_address() | undefined | '_', enable = true :: boolean() | '_', + payload_encode = text :: hex | text | hidden | '_', + extra = #{} :: map() | '_', start_at :: integer() | undefined | '_', end_at :: integer() | undefined | '_' }). diff --git a/apps/emqx/include/http_api.hrl b/apps/emqx/include/http_api.hrl index 08dd08362..ba1438374 100644 --- a/apps/emqx/include/http_api.hrl +++ b/apps/emqx/include/http_api.hrl @@ -57,16 +57,16 @@ -define(ERROR_CODES, [ {?BAD_USERNAME_OR_PWD, <<"Bad username or password">>}, {?BAD_API_KEY_OR_SECRET, <<"Bad API key or secret">>}, - {'BAD_REQUEST', <<"Request parameters are not legal">>}, + {'BAD_REQUEST', <<"Request parameters are invalid">>}, {'NOT_MATCH', <<"Conditions are not matched">>}, {'ALREADY_EXISTS', <<"Resource already existed">>}, - {'BAD_CONFIG_SCHEMA', <<"Configuration data is not legal">>}, + {'BAD_CONFIG_SCHEMA', <<"Configuration data is invalid">>}, {'BAD_LISTENER_ID', <<"Bad listener ID">>}, {'BAD_NODE_NAME', <<"Bad Node Name">>}, {'BAD_RPC', <<"RPC Failed. Check the cluster status and the requested node status">>}, {'BAD_TOPIC', <<"Topic syntax error, Topic needs to comply with the MQTT protocol standard">>}, {'EXCEED_LIMIT', <<"Create resources that exceed the maximum limit or minimum limit">>}, - {'INVALID_PARAMETER', <<"Request parameters is not legal and exceeds the boundary value">>}, + {'INVALID_PARAMETER', <<"Request parameters is invalid and exceeds the boundary value">>}, {'CONFLICT', <<"Conflicting request resources">>}, {'NO_DEFAULT_VALUE', <<"Request parameters do not use default values">>}, {'DEPENDENCY_EXISTS', <<"Resource is dependent by another resource">>}, diff --git a/apps/emqx/priv/bpapi.versions b/apps/emqx/priv/bpapi.versions index d597646d6..975424030 100644 --- a/apps/emqx/priv/bpapi.versions +++ b/apps/emqx/priv/bpapi.versions @@ -1,10 +1,12 @@ %% This file is automatically generated by `make static_checks`, do not edit. {emqx,1}. +{emqx,2}. {emqx_authn,1}. {emqx_authz,1}. {emqx_bridge,1}. {emqx_bridge,2}. {emqx_bridge,3}. +{emqx_bridge,4}. {emqx_broker,1}. {emqx_cm,1}. {emqx_cm,2}. diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 9079322eb..6788b4f40 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -18,25 +18,25 @@ ]}. %% Deps here may duplicate with emqx.git root level rebar.config -%% but there not be any descrpancy. +%% but there may not be any discrepancy. %% This rebar.config is necessary because the app may be used as a %% `git_subdir` dependency in other projects. {deps, [ + {emqx_utils, {path, "../emqx_utils"}}, {lc, {git, "https://github.com/emqx/lc.git", {tag, "0.3.2"}}}, {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}}, - {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}, {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}}, {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}}, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.14.6"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}, - {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.38.0"}}}, + {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.2"}}}, {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}, {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.7"}}} ]}. -{plugins, [{rebar3_proper, "0.12.1"}]}. +{plugins, [{rebar3_proper, "0.12.1"}, rebar3_path_deps]}. {extra_src_dirs, [{"etc", [recursive]}]}. {profiles, [ {test, [ diff --git a/apps/emqx/src/config/emqx_config_logger.erl b/apps/emqx/src/config/emqx_config_logger.erl index 15e4d3959..b0fc1ca67 100644 --- a/apps/emqx/src/config/emqx_config_logger.erl +++ b/apps/emqx/src/config/emqx_config_logger.erl @@ -32,25 +32,15 @@ remove_handler() -> ok = emqx_config_handler:remove_handler(?LOG), ok. -%% refresh logger config when booting, the override config may have changed after node start. +%% refresh logger config when booting, the cluster config may have changed after node start. %% Kernel's app env is confirmed before the node starts, -%% but we only copy cluster-override.conf from other node after this node starts, +%% but we only copy cluster.conf from other node after this node starts, %% so we need to refresh the logger config after this node starts. -%% It will not affect the logger config when cluster-override.conf is unchanged. +%% It will not affect the logger config when cluster.conf is unchanged. refresh_config() -> - Overrides = emqx_config:read_override_confs(), - refresh_config(Overrides). - -refresh_config(#{<<"log">> := _}) -> %% read the checked config LogConfig = emqx:get_config(?LOG, undefined), - Conf = #{log => LogConfig}, - ok = do_refresh_config(Conf); -refresh_config(_) -> - %% No config override found for 'log', do nothing - %% because the 'kernel' app should already be configured - %% from the base configs. i.e. emqx.conf + env vars - ok. + do_refresh_config(#{log => LogConfig}). %% this call is shared between initial config refresh at boot %% and dynamic config update from HTTP API @@ -61,10 +51,9 @@ do_refresh_config(Conf) -> ok = maybe_update_log_level(Level), ok. +%% always refresh config when the override config is changed post_config_update(?LOG, _Req, NewConf, _OldConf, _AppEnvs) -> - ok = do_refresh_config(#{log => NewConf}); -post_config_update(_ConfPath, _Req, _NewConf, _OldConf, _AppEnvs) -> - ok. + do_refresh_config(#{log => NewConf}). maybe_update_log_level(NewLevel) -> OldLevel = emqx_logger:get_primary_log_level(), diff --git a/apps/emqx/src/emqx.app.src b/apps/emqx/src/emqx.app.src index 1cecd7b61..d42478fea 100644 --- a/apps/emqx/src/emqx.app.src +++ b/apps/emqx/src/emqx.app.src @@ -3,7 +3,7 @@ {id, "emqx"}, {description, "EMQX Core"}, % strict semver, bump manually! - {vsn, "5.0.21"}, + {vsn, "5.0.24"}, {modules, []}, {registered, []}, {applications, [ @@ -16,7 +16,6 @@ cowboy, sasl, os_mon, - jiffy, lc, hocon ]}, diff --git a/apps/emqx/src/emqx.erl b/apps/emqx/src/emqx.erl index 6e4aa9922..ffee5fba7 100644 --- a/apps/emqx/src/emqx.erl +++ b/apps/emqx/src/emqx.erl @@ -30,6 +30,12 @@ stop/0 ]). +%% Cluster API +-export([ + cluster_nodes/1, + running_nodes/0 +]). + %% PubSub API -export([ subscribe/1, @@ -102,6 +108,18 @@ is_running() -> _ -> true end. +%%-------------------------------------------------------------------- +%% Cluster API +%%-------------------------------------------------------------------- + +-spec running_nodes() -> [node()]. +running_nodes() -> + mria:running_nodes(). + +-spec cluster_nodes(all | running | cores | stopped) -> [node()]. +cluster_nodes(Type) -> + mria:cluster_nodes(Type). + %%-------------------------------------------------------------------- %% PubSub API %%-------------------------------------------------------------------- @@ -164,29 +182,29 @@ run_hook(HookPoint, Args) -> run_fold_hook(HookPoint, Args, Acc) -> emqx_hooks:run_fold(HookPoint, Args, Acc). --spec get_config(emqx_map_lib:config_key_path()) -> term(). +-spec get_config(emqx_utils_maps:config_key_path()) -> term(). get_config(KeyPath) -> emqx_config:get(KeyPath). --spec get_config(emqx_map_lib:config_key_path(), term()) -> term(). +-spec get_config(emqx_utils_maps:config_key_path(), term()) -> term(). get_config(KeyPath, Default) -> emqx_config:get(KeyPath, Default). --spec get_raw_config(emqx_map_lib:config_key_path()) -> term(). +-spec get_raw_config(emqx_utils_maps:config_key_path()) -> term(). get_raw_config(KeyPath) -> emqx_config:get_raw(KeyPath). --spec get_raw_config(emqx_map_lib:config_key_path(), term()) -> term(). +-spec get_raw_config(emqx_utils_maps:config_key_path(), term()) -> term(). get_raw_config(KeyPath, Default) -> emqx_config:get_raw(KeyPath, Default). --spec update_config(emqx_map_lib:config_key_path(), emqx_config:update_request()) -> +-spec update_config(emqx_utils_maps:config_key_path(), emqx_config:update_request()) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. update_config(KeyPath, UpdateReq) -> update_config(KeyPath, UpdateReq, #{}). -spec update_config( - emqx_map_lib:config_key_path(), + emqx_utils_maps:config_key_path(), emqx_config:update_request(), emqx_config:update_opts() ) -> @@ -198,12 +216,12 @@ update_config([RootName | _] = KeyPath, UpdateReq, Opts) -> {{update, UpdateReq}, Opts} ). --spec remove_config(emqx_map_lib:config_key_path()) -> +-spec remove_config(emqx_utils_maps:config_key_path()) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. remove_config(KeyPath) -> remove_config(KeyPath, #{}). --spec remove_config(emqx_map_lib:config_key_path(), emqx_config:update_opts()) -> +-spec remove_config(emqx_utils_maps:config_key_path(), emqx_config:update_opts()) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. remove_config([RootName | _] = KeyPath, Opts) -> emqx_config_handler:update_config( @@ -212,7 +230,7 @@ remove_config([RootName | _] = KeyPath, Opts) -> {remove, Opts} ). --spec reset_config(emqx_map_lib:config_key_path(), emqx_config:update_opts()) -> +-spec reset_config(emqx_utils_maps:config_key_path(), emqx_config:update_opts()) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. reset_config([RootName | _] = KeyPath, Opts) -> case emqx_config:get_default_value(KeyPath) of diff --git a/apps/emqx/src/emqx_alarm.erl b/apps/emqx/src/emqx_alarm.erl index 84c40ef2a..056f36050 100644 --- a/apps/emqx/src/emqx_alarm.erl +++ b/apps/emqx/src/emqx_alarm.erl @@ -42,7 +42,9 @@ get_alarms/0, get_alarms/1, format/1, - format/2 + format/2, + safe_activate/3, + safe_deactivate/1 ]). %% gen_server callbacks @@ -57,7 +59,6 @@ %% Internal exports (RPC) -export([ - create_activate_alarm/3, do_get_alarms/0 ]). @@ -123,6 +124,9 @@ activate(Name, Details) -> activate(Name, Details, Message) -> gen_server:call(?MODULE, {activate_alarm, Name, Details, Message}). +safe_activate(Name, Details, Message) -> + safe_call({activate_alarm, Name, Details, Message}). + -spec ensure_deactivated(binary() | atom()) -> ok. ensure_deactivated(Name) -> ensure_deactivated(Name, no_details). @@ -155,6 +159,9 @@ deactivate(Name, Details) -> deactivate(Name, Details, Message) -> gen_server:call(?MODULE, {deactivate_alarm, Name, Details, Message}). +safe_deactivate(Name) -> + safe_call({deactivate_alarm, Name, no_details, <<"">>}). + -spec delete_all_deactivated_alarms() -> ok. delete_all_deactivated_alarms() -> gen_server:call(?MODULE, delete_all_deactivated_alarms). @@ -218,17 +225,12 @@ init([]) -> {ok, #{}, get_validity_period()}. handle_call({activate_alarm, Name, Details, Message}, _From, State) -> - Res = mria:transaction( - mria:local_content_shard(), - fun ?MODULE:create_activate_alarm/3, - [Name, Details, Message] - ), - case Res of - {atomic, Alarm} -> + case create_activate_alarm(Name, Details, Message) of + {ok, Alarm} -> do_actions(activate, Alarm, emqx:get_config([alarm, actions])), {reply, ok, State, get_validity_period()}; - {aborted, Reason} -> - {reply, Reason, State, get_validity_period()} + Err -> + {reply, Err, State, get_validity_period()} end; handle_call({deactivate_alarm, Name, Details, Message}, _From, State) -> case mnesia:dirty_read(?ACTIVATED_ALARM, Name) of @@ -283,9 +285,9 @@ get_validity_period() -> emqx:get_config([alarm, validity_period]). create_activate_alarm(Name, Details, Message) -> - case mnesia:read(?ACTIVATED_ALARM, Name) of + case mnesia:dirty_read(?ACTIVATED_ALARM, Name) of [#activated_alarm{name = Name}] -> - mnesia:abort({error, already_existed}); + {error, already_existed}; [] -> Alarm = #activated_alarm{ name = Name, @@ -293,8 +295,8 @@ create_activate_alarm(Name, Details, Message) -> message = normalize_message(Name, iolist_to_binary(Message)), activate_at = erlang:system_time(microsecond) }, - ok = mnesia:write(?ACTIVATED_ALARM, Alarm, write), - Alarm + ok = mria:dirty_write(?ACTIVATED_ALARM, Alarm), + {ok, Alarm} end. do_get_alarms() -> @@ -423,7 +425,7 @@ do_actions(deactivate, Alarm = #deactivated_alarm{name = Name}, [log | More]) -> do_actions(deactivate, Alarm, More); do_actions(Operation, Alarm, [publish | More]) -> Topic = topic(Operation), - {ok, Payload} = emqx_json:safe_encode(normalize(Alarm)), + {ok, Payload} = emqx_utils_json:safe_encode(normalize(Alarm)), Message = emqx_message:make( ?MODULE, 0, @@ -474,3 +476,19 @@ normalize_message(Name, <<"">>) -> list_to_binary(io_lib:format("~p", [Name])); normalize_message(_Name, Message) -> Message. + +safe_call(Req) -> + try + gen_server:call(?MODULE, Req) + catch + _:{timeout, _} = Reason -> + ?SLOG(warning, #{msg => "emqx_alarm_safe_call_timeout", reason => Reason}), + {error, timeout}; + _:Reason:St -> + ?SLOG(error, #{ + msg => "emqx_alarm_safe_call_exception", + reason => Reason, + stacktrace => St + }), + {error, Reason} + end. diff --git a/apps/emqx/src/emqx_authentication_config.erl b/apps/emqx/src/emqx_authentication_config.erl index 5471b66fc..be3b35f57 100644 --- a/apps/emqx/src/emqx_authentication_config.erl +++ b/apps/emqx/src/emqx_authentication_config.erl @@ -277,9 +277,9 @@ atom(Bin) -> binary_to_existing_atom(Bin, utf8). certs_dir(ChainName, ConfigOrID) -> DirName = dir(ChainName, ConfigOrID), SubDir = iolist_to_binary(filename:join(["authn", DirName])), - emqx_misc:safe_filename(SubDir). + emqx_utils:safe_filename(SubDir). dir(ChainName, ID) when is_binary(ID) -> - emqx_misc:safe_filename(iolist_to_binary([to_bin(ChainName), "-", ID])); + emqx_utils:safe_filename(iolist_to_binary([to_bin(ChainName), "-", ID])); dir(ChainName, Config) when is_map(Config) -> dir(ChainName, authenticator_id(Config)). diff --git a/apps/emqx/src/emqx_banned.erl b/apps/emqx/src/emqx_banned.erl index 758c570da..a0ccd93d7 100644 --- a/apps/emqx/src/emqx_banned.erl +++ b/apps/emqx/src/emqx_banned.erl @@ -243,7 +243,7 @@ handle_info(Info, State) -> {noreply, State}. terminate(_Reason, #{expiry_timer := TRef}) -> - emqx_misc:cancel_timer(TRef). + emqx_utils:cancel_timer(TRef). code_change(_OldVsn, State, _Extra) -> {ok, State}. @@ -254,10 +254,10 @@ code_change(_OldVsn, State, _Extra) -> -ifdef(TEST). ensure_expiry_timer(State) -> - State#{expiry_timer := emqx_misc:start_timer(10, expire)}. + State#{expiry_timer := emqx_utils:start_timer(10, expire)}. -else. ensure_expiry_timer(State) -> - State#{expiry_timer := emqx_misc:start_timer(timer:minutes(1), expire)}. + State#{expiry_timer := emqx_utils:start_timer(timer:minutes(1), expire)}. -endif. expire_banned_items(Now) -> diff --git a/apps/emqx/src/emqx_batch.erl b/apps/emqx/src/emqx_batch.erl index 2fe09942c..22e812975 100644 --- a/apps/emqx/src/emqx_batch.erl +++ b/apps/emqx/src/emqx_batch.erl @@ -85,7 +85,7 @@ commit(Batch = #batch{batch_q = Q, commit_fun = Commit}) -> reset(Batch). reset(Batch = #batch{linger_timer = TRef}) -> - _ = emqx_misc:cancel_timer(TRef), + _ = emqx_utils:cancel_timer(TRef), Batch#batch{batch_q = [], linger_timer = undefined}. -spec size(batch()) -> non_neg_integer(). diff --git a/apps/emqx/src/emqx_broker.erl b/apps/emqx/src/emqx_broker.erl index d56620123..5f7c4aaf5 100644 --- a/apps/emqx/src/emqx_broker.erl +++ b/apps/emqx/src/emqx_broker.erl @@ -71,7 +71,7 @@ code_change/3 ]). --import(emqx_tables, [lookup_value/2, lookup_value/3]). +-import(emqx_utils_ets, [lookup_value/2, lookup_value/3]). -ifdef(TEST). -compile(export_all). @@ -92,7 +92,7 @@ start_link(Pool, Id) -> ok = create_tabs(), gen_server:start_link( - {local, emqx_misc:proc_name(?BROKER, Id)}, + {local, emqx_utils:proc_name(?BROKER, Id)}, ?MODULE, [Pool, Id], [] @@ -107,15 +107,15 @@ create_tabs() -> TabOpts = [public, {read_concurrency, true}, {write_concurrency, true}], %% SubOption: {Topic, SubPid} -> SubOption - ok = emqx_tables:new(?SUBOPTION, [ordered_set | TabOpts]), + ok = emqx_utils_ets:new(?SUBOPTION, [ordered_set | TabOpts]), %% Subscription: SubPid -> Topic1, Topic2, Topic3, ... %% duplicate_bag: o(1) insert - ok = emqx_tables:new(?SUBSCRIPTION, [duplicate_bag | TabOpts]), + ok = emqx_utils_ets:new(?SUBSCRIPTION, [duplicate_bag | TabOpts]), %% Subscriber: Topic -> SubPid1, SubPid2, SubPid3, ... %% bag: o(n) insert:( - ok = emqx_tables:new(?SUBSCRIBER, [bag | TabOpts]). + ok = emqx_utils_ets:new(?SUBSCRIBER, [bag | TabOpts]). %%------------------------------------------------------------------------------ %% Subscribe API diff --git a/apps/emqx/src/emqx_broker_helper.erl b/apps/emqx/src/emqx_broker_helper.erl index 91b4c4994..06f249678 100644 --- a/apps/emqx/src/emqx_broker_helper.erl +++ b/apps/emqx/src/emqx_broker_helper.erl @@ -73,11 +73,11 @@ register_sub(SubPid, SubId) when is_pid(SubPid) -> -spec lookup_subid(pid()) -> maybe(emqx_types:subid()). lookup_subid(SubPid) when is_pid(SubPid) -> - emqx_tables:lookup_value(?SUBMON, SubPid). + emqx_utils_ets:lookup_value(?SUBMON, SubPid). -spec lookup_subpid(emqx_types:subid()) -> maybe(pid()). lookup_subpid(SubId) -> - emqx_tables:lookup_value(?SUBID, SubId). + emqx_utils_ets:lookup_value(?SUBID, SubId). -spec get_sub_shard(pid(), emqx_types:topic()) -> non_neg_integer(). get_sub_shard(SubPid, Topic) -> @@ -105,15 +105,15 @@ reclaim_seq(Topic) -> init([]) -> %% Helper table - ok = emqx_tables:new(?HELPER, [{read_concurrency, true}]), + ok = emqx_utils_ets:new(?HELPER, [{read_concurrency, true}]), %% Shards: CPU * 32 true = ets:insert(?HELPER, {shards, emqx_vm:schedulers() * 32}), %% SubSeq: Topic -> SeqId ok = emqx_sequence:create(?SUBSEQ), %% SubId: SubId -> SubPid - ok = emqx_tables:new(?SUBID, [public, {read_concurrency, true}, {write_concurrency, true}]), + ok = emqx_utils_ets:new(?SUBID, [public, {read_concurrency, true}, {write_concurrency, true}]), %% SubMon: SubPid -> SubId - ok = emqx_tables:new(?SUBMON, [public, {read_concurrency, true}, {write_concurrency, true}]), + ok = emqx_utils_ets:new(?SUBMON, [public, {read_concurrency, true}, {write_concurrency, true}]), %% Stats timer ok = emqx_stats:update_interval(broker_stats, fun emqx_broker:stats_fun/0), {ok, #{pmon => emqx_pmon:new()}}. @@ -131,7 +131,7 @@ handle_cast(Msg, State) -> {noreply, State}. handle_info({'DOWN', _MRef, process, SubPid, _Reason}, State = #{pmon := PMon}) -> - SubPids = [SubPid | emqx_misc:drain_down(?BATCH_SIZE)], + SubPids = [SubPid | emqx_utils:drain_down(?BATCH_SIZE)], ok = emqx_pool:async_submit( fun lists:foreach/2, [fun clean_down/1, SubPids] ), diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index 9d4b7eac2..17006f1dc 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -61,7 +61,7 @@ -export([set_field/3]). -import( - emqx_misc, + emqx_utils, [ run_fold/3, pipeline/3, @@ -622,7 +622,7 @@ process_connect( NChannel = Channel#channel{session = Session}, handle_out(connack, {?RC_SUCCESS, sp(false), AckProps}, ensure_connected(NChannel)); {ok, #{session := Session, present := true, pendings := Pendings}} -> - Pendings1 = lists:usort(lists:append(Pendings, emqx_misc:drain_deliver())), + Pendings1 = lists:usort(lists:append(Pendings, emqx_utils:drain_deliver())), NChannel = Channel#channel{ session = Session, resuming = true, @@ -1216,7 +1216,7 @@ handle_call( ) -> ok = emqx_session:takeover(Session), %% TODO: Should not drain deliver here (side effect) - Delivers = emqx_misc:drain_deliver(), + Delivers = emqx_utils:drain_deliver(), AllPendings = lists:append(Delivers, Pendings), disconnect_and_shutdown(takenover, AllPendings, Channel); handle_call(list_authz_cache, Channel) -> @@ -1417,7 +1417,7 @@ ensure_timer(Name, Channel = #channel{timers = Timers}) -> ensure_timer(Name, Time, Channel = #channel{timers = Timers}) -> Msg = maps:get(Name, ?TIMER_TABLE), - TRef = emqx_misc:start_timer(Time, Msg), + TRef = emqx_utils:start_timer(Time, Msg), Channel#channel{timers = Timers#{Name => TRef}}. reset_timer(Name, Channel) -> @@ -1645,7 +1645,7 @@ check_banned(_ConnPkt, #channel{clientinfo = ClientInfo}) -> %% Flapping count_flapping_event(_ConnPkt, Channel = #channel{clientinfo = ClientInfo = #{zone := Zone}}) -> - emqx_config:get_zone_conf(Zone, [flapping_detect, enable]) andalso + is_integer(emqx_config:get_zone_conf(Zone, [flapping_detect, window_time])) andalso emqx_flapping:detect(ClientInfo), {ok, Channel}. @@ -2060,7 +2060,7 @@ clear_keepalive(Channel = #channel{timers = Timers}) -> undefined -> Channel; TRef -> - emqx_misc:cancel_timer(TRef), + emqx_utils:cancel_timer(TRef), Channel#channel{timers = maps:without([alive_timer], Timers)} end. %%-------------------------------------------------------------------- @@ -2256,7 +2256,7 @@ get_mqtt_conf(Zone, Key, Default) -> %%-------------------------------------------------------------------- set_field(Name, Value, Channel) -> - Pos = emqx_misc:index_of(Name, record_info(fields, channel)), + Pos = emqx_utils:index_of(Name, record_info(fields, channel)), setelement(Pos + 1, Channel, Value). get_mqueue(#channel{session = Session}) -> diff --git a/apps/emqx/src/emqx_cm.erl b/apps/emqx/src/emqx_cm.erl index 90b279fa4..265b4e89f 100644 --- a/apps/emqx/src/emqx_cm.erl +++ b/apps/emqx/src/emqx_cm.erl @@ -655,10 +655,10 @@ cast(Msg) -> gen_server:cast(?CM, Msg). init([]) -> TabOpts = [public, {write_concurrency, true}], - ok = emqx_tables:new(?CHAN_TAB, [bag, {read_concurrency, true} | TabOpts]), - ok = emqx_tables:new(?CHAN_CONN_TAB, [bag | TabOpts]), - ok = emqx_tables:new(?CHAN_INFO_TAB, [ordered_set, compressed | TabOpts]), - ok = emqx_tables:new(?CHAN_LIVE_TAB, [ordered_set, {write_concurrency, true} | TabOpts]), + ok = emqx_utils_ets:new(?CHAN_TAB, [bag, {read_concurrency, true} | TabOpts]), + ok = emqx_utils_ets:new(?CHAN_CONN_TAB, [bag | TabOpts]), + ok = emqx_utils_ets:new(?CHAN_INFO_TAB, [ordered_set, compressed | TabOpts]), + ok = emqx_utils_ets:new(?CHAN_LIVE_TAB, [ordered_set, {write_concurrency, true} | TabOpts]), ok = emqx_stats:update_interval(chan_stats, fun ?MODULE:stats_fun/0), State = #{chan_pmon => emqx_pmon:new()}, {ok, State}. @@ -676,7 +676,7 @@ handle_cast(Msg, State) -> handle_info({'DOWN', _MRef, process, Pid, _Reason}, State = #{chan_pmon := PMon}) -> ?tp(emqx_cm_process_down, #{stale_pid => Pid, reason => _Reason}), - ChanPids = [Pid | emqx_misc:drain_down(?BATCH_SIZE)], + ChanPids = [Pid | emqx_utils:drain_down(?BATCH_SIZE)], {Items, PMon1} = emqx_pmon:erase_all(ChanPids, PMon), lists:foreach(fun mark_channel_disconnected/1, ChanPids), ok = emqx_pool:async_submit(fun lists:foreach/2, [fun ?MODULE:clean_down/1, Items]), diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index bf3134568..c94f25ead 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -24,7 +24,7 @@ init_load/2, init_load/3, read_override_conf/1, - read_override_confs/0, + has_deprecated_file/0, delete_override_conf_files/0, check_config/2, fill_defaults/1, @@ -33,8 +33,10 @@ save_configs/5, save_to_app_env/1, save_to_config_map/2, - save_to_override_conf/2 + save_to_override_conf/3 ]). +-export([raw_conf_with_default/4]). +-export([merge_envs/2]). -export([ get_root/1, @@ -142,7 +144,7 @@ -type app_envs() :: [proplists:property()]. %% @doc For the given path, get root value enclosed in a single-key map. --spec get_root(emqx_map_lib:config_key_path()) -> map(). +-spec get_root(emqx_utils_maps:config_key_path()) -> map(). get_root([RootName | _]) -> #{RootName => do_get(?CONF, [RootName], #{})}. @@ -153,14 +155,14 @@ get_root_raw([RootName | _]) -> %% @doc Get a config value for the given path. %% The path should at least include root config name. --spec get(emqx_map_lib:config_key_path()) -> term(). +-spec get(emqx_utils_maps:config_key_path()) -> term(). get(KeyPath) -> do_get(?CONF, KeyPath). --spec get(emqx_map_lib:config_key_path(), term()) -> term(). +-spec get(emqx_utils_maps:config_key_path(), term()) -> term(). get(KeyPath, Default) -> do_get(?CONF, KeyPath, Default). --spec find(emqx_map_lib:config_key_path()) -> - {ok, term()} | {not_found, emqx_map_lib:config_key_path(), term()}. +-spec find(emqx_utils_maps:config_key_path()) -> + {ok, term()} | {not_found, emqx_utils_maps:config_key_path(), term()}. find([]) -> Ref = make_ref(), case do_get(?CONF, [], Ref) of @@ -170,12 +172,12 @@ find([]) -> find(KeyPath) -> atom_conf_path( KeyPath, - fun(AtomKeyPath) -> emqx_map_lib:deep_find(AtomKeyPath, get_root(KeyPath)) end, + fun(AtomKeyPath) -> emqx_utils_maps:deep_find(AtomKeyPath, get_root(KeyPath)) end, {return, {not_found, KeyPath}} ). --spec find_raw(emqx_map_lib:config_key_path()) -> - {ok, term()} | {not_found, emqx_map_lib:config_key_path(), term()}. +-spec find_raw(emqx_utils_maps:config_key_path()) -> + {ok, term()} | {not_found, emqx_utils_maps:config_key_path(), term()}. find_raw([]) -> Ref = make_ref(), case do_get_raw([], Ref) of @@ -183,9 +185,9 @@ find_raw([]) -> Res -> {ok, Res} end; find_raw(KeyPath) -> - emqx_map_lib:deep_find([bin(Key) || Key <- KeyPath], get_root_raw(KeyPath)). + emqx_utils_maps:deep_find([bin(Key) || Key <- KeyPath], get_root_raw(KeyPath)). --spec get_zone_conf(atom(), emqx_map_lib:config_key_path()) -> term(). +-spec get_zone_conf(atom(), emqx_utils_maps:config_key_path()) -> term(). get_zone_conf(Zone, KeyPath) -> case find(?ZONE_CONF_PATH(Zone, KeyPath)) of %% not found in zones, try to find the global config @@ -195,7 +197,7 @@ get_zone_conf(Zone, KeyPath) -> Value end. --spec get_zone_conf(atom(), emqx_map_lib:config_key_path(), term()) -> term(). +-spec get_zone_conf(atom(), emqx_utils_maps:config_key_path(), term()) -> term(). get_zone_conf(Zone, KeyPath, Default) -> case find(?ZONE_CONF_PATH(Zone, KeyPath)) of %% not found in zones, try to find the global config @@ -205,24 +207,24 @@ get_zone_conf(Zone, KeyPath, Default) -> Value end. --spec put_zone_conf(atom(), emqx_map_lib:config_key_path(), term()) -> ok. +-spec put_zone_conf(atom(), emqx_utils_maps:config_key_path(), term()) -> ok. put_zone_conf(Zone, KeyPath, Conf) -> ?MODULE:put(?ZONE_CONF_PATH(Zone, KeyPath), Conf). --spec get_listener_conf(atom(), atom(), emqx_map_lib:config_key_path()) -> term(). +-spec get_listener_conf(atom(), atom(), emqx_utils_maps:config_key_path()) -> term(). get_listener_conf(Type, Listener, KeyPath) -> ?MODULE:get(?LISTENER_CONF_PATH(Type, Listener, KeyPath)). --spec get_listener_conf(atom(), atom(), emqx_map_lib:config_key_path(), term()) -> term(). +-spec get_listener_conf(atom(), atom(), emqx_utils_maps:config_key_path(), term()) -> term(). get_listener_conf(Type, Listener, KeyPath, Default) -> ?MODULE:get(?LISTENER_CONF_PATH(Type, Listener, KeyPath), Default). --spec put_listener_conf(atom(), atom(), emqx_map_lib:config_key_path(), term()) -> ok. +-spec put_listener_conf(atom(), atom(), emqx_utils_maps:config_key_path(), term()) -> ok. put_listener_conf(Type, Listener, KeyPath, Conf) -> ?MODULE:put(?LISTENER_CONF_PATH(Type, Listener, KeyPath), Conf). --spec find_listener_conf(atom(), atom(), emqx_map_lib:config_key_path()) -> - {ok, term()} | {not_found, emqx_map_lib:config_key_path(), term()}. +-spec find_listener_conf(atom(), atom(), emqx_utils_maps:config_key_path()) -> + {ok, term()} | {not_found, emqx_utils_maps:config_key_path(), term()}. find_listener_conf(Type, Listener, KeyPath) -> find(?LISTENER_CONF_PATH(Type, Listener, KeyPath)). @@ -241,20 +243,20 @@ erase(RootName) -> persistent_term:erase(?PERSIS_KEY(?RAW_CONF, bin(RootName))), ok. --spec put(emqx_map_lib:config_key_path(), term()) -> ok. +-spec put(emqx_utils_maps:config_key_path(), term()) -> ok. put(KeyPath, Config) -> Putter = fun(Path, Map, Value) -> - emqx_map_lib:deep_put(Path, Map, Value) + emqx_utils_maps:deep_put(Path, Map, Value) end, do_put(?CONF, Putter, KeyPath, Config). %% Puts value into configuration even if path doesn't exist %% For paths of non-existing atoms use force_put(KeyPath, Config, unsafe) --spec force_put(emqx_map_lib:config_key_path(), term()) -> ok. +-spec force_put(emqx_utils_maps:config_key_path(), term()) -> ok. force_put(KeyPath, Config) -> force_put(KeyPath, Config, safe). --spec force_put(emqx_map_lib:config_key_path(), term(), safe | unsafe) -> ok. +-spec force_put(emqx_utils_maps:config_key_path(), term(), safe | unsafe) -> ok. force_put(KeyPath0, Config, Safety) -> KeyPath = case Safety of @@ -262,19 +264,19 @@ force_put(KeyPath0, Config, Safety) -> unsafe -> [unsafe_atom(Key) || Key <- KeyPath0] end, Putter = fun(Path, Map, Value) -> - emqx_map_lib:deep_force_put(Path, Map, Value) + emqx_utils_maps:deep_force_put(Path, Map, Value) end, do_put(?CONF, Putter, KeyPath, Config). --spec get_default_value(emqx_map_lib:config_key_path()) -> {ok, term()} | {error, term()}. +-spec get_default_value(emqx_utils_maps:config_key_path()) -> {ok, term()} | {error, term()}. get_default_value([RootName | _] = KeyPath) -> BinKeyPath = [bin(Key) || Key <- KeyPath], case find_raw([RootName]) of {ok, RawConf} -> - RawConf1 = emqx_map_lib:deep_remove(BinKeyPath, #{bin(RootName) => RawConf}), + RawConf1 = emqx_utils_maps:deep_remove(BinKeyPath, #{bin(RootName) => RawConf}), try fill_defaults(get_schema_mod(RootName), RawConf1, #{}) of FullConf -> - case emqx_map_lib:deep_find(BinKeyPath, FullConf) of + case emqx_utils_maps:deep_find(BinKeyPath, FullConf) of {not_found, _, _} -> {error, no_default_value}; {ok, Val} -> {ok, Val} end @@ -285,10 +287,10 @@ get_default_value([RootName | _] = KeyPath) -> {error, {rootname_not_found, RootName}} end. --spec get_raw(emqx_map_lib:config_key_path()) -> term(). +-spec get_raw(emqx_utils_maps:config_key_path()) -> term(). get_raw(KeyPath) -> do_get_raw(KeyPath). --spec get_raw(emqx_map_lib:config_key_path(), term()) -> term(). +-spec get_raw(emqx_utils_maps:config_key_path(), term()) -> term(). get_raw(KeyPath, Default) -> do_get_raw(KeyPath, Default). -spec put_raw(map()) -> ok. @@ -301,10 +303,10 @@ put_raw(Config) -> hocon_maps:ensure_plain(Config) ). --spec put_raw(emqx_map_lib:config_key_path(), term()) -> ok. +-spec put_raw(emqx_utils_maps:config_key_path(), term()) -> ok. put_raw(KeyPath, Config) -> Putter = fun(Path, Map, Value) -> - emqx_map_lib:deep_force_put(Path, Map, Value) + emqx_utils_maps:deep_force_put(Path, Map, Value) end, do_put(?RAW_CONF, Putter, KeyPath, Config). @@ -326,9 +328,12 @@ init_load(SchemaMod, ConfFiles) -> %% in the rear of the list overrides prior values. -spec init_load(module(), [string()] | binary() | hocon:config()) -> ok. init_load(SchemaMod, Conf, Opts) when is_list(Conf) orelse is_binary(Conf) -> - init_load(SchemaMod, parse_hocon(Conf), Opts); -init_load(SchemaMod, RawConf, Opts) when is_map(RawConf) -> - ok = save_schema_mod_and_names(SchemaMod), + HasDeprecatedFile = has_deprecated_file(), + RawConf = parse_hocon(HasDeprecatedFile, Conf), + init_load(HasDeprecatedFile, SchemaMod, RawConf, Opts). + +init_load(true, SchemaMod, RawConf, Opts) when is_map(RawConf) -> + %% deprecated conf will be removed in 5.1 %% Merge environment variable overrides on top RawConfWithEnvs = merge_envs(SchemaMod, RawConf), Overrides = read_override_confs(), @@ -338,6 +343,16 @@ init_load(SchemaMod, RawConf, Opts) when is_map(RawConf) -> %% check configs against the schema {AppEnvs, CheckedConf} = check_config(SchemaMod, RawConfAll, #{}), save_to_app_env(AppEnvs), + ok = save_to_config_map(CheckedConf, RawConfAll); +init_load(false, SchemaMod, RawConf, Opts) when is_map(RawConf) -> + ok = save_schema_mod_and_names(SchemaMod), + RootNames = get_root_names(), + %% Merge environment variable overrides on top + RawConfWithEnvs = merge_envs(SchemaMod, RawConf), + RawConfAll = raw_conf_with_default(SchemaMod, RootNames, RawConfWithEnvs, Opts), + %% check configs against the schema + {AppEnvs, CheckedConf} = check_config(SchemaMod, RawConfAll, #{}), + save_to_app_env(AppEnvs), ok = save_to_config_map(CheckedConf, RawConfAll). %% @doc Read merged cluster + local overrides. @@ -374,27 +389,37 @@ schema_default(Schema) -> #{} end. -parse_hocon(Conf) -> +parse_hocon(HasDeprecatedFile, Conf) -> IncDirs = include_dirs(), - case do_parse_hocon(Conf, IncDirs) of + case do_parse_hocon(HasDeprecatedFile, Conf, IncDirs) of {ok, HoconMap} -> HoconMap; {error, Reason} -> ?SLOG(error, #{ - msg => "failed_to_load_hocon_conf", + msg => "failed_to_load_hocon_file", reason => Reason, pwd => file:get_cwd(), include_dirs => IncDirs, config_file => Conf }), - error(failed_to_load_hocon_conf) + error(failed_to_load_hocon_file) end. -do_parse_hocon(Conf, IncDirs) -> +do_parse_hocon(true, Conf, IncDirs) -> Opts = #{format => map, include_dirs => IncDirs}, case is_binary(Conf) of true -> hocon:binary(Conf, Opts); false -> hocon:files(Conf, Opts) + end; +do_parse_hocon(false, Conf, IncDirs) -> + Opts = #{format => map, include_dirs => IncDirs}, + case is_binary(Conf) of + %% only use in test + true -> + hocon:binary(Conf, Opts); + false -> + ClusterFile = cluster_hocon_file(), + hocon:files([ClusterFile | Conf], Opts) end. include_dirs() -> @@ -430,7 +455,7 @@ do_check_config(SchemaMod, RawConf, Opts0) -> Opts = maps:merge(Opts0, Opts1), {AppEnvs, CheckedConf} = hocon_tconf:map_translate(SchemaMod, RawConf, Opts), - {AppEnvs, emqx_map_lib:unsafe_atom_key_map(CheckedConf)}. + {AppEnvs, emqx_utils_maps:unsafe_atom_key_map(CheckedConf)}. fill_defaults(RawConf) -> fill_defaults(RawConf, #{}). @@ -466,10 +491,12 @@ fill_defaults(SchemaMod, RawConf, Opts0) -> %% Delete override config files. -spec delete_override_conf_files() -> ok. delete_override_conf_files() -> - F1 = override_conf_file(#{override_to => local}), - F2 = override_conf_file(#{override_to => cluster}), + F1 = deprecated_conf_file(#{override_to => local}), + F2 = deprecated_conf_file(#{override_to => cluster}), + F3 = cluster_hocon_file(), ok = ensure_file_deleted(F1), - ok = ensure_file_deleted(F2). + ok = ensure_file_deleted(F2), + ok = ensure_file_deleted(F3). ensure_file_deleted(F) -> case file:delete(F) of @@ -480,19 +507,33 @@ ensure_file_deleted(F) -> -spec read_override_conf(map()) -> raw_config(). read_override_conf(#{} = Opts) -> - File = override_conf_file(Opts), + File = + case has_deprecated_file() of + true -> deprecated_conf_file(Opts); + false -> cluster_hocon_file() + end, load_hocon_file(File, map). -override_conf_file(Opts) when is_map(Opts) -> +%% @doc Return `true' if this node is upgraded from older version which used cluster-override.conf for +%% cluster-wide config persistence. +has_deprecated_file() -> + DeprecatedFile = deprecated_conf_file(#{override_to => cluster}), + filelib:is_regular(DeprecatedFile). + +deprecated_conf_file(Opts) when is_map(Opts) -> Key = case maps:get(override_to, Opts, cluster) of local -> local_override_conf_file; cluster -> cluster_override_conf_file end, application:get_env(emqx, Key, undefined); -override_conf_file(Which) when is_atom(Which) -> +deprecated_conf_file(Which) when is_atom(Which) -> application:get_env(emqx, Which, undefined). +%% The newer version cluster-wide config persistence file. +cluster_hocon_file() -> + application:get_env(emqx, cluster_hocon_file, undefined). + -spec save_schema_mod_and_names(module()) -> ok. save_schema_mod_and_names(SchemaMod) -> RootNames = hocon_schema:root_names(SchemaMod), @@ -522,11 +563,15 @@ get_schema_mod(RootName) -> get_root_names() -> maps:get(names, persistent_term:get(?PERSIS_SCHEMA_MODS, #{names => []})). --spec save_configs(app_envs(), config(), raw_config(), raw_config(), update_opts()) -> ok. +-spec save_configs( + app_envs(), config(), raw_config(), raw_config(), update_opts() +) -> ok. + save_configs(AppEnvs, Conf, RawConf, OverrideConf, Opts) -> - %% We first try to save to override.conf, because saving to files is more error prone + %% We first try to save to files, because saving to files is more error prone %% than saving into memory. - ok = save_to_override_conf(OverrideConf, Opts), + HasDeprecatedFile = has_deprecated_file(), + ok = save_to_override_conf(HasDeprecatedFile, OverrideConf, Opts), save_to_app_env(AppEnvs), save_to_config_map(Conf, RawConf). @@ -544,11 +589,12 @@ save_to_config_map(Conf, RawConf) -> ?MODULE:put(Conf), ?MODULE:put_raw(RawConf). --spec save_to_override_conf(raw_config(), update_opts()) -> ok | {error, term()}. -save_to_override_conf(undefined, _) -> +-spec save_to_override_conf(boolean(), raw_config(), update_opts()) -> ok | {error, term()}. +save_to_override_conf(_, undefined, _) -> ok; -save_to_override_conf(RawConf, Opts) -> - case override_conf_file(Opts) of +%% TODO: Remove deprecated override conf file when 5.1 +save_to_override_conf(true, RawConf, Opts) -> + case deprecated_conf_file(Opts) of undefined -> ok; FileName -> @@ -564,6 +610,24 @@ save_to_override_conf(RawConf, Opts) -> }), {error, Reason} end + end; +save_to_override_conf(false, RawConf, _Opts) -> + case cluster_hocon_file() of + undefined -> + ok; + FileName -> + ok = filelib:ensure_dir(FileName), + case file:write_file(FileName, hocon_pp:do(RawConf, #{})) of + ok -> + ok; + {error, Reason} -> + ?SLOG(error, #{ + msg => "failed_to_save_conf_file", + filename => FileName, + reason => Reason + }), + {error, Reason} + end end. add_handlers() -> @@ -645,11 +709,11 @@ do_put(Type, Putter, [RootName | KeyPath], DeepValue) -> do_deep_get(?CONF, KeyPath, Map, Default) -> atom_conf_path( KeyPath, - fun(AtomKeyPath) -> emqx_map_lib:deep_get(AtomKeyPath, Map, Default) end, + fun(AtomKeyPath) -> emqx_utils_maps:deep_get(AtomKeyPath, Map, Default) end, {return, Default} ); do_deep_get(?RAW_CONF, KeyPath, Map, Default) -> - emqx_map_lib:deep_get([bin(Key) || Key <- KeyPath], Map, Default). + emqx_utils_maps:deep_get([bin(Key) || Key <- KeyPath], Map, Default). do_deep_put(?CONF, Putter, KeyPath, Map, Value) -> atom_conf_path( diff --git a/apps/emqx/src/emqx_config_handler.erl b/apps/emqx/src/emqx_config_handler.erl index a0a99b62e..e664a7dd7 100644 --- a/apps/emqx/src/emqx_config_handler.erl +++ b/apps/emqx/src/emqx_config_handler.erl @@ -43,7 +43,6 @@ terminate/2, code_change/3 ]). --export([is_mutable/3]). -define(MOD, {mod}). -define(WKEY, '?'). @@ -230,26 +229,15 @@ process_update_request([_], _Handlers, {remove, _Opts}) -> process_update_request(ConfKeyPath, _Handlers, {remove, Opts}) -> OldRawConf = emqx_config:get_root_raw(ConfKeyPath), BinKeyPath = bin_path(ConfKeyPath), - case check_permissions(remove, BinKeyPath, OldRawConf, Opts) of - allow -> - NewRawConf = emqx_map_lib:deep_remove(BinKeyPath, OldRawConf), - OverrideConf = remove_from_override_config(BinKeyPath, Opts), - {ok, NewRawConf, OverrideConf, Opts}; - {deny, Reason} -> - {error, {permission_denied, Reason}} - end; + NewRawConf = emqx_utils_maps:deep_remove(BinKeyPath, OldRawConf), + OverrideConf = remove_from_override_config(BinKeyPath, Opts), + {ok, NewRawConf, OverrideConf, Opts}; process_update_request(ConfKeyPath, Handlers, {{update, UpdateReq}, Opts}) -> OldRawConf = emqx_config:get_root_raw(ConfKeyPath), case do_update_config(ConfKeyPath, Handlers, OldRawConf, UpdateReq) of {ok, NewRawConf} -> - BinKeyPath = bin_path(ConfKeyPath), - case check_permissions(update, BinKeyPath, NewRawConf, Opts) of - allow -> - OverrideConf = merge_to_override_config(NewRawConf, Opts), - {ok, NewRawConf, OverrideConf, Opts}; - {deny, Reason} -> - {error, {permission_denied, Reason}} - end; + OverrideConf = merge_to_override_config(NewRawConf, Opts), + {ok, NewRawConf, OverrideConf, Opts}; Error -> Error end. @@ -271,8 +259,10 @@ do_update_config( SubOldRawConf = get_sub_config(ConfKeyBin, OldRawConf), SubHandlers = get_sub_handlers(ConfKey, Handlers), case do_update_config(SubConfKeyPath, SubHandlers, SubOldRawConf, UpdateReq, ConfKeyPath) of - {ok, NewUpdateReq} -> merge_to_old_config(#{ConfKeyBin => NewUpdateReq}, OldRawConf); - Error -> Error + {ok, NewUpdateReq} -> + merge_to_old_config(#{ConfKeyBin => NewUpdateReq}, OldRawConf); + Error -> + Error end. check_and_save_configs( @@ -445,7 +435,7 @@ remove_from_override_config(_BinKeyPath, #{persistent := false}) -> undefined; remove_from_override_config(BinKeyPath, Opts) -> OldConf = emqx_config:read_override_conf(Opts), - emqx_map_lib:deep_remove(BinKeyPath, OldConf). + emqx_utils_maps:deep_remove(BinKeyPath, OldConf). %% apply new config on top of override config merge_to_override_config(_RawConf, #{persistent := false}) -> @@ -467,7 +457,7 @@ return_change_result(_ConfKeyPath, {remove, _Opts}) -> return_rawconf(ConfKeyPath, #{rawconf_with_defaults := true}) -> FullRawConf = emqx_config:fill_defaults(emqx_config:get_raw([])), - emqx_map_lib:deep_get(bin_path(ConfKeyPath), FullRawConf); + emqx_utils_maps:deep_get(bin_path(ConfKeyPath), FullRawConf); return_rawconf(ConfKeyPath, _) -> emqx_config:get_raw(ConfKeyPath). @@ -485,16 +475,16 @@ atom(Atom) when is_atom(Atom) -> -dialyzer({nowarn_function, do_remove_handler/2}). do_remove_handler(ConfKeyPath, Handlers) -> - NewHandlers = emqx_map_lib:deep_remove(ConfKeyPath ++ [?MOD], Handlers), + NewHandlers = emqx_utils_maps:deep_remove(ConfKeyPath ++ [?MOD], Handlers), remove_empty_leaf(ConfKeyPath, NewHandlers). remove_empty_leaf([], Handlers) -> Handlers; remove_empty_leaf(KeyPath, Handlers) -> - case emqx_map_lib:deep_find(KeyPath, Handlers) =:= {ok, #{}} of + case emqx_utils_maps:deep_find(KeyPath, Handlers) =:= {ok, #{}} of %% empty leaf true -> - Handlers1 = emqx_map_lib:deep_remove(KeyPath, Handlers), + Handlers1 = emqx_utils_maps:deep_remove(KeyPath, Handlers), SubKeyPath = lists:sublist(KeyPath, length(KeyPath) - 1), remove_empty_leaf(SubKeyPath, Handlers1); false -> @@ -511,7 +501,7 @@ assert_callback_function(Mod) -> end, ok. --spec schema(module(), emqx_map_lib:config_key_path()) -> hocon_schema:schema(). +-spec schema(module(), emqx_utils_maps:config_key_path()) -> hocon_schema:schema(). schema(SchemaModule, [RootKey | _]) -> Roots = hocon_schema:roots(SchemaModule), {Field, Translations} = @@ -546,98 +536,3 @@ load_prev_handlers() -> save_handlers(Handlers) -> application:set_env(emqx, ?MODULE, Handlers). - -check_permissions(_Action, _ConfKeyPath, _NewRawConf, #{override_to := local}) -> - allow; -check_permissions(Action, ConfKeyPath, NewRawConf, _Opts) -> - case emqx_map_lib:deep_find(ConfKeyPath, NewRawConf) of - {ok, NewRaw} -> - LocalOverride = emqx_config:read_override_conf(#{override_to => local}), - case emqx_map_lib:deep_find(ConfKeyPath, LocalOverride) of - {ok, LocalRaw} -> - case is_mutable(Action, NewRaw, LocalRaw) of - ok -> - allow; - {error, Error} -> - ?SLOG(error, #{ - msg => "prevent_remove_local_override_conf", - config_key_path => ConfKeyPath, - error => Error - }), - {deny, "Disable changed from local-override.conf"} - end; - {not_found, _, _} -> - allow - end; - {not_found, _, _} -> - allow - end. - -is_mutable(Action, NewRaw, LocalRaw) -> - try - KeyPath = [], - is_mutable(KeyPath, Action, NewRaw, LocalRaw) - catch - throw:Error -> Error - end. - --define(REMOVE_FAILED, "remove_failed"). --define(UPDATE_FAILED, "update_failed"). - -is_mutable(KeyPath, Action, New = #{}, Local = #{}) -> - maps:foreach( - fun(Key, SubLocal) -> - case maps:find(Key, New) of - error -> ok; - {ok, SubNew} -> is_mutable(KeyPath ++ [Key], Action, SubNew, SubLocal) - end - end, - Local - ); -is_mutable(KeyPath, remove, Update, Origin) -> - throw({error, {?REMOVE_FAILED, KeyPath, Update, Origin}}); -is_mutable(_KeyPath, update, Val, Val) -> - ok; -is_mutable(KeyPath, update, Update, Origin) -> - throw({error, {?UPDATE_FAILED, KeyPath, Update, Origin}}). - --ifdef(TEST). --include_lib("eunit/include/eunit.hrl"). - -is_mutable_update_test() -> - Action = update, - ?assertEqual(ok, is_mutable(Action, #{}, #{})), - ?assertEqual(ok, is_mutable(Action, #{a => #{b => #{c => #{}}}}, #{a => #{b => #{c => #{}}}})), - ?assertEqual(ok, is_mutable(Action, #{a => #{b => #{c => 1}}}, #{a => #{b => #{c => 1}}})), - ?assertEqual( - {error, {?UPDATE_FAILED, [a, b, c], 1, 2}}, - is_mutable(Action, #{a => #{b => #{c => 1}}}, #{a => #{b => #{c => 2}}}) - ), - ?assertEqual( - {error, {?UPDATE_FAILED, [a, b, d], 2, 3}}, - is_mutable(Action, #{a => #{b => #{c => 1, d => 2}}}, #{a => #{b => #{c => 1, d => 3}}}) - ), - ok. - -is_mutable_remove_test() -> - Action = remove, - ?assertEqual(ok, is_mutable(Action, #{}, #{})), - ?assertEqual(ok, is_mutable(Action, #{a => #{b => #{c => #{}}}}, #{a1 => #{b => #{c => #{}}}})), - ?assertEqual(ok, is_mutable(Action, #{a => #{b => #{c => 1}}}, #{a => #{b1 => #{c => 1}}})), - ?assertEqual(ok, is_mutable(Action, #{a => #{b => #{c => 1}}}, #{a => #{b => #{c1 => 1}}})), - - ?assertEqual( - {error, {?REMOVE_FAILED, [a, b, c], 1, 1}}, - is_mutable(Action, #{a => #{b => #{c => 1}}}, #{a => #{b => #{c => 1}}}) - ), - ?assertEqual( - {error, {?REMOVE_FAILED, [a, b, c], 1, 2}}, - is_mutable(Action, #{a => #{b => #{c => 1}}}, #{a => #{b => #{c => 2}}}) - ), - ?assertEqual( - {error, {?REMOVE_FAILED, [a, b, c], 1, 1}}, - is_mutable(Action, #{a => #{b => #{c => 1, d => 2}}}, #{a => #{b => #{c => 1, d => 3}}}) - ), - ok. - --endif. diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl index e5002cab4..27b6f3e84 100644 --- a/apps/emqx/src/emqx_connection.erl +++ b/apps/emqx/src/emqx_connection.erl @@ -77,7 +77,7 @@ -export([set_field/3]). -import( - emqx_misc, + emqx_utils, [start_timer/2] ). @@ -182,10 +182,8 @@ -define(ALARM_SOCK_STATS_KEYS, [send_pend, recv_cnt, recv_oct, send_cnt, send_oct]). -define(ALARM_SOCK_OPTS_KEYS, [high_watermark, high_msgq_watermark, sndbuf, recbuf, buffer]). -%% use macro to do compile time limiter's type check --define(LIMITER_BYTES_IN, bytes_in). --define(LIMITER_MESSAGE_IN, message_in). --define(EMPTY_QUEUE, {[], []}). +-define(LIMITER_BYTES_IN, bytes). +-define(LIMITER_MESSAGE_IN, messages). -dialyzer({no_match, [info/2]}). -dialyzer( @@ -260,7 +258,7 @@ stats(#state{ {error, _} -> [] end, ChanStats = emqx_channel:stats(Channel), - ProcStats = emqx_misc:proc_stats(), + ProcStats = emqx_utils:proc_stats(), lists:append([SockStats, ChanStats, ProcStats]). %% @doc Set TCP keepalive socket options to override system defaults. @@ -392,7 +390,7 @@ run_loop( emqx_channel:info(zone, Channel), [force_shutdown] ), - emqx_misc:tune_heap_size(ShutdownPolicy), + emqx_utils:tune_heap_size(ShutdownPolicy), case activate_socket(State) of {ok, NState} -> hibernate(Parent, NState); @@ -472,7 +470,7 @@ ensure_stats_timer(_Timeout, State) -> -compile({inline, [cancel_stats_timer/1]}). cancel_stats_timer(State = #state{stats_timer = TRef}) when is_reference(TRef) -> ?tp(debug, cancel_stats_timer, #{}), - ok = emqx_misc:cancel_timer(TRef), + ok = emqx_utils:cancel_timer(TRef), State#state{stats_timer = undefined}; cancel_stats_timer(State) -> State. @@ -558,7 +556,7 @@ handle_msg( {incoming, Packet = ?CONNECT_PACKET(ConnPkt)}, State = #state{idle_timer = IdleTimer} ) -> - ok = emqx_misc:cancel_timer(IdleTimer), + ok = emqx_utils:cancel_timer(IdleTimer), Serialize = emqx_frame:serialize_opts(ConnPkt), NState = State#state{ serialize = Serialize, @@ -593,7 +591,7 @@ handle_msg( #state{listener = {Type, Listener}} = State ) -> ActiveN = get_active_n(Type, Listener), - Delivers = [Deliver | emqx_misc:drain_deliver(ActiveN)], + Delivers = [Deliver | emqx_utils:drain_deliver(ActiveN)], with_channel(handle_deliver, [Delivers], State); %% Something sent handle_msg({inet_reply, _Sock, ok}, State = #state{listener = {Type, Listener}}) -> @@ -1073,7 +1071,7 @@ check_oom(State = #state{channel = Channel}) -> emqx_channel:info(zone, Channel), [force_shutdown] ), ?tp(debug, check_oom, #{policy => ShutdownPolicy}), - case emqx_misc:check_oom(ShutdownPolicy) of + case emqx_utils:check_oom(ShutdownPolicy) of {shutdown, Reason} -> %% triggers terminate/2 callback immediately erlang:exit({shutdown, Reason}); @@ -1200,7 +1198,7 @@ inc_counter(Key, Inc) -> %%-------------------------------------------------------------------- set_field(Name, Value, State) -> - Pos = emqx_misc:index_of(Name, record_info(fields, state)), + Pos = emqx_utils:index_of(Name, record_info(fields, state)), setelement(Pos + 1, State, Value). get_state(Pid) -> diff --git a/apps/emqx/src/emqx_crl_cache.erl b/apps/emqx/src/emqx_crl_cache.erl index 79e47a6dc..084313420 100644 --- a/apps/emqx/src/emqx_crl_cache.erl +++ b/apps/emqx/src/emqx_crl_cache.erl @@ -117,7 +117,7 @@ handle_call(Call, _From, State) -> handle_cast({evict, URL}, State0 = #state{refresh_timers = RefreshTimers0}) -> emqx_ssl_crl_cache:delete(URL), MTimer = maps:get(URL, RefreshTimers0, undefined), - emqx_misc:cancel_timer(MTimer), + emqx_utils:cancel_timer(MTimer), RefreshTimers = maps:without([URL], RefreshTimers0), State = State0#state{refresh_timers = RefreshTimers}, ?tp( @@ -223,9 +223,9 @@ ensure_timer(URL, State = #state{refresh_interval = Timeout}) -> ensure_timer(URL, State = #state{refresh_timers = RefreshTimers0}, Timeout) -> ?tp(crl_cache_ensure_timer, #{url => URL, timeout => Timeout}), MTimer = maps:get(URL, RefreshTimers0, undefined), - emqx_misc:cancel_timer(MTimer), + emqx_utils:cancel_timer(MTimer), RefreshTimers = RefreshTimers0#{ - URL => emqx_misc:start_timer( + URL => emqx_utils:start_timer( Timeout, {refresh, URL} ) @@ -297,7 +297,7 @@ handle_cache_overflow(State0) -> {_Time, OldestURL, InsertionTimes} = gb_trees:take_smallest(InsertionTimes0), emqx_ssl_crl_cache:delete(OldestURL), MTimer = maps:get(OldestURL, RefreshTimers0, undefined), - emqx_misc:cancel_timer(MTimer), + emqx_utils:cancel_timer(MTimer), RefreshTimers = maps:remove(OldestURL, RefreshTimers0), CachedURLs = sets:del_element(OldestURL, CachedURLs0), ?tp(debug, crl_cache_overflow, #{oldest_url => OldestURL}), diff --git a/apps/emqx/src/emqx_flapping.erl b/apps/emqx/src/emqx_flapping.erl index 64e4ed6c3..70b1a3232 100644 --- a/apps/emqx/src/emqx_flapping.erl +++ b/apps/emqx/src/emqx_flapping.erl @@ -27,6 +27,10 @@ %% API -export([detect/1]). +-ifdef(TEST). +-export([get_policy/2]). +-endif. + %% gen_server callbacks -export([ init/1, @@ -39,15 +43,6 @@ %% Tab -define(FLAPPING_TAB, ?MODULE). -%% Default Policy --define(FLAPPING_THRESHOLD, 30). --define(FLAPPING_DURATION, 60000). --define(FLAPPING_BANNED_INTERVAL, 300000). --define(DEFAULT_DETECT_POLICY, #{ - max_count => ?FLAPPING_THRESHOLD, - window_time => ?FLAPPING_DURATION, - ban_time => ?FLAPPING_BANNED_INTERVAL -}). -record(flapping, { clientid :: emqx_types:clientid(), @@ -69,7 +64,7 @@ stop() -> gen_server:stop(?MODULE). %% @doc Detect flapping when a MQTT client disconnected. -spec detect(emqx_types:clientinfo()) -> boolean(). detect(#{clientid := ClientId, peerhost := PeerHost, zone := Zone}) -> - Policy = #{max_count := Threshold} = get_policy(Zone), + Policy = #{max_count := Threshold} = get_policy([max_count, window_time, ban_time], Zone), %% The initial flapping record sets the detect_cnt to 0. InitVal = #flapping{ clientid = ClientId, @@ -89,8 +84,22 @@ detect(#{clientid := ClientId, peerhost := PeerHost, zone := Zone}) -> end end. -get_policy(Zone) -> - emqx_config:get_zone_conf(Zone, [flapping_detect]). +get_policy(Keys, Zone) when is_list(Keys) -> + RootKey = flapping_detect, + Conf = emqx_config:get_zone_conf(Zone, [RootKey]), + lists:foldl( + fun(Key, Acc) -> + case maps:find(Key, Conf) of + {ok, V} -> Acc#{Key => V}; + error -> Acc#{Key => emqx_config:get([RootKey, Key])} + end + end, + #{}, + Keys + ); +get_policy(Key, Zone) -> + #{Key := Conf} = get_policy([Key], Zone), + Conf. now_diff(TS) -> erlang:system_time(millisecond) - TS. @@ -99,7 +108,7 @@ now_diff(TS) -> erlang:system_time(millisecond) - TS. %%-------------------------------------------------------------------- init([]) -> - ok = emqx_tables:new(?FLAPPING_TAB, [ + ok = emqx_utils_ets:new(?FLAPPING_TAB, [ public, set, {keypos, #flapping.clientid}, @@ -166,8 +175,7 @@ handle_cast(Msg, State) -> handle_info({timeout, _TRef, {garbage_collect, Zone}}, State) -> Timestamp = - erlang:system_time(millisecond) - - maps:get(window_time, get_policy(Zone)), + erlang:system_time(millisecond) - get_policy(window_time, Zone), MatchSpec = [{{'_', '_', '_', '$1', '_'}, [{'<', '$1', Timestamp}], [true]}], ets:select_delete(?FLAPPING_TAB, MatchSpec), _ = start_timer(Zone), @@ -183,15 +191,19 @@ code_change(_OldVsn, State, _Extra) -> {ok, State}. start_timer(Zone) -> - WindTime = maps:get(window_time, get_policy(Zone)), - emqx_misc:start_timer(WindTime, {garbage_collect, Zone}). + case get_policy(window_time, Zone) of + WindowTime when is_integer(WindowTime) -> + emqx_utils:start_timer(WindowTime, {garbage_collect, Zone}); + disabled -> + ok + end. start_timers() -> - lists:foreach( - fun({Zone, _ZoneConf}) -> + maps:foreach( + fun(Zone, _ZoneConf) -> start_timer(Zone) end, - maps:to_list(emqx:get_config([zones], #{})) + emqx:get_config([zones], #{}) ). fmt_host(PeerHost) -> diff --git a/apps/emqx/src/emqx_guid.erl b/apps/emqx/src/emqx_guid.erl index fea4e70b0..d313723fb 100644 --- a/apps/emqx/src/emqx_guid.erl +++ b/apps/emqx/src/emqx_guid.erl @@ -145,10 +145,10 @@ npid() -> NPid. to_hexstr(I) when byte_size(I) =:= 16 -> - emqx_misc:bin_to_hexstr(I, upper). + emqx_utils:bin_to_hexstr(I, upper). from_hexstr(S) when byte_size(S) =:= 32 -> - emqx_misc:hexstr_to_bin(S). + emqx_utils:hexstr_to_bin(S). to_base62(<>) -> emqx_base62:encode(I). diff --git a/apps/emqx/src/emqx_hooks.erl b/apps/emqx/src/emqx_hooks.erl index 1784d8ea3..0b8dc0941 100644 --- a/apps/emqx/src/emqx_hooks.erl +++ b/apps/emqx/src/emqx_hooks.erl @@ -229,7 +229,7 @@ lookup(HookPoint) -> %%-------------------------------------------------------------------- init([]) -> - ok = emqx_tables:new(?TAB, [{keypos, #hook.name}, {read_concurrency, true}]), + ok = emqx_utils_ets:new(?TAB, [{keypos, #hook.name}, {read_concurrency, true}]), {ok, #{}}. handle_call({add, HookPoint, Callback = #callback{action = {M, F, _}}}, _From, State) -> diff --git a/apps/emqx/src/emqx_limiter/src/emqx_htb_limiter.erl b/apps/emqx/src/emqx_limiter/src/emqx_htb_limiter.erl index 83bc2ec72..53f26deb5 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_htb_limiter.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_htb_limiter.erl @@ -139,7 +139,8 @@ make_token_bucket_limiter(Cfg, Bucket) -> Cfg#{ tokens => emqx_limiter_server:get_initial_val(Cfg), lasttime => ?NOW, - bucket => Bucket + bucket => Bucket, + capacity => emqx_limiter_schema:calc_capacity(Cfg) }. %%@doc create a limiter server's reference @@ -375,7 +376,7 @@ return_pause(infinity, PauseType, Fun, Diff, Limiter) -> {PauseType, ?MINIMUM_PAUSE, make_retry_context(Fun, Diff), Limiter}; return_pause(Rate, PauseType, Fun, Diff, Limiter) -> Val = erlang:round(Diff * emqx_limiter_schema:default_period() / Rate), - Pause = emqx_misc:clamp(Val, ?MINIMUM_PAUSE, ?MAXIMUM_PAUSE), + Pause = emqx_utils:clamp(Val, ?MINIMUM_PAUSE, ?MAXIMUM_PAUSE), {PauseType, Pause, make_retry_context(Fun, Diff), Limiter}. -spec make_retry_context(undefined | retry_fun(Limiter), non_neg_integer()) -> diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_bucket_ref.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_bucket_ref.erl index fe30e41e9..139564df7 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_bucket_ref.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_bucket_ref.erl @@ -23,6 +23,7 @@ %% API -export([ new/3, + infinity_bucket/0, check/3, try_restore/2, available/1 @@ -58,6 +59,10 @@ new(Counter, Index, Rate) -> rate => Rate }. +-spec infinity_bucket() -> bucket_ref(). +infinity_bucket() -> + infinity. + %% @doc check tokens -spec check(pos_integer(), bucket_ref(), Disivisble :: boolean()) -> HasToken :: diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl index f45fc55b6..c762a0f1d 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl @@ -24,6 +24,7 @@ fields/1, to_rate/1, to_capacity/1, + to_burst/1, default_period/0, to_burst_rate/1, to_initial/1, @@ -31,20 +32,20 @@ get_bucket_cfg_path/2, desc/1, types/0, - infinity_value/0 + calc_capacity/1 ]). -define(KILOBYTE, 1024). -define(BUCKET_KEYS, [ - {bytes_in, bucket_infinity}, - {message_in, bucket_infinity}, + {bytes, bucket_infinity}, + {messages, bucket_infinity}, {connection, bucket_limit}, {message_routing, bucket_infinity} ]). -type limiter_type() :: - bytes_in - | message_in + bytes + | messages | connection | message_routing %% internal limiter for unclassified resources @@ -54,8 +55,10 @@ -type bucket_name() :: atom(). -type rate() :: infinity | float(). -type burst_rate() :: 0 | float(). +%% this is a compatible type for the deprecated field and type `capacity`. +-type burst() :: burst_rate(). %% the capacity of the token bucket --type capacity() :: non_neg_integer(). +%%-type capacity() :: non_neg_integer(). %% initial capacity of the token bucket -type initial() :: non_neg_integer(). -type bucket_path() :: list(atom()). @@ -72,13 +75,13 @@ -typerefl_from_string({rate/0, ?MODULE, to_rate}). -typerefl_from_string({burst_rate/0, ?MODULE, to_burst_rate}). --typerefl_from_string({capacity/0, ?MODULE, to_capacity}). +-typerefl_from_string({burst/0, ?MODULE, to_burst}). -typerefl_from_string({initial/0, ?MODULE, to_initial}). -reflect_type([ rate/0, burst_rate/0, - capacity/0, + burst/0, initial/0, failure_strategy/0, bucket_name/0 @@ -90,14 +93,17 @@ namespace() -> limiter. -roots() -> [limiter]. +roots() -> + [{limiter, hoconsc:mk(hoconsc:ref(?MODULE, limiter), #{importance => ?IMPORTANCE_HIDDEN})}]. fields(limiter) -> [ {Type, ?HOCON(?R_REF(node_opts), #{ desc => ?DESC(Type), - default => #{} + default => #{}, + importance => ?IMPORTANCE_HIDDEN, + aliases => alias_of_type(Type) })} || Type <- types() ] ++ @@ -107,6 +113,7 @@ fields(limiter) -> ?R_REF(client_fields), #{ desc => ?DESC(client), + importance => ?IMPORTANCE_HIDDEN, default => maps:from_list([ {erlang:atom_to_binary(Type), #{}} || Type <- types() @@ -124,30 +131,20 @@ fields(node_opts) -> })} ]; fields(client_fields) -> - [ - {Type, - ?HOCON(?R_REF(client_opts), #{ - desc => ?DESC(Type), - default => #{} - })} - || Type <- types() - ]; + client_fields(types(), #{default => #{}}); fields(bucket_infinity) -> - [ - {rate, ?HOCON(rate(), #{desc => ?DESC(rate), default => <<"infinity">>})}, - {capacity, ?HOCON(capacity(), #{desc => ?DESC(capacity), default => <<"infinity">>})}, - {initial, ?HOCON(initial(), #{default => <<"0">>, desc => ?DESC(initial)})} - ]; + fields_of_bucket(<<"infinity">>); fields(bucket_limit) -> - [ - {rate, ?HOCON(rate(), #{desc => ?DESC(rate), default => <<"1000/s">>})}, - {capacity, ?HOCON(capacity(), #{desc => ?DESC(capacity), default => <<"1000">>})}, - {initial, ?HOCON(initial(), #{default => <<"0">>, desc => ?DESC(initial)})} - ]; + fields_of_bucket(<<"1000/s">>); fields(client_opts) -> [ {rate, ?HOCON(rate(), #{default => <<"infinity">>, desc => ?DESC(rate)})}, - {initial, ?HOCON(initial(), #{default => <<"0">>, desc => ?DESC(initial)})}, + {initial, + ?HOCON(initial(), #{ + default => <<"0">>, + desc => ?DESC(initial), + importance => ?IMPORTANCE_HIDDEN + })}, %% low_watermark add for emqx_channel and emqx_session %% both modules consume first and then check %% so we need to use this value to prevent excessive consumption @@ -157,20 +154,24 @@ fields(client_opts) -> initial(), #{ desc => ?DESC(low_watermark), - default => <<"0">> + default => <<"0">>, + importance => ?IMPORTANCE_HIDDEN } )}, - {capacity, - ?HOCON(capacity(), #{ - desc => ?DESC(client_bucket_capacity), - default => <<"infinity">> + {burst, + ?HOCON(burst(), #{ + desc => ?DESC(burst), + default => <<"0">>, + importance => ?IMPORTANCE_HIDDEN, + aliases => [capacity] })}, {divisible, ?HOCON( boolean(), #{ desc => ?DESC(divisible), - default => false + default => false, + importance => ?IMPORTANCE_HIDDEN } )}, {max_retry_time, @@ -178,7 +179,8 @@ fields(client_opts) -> emqx_schema:duration(), #{ desc => ?DESC(max_retry_time), - default => <<"10s">> + default => <<"10s">>, + importance => ?IMPORTANCE_HIDDEN } )}, {failure_strategy, @@ -186,16 +188,18 @@ fields(client_opts) -> failure_strategy(), #{ desc => ?DESC(failure_strategy), - default => force + default => force, + importance => ?IMPORTANCE_HIDDEN } )} ]; fields(listener_fields) -> - bucket_fields(?BUCKET_KEYS, listener_client_fields); + composite_bucket_fields(?BUCKET_KEYS, listener_client_fields); fields(listener_client_fields) -> - client_fields(?BUCKET_KEYS); + {Types, _} = lists:unzip(?BUCKET_KEYS), + client_fields(Types, #{required => false}); fields(Type) -> - bucket_field(Type). + simple_bucket_field(Type). desc(limiter) -> "Settings for the rate limiter."; @@ -230,19 +234,12 @@ get_bucket_cfg_path(Type, BucketName) -> [limiter, Type, bucket, BucketName]. types() -> - [bytes_in, message_in, connection, message_routing, internal]. + [bytes, messages, connection, message_routing, internal]. -%%-------------------------------------------------------------------- -%% Internal functions -%%-------------------------------------------------------------------- - -%% `infinity` to `infinity_value` rules: -%% 1. all infinity capacity will change to infinity_value -%% 2. if the rate of global and bucket both are `infinity`, -%% use `infinity_value` as bucket rate. see `emqx_limiter_server:get_counter_rate/2` -infinity_value() -> - %% 1 TB - 1099511627776. +calc_capacity(#{rate := infinity}) -> + infinity; +calc_capacity(#{rate := Rate, burst := Burst}) -> + erlang:floor(1000 * Rate / default_period()) + Burst. %%-------------------------------------------------------------------- %% Internal functions @@ -251,6 +248,17 @@ infinity_value() -> to_burst_rate(Str) -> to_rate(Str, false, true). +%% The default value of `capacity` is `infinity`, +%% but we have changed `capacity` to `burst` which should not be `infinity` +%% and its default value is 0, so we should convert `infinity` to 0 +to_burst(Str) -> + case to_rate(Str, true, true) of + {ok, infinity} -> + {ok, 0}; + Any -> + Any + end. + %% rate can be: 10 10MB 10MB/s 10MB/2s infinity %% e.g. the bytes_in regex tree is: %% @@ -335,7 +343,7 @@ to_quota(Str, Regex) -> {match, [Quota, ""]} -> {ok, erlang:list_to_integer(Quota)}; {match, ""} -> - {ok, infinity_value()}; + {ok, infinity}; _ -> {error, Str} end @@ -350,7 +358,8 @@ apply_unit("mb", Val) -> Val * ?KILOBYTE * ?KILOBYTE; apply_unit("gb", Val) -> Val * ?KILOBYTE * ?KILOBYTE * ?KILOBYTE; apply_unit(Unit, _) -> throw("invalid unit:" ++ Unit). -bucket_field(Type) when is_atom(Type) -> +%% A bucket with only one type +simple_bucket_field(Type) when is_atom(Type) -> fields(bucket_infinity) ++ [ {client, @@ -358,16 +367,22 @@ bucket_field(Type) when is_atom(Type) -> ?R_REF(?MODULE, client_opts), #{ desc => ?DESC(client), - required => false + required => false, + importance => importance_of_type(Type), + aliases => alias_of_type(Type) } )} ]. -bucket_fields(Types, ClientRef) -> + +%% A bucket with multi types +composite_bucket_fields(Types, ClientRef) -> [ {Type, ?HOCON(?R_REF(?MODULE, Opts), #{ desc => ?DESC(?MODULE, Type), - required => false + required => false, + importance => importance_of_type(Type), + aliases => alias_of_type(Type) })} || {Type, Opts} <- Types ] ++ @@ -382,12 +397,47 @@ bucket_fields(Types, ClientRef) -> )} ]. -client_fields(Types) -> +fields_of_bucket(Default) -> + [ + {rate, ?HOCON(rate(), #{desc => ?DESC(rate), default => Default})}, + {burst, + ?HOCON(burst(), #{ + desc => ?DESC(burst), + default => <<"0">>, + importance => ?IMPORTANCE_HIDDEN, + aliases => [capacity] + })}, + {initial, + ?HOCON(initial(), #{ + default => <<"0">>, + desc => ?DESC(initial), + importance => ?IMPORTANCE_HIDDEN + })} + ]. + +client_fields(Types, Meta) -> [ {Type, - ?HOCON(?R_REF(client_opts), #{ + ?HOCON(?R_REF(client_opts), Meta#{ desc => ?DESC(Type), - required => false + importance => importance_of_type(Type), + aliases => alias_of_type(Type) })} - || {Type, _} <- Types + || Type <- Types ]. + +importance_of_type(interval) -> + ?IMPORTANCE_HIDDEN; +importance_of_type(message_routing) -> + ?IMPORTANCE_HIDDEN; +importance_of_type(connection) -> + ?IMPORTANCE_HIDDEN; +importance_of_type(_) -> + ?DEFAULT_IMPORTANCE. + +alias_of_type(messages) -> + [message_in]; +alias_of_type(bytes) -> + [bytes_in]; +alias_of_type(_) -> + []. diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl index 44663ceeb..58db66f82 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl @@ -118,17 +118,24 @@ connect(_Id, _Type, undefined) -> {ok, emqx_htb_limiter:make_infinity_limiter()}; connect(Id, Type, Cfg) -> case find_limiter_cfg(Type, Cfg) of - {undefined, _} -> + {_ClientCfg, undefined, _NodeCfg} -> {ok, emqx_htb_limiter:make_infinity_limiter()}; + {#{rate := infinity}, #{rate := infinity}, #{rate := infinity}} -> + {ok, emqx_htb_limiter:make_infinity_limiter()}; + {ClientCfg, #{rate := infinity}, #{rate := infinity}} -> + {ok, + emqx_htb_limiter:make_token_bucket_limiter( + ClientCfg, emqx_limiter_bucket_ref:infinity_bucket() + )}; { - #{ - rate := BucketRate, - capacity := BucketSize - }, - #{rate := CliRate, capacity := CliSize} = ClientCfg + #{rate := CliRate} = ClientCfg, + #{rate := BucketRate} = BucketCfg, + _ } -> case emqx_limiter_manager:find_bucket(Id, Type) of {ok, Bucket} -> + BucketSize = emqx_limiter_schema:calc_capacity(BucketCfg), + CliSize = emqx_limiter_schema:calc_capacity(ClientCfg), {ok, if CliRate < BucketRate orelse CliSize < BucketSize -> @@ -493,12 +500,14 @@ make_root(#{rate := Rate, burst := Burst}) -> produced => 0.0 }. -do_add_bucket(Id, #{rate := Rate, capacity := Capacity} = Cfg, #{buckets := Buckets} = State) -> +do_add_bucket(_Id, #{rate := infinity}, #{root := #{rate := infinity}} = State) -> + State; +do_add_bucket(Id, #{rate := Rate} = Cfg, #{buckets := Buckets} = State) -> case maps:get(Id, Buckets, undefined) of undefined -> make_bucket(Id, Cfg, State); Bucket -> - Bucket2 = Bucket#{rate := Rate, capacity := Capacity}, + Bucket2 = Bucket#{rate := Rate, capacity := emqx_limiter_schema:calc_capacity(Cfg)}, State#{buckets := Buckets#{Id := Bucket2}} end. @@ -509,7 +518,7 @@ make_bucket(Id, Cfg, #{index := ?COUNTER_SIZE} = State) -> }); make_bucket( Id, - #{rate := Rate, capacity := Capacity} = Cfg, + #{rate := Rate} = Cfg, #{type := Type, counter := Counter, index := Index, buckets := Buckets} = State ) -> NewIndex = Index + 1, @@ -519,7 +528,7 @@ make_bucket( rate => Rate, obtained => Initial, correction => 0, - capacity => Capacity, + capacity => emqx_limiter_schema:calc_capacity(Cfg), counter => Counter, index => NewIndex }, @@ -541,19 +550,14 @@ do_del_bucket(Id, #{type := Type, buckets := Buckets} = State) -> get_initial_val( #{ initial := Initial, - rate := Rate, - capacity := Capacity + rate := Rate } ) -> - %% initial will nevner be infinity(see the emqx_limiter_schema) - InfVal = emqx_limiter_schema:infinity_value(), if Initial > 0 -> Initial; Rate =/= infinity -> - erlang:min(Rate, Capacity); - Capacity =/= infinity andalso Capacity =/= InfVal -> - Capacity; + Rate; true -> 0 end. @@ -568,11 +572,12 @@ call(Type, Msg) -> end. find_limiter_cfg(Type, #{rate := _} = Cfg) -> - {Cfg, find_client_cfg(Type, maps:get(client, Cfg, undefined))}; + {find_client_cfg(Type, maps:get(client, Cfg, undefined)), Cfg, find_node_cfg(Type)}; find_limiter_cfg(Type, Cfg) -> { + find_client_cfg(Type, emqx_utils_maps:deep_get([client, Type], Cfg, undefined)), maps:get(Type, Cfg, undefined), - find_client_cfg(Type, emqx_map_lib:deep_get([client, Type], Cfg, undefined)) + find_node_cfg(Type) }. find_client_cfg(Type, BucketCfg) -> @@ -585,3 +590,6 @@ merge_client_cfg(NodeCfg, undefined) -> NodeCfg; merge_client_cfg(NodeCfg, BucketCfg) -> maps:merge(NodeCfg, BucketCfg). + +find_node_cfg(Type) -> + emqx:get_config([limiter, Type], #{rate => infinity, burst => 0}). diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 4e5843166..f82aebe7c 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -427,12 +427,12 @@ pre_config_update([listeners, _Type, _Name], {create, _NewConf}, _RawConf) -> pre_config_update([listeners, _Type, _Name], {update, _Request}, undefined) -> {error, not_found}; pre_config_update([listeners, Type, Name], {update, Request}, RawConf) -> - NewConfT = emqx_map_lib:deep_merge(RawConf, Request), + NewConfT = emqx_utils_maps:deep_merge(RawConf, Request), NewConf = ensure_override_limiter_conf(NewConfT, Request), CertsDir = certs_dir(Type, Name), {ok, convert_certs(CertsDir, NewConf)}; pre_config_update([listeners, _Type, _Name], {action, _Action, Updated}, RawConf) -> - NewConf = emqx_map_lib:deep_merge(RawConf, Updated), + NewConf = emqx_utils_maps:deep_merge(RawConf, Updated), {ok, NewConf}; pre_config_update(_Path, _Request, RawConf) -> {ok, RawConf}. @@ -500,7 +500,7 @@ esockd_opts(ListenerId, Type, Opts0) -> ws_opts(Type, ListenerName, Opts) -> WsPaths = [ - {emqx_map_lib:deep_get([websocket, mqtt_path], Opts, "/mqtt"), emqx_ws_connection, #{ + {emqx_utils_maps:deep_get([websocket, mqtt_path], Opts, "/mqtt"), emqx_ws_connection, #{ zone => zone(Opts), listener => {Type, ListenerName}, limiter => limiter(Opts), @@ -538,7 +538,7 @@ esockd_access_rules(StrRules) -> [A, CIDR] = string:tokens(S, " "), %% esockd rules only use words 'allow' and 'deny', both are existing %% comparison of strings may be better, but there is a loss of backward compatibility - case emqx_misc:safe_to_existing_atom(A) of + case emqx_utils:safe_to_existing_atom(A) of {ok, Action} -> [ { @@ -560,7 +560,7 @@ esockd_access_rules(StrRules) -> merge_default(Options) -> case lists:keytake(tcp_options, 1, Options) of {value, {tcp_options, TcpOpts}, Options1} -> - [{tcp_options, emqx_misc:merge_opts(?MQTT_SOCKOPTS, TcpOpts)} | Options1]; + [{tcp_options, emqx_utils:merge_opts(?MQTT_SOCKOPTS, TcpOpts)} | Options1]; false -> [{tcp_options, ?MQTT_SOCKOPTS} | Options] end. diff --git a/apps/emqx/src/emqx_logger_jsonfmt.erl b/apps/emqx/src/emqx_logger_jsonfmt.erl index 22cf75153..0e72fd2b5 100644 --- a/apps/emqx/src/emqx_logger_jsonfmt.erl +++ b/apps/emqx/src/emqx_logger_jsonfmt.erl @@ -62,11 +62,11 @@ %% The JSON object is pretty-printed. %% NOTE: do not use this function for logging. best_effort_json(Input) -> - best_effort_json(Input, [space, {indent, 4}]). + best_effort_json(Input, [pretty, force_utf8]). best_effort_json(Input, Opts) -> Config = #{depth => unlimited, single_line => true}, JsonReady = best_effort_json_obj(Input, Config), - jsx:encode(JsonReady, Opts). + emqx_utils_json:encode(JsonReady, Opts). -spec format(logger:log_event(), config()) -> iodata(). format(#{level := Level, msg := Msg, meta := Meta} = Event, Config0) when is_map(Config0) -> @@ -92,7 +92,7 @@ format(Msg, Meta, Config) -> } end, Data = maps:without([report_cb], Data0), - jiffy:encode(json_obj(Data, Config)). + emqx_utils_json:encode(json_obj(Data, Config)). maybe_format_msg({report, Report} = Msg, #{report_cb := Cb} = Meta, Config) -> case is_map(Report) andalso Cb =:= ?DEFAULT_FORMATTER of @@ -378,15 +378,15 @@ p_config() -> best_effort_json_test() -> ?assertEqual( - <<"{}">>, + <<"{\n \n}">>, emqx_logger_jsonfmt:best_effort_json([]) ), ?assertEqual( - <<"{\n \"key\": []\n}">>, + <<"{\n \"key\" : [\n \n ]\n}">>, emqx_logger_jsonfmt:best_effort_json(#{key => []}) ), ?assertEqual( - <<"[\n {\n \"key\": []\n }\n]">>, + <<"[\n {\n \"key\" : [\n \n ]\n }\n]">>, emqx_logger_jsonfmt:best_effort_json([#{key => []}]) ), ok. diff --git a/apps/emqx/src/emqx_metrics.erl b/apps/emqx/src/emqx_metrics.erl index c2e297623..21a114c0f 100644 --- a/apps/emqx/src/emqx_metrics.erl +++ b/apps/emqx/src/emqx_metrics.erl @@ -541,7 +541,7 @@ init([]) -> CRef = counters:new(?MAX_SIZE, [write_concurrency]), ok = persistent_term:put(?MODULE, CRef), % Create index mapping table - ok = emqx_tables:new(?TAB, [{keypos, 2}, {read_concurrency, true}]), + ok = emqx_utils_ets:new(?TAB, [{keypos, 2}, {read_concurrency, true}]), Metrics = lists:append([ ?BYTES_METRICS, ?PACKET_METRICS, diff --git a/apps/emqx/src/emqx_ocsp_cache.erl b/apps/emqx/src/emqx_ocsp_cache.erl index 4e7ada044..3bb10ee5c 100644 --- a/apps/emqx/src/emqx_ocsp_cache.erl +++ b/apps/emqx/src/emqx_ocsp_cache.erl @@ -110,7 +110,7 @@ register_listener(ListenerID, Opts) -> -spec inject_sni_fun(emqx_listeners:listener_id(), map()) -> map(). inject_sni_fun(ListenerID, Conf0) -> SNIFun = emqx_const_v1:make_sni_fun(ListenerID), - Conf = emqx_map_lib:deep_merge(Conf0, #{ssl_options => #{sni_fun => SNIFun}}), + Conf = emqx_utils_maps:deep_merge(Conf0, #{ssl_options => #{sni_fun => SNIFun}}), ok = ?MODULE:register_listener(ListenerID, Conf), Conf. @@ -120,7 +120,7 @@ inject_sni_fun(ListenerID, Conf0) -> init(_Args) -> logger:set_process_metadata(#{domain => [emqx, ocsp, cache]}), - emqx_tables:new(?CACHE_TAB, [ + emqx_utils_ets:new(?CACHE_TAB, [ named_table, public, {heir, whereis(emqx_kernel_sup), none}, @@ -149,7 +149,7 @@ handle_call({register_listener, ListenerID, Conf}, _From, State0) -> msg => "registering_ocsp_cache", listener_id => ListenerID }), - RefreshInterval0 = emqx_map_lib:deep_get([ssl_options, ocsp, refresh_interval], Conf), + RefreshInterval0 = emqx_utils_maps:deep_get([ssl_options, ocsp, refresh_interval], Conf), RefreshInterval = max(RefreshInterval0, ?MIN_REFRESH_INTERVAL), State = State0#{{refresh_interval, ListenerID} => RefreshInterval}, %% we need to pass the config along because this might be called @@ -476,9 +476,9 @@ ensure_timer(ListenerID, State, Timeout) -> ensure_timer(ListenerID, {refresh, ListenerID}, State, Timeout). ensure_timer(ListenerID, Message, State, Timeout) -> - emqx_misc:cancel_timer(maps:get(?REFRESH_TIMER(ListenerID), State, undefined)), + emqx_utils:cancel_timer(maps:get(?REFRESH_TIMER(ListenerID), State, undefined)), State#{ - ?REFRESH_TIMER(ListenerID) => emqx_misc:start_timer( + ?REFRESH_TIMER(ListenerID) => emqx_utils:start_timer( Timeout, Message ) diff --git a/apps/emqx/src/emqx_os_mon.erl b/apps/emqx/src/emqx_os_mon.erl index c5ce35bf9..4810798eb 100644 --- a/apps/emqx/src/emqx_os_mon.erl +++ b/apps/emqx/src/emqx_os_mon.erl @@ -180,8 +180,8 @@ code_change(_OldVsn, State, _Extra) -> %% Internal functions %%-------------------------------------------------------------------- cancel_outdated_timer(#{mem_time_ref := MemRef, cpu_time_ref := CpuRef}) -> - emqx_misc:cancel_timer(MemRef), - emqx_misc:cancel_timer(CpuRef), + emqx_utils:cancel_timer(MemRef), + emqx_utils:cancel_timer(CpuRef), ok. start_cpu_check_timer() -> @@ -204,7 +204,7 @@ start_mem_check_timer() -> end. start_timer(Interval, Msg) -> - emqx_misc:start_timer(Interval, Msg). + emqx_utils:start_timer(Interval, Msg). update_mem_alarm_status(HWM) when HWM > 1.0 orelse HWM < 0.0 -> ?SLOG(warning, #{msg => "discarded_out_of_range_mem_alarm_threshold", value => HWM}), diff --git a/apps/emqx/src/emqx_pool.erl b/apps/emqx/src/emqx_pool.erl index 1691a533a..1cb5f429c 100644 --- a/apps/emqx/src/emqx_pool.erl +++ b/apps/emqx/src/emqx_pool.erl @@ -57,7 +57,7 @@ -spec start_link(atom(), pos_integer()) -> startlink_ret(). start_link(Pool, Id) -> gen_server:start_link( - {local, emqx_misc:proc_name(?MODULE, Id)}, + {local, emqx_utils:proc_name(?MODULE, Id)}, ?MODULE, [Pool, Id], [{hibernate_after, 1000}] diff --git a/apps/emqx/src/emqx_router.erl b/apps/emqx/src/emqx_router.erl index 7c9cc61b0..42430af5d 100644 --- a/apps/emqx/src/emqx_router.erl +++ b/apps/emqx/src/emqx_router.erl @@ -98,7 +98,7 @@ mnesia(boot) -> -spec start_link(atom(), pos_integer()) -> startlink_ret(). start_link(Pool, Id) -> gen_server:start_link( - {local, emqx_misc:proc_name(?MODULE, Id)}, + {local, emqx_utils:proc_name(?MODULE, Id)}, ?MODULE, [Pool, Id], [{hibernate_after, 1000}] diff --git a/apps/emqx/src/emqx_rpc.erl b/apps/emqx/src/emqx_rpc.erl index e1b5122c4..062bde68b 100644 --- a/apps/emqx/src/emqx_rpc.erl +++ b/apps/emqx/src/emqx_rpc.erl @@ -27,6 +27,8 @@ cast/5, multicall/4, multicall/5, + multicall_on_running/5, + on_running/3, unwrap_erpc/1 ]). @@ -91,6 +93,17 @@ multicall(Nodes, Mod, Fun, Args) -> multicall(Key, Nodes, Mod, Fun, Args) -> gen_rpc:multicall(rpc_nodes([{Key, Node} || Node <- Nodes]), Mod, Fun, Args). +-spec multicall_on_running([node()], module(), atom(), list(), timeout()) -> [term() | {error, _}]. +multicall_on_running(Nodes, Mod, Fun, Args, Timeout) -> + unwrap_erpc(erpc:multicall(Nodes, emqx_rpc, on_running, [Mod, Fun, Args], Timeout)). + +-spec on_running(module(), atom(), list()) -> term(). +on_running(Mod, Fun, Args) -> + case emqx:is_running() of + true -> apply(Mod, Fun, Args); + false -> error(emqx_down) + end. + -spec cast(node(), module(), atom(), list()) -> cast_result(). cast(Node, Mod, Fun, Args) -> %% Note: using a non-ordered cast here, since the generated key is diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 6bfff38d3..540c681b3 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -44,6 +44,7 @@ -type port_number() :: 1..65536. -type server_parse_option() :: #{default_port => port_number(), no_port => boolean()}. -type url() :: binary(). +-type json_binary() :: binary(). -typerefl_from_string({duration/0, emqx_schema, to_duration}). -typerefl_from_string({duration_s/0, emqx_schema, to_duration_s}). @@ -58,6 +59,7 @@ -typerefl_from_string({cipher/0, emqx_schema, to_erl_cipher_suite}). -typerefl_from_string({comma_separated_atoms/0, emqx_schema, to_comma_separated_atoms}). -typerefl_from_string({url/0, emqx_schema, to_url}). +-typerefl_from_string({json_binary/0, emqx_schema, to_json_binary}). -export([ validate_heap_size/1, @@ -84,7 +86,8 @@ to_ip_port/1, to_erl_cipher_suite/1, to_comma_separated_atoms/1, - to_url/1 + to_url/1, + to_json_binary/1 ]). -export([ @@ -112,7 +115,8 @@ ip_port/0, cipher/0, comma_separated_atoms/0, - url/0 + url/0, + json_binary/0 ]). -export([namespace/0, roots/0, roots/1, fields/1, desc/1, tags/0]). @@ -141,25 +145,31 @@ roots(high) -> {"listeners", sc( ref("listeners"), - #{} - )}, - {"zones", - sc( - map("name", ref("zone")), - #{desc => ?DESC(zones)} + #{importance => ?IMPORTANCE_HIGH} )}, {"mqtt", sc( ref("mqtt"), - #{desc => ?DESC(mqtt)} + #{ + desc => ?DESC(mqtt), + importance => ?IMPORTANCE_MEDIUM + } + )}, + {"zones", + sc( + map("name", ref("zone")), + #{ + desc => ?DESC(zones), + importance => ?IMPORTANCE_LOW + } )}, {?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME, authentication(global)}, - %% NOTE: authorization schema here is only to keep emqx app prue + %% NOTE: authorization schema here is only to keep emqx app pure %% the full schema for EMQX node is injected in emqx_conf_schema. {?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME, sc( ref(?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME), - #{} + #{importance => ?IMPORTANCE_HIDDEN} )} ]; roots(medium) -> @@ -182,7 +192,7 @@ roots(medium) -> {"overload_protection", sc( ref("overload_protection"), - #{} + #{importance => ?IMPORTANCE_HIDDEN} )} ]; roots(low) -> @@ -195,12 +205,16 @@ roots(low) -> {"conn_congestion", sc( ref("conn_congestion"), - #{} + #{ + importance => ?IMPORTANCE_HIDDEN + } )}, {"stats", sc( ref("stats"), - #{} + #{ + importance => ?IMPORTANCE_HIDDEN + } )}, {"sysmon", sc( @@ -215,17 +229,17 @@ roots(low) -> {"flapping_detect", sc( ref("flapping_detect"), - #{} + #{importance => ?IMPORTANCE_HIDDEN} )}, {"persistent_session_store", sc( ref("persistent_session_store"), - #{} + #{importance => ?IMPORTANCE_HIDDEN} )}, {"trace", sc( ref("trace"), - #{} + #{importance => ?IMPORTANCE_HIDDEN} )}, {"crl_cache", sc( @@ -335,6 +349,7 @@ fields("stats") -> boolean(), #{ default => true, + importance => ?IMPORTANCE_HIDDEN, desc => ?DESC(stats_enable) } )} @@ -605,8 +620,7 @@ fields("mqtt") -> )} ]; fields("zone") -> - Fields = emqx_zone_schema:roots(), - [{F, ref(emqx_zone_schema, F)} || F <- Fields]; + emqx_zone_schema:zone(); fields("flapping_detect") -> [ {"enable", @@ -614,25 +628,27 @@ fields("flapping_detect") -> boolean(), #{ default => false, + deprecated => {since, "5.0.23"}, desc => ?DESC(flapping_detect_enable) } )}, - {"max_count", - sc( - integer(), - #{ - default => 15, - desc => ?DESC(flapping_detect_max_count) - } - )}, {"window_time", sc( - duration(), + hoconsc:union([disabled, duration()]), #{ - default => <<"1m">>, + default => disabled, + importance => ?IMPORTANCE_HIGH, desc => ?DESC(flapping_detect_window_time) } )}, + {"max_count", + sc( + non_neg_integer(), + #{ + default => 15, + desc => ?DESC(flapping_detect_max_count) + } + )}, {"ban_time", sc( duration(), @@ -1494,12 +1510,14 @@ fields("broker") -> ref("broker_perf"), #{importance => ?IMPORTANCE_HIDDEN} )}, + %% FIXME: Need new design for shared subscription group {"shared_subscription_group", sc( map(name, ref("shared_subscription_group")), #{ example => #{<<"example_group">> => #{<<"strategy">> => <<"random">>}}, - desc => ?DESC(shared_subscription_group_strategy) + desc => ?DESC(shared_subscription_group_strategy), + importance => ?IMPORTANCE_HIDDEN } )} ]; @@ -1615,7 +1633,9 @@ fields("sysmon") -> {"top", sc( ref("sysmon_top"), - #{} + %% Userful monitoring solution when benchmarking, + %% but hardly common enough for regular users. + #{importance => ?IMPORTANCE_HIDDEN} )} ]; fields("sysmon_vm") -> @@ -1849,6 +1869,8 @@ fields("trace") -> {"payload_encode", sc(hoconsc:enum([hex, text, hidden]), #{ default => text, + deprecated => {since, "5.0.22"}, + importance => ?IMPORTANCE_HIDDEN, desc => ?DESC(fields_trace_payload_encode) })} ]. @@ -2205,6 +2227,7 @@ common_ssl_opts_schema(Defaults) -> #{ default => AvailableVersions, desc => ?DESC(common_ssl_opts_schema_versions), + importance => ?IMPORTANCE_HIGH, validator => fun(Inputs) -> validate_tls_versions(AvailableVersions, Inputs) end } )}, @@ -2215,6 +2238,7 @@ common_ssl_opts_schema(Defaults) -> #{ default => <<"emqx_tls_psk:lookup">>, converter => fun ?MODULE:user_lookup_fun_tr/2, + importance => ?IMPORTANCE_HIDDEN, desc => ?DESC(common_ssl_opts_schema_user_lookup_fun) } )}, @@ -2300,8 +2324,6 @@ server_ssl_opts_schema(Defaults, IsRanchListener) -> ref("ocsp"), #{ required => false, - %% TODO: remove after e5.0.2 - importance => ?IMPORTANCE_HIDDEN, validator => fun ocsp_inner_validator/1 } )}, @@ -2310,6 +2332,7 @@ server_ssl_opts_schema(Defaults, IsRanchListener) -> boolean(), #{ default => false, + importance => ?IMPORTANCE_MEDIUM, desc => ?DESC("server_ssl_opts_schema_enable_crl_check") } )} @@ -2321,7 +2344,7 @@ mqtt_ssl_listener_ssl_options_validator(Conf) -> fun ocsp_outer_validator/1, fun crl_outer_validator/1 ], - case emqx_misc:pipeline(Checks, Conf, not_used) of + case emqx_utils:pipeline(Checks, Conf, not_used) of {ok, _, _} -> ok; {error, Reason, _NotUsed} -> @@ -2342,7 +2365,7 @@ ocsp_outer_validator(_Conf) -> ok. ocsp_inner_validator(#{enable_ocsp_stapling := _} = Conf) -> - ocsp_inner_validator(emqx_map_lib:binary_key_map(Conf)); + ocsp_inner_validator(emqx_utils_maps:binary_key_map(Conf)); ocsp_inner_validator(#{<<"enable_ocsp_stapling">> := false} = _Conf) -> ok; ocsp_inner_validator(#{<<"enable_ocsp_stapling">> := true} = Conf) -> @@ -2576,6 +2599,14 @@ to_url(Str) -> Error end. +to_json_binary(Str) -> + case emqx_utils_json:safe_decode(Str) of + {ok, _} -> + {ok, iolist_to_binary(Str)}; + Error -> + Error + end. + to_bar_separated_list(Str) -> {ok, string:tokens(Str, "| ")}. @@ -2642,20 +2673,22 @@ to_atom(Str) when is_list(Str) -> to_atom(Bin) when is_binary(Bin) -> binary_to_atom(Bin, utf8). -validate_heap_size(Siz) -> +validate_heap_size(Siz) when is_integer(Siz) -> MaxSiz = case erlang:system_info(wordsize) of % arch_64 - 8 -> - (1 bsl 59) - 1; + 8 -> (1 bsl 59) - 1; % arch_32 - 4 -> - (1 bsl 27) - 1 + 4 -> (1 bsl 27) - 1 end, case Siz > MaxSiz of - true -> error(io_lib:format("force_shutdown_policy: heap-size ~ts is too large", [Siz])); - false -> ok - end. + true -> + {error, #{reason => max_heap_size_too_large, maximum => MaxSiz}}; + false -> + ok + end; +validate_heap_size(_SizStr) -> + {error, invalid_heap_size}. validate_alarm_actions(Actions) -> UnSupported = lists:filter( @@ -2732,10 +2765,16 @@ str(S) when is_list(S) -> S. authentication(Which) -> - Desc = + {Importance, Desc} = case Which of - global -> ?DESC(global_authentication); - listener -> ?DESC(listener_authentication) + global -> + %% For root level authentication, it is recommended to configure + %% from the dashboard or API. + %% Hence it's considered a low-importance when it comes to + %% configuration importance. + {?IMPORTANCE_LOW, ?DESC(global_authentication)}; + listener -> + {?IMPORTANCE_HIDDEN, ?DESC(listener_authentication)} end, %% poor man's dependency injection %% this is due to the fact that authn is implemented outside of 'emqx' app. @@ -2748,7 +2787,11 @@ authentication(Which) -> Module -> Module:root_type() end, - hoconsc:mk(Type, #{desc => Desc, converter => fun ensure_array/2}). + hoconsc:mk(Type, #{ + desc => Desc, + converter => fun ensure_array/2, + importance => Importance + }). %% the older version schema allows individual element (instead of a chain) in config ensure_array(undefined, _) -> undefined; diff --git a/apps/emqx/src/emqx_sequence.erl b/apps/emqx/src/emqx_sequence.erl index 60596324a..7acc87256 100644 --- a/apps/emqx/src/emqx_sequence.erl +++ b/apps/emqx/src/emqx_sequence.erl @@ -39,7 +39,7 @@ %% @doc Create a sequence. -spec create(name()) -> ok. create(Name) -> - emqx_tables:new(Name, [public, set, {write_concurrency, true}]). + emqx_utils_ets:new(Name, [public, set, {write_concurrency, true}]). %% @doc Next value of the sequence. -spec nextval(name(), key()) -> seqid(). diff --git a/apps/emqx/src/emqx_session.erl b/apps/emqx/src/emqx_session.erl index a13dfe491..8b15340e9 100644 --- a/apps/emqx/src/emqx_session.erl +++ b/apps/emqx/src/emqx_session.erl @@ -941,7 +941,7 @@ age(Now, Ts) -> Now - Ts. %%-------------------------------------------------------------------- set_field(Name, Value, Session) -> - Pos = emqx_misc:index_of(Name, record_info(fields, session)), + Pos = emqx_utils:index_of(Name, record_info(fields, session)), setelement(Pos + 1, Session, Value). get_mqueue(#session{mqueue = Q}) -> diff --git a/apps/emqx/src/emqx_session_router.erl b/apps/emqx/src/emqx_session_router.erl index 0d4972e8c..0435ddca3 100644 --- a/apps/emqx/src/emqx_session_router.erl +++ b/apps/emqx/src/emqx_session_router.erl @@ -95,7 +95,7 @@ create_table(Tab, Storage) -> %%-------------------------------------------------------------------- create_init_tab() -> - emqx_tables:new(?SESSION_INIT_TAB, [ + emqx_utils_ets:new(?SESSION_INIT_TAB, [ public, {read_concurrency, true}, {write_concurrency, true} @@ -104,7 +104,7 @@ create_init_tab() -> -spec start_link(atom(), pos_integer()) -> startlink_ret(). start_link(Pool, Id) -> gen_server:start_link( - {local, emqx_misc:proc_name(?MODULE, Id)}, + {local, emqx_utils:proc_name(?MODULE, Id)}, ?MODULE, [Pool, Id], [{hibernate_after, 1000}] @@ -182,7 +182,7 @@ pending(SessionID, MarkerIDs) -> call(pick(SessionID), {pending, SessionID, MarkerIDs}). buffer(SessionID, STopic, Msg) -> - case emqx_tables:lookup_value(?SESSION_INIT_TAB, SessionID) of + case emqx_utils_ets:lookup_value(?SESSION_INIT_TAB, SessionID) of undefined -> ok; Worker -> emqx_session_router_worker:buffer(Worker, STopic, Msg) end. @@ -194,7 +194,7 @@ resume_begin(From, SessionID) when is_pid(From), is_binary(SessionID) -> -spec resume_end(pid(), binary()) -> {'ok', [emqx_types:message()]} | {'error', term()}. resume_end(From, SessionID) when is_pid(From), is_binary(SessionID) -> - case emqx_tables:lookup_value(?SESSION_INIT_TAB, SessionID) of + case emqx_utils_ets:lookup_value(?SESSION_INIT_TAB, SessionID) of undefined -> ?tp(ps_session_not_found, #{sid => SessionID}), {error, not_found}; @@ -249,7 +249,7 @@ handle_cast({delete_routes, SessionID, Subscriptions}, State) -> ok = lists:foreach(Fun, maps:to_list(Subscriptions)), {noreply, State}; handle_cast({resume_end, SessionID, Pid}, State) -> - case emqx_tables:lookup_value(?SESSION_INIT_TAB, SessionID) of + case emqx_utils_ets:lookup_value(?SESSION_INIT_TAB, SessionID) of undefined -> skip; P when P =:= Pid -> ets:delete(?SESSION_INIT_TAB, SessionID); P when is_pid(P) -> skip @@ -283,7 +283,7 @@ init_resume_worker(RemotePid, SessionID, #{pmon := Pmon} = State) -> error; {ok, Pid} -> Pmon1 = emqx_pmon:monitor(Pid, Pmon), - case emqx_tables:lookup_value(?SESSION_INIT_TAB, SessionID) of + case emqx_utils_ets:lookup_value(?SESSION_INIT_TAB, SessionID) of undefined -> {ok, Pid, State#{pmon => Pmon1}}; {_, OldPid} -> diff --git a/apps/emqx/src/emqx_shared_sub.erl b/apps/emqx/src/emqx_shared_sub.erl index 061f2a42f..d7dc8c5a6 100644 --- a/apps/emqx/src/emqx_shared_sub.erl +++ b/apps/emqx/src/emqx_shared_sub.erl @@ -399,9 +399,11 @@ init([]) -> ok = mria:wait_for_tables([?TAB]), {ok, _} = mnesia:subscribe({table, ?TAB, simple}), {atomic, PMon} = mria:transaction(?SHARED_SUB_SHARD, fun ?MODULE:init_monitors/0), - ok = emqx_tables:new(?SHARED_SUBS, [protected, bag]), - ok = emqx_tables:new(?ALIVE_SUBS, [protected, set, {read_concurrency, true}]), - ok = emqx_tables:new(?SHARED_SUBS_ROUND_ROBIN_COUNTER, [public, set, {write_concurrency, true}]), + ok = emqx_utils_ets:new(?SHARED_SUBS, [protected, bag]), + ok = emqx_utils_ets:new(?ALIVE_SUBS, [protected, set, {read_concurrency, true}]), + ok = emqx_utils_ets:new(?SHARED_SUBS_ROUND_ROBIN_COUNTER, [ + public, set, {write_concurrency, true} + ]), {ok, update_stats(#state{pmon = PMon})}. init_monitors() -> diff --git a/apps/emqx/src/emqx_stats.erl b/apps/emqx/src/emqx_stats.erl index ed901d9a9..ef9109e33 100644 --- a/apps/emqx/src/emqx_stats.erl +++ b/apps/emqx/src/emqx_stats.erl @@ -201,7 +201,7 @@ cast(Msg) -> gen_server:cast(?SERVER, Msg). %%-------------------------------------------------------------------- init(#{tick_ms := TickMs}) -> - ok = emqx_tables:new(?TAB, [public, set, {write_concurrency, true}]), + ok = emqx_utils_ets:new(?TAB, [public, set, {write_concurrency, true}]), Stats = lists:append([ ?CONNECTION_STATS, ?CHANNEL_STATS, @@ -213,7 +213,7 @@ init(#{tick_ms := TickMs}) -> {ok, start_timer(#state{updates = [], tick_ms = TickMs}), hibernate}. start_timer(#state{tick_ms = Ms} = State) -> - State#state{timer = emqx_misc:start_timer(Ms, tick)}. + State#state{timer = emqx_utils:start_timer(Ms, tick)}. handle_call(stop, _From, State) -> {stop, normal, ok, State}; @@ -301,7 +301,7 @@ handle_info(Info, State) -> {noreply, State}. terminate(_Reason, #state{timer = TRef}) -> - emqx_misc:cancel_timer(TRef). + emqx_utils:cancel_timer(TRef). code_change(_OldVsn, State, _Extra) -> {ok, State}. diff --git a/apps/emqx/src/emqx_sys.erl b/apps/emqx/src/emqx_sys.erl index a5f14e32a..509429796 100644 --- a/apps/emqx/src/emqx_sys.erl +++ b/apps/emqx/src/emqx_sys.erl @@ -62,7 +62,7 @@ -endif. -import(emqx_topic, [systop/1]). --import(emqx_misc, [start_timer/2]). +-import(emqx_utils, [start_timer/2]). -record(state, { heartbeat :: maybe(reference()), @@ -222,7 +222,7 @@ handle_info(Info, State) -> terminate(_Reason, #state{heartbeat = TRef1, ticker = TRef2}) -> _ = emqx_config_handler:remove_handler(?CONF_KEY_PATH), unload_event_hooks(sys_event_messages()), - lists:foreach(fun emqx_misc:cancel_timer/1, [TRef1, TRef2]). + lists:foreach(fun emqx_utils:cancel_timer/1, [TRef1, TRef2]). unload_event_hooks([]) -> ok; @@ -348,7 +348,7 @@ publish(Event, Payload) when Event == unsubscribed -> Topic = event_topic(Event, Payload), - safe_publish(Topic, emqx_json:encode(Payload)). + safe_publish(Topic, emqx_utils_json:encode(Payload)). metric_topic(Name) -> translate_topic("metrics/", Name). diff --git a/apps/emqx/src/emqx_sys_mon.erl b/apps/emqx/src/emqx_sys_mon.erl index 6ff68820e..f1190f586 100644 --- a/apps/emqx/src/emqx_sys_mon.erl +++ b/apps/emqx/src/emqx_sys_mon.erl @@ -77,7 +77,7 @@ init([]) -> {ok, start_timer(#{timer => undefined, events => []})}. start_timer(State) -> - State#{timer := emqx_misc:start_timer(timer:seconds(2), reset)}. + State#{timer := emqx_utils:start_timer(timer:seconds(2), reset)}. sysm_opts(VM) -> sysm_opts(maps:to_list(VM), []). @@ -204,7 +204,7 @@ handle_info(Info, State) -> {noreply, State}. terminate(_Reason, #{timer := TRef}) -> - emqx_misc:cancel_timer(TRef), + emqx_utils:cancel_timer(TRef), ok. code_change(_OldVsn, State, _Extra) -> diff --git a/apps/emqx/src/emqx_tls_lib.erl b/apps/emqx/src/emqx_tls_lib.erl index 47797b326..d1c57bf0d 100644 --- a/apps/emqx/src/emqx_tls_lib.erl +++ b/apps/emqx/src/emqx_tls_lib.erl @@ -317,7 +317,9 @@ ensure_ssl_files(Dir, SSL, Opts) -> ensure_ssl_files(_Dir, SSL, [], _Opts) -> {ok, SSL}; ensure_ssl_files(Dir, SSL, [KeyPath | KeyPaths], Opts) -> - case ensure_ssl_file(Dir, KeyPath, SSL, emqx_map_lib:deep_get(KeyPath, SSL, undefined), Opts) of + case + ensure_ssl_file(Dir, KeyPath, SSL, emqx_utils_maps:deep_get(KeyPath, SSL, undefined), Opts) + of {ok, NewSSL} -> ensure_ssl_files(Dir, NewSSL, KeyPaths, Opts); {error, Reason} -> @@ -332,7 +334,7 @@ delete_ssl_files(Dir, NewOpts0, OldOpts0) -> {ok, OldOpts} = ensure_ssl_files(Dir, OldOpts0, #{dry_run => DryRun}), Get = fun (_KP, undefined) -> undefined; - (KP, Opts) -> emqx_map_lib:deep_get(KP, Opts, undefined) + (KP, Opts) -> emqx_utils_maps:deep_get(KP, Opts, undefined) end, lists:foreach( fun(KeyPath) -> delete_old_file(Get(KeyPath, NewOpts), Get(KeyPath, OldOpts)) end, @@ -372,7 +374,7 @@ do_ensure_ssl_file(Dir, KeyPath, SSL, MaybePem, DryRun) -> true -> case save_pem_file(Dir, KeyPath, MaybePem, DryRun) of {ok, Path} -> - NewSSL = emqx_map_lib:deep_put(KeyPath, SSL, Path), + NewSSL = emqx_utils_maps:deep_put(KeyPath, SSL, Path), {ok, NewSSL}; {error, Reason} -> {error, Reason} @@ -482,9 +484,9 @@ is_valid_pem_file(Path) -> %% so they are forced to upload a cert file, or use an existing file path. -spec drop_invalid_certs(map()) -> map(). drop_invalid_certs(#{enable := False} = SSL) when ?IS_FALSE(False) -> - lists:foldl(fun emqx_map_lib:deep_remove/2, SSL, ?SSL_FILE_OPT_PATHS_A); + lists:foldl(fun emqx_utils_maps:deep_remove/2, SSL, ?SSL_FILE_OPT_PATHS_A); drop_invalid_certs(#{<<"enable">> := False} = SSL) when ?IS_FALSE(False) -> - lists:foldl(fun emqx_map_lib:deep_remove/2, SSL, ?SSL_FILE_OPT_PATHS); + lists:foldl(fun emqx_utils_maps:deep_remove/2, SSL, ?SSL_FILE_OPT_PATHS); drop_invalid_certs(#{enable := True} = SSL) when ?IS_TRUE(True) -> do_drop_invalid_certs(?SSL_FILE_OPT_PATHS_A, SSL); drop_invalid_certs(#{<<"enable">> := True} = SSL) when ?IS_TRUE(True) -> @@ -493,7 +495,7 @@ drop_invalid_certs(#{<<"enable">> := True} = SSL) when ?IS_TRUE(True) -> do_drop_invalid_certs([], SSL) -> SSL; do_drop_invalid_certs([KeyPath | KeyPaths], SSL) -> - case emqx_map_lib:deep_get(KeyPath, SSL, undefined) of + case emqx_utils_maps:deep_get(KeyPath, SSL, undefined) of undefined -> do_drop_invalid_certs(KeyPaths, SSL); PemOrPath -> @@ -501,7 +503,7 @@ do_drop_invalid_certs([KeyPath | KeyPaths], SSL) -> true -> do_drop_invalid_certs(KeyPaths, SSL); {error, _} -> - do_drop_invalid_certs(KeyPaths, emqx_map_lib:deep_remove(KeyPath, SSL)) + do_drop_invalid_certs(KeyPaths, emqx_utils_maps:deep_remove(KeyPath, SSL)) end end. @@ -586,7 +588,9 @@ ensure_ssl_file_key(_SSL, []) -> ok; ensure_ssl_file_key(SSL, RequiredKeyPaths) -> NotFoundRef = make_ref(), - Filter = fun(KeyPath) -> NotFoundRef =:= emqx_map_lib:deep_get(KeyPath, SSL, NotFoundRef) end, + Filter = fun(KeyPath) -> + NotFoundRef =:= emqx_utils_maps:deep_get(KeyPath, SSL, NotFoundRef) + end, case lists:filter(Filter, RequiredKeyPaths) of [] -> ok; Miss -> {error, #{reason => ssl_file_option_not_found, which_options => Miss}} diff --git a/apps/emqx/src/emqx_trace/emqx_trace.erl b/apps/emqx/src/emqx_trace/emqx_trace.erl index ea6736038..91194772f 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace.erl @@ -21,6 +21,7 @@ -include_lib("emqx/include/logger.hrl"). -include_lib("kernel/include/file.hrl"). -include_lib("snabbkaffe/include/trace.hrl"). +-include_lib("emqx/include/emqx_trace.hrl"). -export([ publish/1, @@ -54,8 +55,6 @@ -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). --include("emqx_trace.hrl"). - -ifdef(TEST). -export([ log_file/2, @@ -147,7 +146,11 @@ list(Enable) -> -spec create([{Key :: binary(), Value :: binary()}] | #{atom() => binary()}) -> {ok, #?TRACE{}} - | {error, {duplicate_condition, iodata()} | {already_existed, iodata()} | iodata()}. + | {error, + {duplicate_condition, iodata()} + | {already_existed, iodata()} + | {bad_type, any()} + | iodata()}. create(Trace) -> case mnesia:table_info(?TRACE, size) < ?MAX_SIZE of true -> @@ -222,14 +225,16 @@ format(Traces) -> init([]) -> erlang:process_flag(trap_exit, true), + Fields = record_info(fields, ?TRACE), ok = mria:create_table(?TRACE, [ {type, set}, {rlog_shard, ?SHARD}, {storage, disc_copies}, {record_name, ?TRACE}, - {attributes, record_info(fields, ?TRACE)} + {attributes, Fields} ]), ok = mria:wait_for_tables([?TRACE]), + maybe_migrate_trace(Fields), {ok, _} = mnesia:subscribe({table, ?TRACE, simple}), ok = filelib:ensure_dir(filename:join([trace_dir(), dummy])), ok = filelib:ensure_dir(filename:join([zip_dir(), dummy])), @@ -267,7 +272,7 @@ handle_info({timeout, TRef, update_trace}, #{timer := TRef} = State) -> ?tp(update_trace_done, #{}), {noreply, State#{timer => NextTRef}}; handle_info({mnesia_table_event, _Events}, State = #{timer := TRef}) -> - emqx_misc:cancel_timer(TRef), + emqx_utils:cancel_timer(TRef), handle_info({timeout, TRef, update_trace}, State); handle_info(Info, State) -> ?SLOG(error, #{unexpected_info => Info}), @@ -275,7 +280,7 @@ handle_info(Info, State) -> terminate(_Reason, #{timer := TRef}) -> _ = mnesia:unsubscribe({table, ?TRACE, simple}), - emqx_misc:cancel_timer(TRef), + emqx_utils:cancel_timer(TRef), stop_all_trace_handler(), update_trace_handler(), _ = file:del_dir_r(zip_dir()), @@ -297,7 +302,7 @@ update_trace(Traces) -> ok = stop_trace(NeedStop, Started), clean_stale_trace_files(), NextTime = find_closest_time(Traces, Now), - emqx_misc:start_timer(NextTime, update_trace). + emqx_utils:start_timer(NextTime, update_trace). stop_all_trace_handler() -> lists:foreach( @@ -358,9 +363,10 @@ start_trace(Trace) -> name = Name, type = Type, filter = Filter, - start_at = Start + start_at = Start, + payload_encode = PayloadEncode } = Trace, - Who = #{name => Name, type => Type, filter => Filter}, + Who = #{name => Name, type => Type, filter => Filter, payload_encode => PayloadEncode}, emqx_trace_handler:install(Who, debug, log_file(Name, Start)). stop_trace(Finished, Started) -> @@ -490,6 +496,8 @@ to_trace(#{type := ip_address, ip_address := Filter} = Trace, Rec) -> end; to_trace(#{type := Type}, _Rec) -> {error, io_lib:format("required ~s field", [Type])}; +to_trace(#{payload_encode := PayloadEncode} = Trace, Rec) -> + to_trace(maps:remove(payload_encode, Trace), Rec#?TRACE{payload_encode = PayloadEncode}); to_trace(#{start_at := StartAt} = Trace, Rec) -> {ok, Sec} = to_system_second(StartAt), to_trace(maps:remove(start_at, Trace), Rec#?TRACE{start_at = Sec}); @@ -573,3 +581,29 @@ filter_cli_handler(Names) -> now_second() -> os:system_time(second). + +maybe_migrate_trace(Fields) -> + case mnesia:table_info(emqx_trace, attributes) =:= Fields of + true -> + ok; + false -> + TransFun = fun(Trace) -> + case Trace of + {?TRACE, Name, Type, Filter, Enable, StartAt, EndAt} -> + #?TRACE{ + name = Name, + type = Type, + filter = Filter, + enable = Enable, + start_at = StartAt, + end_at = EndAt, + payload_encode = text, + extra = #{} + }; + #?TRACE{} -> + Trace + end + end, + {atomic, ok} = mnesia:transform_table(?TRACE, TransFun, Fields, ?TRACE), + ok + end. diff --git a/apps/emqx/src/emqx_trace/emqx_trace_handler.erl b/apps/emqx/src/emqx_trace/emqx_trace_handler.erl index 9c2d2358e..528bc4d42 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace_handler.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace_handler.erl @@ -44,7 +44,8 @@ -type tracer() :: #{ name := binary(), type := clientid | topic | ip_address, - filter := emqx_types:clientid() | emqx_types:topic() | emqx_trace:ip_address() + filter := emqx_types:clientid() | emqx_types:topic() | emqx_trace:ip_address(), + payload_encode := text | hidden | hex }. -define(CONFIG(_LogFile_), #{ @@ -70,7 +71,12 @@ LogFilePath :: string() ) -> ok | {error, term()}. install(Name, Type, Filter, Level, LogFile) -> - Who = #{type => Type, filter => ensure_bin(Filter), name => ensure_bin(Name)}, + Who = #{ + type => Type, + filter => ensure_bin(Filter), + name => ensure_bin(Name), + payload_encode => payload_encode() + }, install(Who, Level, LogFile). -spec install( @@ -160,14 +166,14 @@ filters(#{type := topic, filter := Filter, name := Name}) -> filters(#{type := ip_address, filter := Filter, name := Name}) -> [{ip_address, {fun ?MODULE:filter_ip_address/2, {ensure_list(Filter), Name}}}]. -formatter(#{type := _Type}) -> +formatter(#{type := _Type, payload_encode := PayloadEncode}) -> {emqx_trace_formatter, #{ %% template is for ?SLOG message not ?TRACE. template => [time, " [", level, "] ", msg, "\n"], single_line => true, max_size => unlimited, depth => unlimited, - payload_encode => payload_encode() + payload_encode => PayloadEncode }}. filter_traces(#{id := Id, level := Level, dst := Dst, filters := Filters}, Acc) -> @@ -190,7 +196,7 @@ handler_id(Name, Type) -> do_handler_id(Name, Type) catch _:_ -> - Hash = emqx_misc:bin_to_hexstr(crypto:hash(md5, Name), lower), + Hash = emqx_utils:bin_to_hexstr(crypto:hash(md5, Name), lower), do_handler_id(Hash, Type) end. diff --git a/apps/emqx/src/emqx_vm_mon.erl b/apps/emqx/src/emqx_vm_mon.erl index 1327a1bb0..d90d4139b 100644 --- a/apps/emqx/src/emqx_vm_mon.erl +++ b/apps/emqx/src/emqx_vm_mon.erl @@ -107,7 +107,7 @@ code_change(_OldVsn, State, _Extra) -> start_check_timer() -> Interval = emqx:get_config([sysmon, vm, process_check_interval]), - emqx_misc:start_timer(Interval, check). + emqx_utils:start_timer(Interval, check). usage(Percent) -> integer_to_list(floor(Percent * 100)) ++ "%". diff --git a/apps/emqx/src/emqx_ws_connection.erl b/apps/emqx/src/emqx_ws_connection.erl index ead609ed8..faf62f98d 100644 --- a/apps/emqx/src/emqx_ws_connection.erl +++ b/apps/emqx/src/emqx_ws_connection.erl @@ -52,7 +52,7 @@ -export([set_field/3]). -import( - emqx_misc, + emqx_utils, [ maybe_apply/2, start_timer/2 @@ -121,8 +121,8 @@ -define(SOCK_STATS, [recv_oct, recv_cnt, send_oct, send_cnt]). -define(ENABLED(X), (X =/= undefined)). --define(LIMITER_BYTES_IN, bytes_in). --define(LIMITER_MESSAGE_IN, message_in). +-define(LIMITER_BYTES_IN, bytes). +-define(LIMITER_MESSAGE_IN, messages). -dialyzer({no_match, [info/2]}). -dialyzer({nowarn_function, [websocket_init/1]}). @@ -172,7 +172,7 @@ stats(WsPid) when is_pid(WsPid) -> stats(#state{channel = Channel}) -> SockStats = emqx_pd:get_counters(?SOCK_STATS), ChanStats = emqx_channel:stats(Channel), - ProcStats = emqx_misc:proc_stats(), + ProcStats = emqx_utils:proc_stats(), lists:append([SockStats, ChanStats, ProcStats]). %% kick|discard|takeover @@ -340,7 +340,7 @@ tune_heap_size(Channel) -> ) of #{enable := false} -> ok; - ShutdownPolicy -> emqx_misc:tune_heap_size(ShutdownPolicy) + ShutdownPolicy -> emqx_utils:tune_heap_size(ShutdownPolicy) end. get_stats_enable(Zone) -> @@ -454,7 +454,7 @@ websocket_info( State = #state{listener = {Type, Listener}} ) -> ActiveN = get_active_n(Type, Listener), - Delivers = [Deliver | emqx_misc:drain_deliver(ActiveN)], + Delivers = [Deliver | emqx_utils:drain_deliver(ActiveN)], with_channel(handle_deliver, [Delivers], State); websocket_info( {timeout, _, limit_timeout}, @@ -678,7 +678,7 @@ check_oom(State = #state{channel = Channel}) -> #{enable := false} -> State; #{enable := true} -> - case emqx_misc:check_oom(ShutdownPolicy) of + case emqx_utils:check_oom(ShutdownPolicy) of Shutdown = {shutdown, _Reason} -> postpone(Shutdown, State); _Other -> @@ -913,7 +913,7 @@ inc_qos_stats_key(_, _) -> undefined. %% Cancel idle timer cancel_idle_timer(State = #state{idle_timer = IdleTimer}) -> - ok = emqx_misc:cancel_timer(IdleTimer), + ok = emqx_utils:cancel_timer(IdleTimer), State#state{idle_timer = undefined}. %%-------------------------------------------------------------------- @@ -1046,7 +1046,7 @@ check_max_connection(Type, Listener) -> %%-------------------------------------------------------------------- set_field(Name, Value, State) -> - Pos = emqx_misc:index_of(Name, record_info(fields, state)), + Pos = emqx_utils:index_of(Name, record_info(fields, state)), setelement(Pos + 1, State, Value). %% ensure lowercase letters in headers diff --git a/apps/emqx/src/emqx_zone_schema.erl b/apps/emqx/src/emqx_zone_schema.erl index c2595725b..5d6720986 100644 --- a/apps/emqx/src/emqx_zone_schema.erl +++ b/apps/emqx/src/emqx_zone_schema.erl @@ -15,8 +15,10 @@ %%-------------------------------------------------------------------- -module(emqx_zone_schema). +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). --export([namespace/0, roots/0, fields/1, desc/1]). +-export([namespace/0, roots/0, fields/1, desc/1, zone/0, zone_without_hidden/0]). namespace() -> zone. @@ -33,6 +35,32 @@ roots() -> "overload_protection" ]. +zone() -> + Fields = roots(), + Hidden = hidden(), + lists:map( + fun(F) -> + case lists:member(F, Hidden) of + true -> + {F, ?HOCON(?R_REF(F), #{importance => ?IMPORTANCE_HIDDEN})}; + false -> + {F, ?HOCON(?R_REF(F), #{})} + end + end, + Fields + ). + +zone_without_hidden() -> + lists:map(fun(F) -> {F, ?HOCON(?R_REF(F), #{})} end, roots() -- hidden()). + +hidden() -> + [ + "stats", + "overload_protection", + "conn_congestion", + "flapping_detect" + ]. + %% zone schemas are clones from the same name from root level %% only not allowed to have default values. fields(Name) -> diff --git a/apps/emqx/src/proto/emqx_proto_v2.erl b/apps/emqx/src/proto/emqx_proto_v2.erl new file mode 100644 index 000000000..a11c8a10e --- /dev/null +++ b/apps/emqx/src/proto/emqx_proto_v2.erl @@ -0,0 +1,86 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_proto_v2). + +-behaviour(emqx_bpapi). + +-include("bpapi.hrl"). + +-export([ + introduced_in/0, + + are_running/1, + is_running/1, + + get_alarms/2, + get_stats/1, + get_metrics/1, + + deactivate_alarm/2, + delete_all_deactivated_alarms/1, + + clean_authz_cache/1, + clean_authz_cache/2, + clean_pem_cache/1 +]). + +introduced_in() -> + "5.0.22". + +-spec is_running(node()) -> boolean() | {badrpc, term()}. +is_running(Node) -> + rpc:call(Node, emqx, is_running, []). + +-spec are_running([node()]) -> emqx_rpc:erpc_multicall(boolean()). +are_running(Nodes) when is_list(Nodes) -> + erpc:multicall(Nodes, emqx, is_running, []). + +-spec get_alarms(node(), all | activated | deactivated) -> [map()]. +get_alarms(Node, Type) -> + rpc:call(Node, emqx_alarm, get_alarms, [Type]). + +-spec get_stats(node()) -> emqx_stats:stats() | {badrpc, _}. +get_stats(Node) -> + rpc:call(Node, emqx_stats, getstats, []). + +-spec get_metrics(node()) -> [{emqx_metrics:metric_name(), non_neg_integer()}] | {badrpc, _}. +get_metrics(Node) -> + rpc:call(Node, emqx_metrics, all, []). + +-spec clean_authz_cache(node(), emqx_types:clientid()) -> + ok + | {error, not_found} + | {badrpc, _}. +clean_authz_cache(Node, ClientId) -> + rpc:call(Node, emqx_authz_cache, drain_cache, [ClientId]). + +-spec clean_authz_cache(node()) -> ok | {badrpc, _}. +clean_authz_cache(Node) -> + rpc:call(Node, emqx_authz_cache, drain_cache, []). + +-spec clean_pem_cache(node()) -> ok | {badrpc, _}. +clean_pem_cache(Node) -> + rpc:call(Node, ssl_pem_cache, clear, []). + +-spec deactivate_alarm(node(), binary() | atom()) -> + ok | {error, not_found} | {badrpc, _}. +deactivate_alarm(Node, Name) -> + rpc:call(Node, emqx_alarm, deactivate, [Name]). + +-spec delete_all_deactivated_alarms(node()) -> ok | {badrpc, _}. +delete_all_deactivated_alarms(Node) -> + rpc:call(Node, emqx_alarm, delete_all_deactivated_alarms, []). diff --git a/apps/emqx/test/emqx_SUITE.erl b/apps/emqx/test/emqx_SUITE.erl index 09d5d8017..64ed2ea19 100644 --- a/apps/emqx/test/emqx_SUITE.erl +++ b/apps/emqx/test/emqx_SUITE.erl @@ -148,6 +148,14 @@ t_run_hook(_) -> ?assertEqual(3, emqx:run_fold_hook(foldl_filter2_hook, [arg], 1)), ?assertEqual(2, emqx:run_fold_hook(foldl_filter2_hook, [arg1], 1)). +t_cluster_nodes(_) -> + Expected = [node()], + ?assertEqual(Expected, emqx:running_nodes()), + ?assertEqual(Expected, emqx:cluster_nodes(running)), + ?assertEqual(Expected, emqx:cluster_nodes(all)), + ?assertEqual(Expected, emqx:cluster_nodes(cores)), + ?assertEqual([], emqx:cluster_nodes(stopped)). + %%-------------------------------------------------------------------- %% Hook fun %%-------------------------------------------------------------------- diff --git a/apps/emqx/test/emqx_authentication_SUITE.erl b/apps/emqx/test/emqx_authentication_SUITE.erl index 2c83162ed..0190ab936 100644 --- a/apps/emqx/test/emqx_authentication_SUITE.erl +++ b/apps/emqx/test/emqx_authentication_SUITE.erl @@ -95,13 +95,17 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> + LogLevel = emqx_logger:get_primary_log_level(), + ok = emqx_logger:set_log_level(debug), application:set_env(ekka, strict_mode, true), emqx_common_test_helpers:boot_modules(all), emqx_common_test_helpers:start_apps([]), - Config. + [{log_level, LogLevel} | Config]. -end_per_suite(_) -> +end_per_suite(Config) -> emqx_common_test_helpers:stop_apps([]), + LogLevel = ?config(log_level), + emqx_logger:set_log_level(LogLevel), ok. init_per_testcase(Case, Config) -> diff --git a/apps/emqx/test/emqx_banned_SUITE.erl b/apps/emqx/test/emqx_banned_SUITE.erl index 80427ac47..0c14f64c9 100644 --- a/apps/emqx/test/emqx_banned_SUITE.erl +++ b/apps/emqx/test/emqx_banned_SUITE.erl @@ -186,9 +186,8 @@ t_session_taken(_) -> false end end, - 3000 + 6000 ), - Publish(), C2 = Connect(), diff --git a/apps/emqx/test/emqx_channel_SUITE.erl b/apps/emqx/test/emqx_channel_SUITE.erl index 633337f80..94a77d2cd 100644 --- a/apps/emqx/test/emqx_channel_SUITE.erl +++ b/apps/emqx/test/emqx_channel_SUITE.erl @@ -162,8 +162,7 @@ limiter_conf() -> Make = fun() -> #{ burst => 0, - rate => infinity, - capacity => infinity + rate => infinity } end, @@ -172,7 +171,7 @@ limiter_conf() -> Acc#{Name => Make()} end, #{}, - [bytes_in, message_in, message_routing, connection, internal] + [bytes, messages, message_routing, connection, internal] ). stats_conf() -> @@ -1137,7 +1136,7 @@ t_ws_cookie_init(_) -> %%-------------------------------------------------------------------- t_flapping_detect(_) -> - emqx_config:put_zone_conf(default, [flapping_detect, enable], true), + emqx_config:put_zone_conf(default, [flapping_detect, window_time], 60000), Parent = self(), ok = meck:expect( emqx_cm, @@ -1258,7 +1257,7 @@ limiter_cfg() -> Client = #{ rate => 5, initial => 0, - capacity => 5, + burst => 0, low_watermark => 1, divisible => false, max_retry_time => timer:seconds(5), @@ -1270,7 +1269,7 @@ limiter_cfg() -> }. bucket_cfg() -> - #{rate => 10, initial => 0, capacity => 10}. + #{rate => 10, initial => 0, burst => 0}. add_bucket() -> emqx_limiter_server:add_bucket(?MODULE, message_routing, bucket_cfg()). diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index e01f18a51..3aad9c6cc 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -313,6 +313,7 @@ stop_apps(Apps) -> ok = emqx_config:delete_override_conf_files(), application:unset_env(emqx, local_override_conf_file), application:unset_env(emqx, cluster_override_conf_file), + application:unset_env(emqx, cluster_hocon_file), application:unset_env(gen_rpc, port_discovery), ok. @@ -461,6 +462,10 @@ force_set_config_file_paths(emqx_conf, [Path] = Paths) -> ok = file:write_file(Path, Bin, [append]), application:set_env(emqx, config_files, Paths); force_set_config_file_paths(emqx, Paths) -> + %% we need init cluster conf, so we can save the cluster conf to the file + application:set_env(emqx, local_override_conf_file, "local_override.conf"), + application:set_env(emqx, cluster_override_conf_file, "cluster_override.conf"), + application:set_env(emqx, cluster_conf_file, "cluster.hocon"), application:set_env(emqx, config_files, Paths); force_set_config_file_paths(_, _) -> ok. @@ -476,7 +481,7 @@ copy_certs(_, _) -> load_config(SchemaModule, Config, Opts) -> ConfigBin = case is_map(Config) of - true -> jsx:encode(Config); + true -> emqx_utils_json:encode(Config); false -> Config end, ok = emqx_config:delete_override_conf_files(), @@ -675,7 +680,8 @@ start_slave(Name, Opts) when is_map(Opts) -> ] ); slave -> - slave:start_link(host(), Name, ebin_path()) + Env = " -env HOCON_ENV_OVERRIDE_PREFIX EMQX_", + slave:start_link(host(), Name, ebin_path() ++ Env) end end, case DoStart() of @@ -748,6 +754,21 @@ setup_node(Node, Opts) when is_map(Opts) -> %% `emqx_conf' app and correctly catch up the config. StartAutocluster = maps:get(start_autocluster, Opts, false), + ct:pal( + "setting up node ~p:\n ~p", + [ + Node, + #{ + start_autocluster => StartAutocluster, + load_apps => LoadApps, + apps => Apps, + env => Env, + join_to => JoinTo, + start_apps => StartApps + } + ] + ), + %% Load env before doing anything to avoid overriding [ok = erpc:call(Node, ?MODULE, load, [App]) || App <- [gen_rpc, ekka, mria, emqx | LoadApps]], @@ -772,10 +793,7 @@ setup_node(Node, Opts) when is_map(Opts) -> end, %% Setting env before starting any applications - [ - ok = rpc:call(Node, application, set_env, [Application, Key, Value]) - || {Application, Key, Value} <- Env - ], + set_envs(Node, Env), %% Here we start the apps EnvHandlerForRpc = @@ -793,8 +811,9 @@ setup_node(Node, Opts) when is_map(Opts) -> node(), integer_to_list(erlang:unique_integer()) ]), + Cookie = atom_to_list(erlang:get_cookie()), os:putenv("EMQX_NODE__DATA_DIR", NodeDataDir), - os:putenv("EMQX_NODE__COOKIE", atom_to_list(erlang:get_cookie())), + os:putenv("EMQX_NODE__COOKIE", Cookie), emqx_config:init_load(SchemaMod), os:unsetenv("EMQX_NODE__DATA_DIR"), os:unsetenv("EMQX_NODE__COOKIE"), @@ -825,7 +844,15 @@ setup_node(Node, Opts) when is_map(Opts) -> ok; _ -> StartAutocluster andalso - (ok = rpc:call(Node, emqx_machine_boot, start_autocluster, [])), + begin + %% Note: we need to re-set the env because + %% starting the apps apparently make some of them + %% to be lost... This is particularly useful for + %% setting extra apps to be restarted after + %% joining. + set_envs(Node, Env), + ok = erpc:call(Node, emqx_machine_boot, start_autocluster, []) + end, case rpc:call(Node, ekka, join, [JoinTo]) of ok -> ok; @@ -882,6 +909,14 @@ merge_opts(Opts1, Opts2) -> Opts2 ). +set_envs(Node, Env) -> + lists:foreach( + fun({Application, Key, Value}) -> + ok = rpc:call(Node, application, set_env, [Application, Key, Value]) + end, + Env + ). + erl_flags() -> %% One core and redirecting logs to master "+S 1:1 -master " ++ atom_to_list(node()) ++ " " ++ ebin_path(). @@ -1006,7 +1041,7 @@ switch_proxy(Switch, Name, ProxyHost, ProxyPort) -> off -> #{<<"enabled">> => false}; on -> #{<<"enabled">> => true} end, - BodyBin = emqx_json:encode(Body), + BodyBin = emqx_utils_json:encode(Body), {ok, {{_, 200, _}, _, _}} = httpc:request( post, {Url, [], "application/json", BodyBin}, @@ -1026,7 +1061,7 @@ timeout_proxy(on, Name, ProxyHost, ProxyPort) -> <<"toxicity">> => 1.0, <<"attributes">> => #{<<"timeout">> => 0} }, - BodyBin = emqx_json:encode(Body), + BodyBin = emqx_utils_json:encode(Body), {ok, {{_, 200, _}, _, _}} = httpc:request( post, {Url, [], "application/json", BodyBin}, @@ -1061,7 +1096,7 @@ latency_up_proxy(on, Name, ProxyHost, ProxyPort) -> <<"jitter">> => 3_000 } }, - BodyBin = emqx_json:encode(Body), + BodyBin = emqx_utils_json:encode(Body), {ok, {{_, 200, _}, _, _}} = httpc:request( post, {Url, [], "application/json", BodyBin}, diff --git a/apps/emqx/test/emqx_common_test_http.erl b/apps/emqx/test/emqx_common_test_http.erl index 575bed5c3..e9064715d 100644 --- a/apps/emqx/test/emqx_common_test_http.erl +++ b/apps/emqx/test/emqx_common_test_http.erl @@ -54,7 +54,7 @@ request_api(Method, Url, QueryParams, Auth, Body, HttpOpts) -> [] -> {NewUrl, [Auth]}; _ -> - {NewUrl, [Auth], "application/json", emqx_json:encode(Body)} + {NewUrl, [Auth], "application/json", emqx_utils_json:encode(Body)} end, do_request_api(Method, Request, HttpOpts). @@ -70,7 +70,7 @@ do_request_api(Method, Request, HttpOpts) -> end. get_http_data(ResponseBody) -> - emqx_json:decode(ResponseBody, [return_maps]). + emqx_utils_json:decode(ResponseBody, [return_maps]). auth_header(User, Pass) -> Encoded = base64:encode_to_string(lists:append([User, ":", Pass])), diff --git a/apps/emqx/test/emqx_config_SUITE.erl b/apps/emqx/test/emqx_config_SUITE.erl index fe8a5fed8..7befd7a16 100644 --- a/apps/emqx/test/emqx_config_SUITE.erl +++ b/apps/emqx/test/emqx_config_SUITE.erl @@ -57,5 +57,5 @@ t_fill_default_values(_) -> WithDefaults ), %% ensure JSON compatible - _ = emqx_json:encode(WithDefaults), + _ = emqx_utils_json:encode(WithDefaults), ok. diff --git a/apps/emqx/test/emqx_config_handler_SUITE.erl b/apps/emqx/test/emqx_config_handler_SUITE.erl index 8126b35c6..deeee8d62 100644 --- a/apps/emqx/test/emqx_config_handler_SUITE.erl +++ b/apps/emqx/test/emqx_config_handler_SUITE.erl @@ -21,8 +21,7 @@ -define(MOD, {mod}). -define(WKEY, '?'). --define(LOCAL_CONF, "/tmp/local-override.conf"). --define(CLUSTER_CONF, "/tmp/cluster-override.conf"). +-define(CLUSTER_CONF, "/tmp/cluster.conf"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). @@ -38,7 +37,6 @@ end_per_suite(_Config) -> emqx_common_test_helpers:stop_apps([]). init_per_testcase(_Case, Config) -> - _ = file:delete(?LOCAL_CONF), _ = file:delete(?CLUSTER_CONF), Config. @@ -200,62 +198,6 @@ t_sub_key_update_remove(_Config) -> ok = emqx_config_handler:remove_handler(KeyPath2), ok. -t_local_override_update_remove(_Config) -> - application:set_env(emqx, local_override_conf_file, ?LOCAL_CONF), - application:set_env(emqx, cluster_override_conf_file, ?CLUSTER_CONF), - KeyPath = [sysmon, os, cpu_high_watermark], - ok = emqx_config_handler:add_handler(KeyPath, ?MODULE), - LocalOpts = #{override_to => local}, - {ok, Res} = emqx:update_config(KeyPath, <<"70%">>, LocalOpts), - ?assertMatch( - #{ - config := 0.7, - post_config_update := #{}, - raw_config := <<"70%">> - }, - Res - ), - ClusterOpts = #{override_to => cluster}, - ?assertMatch( - {error, {permission_denied, _}}, emqx:update_config(KeyPath, <<"71%">>, ClusterOpts) - ), - ?assertMatch(0.7, emqx:get_config(KeyPath)), - - KeyPath2 = [sysmon, os, cpu_low_watermark], - ok = emqx_config_handler:add_handler(KeyPath2, ?MODULE), - ?assertMatch( - {error, {permission_denied, _}}, emqx:update_config(KeyPath2, <<"40%">>, ClusterOpts) - ), - - %% remove - ?assertMatch({error, {permission_denied, _}}, emqx:remove_config(KeyPath)), - ?assertEqual( - {ok, #{post_config_update => #{}}}, - emqx:remove_config(KeyPath, #{override_to => local}) - ), - ?assertEqual( - {ok, #{post_config_update => #{}}}, - emqx:remove_config(KeyPath) - ), - ?assertError({config_not_found, KeyPath}, emqx:get_raw_config(KeyPath)), - OSKey = maps:keys(emqx:get_raw_config([sysmon, os])), - ?assertEqual(false, lists:member(<<"cpu_high_watermark">>, OSKey)), - ?assert(length(OSKey) > 0), - - ?assertEqual( - {ok, #{config => 0.8, post_config_update => #{}, raw_config => <<"80%">>}}, - emqx:reset_config(KeyPath, ClusterOpts) - ), - OSKey1 = maps:keys(emqx:get_raw_config([sysmon, os])), - ?assertEqual(true, lists:member(<<"cpu_high_watermark">>, OSKey1)), - ?assert(length(OSKey1) > 1), - - ok = emqx_config_handler:remove_handler(KeyPath), - ok = emqx_config_handler:remove_handler(KeyPath2), - application:unset_env(emqx, local_override_conf_file), - application:unset_env(emqx, cluster_override_conf_file), - ok. - t_check_failed(_Config) -> KeyPath = [sysmon, os, cpu_check_interval], Opts = #{rawconf_with_defaults => true}, @@ -426,9 +368,9 @@ wait_for_new_pid() -> callback_error(FailedPath, Update, Error) -> Opts = #{rawconf_with_defaults => true}, ok = emqx_config_handler:add_handler(FailedPath, ?MODULE), - Old = emqx:get_raw_config(FailedPath), + Old = emqx:get_raw_config(FailedPath, undefined), ?assertEqual(Error, emqx:update_config(FailedPath, Update, Opts)), - New = emqx:get_raw_config(FailedPath), + New = emqx:get_raw_config(FailedPath, undefined), ?assertEqual(Old, New), ok = emqx_config_handler:remove_handler(FailedPath), ok. diff --git a/apps/emqx/test/emqx_connection_SUITE.erl b/apps/emqx/test/emqx_connection_SUITE.erl index cc9e03168..f24c1c895 100644 --- a/apps/emqx/test/emqx_connection_SUITE.erl +++ b/apps/emqx/test/emqx_connection_SUITE.erl @@ -427,7 +427,7 @@ t_ensure_rate_limit(_) -> fun(_, Client) -> {pause, 3000, undefined, Client} end ), {ok, State2} = emqx_connection:check_limiter( - [{1000, bytes_in}], + [{1000, bytes}], [], WhenOk, [], @@ -496,16 +496,16 @@ t_get_conn_info(_) -> t_oom_shutdown(init, Config) -> ok = snabbkaffe:start_trace(), - ok = meck:new(emqx_misc, [non_strict, passthrough, no_history, no_link]), + ok = meck:new(emqx_utils, [non_strict, passthrough, no_history, no_link]), meck:expect( - emqx_misc, + emqx_utils, check_oom, fun(_) -> {shutdown, "fake_oom"} end ), Config; t_oom_shutdown('end', _Config) -> snabbkaffe:stop(), - meck:unload(emqx_misc), + meck:unload(emqx_utils), ok. t_oom_shutdown(_) -> @@ -703,31 +703,29 @@ handle_call(Pid, Call, St) -> emqx_connection:handle_call(Pid, Call, St). -define(LIMITER_ID, 'tcp:default'). init_limiter() -> - emqx_limiter_container:get_limiter_by_types(?LIMITER_ID, [bytes_in, message_in], limiter_cfg()). + emqx_limiter_container:get_limiter_by_types(?LIMITER_ID, [bytes, messages], limiter_cfg()). limiter_cfg() -> - Infinity = emqx_limiter_schema:infinity_value(), Cfg = bucket_cfg(), Client = #{ - rate => Infinity, + rate => infinity, initial => 0, - capacity => Infinity, + burst => 0, low_watermark => 1, divisible => false, max_retry_time => timer:seconds(5), failure_strategy => force }, - #{bytes_in => Cfg, message_in => Cfg, client => #{bytes_in => Client, message_in => Client}}. + #{bytes => Cfg, messages => Cfg, client => #{bytes => Client, messages => Client}}. bucket_cfg() -> - Infinity = emqx_limiter_schema:infinity_value(), - #{rate => Infinity, initial => 0, capacity => Infinity}. + #{rate => infinity, initial => 0, burst => 0}. add_bucket() -> Cfg = bucket_cfg(), - emqx_limiter_server:add_bucket(?LIMITER_ID, bytes_in, Cfg), - emqx_limiter_server:add_bucket(?LIMITER_ID, message_in, Cfg). + emqx_limiter_server:add_bucket(?LIMITER_ID, bytes, Cfg), + emqx_limiter_server:add_bucket(?LIMITER_ID, messages, Cfg). del_bucket() -> - emqx_limiter_server:del_bucket(?LIMITER_ID, bytes_in), - emqx_limiter_server:del_bucket(?LIMITER_ID, message_in). + emqx_limiter_server:del_bucket(?LIMITER_ID, bytes), + emqx_limiter_server:del_bucket(?LIMITER_ID, messages). diff --git a/apps/emqx/test/emqx_crl_cache_SUITE.erl b/apps/emqx/test/emqx_crl_cache_SUITE.erl index 01f9c7172..dd3eb29e7 100644 --- a/apps/emqx/test/emqx_crl_cache_SUITE.erl +++ b/apps/emqx/test/emqx_crl_cache_SUITE.erl @@ -402,7 +402,7 @@ request(Method, Url, QueryParams, Body) -> Opts = #{return_all => true}, case emqx_mgmt_api_test_util:request_api(Method, Url, QueryParams, AuthHeader, Body, Opts) of {ok, {Reason, Headers, BodyR}} -> - {ok, {Reason, Headers, emqx_json:decode(BodyR, [return_maps])}}; + {ok, {Reason, Headers, emqx_utils_json:decode(BodyR, [return_maps])}}; Error -> Error end. @@ -997,7 +997,7 @@ do_t_update_listener(Config) -> <<"enable_crl_check">> => true } }, - ListenerData1 = emqx_map_lib:deep_merge(ListenerData0, CRLConfig), + ListenerData1 = emqx_utils_maps:deep_merge(ListenerData0, CRLConfig), {ok, {_, _, ListenerData2}} = update_listener_via_api(ListenerId, ListenerData1), ?assertMatch( #{ @@ -1040,7 +1040,7 @@ do_t_validations(_Config) -> {ok, {{_, 200, _}, _, ListenerData0}} = get_listener_via_api(ListenerId), ListenerData1 = - emqx_map_lib:deep_merge( + emqx_utils_maps:deep_merge( ListenerData0, #{ <<"ssl_options">> => @@ -1052,7 +1052,7 @@ do_t_validations(_Config) -> ), {error, {_, _, ResRaw1}} = update_listener_via_api(ListenerId, ListenerData1), #{<<"code">> := <<"BAD_REQUEST">>, <<"message">> := MsgRaw1} = - emqx_json:decode(ResRaw1, [return_maps]), + emqx_utils_json:decode(ResRaw1, [return_maps]), ?assertMatch( #{ <<"mismatches">> := @@ -1064,7 +1064,7 @@ do_t_validations(_Config) -> } } }, - emqx_json:decode(MsgRaw1, [return_maps]) + emqx_utils_json:decode(MsgRaw1, [return_maps]) ), ok. diff --git a/apps/emqx/test/emqx_flapping_SUITE.erl b/apps/emqx/test/emqx_flapping_SUITE.erl index e27ff67e0..877f05995 100644 --- a/apps/emqx/test/emqx_flapping_SUITE.erl +++ b/apps/emqx/test/emqx_flapping_SUITE.erl @@ -101,3 +101,21 @@ t_expired_detecting(_) -> ets:tab2list(emqx_flapping) ) ). + +t_conf_without_window_time(_) -> + %% enable is deprecated, so we need to make sure it won't be used. + Global = emqx_config:get([flapping_detect]), + ?assertNot(maps:is_key(enable, Global)), + %% zones don't have default value, so we need to make sure fallback to global conf. + %% this new_zone will fallback to global conf. + emqx_config:put_zone_conf(new_zone, [flapping_detect], #{}), + ?assertEqual(Global, get_policy(new_zone)), + + emqx_config:put_zone_conf(new_zone_1, [flapping_detect], #{window_time => 100}), + ?assertEqual(100, emqx_flapping:get_policy(window_time, new_zone_1)), + ?assertEqual(maps:get(ban_time, Global), emqx_flapping:get_policy(ban_time, new_zone_1)), + ?assertEqual(maps:get(max_count, Global), emqx_flapping:get_policy(max_count, new_zone_1)), + ok. + +get_policy(Zone) -> + emqx_flapping:get_policy([window_time, ban_time, max_count], Zone). diff --git a/apps/emqx/test/emqx_ocsp_cache_SUITE.erl b/apps/emqx/test/emqx_ocsp_cache_SUITE.erl index 3c3fd0341..15ca29853 100644 --- a/apps/emqx/test/emqx_ocsp_cache_SUITE.erl +++ b/apps/emqx/test/emqx_ocsp_cache_SUITE.erl @@ -143,7 +143,7 @@ init_per_testcase(t_ocsp_responder_error_responses, Config) -> } }, Conf = #{listeners => #{Type => #{Name => ListenerOpts}}}, - ConfBin = emqx_map_lib:binary_key_map(Conf), + ConfBin = emqx_utils_maps:binary_key_map(Conf), hocon_tconf:check_plain(emqx_schema, ConfBin, #{required => false, atom_keys => false}), emqx_config:put_listener_conf(Type, Name, [], ListenerOpts), snabbkaffe:start_trace(), @@ -184,7 +184,7 @@ init_per_testcase(_TestCase, Config) -> } }, Conf = #{listeners => #{Type => #{Name => ListenerOpts}}}, - ConfBin = emqx_map_lib:binary_key_map(Conf), + ConfBin = emqx_utils_maps:binary_key_map(Conf), hocon_tconf:check_plain(emqx_schema, ConfBin, #{required => false, atom_keys => false}), emqx_config:put_listener_conf(Type, Name, [], ListenerOpts), snabbkaffe:start_trace(), @@ -430,7 +430,7 @@ request(Method, Url, QueryParams, Body) -> Opts = #{return_all => true}, case emqx_mgmt_api_test_util:request_api(Method, Url, QueryParams, AuthHeader, Body, Opts) of {ok, {Reason, Headers, BodyR}} -> - {ok, {Reason, Headers, emqx_json:decode(BodyR, [return_maps])}}; + {ok, {Reason, Headers, emqx_utils_json:decode(BodyR, [return_maps])}}; Error -> Error end. @@ -677,9 +677,13 @@ do_t_update_listener(Config) -> %% no ocsp at first ListenerId = "ssl:default", {ok, {{_, 200, _}, _, ListenerData0}} = get_listener_via_api(ListenerId), - ?assertEqual( - undefined, - emqx_map_lib:deep_get([<<"ssl_options">>, <<"ocsp">>], ListenerData0, undefined) + ?assertMatch( + #{ + <<"enable_ocsp_stapling">> := false, + <<"refresh_http_timeout">> := _, + <<"refresh_interval">> := _ + }, + emqx_utils_maps:deep_get([<<"ssl_options">>, <<"ocsp">>], ListenerData0, undefined) ), assert_no_http_get(), @@ -702,7 +706,7 @@ do_t_update_listener(Config) -> } } }, - ListenerData1 = emqx_map_lib:deep_merge(ListenerData0, OCSPConfig), + ListenerData1 = emqx_utils_maps:deep_merge(ListenerData0, OCSPConfig), {ok, {_, _, ListenerData2}} = update_listener_via_api(ListenerId, ListenerData1), ?assertMatch( #{ @@ -722,14 +726,14 @@ do_t_update_listener(Config) -> %% location ?assertNotEqual( IssuerPemPath, - emqx_map_lib:deep_get( + emqx_utils_maps:deep_get( [<<"ssl_options">>, <<"ocsp">>, <<"issuer_pem">>], ListenerData2 ) ), ?assertNotEqual( IssuerPem, - emqx_map_lib:deep_get( + emqx_utils_maps:deep_get( [<<"ssl_options">>, <<"ocsp">>, <<"issuer_pem">>], ListenerData2 ) @@ -818,7 +822,7 @@ do_t_validations(_Config) -> {ok, {{_, 200, _}, _, ListenerData0}} = get_listener_via_api(ListenerId), ListenerData1 = - emqx_map_lib:deep_merge( + emqx_utils_maps:deep_merge( ListenerData0, #{ <<"ssl_options">> => @@ -827,7 +831,7 @@ do_t_validations(_Config) -> ), {error, {_, _, ResRaw1}} = update_listener_via_api(ListenerId, ListenerData1), #{<<"code">> := <<"BAD_REQUEST">>, <<"message">> := MsgRaw1} = - emqx_json:decode(ResRaw1, [return_maps]), + emqx_utils_json:decode(ResRaw1, [return_maps]), ?assertMatch( #{ <<"mismatches">> := @@ -839,11 +843,11 @@ do_t_validations(_Config) -> } } }, - emqx_json:decode(MsgRaw1, [return_maps]) + emqx_utils_json:decode(MsgRaw1, [return_maps]) ), ListenerData2 = - emqx_map_lib:deep_merge( + emqx_utils_maps:deep_merge( ListenerData0, #{ <<"ssl_options">> => @@ -857,7 +861,7 @@ do_t_validations(_Config) -> ), {error, {_, _, ResRaw2}} = update_listener_via_api(ListenerId, ListenerData2), #{<<"code">> := <<"BAD_REQUEST">>, <<"message">> := MsgRaw2} = - emqx_json:decode(ResRaw2, [return_maps]), + emqx_utils_json:decode(ResRaw2, [return_maps]), ?assertMatch( #{ <<"mismatches">> := @@ -869,11 +873,11 @@ do_t_validations(_Config) -> } } }, - emqx_json:decode(MsgRaw2, [return_maps]) + emqx_utils_json:decode(MsgRaw2, [return_maps]) ), ListenerData3a = - emqx_map_lib:deep_merge( + emqx_utils_maps:deep_merge( ListenerData0, #{ <<"ssl_options">> => @@ -886,10 +890,12 @@ do_t_validations(_Config) -> } } ), - ListenerData3 = emqx_map_lib:deep_remove([<<"ssl_options">>, <<"certfile">>], ListenerData3a), + ListenerData3 = emqx_utils_maps:deep_remove( + [<<"ssl_options">>, <<"certfile">>], ListenerData3a + ), {error, {_, _, ResRaw3}} = update_listener_via_api(ListenerId, ListenerData3), #{<<"code">> := <<"BAD_REQUEST">>, <<"message">> := MsgRaw3} = - emqx_json:decode(ResRaw3, [return_maps]), + emqx_utils_json:decode(ResRaw3, [return_maps]), ?assertMatch( #{ <<"mismatches">> := @@ -901,7 +907,7 @@ do_t_validations(_Config) -> } } }, - emqx_json:decode(MsgRaw3, [return_maps]) + emqx_utils_json:decode(MsgRaw3, [return_maps]) ), ok. diff --git a/apps/emqx/test/emqx_ratelimiter_SUITE.erl b/apps/emqx/test/emqx_ratelimiter_SUITE.erl index f3b97d517..26048873e 100644 --- a/apps/emqx/test/emqx_ratelimiter_SUITE.erl +++ b/apps/emqx/test/emqx_ratelimiter_SUITE.erl @@ -72,7 +72,7 @@ t_consume(_) -> Cfg = fun(Cfg) -> Cfg#{ rate := 100, - capacity := 100, + burst := 0, initial := 100, max_retry_time := 1000, failure_strategy := force @@ -89,7 +89,7 @@ t_retry(_) -> Cfg = fun(Cfg) -> Cfg#{ rate := 50, - capacity := 200, + burst := 150, initial := 0, max_retry_time := 1000, failure_strategy := force @@ -109,7 +109,7 @@ t_restore(_) -> Cfg = fun(Cfg) -> Cfg#{ rate := 1, - capacity := 200, + burst := 199, initial := 50, max_retry_time := 100, failure_strategy := force @@ -129,7 +129,7 @@ t_max_retry_time(_) -> Cfg = fun(Cfg) -> Cfg#{ rate := 1, - capacity := 1, + burst := 0, max_retry_time := 500, failure_strategy := drop } @@ -139,8 +139,12 @@ t_max_retry_time(_) -> Begin = ?NOW, Result = emqx_htb_limiter:consume(101, Client), ?assertMatch({drop, _}, Result), - Time = ?NOW - Begin, - ?assert(Time >= 500 andalso Time < 550) + End = ?NOW, + Time = End - Begin, + ?assert( + Time >= 500 andalso Time < 550, + lists:flatten(io_lib:format("Begin:~p, End:~p, Time:~p~n", [Begin, End, Time])) + ) end, with_per_client(Cfg, Case). @@ -150,7 +154,7 @@ t_divisible(_) -> divisible := true, rate := ?RATE("1000/1s"), initial := 600, - capacity := 600 + burst := 0 } end, Case = fun(BucketCfg) -> @@ -176,7 +180,7 @@ t_low_watermark(_) -> low_watermark := 400, rate := ?RATE("1000/1s"), initial := 1000, - capacity := 1000 + burst := 0 } end, Case = fun(BucketCfg) -> @@ -201,8 +205,7 @@ t_infinity_client(_) -> Fun = fun(Cfg) -> Cfg end, Case = fun(Cfg) -> Client = connect(Cfg), - InfVal = emqx_limiter_schema:infinity_value(), - ?assertMatch(#{bucket := #{rate := InfVal}}, Client), + ?assertMatch(infinity, Client), Result = emqx_htb_limiter:check(100000, Client), ?assertEqual({ok, Client}, Result) end, @@ -212,12 +215,12 @@ t_try_restore_agg(_) -> Fun = fun(#{client := Cli} = Bucket) -> Bucket2 = Bucket#{ rate := 1, - capacity := 200, + burst := 199, initial := 50 }, Cli2 = Cli#{ rate := infinity, - capacity := infinity, + burst := 0, divisible := true, max_retry_time := 100, failure_strategy := force @@ -239,11 +242,11 @@ t_short_board(_) -> Bucket2 = Bucket#{ rate := ?RATE("100/1s"), initial := 0, - capacity := 100 + burst := 0 }, Cli2 = Cli#{ rate := ?RATE("600/1s"), - capacity := 600, + burst := 0, initial := 600 }, Bucket2#{client := Cli2} @@ -261,46 +264,45 @@ t_rate(_) -> Bucket2 = Bucket#{ rate := ?RATE("100/100ms"), initial := 0, - capacity := infinity + burst := 0 }, Cli2 = Cli#{ rate := infinity, - capacity := infinity, + burst := 0, initial := 0 }, Bucket2#{client := Cli2} end, Case = fun(Cfg) -> + Time = 1000, Client = connect(Cfg), - Ts1 = erlang:system_time(millisecond), C1 = emqx_htb_limiter:available(Client), - timer:sleep(1000), - Ts2 = erlang:system_time(millisecond), + timer:sleep(1100), C2 = emqx_htb_limiter:available(Client), - ShouldInc = floor((Ts2 - Ts1) / 100) * 100, + ShouldInc = floor(Time / 100) * 100, Inc = C2 - C1, ?assert(in_range(Inc, ShouldInc - 100, ShouldInc + 100), "test bucket rate") end, with_bucket(Fun, Case). t_capacity(_) -> - Capacity = 600, + Capacity = 1200, Fun = fun(#{client := Cli} = Bucket) -> Bucket2 = Bucket#{ rate := ?RATE("100/100ms"), initial := 0, - capacity := 600 + burst := 200 }, Cli2 = Cli#{ rate := infinity, - capacity := infinity, + burst := 0, initial := 0 }, Bucket2#{client := Cli2} end, Case = fun(Cfg) -> Client = connect(Cfg), - timer:sleep(1000), + timer:sleep(1500), C1 = emqx_htb_limiter:available(Client), ?assertEqual(Capacity, C1, "test bucket capacity") end, @@ -318,11 +320,11 @@ t_collaborative_alloc(_) -> Bucket2 = Bucket#{ rate := ?RATE("400/1s"), initial := 0, - capacity := 600 + burst := 200 }, Cli2 = Cli#{ rate := ?RATE("50"), - capacity := 100, + burst := 50, initial := 100 }, Bucket2#{client := Cli2} @@ -363,11 +365,11 @@ t_burst(_) -> Bucket2 = Bucket#{ rate := ?RATE("200/1s"), initial := 0, - capacity := 200 + burst := 0 }, Cli2 = Cli#{ rate := ?RATE("50/1s"), - capacity := 200, + burst := 150, divisible := true }, Bucket2#{client := Cli2} @@ -401,11 +403,11 @@ t_limit_global_with_unlimit_other(_) -> Bucket2 = Bucket#{ rate := infinity, initial := 0, - capacity := infinity + burst := 0 }, Cli2 = Cli#{ rate := infinity, - capacity := infinity, + burst := 0, initial := 0 }, Bucket2#{client := Cli2} @@ -414,7 +416,7 @@ t_limit_global_with_unlimit_other(_) -> Case = fun() -> C1 = counters:new(1, []), start_client({b1, Bucket}, ?NOW + 2000, C1, 20), - timer:sleep(2100), + timer:sleep(2200), check_average_rate(C1, 2, 600) end, @@ -432,7 +434,7 @@ t_check_container(_) -> Cfg#{ rate := ?RATE("1000/1s"), initial := 1000, - capacity := 1000 + burst := 0 } end, Case = fun(#{client := Client} = BucketCfg) -> @@ -565,13 +567,73 @@ t_schema_unit(_) -> ?assertMatch({error, _}, M:to_rate("100MB/1")), ?assertMatch({error, _}, M:to_rate("100/10x")), - ?assertEqual({ok, emqx_limiter_schema:infinity_value()}, M:to_capacity("infinity")), + ?assertEqual({ok, infinity}, M:to_capacity("infinity")), ?assertEqual({ok, 100}, M:to_capacity("100")), ?assertEqual({ok, 100 * 1024}, M:to_capacity("100KB")), ?assertEqual({ok, 100 * 1024 * 1024}, M:to_capacity("100MB")), ?assertEqual({ok, 100 * 1024 * 1024 * 1024}, M:to_capacity("100GB")), ok. +compatibility_for_capacity(_) -> + CfgStr = << + "" + "\n" + "listeners.tcp.default {\n" + " bind = \"0.0.0.0:1883\"\n" + " max_connections = 1024000\n" + " limiter.messages.capacity = infinity\n" + " limiter.client.messages.capacity = infinity\n" + "}\n" + "" + >>, + ?assertMatch( + #{ + messages := #{burst := 0}, + client := #{messages := #{burst := 0}} + }, + parse_and_check(CfgStr) + ). + +compatibility_for_message_in(_) -> + CfgStr = << + "" + "\n" + "listeners.tcp.default {\n" + " bind = \"0.0.0.0:1883\"\n" + " max_connections = 1024000\n" + " limiter.message_in.rate = infinity\n" + " limiter.client.message_in.rate = infinity\n" + "}\n" + "" + >>, + ?assertMatch( + #{ + messages := #{rate := infinity}, + client := #{messages := #{rate := infinity}} + }, + parse_and_check(CfgStr) + ). + +compatibility_for_bytes_in(_) -> + CfgStr = << + "" + "\n" + "listeners.tcp.default {\n" + " bind = \"0.0.0.0:1883\"\n" + " max_connections = 1024000\n" + " limiter.bytes_in.rate = infinity\n" + " limiter.client.bytes_in.rate = infinity\n" + "}\n" + "" + >>, + ?assertMatch( + #{ + bytes := #{rate := infinity}, + client := #{bytes := #{rate := infinity}} + }, + parse_and_check(CfgStr) + ). + %%-------------------------------------------------------------------- %%% Internal functions %%-------------------------------------------------------------------- @@ -748,17 +810,16 @@ connect(Name, Cfg) -> Limiter. make_limiter_cfg() -> - Infinity = emqx_limiter_schema:infinity_value(), Client = #{ - rate => Infinity, + rate => infinity, initial => 0, - capacity => Infinity, + burst => 0, low_watermark => 0, divisible => false, max_retry_time => timer:seconds(5), failure_strategy => force }, - #{client => Client, rate => Infinity, initial => 0, capacity => Infinity}. + #{client => Client, rate => infinity, initial => 0, burst => 0}. add_bucket(Cfg) -> add_bucket(?MODULE, Cfg). @@ -812,3 +873,7 @@ apply_modifier(Pairs, #{default := Template}) -> Acc#{N => M(Template)} end, lists:foldl(Fun, #{}, Pairs). + +parse_and_check(ConfigString) -> + ok = emqx_common_test_helpers:load_config(emqx_schema, ConfigString), + emqx:get_config([listeners, tcp, default, limiter]). diff --git a/apps/emqx/test/emqx_router_SUITE.erl b/apps/emqx/test/emqx_router_SUITE.erl index 298a33fe8..2db0acf82 100644 --- a/apps/emqx/test/emqx_router_SUITE.erl +++ b/apps/emqx/test/emqx_router_SUITE.erl @@ -119,7 +119,7 @@ t_has_routes(_) -> ?R:delete_route(<<"devices/+/messages">>). t_unexpected(_) -> - Router = emqx_misc:proc_name(?R, 1), + Router = emqx_utils:proc_name(?R, 1), ?assertEqual(ignored, gen_server:call(Router, bad_request)), ?assertEqual(ok, gen_server:cast(Router, bad_message)), Router ! bad_info. diff --git a/apps/emqx/test/emqx_tls_lib_tests.erl b/apps/emqx/test/emqx_tls_lib_tests.erl index 5510e4027..ad9598107 100644 --- a/apps/emqx/test/emqx_tls_lib_tests.erl +++ b/apps/emqx/test/emqx_tls_lib_tests.erl @@ -191,7 +191,7 @@ ssl_files_save_delete_test() -> FileKey = maps:get(<<"keyfile">>, SSL), ?assertMatch(<<"/tmp/ssl-test-dir/key-", _:16/binary>>, FileKey), ?assertEqual({ok, bin(test_key())}, file:read_file(FileKey)), - FileIssuerPem = emqx_map_lib:deep_get([<<"ocsp">>, <<"issuer_pem">>], SSL), + FileIssuerPem = emqx_utils_maps:deep_get([<<"ocsp">>, <<"issuer_pem">>], SSL), ?assertMatch(<<"/tmp/ssl-test-dir/ocsp_issuer_pem-", _:16/binary>>, FileIssuerPem), ?assertEqual({ok, bin(test_key())}, file:read_file(FileIssuerPem)), %% no old file to delete @@ -251,8 +251,8 @@ ssl_file_replace_test() -> {ok, SSL3} = emqx_tls_lib:ensure_ssl_files(Dir, SSL1), File1 = maps:get(<<"keyfile">>, SSL2), File2 = maps:get(<<"keyfile">>, SSL3), - IssuerPem1 = emqx_map_lib:deep_get([<<"ocsp">>, <<"issuer_pem">>], SSL2), - IssuerPem2 = emqx_map_lib:deep_get([<<"ocsp">>, <<"issuer_pem">>], SSL3), + IssuerPem1 = emqx_utils_maps:deep_get([<<"ocsp">>, <<"issuer_pem">>], SSL2), + IssuerPem2 = emqx_utils_maps:deep_get([<<"ocsp">>, <<"issuer_pem">>], SSL3), ?assert(filelib:is_regular(File1)), ?assert(filelib:is_regular(File2)), ?assert(filelib:is_regular(IssuerPem1)), diff --git a/apps/emqx/test/emqx_trace_SUITE.erl b/apps/emqx/test/emqx_trace_SUITE.erl index c66808132..140ec79ff 100644 --- a/apps/emqx/test/emqx_trace_SUITE.erl +++ b/apps/emqx/test/emqx_trace_SUITE.erl @@ -22,10 +22,9 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/emqx_trace.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). --record(emqx_trace, {name, type, filter, enable = true, start_at, end_at}). - %%-------------------------------------------------------------------- %% Setups %%-------------------------------------------------------------------- @@ -97,7 +96,9 @@ t_base_create_delete(_Config) -> type => clientid, name => <<"name1">>, start_at => Now, - end_at => Now + 30 * 60 + end_at => Now + 30 * 60, + payload_encode => text, + extra => #{} } ], ?assertEqual(ExpectFormat, emqx_trace:format([TraceRec])), @@ -385,6 +386,48 @@ t_find_closed_time(_Config) -> ?assertEqual(1000, emqx_trace:find_closest_time(Traces, Now)), ok. +t_migrate_trace(_Config) -> + build_new_trace_data(), + build_old_trace_data(), + reload(), + Traces = emqx_trace:format(emqx_trace:list()), + ?assertEqual(2, erlang:length(Traces)), + lists:foreach( + fun(#{name := Name, enable := Enable}) -> + ?assertEqual(true, Enable, Name) + end, + Traces + ), + LoggerIds = logger:get_handler_ids(), + lists:foreach( + fun(Id) -> + ?assertEqual(true, lists:member(Id, LoggerIds), LoggerIds) + end, + [ + trace_topic_test_topic_migrate_new, + trace_topic_test_topic_migrate_old + ] + ), + ok. + +build_new_trace_data() -> + Now = erlang:system_time(second), + {ok, _} = emqx_trace:create([ + {<<"name">>, <<"test_topic_migrate_new">>}, + {<<"type">>, topic}, + {<<"topic">>, <<"/test/migrate/new">>}, + {<<"start_at">>, Now - 10} + ]). + +build_old_trace_data() -> + Now = erlang:system_time(second), + OldAttrs = [name, type, filter, enable, start_at, end_at], + {atomic, ok} = mnesia:transform_table(emqx_trace, ignore, OldAttrs, emqx_trace), + OldTrace = + {emqx_trace, <<"test_topic_migrate_old">>, topic, <<"topic">>, true, Now - 10, Now + 100}, + ok = mnesia:dirty_write(OldTrace), + ok. + reload() -> catch ok = gen_server:stop(emqx_trace), {ok, _Pid} = emqx_trace:start_link(). diff --git a/apps/emqx/test/emqx_ws_connection_SUITE.erl b/apps/emqx/test/emqx_ws_connection_SUITE.erl index de8b1c9af..1ae23361e 100644 --- a/apps/emqx/test/emqx_ws_connection_SUITE.erl +++ b/apps/emqx/test/emqx_ws_connection_SUITE.erl @@ -509,16 +509,16 @@ t_handle_timeout_emit_stats(_) -> t_ensure_rate_limit(_) -> {ok, Rate} = emqx_limiter_schema:to_rate("50MB"), Limiter = init_limiter(#{ - bytes_in => bucket_cfg(), - message_in => bucket_cfg(), - client => #{bytes_in => client_cfg(Rate)} + bytes => bucket_cfg(), + messages => bucket_cfg(), + client => #{bytes => client_cfg(Rate)} }), St = st(#{limiter => Limiter}), %% must bigger than value in emqx_ratelimit_SUITE {ok, Need} = emqx_limiter_schema:to_capacity("1GB"), St1 = ?ws_conn:check_limiter( - [{Need, bytes_in}], + [{Need, bytes}], [], fun(_, _, S) -> S end, [], @@ -699,23 +699,21 @@ init_limiter() -> init_limiter(limiter_cfg()). init_limiter(LimiterCfg) -> - emqx_limiter_container:get_limiter_by_types(?LIMITER_ID, [bytes_in, message_in], LimiterCfg). + emqx_limiter_container:get_limiter_by_types(?LIMITER_ID, [bytes, messages], LimiterCfg). limiter_cfg() -> Cfg = bucket_cfg(), Client = client_cfg(), - #{bytes_in => Cfg, message_in => Cfg, client => #{bytes_in => Client, message_in => Client}}. + #{bytes => Cfg, messages => Cfg, client => #{bytes => Client, messages => Client}}. client_cfg() -> - Infinity = emqx_limiter_schema:infinity_value(), - client_cfg(Infinity). + client_cfg(infinity). client_cfg(Rate) -> - Infinity = emqx_limiter_schema:infinity_value(), #{ rate => Rate, initial => 0, - capacity => Infinity, + burst => 0, low_watermark => 1, divisible => false, max_retry_time => timer:seconds(5), @@ -723,14 +721,13 @@ client_cfg(Rate) -> }. bucket_cfg() -> - Infinity = emqx_limiter_schema:infinity_value(), - #{rate => Infinity, initial => 0, capacity => Infinity}. + #{rate => infinity, initial => 0, burst => 0}. add_bucket() -> Cfg = bucket_cfg(), - emqx_limiter_server:add_bucket(?LIMITER_ID, bytes_in, Cfg), - emqx_limiter_server:add_bucket(?LIMITER_ID, message_in, Cfg). + emqx_limiter_server:add_bucket(?LIMITER_ID, bytes, Cfg), + emqx_limiter_server:add_bucket(?LIMITER_ID, messages, Cfg). del_bucket() -> - emqx_limiter_server:del_bucket(?LIMITER_ID, bytes_in), - emqx_limiter_server:del_bucket(?LIMITER_ID, message_in). + emqx_limiter_server:del_bucket(?LIMITER_ID, bytes), + emqx_limiter_server:del_bucket(?LIMITER_ID, messages). diff --git a/apps/emqx_authn/rebar.config b/apps/emqx_authn/rebar.config index 8fd9cea0f..5f0043c39 100644 --- a/apps/emqx_authn/rebar.config +++ b/apps/emqx_authn/rebar.config @@ -2,6 +2,7 @@ {deps, [ {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}}, {emqx_connector, {path, "../emqx_connector"}} ]}. diff --git a/apps/emqx_authn/src/emqx_authn.app.src b/apps/emqx_authn/src/emqx_authn.app.src index caa59e455..c1d48909c 100644 --- a/apps/emqx_authn/src/emqx_authn.app.src +++ b/apps/emqx_authn/src/emqx_authn.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_authn, [ {description, "EMQX Authentication"}, - {vsn, "0.1.16"}, + {vsn, "0.1.18"}, {modules, []}, {registered, [emqx_authn_sup, emqx_authn_registry]}, {applications, [kernel, stdlib, emqx_resource, emqx_connector, ehttpc, epgsql, mysql, jose]}, diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index ad9cd8579..de856f163 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -872,8 +872,8 @@ lookup_from_local_node(ChainName, AuthenticatorID) -> case emqx_resource:get_instance(ResourceId) of {error, not_found} -> {error, {NodeId, not_found_resource}}; - {ok, _, #{status := Status, metrics := ResourceMetrics}} -> - {ok, {NodeId, Status, Metrics, ResourceMetrics}} + {ok, _, #{status := Status}} -> + {ok, {NodeId, Status, Metrics, emqx_resource:get_metrics(ResourceId)}} end end; {error, Reason} -> @@ -929,7 +929,7 @@ aggregate_metrics([]) -> aggregate_metrics([HeadMetrics | AllMetrics]) -> ErrorLogger = fun(Reason) -> ?SLOG(info, #{msg => "bad_metrics_value", error => Reason}) end, Fun = fun(ElemMap, AccMap) -> - emqx_map_lib:best_effort_recursive_sum(AccMap, ElemMap, ErrorLogger) + emqx_utils_maps:best_effort_recursive_sum(AccMap, ElemMap, ErrorLogger) end, lists:foldl(Fun, HeadMetrics, AllMetrics). @@ -1069,7 +1069,7 @@ update_user(ChainName, AuthenticatorID, UserID, UserInfo0) -> true -> serialize_error({missing_parameter, password}); false -> - UserInfo = emqx_map_lib:safe_atom_key_map(UserInfo0), + UserInfo = emqx_utils_maps:safe_atom_key_map(UserInfo0), case emqx_authentication:update_user(ChainName, AuthenticatorID, UserID, UserInfo) of {ok, User} -> {200, User}; diff --git a/apps/emqx_authn/src/emqx_authn_schema.erl b/apps/emqx_authn/src/emqx_authn_schema.erl index 112ea2076..a7cdaac5f 100644 --- a/apps/emqx_authn/src/emqx_authn_schema.erl +++ b/apps/emqx_authn/src/emqx_authn_schema.erl @@ -18,10 +18,12 @@ -elvis([{elvis_style, invalid_dynamic_call, disable}]). -include_lib("hocon/include/hoconsc.hrl"). +-include("emqx_authn.hrl"). -export([ common_fields/0, roots/0, + validations/0, tags/0, fields/1, authenticator_type/0, @@ -207,3 +209,27 @@ array(Name) -> array(Name, DescId) -> {Name, ?HOCON(?R_REF(Name), #{desc => ?DESC(DescId)})}. + +validations() -> + [ + {check_http_ssl_opts, fun(Conf) -> + CheckFun = fun emqx_authn_http:check_ssl_opts/1, + validation(Conf, CheckFun) + end}, + {check_http_headers, fun(Conf) -> + CheckFun = fun emqx_authn_http:check_headers/1, + validation(Conf, CheckFun) + end} + ]. + +validation(Conf, CheckFun) when is_map(Conf) -> + validation(hocon_maps:get(?CONF_NS, Conf), CheckFun); +validation(undefined, _) -> + ok; +validation([], _) -> + ok; +validation([AuthN | Tail], CheckFun) -> + case CheckFun(#{?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY => AuthN}) of + ok -> validation(Tail, CheckFun); + Error -> Error + end. diff --git a/apps/emqx_authn/src/emqx_authn_utils.erl b/apps/emqx_authn/src/emqx_authn_utils.erl index 1352e3daf..12520251e 100644 --- a/apps/emqx_authn/src/emqx_authn_utils.erl +++ b/apps/emqx_authn/src/emqx_authn_utils.erl @@ -28,6 +28,7 @@ parse_sql/2, render_deep/2, render_str/2, + render_urlencoded_str/2, render_sql_params/2, is_superuser/1, bin/1, @@ -129,6 +130,13 @@ render_str(Template, Credential) -> #{return => full_binary, var_trans => fun handle_var/2} ). +render_urlencoded_str(Template, Credential) -> + emqx_placeholder:proc_tmpl( + Template, + mapping_credential(Credential), + #{return => full_binary, var_trans => fun urlencode_var/2} + ). + render_sql_params(ParamList, Credential) -> emqx_placeholder:proc_tmpl( ParamList, @@ -217,6 +225,11 @@ without_password(Credential, [Name | Rest]) -> without_password(Credential, Rest) end. +urlencode_var({var, _} = Var, Value) -> + emqx_http_lib:uri_encode(handle_var(Var, Value)); +urlencode_var(Var, Value) -> + handle_var(Var, Value). + handle_var({var, _Name}, undefined) -> <<>>; handle_var({var, <<"peerhost">>}, PeerHost) -> diff --git a/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl b/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl index 84f2c9525..b11b89081 100644 --- a/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl +++ b/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl @@ -105,14 +105,16 @@ mnesia(boot) -> %% Hocon Schema %%------------------------------------------------------------------------------ -namespace() -> "authn-scram-builtin_db". +namespace() -> "authn". tags() -> [<<"Authentication">>]. -roots() -> [?CONF_NS]. +%% used for config check when the schema module is resolved +roots() -> + [{?CONF_NS, hoconsc:mk(hoconsc:ref(?MODULE, scram))}]. -fields(?CONF_NS) -> +fields(scram) -> [ {mechanism, emqx_authn_schema:mechanism(scram)}, {backend, emqx_authn_schema:backend(built_in_database)}, @@ -120,7 +122,7 @@ fields(?CONF_NS) -> {iteration_count, fun iteration_count/1} ] ++ emqx_authn_schema:common_fields(). -desc(?CONF_NS) -> +desc(scram) -> "Settings for Salted Challenge Response Authentication Mechanism\n" "(SCRAM) authentication."; desc(_) -> @@ -141,7 +143,7 @@ iteration_count(_) -> undefined. %%------------------------------------------------------------------------------ refs() -> - [hoconsc:ref(?MODULE, ?CONF_NS)]. + [hoconsc:ref(?MODULE, scram)]. create( AuthenticatorID, diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl index 9be1d0a33..eddad92a3 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl @@ -38,6 +38,8 @@ headers/1 ]). +-export([check_headers/1, check_ssl_opts/1]). + -export([ refs/0, union_member_selector/1, @@ -51,34 +53,35 @@ %% Hocon Schema %%------------------------------------------------------------------------------ -namespace() -> "authn-http". +namespace() -> "authn". tags() -> [<<"Authentication">>]. +%% used for config check when the schema module is resolved roots() -> [ {?CONF_NS, hoconsc:mk( - hoconsc:union(fun union_member_selector/1), + hoconsc:union(fun ?MODULE:union_member_selector/1), #{} )} ]. -fields(get) -> +fields(http_get) -> [ {method, #{type => get, required => true, desc => ?DESC(method)}}, {headers, fun headers_no_content_type/1} ] ++ common_fields(); -fields(post) -> +fields(http_post) -> [ {method, #{type => post, required => true, desc => ?DESC(method)}}, {headers, fun headers/1} ] ++ common_fields(). -desc(get) -> +desc(http_get) -> ?DESC(get); -desc(post) -> +desc(http_post) -> ?DESC(post); desc(_) -> undefined. @@ -106,8 +109,8 @@ common_fields() -> validations() -> [ - {check_ssl_opts, fun check_ssl_opts/1}, - {check_headers, fun check_headers/1} + {check_ssl_opts, fun ?MODULE:check_ssl_opts/1}, + {check_headers, fun ?MODULE:check_headers/1} ]. url(type) -> binary(); @@ -156,8 +159,8 @@ request_timeout(_) -> undefined. refs() -> [ - hoconsc:ref(?MODULE, get), - hoconsc:ref(?MODULE, post) + hoconsc:ref(?MODULE, http_get), + hoconsc:ref(?MODULE, http_post) ]. union_member_selector(all_union_members) -> @@ -166,9 +169,9 @@ union_member_selector({value, Value}) -> refs(Value). refs(#{<<"method">> := <<"get">>}) -> - [hoconsc:ref(?MODULE, get)]; + [hoconsc:ref(?MODULE, http_get)]; refs(#{<<"method">> := <<"post">>}) -> - [hoconsc:ref(?MODULE, post)]; + [hoconsc:ref(?MODULE, http_post)]; refs(_) -> throw(#{ field_name => method, @@ -261,21 +264,47 @@ transform_header_name(Headers) -> ). check_ssl_opts(Conf) -> - {BaseUrl, _Path, _Query} = parse_url(get_conf_val("url", Conf)), - case BaseUrl of - <<"https://", _/binary>> -> - case get_conf_val("ssl.enable", Conf) of - true -> ok; - false -> false + case is_backend_http(Conf) of + true -> + Url = get_conf_val("url", Conf), + {BaseUrl, _Path, _Query} = parse_url(Url), + case BaseUrl of + <<"https://", _/binary>> -> + case get_conf_val("ssl.enable", Conf) of + true -> + ok; + false -> + <<"it's required to enable the TLS option to establish a https connection">> + end; + <<"http://", _/binary>> -> + ok end; - <<"http://", _/binary>> -> + false -> ok end. check_headers(Conf) -> - Method = to_bin(get_conf_val("method", Conf)), - Headers = get_conf_val("headers", Conf), - Method =:= <<"post">> orelse (not maps:is_key(<<"content-type">>, Headers)). + case is_backend_http(Conf) of + true -> + Headers = get_conf_val("headers", Conf), + case to_bin(get_conf_val("method", Conf)) of + <<"post">> -> + ok; + <<"get">> -> + case maps:is_key(<<"content-type">>, Headers) of + false -> ok; + true -> <<"HTTP GET requests cannot include content-type header.">> + end + end; + false -> + ok + end. + +is_backend_http(Conf) -> + case get_conf_val("backend", Conf) of + http -> true; + _ -> false + end. parse_url(Url) -> case string:split(Url, "//", leading) of @@ -285,9 +314,9 @@ parse_url(Url) -> BaseUrl = iolist_to_binary([Scheme, "//", HostPort]), case string:split(Remaining, "?", leading) of [Path, QueryString] -> - {BaseUrl, Path, QueryString}; + {BaseUrl, <<"/", Path/binary>>, QueryString}; [Path] -> - {BaseUrl, Path, <<>>} + {BaseUrl, <<"/", Path/binary>>, <<>>} end; [HostPort] -> {iolist_to_binary([Scheme, "//", HostPort]), <<>>, <<>>} @@ -310,7 +339,7 @@ parse_config( method => Method, path => Path, headers => ensure_header_name_type(Headers), - base_path_templete => emqx_authn_utils:parse_str(Path), + base_path_template => emqx_authn_utils:parse_str(Path), base_query_template => emqx_authn_utils:parse_deep( cow_qs:parse_qs(to_bin(Query)) ), @@ -323,12 +352,12 @@ parse_config( generate_request(Credential, #{ method := Method, headers := Headers0, - base_path_templete := BasePathTemplate, + base_path_template := BasePathTemplate, base_query_template := BaseQueryTemplate, body_template := BodyTemplate }) -> Headers = maps:to_list(Headers0), - Path = emqx_authn_utils:render_str(BasePathTemplate, Credential), + Path = emqx_authn_utils:render_urlencoded_str(BasePathTemplate, Credential), Query = emqx_authn_utils:render_deep(BaseQueryTemplate, Credential), Body = emqx_authn_utils:render_deep(BodyTemplate, Credential), case Method of @@ -343,9 +372,9 @@ generate_request(Credential, #{ end. append_query(Path, []) -> - encode_path(Path); + Path; append_query(Path, Query) -> - encode_path(Path) ++ "?" ++ binary_to_list(qs(Query)). + Path ++ "?" ++ binary_to_list(qs(Query)). qs(KVs) -> qs(KVs, []). @@ -357,7 +386,7 @@ qs([{K, V} | More], Acc) -> qs(More, [["&", uri_encode(K), "=", uri_encode(V)] | Acc]). serialize_body(<<"application/json">>, Body) -> - emqx_json:encode(Body); + emqx_utils_json:encode(Body); serialize_body(<<"application/x-www-form-urlencoded">>, Body) -> qs(maps:to_list(Body)). @@ -395,7 +424,7 @@ safely_parse_body(ContentType, Body) -> end. parse_body(<<"application/json", _/binary>>, Body) -> - {ok, emqx_json:decode(Body, [return_maps])}; + {ok, emqx_utils_json:decode(Body, [return_maps])}; parse_body(<<"application/x-www-form-urlencoded", _/binary>>, Body) -> Flags = [<<"result">>, <<"is_superuser">>], RawMap = maps:from_list(cow_qs:parse_qs(Body)), @@ -407,10 +436,6 @@ parse_body(ContentType, _) -> uri_encode(T) -> emqx_http_lib:uri_encode(to_list(T)). -encode_path(Path) -> - Parts = string:split(Path, "/", all), - lists:flatten(["/" ++ Part || Part <- lists:map(fun uri_encode/1, Parts)]). - request_for_log(Credential, #{url := Url} = State) -> SafeCredential = emqx_authn_utils:without_password(Credential), case generate_request(SafeCredential, State) of diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_client.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_client.erl index 5ee923859..23d939f7d 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_client.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_client.erl @@ -99,7 +99,7 @@ handle_info( State1; {StatusLine, Headers, Body} -> try - JWKS = jose_jwk:from(emqx_json:decode(Body, [return_maps])), + JWKS = jose_jwk:from(emqx_utils_json:decode(Body, [return_maps])), {_, JWKs} = JWKS#jose_jwk.keys, State1#{jwks := JWKs} catch diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_connector.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_connector.erl index 1fdd7cef7..fe0349b4a 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_connector.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_connector.erl @@ -35,18 +35,17 @@ callback_mode() -> always_sync. on_start(InstId, Opts) -> - PoolName = emqx_plugin_libs_pool:pool_name(InstId), PoolOpts = [ {pool_size, maps:get(pool_size, Opts, ?DEFAULT_POOL_SIZE)}, {connector_opts, Opts} ], - case emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, PoolOpts) of - ok -> {ok, #{pool_name => PoolName}}; + case emqx_resource_pool:start(InstId, ?MODULE, PoolOpts) of + ok -> {ok, #{pool_name => InstId}}; {error, Reason} -> {error, Reason} end. on_stop(_InstId, #{pool_name := PoolName}) -> - emqx_plugin_libs_pool:stop_pool(PoolName). + emqx_resource_pool:stop(PoolName). on_query(InstId, get_jwks, #{pool_name := PoolName}) -> Result = ecpool:pick_and_do(PoolName, {emqx_authn_jwks_client, get_jwks, []}, no_handover), @@ -72,18 +71,17 @@ on_query(_InstId, {update, Opts}, #{pool_name := PoolName}) -> ok. on_get_status(_InstId, #{pool_name := PoolName}) -> - Func = - fun(Conn) -> - case emqx_authn_jwks_client:get_jwks(Conn) of - {ok, _} -> true; - _ -> false - end - end, - case emqx_plugin_libs_pool:health_check_ecpool_workers(PoolName, Func) of + case emqx_resource_pool:health_check_workers(PoolName, fun health_check/1) of true -> connected; false -> disconnected end. +health_check(Conn) -> + case emqx_authn_jwks_client:get_jwks(Conn) of + {ok, _} -> true; + _ -> false + end. + connect(Opts) -> ConnectorOpts = proplists:get_value(connector_opts, Opts), emqx_authn_jwks_client:start_link(ConnectorOpts). diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl index 73ba6a2c1..0df9014b8 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl @@ -43,36 +43,57 @@ %% Hocon Schema %%------------------------------------------------------------------------------ -namespace() -> "authn-jwt". +namespace() -> "authn". tags() -> [<<"Authentication">>]. +%% used for config check when the schema module is resolved roots() -> [ {?CONF_NS, hoconsc:mk( - hoconsc:union(fun union_member_selector/1), + hoconsc:union(fun ?MODULE:union_member_selector/1), #{} )} ]. -fields('hmac-based') -> +fields(jwt_hmac) -> [ - {use_jwks, sc(hoconsc:enum([false]), #{required => true, desc => ?DESC(use_jwks)})}, + %% for hmac, it's the 'algorithm' field which selects this type + %% use_jwks field can be ignored (kept for backward compatibility) + {use_jwks, + sc( + hoconsc:enum([false]), + #{ + required => false, + desc => ?DESC(use_jwks), + importance => ?IMPORTANCE_HIDDEN + } + )}, {algorithm, sc(hoconsc:enum(['hmac-based']), #{required => true, desc => ?DESC(algorithm)})}, {secret, fun secret/1}, {secret_base64_encoded, fun secret_base64_encoded/1} ] ++ common_fields(); -fields('public-key') -> +fields(jwt_public_key) -> [ - {use_jwks, sc(hoconsc:enum([false]), #{required => true, desc => ?DESC(use_jwks)})}, + %% for public-key, it's the 'algorithm' field which selects this type + %% use_jwks field can be ignored (kept for backward compatibility) + {use_jwks, + sc( + hoconsc:enum([false]), + #{ + required => false, + desc => ?DESC(use_jwks), + importance => ?IMPORTANCE_HIDDEN + } + )}, {algorithm, sc(hoconsc:enum(['public-key']), #{required => true, desc => ?DESC(algorithm)})}, {public_key, fun public_key/1} ] ++ common_fields(); -fields('jwks') -> +fields(jwt_jwks) -> [ {use_jwks, sc(hoconsc:enum([true]), #{required => true, desc => ?DESC(use_jwks)})}, {endpoint, fun endpoint/1}, @@ -85,12 +106,12 @@ fields('jwks') -> }} ] ++ common_fields(). -desc('hmac-based') -> - ?DESC('hmac-based'); -desc('public-key') -> - ?DESC('public-key'); -desc('jwks') -> - ?DESC('jwks'); +desc(jwt_hmac) -> + ?DESC(jwt_hmac); +desc(jwt_public_key) -> + ?DESC(jwt_public_key); +desc(jwt_jwks) -> + ?DESC(jwt_jwks); desc(undefined) -> undefined. @@ -160,9 +181,9 @@ from(_) -> undefined. refs() -> [ - hoconsc:ref(?MODULE, 'hmac-based'), - hoconsc:ref(?MODULE, 'public-key'), - hoconsc:ref(?MODULE, 'jwks') + hoconsc:ref(?MODULE, jwt_hmac), + hoconsc:ref(?MODULE, jwt_public_key), + hoconsc:ref(?MODULE, jwt_jwks) ]. union_member_selector(all_union_members) -> @@ -179,11 +200,11 @@ boolean(<<"false">>) -> false; boolean(Other) -> Other. select_ref(true, _) -> - [hoconsc:ref(?MODULE, 'jwks')]; + [hoconsc:ref(?MODULE, 'jwt_jwks')]; select_ref(false, #{<<"public_key">> := _}) -> - [hoconsc:ref(?MODULE, 'public-key')]; + [hoconsc:ref(?MODULE, jwt_public_key)]; select_ref(false, _) -> - [hoconsc:ref(?MODULE, 'hmac-based')]; + [hoconsc:ref(?MODULE, jwt_hmac)]; select_ref(_, _) -> throw(#{ field_name => use_jwks, @@ -407,7 +428,7 @@ do_verify(_JWT, [], _VerifyClaims) -> do_verify(JWT, [JWK | More], VerifyClaims) -> try jose_jws:verify(JWK, JWT) of {true, Payload, _JWT} -> - Claims0 = emqx_json:decode(Payload, [return_maps]), + Claims0 = emqx_utils_json:decode(Payload, [return_maps]), Claims = try_convert_to_num(Claims0, [<<"exp">>, <<"iat">>, <<"nbf">>]), case verify_claims(Claims, VerifyClaims) of ok -> diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl index 25a3a5976..d57e9e00e 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl @@ -107,14 +107,16 @@ mnesia(boot) -> %% Hocon Schema %%------------------------------------------------------------------------------ -namespace() -> "authn-builtin_db". +namespace() -> "authn". tags() -> [<<"Authentication">>]. -roots() -> [?CONF_NS]. +%% used for config check when the schema module is resolved +roots() -> + [{?CONF_NS, hoconsc:mk(hoconsc:ref(?MODULE, builtin_db))}]. -fields(?CONF_NS) -> +fields(builtin_db) -> [ {mechanism, emqx_authn_schema:mechanism(password_based)}, {backend, emqx_authn_schema:backend(built_in_database)}, @@ -122,8 +124,8 @@ fields(?CONF_NS) -> {password_hash_algorithm, fun emqx_authn_password_hashing:type_rw/1} ] ++ emqx_authn_schema:common_fields(). -desc(?CONF_NS) -> - ?DESC(?CONF_NS); +desc(builtin_db) -> + ?DESC(builtin_db); desc(_) -> undefined. @@ -138,7 +140,7 @@ user_id_type(_) -> undefined. %%------------------------------------------------------------------------------ refs() -> - [hoconsc:ref(?MODULE, ?CONF_NS)]. + [hoconsc:ref(?MODULE, builtin_db)]. create(_AuthenticatorID, Config) -> create(Config). @@ -332,7 +334,7 @@ run_fuzzy_filter( %% Example: data/user-credentials.json import_users_from_json(Bin, #{user_group := UserGroup}) -> - case emqx_json:safe_decode(Bin, [return_maps]) of + case emqx_utils_json:safe_decode(Bin, [return_maps]) of {ok, List} -> trans(fun ?MODULE:import/2, [UserGroup, List]); {error, Reason} -> diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl index 22b930485..1a766b975 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl @@ -44,32 +44,33 @@ %% Hocon Schema %%------------------------------------------------------------------------------ -namespace() -> "authn-mongodb". +namespace() -> "authn". tags() -> [<<"Authentication">>]. +%% used for config check when the schema module is resolved roots() -> [ {?CONF_NS, hoconsc:mk( - hoconsc:union(fun union_member_selector/1), + hoconsc:union(fun ?MODULE:union_member_selector/1), #{} )} ]. -fields(standalone) -> +fields(mongo_single) -> common_fields() ++ emqx_connector_mongo:fields(single); -fields('replica-set') -> +fields(mongo_rs) -> common_fields() ++ emqx_connector_mongo:fields(rs); -fields('sharded-cluster') -> +fields(mongo_sharded) -> common_fields() ++ emqx_connector_mongo:fields(sharded). -desc(standalone) -> - ?DESC(standalone); -desc('replica-set') -> +desc(mongo_single) -> + ?DESC(single); +desc(mongo_rs) -> ?DESC('replica-set'); -desc('sharded-cluster') -> +desc(mongo_sharded) -> ?DESC('sharded-cluster'); desc(_) -> undefined. @@ -126,9 +127,9 @@ is_superuser_field(_) -> undefined. refs() -> [ - hoconsc:ref(?MODULE, standalone), - hoconsc:ref(?MODULE, 'replica-set'), - hoconsc:ref(?MODULE, 'sharded-cluster') + hoconsc:ref(?MODULE, mongo_single), + hoconsc:ref(?MODULE, mongo_rs), + hoconsc:ref(?MODULE, mongo_sharded) ]. create(_AuthenticatorID, Config) -> @@ -254,11 +255,11 @@ union_member_selector({value, Value}) -> refs(Value). refs(#{<<"mongo_type">> := <<"single">>}) -> - [hoconsc:ref(?MODULE, standalone)]; + [hoconsc:ref(?MODULE, mongo_single)]; refs(#{<<"mongo_type">> := <<"rs">>}) -> - [hoconsc:ref(?MODULE, 'replica-set')]; + [hoconsc:ref(?MODULE, mongo_rs)]; refs(#{<<"mongo_type">> := <<"sharded">>}) -> - [hoconsc:ref(?MODULE, 'sharded-cluster')]; + [hoconsc:ref(?MODULE, mongo_sharded)]; refs(_) -> throw(#{ field_name => mongo_type, diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl index bedd169e2..d8e631885 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl @@ -45,14 +45,16 @@ %% Hocon Schema %%------------------------------------------------------------------------------ -namespace() -> "authn-mysql". +namespace() -> "authn". tags() -> [<<"Authentication">>]. -roots() -> [?CONF_NS]. +%% used for config check when the schema module is resolved +roots() -> + [{?CONF_NS, hoconsc:mk(hoconsc:ref(?MODULE, mysql))}]. -fields(?CONF_NS) -> +fields(mysql) -> [ {mechanism, emqx_authn_schema:mechanism(password_based)}, {backend, emqx_authn_schema:backend(mysql)}, @@ -62,8 +64,8 @@ fields(?CONF_NS) -> ] ++ emqx_authn_schema:common_fields() ++ proplists:delete(prepare_statement, emqx_connector_mysql:fields(config)). -desc(?CONF_NS) -> - ?DESC(?CONF_NS); +desc(mysql) -> + ?DESC(mysql); desc(_) -> undefined. @@ -82,7 +84,7 @@ query_timeout(_) -> undefined. %%------------------------------------------------------------------------------ refs() -> - [hoconsc:ref(?MODULE, ?CONF_NS)]. + [hoconsc:ref(?MODULE, mysql)]. create(_AuthenticatorID, Config) -> create(Config). diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl index 3d9e1e08f..d9526cc7b 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl @@ -49,14 +49,16 @@ %% Hocon Schema %%------------------------------------------------------------------------------ -namespace() -> "authn-postgresql". +namespace() -> "authn". tags() -> [<<"Authentication">>]. -roots() -> [?CONF_NS]. +%% used for config check when the schema module is resolved +roots() -> + [{?CONF_NS, hoconsc:mk(hoconsc:ref(?MODULE, postgresql))}]. -fields(?CONF_NS) -> +fields(postgresql) -> [ {mechanism, emqx_authn_schema:mechanism(password_based)}, {backend, emqx_authn_schema:backend(postgresql)}, @@ -66,8 +68,8 @@ fields(?CONF_NS) -> emqx_authn_schema:common_fields() ++ proplists:delete(prepare_statement, emqx_connector_pgsql:fields(config)). -desc(?CONF_NS) -> - ?DESC(?CONF_NS); +desc(postgresql) -> + ?DESC(postgresql); desc(_) -> undefined. @@ -81,7 +83,7 @@ query(_) -> undefined. %%------------------------------------------------------------------------------ refs() -> - [hoconsc:ref(?MODULE, ?CONF_NS)]. + [hoconsc:ref(?MODULE, postgresql)]. create(_AuthenticatorID, Config) -> create(Config). diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl index ff81fd4ca..27d8c540a 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl @@ -44,32 +44,33 @@ %% Hocon Schema %%------------------------------------------------------------------------------ -namespace() -> "authn-redis". +namespace() -> "authn". tags() -> [<<"Authentication">>]. +%% used for config check when the schema module is resolved roots() -> [ {?CONF_NS, hoconsc:mk( - hoconsc:union(fun union_member_selector/1), + hoconsc:union(fun ?MODULE:union_member_selector/1), #{} )} ]. -fields(standalone) -> +fields(redis_single) -> common_fields() ++ emqx_connector_redis:fields(single); -fields(cluster) -> +fields(redis_cluster) -> common_fields() ++ emqx_connector_redis:fields(cluster); -fields(sentinel) -> +fields(redis_sentinel) -> common_fields() ++ emqx_connector_redis:fields(sentinel). -desc(standalone) -> - ?DESC(standalone); -desc(cluster) -> +desc(redis_single) -> + ?DESC(single); +desc(redis_cluster) -> ?DESC(cluster); -desc(sentinel) -> +desc(redis_sentinel) -> ?DESC(sentinel); desc(_) -> "". @@ -93,9 +94,9 @@ cmd(_) -> undefined. refs() -> [ - hoconsc:ref(?MODULE, standalone), - hoconsc:ref(?MODULE, cluster), - hoconsc:ref(?MODULE, sentinel) + hoconsc:ref(?MODULE, redis_single), + hoconsc:ref(?MODULE, redis_cluster), + hoconsc:ref(?MODULE, redis_sentinel) ]. union_member_selector(all_union_members) -> @@ -104,11 +105,11 @@ union_member_selector({value, Value}) -> refs(Value). refs(#{<<"redis_type">> := <<"single">>}) -> - [hoconsc:ref(?MODULE, standalone)]; + [hoconsc:ref(?MODULE, redis_single)]; refs(#{<<"redis_type">> := <<"cluster">>}) -> - [hoconsc:ref(?MODULE, cluster)]; + [hoconsc:ref(?MODULE, redis_cluster)]; refs(#{<<"redis_type">> := <<"sentinel">>}) -> - [hoconsc:ref(?MODULE, sentinel)]; + [hoconsc:ref(?MODULE, redis_sentinel)]; refs(_) -> throw(#{ field_name => redis_type, diff --git a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl index 11e2c6773..c7f718dfc 100644 --- a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl @@ -29,7 +29,7 @@ -define(assertAuthenticatorsMatch(Guard, Path), (fun() -> {ok, 200, Response} = request(get, uri(Path)), - ?assertMatch(Guard, jiffy:decode(Response, [return_maps])) + ?assertMatch(Guard, emqx_utils_json:decode(Response, [return_maps])) end)() ). @@ -234,7 +234,7 @@ test_authenticator(PathPrefix) -> get, uri(PathPrefix ++ [?CONF_NS, "password_based:http", "status"]) ), - {ok, RList} = emqx_json:safe_decode(Res), + {ok, RList} = emqx_utils_json:safe_decode(Res), Snd = fun({_, Val}) -> Val end, LookupVal = fun LookupV(List, RestJson) -> case List of @@ -353,7 +353,7 @@ test_authenticator_users(PathPrefix) -> <<"success">> := 0, <<"nomatch">> := 1 } - } = jiffy:decode(PageData0, [return_maps]); + } = emqx_utils_json:decode(PageData0, [return_maps]); ["listeners", 'tcp:default'] -> #{ <<"metrics">> := #{ @@ -361,7 +361,7 @@ test_authenticator_users(PathPrefix) -> <<"success">> := 0, <<"nomatch">> := 1 } - } = jiffy:decode(PageData0, [return_maps]) + } = emqx_utils_json:decode(PageData0, [return_maps]) end, InvalidUsers = [ @@ -384,7 +384,7 @@ test_authenticator_users(PathPrefix) -> lists:foreach( fun(User) -> {ok, 201, UserData} = request(post, UsersUri, User), - CreatedUser = jiffy:decode(UserData, [return_maps]), + CreatedUser = emqx_utils_json:decode(UserData, [return_maps]), ?assertMatch(#{<<"user_id">> := _}, CreatedUser) end, ValidUsers @@ -411,7 +411,7 @@ test_authenticator_users(PathPrefix) -> <<"success">> := 1, <<"nomatch">> := 1 } - } = jiffy:decode(PageData01, [return_maps]); + } = emqx_utils_json:decode(PageData01, [return_maps]); ["listeners", 'tcp:default'] -> #{ <<"metrics">> := #{ @@ -419,7 +419,7 @@ test_authenticator_users(PathPrefix) -> <<"success">> := 1, <<"nomatch">> := 1 } - } = jiffy:decode(PageData01, [return_maps]) + } = emqx_utils_json:decode(PageData01, [return_maps]) end, {ok, 200, Page1Data} = request(get, UsersUri ++ "?page=1&limit=2"), @@ -433,7 +433,7 @@ test_authenticator_users(PathPrefix) -> <<"count">> := 3 } } = - jiffy:decode(Page1Data, [return_maps]), + emqx_utils_json:decode(Page1Data, [return_maps]), {ok, 200, Page2Data} = request(get, UsersUri ++ "?page=2&limit=2"), @@ -445,7 +445,7 @@ test_authenticator_users(PathPrefix) -> <<"limit">> := 2, <<"count">> := 3 } - } = jiffy:decode(Page2Data, [return_maps]), + } = emqx_utils_json:decode(Page2Data, [return_maps]), ?assertEqual(2, length(Page1Users)), ?assertEqual(1, length(Page2Users)), @@ -465,7 +465,7 @@ test_authenticator_users(PathPrefix) -> <<"limit">> := 3, <<"count">> := 1 } - } = jiffy:decode(Super1Data, [return_maps]), + } = emqx_utils_json:decode(Super1Data, [return_maps]), ?assertEqual( [<<"u2">>], @@ -482,7 +482,7 @@ test_authenticator_users(PathPrefix) -> <<"limit">> := 3, <<"count">> := 2 } - } = jiffy:decode(Super2Data, [return_maps]), + } = emqx_utils_json:decode(Super2Data, [return_maps]), ?assertEqual( [<<"u1">>, <<"u3">>], @@ -509,7 +509,7 @@ test_authenticator_user(PathPrefix) -> {ok, 200, UserData} = request(get, UsersUri ++ "/u1"), - FetchedUser = jiffy:decode(UserData, [return_maps]), + FetchedUser = emqx_utils_json:decode(UserData, [return_maps]), ?assertMatch(#{<<"user_id">> := <<"u1">>}, FetchedUser), ?assertNotMatch(#{<<"password">> := _}, FetchedUser), diff --git a/apps/emqx_authn/test/emqx_authn_http_SUITE.erl b/apps/emqx_authn/test/emqx_authn_http_SUITE.erl index 9a3c7c833..b08167a5b 100644 --- a/apps/emqx_authn/test/emqx_authn_http_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_http_SUITE.erl @@ -41,13 +41,12 @@ -define(SERVER_RESPONSE_JSON(Result), ?SERVER_RESPONSE_JSON(Result, false)). -define(SERVER_RESPONSE_JSON(Result, IsSuperuser), - jiffy:encode(#{ + emqx_utils_json:encode(#{ result => Result, is_superuser => IsSuperuser }) ). --define(SERVER_RESPONSE_URLENCODE(Result), ?SERVER_RESPONSE_URLENCODE(Result, false)). -define(SERVER_RESPONSE_URLENCODE(Result, IsSuperuser), list_to_binary( "result=" ++ @@ -166,17 +165,65 @@ test_user_auth(#{ ?GLOBAL ). +t_authenticate_path_placeholders(_Config) -> + ok = emqx_authn_http_test_server:stop(), + {ok, _} = emqx_authn_http_test_server:start_link(?HTTP_PORT, <<"/[...]">>), + ok = emqx_authn_http_test_server:set_handler( + fun(Req0, State) -> + Req = + case cowboy_req:path(Req0) of + <<"/my/p%20ath//us%20er/auth//">> -> + cowboy_req:reply( + 200, + #{<<"content-type">> => <<"application/json">>}, + emqx_utils_json:encode(#{result => allow, is_superuser => false}), + Req0 + ); + Path -> + ct:pal("Unexpected path: ~p", [Path]), + cowboy_req:reply(403, Req0) + end, + {ok, Req, State} + end + ), + + Credentials = ?CREDENTIALS#{ + username => <<"us er">> + }, + + AuthConfig = maps:merge( + raw_http_auth_config(), + #{ + <<"url">> => <<"http://127.0.0.1:32333/my/p%20ath//${username}/auth//">>, + <<"body">> => #{} + } + ), + {ok, _} = emqx:update_config( + ?PATH, + {create_authenticator, ?GLOBAL, AuthConfig} + ), + + ?assertMatch( + {ok, #{is_superuser := false}}, + emqx_access_control:authenticate(Credentials) + ), + + _ = emqx_authn_test_lib:delete_authenticators( + [authentication], + ?GLOBAL + ). + t_no_value_for_placeholder(_Config) -> Handler = fun(Req0, State) -> {ok, RawBody, Req1} = cowboy_req:read_body(Req0), #{ <<"cert_subject">> := <<"">>, <<"cert_common_name">> := <<"">> - } = jiffy:decode(RawBody, [return_maps]), + } = emqx_utils_json:decode(RawBody, [return_maps]), Req = cowboy_req:reply( 200, #{<<"content-type">> => <<"application/json">>}, - jiffy:encode(#{result => allow, is_superuser => false}), + emqx_utils_json:encode(#{result => allow, is_superuser => false}), Req1 ), {ok, Req, State} @@ -444,7 +491,7 @@ samples() -> Req = cowboy_req:reply( 200, #{<<"content-type">> => <<"application/json">>}, - jiffy:encode(#{result => allow, is_superuser => false}), + emqx_utils_json:encode(#{result => allow, is_superuser => false}), Req0 ), {ok, Req, State} @@ -459,7 +506,7 @@ samples() -> Req = cowboy_req:reply( 200, #{<<"content-type">> => <<"application/json">>}, - jiffy:encode(#{result => allow, is_superuser => true}), + emqx_utils_json:encode(#{result => allow, is_superuser => true}), Req0 ), {ok, Req, State} @@ -512,11 +559,11 @@ samples() -> #{ <<"username">> := <<"plain">>, <<"password">> := <<"plain">> - } = jiffy:decode(RawBody, [return_maps]), + } = emqx_utils_json:decode(RawBody, [return_maps]), Req = cowboy_req:reply( 200, #{<<"content-type">> => <<"application/json">>}, - jiffy:encode(#{result => allow, is_superuser => false}), + emqx_utils_json:encode(#{result => allow, is_superuser => false}), Req1 ), {ok, Req, State} @@ -539,7 +586,7 @@ samples() -> Req = cowboy_req:reply( 200, #{<<"content-type">> => <<"application/json">>}, - jiffy:encode(#{result => allow, is_superuser => false}), + emqx_utils_json:encode(#{result => allow, is_superuser => false}), Req1 ), {ok, Req, State} @@ -565,11 +612,11 @@ samples() -> <<"peerhost">> := <<"127.0.0.1">>, <<"cert_subject">> := <<"cert_subject_data">>, <<"cert_common_name">> := <<"cert_common_name_data">> - } = jiffy:decode(RawBody, [return_maps]), + } = emqx_utils_json:decode(RawBody, [return_maps]), Req = cowboy_req:reply( 200, #{<<"content-type">> => <<"application/json">>}, - jiffy:encode(#{result => allow, is_superuser => false}), + emqx_utils_json:encode(#{result => allow, is_superuser => false}), Req1 ), {ok, Req, State} diff --git a/apps/emqx_authn/test/emqx_authn_https_SUITE.erl b/apps/emqx_authn/test/emqx_authn_https_SUITE.erl index 7d51ff425..c4315b69f 100644 --- a/apps/emqx_authn/test/emqx_authn_https_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_https_SUITE.erl @@ -168,7 +168,7 @@ cowboy_handler(Req0, State) -> Req = cowboy_req:reply( 200, #{<<"content-type">> => <<"application/json">>}, - jiffy:encode(#{result => allow, is_superuser => false}), + emqx_utils_json:encode(#{result => allow, is_superuser => false}), Req0 ), {ok, Req, State}. diff --git a/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl b/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl index 7a51d2bbb..94c07ca96 100644 --- a/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl @@ -467,7 +467,7 @@ jwks_handler(Req0, State) -> Req = cowboy_req:reply( 200, #{<<"content-type">> => <<"application/json">>}, - jiffy:encode(JWKS), + emqx_utils_json:encode(JWKS), Req0 ), {ok, Req, State}. diff --git a/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl b/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl index 65c783298..075ae5cb7 100644 --- a/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl @@ -107,7 +107,7 @@ t_update_with_invalid_config(_Config) -> ?assertMatch( {error, #{ kind := validation_error, - matched_type := "authn-postgresql:authentication", + matched_type := "authn:postgresql", path := "authentication.1.server", reason := required_field }}, diff --git a/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl b/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl index 4f95bef93..1354e06cc 100644 --- a/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl @@ -162,7 +162,7 @@ t_create_invalid_config(_Config) -> ?assertMatch( {error, #{ kind := validation_error, - matched_type := "authn-redis:standalone", + matched_type := "authn:redis_single", path := "authentication.1.server", reason := required_field }}, diff --git a/apps/emqx_authn/test/emqx_authn_schema_SUITE.erl b/apps/emqx_authn/test/emqx_authn_schema_SUITE.erl index 7e67584ac..7a766281b 100644 --- a/apps/emqx_authn/test/emqx_authn_schema_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_schema_SUITE.erl @@ -53,7 +53,7 @@ t_check_schema(_Config) -> ?assertThrow( #{ path := "authentication.1.password_hash_algorithm.name", - matched_type := "authn-builtin_db:authentication/authn-hash:simple", + matched_type := "authn:builtin_db/authn-hash:simple", reason := unable_to_convert_to_enum_symbol }, Check(ConfigNotOk) @@ -72,7 +72,7 @@ t_check_schema(_Config) -> #{ path := "authentication.1.password_hash_algorithm", reason := "algorithm_name_missing", - matched_type := "authn-builtin_db:authentication" + matched_type := "authn:builtin_db" }, Check(ConfigMissingAlgoName) ). diff --git a/apps/emqx_authn/test/emqx_authn_schema_tests.erl b/apps/emqx_authn/test/emqx_authn_schema_tests.erl index 25fcd28e4..622655b2d 100644 --- a/apps/emqx_authn/test/emqx_authn_schema_tests.erl +++ b/apps/emqx_authn/test/emqx_authn_schema_tests.erl @@ -32,19 +32,19 @@ union_member_selector_mongo_test_() -> end}, {"single", fun() -> ?assertMatch( - ?ERR(#{matched_type := "authn-mongodb:standalone"}), + ?ERR(#{matched_type := "authn:mongo_single"}), Check("{mongo_type: single}") ) end}, {"replica-set", fun() -> ?assertMatch( - ?ERR(#{matched_type := "authn-mongodb:replica-set"}), + ?ERR(#{matched_type := "authn:mongo_rs"}), Check("{mongo_type: rs}") ) end}, {"sharded", fun() -> ?assertMatch( - ?ERR(#{matched_type := "authn-mongodb:sharded-cluster"}), + ?ERR(#{matched_type := "authn:mongo_sharded"}), Check("{mongo_type: sharded}") ) end} @@ -61,19 +61,19 @@ union_member_selector_jwt_test_() -> end}, {"jwks", fun() -> ?assertMatch( - ?ERR(#{matched_type := "authn-jwt:jwks"}), + ?ERR(#{matched_type := "authn:jwt_jwks"}), Check("{use_jwks = true}") ) end}, {"publick-key", fun() -> ?assertMatch( - ?ERR(#{matched_type := "authn-jwt:public-key"}), + ?ERR(#{matched_type := "authn:jwt_public_key"}), Check("{use_jwks = false, public_key = 1}") ) end}, {"hmac-based", fun() -> ?assertMatch( - ?ERR(#{matched_type := "authn-jwt:hmac-based"}), + ?ERR(#{matched_type := "authn:jwt_hmac"}), Check("{use_jwks = false}") ) end} @@ -90,19 +90,19 @@ union_member_selector_redis_test_() -> end}, {"single", fun() -> ?assertMatch( - ?ERR(#{matched_type := "authn-redis:standalone"}), + ?ERR(#{matched_type := "authn:redis_single"}), Check("{redis_type = single}") ) end}, {"cluster", fun() -> ?assertMatch( - ?ERR(#{matched_type := "authn-redis:cluster"}), + ?ERR(#{matched_type := "authn:redis_cluster"}), Check("{redis_type = cluster}") ) end}, {"sentinel", fun() -> ?assertMatch( - ?ERR(#{matched_type := "authn-redis:sentinel"}), + ?ERR(#{matched_type := "authn:redis_sentinel"}), Check("{redis_type = sentinel}") ) end} @@ -119,13 +119,13 @@ union_member_selector_http_test_() -> end}, {"get", fun() -> ?assertMatch( - ?ERR(#{matched_type := "authn-http:get"}), + ?ERR(#{matched_type := "authn:http_get"}), Check("{method = get}") ) end}, {"post", fun() -> ?assertMatch( - ?ERR(#{matched_type := "authn-http:post"}), + ?ERR(#{matched_type := "authn:http_post"}), Check("{method = post}") ) end} diff --git a/apps/emqx_authz/rebar.config b/apps/emqx_authz/rebar.config index da2fa7807..9fd61b060 100644 --- a/apps/emqx_authz/rebar.config +++ b/apps/emqx_authz/rebar.config @@ -3,6 +3,7 @@ {erl_opts, [debug_info, nowarn_unused_import]}. {deps, [ {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}}, {emqx_connector, {path, "../emqx_connector"}} ]}. diff --git a/apps/emqx_authz/src/emqx_authz.app.src b/apps/emqx_authz/src/emqx_authz.app.src index 2f8b26894..dd658a6aa 100644 --- a/apps/emqx_authz/src/emqx_authz.app.src +++ b/apps/emqx_authz/src/emqx_authz.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_authz, [ {description, "An OTP application"}, - {vsn, "0.1.16"}, + {vsn, "0.1.18"}, {registered, []}, {mod, {emqx_authz_app, []}}, {applications, [ diff --git a/apps/emqx_authz/src/emqx_authz_api_sources.erl b/apps/emqx_authz/src/emqx_authz_api_sources.erl index 58fa471fc..2220e8f6e 100644 --- a/apps/emqx_authz/src/emqx_authz_api_sources.erl +++ b/apps/emqx_authz/src/emqx_authz_api_sources.erl @@ -344,8 +344,8 @@ lookup_from_local_node(Type) -> case emqx_resource:get_instance(ResourceId) of {error, not_found} -> {error, {NodeId, not_found_resource}}; - {ok, _, #{status := Status, metrics := ResourceMetrics}} -> - {ok, {NodeId, Status, Metrics, ResourceMetrics}} + {ok, _, #{status := Status}} -> + {ok, {NodeId, Status, Metrics, emqx_resource:get_metrics(ResourceId)}} end; _ -> Metrics = emqx_metrics_worker:get_metrics(authz_metrics, Type), @@ -403,7 +403,7 @@ aggregate_metrics([]) -> aggregate_metrics([HeadMetrics | AllMetrics]) -> ErrorLogger = fun(Reason) -> ?SLOG(info, #{msg => "bad_metrics_value", error => Reason}) end, Fun = fun(ElemMap, AccMap) -> - emqx_map_lib:best_effort_recursive_sum(AccMap, ElemMap, ErrorLogger) + emqx_utils_maps:best_effort_recursive_sum(AccMap, ElemMap, ErrorLogger) end, lists:foldl(Fun, HeadMetrics, AllMetrics). diff --git a/apps/emqx_authz/src/emqx_authz_file.erl b/apps/emqx_authz/src/emqx_authz_file.erl index 9aa2d506f..ede4a9582 100644 --- a/apps/emqx_authz/src/emqx_authz_file.erl +++ b/apps/emqx_authz/src/emqx_authz_file.erl @@ -47,7 +47,7 @@ create(#{path := Path} = Source) -> ?SLOG(alert, #{ msg => failed_to_read_acl_file, path => Path, - explain => emqx_misc:explain_posix(Reason) + explain => emqx_utils:explain_posix(Reason) }), throw(failed_to_read_acl_file); {error, Reason} -> diff --git a/apps/emqx_authz/src/emqx_authz_http.erl b/apps/emqx_authz/src/emqx_authz_http.erl index 852a667c8..5747e6eeb 100644 --- a/apps/emqx_authz/src/emqx_authz_http.erl +++ b/apps/emqx_authz/src/emqx_authz_http.erl @@ -161,9 +161,9 @@ parse_url(Url) -> BaseUrl = iolist_to_binary([Scheme, "//", HostPort]), case string:split(Remaining, "?", leading) of [Path, QueryString] -> - {BaseUrl, Path, QueryString}; + {BaseUrl, <<"/", Path/binary>>, QueryString}; [Path] -> - {BaseUrl, Path, <<>>} + {BaseUrl, <<"/", Path/binary>>, <<>>} end; [HostPort] -> {iolist_to_binary([Scheme, "//", HostPort]), <<>>, <<>>} @@ -185,7 +185,7 @@ generate_request( } ) -> Values = client_vars(Client, PubSub, Topic), - Path = emqx_authz_utils:render_str(BasePathTemplate, Values), + Path = emqx_authz_utils:render_urlencoded_str(BasePathTemplate, Values), Query = emqx_authz_utils:render_deep(BaseQueryTemplate, Values), Body = emqx_authz_utils:render_deep(BodyTemplate, Values), case Method of @@ -202,9 +202,9 @@ generate_request( end. append_query(Path, []) -> - encode_path(Path); + to_list(Path); append_query(Path, Query) -> - encode_path(Path) ++ "?" ++ to_list(query_string(Query)). + to_list(Path) ++ "?" ++ to_list(query_string(Query)). query_string(Body) -> query_string(Body, []). @@ -222,12 +222,8 @@ query_string([{K, V} | More], Acc) -> uri_encode(T) -> emqx_http_lib:uri_encode(to_list(T)). -encode_path(Path) -> - Parts = string:split(Path, "/", all), - lists:flatten(["/" ++ Part || Part <- lists:map(fun uri_encode/1, Parts)]). - serialize_body(<<"application/json">>, Body) -> - jsx:encode(Body); + emqx_utils_json:encode(Body); serialize_body(<<"application/x-www-form-urlencoded">>, Body) -> query_string(Body). diff --git a/apps/emqx_authz/src/emqx_authz_rule.erl b/apps/emqx_authz/src/emqx_authz_rule.erl index 306ca9433..bdd0904f7 100644 --- a/apps/emqx_authz/src/emqx_authz_rule.erl +++ b/apps/emqx_authz/src/emqx_authz_rule.erl @@ -185,7 +185,7 @@ match_who(#{peerhost := IpAddress}, {ipaddrs, CIDRs}) -> match_who(ClientInfo, {'and', Principals}) when is_list(Principals) -> lists:foldl( fun(Principal, Permission) -> - match_who(ClientInfo, Principal) andalso Permission + Permission andalso match_who(ClientInfo, Principal) end, true, Principals @@ -193,7 +193,7 @@ match_who(ClientInfo, {'and', Principals}) when is_list(Principals) -> match_who(ClientInfo, {'or', Principals}) when is_list(Principals) -> lists:foldl( fun(Principal, Permission) -> - match_who(ClientInfo, Principal) orelse Permission + Permission orelse match_who(ClientInfo, Principal) end, false, Principals diff --git a/apps/emqx_authz/src/emqx_authz_schema.erl b/apps/emqx_authz/src/emqx_authz_schema.erl index 6630ed526..7aaa68b62 100644 --- a/apps/emqx_authz/src/emqx_authz_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_schema.erl @@ -54,7 +54,7 @@ type_names() -> file, http_get, http_post, - mnesia, + builtin_db, mongo_single, mongo_rs, mongo_sharded, @@ -93,7 +93,7 @@ fields(http_post) -> {method, method(post)}, {headers, fun headers/1} ]; -fields(mnesia) -> +fields(builtin_db) -> authz_common_fields(built_in_database); fields(mongo_single) -> authz_common_fields(mongodb) ++ @@ -191,8 +191,8 @@ desc(http_get) -> ?DESC(http_get); desc(http_post) -> ?DESC(http_post); -desc(mnesia) -> - ?DESC(mnesia); +desc(builtin_db) -> + ?DESC(builtin_db); desc(mongo_single) -> ?DESC(mongo_single); desc(mongo_rs) -> @@ -337,7 +337,7 @@ check_ssl_opts(Conf) -> (#{<<"url">> := Url} = Source) -> case emqx_authz_http:parse_url(Url) of {<<"https", _/binary>>, _, _} -> - case emqx_map_lib:deep_find([<<"ssl">>, <<"enable">>], Source) of + case emqx_utils_maps:deep_find([<<"ssl">>, <<"enable">>], Source) of {ok, true} -> true; {ok, false} -> throw({ssl_not_enable, Url}); _ -> throw({ssl_enable_not_found, Url}) @@ -459,7 +459,7 @@ select_union_member(#{<<"type">> := <<"http">>} = Value) -> }) end; select_union_member(#{<<"type">> := <<"built_in_database">>}) -> - ?R_REF(mnesia); + ?R_REF(builtin_db); select_union_member(#{<<"type">> := Type}) -> select_union_member_loop(Type, type_names()); select_union_member(_) -> @@ -494,7 +494,10 @@ authz_fields() -> default => [], desc => ?DESC(sources), %% doc_lift is force a root level reference instead of nesting sub-structs - extra => #{doc_lift => true} + extra => #{doc_lift => true}, + %% it is recommended to configure authz sources from dashboard + %% hance the importance level for config is low + importance => ?IMPORTANCE_LOW } )} ]. diff --git a/apps/emqx_authz/src/emqx_authz_utils.erl b/apps/emqx_authz/src/emqx_authz_utils.erl index df77673a2..c01505680 100644 --- a/apps/emqx_authz/src/emqx_authz_utils.erl +++ b/apps/emqx_authz/src/emqx_authz_utils.erl @@ -16,7 +16,6 @@ -module(emqx_authz_utils). --include_lib("emqx/include/emqx_placeholder.hrl"). -include_lib("emqx_authz.hrl"). -export([ @@ -28,6 +27,7 @@ update_config/2, parse_deep/2, parse_str/2, + render_urlencoded_str/2, parse_sql/3, render_deep/2, render_str/2, @@ -128,6 +128,13 @@ render_str(Template, Values) -> #{return => full_binary, var_trans => fun handle_var/2} ). +render_urlencoded_str(Template, Values) -> + emqx_placeholder:proc_tmpl( + Template, + client_vars(Values), + #{return => full_binary, var_trans => fun urlencode_var/2} + ). + render_sql_params(ParamList, Values) -> emqx_placeholder:proc_tmpl( ParamList, @@ -144,7 +151,7 @@ parse_http_resp_body(<<"application/x-www-form-urlencoded", _/binary>>, Body) -> end; parse_http_resp_body(<<"application/json", _/binary>>, Body) -> try - result(emqx_json:decode(Body, [return_maps])) + result(emqx_utils_json:decode(Body, [return_maps])) catch _:_ -> error end. @@ -181,6 +188,11 @@ convert_client_var({dn, DN}) -> {cert_subject, DN}; convert_client_var({protocol, Proto}) -> {proto_name, Proto}; convert_client_var(Other) -> Other. +urlencode_var({var, _} = Var, Value) -> + emqx_http_lib:uri_encode(handle_var(Var, Value)); +urlencode_var(Var, Value) -> + handle_var(Var, Value). + handle_var({var, _Name}, undefined) -> <<>>; handle_var({var, <<"peerhost">>}, IpAddr) -> diff --git a/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl index 45e6d7287..ab673b225 100644 --- a/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl @@ -60,19 +60,19 @@ set_special_configs(emqx_authz) -> set_special_configs(_App) -> ok. -t_clean_cahce(_) -> +t_clean_cache(_) -> {ok, C} = emqtt:start_link([{clientid, <<"emqx0">>}, {username, <<"emqx0">>}]), {ok, _} = emqtt:connect(C), {ok, _, _} = emqtt:subscribe(C, <<"a/b/c">>, 0), ok = emqtt:publish(C, <<"a/b/c">>, <<"{\"x\":1,\"y\":1}">>, 0), {ok, 200, Result3} = request(get, uri(["clients", "emqx0", "authorization", "cache"])), - ?assertEqual(2, length(emqx_json:decode(Result3))), + ?assertEqual(2, length(emqx_utils_json:decode(Result3))), request(delete, uri(["authorization", "cache"])), {ok, 200, Result4} = request(get, uri(["clients", "emqx0", "authorization", "cache"])), - ?assertEqual(0, length(emqx_json:decode(Result4))), + ?assertEqual(0, length(emqx_utils_json:decode(Result4))), ok. diff --git a/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl index 2aa2d9545..3775b9a1c 100644 --- a/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl @@ -95,7 +95,7 @@ t_api(_) -> <<"page">> := 1, <<"hasnext">> := false } - } = jsx:decode(Request1), + } = emqx_utils_json:decode(Request1), ?assertEqual(3, length(Rules1)), {ok, 200, Request1_1} = @@ -119,7 +119,7 @@ t_api(_) -> <<"hasnext">> => false } }, - jsx:decode(Request1_1) + emqx_utils_json:decode(Request1_1) ), {ok, 200, Request2} = @@ -128,7 +128,7 @@ t_api(_) -> uri(["authorization", "sources", "built_in_database", "rules", "users", "user1"]), [] ), - #{<<"username">> := <<"user1">>, <<"rules">> := Rules1} = jsx:decode(Request2), + #{<<"username">> := <<"user1">>, <<"rules">> := Rules1} = emqx_utils_json:decode(Request2), {ok, 204, _} = request( @@ -142,7 +142,7 @@ t_api(_) -> uri(["authorization", "sources", "built_in_database", "rules", "users", "user1"]), [] ), - #{<<"username">> := <<"user1">>, <<"rules">> := Rules2} = jsx:decode(Request3), + #{<<"username">> := <<"user1">>, <<"rules">> := Rules2} = emqx_utils_json:decode(Request3), ?assertEqual(0, length(Rules2)), {ok, 204, _} = @@ -202,8 +202,8 @@ t_api(_) -> <<"data">> := [#{<<"clientid">> := <<"client1">>, <<"rules">> := Rules3}], <<"meta">> := #{<<"count">> := 1, <<"limit">> := 100, <<"page">> := 1} } = - jsx:decode(Request4), - #{<<"clientid">> := <<"client1">>, <<"rules">> := Rules3} = jsx:decode(Request5), + emqx_utils_json:decode(Request4), + #{<<"clientid">> := <<"client1">>, <<"rules">> := Rules3} = emqx_utils_json:decode(Request5), ?assertEqual(3, length(Rules3)), {ok, 204, _} = @@ -218,7 +218,7 @@ t_api(_) -> uri(["authorization", "sources", "built_in_database", "rules", "clients", "client1"]), [] ), - #{<<"clientid">> := <<"client1">>, <<"rules">> := Rules4} = jsx:decode(Request6), + #{<<"clientid">> := <<"client1">>, <<"rules">> := Rules4} = emqx_utils_json:decode(Request6), ?assertEqual(0, length(Rules4)), {ok, 204, _} = @@ -252,7 +252,7 @@ t_api(_) -> uri(["authorization", "sources", "built_in_database", "rules", "all"]), [] ), - #{<<"rules">> := Rules5} = jsx:decode(Request7), + #{<<"rules">> := Rules5} = emqx_utils_json:decode(Request7), ?assertEqual(3, length(Rules5)), {ok, 204, _} = @@ -267,7 +267,7 @@ t_api(_) -> uri(["authorization", "sources", "built_in_database", "rules", "all"]), [] ), - #{<<"rules">> := Rules6} = jsx:decode(Request8), + #{<<"rules">> := Rules6} = emqx_utils_json:decode(Request8), ?assertEqual(0, length(Rules6)), {ok, 204, _} = @@ -285,7 +285,7 @@ t_api(_) -> uri(["authorization", "sources", "built_in_database", "rules", "users?page=2&limit=5"]), [] ), - #{<<"data">> := Data1} = jsx:decode(Request9), + #{<<"data">> := Data1} = emqx_utils_json:decode(Request9), ?assertEqual(5, length(Data1)), {ok, 204, _} = @@ -303,7 +303,7 @@ t_api(_) -> uri(["authorization", "sources", "built_in_database", "rules", "clients?limit=5"]), [] ), - #{<<"data">> := Data2} = jsx:decode(Request10), + #{<<"data">> := Data2} = emqx_utils_json:decode(Request10), ?assertEqual(5, length(Data2)), {ok, 400, Msg1} = diff --git a/apps/emqx_authz/test/emqx_authz_api_settings_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_settings_SUITE.erl index 41eba109e..e3412e169 100644 --- a/apps/emqx_authz/test/emqx_authz_api_settings_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_settings_SUITE.erl @@ -76,7 +76,7 @@ t_api(_) -> {ok, 200, Result1} = request(put, uri(["authorization", "settings"]), Settings1), {ok, 200, Result1} = request(get, uri(["authorization", "settings"]), []), - ?assertEqual(Settings1, jsx:decode(Result1)), + ?assertEqual(Settings1, emqx_utils_json:decode(Result1)), Settings2 = #{ <<"no_match">> => <<"allow">>, @@ -90,7 +90,7 @@ t_api(_) -> {ok, 200, Result2} = request(put, uri(["authorization", "settings"]), Settings2), {ok, 200, Result2} = request(get, uri(["authorization", "settings"]), []), - ?assertEqual(Settings2, jsx:decode(Result2)), + ?assertEqual(Settings2, emqx_utils_json:decode(Result2)), ok. diff --git a/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl index 411399d64..7a7dbb7e9 100644 --- a/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl @@ -148,8 +148,8 @@ set_special_configs(_App) -> ok. init_per_testcase(t_api, Config) -> - meck:new(emqx_misc, [non_strict, passthrough, no_history, no_link]), - meck:expect(emqx_misc, gen_id, fun() -> "fake" end), + meck:new(emqx_utils, [non_strict, passthrough, no_history, no_link]), + meck:expect(emqx_utils, gen_id, fun() -> "fake" end), meck:new(emqx, [non_strict, passthrough, no_history, no_link]), meck:expect( @@ -165,7 +165,7 @@ init_per_testcase(_, Config) -> Config. end_per_testcase(t_api, _Config) -> - meck:unload(emqx_misc), + meck:unload(emqx_utils), meck:unload(emqx), ok; end_per_testcase(_, _Config) -> @@ -182,7 +182,7 @@ t_api(_) -> {ok, 404, ErrResult} = request(get, uri(["authorization", "sources", "http"]), []), ?assertMatch( #{<<"code">> := <<"NOT_FOUND">>, <<"message">> := <<"Not found: http">>}, - emqx_json:decode(ErrResult, [return_maps]) + emqx_utils_json:decode(ErrResult, [return_maps]) ), [ @@ -215,7 +215,8 @@ t_api(_) -> ), {ok, 200, Result3} = request(get, uri(["authorization", "sources", "http"]), []), ?assertMatch( - #{<<"type">> := <<"http">>, <<"enable">> := false}, emqx_json:decode(Result3, [return_maps]) + #{<<"type">> := <<"http">>, <<"enable">> := false}, + emqx_utils_json:decode(Result3, [return_maps]) ), Keyfile = emqx_common_test_helpers:app_path( @@ -253,7 +254,7 @@ t_api(_) -> <<"total">> := 0, <<"nomatch">> := 0 } - } = emqx_json:decode(Status4, [return_maps]), + } = emqx_utils_json:decode(Status4, [return_maps]), ?assertMatch( #{ <<"type">> := <<"mongodb">>, @@ -265,7 +266,7 @@ t_api(_) -> <<"verify">> := <<"verify_none">> } }, - emqx_json:decode(Result4, [return_maps]) + emqx_utils_json:decode(Result4, [return_maps]) ), {ok, Cacert} = file:read_file(Cacertfile), @@ -297,7 +298,7 @@ t_api(_) -> <<"verify">> := <<"verify_none">> } }, - emqx_json:decode(Result5, [return_maps]) + emqx_utils_json:decode(Result5, [return_maps]) ), {ok, 200, Status5_1} = request(get, uri(["authorization", "sources", "mongodb", "status"]), []), @@ -308,7 +309,7 @@ t_api(_) -> <<"total">> := 0, <<"nomatch">> := 0 } - } = emqx_json:decode(Status5_1, [return_maps]), + } = emqx_utils_json:decode(Status5_1, [return_maps]), #{ ssl := #{ @@ -355,7 +356,7 @@ t_api(_) -> <<"code">> := <<"BAD_REQUEST">>, <<"message">> := <<"Type mismatch", _/binary>> }, - emqx_json:decode(TypeMismatch, [return_maps]) + emqx_utils_json:decode(TypeMismatch, [return_maps]) ), lists:foreach( @@ -443,7 +444,7 @@ t_api(_) -> <<"total">> := 1, <<"nomatch">> := 0 } - } = emqx_json:decode(Status5, [return_maps]) + } = emqx_utils_json:decode(Status5, [return_maps]) end ), @@ -469,7 +470,7 @@ t_api(_) -> <<"total">> := 2, <<"nomatch">> := 0 } - } = emqx_json:decode(Status6, [return_maps]) + } = emqx_utils_json:decode(Status6, [return_maps]) end ), @@ -495,7 +496,7 @@ t_api(_) -> <<"total">> := 3, <<"nomatch">> := 0 } - } = emqx_json:decode(Status7, [return_maps]) + } = emqx_utils_json:decode(Status7, [return_maps]) end ), ok. @@ -621,7 +622,7 @@ t_aggregate_metrics(_) -> ). get_sources(Result) -> - maps:get(<<"sources">>, emqx_json:decode(Result, [return_maps])). + maps:get(<<"sources">>, emqx_utils_json:decode(Result, [return_maps])). data_dir() -> emqx:data_dir(). diff --git a/apps/emqx_authz/test/emqx_authz_http_SUITE.erl b/apps/emqx_authz/test/emqx_authz_http_SUITE.erl index 0757113f3..702bf2756 100644 --- a/apps/emqx_authz/test/emqx_authz_http_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_http_SUITE.erl @@ -199,7 +199,7 @@ t_query_params(_Config) -> peerhost := <<"127.0.0.1">>, proto_name := <<"MQTT">>, mountpoint := <<"MOUNTPOINT">>, - topic := <<"t">>, + topic := <<"t/1">>, action := <<"publish">> } = cowboy_req:match_qs( [ @@ -241,7 +241,7 @@ t_query_params(_Config) -> ?assertEqual( allow, - emqx_access_control:authorize(ClientInfo, publish, <<"t">>) + emqx_access_control:authorize(ClientInfo, publish, <<"t/1">>) ). t_path(_Config) -> @@ -249,13 +249,13 @@ t_path(_Config) -> fun(Req0, State) -> ?assertEqual( << - "/authz/users/" + "/authz/use%20rs/" "user%20name/" "client%20id/" "127.0.0.1/" "MQTT/" "MOUNTPOINT/" - "t/1/" + "t%2F1/" "publish" >>, cowboy_req:path(Req0) @@ -264,7 +264,7 @@ t_path(_Config) -> end, #{ <<"url">> => << - "http://127.0.0.1:33333/authz/users/" + "http://127.0.0.1:33333/authz/use%20rs/" "${username}/" "${clientid}/" "${peerhost}/" @@ -311,7 +311,7 @@ t_json_body(_Config) -> <<"topic">> := <<"t">>, <<"action">> := <<"publish">> }, - jiffy:decode(RawBody, [return_maps]) + emqx_utils_json:decode(RawBody, [return_maps]) ), {ok, ?AUTHZ_HTTP_RESP(allow, Req1), State} end, @@ -366,7 +366,7 @@ t_placeholder_and_body(_Config) -> <<"CN">> := ?PH_CERT_CN_NAME, <<"CS">> := ?PH_CERT_SUBJECT }, - jiffy:decode(PostVars, [return_maps]) + emqx_utils_json:decode(PostVars, [return_maps]) ), {ok, ?AUTHZ_HTTP_RESP(allow, Req1), State} end, @@ -418,7 +418,7 @@ t_no_value_for_placeholder(_Config) -> #{ <<"mountpoint">> := <<"[]">> }, - jiffy:decode(RawBody, [return_maps]) + emqx_utils_json:decode(RawBody, [return_maps]) ), {ok, ?AUTHZ_HTTP_RESP(allow, Req1), State} end, diff --git a/apps/emqx_auto_subscribe/README.md b/apps/emqx_auto_subscribe/README.md index 96d368715..981e4cb1f 100644 --- a/apps/emqx_auto_subscribe/README.md +++ b/apps/emqx_auto_subscribe/README.md @@ -1,9 +1,54 @@ -emqx_auto_subscribe -===== +# Auto Subscribe -An OTP application +This application can help clients automatically subscribe to topics compiled from user definitions when they connect, and the clients no longer need to send the MQTT `SUBSCRIBE ` request. -Build ------ +# How To Use - $ rebar3 compile +Add the following configuration items to the `emqx.conf` file + +```yaml +auto_subscribe { + topics = [ + { + topic = "c/${clientid}" + }, + { + topic = "client/${clientid}/username/${username}/host/${host}/port/${port}" + qos = 1 + rh = 0 + rap = 0 + nl = 0 + } + ] +} +``` + +This example defines two templates, all of which will be compiled into the final topic by replacing placeholders like `${clientid}` `${port}` with the actual values when the client connects. + +# Configuration + +## Configuration Definition + +| Field | Definition | Value Range | Default | +| -------------- | ----------------------------- | ----------------------------------------------------------- | ------- | +| auto_subscribe | Auto subscribe configuration | topics | topics | +| topics | Subscription Options | Subscription configurations list. See `Subscription Option` | [] | + +## Subscription Option + +| Field | Definition | Value Range | Default | +|-------|---------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------|------------------| +| topic | Required. Topic template. | String, placeholders supported | No default value | +| qos | Optional. Subscription QoS | 0 or 1 or 2. Refer to the MQTT QoS definition | 0 | +| rh | Optional. MQTT version 5.0. Whether to send retain message when a subscription is created. | 0: Not send the retain message
1: Send the retain message | 0 | +| rap | Optional. MQTT version 5.0. When forwarding messages, Whether to send with retain flag | 0: Set retain 0
1: Keep retain flag | 0 | +| nl | Optional. MQTT version 5.0. Whether the message can be forwarded to the client when published by itself | 0: Forwarded to self
1: Not forwarded to self | 0 | + +## Subscription Placeholders + +| Placeholder | Definition | +| ----------- | -------------------------------------- | +| ${clientid} | Client ID | +| ${username} | Client Username | +| ${ip} | Client TCP connection local IP address | +| ${port} | Client TCP connection local Port | diff --git a/apps/emqx_auto_subscribe/rebar.config b/apps/emqx_auto_subscribe/rebar.config index 33e077f50..a19783033 100644 --- a/apps/emqx_auto_subscribe/rebar.config +++ b/apps/emqx_auto_subscribe/rebar.config @@ -1,7 +1,10 @@ %% -*- mode: erlang -*- {erl_opts, [debug_info]}. -{deps, [{emqx, {path, "../emqx"}}]}. +{deps, [ + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}} +]}. {shell, [ {apps, [emqx_auto_subscribe]} diff --git a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.app.src b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.app.src index a273face1..d6f6f4058 100644 --- a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.app.src +++ b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_auto_subscribe, [ {description, "Auto subscribe Application"}, - {vsn, "0.1.3"}, + {vsn, "0.1.4"}, {registered, []}, {mod, {emqx_auto_subscribe_app, []}}, {applications, [ diff --git a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_schema.erl b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_schema.erl index a01e17c1f..98f32c8a4 100644 --- a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_schema.erl +++ b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_schema.erl @@ -31,14 +31,17 @@ namespace() -> "auto_subscribe". roots() -> - ["auto_subscribe"]. + [{"auto_subscribe", ?HOCON(?R_REF("auto_subscribe"), #{importance => ?IMPORTANCE_HIDDEN})}]. fields("auto_subscribe") -> [ {topics, ?HOCON( ?ARRAY(?R_REF("topic")), - #{desc => ?DESC(auto_subscribe), default => []} + #{ + desc => ?DESC(auto_subscribe), + default => [] + } )} ]; fields("topic") -> diff --git a/apps/emqx_auto_subscribe/test/emqx_auto_subscribe_SUITE.erl b/apps/emqx_auto_subscribe/test/emqx_auto_subscribe_SUITE.erl index 5c5a3ee79..9d8d47bf2 100644 --- a/apps/emqx_auto_subscribe/test/emqx_auto_subscribe_SUITE.erl +++ b/apps/emqx_auto_subscribe/test/emqx_auto_subscribe_SUITE.erl @@ -141,7 +141,7 @@ t_update(_) -> Auth = emqx_mgmt_api_test_util:auth_header_(), Body = [#{topic => ?TOPIC_S}], {ok, Response} = emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, Body), - ResponseMap = emqx_json:decode(Response, [return_maps]), + ResponseMap = emqx_utils_json:decode(Response, [return_maps]), ?assertEqual(1, erlang:length(ResponseMap)), BadBody1 = #{topic => ?TOPIC_S}, @@ -177,7 +177,7 @@ t_update(_) -> emqtt:disconnect(Client), {ok, GETResponse} = emqx_mgmt_api_test_util:request_api(get, Path), - GETResponseMap = emqx_json:decode(GETResponse, [return_maps]), + GETResponseMap = emqx_utils_json:decode(GETResponse, [return_maps]), ?assertEqual(1, erlang:length(GETResponseMap)), ok. diff --git a/apps/emqx_bridge/rebar.config b/apps/emqx_bridge/rebar.config index 79f2caf50..864c45e9a 100644 --- a/apps/emqx_bridge/rebar.config +++ b/apps/emqx_bridge/rebar.config @@ -1,7 +1,9 @@ {erl_opts, [debug_info]}. -{deps, [ {emqx, {path, "../emqx"}} - , {emqx_resource, {path, "../../apps/emqx_resource"}} - ]}. +{deps, [ + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}}, + {emqx_resource, {path, "../../apps/emqx_resource"}} + ]}. {shell, [ % {config, "config/sys.config"}, diff --git a/apps/emqx_bridge/src/emqx_bridge.app.src b/apps/emqx_bridge/src/emqx_bridge.app.src index f5bcb23e2..d6c140fef 100644 --- a/apps/emqx_bridge/src/emqx_bridge.app.src +++ b/apps/emqx_bridge/src/emqx_bridge.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_bridge, [ {description, "EMQX bridges"}, - {vsn, "0.1.14"}, + {vsn, "0.1.17"}, {registered, [emqx_bridge_sup]}, {mod, {emqx_bridge_app, []}}, {applications, [ diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index bf91d20f7..08b8222f2 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -34,7 +34,7 @@ unload/0, lookup/1, lookup/2, - lookup/3, + get_metrics/2, create/3, disable_enable/3, remove/2, @@ -69,7 +69,8 @@ T == tdengine; T == dynamo; T == rocketmq; - T == cassandra + T == cassandra; + T == sqlserver ). load() -> @@ -206,7 +207,7 @@ send_message(BridgeId, Message) -> end. query_opts(Config) -> - case emqx_map_lib:deep_get([resource_opts, request_timeout], Config, false) of + case emqx_utils_maps:deep_get([resource_opts, request_timeout], Config, false) of Timeout when is_integer(Timeout) -> %% request_timeout is configured #{timeout => Timeout}; @@ -271,6 +272,9 @@ lookup(Type, Name, RawConf) -> }} end. +get_metrics(Type, Name) -> + emqx_resource:get_metrics(emqx_bridge_resource:resource_id(Type, Name)). + maybe_upgrade(mqtt, Config) -> emqx_bridge_compatible_config:maybe_upgrade(Config); maybe_upgrade(webhook, Config) -> @@ -292,7 +296,7 @@ create(BridgeType, BridgeName, RawConf) -> brige_action => create, bridge_type => BridgeType, bridge_name => BridgeName, - bridge_raw_config => emqx_misc:redact(RawConf) + bridge_raw_config => emqx_utils:redact(RawConf) }), emqx_conf:update( emqx_bridge:config_key_path() ++ [BridgeType, BridgeName], @@ -363,7 +367,7 @@ perform_bridge_changes([{Action, MapConfs} | Tasks], Result0) -> perform_bridge_changes(Tasks, Result). diff_confs(NewConfs, OldConfs) -> - emqx_map_lib:diff_maps( + emqx_utils_maps:diff_maps( flatten_confs(NewConfs), flatten_confs(OldConfs) ). diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index 44a478bca..09d1159bd 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -20,7 +20,7 @@ -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). -include_lib("emqx/include/logger.hrl"). --include_lib("emqx/include/emqx_api_lib.hrl"). +-include_lib("emqx_utils/include/emqx_utils_api.hrl"). -include_lib("emqx_bridge/include/emqx_bridge.hrl"). -import(hoconsc, [mk/2, array/1, enum/1]). @@ -46,6 +46,7 @@ ]). -export([lookup_from_local_node/2]). +-export([get_metrics_from_local_node/2]). -define(BRIDGE_NOT_ENABLED, ?BAD_REQUEST(<<"Forbidden operation, bridge not enabled">>) @@ -63,7 +64,7 @@ {BridgeType, BridgeName} -> EXPR catch - throw:{invalid_bridge_id, Reason} -> + throw:#{reason := Reason} -> ?NOT_FOUND(<<"Invalid bridge ID, ", Reason/binary>>) end ). @@ -219,7 +220,7 @@ info_example_basic(webhook) -> auto_restart_interval => 15000, query_mode => async, inflight_window => 100, - max_queue_bytes => 100 * 1024 * 1024 + max_buffer_bytes => 100 * 1024 * 1024 } }; info_example_basic(mqtt) -> @@ -244,7 +245,7 @@ mqtt_main_example() -> health_check_interval => <<"15s">>, auto_restart_interval => <<"60s">>, query_mode => sync, - max_queue_bytes => 100 * 1024 * 1024 + max_buffer_bytes => 100 * 1024 * 1024 }, ssl => #{ enable => false @@ -467,7 +468,7 @@ schema("/bridges_probe") -> end; '/bridges'(get, _Params) -> Nodes = mria:running_nodes(), - NodeReplies = emqx_bridge_proto_v3:list_bridges_on_nodes(Nodes), + NodeReplies = emqx_bridge_proto_v4:list_bridges_on_nodes(Nodes), case is_ok(NodeReplies) of {ok, NodeBridges} -> AllBridges = [ @@ -524,7 +525,7 @@ schema("/bridges_probe") -> ). '/bridges/:id/metrics'(get, #{bindings := #{id := Id}}) -> - ?TRY_PARSE_ID(Id, lookup_from_all_nodes_metrics(BridgeType, BridgeName, 200)). + ?TRY_PARSE_ID(Id, get_metrics_from_all_nodes(BridgeType, BridgeName)). '/bridges/:id/metrics/reset'(put, #{bindings := #{id := Id}}) -> ?TRY_PARSE_ID( @@ -545,6 +546,8 @@ schema("/bridges_probe") -> case emqx_bridge_resource:create_dry_run(ConnType, maps:remove(<<"type">>, Params1)) of ok -> ?NO_CONTENT; + {error, #{kind := validation_error} = Reason} -> + ?BAD_REQUEST('TEST_FAILED', map_to_json(Reason)); {error, Reason} when not is_tuple(Reason); element(1, Reason) =/= 'exit' -> ?BAD_REQUEST('TEST_FAILED', Reason) end; @@ -564,19 +567,21 @@ maybe_deobfuscate_bridge_probe(#{<<"type">> := BridgeType, <<"name">> := BridgeN maybe_deobfuscate_bridge_probe(Params) -> Params. -lookup_from_all_nodes(BridgeType, BridgeName, SuccCode) -> - FormatFun = fun format_bridge_info/1, - do_lookup_from_all_nodes(BridgeType, BridgeName, SuccCode, FormatFun). - -lookup_from_all_nodes_metrics(BridgeType, BridgeName, SuccCode) -> - FormatFun = fun format_bridge_metrics/1, - do_lookup_from_all_nodes(BridgeType, BridgeName, SuccCode, FormatFun). - -do_lookup_from_all_nodes(BridgeType, BridgeName, SuccCode, FormatFun) -> +get_metrics_from_all_nodes(BridgeType, BridgeName) -> Nodes = mria:running_nodes(), - case is_ok(emqx_bridge_proto_v3:lookup_from_all_nodes(Nodes, BridgeType, BridgeName)) of + Result = do_bpapi_call(all, get_metrics_from_all_nodes, [Nodes, BridgeType, BridgeName]), + case Result of + Metrics when is_list(Metrics) -> + {200, format_bridge_metrics(lists:zip(Nodes, Metrics))}; + {error, Reason} -> + ?INTERNAL_ERROR(Reason) + end. + +lookup_from_all_nodes(BridgeType, BridgeName, SuccCode) -> + Nodes = mria:running_nodes(), + case is_ok(emqx_bridge_proto_v4:lookup_from_all_nodes(Nodes, BridgeType, BridgeName)) of {ok, [{ok, _} | _] = Results} -> - {SuccCode, FormatFun([R || {ok, R} <- Results])}; + {SuccCode, format_bridge_info([R || {ok, R} <- Results])}; {ok, [{error, not_found} | _]} -> ?BRIDGE_NOT_FOUND(BridgeType, BridgeName); {error, Reason} -> @@ -603,6 +608,9 @@ create_or_update_bridge(BridgeType, BridgeName, Conf, HttpStatusCode) -> ?BAD_REQUEST(map_to_json(Reason)) end. +get_metrics_from_local_node(BridgeType, BridgeName) -> + format_metrics(emqx_bridge:get_metrics(BridgeType, BridgeName)). + '/bridges/:id/enable/:enable'(put, #{bindings := #{id := Id, enable := Enable}}) -> ?TRY_PARSE_ID( Id, @@ -662,7 +670,7 @@ create_or_update_bridge(BridgeType, BridgeName, Conf, HttpStatusCode) -> false -> ?BRIDGE_NOT_ENABLED; true -> - case emqx_misc:safe_to_existing_atom(Node, utf8) of + case emqx_utils:safe_to_existing_atom(Node, utf8) of {ok, TargetNode} -> call_operation(TargetNode, OperFunc, [ TargetNode, BridgeType, BridgeName @@ -739,7 +747,7 @@ pick_bridges_by_id(Type, Name, BridgesAllNodes) -> ). format_bridge_info([FirstBridge | _] = Bridges) -> - Res = maps:without([node, metrics], FirstBridge), + Res = maps:remove(node, FirstBridge), NodeStatus = node_status(Bridges), redact(Res#{ status => aggregate_status(NodeStatus), @@ -766,7 +774,7 @@ aggregate_status(AllStatus) -> end. collect_metrics(Bridges) -> - [maps:with([node, metrics], B) || B <- Bridges]. + [#{node => Node, metrics => Metrics} || {Node, Metrics} <- Bridges]. aggregate_metrics(AllMetrics) -> InitMetrics = ?EMPTY_METRICS, @@ -800,9 +808,7 @@ aggregate_metrics( M15 + N15, M16 + N16, M17 + N17 - ); -aggregate_metrics(#{}, Metrics) -> - Metrics. + ). format_resource( #{ @@ -826,63 +832,57 @@ format_resource( ). format_resource_data(ResData) -> - maps:fold(fun format_resource_data/3, #{}, maps:with([status, metrics, error], ResData)). + maps:fold(fun format_resource_data/3, #{}, maps:with([status, error], ResData)). format_resource_data(error, undefined, Result) -> Result; format_resource_data(error, Error, Result) -> - Result#{status_reason => emqx_misc:readable_error_msg(Error)}; -format_resource_data( - metrics, - #{ - counters := #{ - 'dropped' := Dropped, - 'dropped.other' := DroppedOther, - 'dropped.expired' := DroppedExpired, - 'dropped.queue_full' := DroppedQueueFull, - 'dropped.resource_not_found' := DroppedResourceNotFound, - 'dropped.resource_stopped' := DroppedResourceStopped, - 'matched' := Matched, - 'retried' := Retried, - 'late_reply' := LateReply, - 'failed' := SentFailed, - 'success' := SentSucc, - 'received' := Rcvd - }, - gauges := Gauges, - rate := #{ - matched := #{current := Rate, last5m := Rate5m, max := RateMax} - } - }, - Result -) -> - Queued = maps:get('queuing', Gauges, 0), - SentInflight = maps:get('inflight', Gauges, 0), - Result#{ - metrics => - ?METRICS( - Dropped, - DroppedOther, - DroppedExpired, - DroppedQueueFull, - DroppedResourceNotFound, - DroppedResourceStopped, - Matched, - Queued, - Retried, - LateReply, - SentFailed, - SentInflight, - SentSucc, - Rate, - Rate5m, - RateMax, - Rcvd - ) - }; + Result#{status_reason => emqx_utils:readable_error_msg(Error)}; format_resource_data(K, V, Result) -> Result#{K => V}. +format_metrics(#{ + counters := #{ + 'dropped' := Dropped, + 'dropped.other' := DroppedOther, + 'dropped.expired' := DroppedExpired, + 'dropped.queue_full' := DroppedQueueFull, + 'dropped.resource_not_found' := DroppedResourceNotFound, + 'dropped.resource_stopped' := DroppedResourceStopped, + 'matched' := Matched, + 'retried' := Retried, + 'late_reply' := LateReply, + 'failed' := SentFailed, + 'success' := SentSucc, + 'received' := Rcvd + }, + gauges := Gauges, + rate := #{ + matched := #{current := Rate, last5m := Rate5m, max := RateMax} + } +}) -> + Queued = maps:get('queuing', Gauges, 0), + SentInflight = maps:get('inflight', Gauges, 0), + ?METRICS( + Dropped, + DroppedOther, + DroppedExpired, + DroppedQueueFull, + DroppedResourceNotFound, + DroppedResourceStopped, + Matched, + Queued, + Retried, + LateReply, + SentFailed, + SentInflight, + SentSucc, + Rate, + Rate5m, + RateMax, + Rcvd + ). + fill_defaults(Type, RawConf) -> PackedConf = pack_bridge_conf(Type, RawConf), FullConf = emqx_config:fill_defaults(emqx_bridge_schema, PackedConf, #{}), @@ -990,7 +990,7 @@ do_bpapi_call(Node, Call, Args) -> do_bpapi_call_vsn(SupportedVersion, Call, Args) -> case lists:member(SupportedVersion, supported_versions(Call)) of true -> - apply(emqx_bridge_proto_v3, Call, Args); + apply(emqx_bridge_proto_v4, Call, Args); false -> {error, not_implemented} end. @@ -1000,12 +1000,13 @@ maybe_unwrap({error, not_implemented}) -> maybe_unwrap(RpcMulticallResult) -> emqx_rpc:unwrap_erpc(RpcMulticallResult). -supported_versions(start_bridge_to_node) -> [2, 3]; -supported_versions(start_bridges_to_all_nodes) -> [2, 3]; -supported_versions(_Call) -> [1, 2, 3]. +supported_versions(start_bridge_to_node) -> [2, 3, 4]; +supported_versions(start_bridges_to_all_nodes) -> [2, 3, 4]; +supported_versions(get_metrics_from_all_nodes) -> [4]; +supported_versions(_Call) -> [1, 2, 3, 4]. redact(Term) -> - emqx_misc:redact(Term). + emqx_utils:redact(Term). deobfuscate(NewConf, OldConf) -> maps:fold( @@ -1016,7 +1017,7 @@ deobfuscate(NewConf, OldConf) -> {ok, OldV} when is_map(V), is_map(OldV) -> Acc#{K => deobfuscate(V, OldV)}; {ok, OldV} -> - case emqx_misc:is_redacted(K, V) of + case emqx_utils:is_redacted(K, V) of true -> Acc#{K => OldV}; _ -> @@ -1029,6 +1030,6 @@ deobfuscate(NewConf, OldConf) -> ). map_to_json(M) -> - emqx_json:encode( - emqx_map_lib:jsonable_map(M, fun(K, V) -> {K, emqx_map_lib:binary_string(V)} end) + emqx_utils_json:encode( + emqx_utils_maps:jsonable_map(M, fun(K, V) -> {K, emqx_utils_maps:binary_string(V)} end) ). diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl index b43cbe0ec..1ad024c40 100644 --- a/apps/emqx_bridge/src/emqx_bridge_resource.erl +++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl @@ -87,7 +87,7 @@ parse_bridge_id(BridgeId) -> [Type, Name] -> {to_type_atom(Type), validate_name(Name)}; _ -> - invalid_bridge_id( + invalid_data( <<"should be of pattern {type}:{name}, but got ", BridgeId/binary>> ) end. @@ -108,14 +108,14 @@ validate_name(Name0) -> true -> Name0; false -> - invalid_bridge_id(<<"bad name: ", Name0/binary>>) + invalid_data(<<"bad name: ", Name0/binary>>) end; false -> - invalid_bridge_id(<<"only 0-9a-zA-Z_-. is allowed in name: ", Name0/binary>>) + invalid_data(<<"only 0-9a-zA-Z_-. is allowed in name: ", Name0/binary>>) end. --spec invalid_bridge_id(binary()) -> no_return(). -invalid_bridge_id(Reason) -> throw({?FUNCTION_NAME, Reason}). +-spec invalid_data(binary()) -> no_return(). +invalid_data(Reason) -> throw(#{kind => validation_error, reason => Reason}). is_id_char(C) when C >= $0 andalso C =< $9 -> true; is_id_char(C) when C >= $a andalso C =< $z -> true; @@ -130,7 +130,7 @@ to_type_atom(Type) -> erlang:binary_to_existing_atom(Type, utf8) catch _:_ -> - invalid_bridge_id(<<"unknown type: ", Type/binary>>) + invalid_data(<<"unknown bridge type: ", Type/binary>>) end. reset_metrics(ResourceId) -> @@ -157,7 +157,7 @@ create(Type, Name, Conf, Opts0) -> msg => "create bridge", type => Type, name => Name, - config => emqx_misc:redact(Conf) + config => emqx_utils:redact(Conf) }), Opts = override_start_after_created(Conf, Opts0), {ok, _Data} = emqx_resource:create_local( @@ -186,13 +186,13 @@ update(Type, Name, {OldConf, Conf}, Opts0) -> %% without restarting the bridge. %% Opts = override_start_after_created(Conf, Opts0), - case emqx_map_lib:if_only_to_toggle_enable(OldConf, Conf) of + case emqx_utils_maps:if_only_to_toggle_enable(OldConf, Conf) of false -> ?SLOG(info, #{ msg => "update bridge", type => Type, name => Name, - config => emqx_misc:redact(Conf) + config => emqx_utils:redact(Conf) }), case recreate(Type, Name, Conf, Opts) of {ok, _} -> @@ -202,7 +202,7 @@ update(Type, Name, {OldConf, Conf}, Opts0) -> msg => "updating_a_non_existing_bridge", type => Type, name => Name, - config => emqx_misc:redact(Conf) + config => emqx_utils:redact(Conf) }), create(Type, Name, Conf, Opts); {error, Reason} -> @@ -236,19 +236,26 @@ recreate(Type, Name, Conf, Opts) -> ). create_dry_run(Type, Conf0) -> - TmpPath0 = iolist_to_binary([?TEST_ID_PREFIX, emqx_misc:gen_id(8)]), - TmpPath = emqx_misc:safe_filename(TmpPath0), - Conf = emqx_map_lib:safe_atom_key_map(Conf0), + TmpPath0 = iolist_to_binary([?TEST_ID_PREFIX, emqx_utils:gen_id(8)]), + TmpPath = emqx_utils:safe_filename(TmpPath0), + Conf = emqx_utils_maps:safe_atom_key_map(Conf0), case emqx_connector_ssl:convert_certs(TmpPath, Conf) of {error, Reason} -> {error, Reason}; {ok, ConfNew} -> - ParseConf = parse_confs(bin(Type), TmpPath, ConfNew), - Res = emqx_resource:create_dry_run_local( - bridge_to_resource_type(Type), ParseConf - ), - _ = maybe_clear_certs(TmpPath, ConfNew), - Res + try + ParseConf = parse_confs(bin(Type), TmpPath, ConfNew), + Res = emqx_resource:create_dry_run_local( + bridge_to_resource_type(Type), ParseConf + ), + Res + catch + %% validation errors + throw:Reason -> + {error, Reason} + after + _ = maybe_clear_certs(TmpPath, ConfNew) + end end. remove(BridgeId) -> @@ -300,10 +307,18 @@ parse_confs( max_retries := Retry } = Conf ) -> - {BaseUrl, Path} = parse_url(Url), - {ok, BaseUrl2} = emqx_http_lib:uri_parse(BaseUrl), + Url1 = bin(Url), + {BaseUrl, Path} = parse_url(Url1), + BaseUrl1 = + case emqx_http_lib:uri_parse(BaseUrl) of + {ok, BUrl} -> + BUrl; + {error, Reason} -> + Reason1 = emqx_utils:readable_error_msg(Reason), + invalid_data(<<"Invalid URL: ", Url1/binary, ", details: ", Reason1/binary>>) + end, Conf#{ - base_url => BaseUrl2, + base_url => BaseUrl1, request => #{ path => Path, @@ -338,7 +353,7 @@ parse_url(Url) -> {iolist_to_binary([Scheme, "//", HostPort]), <<>>} end; [Url] -> - error({invalid_url, Url}) + invalid_data(<<"Missing scheme in URL: ", Url/binary>>) end. str(Bin) when is_binary(Bin) -> binary_to_list(Bin); diff --git a/apps/emqx_bridge/src/proto/emqx_bridge_proto_v3.erl b/apps/emqx_bridge/src/proto/emqx_bridge_proto_v3.erl index a35db5d96..0b496364a 100644 --- a/apps/emqx_bridge/src/proto/emqx_bridge_proto_v3.erl +++ b/apps/emqx_bridge/src/proto/emqx_bridge_proto_v3.erl @@ -20,6 +20,7 @@ -export([ introduced_in/0, + deprecated_since/0, list_bridges/1, list_bridges_on_nodes/1, @@ -39,6 +40,9 @@ introduced_in() -> "5.0.21". +deprecated_since() -> + "5.0.22". + -spec list_bridges(node()) -> list() | emqx_rpc:badrpc(). list_bridges(Node) -> rpc:call(Node, emqx_bridge, list, [], ?TIMEOUT). diff --git a/apps/emqx_bridge/src/proto/emqx_bridge_proto_v4.erl b/apps/emqx_bridge/src/proto/emqx_bridge_proto_v4.erl new file mode 100644 index 000000000..937065e41 --- /dev/null +++ b/apps/emqx_bridge/src/proto/emqx_bridge_proto_v4.erl @@ -0,0 +1,135 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_proto_v4). + +-behaviour(emqx_bpapi). + +-export([ + introduced_in/0, + + list_bridges_on_nodes/1, + restart_bridge_to_node/3, + start_bridge_to_node/3, + stop_bridge_to_node/3, + lookup_from_all_nodes/3, + get_metrics_from_all_nodes/3, + restart_bridges_to_all_nodes/3, + start_bridges_to_all_nodes/3, + stop_bridges_to_all_nodes/3 +]). + +-include_lib("emqx/include/bpapi.hrl"). + +-define(TIMEOUT, 15000). + +introduced_in() -> + "5.0.22". + +-spec list_bridges_on_nodes([node()]) -> + emqx_rpc:erpc_multicall([emqx_resource:resource_data()]). +list_bridges_on_nodes(Nodes) -> + erpc:multicall(Nodes, emqx_bridge, list, [], ?TIMEOUT). + +-type key() :: atom() | binary() | [byte()]. + +-spec restart_bridge_to_node(node(), key(), key()) -> + term(). +restart_bridge_to_node(Node, BridgeType, BridgeName) -> + rpc:call( + Node, + emqx_bridge_resource, + restart, + [BridgeType, BridgeName], + ?TIMEOUT + ). + +-spec start_bridge_to_node(node(), key(), key()) -> + term(). +start_bridge_to_node(Node, BridgeType, BridgeName) -> + rpc:call( + Node, + emqx_bridge_resource, + start, + [BridgeType, BridgeName], + ?TIMEOUT + ). + +-spec stop_bridge_to_node(node(), key(), key()) -> + term(). +stop_bridge_to_node(Node, BridgeType, BridgeName) -> + rpc:call( + Node, + emqx_bridge_resource, + stop, + [BridgeType, BridgeName], + ?TIMEOUT + ). + +-spec restart_bridges_to_all_nodes([node()], key(), key()) -> + emqx_rpc:erpc_multicall(). +restart_bridges_to_all_nodes(Nodes, BridgeType, BridgeName) -> + erpc:multicall( + Nodes, + emqx_bridge_resource, + restart, + [BridgeType, BridgeName], + ?TIMEOUT + ). + +-spec start_bridges_to_all_nodes([node()], key(), key()) -> + emqx_rpc:erpc_multicall(). +start_bridges_to_all_nodes(Nodes, BridgeType, BridgeName) -> + erpc:multicall( + Nodes, + emqx_bridge_resource, + start, + [BridgeType, BridgeName], + ?TIMEOUT + ). + +-spec stop_bridges_to_all_nodes([node()], key(), key()) -> + emqx_rpc:erpc_multicall(). +stop_bridges_to_all_nodes(Nodes, BridgeType, BridgeName) -> + erpc:multicall( + Nodes, + emqx_bridge_resource, + stop, + [BridgeType, BridgeName], + ?TIMEOUT + ). + +-spec lookup_from_all_nodes([node()], key(), key()) -> + emqx_rpc:erpc_multicall(). +lookup_from_all_nodes(Nodes, BridgeType, BridgeName) -> + erpc:multicall( + Nodes, + emqx_bridge_api, + lookup_from_local_node, + [BridgeType, BridgeName], + ?TIMEOUT + ). + +-spec get_metrics_from_all_nodes([node()], key(), key()) -> + emqx_rpc:erpc_multicall(emqx_metrics_worker:metrics()). +get_metrics_from_all_nodes(Nodes, BridgeType, BridgeName) -> + erpc:multicall( + Nodes, + emqx_bridge_api, + get_metrics_from_local_node, + [BridgeType, BridgeName], + ?TIMEOUT + ). diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_compatible_config.erl b/apps/emqx_bridge/src/schema/emqx_bridge_compatible_config.erl index fe173fa89..595b75ecf 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_compatible_config.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_compatible_config.erl @@ -89,7 +89,7 @@ default_resource_opts() -> <<"inflight_window">> => 100, <<"auto_restart_interval">> => <<"60s">>, <<"health_check_interval">> => <<"15s">>, - <<"max_queue_bytes">> => <<"1GB">>, + <<"max_buffer_bytes">> => <<"1GB">>, <<"query_mode">> => <<"sync">>, %% there is only one underlying MQTT connection %% doesn't make a lot of sense to have a large pool diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl index 6c278a5ec..4b9b7e3fe 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl @@ -137,7 +137,7 @@ namespace() -> "bridge". tags() -> [<<"Bridge">>]. -roots() -> [bridges]. +roots() -> [{bridges, ?HOCON(?R_REF(bridges), #{importance => ?IMPORTANCE_LOW})}]. fields(bridges) -> [ @@ -251,7 +251,7 @@ do_convert_webhook_config( case {MReqTRoot, MReqTResource} of {{ok, ReqTRoot}, {ok, ReqTResource}} -> {_Parsed, ReqTRaw} = max({ReqTRoot, ReqTRootRaw}, {ReqTResource, ReqTResourceRaw}), - Conf1 = emqx_map_lib:deep_merge( + Conf1 = emqx_utils_maps:deep_merge( Conf0, #{ <<"request_timeout">> => ReqTRaw, diff --git a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl index 8899cd24a..d55b92138 100644 --- a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl @@ -414,6 +414,18 @@ t_http_crud_apis(Config) -> }, json(maps:get(<<"message">>, PutFail2)) ), + {ok, 400, _} = request_json( + put, + uri(["bridges", BridgeID]), + ?HTTP_BRIDGE(<<"localhost:1234/foo">>, Name), + Config + ), + {ok, 400, _} = request_json( + put, + uri(["bridges", BridgeID]), + ?HTTP_BRIDGE(<<"htpp://localhost:12341234/foo">>, Name), + Config + ), %% delete the bridge {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), Config), @@ -498,6 +510,22 @@ t_http_crud_apis(Config) -> %% Try create bridge with bad characters as name {ok, 400, _} = request(post, uri(["bridges"]), ?HTTP_BRIDGE(URL1, <<"隋达"/utf8>>), Config), + %% Missing scheme in URL + {ok, 400, _} = request( + post, + uri(["bridges"]), + ?HTTP_BRIDGE(<<"localhost:1234/foo">>, <<"missing_url_scheme">>), + Config + ), + + %% Invalid port + {ok, 400, _} = request( + post, + uri(["bridges"]), + ?HTTP_BRIDGE(<<"http://localhost:12341234/foo">>, <<"invalid_port">>), + Config + ), + {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), Config). t_http_bridges_local_topic(Config) -> @@ -975,7 +1003,7 @@ t_with_redact_update(Config) -> ), %% update with redacted config - BridgeConf = emqx_misc:redact(Template), + BridgeConf = emqx_utils:redact(Template), BridgeID = emqx_bridge_resource:bridge_id(Type, Name), {ok, 200, _} = request(put, uri(["bridges", BridgeID]), BridgeConf, Config), ?assertEqual( @@ -1016,6 +1044,34 @@ t_bridges_probe(Config) -> ) ), + %% Missing scheme in URL + ?assertMatch( + {ok, 400, #{ + <<"code">> := <<"TEST_FAILED">>, + <<"message">> := _ + }}, + request_json( + post, + uri(["bridges_probe"]), + ?HTTP_BRIDGE(<<"203.0.113.3:1234/foo">>), + Config + ) + ), + + %% Invalid port + ?assertMatch( + {ok, 400, #{ + <<"code">> := <<"TEST_FAILED">>, + <<"message">> := _ + }}, + request_json( + post, + uri(["bridges_probe"]), + ?HTTP_BRIDGE(<<"http://203.0.113.3:12341234/foo">>), + Config + ) + ), + {ok, 204, _} = request( post, uri(["bridges_probe"]), @@ -1201,13 +1257,16 @@ t_metrics(Config) -> request_json(get, uri(["bridges", BridgeID, "metrics"]), Config) ), - %% check that metrics isn't returned when listing all bridges + %% check for absence of metrics when listing all bridges {ok, 200, Bridges} = request_json(get, uri(["bridges"]), Config), - ?assert( - lists:all( - fun(E) -> not maps:is_key(<<"metrics">>, E) end, - Bridges - ) + ?assertNotMatch( + [ + #{ + <<"metrics">> := #{}, + <<"node_metrics">> := [_ | _] + } + ], + Bridges ), ok. @@ -1218,7 +1277,7 @@ t_inconsistent_webhook_request_timeouts(Config) -> URL1 = ?URL(Port, "path1"), Name = ?BRIDGE_NAME, BadBridgeParams = - emqx_map_lib:deep_merge( + emqx_utils_maps:deep_merge( ?HTTP_BRIDGE(URL1, Name), #{ <<"request_timeout">> => <<"1s">>, @@ -1301,4 +1360,4 @@ str(S) when is_list(S) -> S; str(S) when is_binary(S) -> binary_to_list(S). json(B) when is_binary(B) -> - emqx_json:decode(B, [return_maps]). + emqx_utils_json:decode(B, [return_maps]). diff --git a/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl index bd6af9323..bd5cda3f0 100644 --- a/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl @@ -201,7 +201,7 @@ t_mqtt_conn_bridge_ingress(_) -> #{ <<"type">> := ?TYPE_MQTT, <<"name">> := ?BRIDGE_NAME_INGRESS - } = jsx:decode(Bridge), + } = emqx_utils_json:decode(Bridge), BridgeIDIngress = emqx_bridge_resource:bridge_id(?TYPE_MQTT, ?BRIDGE_NAME_INGRESS), @@ -270,7 +270,7 @@ t_mqtt_conn_bridge_ingress_downgrades_qos_2(_) -> ?SERVER_CONF(<<"user1">>)#{ <<"type">> => ?TYPE_MQTT, <<"name">> => BridgeName, - <<"ingress">> => emqx_map_lib:deep_merge( + <<"ingress">> => emqx_utils_maps:deep_merge( ?INGRESS_CONF, #{<<"remote">> => #{<<"qos">> => 2}} ) @@ -313,7 +313,7 @@ t_mqtt_conn_bridge_ingress_no_payload_template(_) -> emqx:publish(emqx_message:make(RemoteTopic, Payload)), %% we should receive a message on the local broker, with specified topic Msg = assert_mqtt_msg_received(LocalTopic), - ?assertMatch(#{<<"payload">> := Payload}, jsx:decode(Msg#message.payload)), + ?assertMatch(#{<<"payload">> := Payload}, emqx_utils_json:decode(Msg#message.payload)), %% verify the metrics of the bridge ?assertMetrics( @@ -402,7 +402,7 @@ t_mqtt_conn_bridge_egress_no_payload_template(_) -> Msg = assert_mqtt_msg_received(RemoteTopic), %% the MapMsg is all fields outputed by Rule-Engine. it's a binary coded json here. ?assertMatch(<>, Msg#message.from), - ?assertMatch(#{<<"payload">> := Payload}, jsx:decode(Msg#message.payload)), + ?assertMatch(#{<<"payload">> := Payload}, emqx_utils_json:decode(Msg#message.payload)), %% verify the metrics of the bridge ?retry( @@ -545,7 +545,7 @@ t_ingress_mqtt_bridge_with_rules(_) -> <<"sql">> => <<"SELECT * from \"$bridges/", BridgeIDIngress/binary, "\"">> } ), - #{<<"id">> := RuleId} = jsx:decode(Rule), + #{<<"id">> := RuleId} = emqx_utils_json:decode(Rule), %% we now test if the bridge works as expected @@ -562,7 +562,7 @@ t_ingress_mqtt_bridge_with_rules(_) -> %% and also the rule should be matched, with matched + 1: {ok, 200, Rule1} = request(get, uri(["rules", RuleId]), []), {ok, 200, Metrics} = request(get, uri(["rules", RuleId, "metrics"]), []), - ?assertMatch(#{<<"id">> := RuleId}, jsx:decode(Rule1)), + ?assertMatch(#{<<"id">> := RuleId}, emqx_utils_json:decode(Rule1)), ?assertMatch( #{ <<"metrics">> := #{ @@ -581,7 +581,7 @@ t_ingress_mqtt_bridge_with_rules(_) -> <<"actions.failed.unknown">> := 0 } }, - jsx:decode(Metrics) + emqx_utils_json:decode(Metrics) ), %% we also check if the actions of the rule is triggered @@ -630,7 +630,7 @@ t_egress_mqtt_bridge_with_rules(_) -> <<"sql">> => <<"SELECT * from \"t/1\"">> } ), - #{<<"id">> := RuleId} = jsx:decode(Rule), + #{<<"id">> := RuleId} = emqx_utils_json:decode(Rule), %% we now test if the bridge works as expected LocalTopic = <>, @@ -653,7 +653,7 @@ t_egress_mqtt_bridge_with_rules(_) -> timer:sleep(100), emqx:publish(emqx_message:make(RuleTopic, Payload2)), {ok, 200, Rule1} = request(get, uri(["rules", RuleId]), []), - ?assertMatch(#{<<"id">> := RuleId, <<"name">> := _}, jsx:decode(Rule1)), + ?assertMatch(#{<<"id">> := RuleId, <<"name">> := _}, emqx_utils_json:decode(Rule1)), {ok, 200, Metrics} = request(get, uri(["rules", RuleId, "metrics"]), []), ?assertMatch( #{ @@ -673,7 +673,7 @@ t_egress_mqtt_bridge_with_rules(_) -> <<"actions.failed.unknown">> := 0 } }, - jsx:decode(Metrics) + emqx_utils_json:decode(Metrics) ), %% we should receive a message on the "remote" broker, with specified topic @@ -911,17 +911,17 @@ create_bridge(Config = #{<<"type">> := Type, <<"name">> := Name}) -> <<"type">> := Type, <<"name">> := Name }, - jsx:decode(Bridge) + emqx_utils_json:decode(Bridge) ), emqx_bridge_resource:bridge_id(Type, Name). request_bridge(BridgeID) -> {ok, 200, Bridge} = request(get, uri(["bridges", BridgeID]), []), - jsx:decode(Bridge). + emqx_utils_json:decode(Bridge). request_bridge_metrics(BridgeID) -> {ok, 200, BridgeMetrics} = request(get, uri(["bridges", BridgeID, "metrics"]), []), - jsx:decode(BridgeMetrics). + emqx_utils_json:decode(BridgeMetrics). request(Method, Url, Body) -> request(<<"connector_admin">>, Method, Url, Body). diff --git a/apps/emqx_bridge/test/emqx_bridge_webhook_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_webhook_SUITE.erl index e222190d2..f08c87b6e 100644 --- a/apps/emqx_bridge/test/emqx_bridge_webhook_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_webhook_SUITE.erl @@ -175,7 +175,7 @@ bridge_async_config(#{port := Port} = Config) -> " inflight_window = 100\n" " auto_restart_interval = \"60s\"\n" " health_check_interval = \"15s\"\n" - " max_queue_bytes = \"1GB\"\n" + " max_buffer_bytes = \"1GB\"\n" " query_mode = \"~s\"\n" " request_timeout = \"~s\"\n" " start_after_created = \"true\"\n" diff --git a/apps/emqx_bridge_cassandra/BSL.txt b/apps/emqx_bridge_cassandra/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_bridge_cassandra/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_bridge_cassandra/README.md b/apps/emqx_bridge_cassandra/README.md new file mode 100644 index 000000000..d26bd2fbb --- /dev/null +++ b/apps/emqx_bridge_cassandra/README.md @@ -0,0 +1,39 @@ +# EMQX Cassandra Bridge + +[Apache Cassandra](https://github.com/apache/cassandra) is an open-source, distributed +NoSQL database management system that is designed to manage large amounts of structured +and semi-structured data across many commodity servers, providing high availability +with no single point of failure. +It is commonly used in web and mobile applications, IoT, and other systems that +require storing, querying, and analyzing large amounts of data. + +The application is used to connect EMQX and Cassandra. User can create a rule +and easily ingest IoT data into Cassandra by leveraging +[EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). + + +# Documentation + +- Refer to [Ingest data into Cassandra](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-cassa.html) + for how to use EMQX dashboard to ingest IoT data into Cassandra. +- Refer to [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) + for the EMQX rules engine introduction. + + +# HTTP APIs + +- Several APIs are provided for bridge management, which includes create bridge, + update bridge, get bridge, stop or restart bridge and list bridges etc. + + Refer to [API Docs - Bridges](https://docs.emqx.com/en/enterprise/v5.0/admin/api-docs.html#tag/Bridges) for more detailed information. + + +# Contributing + +Please see our [contributing.md](../../CONTRIBUTING.md). + + +# License + +EMQ Business Source License 1.1, refer to [LICENSE](BSL.txt). + diff --git a/apps/emqx_bridge_cassandra/docker-ct b/apps/emqx_bridge_cassandra/docker-ct new file mode 100644 index 000000000..2626b4068 --- /dev/null +++ b/apps/emqx_bridge_cassandra/docker-ct @@ -0,0 +1,2 @@ +toxiproxy +cassandra diff --git a/apps/emqx_bridge_cassandra/include/emqx_bridge_cassandra.hrl b/apps/emqx_bridge_cassandra/include/emqx_bridge_cassandra.hrl new file mode 100644 index 000000000..eef7c5d2b --- /dev/null +++ b/apps/emqx_bridge_cassandra/include/emqx_bridge_cassandra.hrl @@ -0,0 +1,5 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-define(CASSANDRA_DEFAULT_PORT, 9042). diff --git a/apps/emqx_bridge_cassandra/rebar.config b/apps/emqx_bridge_cassandra/rebar.config new file mode 100644 index 000000000..b8bfc7dd6 --- /dev/null +++ b/apps/emqx_bridge_cassandra/rebar.config @@ -0,0 +1,11 @@ +%% -*- mode: erlang; -*- +{erl_opts, [debug_info]}. +{deps, [ {ecql, {git, "https://github.com/emqx/ecql.git", {tag, "v0.5.1"}}} + , {emqx_connector, {path, "../../apps/emqx_connector"}} + , {emqx_resource, {path, "../../apps/emqx_resource"}} + , {emqx_bridge, {path, "../../apps/emqx_bridge"}} + ]}. + +{shell, [ + {apps, [emqx_bridge_cassandra]} +]}. diff --git a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src new file mode 100644 index 000000000..58e4a1984 --- /dev/null +++ b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src @@ -0,0 +1,9 @@ +{application, emqx_bridge_cassandra, [ + {description, "EMQX Enterprise Cassandra Bridge"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [kernel, stdlib, ecql]}, + {env, []}, + {modules, []}, + {links, []} +]}. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.erl similarity index 95% rename from lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl rename to apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.erl index 78db8352a..e8f7d50ce 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl +++ b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.erl @@ -1,7 +1,7 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_bridge_cassa). +-module(emqx_bridge_cassandra). -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). @@ -63,7 +63,7 @@ values(_Method, Type) -> batch_size => ?DEFAULT_BATCH_SIZE, batch_time => ?DEFAULT_BATCH_TIME, query_mode => sync, - max_queue_bytes => ?DEFAULT_QUEUE_SIZE + max_buffer_bytes => ?DEFAULT_BUFFER_BYTES } }. @@ -88,7 +88,7 @@ fields("config") -> #{desc => ?DESC("local_topic"), default => undefined} )} ] ++ emqx_resource_schema:fields("resource_opts") ++ - (emqx_ee_connector_cassa:fields(config) -- + (emqx_bridge_cassandra_connector:fields(config) -- emqx_connector_schema_lib:prepare_statement_fields()); fields("post") -> fields("post", cassandra); diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl similarity index 71% rename from lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl rename to apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl index a6a77f233..cf6ddff9f 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl +++ b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl @@ -1,24 +1,13 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_ee_connector_cassa). + +-module(emqx_bridge_cassandra_connector). -behaviour(emqx_resource). -include_lib("emqx_connector/include/emqx_connector.hrl"). --include_lib("emqx_ee_connector/include/emqx_ee_connector.hrl"). +-include("emqx_bridge_cassandra.hrl"). -include_lib("typerefl/include/types.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("hocon/include/hoconsc.hrl"). @@ -33,8 +22,9 @@ on_start/2, on_stop/2, on_query/3, - %% TODO: not_supported_now - %%on_batch_query/3, + on_query_async/4, + on_batch_query/3, + on_batch_query_async/4, on_get_status/2 ]). @@ -45,7 +35,7 @@ ]). %% callbacks for query executing --export([query/3, prepared_query/3]). +-export([query/4, prepared_query/4, batch_query/3]). -export([do_get_status/1]). @@ -54,7 +44,7 @@ -type state() :: #{ - poolname := atom(), + pool_name := binary(), prepare_cql := prepares(), params_tokens := params_tokens(), %% returned by ecql:prepare/2 @@ -96,7 +86,7 @@ keyspace(_) -> undefined. %%-------------------------------------------------------------------- %% callbacks for emqx_resource -callback_mode() -> always_sync. +callback_mode() -> async_if_possible. -spec on_start(binary(), hoconsc:config()) -> {ok, state()} | {error, _}. on_start( @@ -112,7 +102,7 @@ on_start( ?SLOG(info, #{ msg => "starting_cassandra_connector", connector => InstId, - config => emqx_misc:redact(Config) + config => emqx_utils:redact(Config) }), Options = [ @@ -134,14 +124,10 @@ on_start( false -> [] end, - %% use InstaId of binary type as Pool name, which is supported in ecpool. - PoolName = InstId, - Prepares = parse_prepare_cql(Config), - InitState = #{poolname => PoolName, prepare_statement => #{}}, - State = maps:merge(InitState, Prepares), - case emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Options ++ SslOpts) of + State = parse_prepare_cql(Config), + case emqx_resource_pool:start(InstId, ?MODULE, Options ++ SslOpts) of ok -> - {ok, init_prepare(State)}; + {ok, init_prepare(State#{pool_name => InstId, prepare_statement => #{}})}; {error, Reason} -> ?tp( cassandra_connector_start_failed, @@ -150,12 +136,12 @@ on_start( {error, Reason} end. -on_stop(InstId, #{poolname := PoolName}) -> +on_stop(InstId, #{pool_name := PoolName}) -> ?SLOG(info, #{ msg => "stopping_cassandra_connector", connector => InstId }), - emqx_plugin_libs_pool:stop_pool(PoolName). + emqx_resource_pool:stop(PoolName). -type request() :: % emqx_bridge.erl @@ -172,7 +158,29 @@ on_stop(InstId, #{poolname := PoolName}) -> on_query( InstId, Request, - #{poolname := PoolName} = State + State +) -> + do_single_query(InstId, Request, sync, State). + +-spec on_query_async( + emqx_resource:resource_id(), + request(), + {function(), list()}, + state() +) -> ok | {error, {recoverable_error | unrecoverable_error, term()}}. +on_query_async( + InstId, + Request, + Callback, + State +) -> + do_single_query(InstId, Request, {async, Callback}, State). + +do_single_query( + InstId, + Request, + Async, + #{pool_name := PoolName} = State ) -> {Type, PreparedKeyOrSQL, Params} = parse_request_to_cql(Request), ?tp( @@ -187,7 +195,59 @@ on_query( } ), {PreparedKeyOrSQL1, Data} = proc_cql_params(Type, PreparedKeyOrSQL, Params, State), - Res = exec_cql_query(InstId, PoolName, Type, PreparedKeyOrSQL1, Data), + Res = exec_cql_query(InstId, PoolName, Type, Async, PreparedKeyOrSQL1, Data), + handle_result(Res). + +-spec on_batch_query( + emqx_resource:resource_id(), + [request()], + state() +) -> ok | {error, {recoverable_error | unrecoverable_error, term()}}. +on_batch_query( + InstId, + Requests, + State +) -> + do_batch_query(InstId, Requests, sync, State). + +-spec on_batch_query_async( + emqx_resource:resource_id(), + [request()], + {function(), list()}, + state() +) -> ok | {error, {recoverable_error | unrecoverable_error, term()}}. +on_batch_query_async( + InstId, + Requests, + Callback, + State +) -> + do_batch_query(InstId, Requests, {async, Callback}, State). + +do_batch_query( + InstId, + Requests, + Async, + #{pool_name := PoolName} = State +) -> + CQLs = + lists:map( + fun(Request) -> + {Type, PreparedKeyOrSQL, Params} = parse_request_to_cql(Request), + proc_cql_params(Type, PreparedKeyOrSQL, Params, State) + end, + Requests + ), + ?tp( + debug, + cassandra_connector_received_cql_batch_query, + #{ + connector => InstId, + cqls => CQLs, + state => State + } + ), + Res = exec_cql_batch_query(InstId, PoolName, Async, CQLs), handle_result(Res). parse_request_to_cql({send_message, Params}) -> @@ -203,17 +263,18 @@ proc_cql_params( Params, #{prepare_statement := Prepares, params_tokens := ParamsTokens} ) -> - PreparedKey = maps:get(PreparedKey0, Prepares), + %% assert + _PreparedKey = maps:get(PreparedKey0, Prepares), Tokens = maps:get(PreparedKey0, ParamsTokens), - {PreparedKey, assign_type_for_params(emqx_plugin_libs_rule:proc_sql(Tokens, Params))}; + {PreparedKey0, assign_type_for_params(emqx_plugin_libs_rule:proc_sql(Tokens, Params))}; proc_cql_params(query, SQL, Params, _State) -> {SQL1, Tokens} = emqx_plugin_libs_rule:preproc_sql(SQL, '?'), {SQL1, assign_type_for_params(emqx_plugin_libs_rule:proc_sql(Tokens, Params))}. -exec_cql_query(InstId, PoolName, Type, PreparedKey, Data) when +exec_cql_query(InstId, PoolName, Type, Async, PreparedKey, Data) when Type == query; Type == prepared_query -> - case ecpool:pick_and_do(PoolName, {?MODULE, Type, [PreparedKey, Data]}, no_handover) of + case ecpool:pick_and_do(PoolName, {?MODULE, Type, [Async, PreparedKey, Data]}, no_handover) of {error, Reason} = Result -> ?tp( error, @@ -226,8 +287,22 @@ exec_cql_query(InstId, PoolName, Type, PreparedKey, Data) when Result end. -on_get_status(_InstId, #{poolname := Pool} = State) -> - case emqx_plugin_libs_pool:health_check_ecpool_workers(Pool, fun ?MODULE:do_get_status/1) of +exec_cql_batch_query(InstId, PoolName, Async, CQLs) -> + case ecpool:pick_and_do(PoolName, {?MODULE, batch_query, [Async, CQLs]}, no_handover) of + {error, Reason} = Result -> + ?tp( + error, + cassandra_connector_query_return, + #{connector => InstId, error => Reason} + ), + Result; + Result -> + ?tp(debug, cassandra_connector_query_return, #{result => Result}), + Result + end. + +on_get_status(_InstId, #{pool_name := PoolName} = State) -> + case emqx_resource_pool:health_check_workers(PoolName, fun ?MODULE:do_get_status/1) of true -> case do_check_prepares(State) of ok -> @@ -248,7 +323,7 @@ do_get_status(Conn) -> do_check_prepares(#{prepare_cql := Prepares}) when is_map(Prepares) -> ok; -do_check_prepares(State = #{poolname := PoolName, prepare_cql := {error, Prepares}}) -> +do_check_prepares(State = #{pool_name := PoolName, prepare_cql := {error, Prepares}}) -> %% retry to prepare case prepare_cql(Prepares, PoolName) of {ok, Sts} -> @@ -261,11 +336,20 @@ do_check_prepares(State = #{poolname := PoolName, prepare_cql := {error, Prepare %%-------------------------------------------------------------------- %% callbacks query -query(Conn, SQL, Params) -> - ecql:query(Conn, SQL, Params). +query(Conn, sync, CQL, Params) -> + ecql:query(Conn, CQL, Params); +query(Conn, {async, Callback}, CQL, Params) -> + ecql:async_query(Conn, CQL, Params, one, Callback). -prepared_query(Conn, PreparedKey, Params) -> - ecql:execute(Conn, PreparedKey, Params). +prepared_query(Conn, sync, PreparedKey, Params) -> + ecql:execute(Conn, PreparedKey, Params); +prepared_query(Conn, {async, Callback}, PreparedKey, Params) -> + ecql:async_execute(Conn, PreparedKey, Params, Callback). + +batch_query(Conn, sync, Rows) -> + ecql:batch(Conn, Rows); +batch_query(Conn, {async, Callback}, Rows) -> + ecql:async_batch(Conn, Rows, Callback). %%-------------------------------------------------------------------- %% callbacks for ecpool @@ -309,7 +393,7 @@ parse_prepare_cql([], Prepares, Tokens) -> params_tokens => Tokens }. -init_prepare(State = #{prepare_cql := Prepares, poolname := PoolName}) -> +init_prepare(State = #{prepare_cql := Prepares, pool_name := PoolName}) -> case maps:size(Prepares) of 0 -> State; @@ -341,17 +425,17 @@ prepare_cql(Prepares, PoolName) -> end. do_prepare_cql(Prepares, PoolName) -> - do_prepare_cql(ecpool:workers(PoolName), Prepares, PoolName, #{}). + do_prepare_cql(ecpool:workers(PoolName), Prepares, #{}). -do_prepare_cql([{_Name, Worker} | T], Prepares, PoolName, _LastSts) -> +do_prepare_cql([{_Name, Worker} | T], Prepares, _LastSts) -> {ok, Conn} = ecpool_worker:client(Worker), case prepare_cql_to_conn(Conn, Prepares) of {ok, Sts} -> - do_prepare_cql(T, Prepares, PoolName, Sts); + do_prepare_cql(T, Prepares, Sts); Error -> Error end; -do_prepare_cql([], _Prepares, _PoolName, LastSts) -> +do_prepare_cql([], _Prepares, LastSts) -> {ok, LastSts}. prepare_cql_to_conn(Conn, Prepares) -> diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE.erl similarity index 89% rename from lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl rename to apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE.erl index f1ea6e930..7865f0415 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl +++ b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_bridge_cassa_SUITE). +-module(emqx_bridge_cassandra_SUITE). -compile(nowarn_export_all). -compile(export_all). @@ -57,7 +57,7 @@ %% CASSA_TCP_HOST=127.0.0.1 CASSA_TCP_PORT=19042 \ %% CASSA_TLS_HOST=127.0.0.1 CASSA_TLS_PORT=19142 \ %% PROXY_HOST=127.0.0.1 ./rebar3 as test ct -c -v --name ct@127.0.0.1 \ -%% --suite lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl +%% --suite apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE.erl %% %%------------------------------------------------------------------------------ @@ -72,10 +72,10 @@ all() -> groups() -> TCs = emqx_common_test_helpers:all(?MODULE), - NonBatchCases = [t_write_timeout], + NonBatchCases = [t_write_timeout, t_simple_sql_query], QueryModeGroups = [{group, async}, {group, sync}], BatchingGroups = [ - %{group, with_batch}, + {group, with_batch}, {group, without_batch} ], [ @@ -261,7 +261,7 @@ create_bridge(Config, Overrides) -> BridgeType = ?config(cassa_bridge_type, Config), Name = ?config(cassa_name, Config), BridgeConfig0 = ?config(cassa_config, Config), - BridgeConfig = emqx_map_lib:deep_merge(BridgeConfig0, Overrides), + BridgeConfig = emqx_utils_maps:deep_merge(BridgeConfig0, Overrides), emqx_bridge:create(BridgeType, Name, BridgeConfig). delete_bridge(Config) -> @@ -273,7 +273,7 @@ create_bridge_http(Params) -> Path = emqx_mgmt_api_test_util:api_path(["bridges"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. @@ -404,12 +404,7 @@ t_setup_via_config_and_publish(Config) -> end, fun(Trace0) -> Trace = ?of_kind(cassandra_connector_query_return, Trace0), - case ?config(enable_batch, Config) of - true -> - ?assertMatch([#{result := {_, [ok]}}], Trace); - false -> - ?assertMatch([#{result := ok}], Trace) - end, + ?assertMatch([#{result := ok}], Trace), ok end ), @@ -448,12 +443,7 @@ t_setup_via_http_api_and_publish(Config) -> end, fun(Trace0) -> Trace = ?of_kind(cassandra_connector_query_return, Trace0), - case ?config(enable_batch, Config) of - true -> - ?assertMatch([#{result := {_, [{ok, 1}]}}], Trace); - false -> - ?assertMatch([#{result := ok}], Trace) - end, + ?assertMatch([#{result := ok}], Trace), ok end ), @@ -540,15 +530,16 @@ t_write_failure(Config) -> fun(Trace0) -> ct:pal("trace: ~p", [Trace0]), Trace = ?of_kind(buffer_worker_flush_nack, Trace0), - ?assertMatch([#{result := {error, _}} | _], Trace), - [#{result := {error, Error}} | _] = Trace, - case Error of - {resource_error, _} -> + [#{result := Result} | _] = Trace, + case Result of + {async_return, {error, {resource_error, _}}} -> ok; - {recoverable_error, disconnected} -> + {async_return, {error, {recoverable_error, disconnected}}} -> + ok; + {error, {resource_error, _}} -> ok; _ -> - ct:fail("unexpected error: ~p", [Error]) + ct:fail("unexpected error: ~p", [Result]) end end ), @@ -576,7 +567,6 @@ t_write_failure(Config) -> % ok. t_simple_sql_query(Config) -> - EnableBatch = ?config(enable_batch, Config), QueryMode = ?config(query_mode, Config), ?assertMatch( {ok, _}, @@ -592,12 +582,7 @@ t_simple_sql_query(Config) -> {ok, Res} = receive_result(Ref, 2_000), Res end, - case EnableBatch of - true -> - ?assertEqual({error, {unrecoverable_error, batch_prepare_not_implemented}}, Result); - false -> - ?assertMatch({ok, {<<"system.local">>, _, [[1]]}}, Result) - end, + ?assertMatch({ok, {<<"system.local">>, _, [[1]]}}, Result), ok. t_missing_data(Config) -> @@ -605,29 +590,31 @@ t_missing_data(Config) -> {ok, _}, create_bridge(Config) ), - %% emqx_ee_connector_cassa will send missed data as a `null` atom + %% emqx_bridge_cassandra_connector will send missed data as a `null` atom %% to ecql driver - {_, {ok, Event}} = - ?wait_async_action( - send_message(Config, #{}), - #{?snk_kind := buffer_worker_flush_ack}, - 2_000 - ), - ?assertMatch( - %% TODO: match error msgs - #{ - result := - {error, {unrecoverable_error, {8704, <<"Expected 8 or 0 byte long for date (4)">>}}} - }, - Event + ?check_trace( + begin + ?wait_async_action( + send_message(Config, #{}), + #{?snk_kind := handle_async_reply, result := {error, {8704, _}}}, + 10_000 + ), + ok + end, + fun(Trace0) -> + %% 1. ecql driver will return `ok` first in async query + Trace = ?of_kind(cassandra_connector_query_return, Trace0), + ?assertMatch([#{result := ok}], Trace), + %% 2. then it will return an error in callback function + Trace1 = ?of_kind(handle_async_reply, Trace0), + ?assertMatch([#{result := {error, {8704, _}}}], Trace1), + ok + end ), ok. t_bad_sql_parameter(Config) -> QueryMode = ?config(query_mode, Config), - EnableBatch = ?config(enable_batch, Config), - Name = ?config(cassa_name, Config), - ResourceId = emqx_bridge_resource:resource_id(cassandra, Name), ?assertMatch( {ok, _}, create_bridge( @@ -656,14 +643,7 @@ t_bad_sql_parameter(Config) -> ct:fail("no response received") end end, - case EnableBatch of - true -> - ?assertEqual({error, {unrecoverable_error, invalid_request}}, Result); - false -> - ?assertMatch( - {error, {unrecoverable_error, _}}, Result - ) - end, + ?assertMatch({error, _}, Result), ok. t_nasty_sql_string(Config) -> diff --git a/lib-ee/emqx_ee_connector/test/emqx_ee_connector_cassa_SUITE.erl b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl similarity index 75% rename from lib-ee/emqx_ee_connector/test/emqx_ee_connector_cassa_SUITE.erl rename to apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl index 95b4407cf..f419283a8 100644 --- a/lib-ee/emqx_ee_connector/test/emqx_ee_connector_cassa_SUITE.erl +++ b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl @@ -1,25 +1,14 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_connector_cassa_SUITE). +-module(emqx_bridge_cassandra_connector_SUITE). -compile(nowarn_export_all). -compile(export_all). --include("emqx_connector.hrl"). --include("emqx_ee_connector.hrl"). +-include("emqx_bridge_cassandra.hrl"). +-include("emqx_connector/include/emqx_connector.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("stdlib/include/assert.hrl"). @@ -27,7 +16,7 @@ %% Cassandra server defined at `.ci/docker-compose-file/docker-compose-cassandra-tcp.yaml` %% You can change it to `127.0.0.1`, if you run this SUITE locally -define(CASSANDRA_HOST, "cassandra"). --define(CASSANDRA_RESOURCE_MOD, emqx_ee_connector_cassa). +-define(CASSANDRA_RESOURCE_MOD, emqx_bridge_cassandra_connector). %% This test SUITE requires a running cassandra instance. If you don't want to %% bring up the whole CI infrastuctucture with the `scripts/ct/run.sh` script @@ -112,15 +101,15 @@ show(Label, What) -> erlang:display({Label, What}), What. -perform_lifecycle_check(PoolName, InitialConfig) -> +perform_lifecycle_check(ResourceId, InitialConfig) -> {ok, #{config := CheckedConfig}} = emqx_resource:check_config(?CASSANDRA_RESOURCE_MOD, InitialConfig), {ok, #{ - state := #{poolname := ReturnedPoolName} = State, + state := #{pool_name := PoolName} = State, status := InitialStatus }} = emqx_resource:create_local( - PoolName, + ResourceId, ?CONNECTOR_RESOURCE_GROUP, ?CASSANDRA_RESOURCE_MOD, CheckedConfig, @@ -132,45 +121,45 @@ perform_lifecycle_check(PoolName, InitialConfig) -> state := State, status := InitialStatus }} = - emqx_resource:get_instance(PoolName), - ?assertEqual({ok, connected}, emqx_resource:health_check(PoolName)), + emqx_resource:get_instance(ResourceId), + ?assertEqual({ok, connected}, emqx_resource:health_check(ResourceId)), % % Perform query as further check that the resource is working as expected (fun() -> - erlang:display({pool_name, PoolName}), - QueryNoParamsResWrapper = emqx_resource:query(PoolName, test_query_no_params()), + erlang:display({pool_name, ResourceId}), + QueryNoParamsResWrapper = emqx_resource:query(ResourceId, test_query_no_params()), ?assertMatch({ok, _}, QueryNoParamsResWrapper) end)(), - ?assertEqual(ok, emqx_resource:stop(PoolName)), + ?assertEqual(ok, emqx_resource:stop(ResourceId)), % Resource will be listed still, but state will be changed and healthcheck will fail % as the worker no longer exists. {ok, ?CONNECTOR_RESOURCE_GROUP, #{ state := State, status := StoppedStatus }} = - emqx_resource:get_instance(PoolName), + emqx_resource:get_instance(ResourceId), ?assertEqual(stopped, StoppedStatus), - ?assertEqual({error, resource_is_stopped}, emqx_resource:health_check(PoolName)), + ?assertEqual({error, resource_is_stopped}, emqx_resource:health_check(ResourceId)), % Resource healthcheck shortcuts things by checking ets. Go deeper by checking pool itself. - ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)), + ?assertEqual({error, not_found}, ecpool:stop_sup_pool(PoolName)), % Can call stop/1 again on an already stopped instance - ?assertEqual(ok, emqx_resource:stop(PoolName)), + ?assertEqual(ok, emqx_resource:stop(ResourceId)), % Make sure it can be restarted and the healthchecks and queries work properly - ?assertEqual(ok, emqx_resource:restart(PoolName)), + ?assertEqual(ok, emqx_resource:restart(ResourceId)), % async restart, need to wait resource timer:sleep(500), {ok, ?CONNECTOR_RESOURCE_GROUP, #{status := InitialStatus}} = - emqx_resource:get_instance(PoolName), - ?assertEqual({ok, connected}, emqx_resource:health_check(PoolName)), + emqx_resource:get_instance(ResourceId), + ?assertEqual({ok, connected}, emqx_resource:health_check(ResourceId)), (fun() -> QueryNoParamsResWrapper = - emqx_resource:query(PoolName, test_query_no_params()), + emqx_resource:query(ResourceId, test_query_no_params()), ?assertMatch({ok, _}, QueryNoParamsResWrapper) end)(), % Stop and remove the resource in one go. - ?assertEqual(ok, emqx_resource:remove_local(PoolName)), - ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)), + ?assertEqual(ok, emqx_resource:remove_local(ResourceId)), + ?assertEqual({error, not_found}, ecpool:stop_sup_pool(PoolName)), % Should not even be able to get the resource data out of ets now unlike just stopping. - ?assertEqual({error, not_found}, emqx_resource:get_instance(PoolName)). + ?assertEqual({error, not_found}, emqx_resource:get_instance(ResourceId)). %%-------------------------------------------------------------------- %% utils diff --git a/apps/emqx_bridge_clickhouse/BSL.txt b/apps/emqx_bridge_clickhouse/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_bridge_clickhouse/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_bridge_clickhouse/README.md b/apps/emqx_bridge_clickhouse/README.md new file mode 100644 index 000000000..ff870e87d --- /dev/null +++ b/apps/emqx_bridge_clickhouse/README.md @@ -0,0 +1,37 @@ +# EMQX ClickHouse Bridge + +[ClickHouse](https://github.com/ClickHouse/ClickHouse) is an open-source, column-based +database management system. It is designed for real-time processing of large volumes of +data and is known for its high performance and scalability. + +The application is used to connect EMQX and ClickHouse. +User can create a rule and easily ingest IoT data into ClickHouse by leveraging +[EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). + + +# Documentation + +- Refer to [Ingest data into ClickHouse](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-clickhouse.html) + for how to use EMQX dashboard to ingest IoT data into ClickHouse. + +- Refer to [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) + for the EMQX rules engine introduction. + + +# HTTP APIs + +- Several APIs are provided for bridge management, which includes create bridge, + update bridge, get bridge, stop or restart bridge and list bridges etc. + + Refer to [API Docs - Bridges](https://docs.emqx.com/en/enterprise/v5.0/admin/api-docs.html#tag/Bridges) + for more detailed information. + + +# Contributing + +Please see our [contributing.md](../../CONTRIBUTING.md). + + +# License + +EMQ Business Source License 1.1, refer to [LICENSE](BSL.txt). diff --git a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src new file mode 100644 index 000000000..a0b409d5b --- /dev/null +++ b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src @@ -0,0 +1,9 @@ +{application, emqx_bridge_clickhouse, [ + {description, "EMQX Enterprise ClickHouse Bridge"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [kernel, stdlib]}, + {env, []}, + {modules, []}, + {links, []} +]}. diff --git a/apps/emqx_bridge_dynamo/BSL.txt b/apps/emqx_bridge_dynamo/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_bridge_dynamo/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_bridge_dynamo/README.md b/apps/emqx_bridge_dynamo/README.md new file mode 100644 index 000000000..48dcb781d --- /dev/null +++ b/apps/emqx_bridge_dynamo/README.md @@ -0,0 +1,40 @@ +# EMQX DynamoDB Bridge + +[Dynamodb](https://aws.amazon.com/dynamodb/) is a high-performance NoSQL database +service provided by Amazon that's designed for scalability and low-latency access +to structured data. + +It's often used in applications that require fast and reliable access to data, +such as mobile, ad tech, and IoT. + +The application is used to connect EMQX and DynamoDB. +User can create a rule and easily ingest IoT data into DynamoDB by leveraging +[EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). + + +# Documentation + +- Refer to [Ingest data into DynamoDB](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-dynamo.html) + for how to use EMQX dashboard to ingest IoT data into DynamoDB. + +- Refer to [Rules engine](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) + for the EMQX rules engine introduction. + + +# HTTP APIs + +- Several APIs are provided for bridge management, which includes create bridge, + update bridge, get bridge, stop or restart bridge and list bridges etc. + + Refer to [API Docs - Bridges](https://docs.emqx.com/en/enterprise/v5.0/admin/api-docs.html#tag/Bridges) + for more detailed information. + + +# Contributing + +Please see our [contributing.md](../../CONTRIBUTING.md). + + +# License + +EMQ Business Source License 1.1, refer to [LICENSE](BSL.txt). diff --git a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src new file mode 100644 index 000000000..51c717220 --- /dev/null +++ b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src @@ -0,0 +1,9 @@ +{application, emqx_bridge_dynamo, [ + {description, "EMQX Enterprise Dynamo Bridge"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [kernel, stdlib]}, + {env, []}, + {modules, []}, + {links, []} +]}. diff --git a/apps/emqx_bridge_gcp_pubsub/BSL.txt b/apps/emqx_bridge_gcp_pubsub/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_bridge_gcp_pubsub/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_bridge_gcp_pubsub/README.md b/apps/emqx_bridge_gcp_pubsub/README.md new file mode 100644 index 000000000..e33c5ab15 --- /dev/null +++ b/apps/emqx_bridge_gcp_pubsub/README.md @@ -0,0 +1,36 @@ +# EMQX GCP Pub/Sub Bridge + +[Google Cloud Pub/Sub](https://cloud.google.com/pubsub) is a messaging service +provided by Google Cloud Platform (GCP). + +The application is used to connect EMQX and GCP Pub/Sub. +User can create a rule and easily ingest IoT data into GCP Pub/Sub by leveraging +[EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). + + +# Documentation + +- Refer to [Ingest data into GCP Pub/Sub](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-gcp-pubsub.html) + for how to use EMQX dashboard to ingest IoT data into GCP Pub/Sub. + +- Refer to [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) + for the EMQX rules engine introduction. + + +# HTTP APIs + +- Several APIs are provided for bridge management, which includes create bridge, + update bridge, get bridge, stop or restart bridge and list bridges etc. + + Refer to [API Docs - Bridges](https://docs.emqx.com/en/enterprise/v5.0/admin/api-docs.html#tag/Bridges) + for more detailed information. + + +# Contributing + +Please see our [contributing.md](../../CONTRIBUTING.md). + + +# License + +EMQ Business Source License 1.1, refer to [LICENSE](BSL.txt). diff --git a/apps/emqx_bridge_gcp_pubsub/rebar.config b/apps/emqx_bridge_gcp_pubsub/rebar.config new file mode 100644 index 000000000..2fd264fc0 --- /dev/null +++ b/apps/emqx_bridge_gcp_pubsub/rebar.config @@ -0,0 +1,10 @@ +%% -*- mode: erlang; -*- +{erl_opts, [debug_info]}. +{deps, [ {emqx_connector, {path, "../../apps/emqx_connector"}} + , {emqx_resource, {path, "../../apps/emqx_resource"}} + , {emqx_bridge, {path, "../../apps/emqx_bridge"}} + ]}. + +{shell, [ + {apps, [emqx_bridge_gcp_pubsub]} +]}. diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src new file mode 100644 index 000000000..86627eb2a --- /dev/null +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src @@ -0,0 +1,13 @@ +{application, emqx_bridge_gcp_pubsub, [ + {description, "EMQX Enterprise GCP Pub/Sub Bridge"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [ + kernel, + stdlib, + ehttpc + ]}, + {env, []}, + {modules, []}, + {links, []} +]}. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_gcp_pubsub.erl b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.erl similarity index 99% rename from lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_gcp_pubsub.erl rename to apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.erl index 180640d65..70109a0ea 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_gcp_pubsub.erl +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_bridge_gcp_pubsub). +-module(emqx_bridge_gcp_pubsub). -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_gcp_pubsub.erl b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_connector.erl similarity index 85% rename from lib-ee/emqx_ee_connector/src/emqx_ee_connector_gcp_pubsub.erl rename to apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_connector.erl index f07cbceab..a3f0ef36b 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_gcp_pubsub.erl +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_connector.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_connector_gcp_pubsub). +-module(emqx_bridge_gcp_pubsub_connector). -behaviour(emqx_resource). @@ -26,9 +26,8 @@ ]). -export([reply_delegator/3]). --type bridge_id() :: binary(). -type jwt_worker() :: binary(). --type service_account_json() :: emqx_ee_bridge_gcp_pubsub:service_account_json(). +-type service_account_json() :: emqx_bridge_gcp_pubsub:service_account_json(). -type config() :: #{ connect_timeout := emqx_schema:duration_ms(), max_retries := non_neg_integer(), @@ -43,7 +42,7 @@ jwt_worker_id := jwt_worker(), max_retries := non_neg_integer(), payload_template := emqx_plugin_libs_rule:tmpl_token(), - pool_name := atom(), + pool_name := binary(), project_id := binary(), pubsub_topic := binary(), request_timeout := timer:time() @@ -86,7 +85,7 @@ on_start( PoolType = random, Transport = tls, TransportOpts = emqx_tls_lib:to_client_opts(#{enable => true, verify => verify_none}), - NTransportOpts = emqx_misc:ipv6_probe(TransportOpts), + NTransportOpts = emqx_utils:ipv6_probe(TransportOpts), PoolOpts = [ {host, Host}, {port, Port}, @@ -102,14 +101,13 @@ on_start( jwt_worker_id := JWTWorkerId, project_id := ProjectId } = ensure_jwt_worker(InstanceId, Config), - PoolName = emqx_plugin_libs_pool:pool_name(InstanceId), State = #{ connect_timeout => ConnectTimeout, instance_id => InstanceId, jwt_worker_id => JWTWorkerId, max_retries => MaxRetries, payload_template => emqx_plugin_libs_rule:preproc_tmpl(PayloadTemplate), - pool_name => PoolName, + pool_name => InstanceId, project_id => ProjectId, pubsub_topic => PubSubTopic, request_timeout => RequestTimeout @@ -118,20 +116,20 @@ on_start( gcp_pubsub_on_start_before_starting_pool, #{ instance_id => InstanceId, - pool_name => PoolName, + pool_name => InstanceId, pool_opts => PoolOpts } ), - ?tp(gcp_pubsub_starting_ehttpc_pool, #{pool_name => PoolName}), - case ehttpc_sup:start_pool(PoolName, PoolOpts) of + ?tp(gcp_pubsub_starting_ehttpc_pool, #{pool_name => InstanceId}), + case ehttpc_sup:start_pool(InstanceId, PoolOpts) of {ok, _} -> {ok, State}; {error, {already_started, _}} -> - ?tp(gcp_pubsub_ehttpc_pool_already_started, #{pool_name => PoolName}), + ?tp(gcp_pubsub_ehttpc_pool_already_started, #{pool_name => InstanceId}), {ok, State}; {error, Reason} -> ?tp(gcp_pubsub_ehttpc_pool_start_failure, #{ - pool_name => PoolName, + pool_name => InstanceId, reason => Reason }), {error, Reason} @@ -140,10 +138,7 @@ on_start( -spec on_stop(manager_id(), state()) -> ok | {error, term()}. on_stop( InstanceId, - _State = #{ - jwt_worker_id := JWTWorkerId, - pool_name := PoolName - } + _State = #{jwt_worker_id := JWTWorkerId, pool_name := PoolName} ) -> ?tp(gcp_pubsub_stop, #{instance_id => InstanceId, jwt_worker_id => JWTWorkerId}), ?SLOG(info, #{ @@ -155,7 +150,7 @@ on_stop( ehttpc_sup:stop_pool(PoolName). -spec on_query( - bridge_id(), + resource_id(), {send_message, map()}, state() ) -> @@ -163,32 +158,32 @@ on_stop( | {ok, status_code(), headers(), body()} | {error, {recoverable_error, term()}} | {error, term()}. -on_query(BridgeId, {send_message, Selected}, State) -> +on_query(ResourceId, {send_message, Selected}, State) -> Requests = [{send_message, Selected}], ?TRACE( "QUERY_SYNC", "gcp_pubsub_received", - #{requests => Requests, connector => BridgeId, state => State} + #{requests => Requests, connector => ResourceId, state => State} ), - do_send_requests_sync(State, Requests, BridgeId). + do_send_requests_sync(State, Requests, ResourceId). -spec on_query_async( - bridge_id(), + resource_id(), {send_message, map()}, {ReplyFun :: function(), Args :: list()}, state() ) -> {ok, pid()}. -on_query_async(BridgeId, {send_message, Selected}, ReplyFunAndArgs, State) -> +on_query_async(ResourceId, {send_message, Selected}, ReplyFunAndArgs, State) -> Requests = [{send_message, Selected}], ?TRACE( "QUERY_ASYNC", "gcp_pubsub_received", - #{requests => Requests, connector => BridgeId, state => State} + #{requests => Requests, connector => ResourceId, state => State} ), - do_send_requests_async(State, Requests, ReplyFunAndArgs, BridgeId). + do_send_requests_async(State, Requests, ReplyFunAndArgs, ResourceId). -spec on_batch_query( - bridge_id(), + resource_id(), [{send_message, map()}], state() ) -> @@ -196,34 +191,30 @@ on_query_async(BridgeId, {send_message, Selected}, ReplyFunAndArgs, State) -> | {ok, status_code(), headers(), body()} | {error, {recoverable_error, term()}} | {error, term()}. -on_batch_query(BridgeId, Requests, State) -> +on_batch_query(ResourceId, Requests, State) -> ?TRACE( "QUERY_SYNC", "gcp_pubsub_received", - #{requests => Requests, connector => BridgeId, state => State} + #{requests => Requests, connector => ResourceId, state => State} ), - do_send_requests_sync(State, Requests, BridgeId). + do_send_requests_sync(State, Requests, ResourceId). -spec on_batch_query_async( - bridge_id(), + resource_id(), [{send_message, map()}], {ReplyFun :: function(), Args :: list()}, state() ) -> {ok, pid()}. -on_batch_query_async(BridgeId, Requests, ReplyFunAndArgs, State) -> +on_batch_query_async(ResourceId, Requests, ReplyFunAndArgs, State) -> ?TRACE( "QUERY_ASYNC", "gcp_pubsub_received", - #{requests => Requests, connector => BridgeId, state => State} + #{requests => Requests, connector => ResourceId, state => State} ), - do_send_requests_async(State, Requests, ReplyFunAndArgs, BridgeId). + do_send_requests_async(State, Requests, ReplyFunAndArgs, ResourceId). -spec on_get_status(manager_id(), state()) -> connected | disconnected. -on_get_status(InstanceId, State) -> - #{ - connect_timeout := Timeout, - pool_name := PoolName - } = State, +on_get_status(InstanceId, #{connect_timeout := Timeout, pool_name := PoolName} = State) -> case do_get_status(InstanceId, PoolName, Timeout) of true -> connected; @@ -245,8 +236,7 @@ on_get_status(InstanceId, State) -> project_id := binary() }. ensure_jwt_worker(InstanceId, #{ - service_account_json := ServiceAccountJSON, - pubsub_topic := PubSubTopic + service_account_json := ServiceAccountJSON }) -> #{ project_id := ProjectId, @@ -276,14 +266,8 @@ ensure_jwt_worker(InstanceId, #{ {ok, Worker0} -> Worker0; Error -> - ?tp( - gcp_pubsub_bridge_jwt_worker_failed_to_start, - #{instance_id => InstanceId, reason => Error} - ), - ?SLOG(error, #{ - msg => "failed_to_start_gcp_pubsub_jwt_worker", - instance_id => InstanceId, - pubsub_topic => PubSubTopic, + ?tp(error, "gcp_pubsub_bridge_jwt_worker_failed_to_start", #{ + connector => InstanceId, reason => Error }), _ = emqx_connector_jwt_sup:ensure_worker_deleted(JWTWorkerId), @@ -301,26 +285,14 @@ ensure_jwt_worker(InstanceId, #{ demonitor(MRef, [flush]), ok; {'DOWN', MRef, process, Worker, Reason} -> - ?tp( - gcp_pubsub_bridge_jwt_worker_failed_to_start, - #{ - resource_id => InstanceId, - reason => Reason - } - ), - ?SLOG(error, #{ - msg => "gcp_pubsub_bridge_jwt_worker_failed_to_start", + ?tp(error, "gcp_pubsub_bridge_jwt_worker_failed_to_start", #{ connector => InstanceId, reason => Reason }), _ = emqx_connector_jwt_sup:ensure_worker_deleted(JWTWorkerId), throw(failed_to_start_jwt_worker) after 10_000 -> - ?tp(gcp_pubsub_bridge_jwt_timeout, #{resource_id => InstanceId}), - ?SLOG(warning, #{ - msg => "gcp_pubsub_bridge_jwt_timeout", - connector => InstanceId - }), + ?tp(warning, "gcp_pubsub_bridge_jwt_timeout", #{connector => InstanceId}), demonitor(MRef, [flush]), _ = emqx_connector_jwt_sup:ensure_worker_deleted(JWTWorkerId), throw(timeout_creating_jwt) @@ -334,14 +306,14 @@ ensure_jwt_worker(InstanceId, #{ encode_payload(_State = #{payload_template := PayloadTemplate}, Selected) -> Interpolated = case PayloadTemplate of - [] -> emqx_json:encode(Selected); + [] -> emqx_utils_json:encode(Selected); _ -> emqx_plugin_libs_rule:proc_tmpl(PayloadTemplate, Selected) end, #{data => base64:encode(Interpolated)}. -spec to_pubsub_request([#{data := binary()}]) -> binary(). to_pubsub_request(Payloads) -> - emqx_json:encode(#{messages => Payloads}). + emqx_utils_json:encode(#{messages => Payloads}). -spec publish_path(state()) -> binary(). publish_path( @@ -569,7 +541,7 @@ reply_delegator(_ResourceId, ReplyFunAndArgs, Result) -> emqx_resource:apply_reply_fun(ReplyFunAndArgs, Result) end. --spec do_get_status(manager_id(), atom(), timer:time()) -> boolean(). +-spec do_get_status(manager_id(), binary(), timer:time()) -> boolean(). do_get_status(InstanceId, PoolName, Timeout) -> Workers = [Worker || {_WorkerName, Worker} <- ehttpc:workers(PoolName)], DoPerWorker = @@ -587,7 +559,7 @@ do_get_status(InstanceId, PoolName, Timeout) -> false end end, - try emqx_misc:pmap(DoPerWorker, Workers, Timeout) of + try emqx_utils:pmap(DoPerWorker, Workers, Timeout) of [_ | _] = Status -> lists:all(fun(St) -> St =:= true end, Status); [] -> diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl b/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_SUITE.erl similarity index 96% rename from lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl rename to apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_SUITE.erl index 75d2d2d8c..55527bf1f 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl +++ b/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_SUITE.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_bridge_gcp_pubsub_SUITE). +-module(emqx_bridge_gcp_pubsub_SUITE). -compile(nowarn_export_all). -compile(export_all). @@ -70,22 +70,13 @@ init_per_suite(Config) -> ok = emqx_connector_test_helpers:start_apps([emqx_resource, emqx_bridge, emqx_rule_engine]), {ok, _} = application:ensure_all_started(emqx_connector), emqx_mgmt_api_test_util:init_suite(), - HTTPHost = "localhost", - HTTPPort = 56000, - HostPort = HTTPHost ++ ":" ++ integer_to_list(HTTPPort), - true = os:putenv("PUBSUB_EMULATOR_HOST", HostPort), - [ - {http_host, HTTPHost}, - {http_port, HTTPPort} - | Config - ]. + Config. end_per_suite(_Config) -> emqx_mgmt_api_test_util:end_suite(), ok = emqx_common_test_helpers:stop_apps([emqx_conf]), ok = emqx_connector_test_helpers:stop_apps([emqx_bridge, emqx_resource, emqx_rule_engine]), _ = application:stop(emqx_connector), - os:unsetenv("PUBSUB_EMULATOR_HOST"), ok. init_per_group(sync_query, Config) -> @@ -113,26 +104,26 @@ init_per_testcase(TestCase, Config0) when 1 -> [{skip_due_to_no_batching, true}]; _ -> - {ok, _} = start_echo_http_server(), delete_all_bridges(), Tid = install_telemetry_handler(TestCase), Config = generate_config(Config0), put(telemetry_table, Tid), - [{telemetry_table, Tid} | Config] + {ok, HttpServer} = start_echo_http_server(), + [{telemetry_table, Tid}, {http_server, HttpServer} | Config] end; init_per_testcase(TestCase, Config0) -> ct:timetrap({seconds, 30}), - {ok, _} = start_echo_http_server(), + {ok, HttpServer} = start_echo_http_server(), delete_all_bridges(), Tid = install_telemetry_handler(TestCase), Config = generate_config(Config0), put(telemetry_table, Tid), - [{telemetry_table, Tid} | Config]. + [{telemetry_table, Tid}, {http_server, HttpServer} | Config]. end_per_testcase(_TestCase, _Config) -> ok = snabbkaffe:stop(), delete_all_bridges(), - ok = emqx_connector_web_hook_server:stop(), + ok = stop_echo_http_server(), emqx_common_test_helpers:call_janitor(), ok. @@ -181,7 +172,7 @@ create_bridge(Config, GCPPubSubConfigOverrides) -> TypeBin = ?BRIDGE_TYPE_BIN, Name = ?config(gcp_pubsub_name, Config), GCPPubSubConfig0 = ?config(gcp_pubsub_config, Config), - GCPPubSubConfig = emqx_map_lib:deep_merge(GCPPubSubConfig0, GCPPubSubConfigOverrides), + GCPPubSubConfig = emqx_utils_maps:deep_merge(GCPPubSubConfig0, GCPPubSubConfigOverrides), ct:pal("creating bridge: ~p", [GCPPubSubConfig]), Res = emqx_bridge:create(TypeBin, Name, GCPPubSubConfig), ct:pal("bridge creation result: ~p", [Res]), @@ -194,7 +185,7 @@ create_bridge_http(Config, GCPPubSubConfigOverrides) -> TypeBin = ?BRIDGE_TYPE_BIN, Name = ?config(gcp_pubsub_name, Config), GCPPubSubConfig0 = ?config(gcp_pubsub_config, Config), - GCPPubSubConfig = emqx_map_lib:deep_merge(GCPPubSubConfig0, GCPPubSubConfigOverrides), + GCPPubSubConfig = emqx_utils_maps:deep_merge(GCPPubSubConfig0, GCPPubSubConfigOverrides), Params = GCPPubSubConfig#{<<"type">> => TypeBin, <<"name">> => Name}, Path = emqx_mgmt_api_test_util:api_path(["bridges"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), @@ -204,7 +195,7 @@ create_bridge_http(Config, GCPPubSubConfigOverrides) -> ct:pal("probe result: ~p", [ProbeResult]), Res = case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res0} -> {ok, emqx_json:decode(Res0, [return_maps])}; + {ok, Res0} -> {ok, emqx_utils_json:decode(Res0, [return_maps])}; Error -> Error end, ct:pal("bridge creation result: ~p", [Res]), @@ -222,7 +213,7 @@ create_rule_and_action_http(Config) -> Path = emqx_mgmt_api_test_util:api_path(["rules"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. @@ -234,7 +225,7 @@ success_http_handler() -> Rep = cowboy_req:reply( 200, #{<<"content-type">> => <<"application/json">>}, - jiffy:encode(#{messageIds => [<<"6058891368195201">>]}), + emqx_utils_json:encode(#{messageIds => [<<"6058891368195201">>]}), Req ), {ok, Rep, State} @@ -242,7 +233,6 @@ success_http_handler() -> start_echo_http_server() -> HTTPHost = "localhost", - HTTPPort = 56000, HTTPPath = <<"/v1/projects/myproject/topics/mytopic:publish">>, ServerSSLOpts = [ @@ -250,14 +240,23 @@ start_echo_http_server() -> {versions, ['tlsv1.2', 'tlsv1.3']}, {ciphers, ["ECDHE-RSA-AES256-GCM-SHA384", "TLS_CHACHA20_POLY1305_SHA256"]} ] ++ certs(), - {ok, _} = emqx_connector_web_hook_server:start_link(HTTPPort, HTTPPath, ServerSSLOpts), + {ok, {HTTPPort, _Pid}} = emqx_connector_web_hook_server:start_link( + random, HTTPPath, ServerSSLOpts + ), ok = emqx_connector_web_hook_server:set_handler(success_http_handler()), + HTTPHost = "localhost", + HostPort = HTTPHost ++ ":" ++ integer_to_list(HTTPPort), + true = os:putenv("PUBSUB_EMULATOR_HOST", HostPort), {ok, #{ - host_port => HTTPHost ++ ":" ++ integer_to_list(HTTPPort), + host_port => HostPort, host => HTTPHost, port => HTTPPort }}. +stop_echo_http_server() -> + os:unsetenv("PUBSUB_EMULATOR_HOST"), + ok = emqx_connector_web_hook_server:stop(). + certs() -> CertsPath = emqx_common_test_helpers:deps_path(emqx, "etc/certs"), [ @@ -274,7 +273,7 @@ gcp_pubsub_config(Config) -> PubSubTopic = proplists:get_value(pubsub_topic, Config, <<"mytopic">>), PipelineSize = proplists:get_value(pipeline_size, Config, 100), ServiceAccountJSON = proplists:get_value(pubsub_topic, Config, generate_service_account_json()), - ServiceAccountJSONStr = emqx_json:encode(ServiceAccountJSON), + ServiceAccountJSONStr = emqx_utils_json:encode(ServiceAccountJSON), GUID = emqx_guid:to_hexstr(emqx_guid:gen()), Name = <<(atom_to_binary(?MODULE))/binary, (GUID)/binary>>, ConfigString = @@ -463,7 +462,7 @@ assert_valid_request_headers(Headers, ServiceAccountJSON) -> end. assert_valid_request_body(Body) -> - BodyMap = emqx_json:decode(Body, [return_maps]), + BodyMap = emqx_utils_json:decode(Body, [return_maps]), ?assertMatch(#{<<"messages">> := [_ | _]}, BodyMap), #{<<"messages">> := Messages} = BodyMap, lists:map( @@ -471,7 +470,7 @@ assert_valid_request_body(Body) -> ?assertMatch(#{<<"data">> := <<_/binary>>}, Msg), #{<<"data">> := Content64} = Msg, Content = base64:decode(Content64), - Decoded = emqx_json:decode(Content, [return_maps]), + Decoded = emqx_utils_json:decode(Content, [return_maps]), ct:pal("decoded payload: ~p", [Decoded]), ?assert(is_map(Decoded)), Decoded @@ -809,7 +808,7 @@ test_publish_success_batch(Config) -> %% making 1-sized batches. also important to note that the pool %% size for the resource (replayq buffering) must be set to 1 to %% avoid further segmentation of batches. - emqx_misc:pmap(fun emqx:publish/1, Messages), + emqx_utils:pmap(fun emqx:publish/1, Messages), DecodedMessages0 = assert_http_request(ServiceAccountJSON), ?assertEqual(BatchSize, length(DecodedMessages0)), DecodedMessages1 = assert_http_request(ServiceAccountJSON), @@ -917,7 +916,7 @@ t_invalid_private_key(Config) -> #{<<"private_key">> => InvalidPrivateKeyPEM} } ), - #{?snk_kind := gcp_pubsub_bridge_jwt_worker_failed_to_start}, + #{?snk_kind := "gcp_pubsub_bridge_jwt_worker_failed_to_start"}, 20_000 ), Res @@ -928,7 +927,7 @@ t_invalid_private_key(Config) -> [#{reason := Reason}] when Reason =:= noproc orelse Reason =:= {shutdown, {error, empty_key}}, - ?of_kind(gcp_pubsub_bridge_jwt_worker_failed_to_start, Trace) + ?of_kind("gcp_pubsub_bridge_jwt_worker_failed_to_start", Trace) ), ?assertMatch( [#{error := empty_key}], @@ -956,14 +955,14 @@ t_jwt_worker_start_timeout(Config) -> #{<<"private_key">> => InvalidPrivateKeyPEM} } ), - #{?snk_kind := gcp_pubsub_bridge_jwt_timeout}, + #{?snk_kind := "gcp_pubsub_bridge_jwt_timeout"}, 20_000 ), Res end, fun(Res, Trace) -> ?assertMatch({ok, _}, Res), - ?assertMatch([_], ?of_kind(gcp_pubsub_bridge_jwt_timeout, Trace)), + ?assertMatch([_], ?of_kind("gcp_pubsub_bridge_jwt_timeout", Trace)), ok end ), @@ -1014,7 +1013,7 @@ t_publish_timeout(Config) -> Rep = cowboy_req:reply( 200, #{<<"content-type">> => <<"application/json">>}, - jiffy:encode(#{messageIds => [<<"6058891368195201">>]}), + emqx_utils_json:encode(#{messageIds => [<<"6058891368195201">>]}), Req ), {ok, Rep, State} @@ -1180,7 +1179,7 @@ t_failure_with_body(Config) -> Rep = cowboy_req:reply( 400, #{<<"content-type">> => <<"application/json">>}, - jiffy:encode(#{}), + emqx_utils_json:encode(#{}), Req ), {ok, Rep, State} @@ -1329,7 +1328,7 @@ t_failed_to_start_jwt_worker(Config) -> fun(Trace) -> ?assertMatch( [#{reason := {error, restarting}}], - ?of_kind(gcp_pubsub_bridge_jwt_worker_failed_to_start, Trace) + ?of_kind("gcp_pubsub_bridge_jwt_worker_failed_to_start", Trace) ), ok end diff --git a/apps/emqx_bridge_hstreamdb/BSL.txt b/apps/emqx_bridge_hstreamdb/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_bridge_hstreamdb/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_bridge_hstreamdb/README.md b/apps/emqx_bridge_hstreamdb/README.md new file mode 100644 index 000000000..3a7c6b49d --- /dev/null +++ b/apps/emqx_bridge_hstreamdb/README.md @@ -0,0 +1,38 @@ +# EMQX HStreamDB Bridge + +[HStreamDB](https://hstream.io/) is streaming database purpose-built to ingest, +store, process, and analyze massive data streams. It is a modern data infrastructure +that unifies messaging, stream processing, and storage to help get value out of +your data in real-time. + +The application is used to connect EMQX and HStreamDB. +User can create a rule and easily ingest IoT data into HStreamDB by leveraging +[EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). + + +# Documentation + +- Refer to [Ingest data into HStreamDB](todo) + for how to use EMQX dashboard to ingest IoT data into HStreamDB. + +- Refer to [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) + for the EMQX rules engine introduction. + + +# HTTP APIs + +- Several APIs are provided for bridge management, which includes create bridge, + update bridge, get bridge, stop or restart bridge and list bridges etc. + + Refer to [API Docs - Bridges](https://docs.emqx.com/en/enterprise/v5.0/admin/api-docs.html#tag/Bridges) + for more detailed information. + + +# Contributing + +Please see our [contributing.md](../../CONTRIBUTING.md). + + +# License + +EMQ Business Source License 1.1, refer to [LICENSE](BSL.txt). diff --git a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src new file mode 100644 index 000000000..1cb3742b3 --- /dev/null +++ b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src @@ -0,0 +1,9 @@ +{application, emqx_bridge_hstreamdb, [ + {description, "EMQX Enterprise HStreamDB Bridge"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [kernel, stdlib]}, + {env, []}, + {modules, []}, + {links, []} +]}. diff --git a/apps/emqx_bridge_influxdb/BSL.txt b/apps/emqx_bridge_influxdb/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_bridge_influxdb/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_bridge_influxdb/README.md b/apps/emqx_bridge_influxdb/README.md new file mode 100644 index 000000000..fe0f14600 --- /dev/null +++ b/apps/emqx_bridge_influxdb/README.md @@ -0,0 +1,49 @@ +# EMQX InfluxDB Bridge + +[InfluxDB](https://github.com/influxdata/influxdb) is an open-source time-series +database that is optimized for storing, retrieving, and querying large volumes of +time-stamped data. +It is commonly used for monitoring and analysis of metrics, events, and real-time +analytics. +InfluxDB is designed to be fast, efficient, and scalable, and it has a SQL-like +query language that makes it easy to extract insights from time-series data. + +The application is used to connect EMQX and InfluxDB. User can create a rule and +easily ingest IoT data into InfluxDB by leveraging +[EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). + + +# Documentation + +- Refer to [Ingest data into InfluxDB](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-influxdb.html) + for how to use EMQX dashboard to ingest IoT data into InfluxDB. + +- Refer to [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) + for the EMQX rules engine introduction. + + +# HTTP APIs + +- Several APIs are provided for bridge management, which includes create bridge, + update bridge, get bridge, stop or restart bridge and list bridges etc. + + Refer to [API Docs - Bridges](https://docs.emqx.com/en/enterprise/v5.0/admin/api-docs.html#tag/Bridges) for more detailed information. + +- [Create bridge API doc](https://docs.emqx.com/en/enterprise/v5.0/admin/api-docs.html#tag/Bridges/paths/~1bridges/post) + list required parameters for creating a InfluxDB bridge. + There are two types of InfluxDB API (`v1` and `v2`), please select the right + version of InfluxDB. Below are several important parameters for `v1`, + - `server`: The IPv4 or IPv6 address or the hostname to connect to. + - `database`: InfluxDB database name + - `write_syntax`: Conf of InfluxDB line protocol to write data points. It is a text-based format that provides the measurement, tag set, field set, and timestamp of a data point, and placeholder supported. + + +# Contributing + +Please see our [contributing.md](../../CONTRIBUTING.md). + + +# License + +EMQ Business Source License 1.1, refer to [LICENSE](BSL.txt). + diff --git a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src new file mode 100644 index 000000000..5443417c3 --- /dev/null +++ b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src @@ -0,0 +1,9 @@ +{application, emqx_bridge_influxdb, [ + {description, "EMQX Enterprise InfluxDB Bridge"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [kernel, stdlib]}, + {env, []}, + {modules, []}, + {links, []} +]}. diff --git a/apps/emqx_bridge_kafka/BSL.txt b/apps/emqx_bridge_kafka/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_bridge_kafka/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_bridge_kafka/README.md b/apps/emqx_bridge_kafka/README.md new file mode 100644 index 000000000..72cbeecc6 --- /dev/null +++ b/apps/emqx_bridge_kafka/README.md @@ -0,0 +1,19 @@ +# Kafka Data Integration Bridge + +This application houses the Kafka Producer and Consumer data +integration bridges for EMQX Enterprise Edition. It provides the +means to connect to Kafka and publish/consume messages to/from it. + +Currently, our Kafka Producer library (`wolff`) has its own `replayq` +buffering implementation, so this bridge does not require buffer +workers from `emqx_resource`. It implements the connection management +and interaction without need for a separate connector app, since it's +not used by authentication and authorization applications. + +## Contributing + +Please see our [contributing.md](../../CONTRIBUTING.md). + +## License + +See [BSL](./BSL.txt). diff --git a/apps/emqx_bridge_kafka/docker-ct b/apps/emqx_bridge_kafka/docker-ct new file mode 100644 index 000000000..5288ee246 --- /dev/null +++ b/apps/emqx_bridge_kafka/docker-ct @@ -0,0 +1,2 @@ +toxiproxy +kafka diff --git a/apps/emqx_bridge_kafka/etc/emqx_bridge_kafka.conf b/apps/emqx_bridge_kafka/etc/emqx_bridge_kafka.conf new file mode 100644 index 000000000..e69de29bb diff --git a/apps/emqx_bridge_kafka/rebar.config b/apps/emqx_bridge_kafka/rebar.config new file mode 100644 index 000000000..fd21fd15b --- /dev/null +++ b/apps/emqx_bridge_kafka/rebar.config @@ -0,0 +1,14 @@ +%% -*- mode: erlang; -*- +{erl_opts, [debug_info]}. +{deps, [ {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "1.7.5"}}} + , {kafka_protocol, {git, "https://github.com/kafka4beam/kafka_protocol.git", {tag, "4.1.2"}}} + , {brod_gssapi, {git, "https://github.com/kafka4beam/brod_gssapi.git", {tag, "v0.1.0-rc1"}}} + , {brod, {git, "https://github.com/kafka4beam/brod.git", {tag, "3.16.8"}}} + , {emqx_connector, {path, "../../apps/emqx_connector"}} + , {emqx_resource, {path, "../../apps/emqx_resource"}} + , {emqx_bridge, {path, "../../apps/emqx_bridge"}} + ]}. + +{shell, [ + {apps, [emqx_bridge_kafka]} +]}. diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src new file mode 100644 index 000000000..a4fbe5673 --- /dev/null +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src @@ -0,0 +1,16 @@ +{application, emqx_bridge_kafka, [ + {description, "EMQX Enterprise Kafka Bridge"}, + {vsn, "0.1.0"}, + {registered, [emqx_bridge_kafka_consumer_sup]}, + {applications, [ + kernel, + stdlib, + telemetry, + wolff, + brod + ]}, + {env, []}, + {modules, []}, + + {links, []} +]}. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl similarity index 99% rename from lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl rename to apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl index f3dfa5964..30f6cd60d 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl @@ -1,7 +1,7 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_bridge_kafka). +-module(emqx_bridge_kafka). -include_lib("emqx_connector/include/emqx_connector.hrl"). -include_lib("typerefl/include/types.hrl"). diff --git a/lib-ee/emqx_ee_bridge/src/kafka/emqx_ee_bridge_kafka_consumer_sup.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_consumer_sup.erl similarity index 98% rename from lib-ee/emqx_ee_bridge/src/kafka/emqx_ee_bridge_kafka_consumer_sup.erl rename to apps/emqx_bridge_kafka/src/emqx_bridge_kafka_consumer_sup.erl index feec8c09b..638c1def6 100644 --- a/lib-ee/emqx_ee_bridge/src/kafka/emqx_ee_bridge_kafka_consumer_sup.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_consumer_sup.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_bridge_kafka_consumer_sup). +-module(emqx_bridge_kafka_consumer_sup). -behaviour(supervisor). diff --git a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl.erl similarity index 97% rename from lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka.erl rename to apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl.erl index c9dcce9a2..22a67c551 100644 --- a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl.erl @@ -3,7 +3,7 @@ %%-------------------------------------------------------------------- %% Kafka connection configuration --module(emqx_bridge_impl_kafka). +-module(emqx_bridge_kafka_impl). -export([ hosts/1, diff --git a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl similarity index 94% rename from lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl rename to apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl index f4dc3456e..fdfa3300c 100644 --- a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl @@ -1,7 +1,7 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_bridge_impl_kafka_consumer). +-module(emqx_bridge_kafka_impl_consumer). -behaviour(emqx_resource). @@ -52,8 +52,9 @@ ssl := _, any() => term() }. --type subscriber_id() :: emqx_ee_bridge_kafka_consumer_sup:child_id(). +-type subscriber_id() :: emqx_bridge_kafka_consumer_sup:child_id(). -type kafka_topic() :: brod:topic(). +-type kafka_message() :: #kafka_message{}. -type state() :: #{ kafka_topics := nonempty_list(kafka_topic()), subscriber_id := subscriber_id(), @@ -129,14 +130,14 @@ on_start(InstanceId, Config) -> ssl := SSL, topic_mapping := _ } = Config, - BootstrapHosts = emqx_bridge_impl_kafka:hosts(BootstrapHosts0), + BootstrapHosts = emqx_bridge_kafka_impl:hosts(BootstrapHosts0), KafkaType = kafka_consumer, %% Note: this is distinct per node. ClientID = make_client_id(InstanceId, KafkaType, BridgeName), ClientOpts0 = case Auth of none -> []; - Auth -> [{sasl, emqx_bridge_impl_kafka:sasl(Auth)}] + Auth -> [{sasl, emqx_bridge_kafka_impl:sasl(Auth)}] end, ClientOpts = add_ssl_opts(ClientOpts0, SSL), case brod:start_client(BootstrapHosts, ClientID, ClientOpts) of @@ -155,7 +156,7 @@ on_start(InstanceId, Config) -> msg => "failed_to_start_kafka_consumer_client", instance_id => InstanceId, kafka_hosts => BootstrapHosts, - reason => emqx_misc:redact(Reason) + reason => emqx_utils:redact(Reason) }), throw(?CLIENT_DOWN_MESSAGE) end, @@ -191,7 +192,7 @@ init(GroupData, State0) -> State = State0#{kafka_topic => KafkaTopic}, {ok, State}. --spec handle_message(#kafka_message{}, consumer_state()) -> {ok, commit, consumer_state()}. +-spec handle_message(kafka_message(), consumer_state()) -> {ok, commit, consumer_state()}. handle_message(Message, State) -> ?tp_span( kafka_consumer_handle_message, @@ -240,13 +241,13 @@ add_ssl_opts(ClientOpts, #{enable := false}) -> add_ssl_opts(ClientOpts, SSL) -> [{ssl, emqx_tls_lib:to_client_opts(SSL)} | ClientOpts]. --spec make_subscriber_id(atom() | binary()) -> emqx_ee_bridge_kafka_consumer_sup:child_id(). +-spec make_subscriber_id(atom() | binary()) -> emqx_bridge_kafka_consumer_sup:child_id(). make_subscriber_id(BridgeName) -> BridgeNameBin = to_bin(BridgeName), <<"kafka_subscriber:", BridgeNameBin/binary>>. ensure_consumer_supervisor_started() -> - Mod = emqx_ee_bridge_kafka_consumer_sup, + Mod = emqx_bridge_kafka_consumer_sup, ChildSpec = #{ id => Mod, @@ -327,7 +328,7 @@ start_consumer(Config, InstanceId, ClientID) -> %% spawns one worker for each assigned topic-partition %% automatically, so we should not spawn duplicate workers. SubscriberId = make_subscriber_id(BridgeName), - case emqx_ee_bridge_kafka_consumer_sup:start_child(SubscriberId, GroupSubscriberConfig) of + case emqx_bridge_kafka_consumer_sup:start_child(SubscriberId, GroupSubscriberConfig) of {ok, _ConsumerPid} -> ?tp( kafka_consumer_subscriber_started, @@ -342,18 +343,18 @@ start_consumer(Config, InstanceId, ClientID) -> ?SLOG(error, #{ msg => "failed_to_start_kafka_consumer", instance_id => InstanceId, - kafka_hosts => emqx_bridge_impl_kafka:hosts(BootstrapHosts0), - reason => emqx_misc:redact(Reason2) + kafka_hosts => emqx_bridge_kafka_impl:hosts(BootstrapHosts0), + reason => emqx_utils:redact(Reason2) }), stop_client(ClientID), throw(failed_to_start_kafka_consumer) end. --spec stop_subscriber(emqx_ee_bridge_kafka_consumer_sup:child_id()) -> ok. +-spec stop_subscriber(emqx_bridge_kafka_consumer_sup:child_id()) -> ok. stop_subscriber(SubscriberId) -> _ = log_when_error( fun() -> - emqx_ee_bridge_kafka_consumer_sup:ensure_child_deleted(SubscriberId) + emqx_bridge_kafka_consumer_sup:ensure_child_deleted(SubscriberId) end, #{ msg => "failed_to_delete_kafka_subscriber", @@ -437,7 +438,7 @@ do_get_status1(ClientID, KafkaTopic, SubscriberId, NPartitions) -> end. are_subscriber_workers_alive(SubscriberId) -> - Children = supervisor:which_children(emqx_ee_bridge_kafka_consumer_sup), + Children = supervisor:which_children(emqx_bridge_kafka_consumer_sup), case lists:keyfind(SubscriberId, 1, Children) of false -> false; @@ -479,7 +480,7 @@ is_dry_run(InstanceId) -> make_client_id(InstanceId, KafkaType, KafkaName) -> case is_dry_run(InstanceId) of false -> - ClientID0 = emqx_bridge_impl_kafka:make_client_id(KafkaType, KafkaName), + ClientID0 = emqx_bridge_kafka_impl:make_client_id(KafkaType, KafkaName), binary_to_atom(ClientID0); true -> %% It is a dry run and we don't want to leak too many diff --git a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl similarity index 98% rename from lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl rename to apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl index 09713a431..7bee2c70d 100644 --- a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl @@ -1,7 +1,7 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_bridge_impl_kafka_producer). +-module(emqx_bridge_kafka_impl_producer). -include_lib("emqx_resource/include/emqx_resource.hrl"). @@ -47,15 +47,15 @@ on_start(InstId, Config) -> BridgeType = ?BRIDGE_TYPE, ResourceId = emqx_bridge_resource:resource_id(BridgeType, BridgeName), _ = maybe_install_wolff_telemetry_handlers(ResourceId), - Hosts = emqx_bridge_impl_kafka:hosts(Hosts0), - ClientId = emqx_bridge_impl_kafka:make_client_id(BridgeType, BridgeName), + Hosts = emqx_bridge_kafka_impl:hosts(Hosts0), + ClientId = emqx_bridge_kafka_impl:make_client_id(BridgeType, BridgeName), ClientConfig = #{ min_metadata_refresh_interval => MinMetaRefreshInterval, connect_timeout => ConnTimeout, client_id => ClientId, request_timeout => MetaReqTimeout, extra_sock_opts => socket_opts(SocketOpts), - sasl => emqx_bridge_impl_kafka:sasl(Auth), + sasl => emqx_bridge_kafka_impl:sasl(Auth), ssl => ssl(SSL) }, case wolff:ensure_supervised_client(ClientId, Hosts, ClientConfig) of diff --git a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl similarity index 96% rename from lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl rename to apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl index 4019a9c42..08fbf5e15 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl +++ b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl @@ -1,7 +1,7 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_bridge_impl_kafka_consumer_SUITE). +-module(emqx_bridge_kafka_impl_consumer_SUITE). -compile(nowarn_export_all). -compile(export_all). @@ -15,6 +15,7 @@ -import(emqx_common_test_helpers, [on_exit/1]). -define(BRIDGE_TYPE_BIN, <<"kafka_consumer">>). +-define(APPS, [emqx_bridge, emqx_resource, emqx_rule_engine, emqx_bridge_kafka]). %%------------------------------------------------------------------------------ %% CT boilerplate @@ -67,7 +68,7 @@ init_per_suite(Config) -> end_per_suite(_Config) -> emqx_mgmt_api_test_util:end_suite(), ok = emqx_common_test_helpers:stop_apps([emqx_conf]), - ok = emqx_connector_test_helpers:stop_apps([emqx_bridge, emqx_resource, emqx_rule_engine]), + ok = emqx_connector_test_helpers:stop_apps(lists:reverse(?APPS)), _ = application:stop(emqx_connector), ok. @@ -228,7 +229,7 @@ common_init_per_group() -> emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), application:load(emqx_bridge), ok = emqx_common_test_helpers:start_apps([emqx_conf]), - ok = emqx_connector_test_helpers:start_apps([emqx_resource, emqx_bridge, emqx_rule_engine]), + ok = emqx_connector_test_helpers:start_apps(?APPS), {ok, _} = application:ensure_all_started(emqx_connector), emqx_mgmt_api_test_util:init_suite(), UniqueNum = integer_to_binary(erlang:unique_integer()), @@ -298,7 +299,7 @@ init_per_testcase(TestCase, Config) when common_init_per_testcase(TestCase, Config) end; init_per_testcase(t_cluster_group = TestCase, Config0) -> - Config = emqx_misc:merge_opts(Config0, [{num_partitions, 6}]), + Config = emqx_utils:merge_opts(Config0, [{num_partitions, 6}]), common_init_per_testcase(TestCase, Config); init_per_testcase(t_multiple_topic_mappings = TestCase, Config0) -> KafkaTopicBase = @@ -408,7 +409,7 @@ start_producers(TestCase, Config) -> DirectKafkaPort = ?config(direct_kafka_port, Config), UseTLS = ?config(use_tls, Config), UseSASL = ?config(use_sasl, Config), - Hosts = emqx_bridge_impl_kafka:hosts( + Hosts = emqx_bridge_kafka_impl:hosts( DirectKafkaHost ++ ":" ++ integer_to_list(DirectKafkaPort) ), SSL = @@ -672,7 +673,7 @@ create_bridge(Config, Overrides) -> Type = ?BRIDGE_TYPE_BIN, Name = ?config(kafka_name, Config), KafkaConfig0 = ?config(kafka_config, Config), - KafkaConfig = emqx_map_lib:deep_merge(KafkaConfig0, Overrides), + KafkaConfig = emqx_utils_maps:deep_merge(KafkaConfig0, Overrides), emqx_bridge:create(Type, Name, KafkaConfig). delete_bridge(Config) -> @@ -695,7 +696,7 @@ create_bridge_api(Config, Overrides) -> TypeBin = ?BRIDGE_TYPE_BIN, Name = ?config(kafka_name, Config), KafkaConfig0 = ?config(kafka_config, Config), - KafkaConfig = emqx_map_lib:deep_merge(KafkaConfig0, Overrides), + KafkaConfig = emqx_utils_maps:deep_merge(KafkaConfig0, Overrides), Params = KafkaConfig#{<<"type">> => TypeBin, <<"name">> => Name}, Path = emqx_mgmt_api_test_util:api_path(["bridges"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), @@ -704,7 +705,7 @@ create_bridge_api(Config, Overrides) -> Res = case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params, Opts) of {ok, {Status, Headers, Body0}} -> - {ok, {Status, Headers, emqx_json:decode(Body0, [return_maps])}}; + {ok, {Status, Headers, emqx_utils_json:decode(Body0, [return_maps])}}; Error -> Error end, @@ -718,7 +719,7 @@ update_bridge_api(Config, Overrides) -> TypeBin = ?BRIDGE_TYPE_BIN, Name = ?config(kafka_name, Config), KafkaConfig0 = ?config(kafka_config, Config), - KafkaConfig = emqx_map_lib:deep_merge(KafkaConfig0, Overrides), + KafkaConfig = emqx_utils_maps:deep_merge(KafkaConfig0, Overrides), BridgeId = emqx_bridge_resource:bridge_id(TypeBin, Name), Params = KafkaConfig#{<<"type">> => TypeBin, <<"name">> => Name}, Path = emqx_mgmt_api_test_util:api_path(["bridges", BridgeId]), @@ -727,7 +728,7 @@ update_bridge_api(Config, Overrides) -> ct:pal("updating bridge (via http): ~p", [Params]), Res = case emqx_mgmt_api_test_util:request_api(put, Path, "", AuthHeader, Params, Opts) of - {ok, {_Status, _Headers, Body0}} -> {ok, emqx_json:decode(Body0, [return_maps])}; + {ok, {_Status, _Headers, Body0}} -> {ok, emqx_utils_json:decode(Body0, [return_maps])}; Error -> Error end, ct:pal("bridge update result: ~p", [Res]), @@ -775,7 +776,7 @@ do_wait_for_expected_published_messages(Messages, Acc, _Timeout) when map_size(M do_wait_for_expected_published_messages(Messages0, Acc0, Timeout) -> receive {publish, Msg0 = #{payload := Payload}} -> - case emqx_json:safe_decode(Payload, [return_maps]) of + case emqx_utils_json:safe_decode(Payload, [return_maps]) of {error, _} -> ct:pal("unexpected message: ~p; discarding", [Msg0]), do_wait_for_expected_published_messages(Messages0, Acc0, Timeout); @@ -876,7 +877,7 @@ ensure_connected(Config) -> consumer_clientid(Config) -> KafkaName = ?config(kafka_name, Config), - binary_to_atom(emqx_bridge_impl_kafka:make_client_id(kafka_consumer, KafkaName)). + binary_to_atom(emqx_bridge_kafka_impl:make_client_id(kafka_consumer, KafkaName)). get_client_connection(Config) -> KafkaHost = ?config(kafka_host, Config), @@ -885,7 +886,7 @@ get_client_connection(Config) -> brod_client:get_connection(ClientID, KafkaHost, KafkaPort). get_subscriber_workers() -> - [{_, SubscriberPid, _, _}] = supervisor:which_children(emqx_ee_bridge_kafka_consumer_sup), + [{_, SubscriberPid, _, _}] = supervisor:which_children(emqx_bridge_kafka_consumer_sup), brod_group_subscriber_v2:get_workers(SubscriberPid). wait_downs(Refs, _Timeout) when map_size(Refs) =:= 0 -> @@ -927,7 +928,7 @@ create_rule_and_action_http(Config) -> AuthHeader = emqx_mgmt_api_test_util:auth_header_(), ct:pal("rule action params: ~p", [Params]), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. @@ -1069,7 +1070,7 @@ cluster(Config) -> Cluster = emqx_common_test_helpers:emqx_cluster( [core, core], [ - {apps, [emqx_conf, emqx_bridge, emqx_rule_engine]}, + {apps, [emqx_conf, emqx_bridge, emqx_rule_engine, emqx_bridge_kafka]}, {listener_ports, []}, {peer_mod, PeerModule}, {priv_data_dir, PrivDataDir}, @@ -1187,7 +1188,7 @@ t_start_and_consume_ok(Config) -> <<"offset">> := OffsetReply, <<"headers">> := #{<<"hkey">> := <<"hvalue">>} }, - emqx_json:decode(PayloadBin, [return_maps]), + emqx_utils_json:decode(PayloadBin, [return_maps]), #{ offset_reply => OffsetReply, kafka_topic => KafkaTopic, @@ -1299,7 +1300,7 @@ t_multiple_topic_mappings(Config) -> %% as configured. Payloads = lists:sort([ - case emqx_json:safe_decode(P, [return_maps]) of + case emqx_utils_json:safe_decode(P, [return_maps]) of {ok, Decoded} -> Decoded; {error, _} -> P end @@ -1440,7 +1441,7 @@ do_t_failed_creation_then_fixed(Config) -> <<"offset">> := _, <<"headers">> := #{<<"hkey">> := <<"hvalue">>} }, - emqx_json:decode(PayloadBin, [return_maps]), + emqx_utils_json:decode(PayloadBin, [return_maps]), #{ kafka_topic => KafkaTopic, payload => Payload @@ -1504,7 +1505,7 @@ do_t_receive_after_recovery(Config) -> _Interval = 500, _NAttempts = 20, begin - GroupId = emqx_bridge_impl_kafka_consumer:consumer_group_id(KafkaNameA), + GroupId = emqx_bridge_kafka_impl_consumer:consumer_group_id(KafkaNameA), {ok, [#{partitions := Partitions}]} = brod:fetch_committed_offsets( KafkaClientId, GroupId ), @@ -1542,7 +1543,7 @@ do_t_receive_after_recovery(Config) -> %% 2) publish messages while the consumer is down. %% we use `pmap' to avoid wolff sending the whole %% batch to a single partition. - emqx_misc:pmap(fun(Msg) -> publish(Config, [Msg]) end, Messages1), + emqx_utils:pmap(fun(Msg) -> publish(Config, [Msg]) end, Messages1), ok end), %% 3) restore and consume messages @@ -1635,7 +1636,7 @@ t_bridge_rule_action_source(Config) -> <<"headers">> := #{<<"hkey">> := <<"hvalue">>}, <<"topic">> := KafkaTopic }, - emqx_json:decode(RawPayload, [return_maps]) + emqx_utils_json:decode(RawPayload, [return_maps]) ), ?retry( _Interval = 200, @@ -1666,7 +1667,7 @@ t_cluster_group(Config) -> || {Name, Opts} <- Cluster ], on_exit(fun() -> - emqx_misc:pmap( + emqx_utils:pmap( fun(N) -> ct:pal("stopping ~p", [N]), ok = emqx_common_test_helpers:stop_slave(N) @@ -1745,8 +1746,12 @@ t_node_joins_existing_cluster(Config) -> ?check_trace( begin [{Name1, Opts1}, {Name2, Opts2} | _] = Cluster, + ct:pal("starting ~p", [Name1]), N1 = emqx_common_test_helpers:start_slave(Name1, Opts1), - on_exit(fun() -> ok = emqx_common_test_helpers:stop_slave(N1) end), + on_exit(fun() -> + ct:pal("stopping ~p", [N1]), + ok = emqx_common_test_helpers:stop_slave(N1) + end), setup_group_subscriber_spy(N1), {{ok, _}, {ok, _}} = ?wait_async_action( @@ -1785,8 +1790,12 @@ t_node_joins_existing_cluster(Config) -> 1, 30_000 ), + ct:pal("starting ~p", [Name2]), N2 = emqx_common_test_helpers:start_slave(Name2, Opts2), - on_exit(fun() -> ok = emqx_common_test_helpers:stop_slave(N2) end), + on_exit(fun() -> + ct:pal("stopping ~p", [N2]), + ok = emqx_common_test_helpers:stop_slave(N2) + end), setup_group_subscriber_spy(N2), Nodes = [N1, N2], wait_for_cluster_rpc(N2), @@ -1873,11 +1882,14 @@ t_cluster_node_down(Config) -> Nodes = [N1, N2 | _] = lists:map( - fun({Name, Opts}) -> emqx_common_test_helpers:start_slave(Name, Opts) end, + fun({Name, Opts}) -> + ct:pal("starting ~p", [Name]), + emqx_common_test_helpers:start_slave(Name, Opts) + end, Cluster ), on_exit(fun() -> - emqx_misc:pmap( + emqx_utils:pmap( fun(N) -> ct:pal("stopping ~p", [N]), ok = emqx_common_test_helpers:stop_slave(N) @@ -1992,7 +2004,7 @@ t_begin_offset_earliest(Config) -> %% the consumers Published = receive_published(#{n => NumMessages}), Payloads = lists:map( - fun(#{payload := P}) -> emqx_json:decode(P, [return_maps]) end, + fun(#{payload := P}) -> emqx_utils_json:decode(P, [return_maps]) end, Published ), ?assert( diff --git a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl similarity index 97% rename from lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl rename to apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl index 9e32f818d..a2111b1a8 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl +++ b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_bridge_impl_kafka_producer_SUITE). +-module(emqx_bridge_kafka_impl_producer_SUITE). -compile(nowarn_export_all). -compile(export_all). @@ -12,7 +12,7 @@ -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -include_lib("brod/include/brod.hrl"). --define(PRODUCER, emqx_bridge_impl_kafka_producer). +-define(PRODUCER, emqx_bridge_kafka_impl_producer). %%------------------------------------------------------------------------------ %% Things for REST API tests @@ -41,6 +41,8 @@ %% to hocon; keeping this as just `kafka' for backwards compatibility. -define(BRIDGE_TYPE, "kafka"). +-define(APPS, [emqx_resource, emqx_bridge, emqx_rule_engine, emqx_bridge_kafka]). + %%------------------------------------------------------------------------------ %% CT boilerplate %%------------------------------------------------------------------------------ @@ -76,7 +78,7 @@ init_per_suite(Config) -> _ = emqx_ee_bridge:module_info(), application:load(emqx_bridge), ok = emqx_common_test_helpers:start_apps([emqx_conf]), - ok = emqx_connector_test_helpers:start_apps([emqx_resource, emqx_bridge, emqx_rule_engine]), + ok = emqx_connector_test_helpers:start_apps(?APPS), {ok, _} = application:ensure_all_started(emqx_connector), emqx_mgmt_api_test_util:init_suite(), wait_until_kafka_is_up(), @@ -96,7 +98,7 @@ init_per_suite(Config) -> end_per_suite(_Config) -> emqx_mgmt_api_test_util:end_suite(), ok = emqx_common_test_helpers:stop_apps([emqx_conf]), - ok = emqx_connector_test_helpers:stop_apps([emqx_bridge, emqx_resource, emqx_rule_engine]), + ok = emqx_connector_test_helpers:stop_apps(lists:reverse(?APPS)), _ = application:stop(emqx_connector), ok. @@ -309,7 +311,7 @@ kafka_bridge_rest_api_helper(Config) -> AtomsAfter = erlang:system_info(atom_count), ?assertEqual(AtomsBefore, AtomsAfter), %% Create a rule that uses the bridge - {ok, 201, _Rule} = http_post( + {ok, 201, Rule} = http_post( ["rules"], #{ <<"name">> => <<"kafka_bridge_rest_api_helper_rule">>, @@ -318,6 +320,7 @@ kafka_bridge_rest_api_helper(Config) -> <<"sql">> => <<"SELECT * from \"kafka_bridge_topic/#\"">> } ), + #{<<"id">> := RuleId} = emqx_utils_json:decode(Rule, [return_maps]), %% counters should be empty before ?assertEqual(0, emqx_resource_metrics:matched_get(ResourceId)), ?assertEqual(0, emqx_resource_metrics:success_get(ResourceId)), @@ -346,6 +349,8 @@ kafka_bridge_rest_api_helper(Config) -> %% Check crucial counters and gauges ?assertEqual(1, emqx_resource_metrics:matched_get(ResourceId)), ?assertEqual(1, emqx_resource_metrics:success_get(ResourceId)), + ?assertEqual(1, emqx_metrics_worker:get(rule_metrics, RuleId, 'actions.success')), + ?assertEqual(0, emqx_metrics_worker:get(rule_metrics, RuleId, 'actions.failed')), ?assertEqual(0, emqx_resource_metrics:dropped_get(ResourceId)), ?assertEqual(0, emqx_resource_metrics:failed_get(ResourceId)), ?assertEqual(0, emqx_resource_metrics:inflight_get(ResourceId)), @@ -797,7 +802,7 @@ api_path(Parts) -> ?HOST ++ filename:join([?BASE_PATH | Parts]). json(Data) -> - {ok, Jsx} = emqx_json:safe_decode(Data, [return_maps]), + {ok, Jsx} = emqx_utils_json:safe_decode(Data, [return_maps]), Jsx. delete_all_bridges() -> diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_kafka_tests.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_tests.erl similarity index 98% rename from lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_kafka_tests.erl rename to apps/emqx_bridge_kafka/test/emqx_bridge_kafka_tests.erl index 1b32f856d..b16df854f 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_kafka_tests.erl +++ b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_tests.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_bridge_kafka_tests). +-module(emqx_bridge_kafka_tests). -include_lib("eunit/include/eunit.hrl"). @@ -92,7 +92,7 @@ kafka_consumer_test() -> ), %% Bad: can't repeat kafka topics. - BadConf1 = emqx_map_lib:deep_put( + BadConf1 = emqx_utils_maps:deep_put( [<<"bridges">>, <<"kafka_consumer">>, <<"my_consumer">>, <<"topic_mapping">>], Conf1, [ @@ -121,7 +121,7 @@ kafka_consumer_test() -> ), %% Bad: there must be at least 1 mapping. - BadConf2 = emqx_map_lib:deep_put( + BadConf2 = emqx_utils_maps:deep_put( [<<"bridges">>, <<"kafka_consumer">>, <<"my_consumer">>, <<"topic_mapping">>], Conf1, [] diff --git a/apps/emqx_bridge_matrix/BSL.txt b/apps/emqx_bridge_matrix/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_bridge_matrix/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_bridge_matrix/README.md b/apps/emqx_bridge_matrix/README.md new file mode 100644 index 000000000..976120ffe --- /dev/null +++ b/apps/emqx_bridge_matrix/README.md @@ -0,0 +1,36 @@ +# EMQX MatrixDB Bridge + +[MatrixDB](http://matrixdb.univ-lyon1.fr/) is a biological database focused on +molecular interactions between extracellular proteins and polysaccharides. + +The application is used to connect EMQX and MatrixDB. +User can create a rule and easily ingest IoT data into MatrixDB by leveraging +[EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). + + +# Documentation + +- Refer to [Ingest data into MatrixDB](todo) + for how to use EMQX dashboard to ingest IoT data into MatrixDB. + +- Refer to [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) + for the EMQX rules engine introduction. + + +# HTTP APIs + +- Several APIs are provided for bridge management, which includes create bridge, + update bridge, get bridge, stop or restart bridge and list bridges etc. + + Refer to [API Docs - Bridges](https://docs.emqx.com/en/enterprise/v5.0/admin/api-docs.html#tag/Bridges) + for more detailed information. + + +# Contributing + +Please see our [contributing.md](../../CONTRIBUTING.md). + + +# License + +EMQ Business Source License 1.1, refer to [LICENSE](BSL.txt). diff --git a/apps/emqx_bridge_matrix/src/emqx_bridge_matrix.app.src b/apps/emqx_bridge_matrix/src/emqx_bridge_matrix.app.src new file mode 100644 index 000000000..e2a17e070 --- /dev/null +++ b/apps/emqx_bridge_matrix/src/emqx_bridge_matrix.app.src @@ -0,0 +1,9 @@ +{application, emqx_bridge_matrix, [ + {description, "EMQX Enterprise MatrixDB Bridge"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [kernel, stdlib]}, + {env, []}, + {modules, []}, + {links, []} +]}. diff --git a/apps/emqx_bridge_mongodb/BSL.txt b/apps/emqx_bridge_mongodb/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_bridge_mongodb/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_bridge_mongodb/README.md b/apps/emqx_bridge_mongodb/README.md new file mode 100644 index 000000000..088c8467f --- /dev/null +++ b/apps/emqx_bridge_mongodb/README.md @@ -0,0 +1,39 @@ +# EMQX MongoDB Bridge + +[MongoDB](https://github.com/mongodb/mongo) is a source-available cross-platform +document-oriented database. It is a NoSQL database that stores flexible JSON-like +documents for faster iteration and better data organization. +It provides high availability and scaling with its built-in replication and sharding +features, and is used in a variety of industries + +The application is used to connect EMQX and MongoDB. +User can create a rule and easily ingest IoT data into MongoDB by leveraging +[EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). + + +# Documentation + +- Refer to [Ingest data into MongoDB](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-mongodb.html) + for how to use EMQX dashboard to ingest IoT data into MongoDB. + +- Refer to [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) + for the EMQX rules engine introduction. + + +# HTTP APIs + +- Several APIs are provided for bridge management, which includes create bridge, + update bridge, get bridge, stop or restart bridge and list bridges etc. + + Refer to [API Docs - Bridges](https://docs.emqx.com/en/enterprise/v5.0/admin/api-docs.html#tag/Bridges) + for more detailed information. + + +# Contributing + +Please see our [contributing.md](../../CONTRIBUTING.md). + + +# License + +EMQ Business Source License 1.1, refer to [LICENSE](BSL.txt). diff --git a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src new file mode 100644 index 000000000..008a9e164 --- /dev/null +++ b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src @@ -0,0 +1,9 @@ +{application, emqx_bridge_mongodb, [ + {description, "EMQX Enterprise MongoDB Bridge"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [kernel, stdlib]}, + {env, []}, + {modules, []}, + {links, []} +]}. diff --git a/apps/emqx_bridge_mysql/BSL.txt b/apps/emqx_bridge_mysql/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_bridge_mysql/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_bridge_mysql/README.md b/apps/emqx_bridge_mysql/README.md new file mode 100644 index 000000000..73f6987b6 --- /dev/null +++ b/apps/emqx_bridge_mysql/README.md @@ -0,0 +1,36 @@ +# EMQX MySQL Bridge + +[MySQL](https://github.com/MySQL/MySQL) is a popular open-source relational database +management system. + +The application is used to connect EMQX and MySQL. +User can create a rule and easily ingest IoT data into MySQL by leveraging +[EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). + + +# Documentation + +- Refer to [Ingest data into MySQL](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-mysql.html) + for how to use EMQX dashboard to ingest IoT data into MySQL. + +- Refer to [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) + for the EMQX rules engine introduction. + + +# HTTP APIs + +- Several APIs are provided for bridge management, which includes create bridge, + update bridge, get bridge, stop or restart bridge and list bridges etc. + + Refer to [API Docs - Bridges](https://docs.emqx.com/en/enterprise/v5.0/admin/api-docs.html#tag/Bridges) + for more detailed information. + + +# Contributing + +Please see our [contributing.md](../../CONTRIBUTING.md). + + +# License + +EMQ Business Source License 1.1, refer to [LICENSE](BSL.txt). diff --git a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src new file mode 100644 index 000000000..2e36587a7 --- /dev/null +++ b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src @@ -0,0 +1,9 @@ +{application, emqx_bridge_mysql, [ + {description, "EMQX Enterprise MySQL Bridge"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [kernel, stdlib]}, + {env, []}, + {modules, []}, + {links, []} +]}. diff --git a/apps/emqx_bridge_pgsql/BSL.txt b/apps/emqx_bridge_pgsql/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_bridge_pgsql/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_bridge_pgsql/README.md b/apps/emqx_bridge_pgsql/README.md new file mode 100644 index 000000000..fc0bd3c6f --- /dev/null +++ b/apps/emqx_bridge_pgsql/README.md @@ -0,0 +1,38 @@ +# EMQX PostgreSQL Bridge + +[PostgreSQL](https://github.com/PostgreSQL/PostgreSQL) is an open-source relational +database management system (RDBMS) that uses and extends the SQL language. +It is known for its reliability, data integrity, and advanced features such as +support for JSON, XML, and other data formats. + +The application is used to connect EMQX and PostgreSQL. +User can create a rule and easily ingest IoT data into PostgreSQL by leveraging +[EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). + + +# Documentation + +- Refer to [Ingest data into PostgreSQL](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-pgsql.html) + for how to use EMQX dashboard to ingest IoT data into PostgreSQL. + +- Refer to [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) + for the EMQX rules engine introduction. + + +# HTTP APIs + +- Several APIs are provided for bridge management, which includes create bridge, + update bridge, get bridge, stop or restart bridge and list bridges etc. + + Refer to [API Docs - Bridges](https://docs.emqx.com/en/enterprise/v5.0/admin/api-docs.html#tag/Bridges) + for more detailed information. + + +# Contributing + +Please see our [contributing.md](../../CONTRIBUTING.md). + + +# License + +EMQ Business Source License 1.1, refer to [LICENSE](BSL.txt). diff --git a/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.app.src b/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.app.src new file mode 100644 index 000000000..c695283f3 --- /dev/null +++ b/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.app.src @@ -0,0 +1,9 @@ +{application, emqx_bridge_pgsql, [ + {description, "EMQX Enterprise PostgreSQL Bridge"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [kernel, stdlib]}, + {env, []}, + {modules, []}, + {links, []} +]}. diff --git a/apps/emqx_bridge_redis/BSL.txt b/apps/emqx_bridge_redis/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_bridge_redis/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_bridge_redis/README.md b/apps/emqx_bridge_redis/README.md new file mode 100644 index 000000000..73ec41f07 --- /dev/null +++ b/apps/emqx_bridge_redis/README.md @@ -0,0 +1,37 @@ +# EMQX Redis Bridge + +[Redis](https://github.com/redis/redis) is an in-memory data structure store, +used as a distributed, in-memory key–value database, cache and message broker, +with optional durability. + +The application is used to connect EMQX and Redis. +User can create a rule and easily ingest IoT data into Redis by leveraging +[EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). + + +# Documentation + +- Refer to [Ingest data into Redis](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-redis.html) + for how to use EMQX dashboard to ingest IoT data into Redis. + +- Refer to [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) + for the EMQX rules engine introduction. + + +# HTTP APIs + +- Several APIs are provided for bridge management, which includes create bridge, + update bridge, get bridge, stop or restart bridge and list bridges etc. + + Refer to [API Docs - Bridges](https://docs.emqx.com/en/enterprise/v5.0/admin/api-docs.html#tag/Bridges) + for more detailed information. + + +# Contributing + +Please see our [contributing.md](../../CONTRIBUTING.md). + + +# License + +EMQ Business Source License 1.1, refer to [LICENSE](BSL.txt). diff --git a/apps/emqx_bridge_redis/src/emqx_bridge_redis.app.src b/apps/emqx_bridge_redis/src/emqx_bridge_redis.app.src new file mode 100644 index 000000000..6b57c6cd7 --- /dev/null +++ b/apps/emqx_bridge_redis/src/emqx_bridge_redis.app.src @@ -0,0 +1,9 @@ +{application, emqx_bridge_redis, [ + {description, "EMQX Enterprise Redis Bridge"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [kernel, stdlib]}, + {env, []}, + {modules, []}, + {links, []} +]}. diff --git a/apps/emqx_bridge_rocketmq/BSL.txt b/apps/emqx_bridge_rocketmq/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_bridge_rocketmq/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_bridge_rocketmq/README.md b/apps/emqx_bridge_rocketmq/README.md new file mode 100644 index 000000000..252e6beac --- /dev/null +++ b/apps/emqx_bridge_rocketmq/README.md @@ -0,0 +1,37 @@ +# EMQX RocketMQ Bridge + +[RocketMQ](https://github.com/apache/rocketmq) is a distributed messaging and +streaming platform developed by the Apache Software Foundation. +It provides reliable, scalable, and high-throughput messaging services for modern cloud-native applications + +The application is used to connect EMQX and RocketMQ. +User can create a rule and easily ingest IoT data into RocketMQ by leveraging +[EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). + + +# Documentation + +- Refer to [Ingest data into RocketMQ](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-rocketmq.html) + for how to use EMQX dashboard to ingest IoT data into RocketMQ. + +- Refer to [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) + for the EMQX rules engine introduction. + + +# HTTP APIs + +- Several APIs are provided for bridge management, which includes create bridge, + update bridge, get bridge, stop or restart bridge and list bridges etc. + + Refer to [API Docs - Bridges](https://docs.emqx.com/en/enterprise/v5.0/admin/api-docs.html#tag/Bridges) + for more detailed information. + + +# Contributing + +Please see our [contributing.md](../../CONTRIBUTING.md). + + +# License + +EMQ Business Source License 1.1, refer to [LICENSE](BSL.txt). diff --git a/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src new file mode 100644 index 000000000..e1916034c --- /dev/null +++ b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src @@ -0,0 +1,9 @@ +{application, emqx_bridge_rocketmq, [ + {description, "EMQX Enterprise RocketMQ Bridge"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [kernel, stdlib]}, + {env, []}, + {modules, []}, + {links, []} +]}. diff --git a/apps/emqx_bridge_tdengine/BSL.txt b/apps/emqx_bridge_tdengine/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_bridge_tdengine/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_bridge_tdengine/README.md b/apps/emqx_bridge_tdengine/README.md new file mode 100644 index 000000000..25faf4c14 --- /dev/null +++ b/apps/emqx_bridge_tdengine/README.md @@ -0,0 +1,39 @@ +# EMQX TDEngine Bridge + +[TDEngine](https://github.com/taosdata/TDengine) is an open-source, cloud-native +time series database (TSDB) optimized for Internet of Things (IoT), Connected Cars, +and Industrial IoT. +It enables efficient, real-time ingestion, processing, and monitoring of petabytes +of data per day, generated by billions of sensors and data collectors. + +The application is used to connect EMQX and TDEngine. +User can create a rule and easily ingest IoT data into TDEngine by leveraging +[EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). + + +# Documentation + +- Refer to [Ingest data into TDEngine](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-tdengine.html) + for how to use EMQX dashboard to ingest IoT data into TDEngine. + +- Refer to [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) + for the EMQX rules engine introduction. + + +# HTTP APIs + +- Several APIs are provided for bridge management, which includes create bridge, + update bridge, get bridge, stop or restart bridge and list bridges etc. + + Refer to [API Docs - Bridges](https://docs.emqx.com/en/enterprise/v5.0/admin/api-docs.html#tag/Bridges) + for more detailed information. + + +# Contributing + +Please see our [contributing.md](../../CONTRIBUTING.md). + + +# License + +EMQ Business Source License 1.1, refer to [LICENSE](BSL.txt). diff --git a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src new file mode 100644 index 000000000..05e8a6f9f --- /dev/null +++ b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src @@ -0,0 +1,9 @@ +{application, emqx_bridge_tdengine, [ + {description, "EMQX Enterprise TDEngine Bridge"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [kernel, stdlib]}, + {env, []}, + {modules, []}, + {links, []} +]}. diff --git a/apps/emqx_bridge_timescale/BSL.txt b/apps/emqx_bridge_timescale/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_bridge_timescale/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_bridge_timescale/README.md b/apps/emqx_bridge_timescale/README.md new file mode 100644 index 000000000..96f70f847 --- /dev/null +++ b/apps/emqx_bridge_timescale/README.md @@ -0,0 +1,38 @@ +# EMQX TimescaleDB Bridge + +[TimescaleDB](https://github.com/timescaleDB/timescaleDB) is an open-source database +designed to make SQL scalable for time-series data. +It is engineered up from PostgreSQL and packaged as a PostgreSQL extension, +providing automatic partitioning across time and space (partitioning key), as well as full SQL support. + +The application is used to connect EMQX and TimescaleDB. +User can create a rule and easily ingest IoT data into TimescaleDB by leveraging +[EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). + + +# Documentation + +- Refer to [Ingest data into TimescaleDB](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-timescaledb.html) + for how to use EMQX dashboard to ingest IoT data into TimescaleDB. + +- Refer to [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) + for the EMQX rules engine introduction. + + +# HTTP APIs + +- Several APIs are provided for bridge management, which includes create bridge, + update bridge, get bridge, stop or restart bridge and list bridges etc. + + Refer to [API Docs - Bridges](https://docs.emqx.com/en/enterprise/v5.0/admin/api-docs.html#tag/Bridges) + for more detailed information. + + +# Contributing + +Please see our [contributing.md](../../CONTRIBUTING.md). + + +# License + +EMQ Business Source License 1.1, refer to [LICENSE](BSL.txt). diff --git a/apps/emqx_bridge_timescale/src/emqx_bridge_timescale.app.src b/apps/emqx_bridge_timescale/src/emqx_bridge_timescale.app.src new file mode 100644 index 000000000..5b4431f73 --- /dev/null +++ b/apps/emqx_bridge_timescale/src/emqx_bridge_timescale.app.src @@ -0,0 +1,9 @@ +{application, emqx_bridge_timescale, [ + {description, "EMQX Enterprise TimescaleDB Bridge"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [kernel, stdlib]}, + {env, []}, + {modules, []}, + {links, []} +]}. diff --git a/apps/emqx_conf/etc/emqx_conf.conf b/apps/emqx_conf/etc/emqx_conf.conf index 86147bf25..a54894dcd 100644 --- a/apps/emqx_conf/etc/emqx_conf.conf +++ b/apps/emqx_conf/etc/emqx_conf.conf @@ -1,12 +1,7 @@ ## NOTE: -## Configs in this file might be overridden by: -## 1. Environment variables which start with 'EMQX_' prefix -## 2. File $EMQX_NODE__DATA_DIR/configs/cluster-override.conf -## 3. File $EMQX_NODE__DATA_DIR/configs/local-override.conf -## -## The *-override.conf files are overwritten at runtime when changes -## are made from EMQX dashboard UI, management HTTP API, or CLI. -## All configuration details can be found in emqx.conf.example +## The EMQX configuration is prioritized (overlayed) in the following order: +## `data/configs/cluster.hocon < etc/emqx.conf < environment variables`. + node { name = "emqx@127.0.0.1" diff --git a/apps/emqx_conf/src/emqx_cluster_rpc.erl b/apps/emqx_conf/src/emqx_cluster_rpc.erl index 0382045d4..c82191bc3 100644 --- a/apps/emqx_conf/src/emqx_cluster_rpc.erl +++ b/apps/emqx_conf/src/emqx_cluster_rpc.erl @@ -501,15 +501,17 @@ log_and_alarm(IsSuccess, Res, #{kind := ?APPLY_KIND_INITIATE} = Meta) -> %% because nothing is committed case IsSuccess of true -> - ?SLOG(debug, Meta#{msg => "cluster_rpc_apply_result", result => emqx_misc:redact(Res)}); + ?SLOG(debug, Meta#{msg => "cluster_rpc_apply_result", result => emqx_utils:redact(Res)}); false -> - ?SLOG(warning, Meta#{msg => "cluster_rpc_apply_result", result => emqx_misc:redact(Res)}) + ?SLOG(warning, Meta#{ + msg => "cluster_rpc_apply_result", result => emqx_utils:redact(Res) + }) end; log_and_alarm(true, Res, Meta) -> - ?SLOG(debug, Meta#{msg => "cluster_rpc_apply_ok", result => emqx_misc:redact(Res)}), + ?SLOG(debug, Meta#{msg => "cluster_rpc_apply_ok", result => emqx_utils:redact(Res)}), do_alarm(deactivate, Res, Meta); log_and_alarm(false, Res, Meta) -> - ?SLOG(error, Meta#{msg => "cluster_rpc_apply_failed", result => emqx_misc:redact(Res)}), + ?SLOG(error, Meta#{msg => "cluster_rpc_apply_failed", result => emqx_utils:redact(Res)}), do_alarm(activate, Res, Meta). do_alarm(Fun, Res, #{tnx_id := Id} = Meta) -> diff --git a/apps/emqx_conf/src/emqx_cluster_rpc_cleaner.erl b/apps/emqx_conf/src/emqx_cluster_rpc_cleaner.erl index bce866c2d..fe72cd65b 100644 --- a/apps/emqx_conf/src/emqx_cluster_rpc_cleaner.erl +++ b/apps/emqx_conf/src/emqx_cluster_rpc_cleaner.erl @@ -73,7 +73,7 @@ handle_info(Info, State) -> {noreply, State}. terminate(_Reason, #{timer := TRef}) -> - emqx_misc:cancel_timer(TRef). + emqx_utils:cancel_timer(TRef). code_change(_OldVsn, State, _Extra) -> {ok, State}. @@ -82,7 +82,7 @@ code_change(_OldVsn, State, _Extra) -> %% Internal functions %%-------------------------------------------------------------------- ensure_timer(State = #{cleanup_ms := Ms}) -> - State#{timer := emqx_misc:start_timer(Ms, del_stale_mfa)}. + State#{timer := emqx_utils:start_timer(Ms, del_stale_mfa)}. %% @doc Keep the latest completed 100 records for querying and troubleshooting. del_stale_mfa(MaxHistory) -> diff --git a/apps/emqx_conf/src/emqx_conf.app.src b/apps/emqx_conf/src/emqx_conf.app.src index 37707431a..234690374 100644 --- a/apps/emqx_conf/src/emqx_conf.app.src +++ b/apps/emqx_conf/src/emqx_conf.app.src @@ -1,6 +1,6 @@ {application, emqx_conf, [ {description, "EMQX configuration management"}, - {vsn, "0.1.15"}, + {vsn, "0.1.17"}, {registered, []}, {mod, {emqx_conf_app, []}}, {applications, [kernel, stdlib, emqx_ctl]}, diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl index d03cf9c27..8632df139 100644 --- a/apps/emqx_conf/src/emqx_conf.erl +++ b/apps/emqx_conf/src/emqx_conf.erl @@ -25,9 +25,15 @@ -export([update/3, update/4]). -export([remove/2, remove/3]). -export([reset/2, reset/3]). --export([dump_schema/1, dump_schema/3]). +-export([dump_schema/2]). -export([schema_module/0]). --export([gen_example_conf/4]). +-export([gen_example_conf/2]). + +%% TODO: move to emqx_dashboard when we stop building api schema at build time +-export([ + hotconf_schema_json/1, + bridge_schema_json/1 +]). %% for rpc -export([get_node_and_config/1]). @@ -43,50 +49,50 @@ add_handler(ConfKeyPath, HandlerName) -> remove_handler(ConfKeyPath) -> emqx_config_handler:remove_handler(ConfKeyPath). --spec get(emqx_map_lib:config_key_path()) -> term(). +-spec get(emqx_utils_maps:config_key_path()) -> term(). get(KeyPath) -> emqx:get_config(KeyPath). --spec get(emqx_map_lib:config_key_path(), term()) -> term(). +-spec get(emqx_utils_maps:config_key_path(), term()) -> term(). get(KeyPath, Default) -> emqx:get_config(KeyPath, Default). --spec get_raw(emqx_map_lib:config_key_path(), term()) -> term(). +-spec get_raw(emqx_utils_maps:config_key_path(), term()) -> term(). get_raw(KeyPath, Default) -> emqx_config:get_raw(KeyPath, Default). --spec get_raw(emqx_map_lib:config_key_path()) -> term(). +-spec get_raw(emqx_utils_maps:config_key_path()) -> term(). get_raw(KeyPath) -> emqx_config:get_raw(KeyPath). %% @doc Returns all values in the cluster. --spec get_all(emqx_map_lib:config_key_path()) -> #{node() => term()}. +-spec get_all(emqx_utils_maps:config_key_path()) -> #{node() => term()}. get_all(KeyPath) -> {ResL, []} = emqx_conf_proto_v2:get_all(KeyPath), maps:from_list(ResL). %% @doc Returns the specified node's KeyPath, or exception if not found --spec get_by_node(node(), emqx_map_lib:config_key_path()) -> term(). +-spec get_by_node(node(), emqx_utils_maps:config_key_path()) -> term(). get_by_node(Node, KeyPath) when Node =:= node() -> emqx:get_config(KeyPath); get_by_node(Node, KeyPath) -> emqx_conf_proto_v2:get_config(Node, KeyPath). %% @doc Returns the specified node's KeyPath, or the default value if not found --spec get_by_node(node(), emqx_map_lib:config_key_path(), term()) -> term(). +-spec get_by_node(node(), emqx_utils_maps:config_key_path(), term()) -> term(). get_by_node(Node, KeyPath, Default) when Node =:= node() -> emqx:get_config(KeyPath, Default); get_by_node(Node, KeyPath, Default) -> emqx_conf_proto_v2:get_config(Node, KeyPath, Default). %% @doc Returns the specified node's KeyPath, or config_not_found if key path not found --spec get_node_and_config(emqx_map_lib:config_key_path()) -> term(). +-spec get_node_and_config(emqx_utils_maps:config_key_path()) -> term(). get_node_and_config(KeyPath) -> {node(), emqx:get_config(KeyPath, config_not_found)}. %% @doc Update all value of key path in cluster-override.conf or local-override.conf. -spec update( - emqx_map_lib:config_key_path(), + emqx_utils_maps:config_key_path(), emqx_config:update_request(), emqx_config:update_opts() ) -> @@ -97,7 +103,7 @@ update(KeyPath, UpdateReq, Opts) -> %% @doc Update the specified node's key path in local-override.conf. -spec update( node(), - emqx_map_lib:config_key_path(), + emqx_utils_maps:config_key_path(), emqx_config:update_request(), emqx_config:update_opts() ) -> @@ -108,13 +114,13 @@ update(Node, KeyPath, UpdateReq, Opts) -> emqx_conf_proto_v2:update(Node, KeyPath, UpdateReq, Opts). %% @doc remove all value of key path in cluster-override.conf or local-override.conf. --spec remove(emqx_map_lib:config_key_path(), emqx_config:update_opts()) -> +-spec remove(emqx_utils_maps:config_key_path(), emqx_config:update_opts()) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. remove(KeyPath, Opts) -> emqx_conf_proto_v2:remove_config(KeyPath, Opts). %% @doc remove the specified node's key path in local-override.conf. --spec remove(node(), emqx_map_lib:config_key_path(), emqx_config:update_opts()) -> +-spec remove(node(), emqx_utils_maps:config_key_path(), emqx_config:update_opts()) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. remove(Node, KeyPath, Opts) when Node =:= node() -> emqx:remove_config(KeyPath, Opts#{override_to => local}); @@ -122,13 +128,13 @@ remove(Node, KeyPath, Opts) -> emqx_conf_proto_v2:remove_config(Node, KeyPath, Opts). %% @doc reset all value of key path in cluster-override.conf or local-override.conf. --spec reset(emqx_map_lib:config_key_path(), emqx_config:update_opts()) -> +-spec reset(emqx_utils_maps:config_key_path(), emqx_config:update_opts()) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. reset(KeyPath, Opts) -> emqx_conf_proto_v2:reset(KeyPath, Opts). %% @doc reset the specified node's key path in local-override.conf. --spec reset(node(), emqx_map_lib:config_key_path(), emqx_config:update_opts()) -> +-spec reset(node(), emqx_utils_maps:config_key_path(), emqx_config:update_opts()) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. reset(Node, KeyPath, Opts) when Node =:= node() -> emqx:reset_config(KeyPath, Opts#{override_to => local}); @@ -136,24 +142,22 @@ reset(Node, KeyPath, Opts) -> emqx_conf_proto_v2:reset(Node, KeyPath, Opts). %% @doc Called from build script. --spec dump_schema(file:name_all()) -> ok. -dump_schema(Dir) -> - I18nFile = emqx_dashboard:i18n_file(), - dump_schema(Dir, emqx_conf_schema, I18nFile). - -dump_schema(Dir, SchemaModule, I18nFile) -> +%% TODO: move to a external escript after all refactoring is done +dump_schema(Dir, SchemaModule) -> + _ = application:load(emqx_dashboard), + ok = emqx_dashboard_desc_cache:init(), lists:foreach( fun(Lang) -> - gen_config_md(Dir, I18nFile, SchemaModule, Lang), - gen_api_schema_json(Dir, I18nFile, Lang), - gen_example_conf(Dir, I18nFile, SchemaModule, Lang), - gen_schema_json(Dir, I18nFile, SchemaModule, Lang) + ok = gen_config_md(Dir, SchemaModule, Lang), + ok = gen_api_schema_json(Dir, Lang), + ok = gen_schema_json(Dir, SchemaModule, Lang) end, ["en", "zh"] - ). + ), + ok = gen_example_conf(Dir, SchemaModule). %% for scripts/spellcheck. -gen_schema_json(Dir, I18nFile, SchemaModule, Lang) -> +gen_schema_json(Dir, SchemaModule, Lang) -> SchemaJsonFile = filename:join([Dir, "schema-" ++ Lang ++ ".json"]), io:format(user, "===< Generating: ~s~n", [SchemaJsonFile]), %% EMQX_SCHEMA_FULL_DUMP is quite a hidden API @@ -164,40 +168,62 @@ gen_schema_json(Dir, I18nFile, SchemaModule, Lang) -> false -> ?IMPORTANCE_LOW end, io:format(user, "===< Including fields from importance level: ~p~n", [IncludeImportance]), - Opts = #{desc_file => I18nFile, lang => Lang, include_importance_up_from => IncludeImportance}, + Opts = #{ + include_importance_up_from => IncludeImportance, + desc_resolver => make_desc_resolver(Lang) + }, JsonMap = hocon_schema_json:gen(SchemaModule, Opts), - IoData = jsx:encode(JsonMap, [space, {indent, 4}]), + IoData = emqx_utils_json:encode(JsonMap, [pretty, force_utf8]), ok = file:write_file(SchemaJsonFile, IoData). -gen_api_schema_json(Dir, I18nFile, Lang) -> - emqx_dashboard:init_i18n(I18nFile, list_to_binary(Lang)), +%% TODO: delete this function when we stop generating this JSON at build time. +gen_api_schema_json(Dir, Lang) -> gen_api_schema_json_hotconf(Dir, Lang), - gen_api_schema_json_bridge(Dir, Lang), - emqx_dashboard:clear_i18n(). + gen_api_schema_json_bridge(Dir, Lang). +%% TODO: delete this function when we stop generating this JSON at build time. gen_api_schema_json_hotconf(Dir, Lang) -> - SchemaInfo = #{title => <<"EMQX Hot Conf API Schema">>, version => <<"0.1.0">>}, File = schema_filename(Dir, "hot-config-schema-", Lang), - ok = do_gen_api_schema_json(File, emqx_mgmt_api_configs, SchemaInfo). + IoData = hotconf_schema_json(Lang), + ok = write_api_schema_json_file(File, IoData). +%% TODO: delete this function when we stop generating this JSON at build time. gen_api_schema_json_bridge(Dir, Lang) -> - SchemaInfo = #{title => <<"EMQX Data Bridge API Schema">>, version => <<"0.1.0">>}, File = schema_filename(Dir, "bridge-api-", Lang), - ok = do_gen_api_schema_json(File, emqx_bridge_api, SchemaInfo). + IoData = bridge_schema_json(Lang), + ok = write_api_schema_json_file(File, IoData). + +%% TODO: delete this function when we stop generating this JSON at build time. +write_api_schema_json_file(File, IoData) -> + io:format(user, "===< Generating: ~s~n", [File]), + file:write_file(File, IoData). + +%% TODO: move this function to emqx_dashboard when we stop generating this JSON at build time. +hotconf_schema_json(Lang) -> + SchemaInfo = #{title => <<"EMQX Hot Conf API Schema">>, version => <<"0.1.0">>}, + gen_api_schema_json_iodata(emqx_mgmt_api_configs, SchemaInfo, Lang). + +%% TODO: move this function to emqx_dashboard when we stop generating this JSON at build time. +bridge_schema_json(Lang) -> + SchemaInfo = #{title => <<"EMQX Data Bridge API Schema">>, version => <<"0.1.0">>}, + gen_api_schema_json_iodata(emqx_bridge_api, SchemaInfo, Lang). schema_filename(Dir, Prefix, Lang) -> Filename = Prefix ++ Lang ++ ".json", filename:join([Dir, Filename]). -gen_config_md(Dir, I18nFile, SchemaModule, Lang) -> +%% TODO: remove it and also remove hocon_md.erl and friends. +%% markdown generation from schema is a failure and we are moving to an interactive +%% viewer like swagger UI. +gen_config_md(Dir, SchemaModule, Lang) -> SchemaMdFile = filename:join([Dir, "config-" ++ Lang ++ ".md"]), io:format(user, "===< Generating: ~s~n", [SchemaMdFile]), - ok = gen_doc(SchemaMdFile, SchemaModule, I18nFile, Lang). + ok = gen_doc(SchemaMdFile, SchemaModule, Lang). -gen_example_conf(Dir, I18nFile, SchemaModule, Lang) -> - SchemaMdFile = filename:join([Dir, "emqx.conf." ++ Lang ++ ".example"]), +gen_example_conf(Dir, SchemaModule) -> + SchemaMdFile = filename:join([Dir, "emqx.conf.example"]), io:format(user, "===< Generating: ~s~n", [SchemaMdFile]), - ok = gen_example(SchemaMdFile, SchemaModule, I18nFile, Lang). + ok = gen_example(SchemaMdFile, SchemaModule). %% @doc return the root schema module. -spec schema_module() -> module(). @@ -211,35 +237,48 @@ schema_module() -> %% Internal functions %%-------------------------------------------------------------------- --spec gen_doc(file:name_all(), module(), file:name_all(), string()) -> ok. -gen_doc(File, SchemaModule, I18nFile, Lang) -> +%% @doc Make a resolver function that can be used to lookup the description by hocon_schema_json dump. +make_desc_resolver(Lang) -> + fun + ({desc, Namespace, Id}) -> + emqx_dashboard_desc_cache:lookup(Lang, Namespace, Id, desc); + (Desc) -> + unicode:characters_to_binary(Desc) + end. + +-spec gen_doc(file:name_all(), module(), string()) -> ok. +gen_doc(File, SchemaModule, Lang) -> Version = emqx_release:version(), Title = "# " ++ emqx_release:description() ++ " Configuration\n\n" ++ "", BodyFile = filename:join([rel, "emqx_conf.template." ++ Lang ++ ".md"]), {ok, Body} = file:read_file(BodyFile), - Opts = #{title => Title, body => Body, desc_file => I18nFile, lang => Lang}, + Resolver = make_desc_resolver(Lang), + Opts = #{title => Title, body => Body, desc_resolver => Resolver}, Doc = hocon_schema_md:gen(SchemaModule, Opts), file:write_file(File, Doc). -gen_example(File, SchemaModule, I18nFile, Lang) -> +gen_example(File, SchemaModule) -> + %% we do not generate description in example files + %% so there is no need for a desc_resolver Opts = #{ title => <<"EMQX Configuration Example">>, body => <<"">>, - desc_file => I18nFile, - lang => Lang, include_importance_up_from => ?IMPORTANCE_MEDIUM }, Example = hocon_schema_example:gen(SchemaModule, Opts), file:write_file(File, Example). -%% Only gen hot_conf schema, not all configuration fields. -do_gen_api_schema_json(File, SchemaMod, SchemaInfo) -> - io:format(user, "===< Generating: ~s~n", [File]), +%% TODO: move this to emqx_dashboard when we stop generating +%% this JSON at build time. +gen_api_schema_json_iodata(SchemaMod, SchemaInfo, Lang) -> {ApiSpec0, Components0} = emqx_dashboard_swagger:spec( SchemaMod, - #{schema_converter => fun hocon_schema_to_spec/2} + #{ + schema_converter => fun hocon_schema_to_spec/2, + i18n_lang => Lang + } ), ApiSpec = lists:foldl( fun({Path, Spec, _, _}, Acc) -> @@ -268,22 +307,14 @@ do_gen_api_schema_json(File, SchemaMod, SchemaInfo) -> ApiSpec0 ), Components = lists:foldl(fun(M, Acc) -> maps:merge(M, Acc) end, #{}, Components0), - IoData = jsx:encode( + emqx_utils_json:encode( #{ info => SchemaInfo, paths => ApiSpec, components => #{schemas => Components} }, - [space, {indent, 4}] - ), - file:write_file(File, IoData). - --define(INIT_SCHEMA, #{ - fields => #{}, - translations => #{}, - validations => [], - namespace => undefined -}). + [pretty, force_utf8] + ). -define(TO_REF(_N_, _F_), iolist_to_binary([to_bin(_N_), ".", to_bin(_F_)])). -define(TO_COMPONENTS_SCHEMA(_M_, _F_), diff --git a/apps/emqx_conf/src/emqx_conf_app.erl b/apps/emqx_conf/src/emqx_conf_app.erl index 51fc5c2e2..35a79ea6e 100644 --- a/apps/emqx_conf/src/emqx_conf_app.erl +++ b/apps/emqx_conf/src/emqx_conf_app.erl @@ -60,7 +60,14 @@ get_override_config_file() -> TnxId = emqx_cluster_rpc:get_node_tnx_id(Node), WallClock = erlang:statistics(wall_clock), Conf = emqx_config_handler:get_raw_cluster_override_conf(), - #{wall_clock => WallClock, conf => Conf, tnx_id => TnxId, node => Node} + HasDeprecateFile = emqx_config:has_deprecated_file(), + #{ + wall_clock => WallClock, + conf => Conf, + tnx_id => TnxId, + node => Node, + has_deprecated_file => HasDeprecateFile + } end, case mria:ro_transaction(?CLUSTER_RPC_SHARD, Fun) of {atomic, Res} -> {ok, Res}; @@ -153,10 +160,10 @@ copy_override_conf_from_core_node() -> {ok, ?DEFAULT_INIT_TXN_ID}; false -> %% retry in some time - Jitter = rand:uniform(2_000), - Timeout = 10_000 + Jitter, + Jitter = rand:uniform(2000), + Timeout = 10000 + Jitter, ?SLOG(info, #{ - msg => "copy_override_conf_from_core_node_retry", + msg => "copy_cluster_conf_from_core_node_retry", timeout => Timeout, nodes => Nodes, failed => Failed, @@ -168,18 +175,16 @@ copy_override_conf_from_core_node() -> _ -> [{ok, Info} | _] = lists:sort(fun conf_sort/2, Ready), #{node := Node, conf := RawOverrideConf, tnx_id := TnxId} = Info, + HasDeprecatedFile = maps:get(has_deprecated_file, Info, false), ?SLOG(debug, #{ - msg => "copy_override_conf_from_core_node_success", + msg => "copy_cluster_conf_from_core_node_success", node => Node, - cluster_override_conf_file => application:get_env( - emqx, cluster_override_conf_file - ), - local_override_conf_file => application:get_env( - emqx, local_override_conf_file - ), - data_dir => emqx:data_dir() + has_deprecated_file => HasDeprecatedFile, + data_dir => emqx:data_dir(), + tnx_id => TnxId }), ok = emqx_config:save_to_override_conf( + HasDeprecatedFile, RawOverrideConf, #{override_to => cluster} ), diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 58bcf9700..f3f014321 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -98,7 +98,10 @@ roots() -> {"rpc", sc( ?R_REF("rpc"), - #{translate_to => ["gen_rpc"]} + #{ + translate_to => ["gen_rpc"], + importance => ?IMPORTANCE_LOW + } )} ] ++ emqx_schema:roots(medium) ++ @@ -132,7 +135,7 @@ fields("cluster") -> )}, {"core_nodes", sc( - emqx_schema:comma_separated_atoms(), + node_array(), #{ mapping => "mria.core_nodes", default => [], @@ -200,7 +203,7 @@ fields(cluster_static) -> [ {"seeds", sc( - hoconsc:array(atom()), + node_array(), #{ default => [], desc => ?DESC(cluster_static_seeds), @@ -454,7 +457,7 @@ fields("node") -> mapping => "vm_args.+e", desc => ?DESC(max_ets_tables), default => 262144, - importance => ?IMPORTANCE_LOW, + importance => ?IMPORTANCE_HIDDEN, 'readOnly' => true } )}, @@ -500,7 +503,7 @@ fields("node") -> mapping => "vm_args.-env ERL_CRASH_DUMP", desc => ?DESC(node_crash_dump_file), default => crash_dump_file_default(), - importance => ?IMPORTANCE_LOW, + importance => ?IMPORTANCE_HIDDEN, 'readOnly' => true } )}, @@ -511,7 +514,7 @@ fields("node") -> mapping => "vm_args.-env ERL_CRASH_DUMP_SECONDS", default => <<"30s">>, desc => ?DESC(node_crash_dump_seconds), - importance => ?IMPORTANCE_LOW, + importance => ?IMPORTANCE_HIDDEN, 'readOnly' => true } )}, @@ -522,7 +525,7 @@ fields("node") -> mapping => "vm_args.-env ERL_CRASH_DUMP_BYTES", default => <<"100MB">>, desc => ?DESC(node_crash_dump_bytes), - importance => ?IMPORTANCE_LOW, + importance => ?IMPORTANCE_HIDDEN, 'readOnly' => true } )}, @@ -533,7 +536,7 @@ fields("node") -> mapping => "vm_args.-kernel net_ticktime", default => <<"2m">>, 'readOnly' => true, - importance => ?IMPORTANCE_LOW, + importance => ?IMPORTANCE_HIDDEN, desc => ?DESC(node_dist_net_ticktime) } )}, @@ -544,7 +547,7 @@ fields("node") -> mapping => "emqx_machine.backtrace_depth", default => 23, 'readOnly' => true, - importance => ?IMPORTANCE_LOW, + importance => ?IMPORTANCE_HIDDEN, desc => ?DESC(node_backtrace_depth) } )}, @@ -555,7 +558,7 @@ fields("node") -> mapping => "emqx_machine.applications", default => [], 'readOnly' => true, - importance => ?IMPORTANCE_LOW, + importance => ?IMPORTANCE_HIDDEN, desc => ?DESC(node_applications) } )}, @@ -565,7 +568,7 @@ fields("node") -> #{ desc => ?DESC(node_etc_dir), 'readOnly' => true, - importance => ?IMPORTANCE_LOW, + importance => ?IMPORTANCE_HIDDEN, deprecated => {since, "5.0.8"} } )}, @@ -574,7 +577,7 @@ fields("node") -> ?R_REF("cluster_call"), #{ 'readOnly' => true, - importance => ?IMPORTANCE_LOW + importance => ?IMPORTANCE_HIDDEN } )}, {"db_backend", @@ -588,7 +591,7 @@ fields("node") -> desc => ?DESC(db_backend) } )}, - {"db_role", + {"role", sc( hoconsc:enum([core, replicant]), #{ @@ -596,6 +599,7 @@ fields("node") -> default => core, 'readOnly' => true, importance => ?IMPORTANCE_HIGH, + aliases => [db_role], desc => ?DESC(db_role) } )}, @@ -617,7 +621,7 @@ fields("node") -> mapping => "mria.tlog_push_mode", default => async, 'readOnly' => true, - importance => ?IMPORTANCE_LOW, + importance => ?IMPORTANCE_HIDDEN, desc => ?DESC(db_tlog_push_mode) } )}, @@ -1025,7 +1029,8 @@ translation("emqx") -> [ {"config_files", fun tr_config_files/1}, {"cluster_override_conf_file", fun tr_cluster_override_conf_file/1}, - {"local_override_conf_file", fun tr_local_override_conf_file/1} + {"local_override_conf_file", fun tr_local_override_conf_file/1}, + {"cluster_hocon_file", fun tr_cluster_hocon_file/1} ]; translation("gen_rpc") -> [{"default_client_driver", fun tr_default_config_driver/1}]; @@ -1073,12 +1078,15 @@ tr_config_files(_Conf) -> end. tr_cluster_override_conf_file(Conf) -> - tr_override_conf_file(Conf, "cluster-override.conf"). + tr_conf_file(Conf, "cluster-override.conf"). tr_local_override_conf_file(Conf) -> - tr_override_conf_file(Conf, "local-override.conf"). + tr_conf_file(Conf, "local-override.conf"). -tr_override_conf_file(Conf, Filename) -> +tr_cluster_hocon_file(Conf) -> + tr_conf_file(Conf, "cluster.hocon"). + +tr_conf_file(Conf, Filename) -> DataDir = conf_get("node.data_dir", Conf), %% assert, this config is not nullable [_ | _] = DataDir, @@ -1278,7 +1286,10 @@ emqx_schema_high_prio_roots() -> {"authorization", sc( ?R_REF("authorization"), - #{desc => ?DESC(authorization)} + #{ + desc => ?DESC(authorization), + importance => ?IMPORTANCE_HIGH + } )}, lists:keyreplace("authorization", 1, Roots, Authz). @@ -1301,3 +1312,6 @@ validator_string_re(Val, RE, Error) -> catch _:_ -> {error, Error} end. + +node_array() -> + hoconsc:union([emqx_schema:comma_separated_atoms(), hoconsc:array(atom())]). diff --git a/apps/emqx_conf/src/proto/emqx_conf_proto_v1.erl b/apps/emqx_conf/src/proto/emqx_conf_proto_v1.erl index 84687e314..b66307a1b 100644 --- a/apps/emqx_conf/src/proto/emqx_conf_proto_v1.erl +++ b/apps/emqx_conf/src/proto/emqx_conf_proto_v1.erl @@ -38,22 +38,22 @@ -include_lib("emqx/include/bpapi.hrl"). --type update_config_key_path() :: [emqx_map_lib:config_key(), ...]. +-type update_config_key_path() :: [emqx_utils_maps:config_key(), ...]. introduced_in() -> "5.0.0". --spec get_config(node(), emqx_map_lib:config_key_path()) -> +-spec get_config(node(), emqx_utils_maps:config_key_path()) -> term() | emqx_rpc:badrpc(). get_config(Node, KeyPath) -> rpc:call(Node, emqx, get_config, [KeyPath]). --spec get_config(node(), emqx_map_lib:config_key_path(), _Default) -> +-spec get_config(node(), emqx_utils_maps:config_key_path(), _Default) -> term() | emqx_rpc:badrpc(). get_config(Node, KeyPath, Default) -> rpc:call(Node, emqx, get_config, [KeyPath, Default]). --spec get_all(emqx_map_lib:config_key_path()) -> emqx_rpc:multicall_result(). +-spec get_all(emqx_utils_maps:config_key_path()) -> emqx_rpc:multicall_result(). get_all(KeyPath) -> rpc:multicall(emqx_conf, get_node_and_config, [KeyPath], 5000). diff --git a/apps/emqx_conf/src/proto/emqx_conf_proto_v2.erl b/apps/emqx_conf/src/proto/emqx_conf_proto_v2.erl index dd8d2fedd..97446ee9f 100644 --- a/apps/emqx_conf/src/proto/emqx_conf_proto_v2.erl +++ b/apps/emqx_conf/src/proto/emqx_conf_proto_v2.erl @@ -44,19 +44,19 @@ introduced_in() -> -spec sync_data_from_node(node()) -> {ok, binary()} | emqx_rpc:badrpc(). sync_data_from_node(Node) -> rpc:call(Node, emqx_conf_app, sync_data_from_node, [], 20000). --type update_config_key_path() :: [emqx_map_lib:config_key(), ...]. +-type update_config_key_path() :: [emqx_utils_maps:config_key(), ...]. --spec get_config(node(), emqx_map_lib:config_key_path()) -> +-spec get_config(node(), emqx_utils_maps:config_key_path()) -> term() | emqx_rpc:badrpc(). get_config(Node, KeyPath) -> rpc:call(Node, emqx, get_config, [KeyPath]). --spec get_config(node(), emqx_map_lib:config_key_path(), _Default) -> +-spec get_config(node(), emqx_utils_maps:config_key_path(), _Default) -> term() | emqx_rpc:badrpc(). get_config(Node, KeyPath, Default) -> rpc:call(Node, emqx, get_config, [KeyPath, Default]). --spec get_all(emqx_map_lib:config_key_path()) -> emqx_rpc:multicall_result(). +-spec get_all(emqx_utils_maps:config_key_path()) -> emqx_rpc:multicall_result(). get_all(KeyPath) -> rpc:multicall(emqx_conf, get_node_and_config, [KeyPath], 5000). diff --git a/apps/emqx_conf/test/emqx_conf_app_SUITE.erl b/apps/emqx_conf/test/emqx_conf_app_SUITE.erl index c34eb9dc3..583405158 100644 --- a/apps/emqx_conf/test/emqx_conf_app_SUITE.erl +++ b/apps/emqx_conf/test/emqx_conf_app_SUITE.erl @@ -27,7 +27,7 @@ all() -> t_copy_conf_override_on_restarts(_Config) -> ct:timetrap({seconds, 120}), snabbkaffe:fix_ct_logging(), - Cluster = cluster([core, core, core]), + Cluster = cluster([cluster_spec({core, 1}), cluster_spec({core, 2}), cluster_spec({core, 3})]), %% 1. Start all nodes Nodes = start_cluster(Cluster), @@ -41,7 +41,7 @@ t_copy_conf_override_on_restarts(_Config) -> %% crash and eventually all nodes should be ready. start_cluster_async(Cluster), - timer:sleep(15_000), + timer:sleep(15000), assert_config_load_done(Nodes), @@ -50,23 +50,48 @@ t_copy_conf_override_on_restarts(_Config) -> stop_cluster(Nodes) end. -t_copy_data_dir(_Config) -> +t_copy_new_data_dir(_Config) -> net_kernel:start(['master1@127.0.0.1', longnames]), ct:timetrap({seconds, 120}), snabbkaffe:fix_ct_logging(), - Cluster = cluster([{core, copy1}, {core, copy2}, {core, copy3}]), + Cluster = cluster([cluster_spec({core, 4}), cluster_spec({core, 5}), cluster_spec({core, 6})]), %% 1. Start all nodes [First | Rest] = Nodes = start_cluster(Cluster), try + File = "/configs/cluster.hocon", assert_config_load_done(Nodes), - rpc:call(First, ?MODULE, create_data_dir, []), + rpc:call(First, ?MODULE, create_data_dir, [File]), {[ok, ok, ok], []} = rpc:multicall(Nodes, application, stop, [emqx_conf]), {[ok, ok, ok], []} = rpc:multicall(Nodes, ?MODULE, set_data_dir_env, []), ok = rpc:call(First, application, start, [emqx_conf]), {[ok, ok], []} = rpc:multicall(Rest, application, start, [emqx_conf]), - assert_data_copy_done(Nodes), + assert_data_copy_done(Nodes, File), + stop_cluster(Nodes), + ok + after + stop_cluster(Nodes) + end. + +t_copy_deprecated_data_dir(_Config) -> + net_kernel:start(['master2@127.0.0.1', longnames]), + ct:timetrap({seconds, 120}), + snabbkaffe:fix_ct_logging(), + Cluster = cluster([cluster_spec({core, 7}), cluster_spec({core, 8}), cluster_spec({core, 9})]), + + %% 1. Start all nodes + [First | Rest] = Nodes = start_cluster(Cluster), + try + File = "/configs/cluster-override.conf", + assert_config_load_done(Nodes), + rpc:call(First, ?MODULE, create_data_dir, [File]), + {[ok, ok, ok], []} = rpc:multicall(Nodes, application, stop, [emqx_conf]), + {[ok, ok, ok], []} = rpc:multicall(Nodes, ?MODULE, set_data_dir_env, []), + ok = rpc:call(First, application, start, [emqx_conf]), + {[ok, ok], []} = rpc:multicall(Rest, application, start, [emqx_conf]), + + assert_data_copy_done(Nodes, File), stop_cluster(Nodes), ok after @@ -77,7 +102,7 @@ t_copy_data_dir(_Config) -> %% Helper functions %%------------------------------------------------------------------------------ -create_data_dir() -> +create_data_dir(File) -> Node = atom_to_list(node()), ok = filelib:ensure_dir(Node ++ "/certs/"), ok = filelib:ensure_dir(Node ++ "/authz/"), @@ -85,7 +110,7 @@ create_data_dir() -> ok = file:write_file(Node ++ "/certs/fake-cert", list_to_binary(Node)), ok = file:write_file(Node ++ "/authz/fake-authz", list_to_binary(Node)), Telemetry = <<"telemetry.enable = false">>, - ok = file:write_file(Node ++ "/configs/cluster-override.conf", Telemetry). + ok = file:write_file(Node ++ File, Telemetry). set_data_dir_env() -> Node = atom_to_list(node()), @@ -100,14 +125,17 @@ set_data_dir_env() -> ok = file:write_file(NewConfigFile, DataDir, [append]), application:set_env(emqx, config_files, [NewConfigFile]), application:set_env(emqx, data_dir, Node), + %% We set env both cluster.hocon and cluster-override.conf, but only one will be used + application:set_env(emqx, cluster_hocon_file, Node ++ "/configs/cluster.hocon"), application:set_env(emqx, cluster_override_conf_file, Node ++ "/configs/cluster-override.conf"), ok. -assert_data_copy_done([First0 | Rest]) -> +assert_data_copy_done([First0 | Rest], File) -> First = atom_to_list(First0), {ok, FakeCertFile} = file:read_file(First ++ "/certs/fake-cert"), {ok, FakeAuthzFile} = file:read_file(First ++ "/authz/fake-authz"), - {ok, FakeOverrideFile} = file:read_file(First ++ "/configs/cluster-override.conf"), + {ok, FakeOverrideFile} = file:read_file(First ++ File), + {ok, ExpectFake} = hocon:binary(FakeOverrideFile), lists:foreach( fun(Node0) -> Node = atom_to_list(Node0), @@ -117,8 +145,8 @@ assert_data_copy_done([First0 | Rest]) -> #{node => Node} ), ?assertEqual( - {ok, FakeOverrideFile}, - file:read_file(Node ++ "/configs/cluster-override.conf"), + {ok, ExpectFake}, + hocon:files([Node ++ File]), #{node => Node} ), ?assertEqual( @@ -173,3 +201,6 @@ cluster(Specs) -> ok end} ]). + +cluster_spec({Type, Num}) -> + {Type, list_to_atom(atom_to_list(?MODULE) ++ integer_to_list(Num))}. diff --git a/apps/emqx_conf/test/emqx_conf_schema_tests.erl b/apps/emqx_conf/test/emqx_conf_schema_tests.erl index 3653b9d19..667d1766f 100644 --- a/apps/emqx_conf/test/emqx_conf_schema_tests.erl +++ b/apps/emqx_conf/test/emqx_conf_schema_tests.erl @@ -6,6 +6,116 @@ -include_lib("eunit/include/eunit.hrl"). +%% erlfmt-ignore +-define(BASE_CONF, + """ + node { + name = \"emqx1@127.0.0.1\" + cookie = \"emqxsecretcookie\" + data_dir = \"data\" + } + cluster { + name = emqxcl + discovery_strategy = static + static.seeds = ~p + core_nodes = ~p + } + """). + +array_nodes_test() -> + ExpectNodes = ['emqx1@127.0.0.1', 'emqx2@127.0.0.1'], + lists:foreach( + fun(Nodes) -> + ConfFile = to_bin(?BASE_CONF, [Nodes, Nodes]), + {ok, Conf} = hocon:binary(ConfFile, #{format => richmap}), + ConfList = hocon_tconf:generate(emqx_conf_schema, Conf), + ClusterDiscovery = proplists:get_value( + cluster_discovery, proplists:get_value(ekka, ConfList) + ), + ?assertEqual( + {static, [{seeds, ExpectNodes}]}, + ClusterDiscovery, + Nodes + ), + ?assertEqual( + ExpectNodes, + proplists:get_value(core_nodes, proplists:get_value(mria, ConfList)), + Nodes + ) + end, + [["emqx1@127.0.0.1", "emqx2@127.0.0.1"], "emqx1@127.0.0.1, emqx2@127.0.0.1"] + ), + ok. + +%% erlfmt-ignore +-define(BASE_AUTHN_ARRAY, + """ + authentication = [ + {backend = \"http\" + body {password = \"${password}\", username = \"${username}\"} + connect_timeout = \"15s\" + enable_pipelining = 100 + headers {\"content-type\" = \"application/json\"} + mechanism = \"password_based\" + method = \"~p\" + pool_size = 8 + request_timeout = \"5s\" + ssl {enable = ~p, verify = \"verify_peer\"} + url = \"~ts\" + } + ] + """ +). + +-define(ERROR(Reason), + {emqx_conf_schema, [ + #{ + kind := validation_error, + reason := integrity_validation_failure, + result := _, + validation_name := Reason + } + ]} +). + +authn_validations_test() -> + BaseConf = to_bin(?BASE_CONF, ["emqx1@127.0.0.1", "emqx1@127.0.0.1"]), + + OKHttps = to_bin(?BASE_AUTHN_ARRAY, [post, true, <<"https://127.0.0.1:8080">>]), + Conf0 = <>, + {ok, ConfMap0} = hocon:binary(Conf0, #{format => richmap}), + ?assert(is_list(hocon_tconf:generate(emqx_conf_schema, ConfMap0))), + + OKHttp = to_bin(?BASE_AUTHN_ARRAY, [post, false, <<"http://127.0.0.1:8080">>]), + Conf1 = <>, + {ok, ConfMap1} = hocon:binary(Conf1, #{format => richmap}), + ?assert(is_list(hocon_tconf:generate(emqx_conf_schema, ConfMap1))), + + DisableSSLWithHttps = to_bin(?BASE_AUTHN_ARRAY, [post, false, <<"https://127.0.0.1:8080">>]), + Conf2 = <>, + {ok, ConfMap2} = hocon:binary(Conf2, #{format => richmap}), + ?assertThrow( + ?ERROR(check_http_ssl_opts), + hocon_tconf:generate(emqx_conf_schema, ConfMap2) + ), + + BadHeader = to_bin(?BASE_AUTHN_ARRAY, [get, true, <<"https://127.0.0.1:8080">>]), + Conf3 = <>, + {ok, ConfMap3} = hocon:binary(Conf3, #{format => richmap}), + ?assertThrow( + ?ERROR(check_http_headers), + hocon_tconf:generate(emqx_conf_schema, ConfMap3) + ), + + BadHeaderWithTuple = binary:replace(BadHeader, [<<"[">>, <<"]">>], <<"">>, [global]), + Conf4 = <>, + {ok, ConfMap4} = hocon:binary(Conf4, #{format => richmap}), + ?assertThrow( + ?ERROR(check_http_headers), + hocon_tconf:generate(emqx_conf_schema, ConfMap4) + ), + ok. + doc_gen_test() -> %% the json file too large to encode. { @@ -26,3 +136,6 @@ doc_gen_test() -> ok end }. + +to_bin(Format, Args) -> + iolist_to_binary(io_lib:format(Format, Args)). diff --git a/apps/emqx_connector/rebar.config b/apps/emqx_connector/rebar.config index 2ce6b00f8..03be87356 100644 --- a/apps/emqx_connector/rebar.config +++ b/apps/emqx_connector/rebar.config @@ -7,6 +7,7 @@ {deps, [ {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}}, {emqx_resource, {path, "../emqx_resource"}}, {eldap2, {git, "https://github.com/emqx/eldap2", {tag, "v0.2.2"}}}, {mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.2"}}}, diff --git a/apps/emqx_connector/src/emqx_connector.app.src b/apps/emqx_connector/src/emqx_connector.app.src index 08f97c243..c0a19824c 100644 --- a/apps/emqx_connector/src/emqx_connector.app.src +++ b/apps/emqx_connector/src/emqx_connector.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_connector, [ {description, "EMQX Data Integration Connectors"}, - {vsn, "0.1.18"}, + {vsn, "0.1.21"}, {registered, []}, {mod, {emqx_connector_app, []}}, {applications, [ diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl index 401fc8812..5b71e3f3a 100644 --- a/apps/emqx_connector/src/emqx_connector_http.erl +++ b/apps/emqx_connector/src/emqx_connector_http.erl @@ -47,7 +47,7 @@ namespace/0 ]). --export([check_ssl_opts/2, validate_method/1]). +-export([check_ssl_opts/2, validate_method/1, join_paths/2]). -type connect_timeout() :: emqx_schema:duration() | infinity. -type pool_type() :: random | hash. @@ -219,7 +219,7 @@ on_start( SSLOpts = emqx_tls_lib:to_client_opts(maps:get(ssl, Config)), {tls, SSLOpts} end, - NTransportOpts = emqx_misc:ipv6_probe(TransportOpts), + NTransportOpts = emqx_utils:ipv6_probe(TransportOpts), PoolOpts = [ {host, Host}, {port, Port}, @@ -231,9 +231,8 @@ on_start( {transport_opts, NTransportOpts}, {enable_pipelining, maps:get(enable_pipelining, Config, ?DEFAULT_PIPELINE_SIZE)} ], - PoolName = emqx_plugin_libs_pool:pool_name(InstId), State = #{ - pool_name => PoolName, + pool_name => InstId, pool_type => PoolType, host => Host, port => Port, @@ -241,7 +240,7 @@ on_start( base_path => BasePath, request => preprocess_request(maps:get(request, Config, undefined)) }, - case ehttpc_sup:start_pool(PoolName, PoolOpts) of + case ehttpc_sup:start_pool(InstId, PoolOpts) of {ok, _} -> {ok, State}; {error, {already_started, _}} -> {ok, State}; {error, Reason} -> {error, Reason} @@ -425,7 +424,7 @@ do_get_status(PoolName, Timeout) -> Error end end, - try emqx_misc:pmap(DoPerWorker, Workers, Timeout) of + try emqx_utils:pmap(DoPerWorker, Workers, Timeout) of % we crash in case of non-empty lists since we don't know what to do in that case [_ | _] = Results -> case [E || {error, _} = E <- Results] of @@ -458,7 +457,7 @@ preprocess_request( } = Req ) -> #{ - method => emqx_plugin_libs_rule:preproc_tmpl(bin(Method)), + method => emqx_plugin_libs_rule:preproc_tmpl(to_bin(Method)), path => emqx_plugin_libs_rule:preproc_tmpl(Path), body => maybe_preproc_tmpl(body, Req), headers => preproc_headers(Headers), @@ -471,8 +470,8 @@ preproc_headers(Headers) when is_map(Headers) -> fun(K, V, Acc) -> [ { - emqx_plugin_libs_rule:preproc_tmpl(bin(K)), - emqx_plugin_libs_rule:preproc_tmpl(bin(V)) + emqx_plugin_libs_rule:preproc_tmpl(to_bin(K)), + emqx_plugin_libs_rule:preproc_tmpl(to_bin(V)) } | Acc ] @@ -484,8 +483,8 @@ preproc_headers(Headers) when is_list(Headers) -> lists:map( fun({K, V}) -> { - emqx_plugin_libs_rule:preproc_tmpl(bin(K)), - emqx_plugin_libs_rule:preproc_tmpl(bin(V)) + emqx_plugin_libs_rule:preproc_tmpl(to_bin(K)), + emqx_plugin_libs_rule:preproc_tmpl(to_bin(V)) } end, Headers @@ -516,7 +515,7 @@ process_request( }. process_request_body(undefined, Msg) -> - emqx_json:encode(Msg); + emqx_utils_json:encode(Msg); process_request_body(BodyTks, Msg) -> emqx_plugin_libs_rule:proc_tmpl(BodyTks, Msg). @@ -553,15 +552,41 @@ formalize_request(Method, BasePath, {Path, Headers, _Body}) when -> formalize_request(Method, BasePath, {Path, Headers}); formalize_request(_Method, BasePath, {Path, Headers, Body}) -> - {filename:join(BasePath, Path), Headers, Body}; + {join_paths(BasePath, Path), Headers, Body}; formalize_request(_Method, BasePath, {Path, Headers}) -> - {filename:join(BasePath, Path), Headers}. + {join_paths(BasePath, Path), Headers}. -bin(Bin) when is_binary(Bin) -> +%% By default, we cannot treat HTTP paths as "file" or "resource" paths, +%% because an HTTP server may handle paths like +%% "/a/b/c/", "/a/b/c" and "/a//b/c" differently. +%% +%% So we try to avoid unneccessary path normalization. +%% +%% See also: `join_paths_test_/0` +join_paths(Path1, Path2) -> + do_join_paths(lists:reverse(to_list(Path1)), to_list(Path2)). + +%% "abc/" + "/cde" +do_join_paths([$/ | Path1], [$/ | Path2]) -> + lists:reverse(Path1) ++ [$/ | Path2]; +%% "abc/" + "cde" +do_join_paths([$/ | Path1], Path2) -> + lists:reverse(Path1) ++ [$/ | Path2]; +%% "abc" + "/cde" +do_join_paths(Path1, [$/ | Path2]) -> + lists:reverse(Path1) ++ [$/ | Path2]; +%% "abc" + "cde" +do_join_paths(Path1, Path2) -> + lists:reverse(Path1) ++ [$/ | Path2]. + +to_list(List) when is_list(List) -> List; +to_list(Bin) when is_binary(Bin) -> binary_to_list(Bin). + +to_bin(Bin) when is_binary(Bin) -> Bin; -bin(Str) when is_list(Str) -> +to_bin(Str) when is_list(Str) -> list_to_binary(Str); -bin(Atom) when is_atom(Atom) -> +to_bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8). reply_delegator(ReplyFunAndArgs, Result) -> @@ -603,7 +628,7 @@ is_sensitive_key(_) -> %% Function that will do a deep traversal of Data and remove sensitive %% information (i.e., passwords) redact(Data) -> - emqx_misc:redact(Data, fun is_sensitive_key/1). + emqx_utils:redact(Data, fun is_sensitive_key/1). %% because the body may contain some sensitive data %% and at the same time the redact function will not scan the binary data @@ -642,4 +667,21 @@ redact_test_() -> ?_assertNotEqual(TestData2, redact(TestData2)) ]. +join_paths_test_() -> + [ + ?_assertEqual("abc/cde", join_paths("abc", "cde")), + ?_assertEqual("abc/cde", join_paths("abc", "/cde")), + ?_assertEqual("abc/cde", join_paths("abc/", "cde")), + ?_assertEqual("abc/cde", join_paths("abc/", "/cde")), + + ?_assertEqual("/", join_paths("", "")), + ?_assertEqual("/cde", join_paths("", "cde")), + ?_assertEqual("/cde", join_paths("", "/cde")), + ?_assertEqual("/cde", join_paths("/", "cde")), + ?_assertEqual("/cde", join_paths("/", "/cde")), + + ?_assertEqual("//cde/", join_paths("/", "//cde/")), + ?_assertEqual("abc///cde/", join_paths("abc//", "//cde/")) + ]. + -endif. diff --git a/apps/emqx_connector/src/emqx_connector_ldap.erl b/apps/emqx_connector/src/emqx_connector_ldap.erl index 82d622e09..e2121de22 100644 --- a/apps/emqx_connector/src/emqx_connector_ldap.erl +++ b/apps/emqx_connector/src/emqx_connector_ldap.erl @@ -65,7 +65,7 @@ on_start( ?SLOG(info, #{ msg => "starting_ldap_connector", connector => InstId, - config => emqx_misc:redact(Config) + config => emqx_utils:redact(Config) }), Servers = emqx_schema:parse_servers(Servers0, ?LDAP_HOST_OPTIONS), SslOpts = @@ -87,20 +87,19 @@ on_start( {pool_size, PoolSize}, {auto_reconnect, ?AUTO_RECONNECT_INTERVAL} ], - PoolName = emqx_plugin_libs_pool:pool_name(InstId), - case emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Opts ++ SslOpts) of - ok -> {ok, #{poolname => PoolName}}; + case emqx_resource_pool:start(InstId, ?MODULE, Opts ++ SslOpts) of + ok -> {ok, #{pool_name => InstId}}; {error, Reason} -> {error, Reason} end. -on_stop(InstId, #{poolname := PoolName}) -> +on_stop(InstId, #{pool_name := PoolName}) -> ?SLOG(info, #{ msg => "stopping_ldap_connector", connector => InstId }), - emqx_plugin_libs_pool:stop_pool(PoolName). + emqx_resource_pool:stop(PoolName). -on_query(InstId, {search, Base, Filter, Attributes}, #{poolname := PoolName} = State) -> +on_query(InstId, {search, Base, Filter, Attributes}, #{pool_name := PoolName} = State) -> Request = {Base, Filter, Attributes}, ?TRACE( "QUERY", diff --git a/apps/emqx_connector/src/emqx_connector_mongo.erl b/apps/emqx_connector/src/emqx_connector_mongo.erl index ead3e2e49..a65a32842 100644 --- a/apps/emqx_connector/src/emqx_connector_mongo.erl +++ b/apps/emqx_connector/src/emqx_connector_mongo.erl @@ -162,7 +162,7 @@ on_start( rs -> "starting_mongodb_replica_set_connector"; sharded -> "starting_mongodb_sharded_connector" end, - ?SLOG(info, #{msg => Msg, connector => InstId, config => emqx_misc:redact(Config)}), + ?SLOG(info, #{msg => Msg, connector => InstId, config => emqx_utils:redact(Config)}), NConfig = #{hosts := Hosts} = maybe_resolve_srv_and_txt_records(Config), SslOpts = case maps:get(enable, SSL) of @@ -182,12 +182,11 @@ on_start( {options, init_topology_options(maps:to_list(Topology), [])}, {worker_options, init_worker_options(maps:to_list(NConfig), SslOpts)} ], - PoolName = emqx_plugin_libs_pool:pool_name(InstId), Collection = maps:get(collection, Config, <<"mqtt">>), - case emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Opts) of + case emqx_resource_pool:start(InstId, ?MODULE, Opts) of ok -> {ok, #{ - poolname => PoolName, + pool_name => InstId, type => Type, collection => Collection }}; @@ -195,17 +194,17 @@ on_start( {error, Reason} end. -on_stop(InstId, #{poolname := PoolName}) -> +on_stop(InstId, #{pool_name := PoolName}) -> ?SLOG(info, #{ msg => "stopping_mongodb_connector", connector => InstId }), - emqx_plugin_libs_pool:stop_pool(PoolName). + emqx_resource_pool:stop(PoolName). on_query( InstId, {send_message, Document}, - #{poolname := PoolName, collection := Collection} = State + #{pool_name := PoolName, collection := Collection} = State ) -> Request = {insert, Collection, Document}, ?TRACE( @@ -234,7 +233,7 @@ on_query( on_query( InstId, {Action, Collection, Filter, Projector}, - #{poolname := PoolName} = State + #{pool_name := PoolName} = State ) -> Request = {Action, Collection, Filter, Projector}, ?TRACE( @@ -263,8 +262,7 @@ on_query( {ok, Result} end. --dialyzer({nowarn_function, [on_get_status/2]}). -on_get_status(InstId, #{poolname := PoolName} = _State) -> +on_get_status(InstId, #{pool_name := PoolName}) -> case health_check(PoolName) of true -> ?tp(debug, emqx_connector_mongo_health_check, #{ @@ -281,8 +279,10 @@ on_get_status(InstId, #{poolname := PoolName} = _State) -> end. health_check(PoolName) -> - emqx_plugin_libs_pool:health_check_ecpool_workers( - PoolName, fun ?MODULE:check_worker_health/1, ?HEALTH_CHECK_TIMEOUT + timer:seconds(1) + emqx_resource_pool:health_check_workers( + PoolName, + fun ?MODULE:check_worker_health/1, + ?HEALTH_CHECK_TIMEOUT + timer:seconds(1) ). %% =================================================================== diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl index 8f2d06517..5b488825b 100644 --- a/apps/emqx_connector/src/emqx_connector_mqtt.erl +++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl @@ -149,7 +149,7 @@ on_start(InstanceId, Conf) -> ?SLOG(info, #{ msg => "starting_mqtt_connector", connector => InstanceId, - config => emqx_misc:redact(Conf) + config => emqx_utils:redact(Conf) }), BasicConf = basic_config(Conf), BridgeConf = BasicConf#{ diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl index fe495252a..45d459e70 100644 --- a/apps/emqx_connector/src/emqx_connector_mysql.erl +++ b/apps/emqx_connector/src/emqx_connector_mysql.erl @@ -51,7 +51,7 @@ -type sqls() :: #{atom() => binary()}. -type state() :: #{ - poolname := atom(), + pool_name := binary(), prepare_statement := prepares(), params_tokens := params_tokens(), batch_inserts := sqls(), @@ -102,7 +102,7 @@ on_start( ?SLOG(info, #{ msg => "starting_mysql_connector", connector => InstId, - config => emqx_misc:redact(Config) + config => emqx_utils:redact(Config) }), SslOpts = case maps:get(enable, SSL) of @@ -123,13 +123,10 @@ on_start( {pool_size, PoolSize} ] ), - - PoolName = emqx_plugin_libs_pool:pool_name(InstId), - Prepares = parse_prepare_sql(Config), - State = maps:merge(#{poolname => PoolName}, Prepares), - case emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Options ++ SslOpts) of + State = parse_prepare_sql(Config), + case emqx_resource_pool:start(InstId, ?MODULE, Options ++ SslOpts) of ok -> - {ok, init_prepare(State)}; + {ok, init_prepare(State#{pool_name => InstId})}; {error, Reason} -> ?tp( mysql_connector_start_failed, @@ -143,12 +140,12 @@ maybe_add_password_opt(undefined, Options) -> maybe_add_password_opt(Password, Options) -> [{password, Password} | Options]. -on_stop(InstId, #{poolname := PoolName}) -> +on_stop(InstId, #{pool_name := PoolName}) -> ?SLOG(info, #{ msg => "stopping_mysql_connector", connector => InstId }), - emqx_plugin_libs_pool:stop_pool(PoolName). + emqx_resource_pool:stop(PoolName). on_query(InstId, {TypeOrKey, SQLOrKey}, State) -> on_query(InstId, {TypeOrKey, SQLOrKey, [], default_timeout}, State); @@ -157,7 +154,7 @@ on_query(InstId, {TypeOrKey, SQLOrKey, Params}, State) -> on_query( InstId, {TypeOrKey, SQLOrKey, Params, Timeout}, - #{poolname := PoolName, prepare_statement := Prepares} = State + #{pool_name := PoolName, prepare_statement := Prepares} = State ) -> MySqlFunction = mysql_function(TypeOrKey), {SQLOrKey2, Data} = proc_sql_params(TypeOrKey, SQLOrKey, Params, State), @@ -216,8 +213,8 @@ mysql_function(prepared_query) -> mysql_function(_) -> mysql_function(prepared_query). -on_get_status(_InstId, #{poolname := Pool} = State) -> - case emqx_plugin_libs_pool:health_check_ecpool_workers(Pool, fun ?MODULE:do_get_status/1) of +on_get_status(_InstId, #{pool_name := PoolName} = State) -> + case emqx_resource_pool:health_check_workers(PoolName, fun ?MODULE:do_get_status/1) of true -> case do_check_prepares(State) of ok -> @@ -238,7 +235,7 @@ do_get_status(Conn) -> do_check_prepares(#{prepare_statement := Prepares}) when is_map(Prepares) -> ok; -do_check_prepares(State = #{poolname := PoolName, prepare_statement := {error, Prepares}}) -> +do_check_prepares(State = #{pool_name := PoolName, prepare_statement := {error, Prepares}}) -> %% retry to prepare case prepare_sql(Prepares, PoolName) of ok -> @@ -253,7 +250,7 @@ do_check_prepares(State = #{poolname := PoolName, prepare_statement := {error, P connect(Options) -> mysql:start_link(Options). -init_prepare(State = #{prepare_statement := Prepares, poolname := PoolName}) -> +init_prepare(State = #{prepare_statement := Prepares, pool_name := PoolName}) -> case maps:size(Prepares) of 0 -> State; @@ -409,7 +406,7 @@ on_sql_query( SQLOrKey, Params, Timeout, - #{poolname := PoolName} = State + #{pool_name := PoolName} = State ) -> LogMeta = #{connector => InstId, sql => SQLOrKey, state => State}, ?TRACE("QUERY", "mysql_connector_received", LogMeta), @@ -437,7 +434,7 @@ do_sql_query(SQLFunc, Conn, SQLOrKey, Params, Timeout, LogMeta) -> error, LogMeta#{msg => "mysql_connector_do_sql_query_failed", reason => disconnected} ), - %% kill the poll worker to trigger reconnection + %% kill the pool worker to trigger reconnection _ = exit(Conn, restart), {error, {recoverable_error, disconnected}}; {error, not_prepared} = Error -> diff --git a/apps/emqx_connector/src/emqx_connector_pgsql.erl b/apps/emqx_connector/src/emqx_connector_pgsql.erl index 14cbbc80f..ddbf9491d 100644 --- a/apps/emqx_connector/src/emqx_connector_pgsql.erl +++ b/apps/emqx_connector/src/emqx_connector_pgsql.erl @@ -56,7 +56,7 @@ -type state() :: #{ - poolname := atom(), + pool_name := binary(), prepare_sql := prepares(), params_tokens := params_tokens(), prepare_statement := epgsql:statement() @@ -95,7 +95,7 @@ on_start( ?SLOG(info, #{ msg => "starting_postgresql_connector", connector => InstId, - config => emqx_misc:redact(Config) + config => emqx_utils:redact(Config) }), SslOpts = case maps:get(enable, SSL) of @@ -120,13 +120,10 @@ on_start( {auto_reconnect, ?AUTO_RECONNECT_INTERVAL}, {pool_size, PoolSize} ], - PoolName = emqx_plugin_libs_pool:pool_name(InstId), - Prepares = parse_prepare_sql(Config), - InitState = #{poolname => PoolName, prepare_statement => #{}}, - State = maps:merge(InitState, Prepares), - case emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Options ++ SslOpts) of + State = parse_prepare_sql(Config), + case emqx_resource_pool:start(InstId, ?MODULE, Options ++ SslOpts) of ok -> - {ok, init_prepare(State)}; + {ok, init_prepare(State#{pool_name => InstId, prepare_statement => #{}})}; {error, Reason} -> ?tp( pgsql_connector_start_failed, @@ -135,19 +132,19 @@ on_start( {error, Reason} end. -on_stop(InstId, #{poolname := PoolName}) -> +on_stop(InstId, #{pool_name := PoolName}) -> ?SLOG(info, #{ msg => "stopping postgresql connector", connector => InstId }), - emqx_plugin_libs_pool:stop_pool(PoolName). + emqx_resource_pool:stop(PoolName). -on_query(InstId, {TypeOrKey, NameOrSQL}, #{poolname := _PoolName} = State) -> +on_query(InstId, {TypeOrKey, NameOrSQL}, State) -> on_query(InstId, {TypeOrKey, NameOrSQL, []}, State); on_query( InstId, {TypeOrKey, NameOrSQL, Params}, - #{poolname := PoolName} = State + #{pool_name := PoolName} = State ) -> ?SLOG(debug, #{ msg => "postgresql connector received sql query", @@ -174,7 +171,7 @@ pgsql_query_type(_) -> on_batch_query( InstId, BatchReq, - #{poolname := PoolName, params_tokens := Tokens, prepare_statement := Sts} = State + #{pool_name := PoolName, params_tokens := Tokens, prepare_statement := Sts} = State ) -> case BatchReq of [{Key, _} = Request | _] -> @@ -258,8 +255,8 @@ on_sql_query(InstId, PoolName, Type, NameOrSQL, Data) -> {error, {unrecoverable_error, invalid_request}} end. -on_get_status(_InstId, #{poolname := Pool} = State) -> - case emqx_plugin_libs_pool:health_check_ecpool_workers(Pool, fun ?MODULE:do_get_status/1) of +on_get_status(_InstId, #{pool_name := PoolName} = State) -> + case emqx_resource_pool:health_check_workers(PoolName, fun ?MODULE:do_get_status/1) of true -> case do_check_prepares(State) of ok -> @@ -280,7 +277,7 @@ do_get_status(Conn) -> do_check_prepares(#{prepare_sql := Prepares}) when is_map(Prepares) -> ok; -do_check_prepares(State = #{poolname := PoolName, prepare_sql := {error, Prepares}}) -> +do_check_prepares(State = #{pool_name := PoolName, prepare_sql := {error, Prepares}}) -> %% retry to prepare case prepare_sql(Prepares, PoolName) of {ok, Sts} -> @@ -358,7 +355,7 @@ parse_prepare_sql([], Prepares, Tokens) -> params_tokens => Tokens }. -init_prepare(State = #{prepare_sql := Prepares, poolname := PoolName}) -> +init_prepare(State = #{prepare_sql := Prepares, pool_name := PoolName}) -> case maps:size(Prepares) of 0 -> State; @@ -389,17 +386,17 @@ prepare_sql(Prepares, PoolName) -> end. do_prepare_sql(Prepares, PoolName) -> - do_prepare_sql(ecpool:workers(PoolName), Prepares, PoolName, #{}). + do_prepare_sql(ecpool:workers(PoolName), Prepares, #{}). -do_prepare_sql([{_Name, Worker} | T], Prepares, PoolName, _LastSts) -> +do_prepare_sql([{_Name, Worker} | T], Prepares, _LastSts) -> {ok, Conn} = ecpool_worker:client(Worker), case prepare_sql_to_conn(Conn, Prepares) of {ok, Sts} -> - do_prepare_sql(T, Prepares, PoolName, Sts); + do_prepare_sql(T, Prepares, Sts); Error -> Error end; -do_prepare_sql([], _Prepares, _PoolName, LastSts) -> +do_prepare_sql([], _Prepares, LastSts) -> {ok, LastSts}. prepare_sql_to_conn(Conn, Prepares) -> diff --git a/apps/emqx_connector/src/emqx_connector_redis.erl b/apps/emqx_connector/src/emqx_connector_redis.erl index c70e766af..e2155eb49 100644 --- a/apps/emqx_connector/src/emqx_connector_redis.erl +++ b/apps/emqx_connector/src/emqx_connector_redis.erl @@ -123,7 +123,7 @@ on_start( ?SLOG(info, #{ msg => "starting_redis_connector", connector => InstId, - config => emqx_misc:redact(Config) + config => emqx_utils:redact(Config) }), ConfKey = case Type of @@ -153,11 +153,10 @@ on_start( false -> [{ssl, false}] end ++ [{sentinel, maps:get(sentinel, Config, undefined)}], - PoolName = InstId, - State = #{poolname => PoolName, type => Type}, + State = #{pool_name => InstId, type => Type}, case Type of cluster -> - case eredis_cluster:start_pool(PoolName, Opts ++ [{options, Options}]) of + case eredis_cluster:start_pool(InstId, Opts ++ [{options, Options}]) of {ok, _} -> {ok, State}; {ok, _, _} -> @@ -166,22 +165,20 @@ on_start( {error, Reason} end; _ -> - case - emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Opts ++ [{options, Options}]) - of + case emqx_resource_pool:start(InstId, ?MODULE, Opts ++ [{options, Options}]) of ok -> {ok, State}; {error, Reason} -> {error, Reason} end end. -on_stop(InstId, #{poolname := PoolName, type := Type}) -> +on_stop(InstId, #{pool_name := PoolName, type := Type}) -> ?SLOG(info, #{ msg => "stopping_redis_connector", connector => InstId }), case Type of cluster -> eredis_cluster:stop_pool(PoolName); - _ -> emqx_plugin_libs_pool:stop_pool(PoolName) + _ -> emqx_resource_pool:stop(PoolName) end. on_query(InstId, {cmd, _} = Query, State) -> @@ -189,7 +186,7 @@ on_query(InstId, {cmd, _} = Query, State) -> on_query(InstId, {cmds, _} = Query, State) -> do_query(InstId, Query, State). -do_query(InstId, Query, #{poolname := PoolName, type := Type} = State) -> +do_query(InstId, Query, #{pool_name := PoolName, type := Type} = State) -> ?TRACE( "QUERY", "redis_connector_received", @@ -227,7 +224,7 @@ is_unrecoverable_error({error, invalid_cluster_command}) -> is_unrecoverable_error(_) -> false. -on_get_status(_InstId, #{type := cluster, poolname := PoolName}) -> +on_get_status(_InstId, #{type := cluster, pool_name := PoolName}) -> case eredis_cluster:pool_exists(PoolName) of true -> Health = eredis_cluster:ping_all(PoolName), @@ -235,8 +232,8 @@ on_get_status(_InstId, #{type := cluster, poolname := PoolName}) -> false -> disconnected end; -on_get_status(_InstId, #{poolname := Pool}) -> - Health = emqx_plugin_libs_pool:health_check_ecpool_workers(Pool, fun ?MODULE:do_get_status/1), +on_get_status(_InstId, #{pool_name := PoolName}) -> + Health = emqx_resource_pool:health_check_workers(PoolName, fun ?MODULE:do_get_status/1), status_result(Health). do_get_status(Conn) -> diff --git a/apps/emqx_connector/src/emqx_connector_ssl.erl b/apps/emqx_connector/src/emqx_connector_ssl.erl index 54dc0e022..e07d95d51 100644 --- a/apps/emqx_connector/src/emqx_connector_ssl.erl +++ b/apps/emqx_connector/src/emqx_connector_ssl.erl @@ -74,7 +74,7 @@ new_ssl_config(Config, _NewSSL) -> normalize_key_to_bin(undefined) -> undefined; normalize_key_to_bin(Map) when is_map(Map) -> - emqx_map_lib:binary_key_map(Map). + emqx_utils_maps:binary_key_map(Map). try_map_get(Key, Map, Default) when is_map(Map) -> maps:get(Key, Map, Default); diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl index defbbaea2..df1114483 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl @@ -85,7 +85,7 @@ to_remote_msg(MapMsg, #{ qos = QoS, retain = Retain, topic = topic(Mountpoint, Topic), - props = emqx_misc:pub_props_to_packet(PubProps), + props = emqx_utils:pub_props_to_packet(PubProps), payload = Payload }; to_remote_msg(#message{topic = Topic} = Msg, #{mountpoint := Mountpoint}) -> @@ -112,7 +112,7 @@ to_broker_msg( Retain = replace_simple_var(RetainToken, MapMsg), PubProps = maps:get(pub_props, MapMsg, #{}), set_headers( - Props#{properties => emqx_misc:pub_props_to_packet(PubProps)}, + Props#{properties => emqx_utils:pub_props_to_packet(PubProps)}, emqx_message:set_flags( #{dup => Dup, retain => Retain}, emqx_message:make(bridge, QoS, topic(Mountpoint, Topic), Payload) @@ -123,7 +123,7 @@ process_payload(From, MapMsg) -> do_process_payload(maps:get(payload, From, undefined), MapMsg). do_process_payload(undefined, Msg) -> - emqx_json:encode(Msg); + emqx_utils_json:encode(Msg); do_process_payload(Tks, Msg) -> replace_vars_in_str(Tks, Msg). diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl index 8e41a9f0f..880a99313 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl @@ -124,7 +124,7 @@ start_link(Name, BridgeOpts) -> {error, Reason} = Error -> ?SLOG(error, #{ msg => "client_start_failed", - config => emqx_misc:redact(BridgeOpts), + config => emqx_utils:redact(BridgeOpts), reason => Reason }), Error @@ -410,7 +410,7 @@ handle_disconnect(_Reason) -> ok. maybe_publish_local(Msg, Vars, Props) -> - case emqx_map_lib:deep_get([local, topic], Vars, undefined) of + case emqx_utils_maps:deep_get([local, topic], Vars, undefined) of %% local topic is not set, discard it undefined -> ok; diff --git a/apps/emqx_connector/test/emqx_connector_mongo_SUITE.erl b/apps/emqx_connector/test/emqx_connector_mongo_SUITE.erl index 2be30466c..9067c85de 100644 --- a/apps/emqx_connector/test/emqx_connector_mongo_SUITE.erl +++ b/apps/emqx_connector/test/emqx_connector_mongo_SUITE.erl @@ -64,15 +64,15 @@ t_lifecycle(_Config) -> mongo_config() ). -perform_lifecycle_check(PoolName, InitialConfig) -> +perform_lifecycle_check(ResourceId, InitialConfig) -> {ok, #{config := CheckedConfig}} = emqx_resource:check_config(?MONGO_RESOURCE_MOD, InitialConfig), {ok, #{ - state := #{poolname := ReturnedPoolName} = State, + state := #{pool_name := PoolName} = State, status := InitialStatus }} = emqx_resource:create_local( - PoolName, + ResourceId, ?CONNECTOR_RESOURCE_GROUP, ?MONGO_RESOURCE_MOD, CheckedConfig, @@ -84,39 +84,39 @@ perform_lifecycle_check(PoolName, InitialConfig) -> state := State, status := InitialStatus }} = - emqx_resource:get_instance(PoolName), - ?assertEqual({ok, connected}, emqx_resource:health_check(PoolName)), + emqx_resource:get_instance(ResourceId), + ?assertEqual({ok, connected}, emqx_resource:health_check(ResourceId)), % % Perform query as further check that the resource is working as expected - ?assertMatch({ok, []}, emqx_resource:query(PoolName, test_query_find())), - ?assertMatch({ok, undefined}, emqx_resource:query(PoolName, test_query_find_one())), - ?assertEqual(ok, emqx_resource:stop(PoolName)), + ?assertMatch({ok, []}, emqx_resource:query(ResourceId, test_query_find())), + ?assertMatch({ok, undefined}, emqx_resource:query(ResourceId, test_query_find_one())), + ?assertEqual(ok, emqx_resource:stop(ResourceId)), % Resource will be listed still, but state will be changed and healthcheck will fail % as the worker no longer exists. {ok, ?CONNECTOR_RESOURCE_GROUP, #{ state := State, status := StoppedStatus }} = - emqx_resource:get_instance(PoolName), + emqx_resource:get_instance(ResourceId), ?assertEqual(stopped, StoppedStatus), - ?assertEqual({error, resource_is_stopped}, emqx_resource:health_check(PoolName)), + ?assertEqual({error, resource_is_stopped}, emqx_resource:health_check(ResourceId)), % Resource healthcheck shortcuts things by checking ets. Go deeper by checking pool itself. - ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)), + ?assertEqual({error, not_found}, ecpool:stop_sup_pool(PoolName)), % Can call stop/1 again on an already stopped instance - ?assertEqual(ok, emqx_resource:stop(PoolName)), + ?assertEqual(ok, emqx_resource:stop(ResourceId)), % Make sure it can be restarted and the healthchecks and queries work properly - ?assertEqual(ok, emqx_resource:restart(PoolName)), + ?assertEqual(ok, emqx_resource:restart(ResourceId)), % async restart, need to wait resource timer:sleep(500), {ok, ?CONNECTOR_RESOURCE_GROUP, #{status := InitialStatus}} = - emqx_resource:get_instance(PoolName), - ?assertEqual({ok, connected}, emqx_resource:health_check(PoolName)), - ?assertMatch({ok, []}, emqx_resource:query(PoolName, test_query_find())), - ?assertMatch({ok, undefined}, emqx_resource:query(PoolName, test_query_find_one())), + emqx_resource:get_instance(ResourceId), + ?assertEqual({ok, connected}, emqx_resource:health_check(ResourceId)), + ?assertMatch({ok, []}, emqx_resource:query(ResourceId, test_query_find())), + ?assertMatch({ok, undefined}, emqx_resource:query(ResourceId, test_query_find_one())), % Stop and remove the resource in one go. - ?assertEqual(ok, emqx_resource:remove_local(PoolName)), - ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)), + ?assertEqual(ok, emqx_resource:remove_local(ResourceId)), + ?assertEqual({error, not_found}, ecpool:stop_sup_pool(PoolName)), % Should not even be able to get the resource data out of ets now unlike just stopping. - ?assertEqual({error, not_found}, emqx_resource:get_instance(PoolName)). + ?assertEqual({error, not_found}, emqx_resource:get_instance(ResourceId)). % %%------------------------------------------------------------------------------ % %% Helpers diff --git a/apps/emqx_connector/test/emqx_connector_mysql_SUITE.erl b/apps/emqx_connector/test/emqx_connector_mysql_SUITE.erl index dc5826766..a0455c92c 100644 --- a/apps/emqx_connector/test/emqx_connector_mysql_SUITE.erl +++ b/apps/emqx_connector/test/emqx_connector_mysql_SUITE.erl @@ -64,14 +64,14 @@ t_lifecycle(_Config) -> mysql_config() ). -perform_lifecycle_check(PoolName, InitialConfig) -> +perform_lifecycle_check(ResourceId, InitialConfig) -> {ok, #{config := CheckedConfig}} = emqx_resource:check_config(?MYSQL_RESOURCE_MOD, InitialConfig), {ok, #{ - state := #{poolname := ReturnedPoolName} = State, + state := #{pool_name := PoolName} = State, status := InitialStatus }} = emqx_resource:create_local( - PoolName, + ResourceId, ?CONNECTOR_RESOURCE_GROUP, ?MYSQL_RESOURCE_MOD, CheckedConfig, @@ -83,53 +83,53 @@ perform_lifecycle_check(PoolName, InitialConfig) -> state := State, status := InitialStatus }} = - emqx_resource:get_instance(PoolName), - ?assertEqual({ok, connected}, emqx_resource:health_check(PoolName)), + emqx_resource:get_instance(ResourceId), + ?assertEqual({ok, connected}, emqx_resource:health_check(ResourceId)), % % Perform query as further check that the resource is working as expected - ?assertMatch({ok, _, [[1]]}, emqx_resource:query(PoolName, test_query_no_params())), - ?assertMatch({ok, _, [[1]]}, emqx_resource:query(PoolName, test_query_with_params())), + ?assertMatch({ok, _, [[1]]}, emqx_resource:query(ResourceId, test_query_no_params())), + ?assertMatch({ok, _, [[1]]}, emqx_resource:query(ResourceId, test_query_with_params())), ?assertMatch( {ok, _, [[1]]}, emqx_resource:query( - PoolName, + ResourceId, test_query_with_params_and_timeout() ) ), - ?assertEqual(ok, emqx_resource:stop(PoolName)), + ?assertEqual(ok, emqx_resource:stop(ResourceId)), % Resource will be listed still, but state will be changed and healthcheck will fail % as the worker no longer exists. {ok, ?CONNECTOR_RESOURCE_GROUP, #{ state := State, status := StoppedStatus }} = - emqx_resource:get_instance(PoolName), + emqx_resource:get_instance(ResourceId), ?assertEqual(stopped, StoppedStatus), - ?assertEqual({error, resource_is_stopped}, emqx_resource:health_check(PoolName)), + ?assertEqual({error, resource_is_stopped}, emqx_resource:health_check(ResourceId)), % Resource healthcheck shortcuts things by checking ets. Go deeper by checking pool itself. - ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)), + ?assertEqual({error, not_found}, ecpool:stop_sup_pool(PoolName)), % Can call stop/1 again on an already stopped instance - ?assertEqual(ok, emqx_resource:stop(PoolName)), + ?assertEqual(ok, emqx_resource:stop(ResourceId)), % Make sure it can be restarted and the healthchecks and queries work properly - ?assertEqual(ok, emqx_resource:restart(PoolName)), + ?assertEqual(ok, emqx_resource:restart(ResourceId)), % async restart, need to wait resource timer:sleep(500), {ok, ?CONNECTOR_RESOURCE_GROUP, #{status := InitialStatus}} = - emqx_resource:get_instance(PoolName), - ?assertEqual({ok, connected}, emqx_resource:health_check(PoolName)), - ?assertMatch({ok, _, [[1]]}, emqx_resource:query(PoolName, test_query_no_params())), - ?assertMatch({ok, _, [[1]]}, emqx_resource:query(PoolName, test_query_with_params())), + emqx_resource:get_instance(ResourceId), + ?assertEqual({ok, connected}, emqx_resource:health_check(ResourceId)), + ?assertMatch({ok, _, [[1]]}, emqx_resource:query(ResourceId, test_query_no_params())), + ?assertMatch({ok, _, [[1]]}, emqx_resource:query(ResourceId, test_query_with_params())), ?assertMatch( {ok, _, [[1]]}, emqx_resource:query( - PoolName, + ResourceId, test_query_with_params_and_timeout() ) ), % Stop and remove the resource in one go. - ?assertEqual(ok, emqx_resource:remove_local(PoolName)), - ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)), + ?assertEqual(ok, emqx_resource:remove_local(ResourceId)), + ?assertEqual({error, not_found}, ecpool:stop_sup_pool(PoolName)), % Should not even be able to get the resource data out of ets now unlike just stopping. - ?assertEqual({error, not_found}, emqx_resource:get_instance(PoolName)). + ?assertEqual({error, not_found}, emqx_resource:get_instance(ResourceId)). % %%------------------------------------------------------------------------------ % %% Helpers diff --git a/apps/emqx_connector/test/emqx_connector_pgsql_SUITE.erl b/apps/emqx_connector/test/emqx_connector_pgsql_SUITE.erl index 2f77ca38d..a4ac4f932 100644 --- a/apps/emqx_connector/test/emqx_connector_pgsql_SUITE.erl +++ b/apps/emqx_connector/test/emqx_connector_pgsql_SUITE.erl @@ -64,15 +64,15 @@ t_lifecycle(_Config) -> pgsql_config() ). -perform_lifecycle_check(PoolName, InitialConfig) -> +perform_lifecycle_check(ResourceId, InitialConfig) -> {ok, #{config := CheckedConfig}} = emqx_resource:check_config(?PGSQL_RESOURCE_MOD, InitialConfig), {ok, #{ - state := #{poolname := ReturnedPoolName} = State, + state := #{pool_name := PoolName} = State, status := InitialStatus }} = emqx_resource:create_local( - PoolName, + ResourceId, ?CONNECTOR_RESOURCE_GROUP, ?PGSQL_RESOURCE_MOD, CheckedConfig, @@ -84,39 +84,39 @@ perform_lifecycle_check(PoolName, InitialConfig) -> state := State, status := InitialStatus }} = - emqx_resource:get_instance(PoolName), - ?assertEqual({ok, connected}, emqx_resource:health_check(PoolName)), + emqx_resource:get_instance(ResourceId), + ?assertEqual({ok, connected}, emqx_resource:health_check(ResourceId)), % % Perform query as further check that the resource is working as expected - ?assertMatch({ok, _, [{1}]}, emqx_resource:query(PoolName, test_query_no_params())), - ?assertMatch({ok, _, [{1}]}, emqx_resource:query(PoolName, test_query_with_params())), - ?assertEqual(ok, emqx_resource:stop(PoolName)), + ?assertMatch({ok, _, [{1}]}, emqx_resource:query(ResourceId, test_query_no_params())), + ?assertMatch({ok, _, [{1}]}, emqx_resource:query(ResourceId, test_query_with_params())), + ?assertEqual(ok, emqx_resource:stop(ResourceId)), % Resource will be listed still, but state will be changed and healthcheck will fail % as the worker no longer exists. {ok, ?CONNECTOR_RESOURCE_GROUP, #{ state := State, status := StoppedStatus }} = - emqx_resource:get_instance(PoolName), + emqx_resource:get_instance(ResourceId), ?assertEqual(stopped, StoppedStatus), - ?assertEqual({error, resource_is_stopped}, emqx_resource:health_check(PoolName)), + ?assertEqual({error, resource_is_stopped}, emqx_resource:health_check(ResourceId)), % Resource healthcheck shortcuts things by checking ets. Go deeper by checking pool itself. - ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)), + ?assertEqual({error, not_found}, ecpool:stop_sup_pool(PoolName)), % Can call stop/1 again on an already stopped instance - ?assertEqual(ok, emqx_resource:stop(PoolName)), + ?assertEqual(ok, emqx_resource:stop(ResourceId)), % Make sure it can be restarted and the healthchecks and queries work properly - ?assertEqual(ok, emqx_resource:restart(PoolName)), + ?assertEqual(ok, emqx_resource:restart(ResourceId)), % async restart, need to wait resource timer:sleep(500), {ok, ?CONNECTOR_RESOURCE_GROUP, #{status := InitialStatus}} = - emqx_resource:get_instance(PoolName), - ?assertEqual({ok, connected}, emqx_resource:health_check(PoolName)), - ?assertMatch({ok, _, [{1}]}, emqx_resource:query(PoolName, test_query_no_params())), - ?assertMatch({ok, _, [{1}]}, emqx_resource:query(PoolName, test_query_with_params())), + emqx_resource:get_instance(ResourceId), + ?assertEqual({ok, connected}, emqx_resource:health_check(ResourceId)), + ?assertMatch({ok, _, [{1}]}, emqx_resource:query(ResourceId, test_query_no_params())), + ?assertMatch({ok, _, [{1}]}, emqx_resource:query(ResourceId, test_query_with_params())), % Stop and remove the resource in one go. - ?assertEqual(ok, emqx_resource:remove_local(PoolName)), - ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)), + ?assertEqual(ok, emqx_resource:remove_local(ResourceId)), + ?assertEqual({error, not_found}, ecpool:stop_sup_pool(PoolName)), % Should not even be able to get the resource data out of ets now unlike just stopping. - ?assertEqual({error, not_found}, emqx_resource:get_instance(PoolName)). + ?assertEqual({error, not_found}, emqx_resource:get_instance(ResourceId)). % %%------------------------------------------------------------------------------ % %% Helpers diff --git a/apps/emqx_connector/test/emqx_connector_redis_SUITE.erl b/apps/emqx_connector/test/emqx_connector_redis_SUITE.erl index 3a134ad35..e6df4f711 100644 --- a/apps/emqx_connector/test/emqx_connector_redis_SUITE.erl +++ b/apps/emqx_connector/test/emqx_connector_redis_SUITE.erl @@ -102,14 +102,14 @@ t_sentinel_lifecycle(_Config) -> [<<"PING">>] ). -perform_lifecycle_check(PoolName, InitialConfig, RedisCommand) -> +perform_lifecycle_check(ResourceId, InitialConfig, RedisCommand) -> {ok, #{config := CheckedConfig}} = emqx_resource:check_config(?REDIS_RESOURCE_MOD, InitialConfig), {ok, #{ - state := #{poolname := ReturnedPoolName} = State, + state := #{pool_name := PoolName} = State, status := InitialStatus }} = emqx_resource:create_local( - PoolName, + ResourceId, ?CONNECTOR_RESOURCE_GROUP, ?REDIS_RESOURCE_MOD, CheckedConfig, @@ -121,49 +121,49 @@ perform_lifecycle_check(PoolName, InitialConfig, RedisCommand) -> state := State, status := InitialStatus }} = - emqx_resource:get_instance(PoolName), - ?assertEqual({ok, connected}, emqx_resource:health_check(PoolName)), + emqx_resource:get_instance(ResourceId), + ?assertEqual({ok, connected}, emqx_resource:health_check(ResourceId)), % Perform query as further check that the resource is working as expected - ?assertEqual({ok, <<"PONG">>}, emqx_resource:query(PoolName, {cmd, RedisCommand})), + ?assertEqual({ok, <<"PONG">>}, emqx_resource:query(ResourceId, {cmd, RedisCommand})), ?assertEqual( {ok, [{ok, <<"PONG">>}, {ok, <<"PONG">>}]}, - emqx_resource:query(PoolName, {cmds, [RedisCommand, RedisCommand]}) + emqx_resource:query(ResourceId, {cmds, [RedisCommand, RedisCommand]}) ), ?assertMatch( {error, {unrecoverable_error, [{ok, <<"PONG">>}, {error, _}]}}, emqx_resource:query( - PoolName, + ResourceId, {cmds, [RedisCommand, [<<"INVALID_COMMAND">>]]}, #{timeout => 500} ) ), - ?assertEqual(ok, emqx_resource:stop(PoolName)), + ?assertEqual(ok, emqx_resource:stop(ResourceId)), % Resource will be listed still, but state will be changed and healthcheck will fail % as the worker no longer exists. {ok, ?CONNECTOR_RESOURCE_GROUP, #{ state := State, status := StoppedStatus }} = - emqx_resource:get_instance(PoolName), + emqx_resource:get_instance(ResourceId), ?assertEqual(stopped, StoppedStatus), - ?assertEqual({error, resource_is_stopped}, emqx_resource:health_check(PoolName)), + ?assertEqual({error, resource_is_stopped}, emqx_resource:health_check(ResourceId)), % Resource healthcheck shortcuts things by checking ets. Go deeper by checking pool itself. - ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)), + ?assertEqual({error, not_found}, ecpool:stop_sup_pool(PoolName)), % Can call stop/1 again on an already stopped instance - ?assertEqual(ok, emqx_resource:stop(PoolName)), + ?assertEqual(ok, emqx_resource:stop(ResourceId)), % Make sure it can be restarted and the healthchecks and queries work properly - ?assertEqual(ok, emqx_resource:restart(PoolName)), + ?assertEqual(ok, emqx_resource:restart(ResourceId)), % async restart, need to wait resource timer:sleep(500), {ok, ?CONNECTOR_RESOURCE_GROUP, #{status := InitialStatus}} = - emqx_resource:get_instance(PoolName), - ?assertEqual({ok, connected}, emqx_resource:health_check(PoolName)), - ?assertEqual({ok, <<"PONG">>}, emqx_resource:query(PoolName, {cmd, RedisCommand})), + emqx_resource:get_instance(ResourceId), + ?assertEqual({ok, connected}, emqx_resource:health_check(ResourceId)), + ?assertEqual({ok, <<"PONG">>}, emqx_resource:query(ResourceId, {cmd, RedisCommand})), % Stop and remove the resource in one go. - ?assertEqual(ok, emqx_resource:remove_local(PoolName)), - ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)), + ?assertEqual(ok, emqx_resource:remove_local(ResourceId)), + ?assertEqual({error, not_found}, ecpool:stop_sup_pool(PoolName)), % Should not even be able to get the resource data out of ets now unlike just stopping. - ?assertEqual({error, not_found}, emqx_resource:get_instance(PoolName)). + ?assertEqual({error, not_found}, emqx_resource:get_instance(ResourceId)). % %%------------------------------------------------------------------------------ % %% Helpers diff --git a/apps/emqx_connector/test/emqx_connector_web_hook_server.erl b/apps/emqx_connector/test/emqx_connector_web_hook_server.erl index b68ebcbba..bdc6e100c 100644 --- a/apps/emqx_connector/test/emqx_connector_web_hook_server.erl +++ b/apps/emqx_connector/test/emqx_connector_web_hook_server.erl @@ -29,7 +29,14 @@ start_link(Port, Path) -> start_link(Port, Path, false). start_link(Port, Path, SSLOpts) -> - supervisor:start_link({local, ?MODULE}, ?MODULE, [Port, Path, SSLOpts]). + case Port of + random -> + PickedPort = pick_port_number(56000), + {ok, Pid} = supervisor:start_link({local, ?MODULE}, ?MODULE, [PickedPort, Path, SSLOpts]), + {ok, {PickedPort, Pid}}; + _ -> + supervisor:start_link({local, ?MODULE}, ?MODULE, [Port, Path, SSLOpts]) + end. stop() -> try @@ -103,3 +110,20 @@ default_handler(Req0, State) -> Req0 ), {ok, Req, State}. + +pick_port_number(Port) -> + case is_port_in_use(Port) of + true -> + pick_port_number(Port + 1); + false -> + Port + end. + +is_port_in_use(Port) -> + case gen_tcp:listen(Port, [{reuseaddr, true}, {active, false}]) of + {ok, ListenSocket} -> + gen_tcp:close(ListenSocket), + false; + {error, eaddrinuse} -> + true + end. diff --git a/apps/emqx_dashboard/rebar.config b/apps/emqx_dashboard/rebar.config index 9657d0bbf..440fde465 100644 --- a/apps/emqx_dashboard/rebar.config +++ b/apps/emqx_dashboard/rebar.config @@ -1,6 +1,9 @@ %% -*- mode: erlang -*- -{deps, [{emqx, {path, "../emqx"}}]}. +{deps, [ + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}} +]}. {edoc_opts, [{preprocess, true}]}. {erl_opts, [ diff --git a/apps/emqx_dashboard/src/emqx_dashboard.app.src b/apps/emqx_dashboard/src/emqx_dashboard.app.src index 8a4764c84..8c7e424e0 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.app.src +++ b/apps/emqx_dashboard/src/emqx_dashboard.app.src @@ -2,7 +2,7 @@ {application, emqx_dashboard, [ {description, "EMQX Web Dashboard"}, % strict semver, bump manually! - {vsn, "5.0.16"}, + {vsn, "5.0.19"}, {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [kernel, stdlib, mnesia, minirest, emqx, emqx_ctl]}, diff --git a/apps/emqx_dashboard/src/emqx_dashboard.erl b/apps/emqx_dashboard/src/emqx_dashboard.erl index 53b661ea8..13fd18267 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard.erl @@ -16,22 +16,13 @@ -module(emqx_dashboard). --define(APP, ?MODULE). - -export([ start_listeners/0, start_listeners/1, stop_listeners/1, stop_listeners/0, - list_listeners/0 -]). - --export([ - init_i18n/2, - init_i18n/0, - get_i18n/0, - i18n_file/0, - clear_i18n/0 + list_listeners/0, + wait_for_listeners/0 ]). %% Authorization @@ -88,30 +79,34 @@ start_listeners(Listeners) -> dispatch => Dispatch, middlewares => [?EMQX_MIDDLE, cowboy_router, cowboy_handler] }, - Res = + {OkListeners, ErrListeners} = lists:foldl( - fun({Name, Protocol, Bind, RanchOptions, ProtoOpts}, Acc) -> + fun({Name, Protocol, Bind, RanchOptions, ProtoOpts}, {OkAcc, ErrAcc}) -> Minirest = BaseMinirest#{protocol => Protocol, protocol_options => ProtoOpts}, case minirest:start(Name, RanchOptions, Minirest) of {ok, _} -> ?ULOG("Listener ~ts on ~ts started.~n", [ Name, emqx_listeners:format_bind(Bind) ]), - Acc; + {[Name | OkAcc], ErrAcc}; {error, _Reason} -> %% Don't record the reason because minirest already does(too much logs noise). - [Name | Acc] + {OkAcc, [Name | ErrAcc]} end end, - [], + {[], []}, listeners(Listeners) ), - case Res of - [] -> ok; - _ -> {error, Res} + case ErrListeners of + [] -> + optvar:set(emqx_dashboard_listeners_ready, OkListeners), + ok; + _ -> + {error, ErrListeners} end. stop_listeners(Listeners) -> + optvar:unset(emqx_dashboard_listeners_ready), [ begin case minirest:stop(Name) of @@ -127,23 +122,8 @@ stop_listeners(Listeners) -> ], ok. -get_i18n() -> - application:get_env(emqx_dashboard, i18n). - -init_i18n(File, Lang) when is_atom(Lang) -> - init_i18n(File, atom_to_binary(Lang)); -init_i18n(File, Lang) when is_binary(Lang) -> - Cache = hocon_schema:new_desc_cache(File), - application:set_env(emqx_dashboard, i18n, #{lang => Lang, cache => Cache}). - -clear_i18n() -> - case application:get_env(emqx_dashboard, i18n) of - {ok, #{cache := Cache}} -> - hocon_schema:delete_desc_cache(Cache), - application:unset_env(emqx_dashboard, i18n); - undefined -> - ok - end. +wait_for_listeners() -> + optvar:read(emqx_dashboard_listeners_ready). %%-------------------------------------------------------------------- %% internal @@ -185,11 +165,6 @@ ip_port(error, Opts) -> {Opts#{port => 18083}, 18083}; ip_port({Port, Opts}, _) when is_integer(Port) -> {Opts#{port => Port}, Port}; ip_port({{IP, Port}, Opts}, _) -> {Opts#{port => Port, ip => IP}, {IP, Port}}. -init_i18n() -> - File = i18n_file(), - Lang = emqx_conf:get([dashboard, i18n_lang], en), - init_i18n(File, Lang). - ranch_opts(Options) -> Keys = [ handshake_timeout, @@ -253,12 +228,6 @@ return_unauthorized(Code, Message) -> }, #{code => Code, message => Message}}. -i18n_file() -> - case application:get_env(emqx_dashboard, i18n_file) of - undefined -> filename:join([code:priv_dir(emqx_dashboard), "i18n.conf"]); - {ok, File} -> File - end. - listeners() -> emqx_conf:get([dashboard, listeners], #{}). diff --git a/apps/emqx_dashboard/src/emqx_dashboard_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_api.erl index d5655d99d..108cde379 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_api.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_api.erl @@ -18,7 +18,6 @@ -behaviour(minirest_api). --include("emqx_dashboard.hrl"). -include_lib("hocon/include/hoconsc.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("typerefl/include/types.hrl"). diff --git a/apps/emqx_dashboard/src/emqx_dashboard_desc_cache.erl b/apps/emqx_dashboard/src/emqx_dashboard_desc_cache.erl new file mode 100644 index 000000000..9d8d1905d --- /dev/null +++ b/apps/emqx_dashboard/src/emqx_dashboard_desc_cache.erl @@ -0,0 +1,110 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @doc This module is used to cache the description of the configuration items. +-module(emqx_dashboard_desc_cache). + +-export([init/0]). + +%% internal exports +-export([load_desc/2, lookup/4, lookup/5]). + +-include_lib("emqx/include/logger.hrl"). + +%% @doc Global ETS table to cache the description of the configuration items. +%% The table is owned by the emqx_dashboard_sup the root supervisor of emqx_dashboard. +%% The cache is initialized with the default language (English) and +%% all the desc..hocon files in the www/static directory (extracted from dashboard package). +init() -> + ok = ensure_app_loaded(emqx_dashboard), + PrivDir = code:priv_dir(emqx_dashboard), + EngDesc = filename:join([PrivDir, "desc.en.hocon"]), + WwwStaticDir = filename:join([PrivDir, "www", "static"]), + OtherLangDesc0 = filelib:wildcard("desc.*.hocon", WwwStaticDir), + OtherLangDesc = lists:map(fun(F) -> filename:join([WwwStaticDir, F]) end, OtherLangDesc0), + Files = [EngDesc | OtherLangDesc], + ?MODULE = ets:new(?MODULE, [named_table, public, set, {read_concurrency, true}]), + ok = lists:foreach(fun(F) -> load_desc(?MODULE, F) end, Files). + +%% @doc Load the description of the configuration items from the file. +%% Load is incremental, so it can be called multiple times. +%% NOTE: no garbage collection is done, because stale entries are harmless. +load_desc(EtsTab, File) -> + ?SLOG(info, #{msg => "loading desc", file => File}), + {ok, Descs} = hocon:load(File), + ["desc", Lang, "hocon"] = string:tokens(filename:basename(File), "."), + Insert = fun(Namespace, Id, Tag, Text) -> + Key = {bin(Lang), bin(Namespace), bin(Id), bin(Tag)}, + true = ets:insert(EtsTab, {Key, bin(Text)}), + ok + end, + walk_ns(Insert, maps:to_list(Descs)). + +%% @doc Lookup the description of the configuration item from the global cache. +lookup(Lang, Namespace, Id, Tag) -> + lookup(?MODULE, Lang, Namespace, Id, Tag). + +%% @doc Lookup the description of the configuration item from the given cache. +lookup(EtsTab, Lang0, Namespace, Id, Tag) -> + Lang = bin(Lang0), + try ets:lookup(EtsTab, {Lang, bin(Namespace), bin(Id), bin(Tag)}) of + [{_, Desc}] -> + Desc; + [] when Lang =/= <<"en">> -> + %% fallback to English + lookup(EtsTab, <<"en">>, Namespace, Id, Tag); + _ -> + %% undefined but not <<>> + undefined + catch + error:badarg -> + %% schema is not initialized + %% most likely in test cases + undefined + end. + +%% The desc files are of names like: +%% desc.en.hocon or desc.zh.hocon +%% And with content like: +%% namespace.id.desc = "description" +%% namespace.id.label = "label" +walk_ns(_Insert, []) -> + ok; +walk_ns(Insert, [{Namespace, Ids} | Rest]) -> + walk_id(Insert, Namespace, maps:to_list(Ids)), + walk_ns(Insert, Rest). + +walk_id(_Insert, _Namespace, []) -> + ok; +walk_id(Insert, Namespace, [{Id, Tags} | Rest]) -> + walk_tag(Insert, Namespace, Id, maps:to_list(Tags)), + walk_id(Insert, Namespace, Rest). + +walk_tag(_Insert, _Namespace, _Id, []) -> + ok; +walk_tag(Insert, Namespace, Id, [{Tag, Text} | Rest]) -> + ok = Insert(Namespace, Id, Tag, Text), + walk_tag(Insert, Namespace, Id, Rest). + +bin(A) when is_atom(A) -> atom_to_binary(A, utf8); +bin(B) when is_binary(B) -> B; +bin(L) when is_list(L) -> list_to_binary(L). + +ensure_app_loaded(App) -> + case application:load(App) of + ok -> ok; + {error, {already_loaded, _}} -> ok + end. diff --git a/apps/emqx_dashboard/src/emqx_dashboard_listener.erl b/apps/emqx_dashboard/src/emqx_dashboard_listener.erl index eac4f845f..6a306c288 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_listener.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_listener.erl @@ -15,9 +15,11 @@ %%-------------------------------------------------------------------- -module(emqx_dashboard_listener). --include_lib("emqx/include/logger.hrl"). -behaviour(emqx_config_handler). +-include_lib("emqx/include/logger.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + %% API -export([add_handler/0, remove_handler/0]). -export([pre_config_update/3, post_config_update/5]). @@ -54,12 +56,10 @@ init([]) -> {ok, undefined, {continue, regenerate_dispatch}}. handle_continue(regenerate_dispatch, _State) -> - NewState = regenerate_minirest_dispatch(), - {noreply, NewState, hibernate}. + %% initialize the swagger dispatches + ready = regenerate_minirest_dispatch(), + {noreply, ready, hibernate}. -handle_call(is_ready, _From, retry) -> - NewState = regenerate_minirest_dispatch(), - {reply, NewState, NewState, hibernate}; handle_call(is_ready, _From, State) -> {reply, State, State, hibernate}; handle_call(_Request, _From, State) -> @@ -68,6 +68,9 @@ handle_call(_Request, _From, State) -> handle_cast(_Request, State) -> {noreply, State, hibernate}. +handle_info(i18n_lang_changed, _State) -> + NewState = regenerate_minirest_dispatch(), + {noreply, NewState, hibernate}; handle_info({update_listeners, OldListeners, NewListeners}, _State) -> ok = emqx_dashboard:stop_listeners(OldListeners), ok = emqx_dashboard:start_listeners(NewListeners), @@ -83,29 +86,26 @@ terminate(_Reason, _State) -> code_change(_OldVsn, State, _Extra) -> {ok, State}. -%% generate dispatch is very slow. +%% generate dispatch is very slow, takes about 1s. regenerate_minirest_dispatch() -> - try - emqx_dashboard:init_i18n(), - lists:foreach( - fun(Listener) -> - minirest:update_dispatch(element(1, Listener)) - end, - emqx_dashboard:list_listeners() - ), - ready - catch - T:E:S -> - ?SLOG(error, #{ - msg => "regenerate_minirest_dispatch_failed", - reason => E, - type => T, - stacktrace => S - }), - retry - after - emqx_dashboard:clear_i18n() - end. + %% optvar:read waits for the var to be set + Names = emqx_dashboard:wait_for_listeners(), + {Time, ok} = timer:tc(fun() -> do_regenerate_minirest_dispatch(Names) end), + Lang = emqx:get_config([dashboard, i18n_lang]), + ?tp(info, regenerate_minirest_dispatch, #{ + elapsed => erlang:convert_time_unit(Time, microsecond, millisecond), + listeners => Names, + i18n_lang => Lang + }), + ready. + +do_regenerate_minirest_dispatch(Names) -> + lists:foreach( + fun(Name) -> + ok = minirest:update_dispatch(Name) + end, + Names + ). add_handler() -> Roots = emqx_dashboard_schema:roots(), @@ -117,9 +117,15 @@ remove_handler() -> ok = emqx_config_handler:remove_handler(Roots), ok. +pre_config_update(_Path, {change_i18n_lang, NewLang}, RawConf) -> + %% e.g. emqx_conf:update([dashboard], {change_i18n_lang, zh}, #{}). + %% TODO: check if there is such a language (all languages are cached in emqx_dashboard_desc_cache) + Update = #{<<"i18n_lang">> => NewLang}, + NewConf = emqx_utils_maps:deep_merge(RawConf, Update), + {ok, NewConf}; pre_config_update(_Path, UpdateConf0, RawConf) -> UpdateConf = remove_sensitive_data(UpdateConf0), - NewConf = emqx_map_lib:deep_merge(RawConf, UpdateConf), + NewConf = emqx_utils_maps:deep_merge(RawConf, UpdateConf), ensure_ssl_cert(NewConf). -define(SENSITIVE_PASSWORD, <<"******">>). @@ -134,11 +140,13 @@ remove_sensitive_data(Conf0) -> end, case Conf1 of #{<<"listeners">> := #{<<"https">> := #{<<"password">> := ?SENSITIVE_PASSWORD}}} -> - emqx_map_lib:deep_remove([<<"listeners">>, <<"https">>, <<"password">>], Conf1); + emqx_utils_maps:deep_remove([<<"listeners">>, <<"https">>, <<"password">>], Conf1); _ -> Conf1 end. +post_config_update(_, {change_i18n_lang, _}, _NewConf, _OldConf, _AppEnvs) -> + delay_job(i18n_lang_changed); post_config_update(_, _Req, NewConf, OldConf, _AppEnvs) -> OldHttp = get_listener(http, OldConf), OldHttps = get_listener(https, OldConf), @@ -148,11 +156,16 @@ post_config_update(_, _Req, NewConf, OldConf, _AppEnvs) -> {StopHttps, StartHttps} = diff_listeners(https, OldHttps, NewHttps), Stop = maps:merge(StopHttp, StopHttps), Start = maps:merge(StartHttp, StartHttps), - _ = erlang:send_after(500, ?MODULE, {update_listeners, Stop, Start}), + delay_job({update_listeners, Stop, Start}). + +%% in post_config_update, the config is not yet persisted to persistent_term +%% so we need to delegate the listener update to the gen_server a bit later +delay_job(Msg) -> + _ = erlang:send_after(500, ?MODULE, Msg), ok. get_listener(Type, Conf) -> - emqx_map_lib:deep_get([listeners, Type], Conf, undefined). + emqx_utils_maps:deep_get([listeners, Type], Conf, undefined). diff_listeners(_, Listener, Listener) -> {#{}, #{}}; diff_listeners(Type, undefined, Start) -> {#{}, #{Type => Start}}; @@ -162,13 +175,14 @@ diff_listeners(Type, Stop, Start) -> {#{Type => Stop}, #{Type => Start}}. -define(DIR, <<"dashboard">>). ensure_ssl_cert(#{<<"listeners">> := #{<<"https">> := #{<<"enable">> := true}}} = Conf) -> - Https = emqx_map_lib:deep_get([<<"listeners">>, <<"https">>], Conf, undefined), + Https = emqx_utils_maps:deep_get([<<"listeners">>, <<"https">>], Conf, undefined), Opts = #{required_keys => [[<<"keyfile">>], [<<"certfile">>], [<<"cacertfile">>]]}, case emqx_tls_lib:ensure_ssl_files(?DIR, Https, Opts) of {ok, undefined} -> {error, <<"ssl_cert_not_found">>}; {ok, NewHttps} -> - {ok, emqx_map_lib:deep_merge(Conf, #{<<"listeners">> => #{<<"https">> => NewHttps}})}; + {ok, + emqx_utils_maps:deep_merge(Conf, #{<<"listeners">> => #{<<"https">> => NewHttps}})}; {error, Reason} -> ?SLOG(error, Reason#{msg => "bad_ssl_config"}), {error, Reason} diff --git a/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl index aaee92b8c..c0e162b62 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl @@ -122,7 +122,7 @@ fields(sampler_current) -> monitor(get, #{query_string := QS, bindings := Bindings}) -> Latest = maps:get(<<"latest">>, QS, infinity), RawNode = maps:get(node, Bindings, <<"all">>), - emqx_api_lib:with_node_or_cluster(RawNode, dashboard_samplers_fun(Latest)). + emqx_utils_api:with_node_or_cluster(RawNode, dashboard_samplers_fun(Latest)). dashboard_samplers_fun(Latest) -> fun(NodeOrCluster) -> @@ -132,9 +132,11 @@ dashboard_samplers_fun(Latest) -> end end. +monitor_current(get, #{bindings := []}) -> + emqx_utils_api:with_node_or_cluster(erlang:node(), fun emqx_dashboard_monitor:current_rate/1); monitor_current(get, #{bindings := Bindings}) -> RawNode = maps:get(node, Bindings, <<"all">>), - emqx_api_lib:with_node_or_cluster(RawNode, fun current_rate/1). + emqx_utils_api:with_node_or_cluster(RawNode, fun current_rate/1). current_rate(Node) -> case emqx_dashboard_monitor:current_rate(Node) of diff --git a/apps/emqx_dashboard/src/emqx_dashboard_schema.erl b/apps/emqx_dashboard/src/emqx_dashboard_schema.erl index 7df661fb2..319c9cee1 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_schema.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_schema.erl @@ -42,6 +42,7 @@ fields("dashboard") -> #{ default => <<"10s">>, desc => ?DESC(sample_interval), + importance => ?IMPORTANCE_HIDDEN, validator => fun validate_sample_interval/1 } )}, @@ -61,6 +62,7 @@ fields("dashboard") -> #{ desc => ?DESC(bootstrap_users_file), required => false, + importance => ?IMPORTANCE_HIDDEN, default => <<>> %% deprecated => {since, "5.1.0"} } @@ -95,13 +97,15 @@ fields("https") -> [ enable(false), bind(18084) - | common_listener_fields() ++ - exclude_fields( - ["fail_if_no_peer_cert"], - emqx_schema:server_ssl_opts_schema(#{}, true) - ) + | common_listener_fields() ++ server_ssl_opts() ]. +server_ssl_opts() -> + Opts0 = emqx_schema:server_ssl_opts_schema(#{}, true), + Opts1 = exclude_fields(["fail_if_no_peer_cert"], Opts0), + {value, {_, Meta}, Opts2} = lists:keytake("password", 1, Opts1), + [{"password", Meta#{importance => ?IMPORTANCE_HIDDEN}} | Opts2]. + exclude_fields([], Fields) -> Fields; exclude_fields([FieldName | Rest], Fields) -> @@ -210,6 +214,7 @@ default_username(default) -> <<"admin">>; default_username(required) -> true; default_username(desc) -> ?DESC(default_username); default_username('readOnly') -> true; +default_username(importance) -> ?IMPORTANCE_HIDDEN; default_username(_) -> undefined. default_password(type) -> binary(); @@ -219,6 +224,7 @@ default_password('readOnly') -> true; default_password(sensitive) -> true; default_password(converter) -> fun emqx_schema:password_converter/2; default_password(desc) -> ?DESC(default_password); +default_password(importance) -> ?IMPORTANCE_HIDDEN; default_password(_) -> undefined. cors(type) -> boolean(); @@ -227,10 +233,13 @@ cors(required) -> false; cors(desc) -> ?DESC(cors); cors(_) -> undefined. +%% TODO: change it to string type +%% It will be up to the dashboard package which languagues to support i18n_lang(type) -> ?ENUM([en, zh]); i18n_lang(default) -> en; i18n_lang('readOnly') -> true; i18n_lang(desc) -> ?DESC(i18n_lang); +i18n_lang(importance) -> ?IMPORTANCE_HIDDEN; i18n_lang(_) -> undefined. validate_sample_interval(Second) -> diff --git a/apps/emqx_dashboard/src/emqx_dashboard_schema_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_schema_api.erl new file mode 100644 index 000000000..898d95b3c --- /dev/null +++ b/apps/emqx_dashboard/src/emqx_dashboard_schema_api.erl @@ -0,0 +1,84 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% This module is for dashboard to retrieve the schema hot config and bridges. +-module(emqx_dashboard_schema_api). + +-behaviour(minirest_api). + +-include_lib("hocon/include/hoconsc.hrl"). + +%% minirest API +-export([api_spec/0, paths/0, schema/1]). + +-export([get_schema/2]). + +-define(TAGS, [<<"dashboard">>]). +-define(BAD_REQUEST, 'BAD_REQUEST'). + +%%-------------------------------------------------------------------- +%% minirest API and schema +%%-------------------------------------------------------------------- + +api_spec() -> + emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). + +paths() -> + ["/schemas/:name"]. + +%% This is a rather hidden API, so we don't need to add translations for the description. +schema("/schemas/:name") -> + #{ + 'operationId' => get_schema, + get => #{ + parameters => [ + {name, hoconsc:mk(hoconsc:enum([hotconf, bridges]), #{in => path})}, + {lang, + hoconsc:mk(typerefl:string(), #{ + in => query, + default => <<"en">>, + desc => <<"The language of the schema.">> + })} + ], + desc => << + "Get the schema JSON of the specified name. " + "NOTE: you should never need to make use of this API " + "unless you are building a multi-lang dashboaard." + >>, + tags => ?TAGS, + security => [], + responses => #{ + 200 => hoconsc:mk(binary(), #{desc => <<"The JSON schema of the specified name.">>}) + } + } + }. + +%%-------------------------------------------------------------------- +%% API Handler funcs +%%-------------------------------------------------------------------- + +get_schema(get, #{ + bindings := #{name := Name}, + query_string := #{<<"lang">> := Lang} +}) -> + {200, gen_schema(Name, iolist_to_binary(Lang))}; +get_schema(get, _) -> + {400, ?BAD_REQUEST, <<"unknown">>}. + +gen_schema(hotconf, Lang) -> + emqx_conf:hotconf_schema_json(Lang); +gen_schema(bridges, Lang) -> + emqx_conf:bridge_schema_json(Lang). diff --git a/apps/emqx_dashboard/src/emqx_dashboard_sup.erl b/apps/emqx_dashboard/src/emqx_dashboard_sup.erl index 896b44859..04d8ed1d5 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_sup.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_sup.erl @@ -28,6 +28,8 @@ start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). init([]) -> + %% supervisor owns the cache table + ok = emqx_dashboard_desc_cache:init(), {ok, {{one_for_one, 5, 100}, [ ?CHILD(emqx_dashboard_listener, brutal_kill), diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 95f28e387..e42d9c3ae 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -88,7 +88,8 @@ -type spec_opts() :: #{ check_schema => boolean() | filter(), translate_body => boolean(), - schema_converter => fun((hocon_schema:schema(), Module :: atom()) -> map()) + schema_converter => fun((hocon_schema:schema(), Module :: atom()) -> map()), + i18n_lang => atom() | string() | binary() }. -type route_path() :: string() | binary(). @@ -249,8 +250,16 @@ parse_spec_ref(Module, Path, Options) -> erlang:apply(Module, schema, [Path]) %% better error message catch - error:Reason -> - throw({error, #{mfa => {Module, schema, [Path]}, reason => Reason}}) + error:Reason:Stacktrace -> + %% raise a new error with the same stacktrace. + %% it's a bug if this happens. + %% i.e. if a path is listed in the spec but the module doesn't + %% implement it or crashes when trying to build the schema. + erlang:raise( + error, + #{mfa => {Module, schema, [Path]}, reason => Reason}, + Stacktrace + ) end, {Specs, Refs} = maps:fold( fun(Method, Meta, {Acc, RefsAcc}) -> @@ -345,11 +354,11 @@ check_request_body(#{body := Body}, Spec, _Module, _CheckFun, false) when is_map %% tags, description, summary, security, deprecated meta_to_spec(Meta, Module, Options) -> - {Params, Refs1} = parameters(maps:get(parameters, Meta, []), Module), + {Params, Refs1} = parameters(maps:get(parameters, Meta, []), Module, Options), {RequestBody, Refs2} = request_body(maps:get('requestBody', Meta, []), Module, Options), {Responses, Refs3} = responses(maps:get(responses, Meta, #{}), Module, Options), { - generate_method_desc(to_spec(Meta, Params, RequestBody, Responses)), + generate_method_desc(to_spec(Meta, Params, RequestBody, Responses), Options), lists:usort(Refs1 ++ Refs2 ++ Refs3) }. @@ -360,13 +369,13 @@ to_spec(Meta, Params, RequestBody, Responses) -> Spec = to_spec(Meta, Params, [], Responses), maps:put('requestBody', RequestBody, Spec). -generate_method_desc(Spec = #{desc := _Desc}) -> - Spec1 = trans_description(maps:remove(desc, Spec), Spec), +generate_method_desc(Spec = #{desc := _Desc}, Options) -> + Spec1 = trans_description(maps:remove(desc, Spec), Spec, Options), trans_tags(Spec1); -generate_method_desc(Spec = #{description := _Desc}) -> - Spec1 = trans_description(Spec, Spec), +generate_method_desc(Spec = #{description := _Desc}, Options) -> + Spec1 = trans_description(Spec, Spec, Options), trans_tags(Spec1); -generate_method_desc(Spec) -> +generate_method_desc(Spec, _Options) -> trans_tags(Spec). trans_tags(Spec = #{tags := Tags}) -> @@ -374,7 +383,7 @@ trans_tags(Spec = #{tags := Tags}) -> trans_tags(Spec) -> Spec. -parameters(Params, Module) -> +parameters(Params, Module, Options) -> {SpecList, AllRefs} = lists:foldl( fun(Param, {Acc, RefsAcc}) -> @@ -400,7 +409,7 @@ parameters(Params, Module) -> Type ), Spec1 = trans_required(Spec0, Required, In), - Spec2 = trans_description(Spec1, Type), + Spec2 = trans_description(Spec1, Type, Options), {[Spec2 | Acc], Refs ++ RefsAcc} end end, @@ -444,38 +453,38 @@ trans_required(Spec, true, _) -> Spec#{required => true}; trans_required(Spec, _, path) -> Spec#{required => true}; trans_required(Spec, _, _) -> Spec. -trans_desc(Init, Hocon, Func, Name) -> - Spec0 = trans_description(Init, Hocon), +trans_desc(Init, Hocon, Func, Name, Options) -> + Spec0 = trans_description(Init, Hocon, Options), case Func =:= fun hocon_schema_to_spec/2 of true -> Spec0; false -> - Spec1 = trans_label(Spec0, Hocon, Name), + Spec1 = trans_label(Spec0, Hocon, Name, Options), case Spec1 of #{description := _} -> Spec1; _ -> Spec1#{description => <>} end end. -trans_description(Spec, Hocon) -> +trans_description(Spec, Hocon, Options) -> Desc = case desc_struct(Hocon) of undefined -> undefined; - ?DESC(_, _) = Struct -> get_i18n(<<"desc">>, Struct, undefined); - Struct -> to_bin(Struct) + ?DESC(_, _) = Struct -> get_i18n(<<"desc">>, Struct, undefined, Options); + Text -> to_bin(Text) end, case Desc of undefined -> Spec; Desc -> Desc1 = binary:replace(Desc, [<<"\n">>], <<"
">>, [global]), - maybe_add_summary_from_label(Spec#{description => Desc1}, Hocon) + maybe_add_summary_from_label(Spec#{description => Desc1}, Hocon, Options) end. -maybe_add_summary_from_label(Spec, Hocon) -> +maybe_add_summary_from_label(Spec, Hocon, Options) -> Label = case desc_struct(Hocon) of - ?DESC(_, _) = Struct -> get_i18n(<<"label">>, Struct, undefined); + ?DESC(_, _) = Struct -> get_i18n(<<"label">>, Struct, undefined, Options); _ -> undefined end, case Label of @@ -483,29 +492,44 @@ maybe_add_summary_from_label(Spec, Hocon) -> _ -> Spec#{summary => Label} end. -get_i18n(Key, Struct, Default) -> - {ok, #{cache := Cache, lang := Lang}} = emqx_dashboard:get_i18n(), - Desc = hocon_schema:resolve_schema(Struct, Cache), - emqx_map_lib:deep_get([Key, Lang], Desc, Default). +get_i18n(Tag, ?DESC(Namespace, Id), Default, Options) -> + Lang = get_lang(Options), + case emqx_dashboard_desc_cache:lookup(Lang, Namespace, Id, Tag) of + undefined -> + Default; + Text -> + Text + end. -trans_label(Spec, Hocon, Default) -> +%% So far i18n_lang in options is only used at build time. +%% At runtime, it's still the global config which controls the language. +get_lang(#{i18n_lang := Lang}) -> Lang; +get_lang(_) -> emqx:get_config([dashboard, i18n_lang]). + +trans_label(Spec, Hocon, Default, Options) -> Label = case desc_struct(Hocon) of - ?DESC(_, _) = Struct -> get_i18n(<<"label">>, Struct, Default); + ?DESC(_, _) = Struct -> get_i18n(<<"label">>, Struct, Default, Options); _ -> Default end, Spec#{label => Label}. desc_struct(Hocon) -> - case hocon_schema:field_schema(Hocon, desc) of - undefined -> - case hocon_schema:field_schema(Hocon, description) of - undefined -> get_ref_desc(Hocon); - Struct1 -> Struct1 - end; - Struct -> - Struct - end. + R = + case hocon_schema:field_schema(Hocon, desc) of + undefined -> + case hocon_schema:field_schema(Hocon, description) of + undefined -> get_ref_desc(Hocon); + Struct1 -> Struct1 + end; + Struct -> + Struct + end, + ensure_bin(R). + +ensure_bin(undefined) -> undefined; +ensure_bin(?DESC(_Namespace, _Id) = Desc) -> Desc; +ensure_bin(Text) -> to_bin(Text). get_ref_desc(?R_REF(Mod, Name)) -> case erlang:function_exported(Mod, desc, 1) of @@ -536,7 +560,7 @@ responses(Responses, Module, Options) -> {Spec, Refs}. response(Status, ?DESC(_Mod, _Id) = Schema, {Acc, RefsAcc, Module, Options}) -> - Desc = trans_description(#{}, #{desc => Schema}), + Desc = trans_description(#{}, #{desc => Schema}, Options), {Acc#{integer_to_binary(Status) => Desc}, RefsAcc, Module, Options}; response(Status, Bin, {Acc, RefsAcc, Module, Options}) when is_binary(Bin) -> {Acc#{integer_to_binary(Status) => #{description => Bin}}, RefsAcc, Module, Options}; @@ -565,7 +589,7 @@ response(Status, Schema, {Acc, RefsAcc, Module, Options}) -> Hocon = hocon_schema:field_schema(Schema, type), Examples = hocon_schema:field_schema(Schema, examples), {Spec, Refs} = hocon_schema_to_spec(Hocon, Module), - Init = trans_description(#{}, Schema), + Init = trans_description(#{}, Schema, Options), Content = content(Spec, Examples), { Acc#{integer_to_binary(Status) => Init#{<<"content">> => Content}}, @@ -575,7 +599,7 @@ response(Status, Schema, {Acc, RefsAcc, Module, Options}) -> }; false -> {Props, Refs} = parse_object(Schema, Module, Options), - Init = trans_description(#{}, Schema), + Init = trans_description(#{}, Schema, Options), Content = Init#{<<"content">> => content(Props)}, {Acc#{integer_to_binary(Status) => Content}, Refs ++ RefsAcc, Module, Options} end. @@ -602,7 +626,7 @@ components(Options, [{Module, Field} | Refs], SpecAcc, SubRefsAcc) -> %% parameters in ref only have one value, not array components(Options, [{Module, Field, parameter} | Refs], SpecAcc, SubRefsAcc) -> Props = hocon_schema_fields(Module, Field), - {[Param], SubRefs} = parameters(Props, Module), + {[Param], SubRefs} = parameters(Props, Module, Options), Namespace = namespace(Module), NewSpecAcc = SpecAcc#{?TO_REF(Namespace, Field) => Param}, components(Options, Refs, NewSpecAcc, SubRefs ++ SubRefsAcc). @@ -756,7 +780,7 @@ typename_to_spec("log_level()", _Mod) -> }; typename_to_spec("rate()", _Mod) -> #{type => string, example => <<"10MB">>}; -typename_to_spec("capacity()", _Mod) -> +typename_to_spec("burst()", _Mod) -> #{type => string, example => <<"100MB">>}; typename_to_spec("burst_rate()", _Mod) -> %% 0/0s = no burst @@ -767,6 +791,8 @@ typename_to_spec("initial()", _Mod) -> #{type => string, example => <<"0MB">>}; typename_to_spec("bucket_name()", _Mod) -> #{type => string, example => <<"retainer">>}; +typename_to_spec("json_binary()", _Mod) -> + #{type => string, example => <<"{\"a\": [1,true]}">>}; typename_to_spec(Name, Mod) -> Spec = range(Name), Spec1 = remote_module_type(Spec, Name, Mod), @@ -879,7 +905,7 @@ parse_object_loop([{Name, Hocon} | Rest], Module, Options, Props, Required, Refs HoconType = hocon_schema:field_schema(Hocon, type), Init0 = init_prop([default | ?DEFAULT_FIELDS], #{}, Hocon), SchemaToSpec = schema_converter(Options), - Init = trans_desc(Init0, Hocon, SchemaToSpec, NameBin), + Init = trans_desc(Init0, Hocon, SchemaToSpec, NameBin, Options), {Prop, Refs1} = SchemaToSpec(HoconType, Module), NewRequiredAcc = case is_required(Hocon) of @@ -925,4 +951,4 @@ schema_converter(Options) -> maps:get(schema_converter, Options, fun hocon_schema_to_spec/2). hocon_error_msg(Reason) -> - emqx_misc:readable_error_msg(Reason). + emqx_utils:readable_error_msg(Reason). diff --git a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl index e951a9a2a..1f14b02c0 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl @@ -142,7 +142,7 @@ t_swagger_json(_Config) -> %% with auth Auth = auth_header_(<<"admin">>, <<"public_www1">>), {ok, 200, Body1} = request_api(get, Url, Auth), - ?assert(jsx:is_json(Body1)), + ?assert(emqx_utils_json:is_json(Body1)), %% without auth {ok, {{"HTTP/1.1", 200, "OK"}, _Headers, Body2}} = httpc:request(get, {Url, []}, [], [{body_format, binary}]), @@ -246,5 +246,5 @@ api_path(Parts) -> ?HOST ++ filename:join([?BASE_PATH | Parts]). json(Data) -> - {ok, Jsx} = emqx_json:safe_decode(Data, [return_maps]), + {ok, Jsx} = emqx_utils_json:safe_decode(Data, [return_maps]), Jsx. diff --git a/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl b/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl index f70f464b3..70b40e1ff 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl @@ -83,7 +83,7 @@ request(Username, Method, Url, Body) -> -> {Url, [auth_header(Username)]}; _ -> - {Url, [auth_header(Username)], "application/json", jsx:encode(Body)} + {Url, [auth_header(Username)], "application/json", emqx_utils_json:encode(Body)} end, ct:pal("Method: ~p, Request: ~p", [Method, Request]), case httpc:request(Method, Request, [], [{body_format, binary}]) of diff --git a/apps/emqx_dashboard/test/emqx_dashboard_error_code_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_error_code_SUITE.erl index 19d3f471e..588a69065 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_error_code_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_error_code_SUITE.erl @@ -57,7 +57,7 @@ t_look_up_code(_) -> t_description_code(_) -> {error, not_found} = emqx_dashboard_error_code:description('_____NOT_EXIST_NAME'), - {ok, <<"Request parameters are not legal">>} = + {ok, <<"Request parameters are invalid">>} = emqx_dashboard_error_code:description('BAD_REQUEST'), ok. @@ -79,7 +79,7 @@ t_api_code(_) -> Url = ?SERVER ++ "/error_codes/BAD_REQUEST", {ok, #{ <<"code">> := <<"BAD_REQUEST">>, - <<"description">> := <<"Request parameters are not legal">> + <<"description">> := <<"Request parameters are invalid">> }} = request(Url), ok. @@ -100,7 +100,7 @@ request(Url) -> {ok, {{"HTTP/1.1", Code, _}, _, Return}} when Code >= 200 andalso Code =< 299 -> - {ok, emqx_json:decode(Return, [return_maps])}; + {ok, emqx_utils_json:decode(Return, [return_maps])}; {ok, {Reason, _, _}} -> {error, Reason} end. diff --git a/apps/emqx_dashboard/test/emqx_dashboard_listener_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_listener_SUITE.erl new file mode 100644 index 000000000..7f28841fc --- /dev/null +++ b/apps/emqx_dashboard/test/emqx_dashboard_listener_SUITE.erl @@ -0,0 +1,51 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_dashboard_listener_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + emqx_mgmt_api_test_util:init_suite([emqx_conf]), + ok = change_i18n_lang(en), + Config. + +end_per_suite(_Config) -> + ok = change_i18n_lang(en), + emqx_mgmt_api_test_util:end_suite([emqx_conf]). + +t_change_i18n_lang(_Config) -> + ?check_trace( + begin + ok = change_i18n_lang(zh), + {ok, _} = ?block_until(#{?snk_kind := regenerate_minirest_dispatch}, 10_000), + ok + end, + fun(ok, Trace) -> + ?assertMatch([#{i18n_lang := zh}], ?of_kind(regenerate_minirest_dispatch, Trace)) + end + ), + ok. + +change_i18n_lang(Lang) -> + {ok, _} = emqx_conf:update([dashboard], {change_i18n_lang, Lang}, #{}), + ok. diff --git a/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl index f35652f8e..a24fc2337 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl @@ -137,10 +137,10 @@ do_request_api(Method, Request) -> Code >= 200 andalso Code =< 299 -> ct:pal("Resp ~p ~p~n", [Code, Return]), - {ok, emqx_json:decode(Return, [return_maps])}; + {ok, emqx_utils_json:decode(Return, [return_maps])}; {ok, {{"HTTP/1.1", Code, _}, _, Return}} -> ct:pal("Resp ~p ~p~n", [Code, Return]), - {error, {Code, emqx_json:decode(Return, [return_maps])}}; + {error, {Code, emqx_utils_json:decode(Return, [return_maps])}}; {error, Reason} -> {error, Reason} end. diff --git a/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl index 472e90405..81b3f4402 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl @@ -64,7 +64,6 @@ groups() -> init_per_suite(Config) -> emqx_mgmt_api_test_util:init_suite([emqx_conf]), - emqx_dashboard:init_i18n(), Config. end_per_suite(_Config) -> diff --git a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl index 3150ed097..f2ba56e08 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl @@ -33,7 +33,6 @@ init_per_suite(Config) -> mria:start(), application:load(emqx_dashboard), emqx_common_test_helpers:start_apps([emqx_conf, emqx_dashboard], fun set_special_configs/1), - emqx_dashboard:init_i18n(), Config. set_special_configs(emqx_dashboard) -> @@ -308,8 +307,11 @@ t_nest_ref(_Config) -> t_none_ref(_Config) -> Path = "/ref/none", - ?assertThrow( - {error, #{mfa := {?MODULE, schema, [Path]}}}, + ?assertError( + #{ + mfa := {?MODULE, schema, [Path]}, + reason := function_clause + }, emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path, #{}) ), ok. diff --git a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl index 4d1501dae..cda533cc2 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl @@ -33,7 +33,6 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> emqx_mgmt_api_test_util:init_suite([emqx_conf]), - emqx_dashboard:init_i18n(), Config. end_per_suite(Config) -> @@ -278,11 +277,11 @@ t_bad_ref(_Config) -> t_none_ref(_Config) -> Path = "/ref/none", - ?assertThrow( - {error, #{ + ?assertError( + #{ mfa := {?MODULE, schema, ["/ref/none"]}, reason := function_clause - }}, + }, validate(Path, #{}, []) ), ok. diff --git a/apps/emqx_exhook/rebar.config b/apps/emqx_exhook/rebar.config index fad539ed1..7abc601b4 100644 --- a/apps/emqx_exhook/rebar.config +++ b/apps/emqx_exhook/rebar.config @@ -5,7 +5,8 @@ ]}. {deps, [ - {emqx, {path, "../emqx"}} + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}} ]}. {grpc, [ diff --git a/apps/emqx_exhook/src/emqx_exhook.app.src b/apps/emqx_exhook/src/emqx_exhook.app.src index 8ca15a907..194c91206 100644 --- a/apps/emqx_exhook/src/emqx_exhook.app.src +++ b/apps/emqx_exhook/src/emqx_exhook.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_exhook, [ {description, "EMQX Extension for Hook"}, - {vsn, "5.0.11"}, + {vsn, "5.0.12"}, {modules, []}, {registered, []}, {mod, {emqx_exhook_app, []}}, diff --git a/apps/emqx_exhook/src/emqx_exhook_api.erl b/apps/emqx_exhook/src/emqx_exhook_api.erl index aa5d1897f..9bfae9579 100644 --- a/apps/emqx_exhook/src/emqx_exhook_api.erl +++ b/apps/emqx_exhook/src/emqx_exhook_api.erl @@ -478,7 +478,7 @@ call_cluster(Fun) -> %%-------------------------------------------------------------------- %% Internal Funcs %%-------------------------------------------------------------------- -err_msg(Msg) -> emqx_misc:readable_error_msg(Msg). +err_msg(Msg) -> emqx_utils:readable_error_msg(Msg). get_raw_config() -> RawConfig = emqx:get_raw_config([exhook, servers], []), diff --git a/apps/emqx_exhook/src/emqx_exhook_mgr.erl b/apps/emqx_exhook/src/emqx_exhook_mgr.erl index 77937a835..0647c80ea 100644 --- a/apps/emqx_exhook/src/emqx_exhook_mgr.erl +++ b/apps/emqx_exhook/src/emqx_exhook_mgr.erl @@ -507,11 +507,11 @@ sort_name_by_order(Names, Orders) -> lists:sort( fun (A, B) when is_binary(A) -> - emqx_map_lib:deep_get([A, order], Orders) < - emqx_map_lib:deep_get([B, order], Orders); + emqx_utils_maps:deep_get([A, order], Orders) < + emqx_utils_maps:deep_get([B, order], Orders); (#{name := A}, #{name := B}) -> - emqx_map_lib:deep_get([A, order], Orders) < - emqx_map_lib:deep_get([B, order], Orders) + emqx_utils_maps:deep_get([A, order], Orders) < + emqx_utils_maps:deep_get([B, order], Orders) end, Names ). diff --git a/apps/emqx_exhook/src/emqx_exhook_schema.erl b/apps/emqx_exhook/src/emqx_exhook_schema.erl index 07373288d..f6cc896f3 100644 --- a/apps/emqx_exhook/src/emqx_exhook_schema.erl +++ b/apps/emqx_exhook/src/emqx_exhook_schema.erl @@ -31,7 +31,8 @@ namespace() -> exhook. -roots() -> [exhook]. +roots() -> + [{exhook, ?HOCON(?R_REF(exhook), #{importance => ?IMPORTANCE_LOW})}]. fields(exhook) -> [ diff --git a/apps/emqx_exhook/test/emqx_exhook_api_SUITE.erl b/apps/emqx_exhook/test/emqx_exhook_api_SUITE.erl index 8a4fb7a44..c03b3f231 100644 --- a/apps/emqx_exhook/test/emqx_exhook_api_SUITE.erl +++ b/apps/emqx_exhook/test/emqx_exhook_api_SUITE.erl @@ -310,8 +310,8 @@ t_update(Cfg) -> ?assertMatch([], emqx_exhook_mgr:running()). decode_json(Data) -> - BinJosn = emqx_json:decode(Data, [return_maps]), - emqx_map_lib:unsafe_atom_key_map(BinJosn). + BinJosn = emqx_utils_json:decode(Data, [return_maps]), + emqx_utils_maps:unsafe_atom_key_map(BinJosn). request_api(Method, Url, Auth) -> request_api(Method, Url, [], Auth, []). @@ -332,7 +332,7 @@ request_api(Method, Url, QueryParams, Auth, Body) -> "" -> Url; _ -> Url ++ "?" ++ QueryParams end, - do_request_api(Method, {NewUrl, [Auth], "application/json", emqx_json:encode(Body)}). + do_request_api(Method, {NewUrl, [Auth], "application/json", emqx_utils_json:encode(Body)}). do_request_api(Method, Request) -> case httpc:request(Method, Request, [], [{body_format, binary}]) of diff --git a/apps/emqx_gateway/.gitignore b/apps/emqx_gateway/.gitignore index 5bff8a84d..a81bb07da 100644 --- a/apps/emqx_gateway/.gitignore +++ b/apps/emqx_gateway/.gitignore @@ -18,8 +18,4 @@ _build rebar3.crashdump *~ rebar.lock -src/exproto/emqx_exproto_pb.erl -src/exproto/emqx_exproto_v_1_connection_adapter_bhvr.erl -src/exproto/emqx_exproto_v_1_connection_adapter_client.erl -src/exproto/emqx_exproto_v_1_connection_handler_bhvr.erl -src/exproto/emqx_exproto_v_1_connection_handler_client.erl + diff --git a/apps/emqx_gateway/Makefile b/apps/emqx_gateway/Makefile deleted file mode 100644 index b2a54f7dd..000000000 --- a/apps/emqx_gateway/Makefile +++ /dev/null @@ -1,28 +0,0 @@ -## shallow clone for speed - -REBAR_GIT_CLONE_OPTIONS += --depth 1 -export REBAR_GIT_CLONE_OPTIONS - -REBAR = rebar3 -all: compile - -compile: - $(REBAR) compile - -clean: distclean - -ct: - $(REBAR) as test ct -v - -eunit: - $(REBAR) as test eunit - -xref: - $(REBAR) xref - -cover: - $(REBAR) cover - -distclean: - @rm -rf _build - @rm -f data/app.*.config data/vm.*.args rebar.lock diff --git a/apps/emqx_gateway/README.md b/apps/emqx_gateway/README.md index be8f6cb35..ebab3a7a9 100644 --- a/apps/emqx_gateway/README.md +++ b/apps/emqx_gateway/README.md @@ -1,332 +1,83 @@ -# emqx_gateway +# Gateway -EMQX Gateway +EMQX Gateway is an application framework that manages all gateways within EMQX. -## Concept +It provides a set of standards to define how to implement a certain type of +protocol access on EMQX. For example: - EMQX Gateway Management - - Gateway-Registry (or Gateway Type) - - *Load - - *UnLoad - - *List +- Frame parsing +- Access authentication +- Publish and subscribe +- Configuration & Schema +- HTTP/CLI management interfaces - - Gateway - - *Create - - *Delete - - *Update - - *Stop-And-Start - - *Hot-Upgrade - - *Satrt/Enable - - *Stop/Disable - - Listener +The emqx_gateway application depends on `emqx`, `emqx_authn`, `emqx_authz`, `emqx_ctl` that +provide the foundation for protocol access. -## ROADMAP +More introduction: [Extended Protocol Gateway](https://www.emqx.io/docs/en/v5.0/gateway/gateway.html) -Gateway v0.1: "Basic Functionals" - - Management support - - Conn/Frame/Protocol Template - - Support Stomp/MQTT-SN/CoAP/LwM2M/ExProto +## Usage -Gateway v0.2: "Integration & Friendly Management" - - Hooks & Metrics & Statistic - - HTTP APIs - - Management in the cluster - - Integrate with AuthN - - Integrate with `emqx_config` - - Improve hocon config - - Mountpoint & ClientInfo's Metadata - - The Concept Review +This application is just a Framework, we provide some standard implementations, +such as [Stomp](../emqx_stomp/README.md), [MQTT-SN](../emqx_mqttsn/README.md), +[CoAP](../emqx_coap/README.md) and [LwM2M](../emqx_lwm2m/README.md) gateway. -Gateway v0.3: "Fault tolerance and high availability" - - A common session modoule for message delivery policy - - The restart mechanism for gateway-instance - - Consistency of cluster state - - Configuration hot update - -Gateway v1.0: "Best practices for each type of protocol" - - CoAP - - Stomp - - MQTT-SN - - LwM2M - -### Compatible with EMQX - -> Why we need to compatible - -1. Authentication -2. Hooks/Event system -3. Messages Mode & Rule Engine -4. Cluster registration -5. Metrics & Statistic - -> How to do it - -> - -### User Interface - -#### Configurations +These applications are all packaged by default in the EMQX distribution. If you +need to start a certain gateway, you only need to enable it via +Dashboard, HTTP API or emqx.conf file. +For instance, enable the Stomp gateway in emqx.conf: ```hocon -gateway { +gateway.stomp { - ## ... some confs for top scope - .. - ## End. + mountpoint = "stomp/" - ## Gateway Instances - - lwm2m[.name] { - - ## variable support - mountpoint: lwm2m/%e/ - - lifetime_min: 1s - lifetime_max: 86400s - #qmode_time_window: 22 - #auto_observe: off - - #update_msg_publish_condition: contains_object_list - - xml_dir: {{ platform_etc_dir }}/lwm2m_xml - - clientinfo_override: { - username: ${register.opts.uname} - password: ${register.opts.passwd} - clientid: ${epn} - } - - #authenticator: allow_anonymous - authenticator: [ - { - type: auth-http - method: post - //?? how to generate clientinfo ?? - params: $client.credential - } - ] - - translator: { - downlink: "dn/#" - uplink: { - notify: "up/notify" - response: "up/resp" - register: "up/resp" - update: "up/reps" - } - } - - %% ?? listener.$type.name ?? - listener.udp[.name] { - listen_on: 0.0.0.0:5683 - max_connections: 1024000 - max_conn_rate: 1000 - ## ?? udp keepalive in socket level ??? - #keepalive: - ## ?? udp proxy-protocol in socket level ??? - #proxy_protocol: on - #proxy_timeout: 30s - recbuf: 2KB - sndbuf: 2KB - buffer: 2KB - tune_buffer: off - #access: allow all - read_packets: 20 - } - - listener.dtls[.name] { - listen_on: 0.0.0.0:5684 - ... - } - } - - ## The CoAP Gateway - coap[.name] { - - #enable_stats: on - - authenticator: [ - ... - ] - - listener.udp[.name] { - ... - } - - listener.dtls[.name] { - ... - } -} - - ## The Stomp Gateway - stomp[.name] { - - allow_anonymous: true - - default_user.login: guest - default_user.passcode: guest - - frame.max_headers: 10 - frame.max_header_length: 1024 - frame.max_body_length: 8192 - - listener.tcp[.name] { - ... - } - - listener.ssl[.name] { - ... - } - } - - exproto[.name] { - - proto_name: DL-648 - - authenticators: [...] - - adapter: { - type: grpc - options: { - listen_on: 9100 - } - } - - handler: { - type: grpc - options: { - url: - } - } - - listener.tcp[.name] { - ... - } - } - - ## ============================ Enterpise gateways - - ## The JT/T 808 Gateway - jtt808[.name] { - - idle_timeout: 30s - enable_stats: on - max_packet_size: 8192 - - clientinfo_override: { - clientid: $phone - username: xxx - password: xxx - } - - authenticator: [ - { - type: auth-http - method: post - params: $clientinfo.credential - } - ] - - translator: { - subscribe: [jt808/%c/dn] - publish: [jt808/%c/up] - } - - listener.tcp[.name] { - ... - } - - listener.ssl[.name] { - ... - } - } - - gbt32960[.name] { - - frame.max_length: 8192 - retx_interval: 8s - retx_max_times: 3 - message_queue_len: 10 - - authenticators: [...] - - translator: { - ## upstream - login: gbt32960/${vin}/upstream/vlogin - logout: gbt32960/${vin}/upstream/vlogout - informing: gbt32960/${vin}/upstream/info - reinforming: gbt32960/${vin}/upstream/reinfo - ## downstream - downstream: gbt32960/${vin}/dnstream - response: gbt32960/${vin}/upstream/response - } - - listener.tcp[.name] { - ... - } - - listener.ssl[.name] { - ... - } - } - - privtcp[.name] { - - max_packet_size: 65535 - idle_timeout: 15s - - enable_stats: on - - force_gc_policy: 1000|1MB - force_shutdown_policy: 8000|800MB - - translator: { - up_topic: tcp/%c/up - dn_topic: tcp/%c/dn - } - - listener.tcp[.name]: { - ... - } + listeners.tcp.default { + bind = 61613 + acceptors = 16 + max_connections = 1024000 + max_conn_rate = 1000 } } ``` -#### CLI +## How to develop your Gateway application -##### Gateway +There are three ways to develop a Gateway application to accept your private protocol +clients. -```bash -## List all started gateway and gateway-instance -emqx_ctl gateway list -emqx_ctl gateway lookup -emqx_ctl gateway stop -emqx_ctl gateway start +### Raw Erlang Application -emqx_ctl gateway-registry re-searching -emqx_ctl gateway-registry list +This approach is the same as in EMQX 4.x. You need to implement an Erlang application, +which is packaged in EMQX as a Plugin or as a source code dependency. +In this approach, you do not need to respect any specifications of emqx_gateway, +and you can freely implement the features you need. -emqx_ctl gateway-clients list -emqx_ctl gateway-clients show -emqx_ctl gateway-clients kick -## Banned ?? -emqx_ctl gateway-banned +### Respect emqx_gateway framework -## Metrics -emqx_ctl gateway-metrics [] -``` +Similar to the first approach, you still need to implement an application using Erlang +and package it into EMQX. +The only difference is that you need to follow the standard behaviors(callbacks) provided +by emqx_gateway. -#### Management by HTTP-API/Dashboard/ +This is the approach we recommend. In this approach, your implementation can be managed +by the emqx_gateway framework, even if it may require you to understand more details about it. -#### How to integrate a protocol to your platform -### Develop your protocol gateway +### Use ExProto Gateway (Non-Erlang developers) -There are 3 way to create your protocol gateway for EMQX 5.0: +If you want to implement your gateway using other programming languages such as +Java, Python, Go, etc. -1. Use Erlang to create a new emqx plugin to handle all of protocol packets (same as v5.0 before) +You need to implement a gRPC service in the other programming language to parse +your device protocol and integrate it with EMQX. -2. Based on the emqx-gateway-impl-bhvr and emqx-gateway -3. Use the gRPC Gateway +## Contributing + +Please see our [contributing.md](../../CONTRIBUTING.md). + +## License + +See [LICENSE](../../APL.txt) diff --git a/apps/emqx_gateway/include/emqx_gateway.hrl b/apps/emqx_gateway/include/emqx_gateway.hrl index 3466ecd98..c880aca26 100644 --- a/apps/emqx_gateway/include/emqx_gateway.hrl +++ b/apps/emqx_gateway/include/emqx_gateway.hrl @@ -37,4 +37,11 @@ config => emqx_config:config() }. +-type gateway_def() :: + #{ + name := gateway_name(), + callback_module := module(), + config_schema_module := module() + }. + -endif. diff --git a/apps/emqx_gateway/rebar.config b/apps/emqx_gateway/rebar.config index 272783758..2340a2dd8 100644 --- a/apps/emqx_gateway/rebar.config +++ b/apps/emqx_gateway/rebar.config @@ -1,38 +1,6 @@ %% -*- mode: erlang -*- - {erl_opts, [debug_info]}. {deps, [ - {emqx, {path, "../emqx"}} + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}} ]}. - -{plugins, [ - {grpc_plugin, {git, "https://github.com/HJianBo/grpc_plugin", {tag, "v0.10.2"}}} -]}. - -{grpc, [ - {protos, ["src/exproto/protos"]}, - {out_dir, "src/exproto/"}, - {gpb_opts, [ - {module_name_prefix, "emqx_"}, - {module_name_suffix, "_pb"} - ]} -]}. - -{provider_hooks, [ - {pre, [ - {compile, {grpc, gen}}, - {clean, {grpc, clean}} - ]} -]}. - -{xref_ignores, [emqx_exproto_pb]}. - -{cover_excl_mods, [ - emqx_exproto_pb, - emqx_exproto_v_1_connection_adapter_client, - emqx_exproto_v_1_connection_adapter_bhvr, - emqx_exproto_v_1_connection_handler_client, - emqx_exproto_v_1_connection_handler_bhvr -]}. - -{project_plugins, [erlfmt]}. diff --git a/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl b/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl index 7f37061ac..4145a92a7 100644 --- a/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl +++ b/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl @@ -173,7 +173,7 @@ stats(#state{ end, ConnStats = emqx_pd:get_counters(?CONN_STATS), ChanStats = ChannMod:stats(Channel), - ProcStats = emqx_misc:proc_stats(), + ProcStats = emqx_utils:proc_stats(), lists:append([SockStats, ConnStats, ChanStats, ProcStats]). call(Pid, Req) -> @@ -297,7 +297,7 @@ init_state(WrappedSock, Peername, Options, FrameMod, ChannMod) -> StatsTimer = emqx_gateway_utils:stats_timer(Options), IdleTimeout = emqx_gateway_utils:idle_timeout(Options), OomPolicy = emqx_gateway_utils:oom_policy(Options), - IdleTimer = emqx_misc:start_timer(IdleTimeout, idle_timeout), + IdleTimer = emqx_utils:start_timer(IdleTimeout, idle_timeout), #state{ socket = WrappedSock, peername = Peername, @@ -327,7 +327,7 @@ run_loop( } ) -> emqx_logger:set_metadata_peername(esockd:format(Peername)), - _ = emqx_misc:tune_heap_size(OomPolicy), + _ = emqx_utils:tune_heap_size(OomPolicy), case activate_socket(State) of {ok, NState} -> hibernate(Parent, NState); @@ -383,14 +383,14 @@ wakeup_from_hib(Parent, State) -> %% Ensure/cancel stats timer ensure_stats_timer(Timeout, State = #state{stats_timer = undefined}) -> - State#state{stats_timer = emqx_misc:start_timer(Timeout, emit_stats)}; + State#state{stats_timer = emqx_utils:start_timer(Timeout, emit_stats)}; ensure_stats_timer(_Timeout, State) -> State. cancel_stats_timer(State = #state{stats_timer = TRef}) when is_reference(TRef) -> - ok = emqx_misc:cancel_timer(TRef), + ok = emqx_utils:cancel_timer(TRef), State#state{stats_timer = undefined}; cancel_stats_timer(State) -> State. @@ -471,7 +471,7 @@ handle_msg( State = #state{idle_timer = IdleTimer} ) -> IdleTimer /= undefined andalso - emqx_misc:cancel_timer(IdleTimer), + emqx_utils:cancel_timer(IdleTimer), NState = State#state{idle_timer = undefined}, handle_incoming(Packet, NState); handle_msg({outgoing, Data}, State) -> @@ -501,7 +501,7 @@ handle_msg( Deliver = {deliver, _Topic, _Msg}, State = #state{active_n = ActiveN} ) -> - Delivers = [Deliver | emqx_misc:drain_deliver(ActiveN)], + Delivers = [Deliver | emqx_utils:drain_deliver(ActiveN)], with_channel(handle_deliver, [Delivers], State); %% Something sent %% TODO: Who will deliver this message? @@ -904,7 +904,7 @@ handle_info(Info, State) -> %% msg => "reach_rate_limit", %% pause => Time %% }), -%% TRef = emqx_misc:start_timer(Time, limit_timeout), +%% TRef = emqx_utils:start_timer(Time, limit_timeout), %% State#state{ %% sockstate = blocked, %% limiter = Limiter1, @@ -928,7 +928,7 @@ run_gc(Stats, State = #state{gc_state = GcSt}) -> end. check_oom(State = #state{oom_policy = OomPolicy}) -> - case ?ENABLED(OomPolicy) andalso emqx_misc:check_oom(OomPolicy) of + case ?ENABLED(OomPolicy) andalso emqx_utils:check_oom(OomPolicy) of {shutdown, Reason} -> %% triggers terminate/2 callback immediately erlang:exit({shutdown, Reason}); diff --git a/apps/emqx_gateway/src/coap/README.md b/apps/emqx_gateway/src/coap/README.md deleted file mode 100644 index 045db529d..000000000 --- a/apps/emqx_gateway/src/coap/README.md +++ /dev/null @@ -1,443 +0,0 @@ - -# Table of Contents - -1. [EMQX 5.0 CoAP Gateway](#org61e5bb8) - 1. [Features](#orgeddbc94) - 1. [PubSub Handler](#orgfc7be2d) - 2. [MQTT Handler](#org55be508) - 3. [Heartbeat](#org3d1a32e) - 4. [Query String](#org9a6b996) - 2. [Implementation](#org9985dfe) - 1. [Request/Response flow](#orge94210c) - 3. [Example](#ref_example) - - - - - -# EMQX 5.0 CoAP Gateway - -emqx-coap is a CoAP Gateway for EMQX. It translates CoAP messages into MQTT messages and make it possible to communiate between CoAP clients and MQTT clients. - - - - -## Features - -- Partially achieves [Publish-Subscribe Broker for the Constrained Application Protocol (CoAP)](https://datatracker.ietf.org/doc/html/draft-ietf-core-coap-pubsub-09) - we called this as ps handler, include following functions: - - Publish - - Subscribe - - UnSubscribe -- Long connection and authorization verification called as MQTT handler - - - - -### PubSub Handler - -1. Publish - - Method: POST\ - URI Schema: ps/{+topic}{?q\*}\ - q\*: [Shared Options](#orgc50043b)\ - Response: - - - 2.04 "Changed" when success - - 4.00 "Bad Request" when error - - 4.01 "Unauthorized" when with wrong auth uri query - -2. Subscribe - - Method: GET - Options: - - - Observer = 0 - - URI Schema: ps/{+topic}{?q\*}\ - q\*: see [Shared Options](#orgc50043b)\ - Response: - - - 2.05 "Content" when success - - 4.00 "Bad Request" when error - - 4.01 "Unauthorized" when with wrong auth uri query - -``` - Client1 Client2 Broker - | | Subscribe | - | | ----- GET /ps/topic1 Observe:0 Token:XX ----> | - | | | - | | <---------- 2.05 Content Observe:10---------- | - | | | - | | | - | | Publish | - | ---------|----------- PUT /ps/topic1 "1033.3" --------> | - | | Notify | - | | <---------- 2.05 Content Observe:11 --------- | - | | | -``` - -3. UnSubscribe - - Method : GET - Options: - - - Observe = 1 - - URI Schema: ps/{+topic}{?q\*}\ - q\*: see [Shared Options](#orgc50043b)\ - Response: - - - 2.07 "No Content" when success - - 4.00 "Bad Request" when error - - 4.01 "Unauthorized" when with wrong auth uri query - - - - -### MQTT Handler - - Establishing a connection is optional. If the CoAP client needs to use connection-based operations, it must first establish a connection. -At the same time, the connectionless mode and the connected mode cannot be mixed. -In connection mode, the Publish/Subscribe/UnSubscribe sent by the client must be has Token and ClientId in query string. -If the Token and Clientid is wrong/miss, EMQX will reset the request. -The communication token is the data carried in the response payload after the client successfully establishes a connection. -After obtaining the token, the client's subsequent request must attach "token=Token" to the Query String -ClientId is necessary when there is a connection, and is a unique identifier defined by the client. -The server manages the client through the ClientId. If the ClientId is wrong, EMQX will reset the request. - -1. Create a Connection - - Method: POST - URI Schema: mqtt/connection{?q\*} - q\*: - - - clientid := client uid - - username - - password - - Response: - - - 2.01 "Created" when success - - 4.00 "Bad Request" when error - - 4.01 "Unauthorized" wrong username or password - - Payload: Token if success - -2. Close a Connection - - Method : DELETE - URI Schema: mqtt/connection{?q\*} - q\*: - - - clientid := client uid - - token - - Response: - - - 2.01 "Deleted" when success - - 4.00 "Bad Request" when error - - 4.01 "Unauthorized" wrong clientid or token - - - - -### Heartbeat - -The Coap client can maintain the "connection" with the server through the heartbeat, -regardless of whether it is authenticated or not, -so that the server will not release related resources -Method : PUT -URI Schema: mqtt/connection{?q\*} -q\*: - -- clientid if authenticated -- token if authenticated - -Response: - -- 2.01 "Changed" when success -- 4.00 "Bad Request" when error -- 4.01 "Unauthorized" wrong clientid or token - - - - -### Query String - -CoAP gateway uses some options in query string to conversion between MQTT CoAP. - -1. Shared Options - - - clientid - - token - -2. Connect Options - - - username - - password - -3. Publish - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
OptionTypeDefault
retainbooleanfalse
qosMQTT QosSee here
expiryMessage Expiry Interval0(Never expiry)
- -4. Subscribe - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
OptionTypeDefault
qosMQTT QosSee here
nlMQTT Subscribe No Local0
rhMQTT Subscribe Retain Handing0
- -5. MQTT Qos <=> CoAP non/con - - 1.notif_type - Control the type of notify messages when the observed object has changed.Can be: - - - non - - con - - qos - in this value, MQTT Qos0 -> non, Qos1/Qos2 -> con - - 2.subscribe_qos - Control the qos of subscribe.Can be: - - - qos0 - - qos1 - - qos2 - - coap - in this value, CoAP non -> qos0, con -> qos1 - - 3.publish_qos - like subscribe_qos, but control the qos of the publish MQTT message - - - - -## Implementation - - - - -### Request/Response flow - -![img](./doc/flow.png) - -1. Authorization check - - Check whether the clientid and token in the query string match the current connection - -2. Session - - Manager the "Transport Manager" "Observe Resources Manager" and next message id - -3. Transport Mnager - - Manager "Transport" create/close/dispatch - -4. Observe resources Mnager - - Mnager observe topic and token - -5. Transport - - ![img](./doc/transport.png) - - 1. Shared State - - ![img](./doc/shared_state.png) - -6. Handler - - 1. pubsub - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
MethodObserveAction
GET0subscribe and reply result
GET1unsubscribe and reply result
POSTXpublish and reply result
- - 2. mqtt - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
MethodAction
PUTreply result
POSTreturn create connection action
DELETEreturn close connection action
- - - -## Example -1. Create Connection -``` -coap-client -m post -e "" "coap://127.0.0.1/mqtt/connection?clientid=123&username=admin&password=public" -``` -Server will return token **X** in payload - -2. Update Connection -``` -coap-client -m put -e "" "coap://127.0.0.1/mqtt/connection?clientid=123&token=X" -``` - -3. Publish -``` -coap-client -m post -e "Hellow" "obstoken" "coap://127.0.0.1/ps/coap/test?clientid=123&username=admin&password=public" -``` -if you want to publish with auth, you must first establish a connection, and then post publish request on the same socket, so libcoap client can't simulation publish with a token - -``` -coap-client -m post -e "Hellow" "coap://127.0.0.1/ps/coap/test?clientid=123&token=X" -``` - -4. Subscribe -``` -coap-client -m get -s 60 -O 6,0x00 -o - -T "obstoken" "coap://127.0.0.1/ps/coap/test?clientid=123&username=admin&password=public" -``` -**Or** - -``` -coap-client -m get -s 60 -O 6,0x00 -o - -T "obstoken" "coap://127.0.0.1/ps/coap/test?clientid=123&token=X" -``` -5. Close Connection -``` -coap-client -m delete -e "" "coap://127.0.0.1/mqtt/connection?clientid=123&token=X -``` - diff --git a/apps/emqx_gateway/src/emqx_gateway.app.src b/apps/emqx_gateway/src/emqx_gateway.app.src index ced013497..0419666b4 100644 --- a/apps/emqx_gateway/src/emqx_gateway.app.src +++ b/apps/emqx_gateway/src/emqx_gateway.app.src @@ -1,10 +1,10 @@ %% -*- mode: erlang -*- {application, emqx_gateway, [ {description, "The Gateway management application"}, - {vsn, "0.1.14"}, + {vsn, "0.1.15"}, {registered, []}, {mod, {emqx_gateway_app, []}}, - {applications, [kernel, stdlib, grpc, emqx, emqx_authn, emqx_ctl]}, + {applications, [kernel, stdlib, emqx, emqx_authn, emqx_ctl]}, {env, []}, {modules, []}, {licenses, ["Apache 2.0"]}, diff --git a/apps/emqx_gateway/src/emqx_gateway_api.erl b/apps/emqx_gateway/src/emqx_gateway_api.erl index 62f723d59..bc44daca8 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api.erl @@ -395,7 +395,7 @@ fields(Gw) when Gw == exproto -> [{name, mk(Gw, #{desc => ?DESC(gateway_name)})}] ++ - convert_listener_struct(emqx_gateway_schema:fields(Gw)); + convert_listener_struct(emqx_gateway_schema:gateway_schema(Gw)); fields(Gw) when Gw == update_stomp; Gw == update_mqttsn; @@ -405,7 +405,7 @@ fields(Gw) when -> "update_" ++ GwStr = atom_to_list(Gw), Gw1 = list_to_existing_atom(GwStr), - remove_listener_and_authn(emqx_gateway_schema:fields(Gw1)); + remove_listener_and_authn(emqx_gateway_schema:gateway_schema(Gw1)); fields(Listener) when Listener == tcp_listener; Listener == ssl_listener; diff --git a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl index e64e918b4..cd387e3bb 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl @@ -115,7 +115,7 @@ clients(get, #{ fun ?MODULE:format_channel_info/2 ); Node0 -> - case emqx_misc:safe_to_existing_atom(Node0) of + case emqx_utils:safe_to_existing_atom(Node0) of {ok, Node1} -> QStringWithoutNode = maps:without([<<"node">>], QString), emqx_mgmt_api:node_query( diff --git a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl index 14b80a500..d90bf3689 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl @@ -96,7 +96,7 @@ listeners(post, #{bindings := #{name := Name0}, body := LConf}) -> LName = binary_to_atom(maps:get(<<"name">>, LConf)), Path = [listeners, Type, LName], - case emqx_map_lib:deep_get(Path, RunningConf, undefined) of + case emqx_utils_maps:deep_get(Path, RunningConf, undefined) of undefined -> ListenerId = emqx_gateway_utils:listener_id( GwName, Type, LName diff --git a/apps/emqx_gateway/src/emqx_gateway_app.erl b/apps/emqx_gateway/src/emqx_gateway_app.erl index cb5a16fde..f0406bcaa 100644 --- a/apps/emqx_gateway/src/emqx_gateway_app.erl +++ b/apps/emqx_gateway/src/emqx_gateway_app.erl @@ -41,35 +41,38 @@ stop(_State) -> %% Internal funcs load_default_gateway_applications() -> - Apps = gateway_type_searching(), - lists:foreach(fun reg/1, Apps). + lists:foreach( + fun(Def) -> + load_gateway_application(Def) + end, + emqx_gateway_utils:find_gateway_definitions() + ). -gateway_type_searching() -> - %% FIXME: Hardcoded apps - [ - emqx_stomp_impl, - emqx_sn_impl, - emqx_exproto_impl, - emqx_coap_impl, - emqx_lwm2m_impl - ]. - -reg(Mod) -> - try - Mod:reg(), - ?SLOG(debug, #{ - msg => "register_gateway_succeed", - callback_module => Mod - }) - catch - Class:Reason:Stk -> +load_gateway_application( + #{ + name := Name, + callback_module := CbMod, + config_schema_module := SchemaMod + } +) -> + RegistryOptions = [{cbkmod, CbMod}, {schema, SchemaMod}], + case emqx_gateway_registry:reg(Name, RegistryOptions) of + ok -> + ?SLOG(debug, #{ + msg => "register_gateway_succeed", + callback_module => CbMod + }); + {error, already_registered} -> ?SLOG(error, #{ - msg => "failed_to_register_gateway", - callback_module => Mod, - reason => {Class, Reason}, - stacktrace => Stk + msg => "gateway_already_registered", + name => Name, + callback_module => CbMod }) - end. + end; +load_gateway_application(_) -> + ?SLOG(error, #{ + msg => "invalid_gateway_defination" + }). load_gateway_by_default() -> load_gateway_by_default(confs()). diff --git a/apps/emqx_gateway/src/emqx_gateway_cli.erl b/apps/emqx_gateway/src/emqx_gateway_cli.erl index df808f295..fb4261065 100644 --- a/apps/emqx_gateway/src/emqx_gateway_cli.erl +++ b/apps/emqx_gateway/src/emqx_gateway_cli.erl @@ -74,7 +74,7 @@ gateway(["load", Name, Conf]) -> case emqx_gateway_conf:load_gateway( bin(Name), - emqx_json:decode(Conf, [return_maps]) + emqx_utils_json:decode(Conf, [return_maps]) ) of {ok, _} -> diff --git a/apps/emqx_gateway/src/emqx_gateway_cm.erl b/apps/emqx_gateway/src/emqx_gateway_cm.erl index 71ec4bf59..837600811 100644 --- a/apps/emqx_gateway/src/emqx_gateway_cm.erl +++ b/apps/emqx_gateway/src/emqx_gateway_cm.erl @@ -766,9 +766,9 @@ init(Options) -> TabOpts = [public, {write_concurrency, true}], {ChanTab, ConnTab, InfoTab} = cmtabs(GwName), - ok = emqx_tables:new(ChanTab, [bag, {read_concurrency, true} | TabOpts]), - ok = emqx_tables:new(ConnTab, [bag | TabOpts]), - ok = emqx_tables:new(InfoTab, [ordered_set, compressed | TabOpts]), + ok = emqx_utils_ets:new(ChanTab, [bag, {read_concurrency, true} | TabOpts]), + ok = emqx_utils_ets:new(ConnTab, [bag | TabOpts]), + ok = emqx_utils_ets:new(InfoTab, [ordered_set, compressed | TabOpts]), %% Start link cm-registry process %% XXX: Should I hang it under a higher level supervisor? @@ -802,7 +802,7 @@ handle_info( {'DOWN', _MRef, process, Pid, _Reason}, State = #state{gwname = GwName, chan_pmon = PMon} ) -> - ChanPids = [Pid | emqx_misc:drain_down(?DEFAULT_BATCH_SIZE)], + ChanPids = [Pid | emqx_utils:drain_down(?DEFAULT_BATCH_SIZE)], {Items, PMon1} = emqx_pmon:erase_all(ChanPids, PMon), CmTabs = cmtabs(GwName), diff --git a/apps/emqx_gateway/src/emqx_gateway_conf.erl b/apps/emqx_gateway/src/emqx_gateway_conf.erl index 07a4c1e2c..56a3e2068 100644 --- a/apps/emqx_gateway/src/emqx_gateway_conf.erl +++ b/apps/emqx_gateway/src/emqx_gateway_conf.erl @@ -106,7 +106,7 @@ unconvert_listeners(Ls) when is_list(Ls) -> {[Type, Name], Lis1} = maps_key_take([<<"type">>, <<"name">>], Lis), _ = vaildate_listener_name(Name), NLis1 = maps:without([<<"id">>, <<"running">>], Lis1), - emqx_map_lib:deep_merge(Acc, #{Type => #{Name => NLis1}}) + emqx_utils_maps:deep_merge(Acc, #{Type => #{Name => NLis1}}) end, #{}, Ls @@ -160,8 +160,8 @@ gateway(GwName0) -> RawConf = emqx_config:fill_defaults( emqx_config:get_root_raw(Path) ), - Confs = emqx_map_lib:jsonable_map( - emqx_map_lib:deep_get(Path, RawConf) + Confs = emqx_utils_maps:jsonable_map( + emqx_utils_maps:deep_get(Path, RawConf) ), LsConf = maps:get(<<"listeners">>, Confs, #{}), Confs#{<<"listeners">> => convert_listeners(GwName, LsConf)}. @@ -198,8 +198,8 @@ listeners(GwName0) -> RawConf = emqx_config:fill_defaults( emqx_config:get_root_raw([<<"gateway">>]) ), - Listeners = emqx_map_lib:jsonable_map( - emqx_map_lib:deep_get( + Listeners = emqx_utils_maps:jsonable_map( + emqx_utils_maps:deep_get( [<<"gateway">>, GwName, <<"listeners">>], RawConf ) ), @@ -213,12 +213,12 @@ listener(ListenerId) -> ), try Path = [<<"gateway">>, GwName, <<"listeners">>, Type, LName], - LConf = emqx_map_lib:deep_get(Path, RootConf), + LConf = emqx_utils_maps:deep_get(Path, RootConf), Running = emqx_gateway_utils:is_running( binary_to_existing_atom(ListenerId), LConf ), {ok, - emqx_map_lib:jsonable_map( + emqx_utils_maps:jsonable_map( LConf#{ id => ListenerId, type => Type, @@ -305,8 +305,8 @@ ret_ok_err({ok, _}) -> ok; ret_ok_err(Err) -> Err. ret_gw(GwName, {ok, #{raw_config := GwConf}}) -> - GwConf1 = emqx_map_lib:deep_get([bin(GwName)], GwConf), - LsConf = emqx_map_lib:deep_get( + GwConf1 = emqx_utils_maps:deep_get([bin(GwName)], GwConf), + LsConf = emqx_utils_maps:deep_get( [bin(GwName), <<"listeners">>], GwConf, #{} @@ -331,7 +331,7 @@ ret_gw(_GwName, Err) -> Err. ret_authn(GwName, {ok, #{raw_config := GwConf}}) -> - Authn = emqx_map_lib:deep_get( + Authn = emqx_utils_maps:deep_get( [bin(GwName), <<"authentication">>], GwConf ), @@ -340,7 +340,7 @@ ret_authn(_GwName, Err) -> Err. ret_authn(GwName, {LType, LName}, {ok, #{raw_config := GwConf}}) -> - Authn = emqx_map_lib:deep_get( + Authn = emqx_utils_maps:deep_get( [ bin(GwName), <<"listeners">>, @@ -355,7 +355,7 @@ ret_authn(_, _, Err) -> Err. ret_listener_or_err(GwName, {LType, LName}, {ok, #{raw_config := GwConf}}) -> - LConf = emqx_map_lib:deep_get( + LConf = emqx_utils_maps:deep_get( [bin(GwName), <<"listeners">>, bin(LType), bin(LName)], GwConf ), @@ -377,7 +377,7 @@ pre_config_update(_, {load_gateway, GwName, Conf}, RawConf) -> case maps:get(GwName, RawConf, undefined) of undefined -> NConf = tune_gw_certs(fun convert_certs/2, GwName, Conf), - {ok, emqx_map_lib:deep_put([GwName], RawConf, NConf)}; + {ok, emqx_utils_maps:deep_put([GwName], RawConf, NConf)}; _ -> badres_gateway(already_exist, GwName) end; @@ -389,7 +389,7 @@ pre_config_update(_, {update_gateway, GwName, Conf}, RawConf) -> Conf1 = maps:without([<<"listeners">>, ?AUTHN_BIN], Conf), NConf = tune_gw_certs(fun convert_certs/2, GwName, Conf1), NConf1 = maps:merge(GwRawConf, NConf), - {ok, emqx_map_lib:deep_put([GwName], RawConf, NConf1)} + {ok, emqx_utils_maps:deep_put([GwName], RawConf, NConf1)} end; pre_config_update(_, {unload_gateway, GwName}, RawConf) -> _ = tune_gw_certs( @@ -400,7 +400,7 @@ pre_config_update(_, {unload_gateway, GwName}, RawConf) -> {ok, maps:remove(GwName, RawConf)}; pre_config_update(_, {add_listener, GwName, {LType, LName}, Conf}, RawConf) -> case - emqx_map_lib:deep_get( + emqx_utils_maps:deep_get( [GwName, <<"listeners">>, LType, LName], RawConf, undefined ) of @@ -408,7 +408,7 @@ pre_config_update(_, {add_listener, GwName, {LType, LName}, Conf}, RawConf) -> NConf = convert_certs(certs_dir(GwName), Conf), NListener = #{LType => #{LName => NConf}}, {ok, - emqx_map_lib:deep_merge( + emqx_utils_maps:deep_merge( RawConf, #{GwName => #{<<"listeners">> => NListener}} )}; @@ -417,7 +417,7 @@ pre_config_update(_, {add_listener, GwName, {LType, LName}, Conf}, RawConf) -> end; pre_config_update(_, {update_listener, GwName, {LType, LName}, Conf}, RawConf) -> case - emqx_map_lib:deep_get( + emqx_utils_maps:deep_get( [GwName, <<"listeners">>, LType, LName], RawConf, undefined ) of @@ -425,7 +425,7 @@ pre_config_update(_, {update_listener, GwName, {LType, LName}, Conf}, RawConf) - badres_listener(not_found, GwName, LType, LName); OldConf -> NConf = convert_certs(certs_dir(GwName), Conf, OldConf), - NRawConf = emqx_map_lib:deep_put( + NRawConf = emqx_utils_maps:deep_put( [GwName, <<"listeners">>, LType, LName], RawConf, NConf @@ -434,22 +434,22 @@ pre_config_update(_, {update_listener, GwName, {LType, LName}, Conf}, RawConf) - end; pre_config_update(_, {remove_listener, GwName, {LType, LName}}, RawConf) -> Path = [GwName, <<"listeners">>, LType, LName], - case emqx_map_lib:deep_get(Path, RawConf, undefined) of + case emqx_utils_maps:deep_get(Path, RawConf, undefined) of undefined -> {ok, RawConf}; OldConf -> clear_certs(certs_dir(GwName), OldConf), - {ok, emqx_map_lib:deep_remove(Path, RawConf)} + {ok, emqx_utils_maps:deep_remove(Path, RawConf)} end; pre_config_update(_, {add_authn, GwName, Conf}, RawConf) -> case - emqx_map_lib:deep_get( + emqx_utils_maps:deep_get( [GwName, ?AUTHN_BIN], RawConf, undefined ) of undefined -> {ok, - emqx_map_lib:deep_merge( + emqx_utils_maps:deep_merge( RawConf, #{GwName => #{?AUTHN_BIN => Conf}} )}; @@ -458,7 +458,7 @@ pre_config_update(_, {add_authn, GwName, Conf}, RawConf) -> end; pre_config_update(_, {add_authn, GwName, {LType, LName}, Conf}, RawConf) -> case - emqx_map_lib:deep_get( + emqx_utils_maps:deep_get( [GwName, <<"listeners">>, LType, LName], RawConf, undefined @@ -477,25 +477,25 @@ pre_config_update(_, {add_authn, GwName, {LType, LName}, Conf}, RawConf) -> #{LType => #{LName => NListener}} } }, - {ok, emqx_map_lib:deep_merge(RawConf, NGateway)}; + {ok, emqx_utils_maps:deep_merge(RawConf, NGateway)}; _ -> badres_listener_authn(already_exist, GwName, LType, LName) end end; pre_config_update(_, {update_authn, GwName, Conf}, RawConf) -> case - emqx_map_lib:deep_get( + emqx_utils_maps:deep_get( [GwName, ?AUTHN_BIN], RawConf, undefined ) of undefined -> badres_authn(not_found, GwName); _Authn -> - {ok, emqx_map_lib:deep_put([GwName, ?AUTHN_BIN], RawConf, Conf)} + {ok, emqx_utils_maps:deep_put([GwName, ?AUTHN_BIN], RawConf, Conf)} end; pre_config_update(_, {update_authn, GwName, {LType, LName}, Conf}, RawConf) -> case - emqx_map_lib:deep_get( + emqx_utils_maps:deep_get( [GwName, <<"listeners">>, LType, LName], RawConf, undefined @@ -514,7 +514,7 @@ pre_config_update(_, {update_authn, GwName, {LType, LName}, Conf}, RawConf) -> Listener ), {ok, - emqx_map_lib:deep_put( + emqx_utils_maps:deep_put( [GwName, <<"listeners">>, LType, LName], RawConf, NListener @@ -523,12 +523,12 @@ pre_config_update(_, {update_authn, GwName, {LType, LName}, Conf}, RawConf) -> end; pre_config_update(_, {remove_authn, GwName}, RawConf) -> {ok, - emqx_map_lib:deep_remove( + emqx_utils_maps:deep_remove( [GwName, ?AUTHN_BIN], RawConf )}; pre_config_update(_, {remove_authn, GwName, {LType, LName}}, RawConf) -> Path = [GwName, <<"listeners">>, LType, LName, ?AUTHN_BIN], - {ok, emqx_map_lib:deep_remove(Path, RawConf)}; + {ok, emqx_utils_maps:deep_remove(Path, RawConf)}; pre_config_update(_, UnknownReq, _RawConf) -> logger:error("Unknown configuration update request: ~0p", [UnknownReq]), {error, badreq}. diff --git a/apps/emqx_gateway/src/emqx_gateway_http.erl b/apps/emqx_gateway/src/emqx_gateway_http.erl index a0155a126..7aaaee9cb 100644 --- a/apps/emqx_gateway/src/emqx_gateway_http.erl +++ b/apps/emqx_gateway/src/emqx_gateway_http.erl @@ -240,7 +240,7 @@ authn(GwName) -> ChainName = emqx_gateway_utils:global_chain(GwName), wrap_chain_name( ChainName, - emqx_map_lib:jsonable_map(emqx:get_raw_config(Path)) + emqx_utils_maps:jsonable_map(emqx:get_raw_config(Path)) ). -spec authn(gateway_name(), binary()) -> map(). @@ -250,7 +250,7 @@ authn(GwName, ListenerId) -> ChainName = emqx_gateway_utils:listener_chain(GwName, Type, Name), wrap_chain_name( ChainName, - emqx_map_lib:jsonable_map(emqx:get_raw_config(Path)) + emqx_utils_maps:jsonable_map(emqx:get_raw_config(Path)) ). wrap_chain_name(ChainName, Conf) -> @@ -404,7 +404,7 @@ return_http_error(Code, Msg) -> -spec reason2msg({atom(), map()} | any()) -> error | string(). reason2msg({badconf, #{key := Key, value := Value, reason := Reason}}) -> NValue = - case emqx_json:safe_encode(Value) of + case emqx_utils_json:safe_encode(Value) of {ok, Str} -> Str; {error, _} -> emqx_gateway_utils:stringfy(Value) end, @@ -495,7 +495,7 @@ reason2msg( reason2msg( {#{roots := [{gateway, _}]}, [_ | _]} = Error ) -> - Bin = emqx_misc:readable_error_msg(Error), + Bin = emqx_utils:readable_error_msg(Error), <<"Invalid configurations: ", Bin/binary>>; reason2msg(_) -> error. diff --git a/apps/emqx_gateway/src/emqx_gateway_metrics.erl b/apps/emqx_gateway/src/emqx_gateway_metrics.erl index e94510387..0aa2ff210 100644 --- a/apps/emqx_gateway/src/emqx_gateway_metrics.erl +++ b/apps/emqx_gateway/src/emqx_gateway_metrics.erl @@ -89,7 +89,7 @@ tabname(GwName) -> init([GwName]) -> TabOpts = [public, {write_concurrency, true}], - ok = emqx_tables:new(tabname(GwName), [set | TabOpts]), + ok = emqx_utils_ets:new(tabname(GwName), [set | TabOpts]), {ok, #state{}}. handle_call(_Request, _From, State) -> diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl index 28c1e6f89..8c80fc1fa 100644 --- a/apps/emqx_gateway/src/emqx_gateway_schema.erl +++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl @@ -53,329 +53,29 @@ -export([proxy_protocol_opts/0]). +-export([mountpoint/0, mountpoint/1, gateway_common_options/0, gateway_schema/1]). + namespace() -> gateway. tags() -> [<<"Gateway">>]. -roots() -> [gateway]. +roots() -> + [{gateway, sc(ref(?MODULE, gateway), #{importance => ?IMPORTANCE_HIDDEN})}]. fields(gateway) -> - [ - {stomp, - sc( - ref(stomp), - #{ - required => {false, recursively}, - desc => ?DESC(stomp) - } - )}, - {mqttsn, - sc( - ref(mqttsn), - #{ - required => {false, recursively}, - desc => ?DESC(mqttsn) - } - )}, - {coap, - sc( - ref(coap), - #{ - required => {false, recursively}, - desc => ?DESC(coap) - } - )}, - {lwm2m, - sc( - ref(lwm2m), - #{ - required => {false, recursively}, - desc => ?DESC(lwm2m) - } - )}, - {exproto, - sc( - ref(exproto), - #{ - required => {false, recursively}, - desc => ?DESC(exproto) - } - )} - ]; -fields(stomp) -> - [ - {frame, sc(ref(stomp_frame))}, - {mountpoint, mountpoint()}, - {listeners, sc(ref(tcp_listeners), #{desc => ?DESC(tcp_listeners)})} - ] ++ gateway_common_options(); -fields(stomp_frame) -> - [ - {max_headers, - sc( - non_neg_integer(), - #{ - default => 10, - desc => ?DESC(stom_frame_max_headers) - } - )}, - {max_headers_length, - sc( - non_neg_integer(), - #{ - default => 1024, - desc => ?DESC(stomp_frame_max_headers_length) - } - )}, - {max_body_length, - sc( - integer(), - #{ - default => 65536, - desc => ?DESC(stom_frame_max_body_length) - } - )} - ]; -fields(mqttsn) -> - [ - {gateway_id, - sc( - integer(), - #{ - default => 1, - required => true, - desc => ?DESC(mqttsn_gateway_id) - } - )}, - {broadcast, - sc( - boolean(), - #{ - default => false, - desc => ?DESC(mqttsn_broadcast) - } - )}, - %% TODO: rename - {enable_qos3, - sc( - boolean(), - #{ - default => true, - desc => ?DESC(mqttsn_enable_qos3) - } - )}, - {subs_resume, - sc( - boolean(), - #{ - default => false, - desc => ?DESC(mqttsn_subs_resume) - } - )}, - {predefined, - sc( - hoconsc:array(ref(mqttsn_predefined)), - #{ - default => [], - required => {false, recursively}, - desc => ?DESC(mqttsn_predefined) - } - )}, - {mountpoint, mountpoint()}, - {listeners, sc(ref(udp_listeners), #{desc => ?DESC(udp_listeners)})} - ] ++ gateway_common_options(); -fields(mqttsn_predefined) -> - [ - {id, - sc(integer(), #{ - required => true, - desc => ?DESC(mqttsn_predefined_id) - })}, - - {topic, - sc(binary(), #{ - required => true, - desc => ?DESC(mqttsn_predefined_topic) - })} - ]; -fields(coap) -> - [ - {heartbeat, - sc( - duration(), - #{ - default => <<"30s">>, - desc => ?DESC(coap_heartbeat) - } - )}, - {connection_required, - sc( - boolean(), - #{ - default => false, - desc => ?DESC(coap_connection_required) - } - )}, - {notify_type, - sc( - hoconsc:enum([non, con, qos]), - #{ - default => qos, - desc => ?DESC(coap_notify_type) - } - )}, - {subscribe_qos, - sc( - hoconsc:enum([qos0, qos1, qos2, coap]), - #{ - default => coap, - desc => ?DESC(coap_subscribe_qos) - } - )}, - {publish_qos, - sc( - hoconsc:enum([qos0, qos1, qos2, coap]), - #{ - default => coap, - desc => ?DESC(coap_publish_qos) - } - )}, - {mountpoint, mountpoint()}, - {listeners, - sc( - ref(udp_listeners), - #{desc => ?DESC(udp_listeners)} - )} - ] ++ gateway_common_options(); -fields(lwm2m) -> - [ - {xml_dir, - sc( - binary(), - #{ - %% since this is not packaged with emqx, nor - %% present in the packages, we must let the user - %% specify it rather than creating a dynamic - %% default (especially difficult to handle when - %% generating docs). - example => <<"/etc/emqx/lwm2m_xml">>, - required => true, - desc => ?DESC(lwm2m_xml_dir) - } - )}, - {lifetime_min, - sc( - duration(), - #{ - default => <<"15s">>, - desc => ?DESC(lwm2m_lifetime_min) - } - )}, - {lifetime_max, - sc( - duration(), - #{ - default => <<"86400s">>, - desc => ?DESC(lwm2m_lifetime_max) - } - )}, - {qmode_time_window, - sc( - duration_s(), - #{ - default => <<"22s">>, - desc => ?DESC(lwm2m_qmode_time_window) - } - )}, - %% TODO: Support config resource path - {auto_observe, - sc( - boolean(), - #{ - default => false, - desc => ?DESC(lwm2m_auto_observe) - } - )}, - %% FIXME: not working now - {update_msg_publish_condition, - sc( - hoconsc:enum([always, contains_object_list]), - #{ - default => contains_object_list, - desc => ?DESC(lwm2m_update_msg_publish_condition) - } - )}, - {translators, - sc( - ref(lwm2m_translators), - #{ - required => true, - desc => ?DESC(lwm2m_translators) - } - )}, - {mountpoint, mountpoint("lwm2m/${endpoint_name}/")}, - {listeners, sc(ref(udp_listeners), #{desc => ?DESC(udp_listeners)})} - ] ++ gateway_common_options(); -fields(exproto) -> - [ - {server, - sc( - ref(exproto_grpc_server), - #{ - required => true, - desc => ?DESC(exproto_server) - } - )}, - {handler, - sc( - ref(exproto_grpc_handler), - #{ - required => true, - desc => ?DESC(exproto_handler) - } - )}, - {mountpoint, mountpoint()}, - {listeners, sc(ref(tcp_udp_listeners), #{desc => ?DESC(tcp_udp_listeners)})} - ] ++ gateway_common_options(); -fields(exproto_grpc_server) -> - [ - {bind, - sc( - hoconsc:union([ip_port(), integer()]), - #{ - required => true, - desc => ?DESC(exproto_grpc_server_bind) - } - )}, - {ssl_options, - sc( - ref(ssl_server_opts), - #{ - required => {false, recursively}, - desc => ?DESC(exproto_grpc_server_ssl) - } - )} - ]; -fields(exproto_grpc_handler) -> - [ - {address, sc(binary(), #{required => true, desc => ?DESC(exproto_grpc_handler_address)})}, - {ssl_options, - sc( - ref(emqx_schema, "ssl_client_opts"), - #{ - required => {false, recursively}, - desc => ?DESC(exproto_grpc_handler_ssl) - } - )} - ]; -fields(ssl_server_opts) -> - emqx_schema:server_ssl_opts_schema( - #{ - depth => 10, - reuse_sessions => true, - versions => tls_all_available - }, - true + lists:map( + fun(#{name := Name, config_schema_module := Mod}) -> + {Name, + sc( + ref(Mod, Name), + #{ + required => {false, recursively}, + desc => ?DESC(Name) + } + )} + end, + emqx_gateway_utils:find_gateway_definitions() ); fields(clientinfo_override) -> [ @@ -389,68 +89,6 @@ fields(clientinfo_override) -> })}, {clientid, sc(binary(), #{desc => ?DESC(gateway_common_clientinfo_override_clientid)})} ]; -fields(lwm2m_translators) -> - [ - {command, - sc( - ref(translator), - #{ - desc => ?DESC(lwm2m_translators_command), - required => true - } - )}, - {response, - sc( - ref(translator), - #{ - desc => ?DESC(lwm2m_translators_response), - required => true - } - )}, - {notify, - sc( - ref(translator), - #{ - desc => ?DESC(lwm2m_translators_notify), - required => true - } - )}, - {register, - sc( - ref(translator), - #{ - desc => ?DESC(lwm2m_translators_register), - required => true - } - )}, - {update, - sc( - ref(translator), - #{ - desc => ?DESC(lwm2m_translators_update), - required => true - } - )} - ]; -fields(translator) -> - [ - {topic, - sc( - binary(), - #{ - required => true, - desc => ?DESC(translator_topic) - } - )}, - {qos, - sc( - emqx_schema:qos(), - #{ - default => 0, - desc => ?DESC(translator_qos) - } - )} - ]; fields(udp_listeners) -> [ {udp, sc(map(name, ref(udp_listener)), #{desc => ?DESC(listener_name_to_settings_map)})}, @@ -522,37 +160,8 @@ fields(dtls_opts) -> desc(gateway) -> "EMQX Gateway configuration root."; -desc(stomp) -> - "The STOMP protocol gateway provides EMQX with the ability to access STOMP\n" - "(Simple (or Streaming) Text Orientated Messaging Protocol) protocol."; -desc(stomp_frame) -> - "Size limits for the STOMP frames."; -desc(mqttsn) -> - "The MQTT-SN (MQTT for Sensor Networks) protocol gateway."; -desc(mqttsn_predefined) -> - "The pre-defined topic name corresponding to the pre-defined topic\n" - "ID of N.\n\n" - "Note: the pre-defined topic ID of 0 is reserved."; -desc(coap) -> - "The CoAP protocol gateway provides EMQX with the access capability of the CoAP protocol.\n" - "It allows publishing, subscribing, and receiving messages to EMQX in accordance\n" - "with a certain defined CoAP message format."; -desc(lwm2m) -> - "The LwM2M protocol gateway."; -desc(exproto) -> - "Settings for EMQX extension protocol (exproto)."; -desc(exproto_grpc_server) -> - "Settings for the exproto gRPC server."; -desc(exproto_grpc_handler) -> - "Settings for the exproto gRPC connection handler."; -desc(ssl_server_opts) -> - "SSL configuration for the server."; desc(clientinfo_override) -> "ClientInfo override."; -desc(lwm2m_translators) -> - "MQTT topics that correspond to LwM2M events."; -desc(translator) -> - "MQTT topic that corresponds to a particular type of event."; desc(udp_listeners) -> "Settings for the UDP listeners."; desc(tcp_listeners) -> @@ -715,8 +324,18 @@ proxy_protocol_opts() -> )} ]. -sc(Type) -> - sc(Type, #{}). +%%-------------------------------------------------------------------- +%% dynamic schemas + +%% FIXME: don't hardcode the gateway names +gateway_schema(stomp) -> emqx_stomp_schema:fields(stomp); +gateway_schema(mqttsn) -> emqx_mqttsn_schema:fields(mqttsn); +gateway_schema(coap) -> emqx_coap_schema:fields(coap); +gateway_schema(lwm2m) -> emqx_lwm2m_schema:fields(lwm2m); +gateway_schema(exproto) -> emqx_exproto_schema:fields(exproto). + +%%-------------------------------------------------------------------- +%% helpers sc(Type, Meta) -> hoconsc:mk(Type, Meta). diff --git a/apps/emqx_gateway/src/emqx_gateway_utils.erl b/apps/emqx_gateway/src/emqx_gateway_utils.erl index cee5baaa8..7a0188387 100644 --- a/apps/emqx_gateway/src/emqx_gateway_utils.erl +++ b/apps/emqx_gateway/src/emqx_gateway_utils.erl @@ -46,7 +46,8 @@ global_chain/1, listener_chain/3, make_deprecated_paths/1, - make_compatible_schema/2 + make_compatible_schema/2, + find_gateway_definitions/0 ]). -export([stringfy/1]). @@ -222,7 +223,7 @@ merge_default(Udp, Options) -> case lists:keytake(Key, 1, Options) of {value, {Key, TcpOpts}, Options1} -> [ - {Key, emqx_misc:merge_opts(Default, TcpOpts)} + {Key, emqx_utils:merge_opts(Default, TcpOpts)} | Options1 ]; false -> @@ -481,7 +482,7 @@ frame_options(Options) -> -spec init_gc_state(map()) -> emqx_gc:gc_state() | undefined. init_gc_state(Options) -> - emqx_misc:maybe_apply(fun emqx_gc:init/1, force_gc_policy(Options)). + emqx_utils:maybe_apply(fun emqx_gc:init/1, force_gc_policy(Options)). -spec force_gc_policy(map()) -> emqx_gc:opts() | undefined. force_gc_policy(Options) -> @@ -562,3 +563,82 @@ make_compatible_schema2(Path, SchemaFun) -> end, Schema ). + +-spec find_gateway_definitions() -> list(gateway_def()). +find_gateway_definitions() -> + lists:flatten( + lists:map( + fun(App) -> + gateways(find_attrs(App, gateway)) + end, + ignore_lib_apps(application:loaded_applications()) + ) + ). + +gateways([]) -> + []; +gateways([ + {_App, _Mod, + Defination = + #{ + name := Name, + callback_module := CbMod, + config_schema_module := SchemaMod + }} + | More +]) when is_atom(Name), is_atom(CbMod), is_atom(SchemaMod) -> + [Defination | gateways(More)]. + +find_attrs(App, Def) -> + [ + {App, Mod, Attr} + || {ok, Modules} <- [application:get_key(App, modules)], + Mod <- Modules, + {Name, Attrs} <- module_attributes(Mod), + Name =:= Def, + Attr <- Attrs + ]. + +module_attributes(Module) -> + try + apply(Module, module_info, [attributes]) + catch + error:undef -> [] + end. + +ignore_lib_apps(Apps) -> + LibApps = [ + kernel, + stdlib, + sasl, + appmon, + eldap, + erts, + syntax_tools, + ssl, + crypto, + mnesia, + os_mon, + inets, + goldrush, + gproc, + runtime_tools, + snmp, + otp_mibs, + public_key, + asn1, + ssh, + hipe, + common_test, + observer, + webtool, + xmerl, + tools, + test_server, + compiler, + debugger, + eunit, + et, + wx + ], + [AppName || {AppName, _, _} <- Apps, not lists:member(AppName, LibApps)]. diff --git a/apps/emqx_gateway/src/lwm2m/.gitignore b/apps/emqx_gateway/src/lwm2m/.gitignore deleted file mode 100644 index be6914be3..000000000 --- a/apps/emqx_gateway/src/lwm2m/.gitignore +++ /dev/null @@ -1,25 +0,0 @@ -deps/ -ebin/ -_rel/ -.erlang.mk/ -*.d -*.o -*.exe -data/ -*.iml -.idea/ -logs/ -*.beam -emqx_coap.d -erlang.mk -integration_test/emqx-rel/ -integration_test/build_wakaama/ -integration_test/case*.txt -integration_test/paho/ -integration_test/wakaama/ -_build/ -rebar.lock -rebar3.crashdump -*.conf.rendered -.rebar3/ -*.swp diff --git a/apps/emqx_gateway/src/lwm2m/README.md b/apps/emqx_gateway/src/lwm2m/README.md deleted file mode 100644 index bf7626c6f..000000000 --- a/apps/emqx_gateway/src/lwm2m/README.md +++ /dev/null @@ -1,357 +0,0 @@ - -# LwM2M Gateway - -[The LwM2M Specifications](http://www.openmobilealliance.org/release/LightweightM2M) is a Lightweight Machine to Machine protocol. - -With `emqx_lwm2m`, user is able to send LwM2M commands(READ/WRITE/EXECUTE/...) and get LwM2M response in MQTT way. `emqx_lwm2m` transforms data between MQTT and LwM2M protocol. - -emqx_lwm2m needs object definitions to parse data from lwm2m devices. Object definitions are declared by organizations in XML format, you could find those XMLs from [LwM2MRegistry](http://www.openmobilealliance.org/wp/OMNA/LwM2M/LwM2MRegistry.html), download and put them into the directory specified by `lwm2m.xml_dir`. If no associated object definition is found, response from device will be discarded and report an error message in log. - -## Load emqx_lwm2m - -``` -./bin/emqx_ctl plugins load emqx_lwm2m -``` - -## Test emqx-lwm2m using *wakaama* - -[wakaama](https://github.com/eclipse/wakaama) is an easy-to-use lwm2m client command line tool. - -Start *lwm2mclient* using an endpoint name `ep1`: -``` -./lwm2mclient -n ep1 -h 127.0.0.1 -p 5683 -4 -``` - -To send an LwM2M DISCOVER command to *lwm2mclient*, publish an MQTT message to topic `lwm2m//dn` (where `` is the endpoint name of the client), with following payload: - -```json -{ - "reqID": "2", - "msgType": "discover", - "data": { - "path": "/3/0" - } -} -``` - -The MQTT message will be translated to an LwM2M DISCOVER command and sent to the *lwm2mclient*. Then the response of *lwm2mclient* will be in turn translated to an MQTT message, with topic `lwm2m//up/resp`, with following payload: - -```json -{ - "reqID": "2", - "msgType": "discover", - "data": { - "code":"2.05", - "codeMsg": "content", - "content": [ - ";dim=8", - "", - "", - "", - "" - ] - } -} -``` - -## LwM2M <--> MQTT Mapping - -### Register/Update (LwM2M Client Registration Interface) - -- **LwM2M Register and Update message will be converted to following MQTT message:** - - - **Method:** PUBLISH - - **Topic:** `lwm2m/{?EndpointName}/up/resp` (configurable) - - **Payload**: - - MsgType **register** and **update**: - ```json - { - "msgType": {?MsgType}, - "data": { - "ep": {?EndpointName}, - "lt": {?LifeTime}, - "sms": {?MSISDN}, - "lwm2m": {?Lwm2mVersion}, - "b": {?Binding}, - "alternatePath": {?AlternatePath}, - "objectList": {?ObjectList} - } - } - ``` - - {?EndpointName}: String, the endpoint name of the LwM2M client - - {?MsgType}: String, could be: - - "register": LwM2M Register - - "update": LwM2M Update - - "data" contains the query options and the object-list of the register message - - The *update* message is only published if the object-list changed. - -### Downlink Command and Uplink Response (LwM2M Device Management & Service Enablement Interface) - -- **To send a downlink command to device, publish following MQTT message:** - - **Method:** PUBLISH - - **Topic:** `lwm2m/{?EndpointName}/dn` - - **Request Payload**: - ```json - { - "reqID": {?ReqID}, - "msgType": {?MsgType}, - "data": {?Data} - } - ``` - - {?ReqID}: Integer, request-id, used for matching the response to the request - - {?MsgType}: String, can be one of the following: - - "read": LwM2M Read - - "discover": LwM2M Discover - - "write": LwM2M Write - - "write-attr": LwM2M Write Attributes - - "execute": LwM2M Execute - - "create": LwM2M Create - - "delete": LwM2M Delete - - {?Data}: JSON Object, its value depends on the {?MsgType}: - - **If {?MsgType} = "read" or "discover"**: - ```json - { - "path": {?ResourcePath} - } - ``` - - {?ResourcePath}: String, LwM2M full resource path. e.g. "3/0", "/3/0/0", "/3/0/6/0" - - **If {?MsgType} = "write" (single write)**: - ```json - { - "path": {?ResourcePath}, - "type": {?ValueType}, - "value": {?Value} - } - ``` - - {?ValueType}: String, can be: "Time", "String", "Integer", "Float", "Boolean", "Opaque", "Objlnk" - - {?Value}: Value of the resource, depends on "type". - - **If {?MsgType} = "write" (batch write)**: - ```json - { - "basePath": {?BasePath}, - "content": [ - { - "path": {?ResourcePath}, - "type": {?ValueType}, - "value": {?Value} - } - ] - } - ``` - - The full path is concatenation of "basePath" and "path". - - **If {?MsgType} = "write-attr"**: - ```json - { - "path": {?ResourcePath}, - "pmin": {?PeriodMin}, - "pmax": {?PeriodMax}, - "gt": {?GreaterThan}, - "lt": {?LessThan}, - "st": {?Step} - } - ``` - - {?PeriodMin}: Number, LwM2M Notification Class Attribute - Minimum Period. - - {?PeriodMax}: Number, LwM2M Notification Class Attribute - Maximum Period. - - {?GreaterThan}: Number, LwM2M Notification Class Attribute - Greater Than. - - {?LessThan}: Number, LwM2M Notification Class Attribute - Less Than. - - {?Step}: Number, LwM2M Notification Class Attribute - Step. - - - **If {?MsgType} = "execute"**: - ```json - { - "path": {?ResourcePath}, - "args": {?Arguments} - } - ``` - - {?Arguments}: String, LwM2M Execute Arguments. - - - **If {?MsgType} = "create"**: - ```json - { - "basePath": "/{?ObjectID}", - "content": [ - { - "path": {?ResourcePath}, - "type": {?ValueType}, - "value": {?Value} - } - ] - } - ``` - - {?ObjectID}: Integer, LwM2M Object ID - - - **If {?MsgType} = "delete"**: - ```json - { - "path": "{?ObjectID}/{?ObjectInstanceID}" - } - ``` - - {?ObjectInstanceID}: Integer, LwM2M Object Instance ID - -- **The response of LwM2M will be converted to following MQTT message:** - - **Method:** PUBLISH - - **Topic:** `"lwm2m/{?EndpointName}/up/resp"` - - **Response Payload:** - - ```json - { - "reqID": {?ReqID}, - "imei": {?IMEI}, - "imsi": {?IMSI}, - "msgType": {?MsgType}, - "data": {?Data} - } - ``` - - - {?MsgType}: String, can be: - - "read": LwM2M Read - - "discover": LwM2M Discover - - "write": LwM2M Write - - "write-attr": LwM2M Write Attributes - - "execute": LwM2M Execute - - "create": LwM2M Create - - "delete": LwM2M Delete - - **"ack"**: [CoAP Empty ACK](https://tools.ietf.org/html/rfc7252#section-5.2.2) - - {?Data}: JSON Object, its value depends on {?MsgType}: - - **If {?MsgType} = "write", "write-attr", "execute", "create", "delete", or "read"(when response without content)**: - ```json - { - "code": {?StatusCode}, - "codeMsg": {?CodeMsg}, - "reqPath": {?RequestPath} - } - ``` - - {?StatusCode}: String, LwM2M status code, e.g. "2.01", "4.00", etc. - - {?CodeMsg}: String, LwM2M response message, e.g. "content", "bad_request" - - {?RequestPath}: String, the requested "path" or "basePath" - - - **If {?MsgType} = "discover"**: - ```json - { - "code": {?StatusCode}, - "codeMsg": {?CodeMsg}, - "reqPath": {?RequestPath}, - "content": [ - {?Link}, - ... - ] - } - ``` - - {?Link}: String(LwM2M link format) e.g. `""`, `"<3/0/1>;dim=8"` - - - **If {?MsgType} = "read"(when response with content)**: - ```json - { - "code": {?StatusCode}, - "codeMsg": {?CodeMsg}, - "content": {?Content} - } - ``` - - {?Content} - ```json - [ - { - "path": {?ResourcePath}, - "value": {?Value} - } - ] - ``` - - - **If {?MsgType} = "ack", "data" does not exists** - -### Observe (Information Reporting Interface - Observe/Cancel-Observe) - -- **To observe/cancel-observe LwM2M client, send following MQTT PUBLISH:** - - **Method:** PUBLISH - - **Topic:** `lwm2m/{?EndpointName}/dn` - - **Request Payload**: - ```json - { - "reqID": {?ReqID}, - "msgType": {?MsgType}, - "data": { - "path": {?ResourcePath} - } - } - ``` - - {?ResourcePath}: String, the LwM2M resource to be observed/cancel-observed. - - {?MsgType}: String, can be: - - "observe": LwM2M Observe - - "cancel-observe": LwM2M Cancel Observe - - {?ReqID}: Integer, request-id, is the {?ReqID} in the request - -- **Responses will be converted to following MQTT message:** - - **Method:** PUBLISH - - **Topic:** `lwm2m/{?EndpointName}/up/resp` - - **Response Payload**: - ```json - { - "reqID": {?ReqID}, - "msgType": {?MsgType}, - "data": { - "code": {?StatusCode}, - "codeMsg": {?CodeMsg}, - "reqPath": {?RequestPath}, - "content": [ - { - "path": {?ResourcePath}, - "value": {?Value} - } - ] - } - } - ``` - - {?MsgType}: String, can be: - - "observe": LwM2M Observe - - "cancel-observe": LwM2M Cancel Observe - - **"ack"**: [CoAP Empty ACK](https://tools.ietf.org/html/rfc7252#section-5.2.2) - -### Notification (Information Reporting Interface - Notify) - -- **The notifications from LwM2M clients will be converted to MQTT PUBLISH:** - - **Method:** PUBLISH - - **Topic:** `lwm2m/{?EndpiontName}/up/notify` - - **Notification Payload**: - ```json - { - "reqID": {?ReqID}, - "msgType": {?MsgType}, - "seqNum": {?ObserveSeqNum}, - "data": { - "code": {?StatusCode}, - "codeMsg": {?CodeMsg}, - "reqPath": {?RequestPath}, - "content": [ - { - "path": {?ResourcePath}, - "value": {?Value} - } - ] - } - } - ``` - - {?MsgType}: String, must be "notify" - - {?ObserveSeqNum}: Number, value of "Observe" option in CoAP message - - "content": same to the "content" field contains in the response of "read" command - -## Feature limitations - -- emqx_lwm2m implements LwM2M gateway to EMQX, not a full-featured and independent LwM2M server. -- emqx_lwm2m does not include LwM2M bootstrap server. -- emqx_lwm2m supports UDP binding, no SMS binding yet. -- Firmware object is not fully supported now since mqtt to coap block-wise transfer is not available. -- Object Versioning is not supported now. - -## DTLS - -emqx-lwm2m support DTLS to secure UDP data. - -Please config lwm2m.certfile and lwm2m.keyfile in emqx_lwm2m.conf. If certfile or keyfile are invalid, DTLS will be turned off and you could read a error message in the log. - -## License - -Apache License Version 2.0 - -## Author - -EMQX-Men Team. diff --git a/apps/emqx_gateway/src/mqttsn/README.md b/apps/emqx_gateway/src/mqttsn/README.md deleted file mode 100644 index 67938b748..000000000 --- a/apps/emqx_gateway/src/mqttsn/README.md +++ /dev/null @@ -1,110 +0,0 @@ -# MQTT-SN Gateway - -EMQX MQTT-SN Gateway. - -## Configure Plugin - - -File: etc/emqx_sn.conf - -``` -## The UDP port which emq-sn is listening on. -## -## Value: IP:Port | Port -## -## Examples: 1884, 127.0.0.1:1884, ::1:1884 -mqtt.sn.port = 1884 - -## The duration(seconds) that emq-sn broadcast ADVERTISE message through. -## -## Value: Second -mqtt.sn.advertise_duration = 900 - -## The MQTT-SN Gateway id in ADVERTISE message. -## -## Value: Number -mqtt.sn.gateway_id = 1 - -## To control whether write statistics data into ETS table for dashboard to read. -## -## Value: on | off -mqtt.sn.enable_stats = off - -## To control whether accept and process the received publish message with qos=-1. -## -## Value: on | off -mqtt.sn.enable_qos3 = off - -## The pre-defined topic name corresponding to the pre-defined topic id of N. -## Note that the pre-defined topic id of 0 is reserved. -mqtt.sn.predefined.topic.0 = reserved -mqtt.sn.predefined.topic.1 = /predefined/topic/name/hello -mqtt.sn.predefined.topic.2 = /predefined/topic/name/nice - -## Default username for MQTT-SN. This parameter is optional. If specified, -## emq-sn will connect EMQ core with this username. It is useful if any auth -## plug-in is enabled. -## -## Value: String -mqtt.sn.username = mqtt_sn_user - -## This parameter is optional. Pair with username above. -## -## Value: String -mqtt.sn.password = abc -``` - -- mqtt.sn.port - * The UDP port which emqx-sn is listening on. -- mqtt.sn.advertise_duration - * The duration(seconds) that emqx-sn broadcast ADVERTISE message through. -- mqtt.sn.gateway_id - * Gateway id in ADVERTISE message. -- mqtt.sn.enable_stats - * To control whether write statistics data into ETS table for dashboard to read. -- mqtt.sn.enable_qos3 - * To control whether accept and process the received publish message with qos=-1. -- mqtt.sn.predefined.topic.N - * The pre-defined topic name corresponding to the pre-defined topic id of N. Note that the pre-defined topic id of 0 is reserved. -- mqtt.sn.username - * This parameter is optional. If specified, emqx-sn will connect EMQX core with this username. It is useful if any auth plug-in is enabled. -- mqtt.sn.password - * This parameter is optional. Pair with username above. - -## Load Plugin - -``` -./bin/emqx_ctl plugins load emqx_sn -``` - -## Client - -### NOTE -- Topic ID is per-client, and will be cleared if client disconnected with broker or keepalive failure is detected in broker. -- Please register your topics again each time connected with broker. -- If your udp socket(mqtt-sn client) has successfully connected to broker, don't try to send another CONNECT on this socket again, which will lead to confusing behaviour. If you want to start from beging, please do as following: - + destroy your present socket and create a new socket to connect again - + or send DISCONNECT on the same socket and connect again. - -### Library - -- https://github.com/eclipse/paho.mqtt-sn.embedded-c/ -- https://github.com/ty4tw/MQTT-SN -- https://github.com/njh/mqtt-sn-tools -- https://github.com/arobenko/mqtt-sn - -### sleeping device - -PINGREQ must have a ClientId which is identical to the one in CONNECT message. Without ClientId, emqx-sn will ignore such PINGREQ. - -### pre-defined topics - -The mapping of a pre-defined topic id and topic name should be known inadvance by both client's application and gateway. We define this mapping info in emqx_sn.conf file, and which shall be kept equivalent in all client's side. - -## License - -Apache License Version 2.0 - -## Author - -EMQX Team. diff --git a/apps/emqx_gateway/src/stomp/README.md b/apps/emqx_gateway/src/stomp/README.md deleted file mode 100644 index d96999aec..000000000 --- a/apps/emqx_gateway/src/stomp/README.md +++ /dev/null @@ -1,73 +0,0 @@ - -# emqx-stomp - - -The plugin adds STOMP 1.0/1.1/1.2 protocol supports to the EMQX broker. - -The STOMP clients could PubSub to the MQTT clients. - -## Configuration - -etc/emqx_stomp.conf - -``` -## The Port that stomp listener will bind. -## -## Value: Port -stomp.listener = 61613 - -## The acceptor pool for stomp listener. -## -## Value: Number -stomp.listener.acceptors = 4 - -## Maximum number of concurrent stomp connections. -## -## Value: Number -stomp.listener.max_connections = 512 - -## Default login user -## -## Value: String -stomp.default_user.login = guest - -## Default login password -## -## Value: String -stomp.default_user.passcode = guest - -## Allow anonymous authentication. -## -## Value: true | false -stomp.allow_anonymous = true - -## Maximum numbers of frame headers. -## -## Value: Number -stomp.frame.max_headers = 10 - -## Maximum length of frame header. -## -## Value: Number -stomp.frame.max_header_length = 1024 - -## Maximum body length of frame. -## -## Value: Number -stomp.frame.max_body_length = 8192 -``` - -## Load the Plugin - -``` -./bin/emqx_ctl plugins load emqx_stomp -``` - -## License - -Apache License Version 2.0 - -## Author - -EMQX Team. - diff --git a/apps/emqx_gateway/test/emqx_gateway_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_SUITE.erl index f611988a0..9f8c7911c 100644 --- a/apps/emqx_gateway/test/emqx_gateway_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_SUITE.erl @@ -33,6 +33,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Conf) -> emqx_config:erase(gateway), + emqx_gateway_test_utils:load_all_gateway_apps(), emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), emqx_common_test_helpers:start_apps([emqx_authn, emqx_gateway]), Conf. @@ -67,11 +68,11 @@ end_per_testcase(_TestCase, _Config) -> t_registered_gateway(_) -> [ - {coap, #{cbkmod := emqx_coap_impl}}, - {exproto, #{cbkmod := emqx_exproto_impl}}, - {lwm2m, #{cbkmod := emqx_lwm2m_impl}}, - {mqttsn, #{cbkmod := emqx_sn_impl}}, - {stomp, #{cbkmod := emqx_stomp_impl}} + {coap, #{cbkmod := emqx_gateway_coap}}, + {exproto, #{cbkmod := emqx_gateway_exproto}}, + {lwm2m, #{cbkmod := emqx_gateway_lwm2m}}, + {mqttsn, #{cbkmod := emqx_gateway_mqttsn}}, + {stomp, #{cbkmod := emqx_gateway_stomp}} ] = emqx_gateway:registered_gateway(). t_load_unload_list_lookup(_) -> @@ -187,7 +188,14 @@ read_lwm2m_conf(DataDir) -> Conf. setup_fake_usage_data(Lwm2mDataDir) -> - XmlDir = emqx_common_test_helpers:deps_path(emqx_gateway, "src/lwm2m/lwm2m_xml"), + XmlDir = filename:join( + [ + emqx_common_test_helpers:proj_root(), + "apps", + "emqx_gateway_lwm2m", + "lwm2m_xml" + ] + ), Lwm2mConf = read_lwm2m_conf(Lwm2mDataDir), ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, Lwm2mConf), emqx_config:put([gateway, lwm2m, xml_dir], XmlDir), diff --git a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl index 7aac45d61..fb648062a 100644 --- a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl @@ -46,6 +46,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Conf) -> application:load(emqx), + emqx_gateway_test_utils:load_all_gateway_apps(), emqx_config:delete_override_conf_files(), emqx_config:erase(gateway), emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), @@ -163,7 +164,7 @@ t_gateway_stomp(_) -> {204, _} = request(put, "/gateways/stomp", GwConf), {200, ConfResp} = request(get, "/gateways/stomp"), assert_confs(GwConf, ConfResp), - GwConf2 = emqx_map_lib:deep_merge(GwConf, #{frame => #{max_headers => 10}}), + GwConf2 = emqx_utils_maps:deep_merge(GwConf, #{frame => #{max_headers => 10}}), {204, _} = request(put, "/gateways/stomp", maps:without([name, listeners], GwConf2)), {200, ConfResp2} = request(get, "/gateways/stomp"), assert_confs(GwConf2, ConfResp2), @@ -185,7 +186,7 @@ t_gateway_mqttsn(_) -> {204, _} = request(put, "/gateways/mqttsn", GwConf), {200, ConfResp} = request(get, "/gateways/mqttsn"), assert_confs(GwConf, ConfResp), - GwConf2 = emqx_map_lib:deep_merge(GwConf, #{predefined => []}), + GwConf2 = emqx_utils_maps:deep_merge(GwConf, #{predefined => []}), {204, _} = request(put, "/gateways/mqttsn", maps:without([name, listeners], GwConf2)), {200, ConfResp2} = request(get, "/gateways/mqttsn"), assert_confs(GwConf2, ConfResp2), @@ -205,7 +206,7 @@ t_gateway_coap(_) -> {204, _} = request(put, "/gateways/coap", GwConf), {200, ConfResp} = request(get, "/gateways/coap"), assert_confs(GwConf, ConfResp), - GwConf2 = emqx_map_lib:deep_merge(GwConf, #{heartbeat => <<"10s">>}), + GwConf2 = emqx_utils_maps:deep_merge(GwConf, #{heartbeat => <<"10s">>}), {204, _} = request(put, "/gateways/coap", maps:without([name, listeners], GwConf2)), {200, ConfResp2} = request(get, "/gateways/coap"), assert_confs(GwConf2, ConfResp2), @@ -214,9 +215,17 @@ t_gateway_coap(_) -> t_gateway_lwm2m(_) -> {200, Gw} = request(get, "/gateways/lwm2m"), assert_gw_unloaded(Gw), + XmlDir = filename:join( + [ + emqx_common_test_helpers:proj_root(), + "apps", + "emqx_gateway_lwm2m", + "lwm2m_xml" + ] + ), GwConf = #{ name => <<"lwm2m">>, - xml_dir => <<"../../lib/emqx_gateway/src/lwm2m/lwm2m_xml">>, + xml_dir => list_to_binary(XmlDir), lifetime_min => <<"1s">>, lifetime_max => <<"1000s">>, qmode_time_window => <<"30s">>, @@ -235,7 +244,7 @@ t_gateway_lwm2m(_) -> {204, _} = request(put, "/gateways/lwm2m", GwConf), {200, ConfResp} = request(get, "/gateways/lwm2m"), assert_confs(GwConf, ConfResp), - GwConf2 = emqx_map_lib:deep_merge(GwConf, #{qmode_time_window => <<"10s">>}), + GwConf2 = emqx_utils_maps:deep_merge(GwConf, #{qmode_time_window => <<"10s">>}), {204, _} = request(put, "/gateways/lwm2m", maps:without([name, listeners], GwConf2)), {200, ConfResp2} = request(get, "/gateways/lwm2m"), assert_confs(GwConf2, ConfResp2), @@ -255,7 +264,7 @@ t_gateway_exproto(_) -> {204, _} = request(put, "/gateways/exproto", GwConf), {200, ConfResp} = request(get, "/gateways/exproto"), assert_confs(GwConf, ConfResp), - GwConf2 = emqx_map_lib:deep_merge(GwConf, #{server => #{bind => <<"9200">>}}), + GwConf2 = emqx_utils_maps:deep_merge(GwConf, #{server => #{bind => <<"9200">>}}), {204, _} = request(put, "/gateways/exproto", maps:without([name, listeners], GwConf2)), {200, ConfResp2} = request(get, "/gateways/exproto"), assert_confs(GwConf2, ConfResp2), @@ -284,7 +293,7 @@ t_gateway_exproto_with_ssl(_) -> {204, _} = request(put, "/gateways/exproto", GwConf), {200, ConfResp} = request(get, "/gateways/exproto"), assert_confs(GwConf, ConfResp), - GwConf2 = emqx_map_lib:deep_merge(GwConf, #{ + GwConf2 = emqx_utils_maps:deep_merge(GwConf, #{ server => #{ bind => <<"9200">>, ssl_options => SslCliOpts diff --git a/apps/emqx_gateway/test/emqx_gateway_auth_ct.erl b/apps/emqx_gateway/test/emqx_gateway_auth_ct.erl index d75bf80eb..0ed66a38d 100644 --- a/apps/emqx_gateway/test/emqx_gateway_auth_ct.erl +++ b/apps/emqx_gateway/test/emqx_gateway_auth_ct.erl @@ -153,7 +153,7 @@ on_start_auth(authn_http) -> Handler = fun(Req0, State) -> ct:pal("Authn Req:~p~nState:~p~n", [Req0, State]), Headers = #{<<"content-type">> => <<"application/json">>}, - Response = jiffy:encode(#{result => allow, is_superuser => false}), + Response = emqx_utils_json:encode(#{result => allow, is_superuser => false}), case cowboy_req:match_qs([username, password], Req0) of #{ username := <<"admin">>, diff --git a/apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl index 2427a10ee..149e6acd6 100644 --- a/apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl @@ -66,6 +66,7 @@ end_per_group(AuthName, Conf) -> Conf. init_per_suite(Config) -> + emqx_gateway_test_utils:load_all_gateway_apps(), emqx_config:erase(gateway), init_gateway_conf(), emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_authn, emqx_gateway]), @@ -265,7 +266,7 @@ t_case_exproto(_) -> Mod:send(Sock, ConnBin), {ok, Recv} = Mod:recv(Sock, 5000), - C = ?FUNCTOR(Bin, emqx_json:decode(Bin, [return_maps])), + C = ?FUNCTOR(Bin, emqx_utils_json:decode(Bin, [return_maps])), ?assertEqual(C(Expect), C(Recv)) end ) @@ -281,7 +282,7 @@ t_case_exproto(_) -> disable_authn(GwName, Type, Name) -> RawCfg = emqx_conf:get_raw([gateway, GwName], #{}), - ListenerCfg = emqx_map_lib:deep_get( + ListenerCfg = emqx_utils_maps:deep_get( [<<"listeners">>, atom_to_binary(Type), atom_to_binary(Name)], RawCfg ), {ok, _} = emqx_gateway_conf:update_listener(GwName, {Type, Name}, ListenerCfg#{ diff --git a/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl index 2e44415aa..c62e840df 100644 --- a/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl @@ -66,6 +66,7 @@ end_per_group(AuthName, Conf) -> init_per_suite(Config) -> emqx_config:erase(gateway), + emqx_gateway_test_utils:load_all_gateway_apps(), init_gateway_conf(), meck:new(emqx_authz_file, [non_strict, passthrough, no_history, no_link]), meck:expect(emqx_authz_file, create, fun(S) -> S end), @@ -164,7 +165,7 @@ t_case_lwm2m(_) -> Test("lwm2m", fun(SubTopic, Msg) -> ?assertEqual(true, lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics())), Payload = emqx_message:payload(Msg), - Cmd = emqx_json:decode(Payload, [return_maps]), + Cmd = emqx_utils_json:decode(Payload, [return_maps]), ?assertMatch(#{<<"msgType">> := <<"register">>, <<"data">> := _}, Cmd) end), @@ -225,7 +226,7 @@ t_case_sn_subscribe(_) -> ) end, Sub(<<"/subscribe">>, fun(Data) -> - {ok, Msg, _, _} = emqx_sn_frame:parse(Data, undefined), + {ok, Msg, _, _} = emqx_mqttsn_frame:parse(Data, undefined), ?assertMatch({mqtt_sn_message, _, {_, 3, 0, Payload}}, Msg) end), Sub(<<"/badsubscribe">>, fun(Data) -> @@ -349,7 +350,7 @@ t_case_exproto_publish(_) -> Mod:send(Sock, ConnBin), {ok, Recv} = Mod:recv(Sock, 5000), - C = ?FUNCTOR(Bin, emqx_json:decode(Bin, [return_maps])), + C = ?FUNCTOR(Bin, emqx_utils_json:decode(Bin, [return_maps])), ?assertEqual(C(SvrMod:frame_connack(0)), C(Recv)), Send = fun() -> @@ -386,7 +387,7 @@ t_case_exproto_subscribe(_) -> Mod:send(Sock, ConnBin), {ok, Recv} = Mod:recv(Sock, WaitTime), - C = ?FUNCTOR(Bin, emqx_json:decode(Bin, [return_maps])), + C = ?FUNCTOR(Bin, emqx_utils_json:decode(Bin, [return_maps])), ?assertEqual(C(SvrMod:frame_connack(0)), C(Recv)), SubBin = SvrMod:frame_subscribe(Topic, 0), diff --git a/apps/emqx_gateway/test/emqx_gateway_cli_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_cli_SUITE.erl index c66785e00..641528eda 100644 --- a/apps/emqx_gateway/test/emqx_gateway_cli_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_cli_SUITE.erl @@ -62,6 +62,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Conf) -> emqx_config:erase(gateway), + emqx_gateway_test_utils:load_all_gateway_apps(), emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_authn, emqx_gateway]), Conf. @@ -116,11 +117,11 @@ t_gateway_registry_usage(_) -> t_gateway_registry_list(_) -> emqx_gateway_cli:'gateway-registry'(["list"]), ?assertEqual( - "Registered Name: coap, Callback Module: emqx_coap_impl\n" - "Registered Name: exproto, Callback Module: emqx_exproto_impl\n" - "Registered Name: lwm2m, Callback Module: emqx_lwm2m_impl\n" - "Registered Name: mqttsn, Callback Module: emqx_sn_impl\n" - "Registered Name: stomp, Callback Module: emqx_stomp_impl\n", + "Registered Name: coap, Callback Module: emqx_gateway_coap\n" + "Registered Name: exproto, Callback Module: emqx_gateway_exproto\n" + "Registered Name: lwm2m, Callback Module: emqx_gateway_lwm2m\n" + "Registered Name: mqttsn, Callback Module: emqx_gateway_mqttsn\n" + "Registered Name: stomp, Callback Module: emqx_gateway_stomp\n", acc_print() ). diff --git a/apps/emqx_gateway/test/emqx_gateway_cm_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_cm_SUITE.erl index c5e8d9a92..8b0dacd75 100644 --- a/apps/emqx_gateway/test/emqx_gateway_cm_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_cm_SUITE.erl @@ -34,6 +34,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Conf) -> emqx_config:erase(gateway), + emqx_gateway_test_utils:load_all_gateway_apps(), emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), emqx_common_test_helpers:start_apps([]), diff --git a/apps/emqx_gateway/test/emqx_gateway_cm_registry_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_cm_registry_SUITE.erl index 77f4058e7..35e32d3da 100644 --- a/apps/emqx_gateway/test/emqx_gateway_cm_registry_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_cm_registry_SUITE.erl @@ -34,6 +34,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Conf) -> emqx_config:erase(gateway), + emqx_gateway_test_utils:load_all_gateway_apps(), emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), emqx_common_test_helpers:start_apps([]), Conf. diff --git a/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl index 6f6c2c45a..1e947e793 100644 --- a/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl @@ -37,6 +37,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Conf) -> + emqx_gateway_test_utils:load_all_gateway_apps(), emqx_common_test_helpers:load_config(emqx_gateway_schema, <<"gateway {}">>), emqx_common_test_helpers:start_apps([emqx_conf, emqx_authn, emqx_gateway]), Conf. @@ -412,7 +413,7 @@ t_load_gateway_with_certs_content(_) -> ), {ok, _} = emqx_gateway_conf:load_gateway(<<"stomp">>, StompConf), assert_confs(StompConf, emqx:get_raw_config([gateway, stomp])), - SslConf = emqx_map_lib:deep_get( + SslConf = emqx_utils_maps:deep_get( [<<"listeners">>, <<"ssl">>, <<"default">>, <<"ssl_options">>], emqx:get_raw_config([gateway, stomp]) ), @@ -435,7 +436,7 @@ t_load_gateway_with_certs_content(_) -> % ), % {ok, _} = emqx_gateway_conf:load_gateway(<<"stomp">>, StompConf), % assert_confs(StompConf, emqx:get_raw_config([gateway, stomp])), -% SslConf = emqx_map_lib:deep_get( +% SslConf = emqx_utils_maps:deep_get( % [<<"listeners">>, <<"ssl">>, <<"default">>, <<"ssl_options">>], % emqx:get_raw_config([gateway, stomp]) % ), @@ -470,7 +471,7 @@ t_add_listener_with_certs_content(_) -> emqx:get_raw_config([gateway, stomp]) ), - SslConf = emqx_map_lib:deep_get( + SslConf = emqx_utils_maps:deep_get( [<<"listeners">>, <<"ssl">>, <<"default">>, <<"ssl_options">>], emqx:get_raw_config([gateway, stomp]) ), diff --git a/apps/emqx_gateway/test/emqx_gateway_ctx_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_ctx_SUITE.erl index 0aa3172f1..35ce5fb31 100644 --- a/apps/emqx_gateway/test/emqx_gateway_ctx_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_ctx_SUITE.erl @@ -28,6 +28,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Conf) -> + emqx_gateway_test_utils:load_all_gateway_apps(), ok = meck:new(emqx_access_control, [passthrough, no_history, no_link]), ok = meck:expect( emqx_access_control, diff --git a/apps/emqx_gateway/test/emqx_gateway_metrics_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_metrics_SUITE.erl index 211315e6c..b82e049d3 100644 --- a/apps/emqx_gateway/test/emqx_gateway_metrics_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_metrics_SUITE.erl @@ -33,6 +33,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Conf) -> emqx_config:erase(gateway), + emqx_gateway_test_utils:load_all_gateway_apps(), emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), emqx_common_test_helpers:start_apps([]), Conf. diff --git a/apps/emqx_gateway/test/emqx_gateway_registry_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_registry_SUITE.erl index cc5f7bf37..a51621688 100644 --- a/apps/emqx_gateway/test/emqx_gateway_registry_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_registry_SUITE.erl @@ -37,6 +37,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). %%-------------------------------------------------------------------- init_per_suite(Cfg) -> + emqx_gateway_test_utils:load_all_gateway_apps(), ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), emqx_common_test_helpers:start_apps([emqx_authn, emqx_gateway]), Cfg. diff --git a/apps/emqx_gateway/test/emqx_gateway_test_utils.erl b/apps/emqx_gateway/test/emqx_gateway_test_utils.erl index deb602bc7..bb378ef10 100644 --- a/apps/emqx_gateway/test/emqx_gateway_test_utils.erl +++ b/apps/emqx_gateway/test/emqx_gateway_test_utils.erl @@ -101,6 +101,12 @@ assert_fields_exist(Ks, Map) -> end, Ks ). +load_all_gateway_apps() -> + application:load(emqx_gateway_stomp), + application:load(emqx_gateway_mqttsn), + application:load(emqx_gateway_coap), + application:load(emqx_gateway_lwm2m), + application:load(emqx_gateway_exproto). %%-------------------------------------------------------------------- %% http @@ -153,8 +159,8 @@ do_request(Mth, Req) -> <<>> -> #{}; _ -> - emqx_map_lib:unsafe_atom_key_map( - emqx_json:decode(Resp, [return_maps]) + emqx_utils_maps:unsafe_atom_key_map( + emqx_utils_json:decode(Resp, [return_maps]) ) end, {Code, NResp}; @@ -166,7 +172,7 @@ req(Path, Qs) -> {url(Path, Qs), auth([])}. req(Path, Qs, Body) -> - {url(Path, Qs), auth([]), "application/json", emqx_json:encode(Body)}. + {url(Path, Qs), auth([]), "application/json", emqx_utils_json:encode(Body)}. url(Path, []) -> lists:concat([?http_api_host, Path]); diff --git a/apps/emqx_gateway_coap/.gitignore b/apps/emqx_gateway_coap/.gitignore new file mode 100644 index 000000000..f1c455451 --- /dev/null +++ b/apps/emqx_gateway_coap/.gitignore @@ -0,0 +1,19 @@ +.rebar3 +_* +.eunit +*.o +*.beam +*.plt +*.swp +*.swo +.erlang.cookie +ebin +log +erl_crash.dump +.rebar +logs +_build +.idea +*.iml +rebar3.crashdump +*~ diff --git a/apps/emqx_gateway_coap/README.md b/apps/emqx_gateway_coap/README.md new file mode 100644 index 000000000..405366e89 --- /dev/null +++ b/apps/emqx_gateway_coap/README.md @@ -0,0 +1,31 @@ +# emqx_coap + +The CoAP gateway implements publish, subscribe, and receive messages as standard +with [Publish-Subscribe Broker for the CoAP](https://datatracker.ietf.org/doc/html/draft-ietf-core-coap-pubsub-09). + +## Quick Start + +In EMQX 5.0, CoAP gateways can be configured and enabled through the Dashboard. + +It can also be enabled via the HTTP API or emqx.conf, e.g. In emqx.conf: + +```properties +gateway.coap { + + mountpoint = "coap/" + + connection_required = false + + listeners.udp.default { + bind = "5683" + max_connections = 1024000 + max_conn_rate = 1000 + } +} +``` + +> Note: +> Configuring the gateway via emqx.conf requires changes on a per-node basis, +> but configuring it via Dashboard or the HTTP API will take effect across the cluster. + +More documentations: [CoAP Gateway](https://www.emqx.io/docs/en/v5.0/gateway/coap.html) diff --git a/apps/emqx_gateway/src/coap/doc/flow.png b/apps/emqx_gateway_coap/doc/flow.png similarity index 100% rename from apps/emqx_gateway/src/coap/doc/flow.png rename to apps/emqx_gateway_coap/doc/flow.png diff --git a/apps/emqx_gateway/src/coap/doc/shared_state.png b/apps/emqx_gateway_coap/doc/shared_state.png similarity index 100% rename from apps/emqx_gateway/src/coap/doc/shared_state.png rename to apps/emqx_gateway_coap/doc/shared_state.png diff --git a/apps/emqx_gateway/src/coap/doc/transport.png b/apps/emqx_gateway_coap/doc/transport.png similarity index 100% rename from apps/emqx_gateway/src/coap/doc/transport.png rename to apps/emqx_gateway_coap/doc/transport.png diff --git a/apps/emqx_gateway/src/coap/include/emqx_coap.hrl b/apps/emqx_gateway_coap/include/emqx_coap.hrl similarity index 100% rename from apps/emqx_gateway/src/coap/include/emqx_coap.hrl rename to apps/emqx_gateway_coap/include/emqx_coap.hrl diff --git a/apps/emqx_gateway_coap/rebar.config b/apps/emqx_gateway_coap/rebar.config new file mode 100644 index 000000000..3b070a72a --- /dev/null +++ b/apps/emqx_gateway_coap/rebar.config @@ -0,0 +1,6 @@ +{erl_opts, [debug_info]}. +{deps, [ + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}}, + {emqx_gateway, {path, "../emqx_gateway"}} +]}. diff --git a/apps/emqx_gateway/src/coap/emqx_coap_api.erl b/apps/emqx_gateway_coap/src/emqx_coap_api.erl similarity index 98% rename from apps/emqx_gateway/src/coap/emqx_coap_api.erl rename to apps/emqx_gateway_coap/src/emqx_coap_api.erl index 0f4c7a053..b4fce5473 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_api.erl +++ b/apps/emqx_gateway_coap/src/emqx_coap_api.erl @@ -18,10 +18,10 @@ -behaviour(minirest_api). +-include("emqx_coap.hrl"). -include_lib("hocon/include/hoconsc.hrl"). -include_lib("typerefl/include/types.hrl"). -include_lib("emqx/include/logger.hrl"). --include("src/coap/include/emqx_coap.hrl"). %% API -export([api_spec/0, paths/0, schema/1, namespace/0]). @@ -34,9 +34,12 @@ -import(hoconsc, [mk/2, enum/1]). -import(emqx_dashboard_swagger, [error_codes/2]). +-elvis([{elvis_style, atom_naming_convention, disable}]). + %%-------------------------------------------------------------------- %% API %%-------------------------------------------------------------------- + namespace() -> "gateway_coap". api_spec() -> diff --git a/apps/emqx_gateway/src/coap/emqx_coap_channel.erl b/apps/emqx_gateway_coap/src/emqx_coap_channel.erl similarity index 99% rename from apps/emqx_gateway/src/coap/emqx_coap_channel.erl rename to apps/emqx_gateway_coap/src/emqx_coap_channel.erl index d6b8594b1..b90fd630d 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_channel.erl +++ b/apps/emqx_gateway_coap/src/emqx_coap_channel.erl @@ -45,8 +45,8 @@ -export_type([channel/0]). +-include("emqx_coap.hrl"). -include_lib("emqx/include/logger.hrl"). --include("src/coap/include/emqx_coap.hrl"). -include_lib("emqx/include/emqx_authentication.hrl"). -define(AUTHN, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM). @@ -111,7 +111,7 @@ info(conn_state, #channel{conn_state = ConnState}) -> info(clientinfo, #channel{clientinfo = ClientInfo}) -> ClientInfo; info(session, #channel{session = Session}) -> - emqx_misc:maybe_apply(fun emqx_coap_session:info/1, Session); + emqx_utils:maybe_apply(fun emqx_coap_session:info/1, Session); info(clientid, #channel{clientinfo = #{clientid := ClientId}}) -> ClientId; info(ctx, #channel{ctx = Ctx}) -> @@ -366,7 +366,7 @@ ensure_timer(Name, Time, Msg, #channel{timers = Timers} = Channel) -> end. make_timer(Name, Time, Msg, Channel = #channel{timers = Timers}) -> - TRef = emqx_misc:start_timer(Time, Msg), + TRef = emqx_utils:start_timer(Time, Msg), Channel#channel{timers = Timers#{Name => TRef}}. ensure_keepalive_timer(Channel) -> @@ -710,7 +710,7 @@ process_connection( ) -> Queries = emqx_coap_message:get_option(uri_query, Req), case - emqx_misc:pipeline( + emqx_utils:pipeline( [ fun enrich_conninfo/2, fun run_conn_hooks/2, diff --git a/apps/emqx_gateway/src/coap/emqx_coap_frame.erl b/apps/emqx_gateway_coap/src/emqx_coap_frame.erl similarity index 99% rename from apps/emqx_gateway/src/coap/emqx_coap_frame.erl rename to apps/emqx_gateway_coap/src/emqx_coap_frame.erl index 4d2479d75..535d07a94 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_frame.erl +++ b/apps/emqx_gateway_coap/src/emqx_coap_frame.erl @@ -29,7 +29,7 @@ is_message/1 ]). --include("src/coap/include/emqx_coap.hrl"). +-include("emqx_coap.hrl"). -include_lib("emqx/include/types.hrl"). -define(VERSION, 1). @@ -55,6 +55,8 @@ -define(OPTION_PROXY_SCHEME, 39). -define(OPTION_SIZE1, 60). +-elvis([{elvis_style, no_if_expression, disable}]). + %%-------------------------------------------------------------------- %% API %%-------------------------------------------------------------------- diff --git a/apps/emqx_gateway/src/coap/emqx_coap_medium.erl b/apps/emqx_gateway_coap/src/emqx_coap_medium.erl similarity index 98% rename from apps/emqx_gateway/src/coap/emqx_coap_medium.erl rename to apps/emqx_gateway_coap/src/emqx_coap_medium.erl index 8f5028f25..b6bd8e764 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_medium.erl +++ b/apps/emqx_gateway_coap/src/emqx_coap_medium.erl @@ -20,7 +20,7 @@ -module(emqx_coap_medium). --include("src/coap/include/emqx_coap.hrl"). +-include("emqx_coap.hrl"). %% API -export([ diff --git a/apps/emqx_gateway/src/coap/emqx_coap_message.erl b/apps/emqx_gateway_coap/src/emqx_coap_message.erl similarity index 99% rename from apps/emqx_gateway/src/coap/emqx_coap_message.erl rename to apps/emqx_gateway_coap/src/emqx_coap_message.erl index 99c9e0840..ee17231a7 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_message.erl +++ b/apps/emqx_gateway_coap/src/emqx_coap_message.erl @@ -43,7 +43,7 @@ set_payload_block/3, set_payload_block/4 ]). --include("src/coap/include/emqx_coap.hrl"). +-include("emqx_coap.hrl"). request(Type, Method) -> request(Type, Method, <<>>, []). diff --git a/apps/emqx_gateway/src/coap/handler/emqx_coap_mqtt_handler.erl b/apps/emqx_gateway_coap/src/emqx_coap_mqtt_handler.erl similarity index 96% rename from apps/emqx_gateway/src/coap/handler/emqx_coap_mqtt_handler.erl rename to apps/emqx_gateway_coap/src/emqx_coap_mqtt_handler.erl index 59825a745..4bcf71b1a 100644 --- a/apps/emqx_gateway/src/coap/handler/emqx_coap_mqtt_handler.erl +++ b/apps/emqx_gateway_coap/src/emqx_coap_mqtt_handler.erl @@ -16,7 +16,7 @@ -module(emqx_coap_mqtt_handler). --include("src/coap/include/emqx_coap.hrl"). +-include("emqx_coap.hrl"). -export([handle_request/4]). -import(emqx_coap_message, [response/2, response/3]). diff --git a/apps/emqx_gateway/src/coap/emqx_coap_observe_res.erl b/apps/emqx_gateway_coap/src/emqx_coap_observe_res.erl similarity index 100% rename from apps/emqx_gateway/src/coap/emqx_coap_observe_res.erl rename to apps/emqx_gateway_coap/src/emqx_coap_observe_res.erl diff --git a/apps/emqx_gateway/src/coap/handler/emqx_coap_pubsub_handler.erl b/apps/emqx_gateway_coap/src/emqx_coap_pubsub_handler.erl similarity index 99% rename from apps/emqx_gateway/src/coap/handler/emqx_coap_pubsub_handler.erl rename to apps/emqx_gateway_coap/src/emqx_coap_pubsub_handler.erl index 5e14ba9e4..da1f5e0ef 100644 --- a/apps/emqx_gateway/src/coap/handler/emqx_coap_pubsub_handler.erl +++ b/apps/emqx_gateway_coap/src/emqx_coap_pubsub_handler.erl @@ -18,7 +18,7 @@ -module(emqx_coap_pubsub_handler). -include_lib("emqx/include/emqx_mqtt.hrl"). --include("src/coap/include/emqx_coap.hrl"). +-include("emqx_coap.hrl"). -export([handle_request/4]). diff --git a/apps/emqx_gateway_coap/src/emqx_coap_schema.erl b/apps/emqx_gateway_coap/src/emqx_coap_schema.erl new file mode 100644 index 000000000..b7ce88451 --- /dev/null +++ b/apps/emqx_gateway_coap/src/emqx_coap_schema.erl @@ -0,0 +1,95 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_coap_schema). + +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("typerefl/include/types.hrl"). + +-type duration() :: non_neg_integer(). + +-typerefl_from_string({duration/0, emqx_schema, to_duration}). + +-reflect_type([duration/0]). + +%% config schema provides +-export([fields/1, desc/1]). + +fields(coap) -> + [ + {heartbeat, + sc( + duration(), + #{ + default => <<"30s">>, + desc => ?DESC(coap_heartbeat) + } + )}, + {connection_required, + sc( + boolean(), + #{ + default => false, + desc => ?DESC(coap_connection_required) + } + )}, + {notify_type, + sc( + hoconsc:enum([non, con, qos]), + #{ + default => qos, + desc => ?DESC(coap_notify_type) + } + )}, + {subscribe_qos, + sc( + hoconsc:enum([qos0, qos1, qos2, coap]), + #{ + default => coap, + desc => ?DESC(coap_subscribe_qos) + } + )}, + {publish_qos, + sc( + hoconsc:enum([qos0, qos1, qos2, coap]), + #{ + default => coap, + desc => ?DESC(coap_publish_qos) + } + )}, + {mountpoint, emqx_gateway_schema:mountpoint()}, + {listeners, + sc( + ref(emqx_gateway_schema, udp_listeners), + #{desc => ?DESC(udp_listeners)} + )} + ] ++ emqx_gateway_schema:gateway_common_options(). + +desc(coap) -> + "The CoAP protocol gateway provides EMQX with the access capability of the CoAP protocol.\n" + "It allows publishing, subscribing, and receiving messages to EMQX in accordance\n" + "with a certain defined CoAP message format."; +desc(_) -> + undefined. + +%%-------------------------------------------------------------------- +%% helpers + +sc(Type, Meta) -> + hoconsc:mk(Type, Meta). + +ref(Mod, Field) -> + hoconsc:ref(Mod, Field). diff --git a/apps/emqx_gateway/src/coap/emqx_coap_session.erl b/apps/emqx_gateway_coap/src/emqx_coap_session.erl similarity index 99% rename from apps/emqx_gateway/src/coap/emqx_coap_session.erl rename to apps/emqx_gateway_coap/src/emqx_coap_session.erl index 253f34d4d..5ae169675 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_session.erl +++ b/apps/emqx_gateway_coap/src/emqx_coap_session.erl @@ -15,10 +15,10 @@ %%-------------------------------------------------------------------- -module(emqx_coap_session). +-include("emqx_coap.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/logger.hrl"). --include("src/coap/include/emqx_coap.hrl"). %% API -export([ @@ -81,7 +81,7 @@ %%%------------------------------------------------------------------- -spec new() -> session(). new() -> - _ = emqx_misc:rand_seed(), + _ = emqx_utils:rand_seed(), #session{ transport_manager = emqx_coap_tm:new(), observe_manager = emqx_coap_observe_res:new_manager(), diff --git a/apps/emqx_gateway/src/coap/emqx_coap_tm.erl b/apps/emqx_gateway_coap/src/emqx_coap_tm.erl similarity index 97% rename from apps/emqx_gateway/src/coap/emqx_coap_tm.erl rename to apps/emqx_gateway_coap/src/emqx_coap_tm.erl index 1a0004f8c..68a7ae237 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_tm.erl +++ b/apps/emqx_gateway_coap/src/emqx_coap_tm.erl @@ -29,8 +29,8 @@ -export_type([manager/0, event_result/1]). +-include("emqx_coap.hrl"). -include_lib("emqx/include/logger.hrl"). --include("src/coap/include/emqx_coap.hrl"). -type direction() :: in | out. @@ -80,6 +80,8 @@ -import(emqx_coap_medium, [empty/0, iter/4, reset/1, proto_out/2]). +-elvis([{elvis_style, no_if_expression, disable}]). + %%-------------------------------------------------------------------- %% API %%-------------------------------------------------------------------- @@ -270,12 +272,12 @@ cancel_state_timer(#state_machine{timers = Timers} = Machine) -> undefined -> Machine; Ref -> - _ = emqx_misc:cancel_timer(Ref), + _ = emqx_utils:cancel_timer(Ref), Machine#state_machine{timers = maps:remove(state_timer, Timers)} end. process_timer(SeqId, {Type, Interval, Msg}, Timers) -> - Ref = emqx_misc:start_timer(Interval, {state_machine, {SeqId, Type, Msg}}), + Ref = emqx_utils:start_timer(Interval, {state_machine, {SeqId, Type, Msg}}), Timers#{Type => Ref}. -spec delete_machine(manager_key(), manager()) -> manager(). @@ -291,7 +293,7 @@ delete_machine(Id, Manager) -> } -> lists:foreach( fun({_, Ref}) -> - emqx_misc:cancel_timer(Ref) + emqx_utils:cancel_timer(Ref) end, maps:to_list(Timers) ), @@ -401,9 +403,9 @@ alloc_message_id(MsgId, TM) -> next_message_id(MsgId) -> Next = MsgId + 1, - if - Next >= ?MAX_MESSAGE_ID -> - 1; + case Next >= ?MAX_MESSAGE_ID of true -> + 1; + false -> Next end. diff --git a/apps/emqx_gateway/src/coap/emqx_coap_transport.erl b/apps/emqx_gateway_coap/src/emqx_coap_transport.erl similarity index 96% rename from apps/emqx_gateway/src/coap/emqx_coap_transport.erl rename to apps/emqx_gateway_coap/src/emqx_coap_transport.erl index 1e6c5238a..daea13ba8 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_transport.erl +++ b/apps/emqx_gateway_coap/src/emqx_coap_transport.erl @@ -16,8 +16,8 @@ -module(emqx_coap_transport). +-include("emqx_coap.hrl"). -include_lib("emqx/include/logger.hrl"). --include("src/coap/include/emqx_coap.hrl"). -define(ACK_TIMEOUT, 2000). -define(ACK_RANDOM_FACTOR, 1000). @@ -60,6 +60,12 @@ reply/2 ]). +-elvis([{elvis_style, atom_naming_convention, disable}]). +-elvis([{elvis_style, no_if_expression, disable}]). + +%%-------------------------------------------------------------------- +%% APIs + -spec new() -> transport(). new() -> new(undefined). @@ -113,7 +119,7 @@ idle(out, #coap_message{type = non} = Msg, _) -> timeouts => [{stop_timeout, ?NON_LIFETIME}] }); idle(out, Msg, Transport) -> - _ = emqx_misc:rand_seed(), + _ = emqx_utils:rand_seed(), Timeout = ?ACK_TIMEOUT + rand:uniform(?ACK_RANDOM_FACTOR), out(Msg, #{ next => wait_ack, diff --git a/apps/emqx_gateway_coap/src/emqx_gateway_coap.app.src b/apps/emqx_gateway_coap/src/emqx_gateway_coap.app.src new file mode 100644 index 000000000..decd13bef --- /dev/null +++ b/apps/emqx_gateway_coap/src/emqx_gateway_coap.app.src @@ -0,0 +1,10 @@ +{application, emqx_gateway_coap, [ + {description, "CoAP Gateway"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [kernel, stdlib, emqx, emqx_gateway]}, + {env, []}, + {modules, []}, + {licenses, ["Apache 2.0"]}, + {links, []} +]}. diff --git a/apps/emqx_gateway/src/coap/emqx_coap_impl.erl b/apps/emqx_gateway_coap/src/emqx_gateway_coap.erl similarity index 86% rename from apps/emqx_gateway/src/coap/emqx_coap_impl.erl rename to apps/emqx_gateway_coap/src/emqx_gateway_coap.erl index bebcef237..6c495fbdb 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_impl.erl +++ b/apps/emqx_gateway_coap/src/emqx_gateway_coap.erl @@ -14,13 +14,29 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_coap_impl). - --behaviour(emqx_gateway_impl). +%% @doc The CoAP Gateway implement +-module(emqx_gateway_coap). -include_lib("emqx/include/logger.hrl"). -include_lib("emqx_gateway/include/emqx_gateway.hrl"). +%% define a gateway named stomp +-gateway(#{ + name => coap, + callback_module => ?MODULE, + config_schema_module => emqx_coap_schema +}). + +%% callback_module must implement the emqx_gateway_impl behaviour +-behaviour(emqx_gateway_impl). + +%% callback for emqx_gateway_impl +-export([ + on_gateway_load/2, + on_gateway_update/3, + on_gateway_unload/2 +]). + -import( emqx_gateway_utils, [ @@ -30,31 +46,8 @@ ] ). -%% APIs --export([ - reg/0, - unreg/0 -]). - --export([ - on_gateway_load/2, - on_gateway_update/3, - on_gateway_unload/2 -]). - %%-------------------------------------------------------------------- -%% APIs -%%-------------------------------------------------------------------- - -reg() -> - RegistryOptions = [{cbkmod, ?MODULE}], - emqx_gateway_registry:reg(coap, RegistryOptions). - -unreg() -> - emqx_gateway_registry:unreg(coap). - -%%-------------------------------------------------------------------- -%% emqx_gateway_registry callbacks +%% emqx_gateway_impl callbacks %%-------------------------------------------------------------------- on_gateway_load( diff --git a/apps/emqx_gateway/test/emqx_coap_SUITE.erl b/apps/emqx_gateway_coap/test/emqx_coap_SUITE.erl similarity index 99% rename from apps/emqx_gateway/test/emqx_coap_SUITE.erl rename to apps/emqx_gateway_coap/test/emqx_coap_SUITE.erl index db99c3df1..9b6f7ce1f 100644 --- a/apps/emqx_gateway/test/emqx_coap_SUITE.erl +++ b/apps/emqx_gateway_coap/test/emqx_coap_SUITE.erl @@ -56,6 +56,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> + application:load(emqx_gateway_coap), ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), emqx_mgmt_api_test_util:init_suite([emqx_authn, emqx_gateway]), ok = meck:new(emqx_access_control, [passthrough, no_history, no_link]), diff --git a/apps/emqx_gateway/test/emqx_coap_api_SUITE.erl b/apps/emqx_gateway_coap/test/emqx_coap_api_SUITE.erl similarity index 98% rename from apps/emqx_gateway/test/emqx_coap_api_SUITE.erl rename to apps/emqx_gateway_coap/test/emqx_coap_api_SUITE.erl index 6c1354bc0..cec09a016 100644 --- a/apps/emqx_gateway/test/emqx_coap_api_SUITE.erl +++ b/apps/emqx_gateway_coap/test/emqx_coap_api_SUITE.erl @@ -19,7 +19,7 @@ -compile(export_all). -compile(nowarn_export_all). --include("src/coap/include/emqx_coap.hrl"). +-include("emqx_coap.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). @@ -56,6 +56,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> + application:load(emqx_gateway_coap), ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), emqx_mgmt_api_test_util:init_suite([emqx_authn, emqx_gateway]), Config. @@ -91,7 +92,7 @@ t_send_request_api(_) -> Req ), #{<<"token">> := RToken, <<"payload">> := RPayload} = - emqx_json:decode(Response, [return_maps]), + emqx_utils_json:decode(Response, [return_maps]), ?assertEqual(Token, RToken), ?assertEqual(Payload, RPayload) end, diff --git a/apps/emqx_gateway_exproto/.gitignore b/apps/emqx_gateway_exproto/.gitignore new file mode 100644 index 000000000..922b0f989 --- /dev/null +++ b/apps/emqx_gateway_exproto/.gitignore @@ -0,0 +1,24 @@ +.rebar3 +_* +.eunit +*.o +*.beam +*.plt +*.swp +*.swo +.erlang.cookie +ebin +log +erl_crash.dump +.rebar +logs +_build +.idea +*.iml +rebar3.crashdump +*~ +src/emqx_exproto_pb.erl +src/emqx_exproto_v_1_connection_adapter_bhvr.erl +src/emqx_exproto_v_1_connection_adapter_client.erl +src/emqx_exproto_v_1_connection_handler_bhvr.erl +src/emqx_exproto_v_1_connection_handler_client.erl diff --git a/apps/emqx_gateway/src/exproto/README.md b/apps/emqx_gateway_exproto/README.md similarity index 100% rename from apps/emqx_gateway/src/exproto/README.md rename to apps/emqx_gateway_exproto/README.md diff --git a/apps/emqx_gateway/src/exproto/include/emqx_exproto.hrl b/apps/emqx_gateway_exproto/include/emqx_exproto.hrl similarity index 100% rename from apps/emqx_gateway/src/exproto/include/emqx_exproto.hrl rename to apps/emqx_gateway_exproto/include/emqx_exproto.hrl diff --git a/apps/emqx_gateway/src/exproto/protos/exproto.proto b/apps/emqx_gateway_exproto/priv/protos/exproto.proto similarity index 100% rename from apps/emqx_gateway/src/exproto/protos/exproto.proto rename to apps/emqx_gateway_exproto/priv/protos/exproto.proto diff --git a/apps/emqx_gateway_exproto/rebar.config b/apps/emqx_gateway_exproto/rebar.config new file mode 100644 index 000000000..473fa9b67 --- /dev/null +++ b/apps/emqx_gateway_exproto/rebar.config @@ -0,0 +1,36 @@ +{erl_opts, [debug_info]}. +{deps, [ + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}}, + {emqx_gateway, {path, "../emqx_gateway"}} +]}. + +{plugins, [ + {grpc_plugin, {git, "https://github.com/HJianBo/grpc_plugin", {tag, "v0.10.2"}}} +]}. + +{grpc, [ + {protos, ["priv/protos"]}, + {out_dir, "src"}, + {gpb_opts, [ + {module_name_prefix, "emqx_"}, + {module_name_suffix, "_pb"} + ]} +]}. + +{provider_hooks, [ + {pre, [ + {compile, {grpc, gen}}, + {clean, {grpc, clean}} + ]} +]}. + +{xref_ignores, [emqx_exproto_pb]}. + +{cover_excl_mods, [ + emqx_exproto_pb, + emqx_exproto_v_1_connection_adapter_client, + emqx_exproto_v_1_connection_adapter_bhvr, + emqx_exproto_v_1_connection_handler_client, + emqx_exproto_v_1_connection_handler_bhvr +]}. diff --git a/apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl b/apps/emqx_gateway_exproto/src/emqx_exproto_channel.erl similarity index 99% rename from apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl rename to apps/emqx_gateway_exproto/src/emqx_exproto_channel.erl index 301154df0..3b2c8d73b 100644 --- a/apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl +++ b/apps/emqx_gateway_exproto/src/emqx_exproto_channel.erl @@ -15,7 +15,8 @@ %%-------------------------------------------------------------------- -module(emqx_exproto_channel). --include("src/exproto/include/emqx_exproto.hrl"). + +-include("emqx_exproto.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/types.hrl"). @@ -680,14 +681,14 @@ ensure_timer(Name, Channel = #channel{timers = Timers}) -> ensure_timer(Name, Time, Channel = #channel{timers = Timers}) -> Msg = maps:get(Name, ?TIMER_TABLE), - TRef = emqx_misc:start_timer(Time, Msg), + TRef = emqx_utils:start_timer(Time, Msg), Channel#channel{timers = Timers#{Name => TRef}}. reset_timer(Name, Channel) -> ensure_timer(Name, remove_timer_ref(Name, Channel)). cancel_timer(Name, Channel = #channel{timers = Timers}) -> - emqx_misc:cancel_timer(maps:get(Name, Timers, undefined)), + emqx_utils:cancel_timer(maps:get(Name, Timers, undefined)), remove_timer_ref(Name, Channel). remove_timer_ref(Name, Channel = #channel{timers = Timers}) -> @@ -791,4 +792,4 @@ proto_name_to_protocol(ProtoName) when is_binary(ProtoName) -> binary_to_atom(ProtoName). anonymous_clientid() -> - iolist_to_binary(["exproto-", emqx_misc:gen_id()]). + iolist_to_binary(["exproto-", emqx_utils:gen_id()]). diff --git a/apps/emqx_gateway/src/exproto/emqx_exproto_frame.erl b/apps/emqx_gateway_exproto/src/emqx_exproto_frame.erl similarity index 100% rename from apps/emqx_gateway/src/exproto/emqx_exproto_frame.erl rename to apps/emqx_gateway_exproto/src/emqx_exproto_frame.erl diff --git a/apps/emqx_gateway/src/exproto/emqx_exproto_gcli.erl b/apps/emqx_gateway_exproto/src/emqx_exproto_gcli.erl similarity index 99% rename from apps/emqx_gateway/src/exproto/emqx_exproto_gcli.erl rename to apps/emqx_gateway_exproto/src/emqx_exproto_gcli.erl index af15ef9d3..34883cdce 100644 --- a/apps/emqx_gateway/src/exproto/emqx_exproto_gcli.erl +++ b/apps/emqx_gateway_exproto/src/emqx_exproto_gcli.erl @@ -50,7 +50,7 @@ start_link(Pool, Id) -> gen_server:start_link( - {local, emqx_misc:proc_name(?MODULE, Id)}, + {local, emqx_utils:proc_name(?MODULE, Id)}, ?MODULE, [Pool, Id], [] diff --git a/apps/emqx_gateway/src/exproto/emqx_exproto_gsvr.erl b/apps/emqx_gateway_exproto/src/emqx_exproto_gsvr.erl similarity index 99% rename from apps/emqx_gateway/src/exproto/emqx_exproto_gsvr.erl rename to apps/emqx_gateway_exproto/src/emqx_exproto_gsvr.erl index 13bd49e55..5bbe7bf37 100644 --- a/apps/emqx_gateway/src/exproto/emqx_exproto_gsvr.erl +++ b/apps/emqx_gateway_exproto/src/emqx_exproto_gsvr.erl @@ -19,7 +19,7 @@ % -behaviour(emqx_exproto_v_1_connection_adapter_bhvr). --include("src/exproto/include/emqx_exproto.hrl"). +-include("emqx_exproto.hrl"). -include_lib("emqx/include/logger.hrl"). -define(IS_QOS(X), (X =:= 0 orelse X =:= 1 orelse X =:= 2)). diff --git a/apps/emqx_gateway_exproto/src/emqx_exproto_schema.erl b/apps/emqx_gateway_exproto/src/emqx_exproto_schema.erl new file mode 100644 index 000000000..eb44c030b --- /dev/null +++ b/apps/emqx_gateway_exproto/src/emqx_exproto_schema.erl @@ -0,0 +1,117 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_exproto_schema). + +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("typerefl/include/types.hrl"). + +-type ip_port() :: tuple() | integer(). + +-typerefl_from_string({ip_port/0, emqx_schema, to_ip_port}). + +-reflect_type([ + ip_port/0 +]). + +%% config schema provides +-export([fields/1, desc/1]). + +fields(exproto) -> + [ + {server, + sc( + ref(exproto_grpc_server), + #{ + required => true, + desc => ?DESC(exproto_server) + } + )}, + {handler, + sc( + ref(exproto_grpc_handler), + #{ + required => true, + desc => ?DESC(exproto_handler) + } + )}, + {mountpoint, emqx_gateway_schema:mountpoint()}, + {listeners, + sc(ref(emqx_gateway_schema, tcp_udp_listeners), #{desc => ?DESC(tcp_udp_listeners)})} + ] ++ emqx_gateway_schema:gateway_common_options(); +fields(exproto_grpc_server) -> + [ + {bind, + sc( + hoconsc:union([ip_port(), integer()]), + #{ + required => true, + desc => ?DESC(exproto_grpc_server_bind) + } + )}, + {ssl_options, + sc( + ref(ssl_server_opts), + #{ + required => {false, recursively}, + desc => ?DESC(exproto_grpc_server_ssl) + } + )} + ]; +fields(exproto_grpc_handler) -> + [ + {address, sc(binary(), #{required => true, desc => ?DESC(exproto_grpc_handler_address)})}, + {ssl_options, + sc( + ref(emqx_schema, "ssl_client_opts"), + #{ + required => {false, recursively}, + desc => ?DESC(exproto_grpc_handler_ssl) + } + )} + ]; +fields(ssl_server_opts) -> + emqx_schema:server_ssl_opts_schema( + #{ + depth => 10, + reuse_sessions => true, + versions => tls_all_available + }, + true + ). + +desc(exproto) -> + "Settings for EMQX extension protocol (exproto)."; +desc(exproto_grpc_server) -> + "Settings for the exproto gRPC server."; +desc(exproto_grpc_handler) -> + "Settings for the exproto gRPC connection handler."; +desc(ssl_server_opts) -> + "SSL configuration for the server."; +desc(_) -> + undefined. + +%%-------------------------------------------------------------------- +%% helpers + +sc(Type, Meta) -> + hoconsc:mk(Type, Meta). + +ref(StructName) -> + ref(?MODULE, StructName). + +ref(Mod, Field) -> + hoconsc:ref(Mod, Field). diff --git a/apps/emqx_gateway_exproto/src/emqx_gateway_exproto.app.src b/apps/emqx_gateway_exproto/src/emqx_gateway_exproto.app.src new file mode 100644 index 000000000..09cf58338 --- /dev/null +++ b/apps/emqx_gateway_exproto/src/emqx_gateway_exproto.app.src @@ -0,0 +1,10 @@ +{application, emqx_gateway_exproto, [ + {description, "ExProto Gateway"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [kernel, stdlib, grpc, emqx, emqx_gateway]}, + {env, []}, + {modules, []}, + {licenses, ["Apache 2.0"]}, + {links, []} +]}. diff --git a/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl b/apps/emqx_gateway_exproto/src/emqx_gateway_exproto.erl similarity index 91% rename from apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl rename to apps/emqx_gateway_exproto/src/emqx_gateway_exproto.erl index 0c25e5e08..ff105b931 100644 --- a/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl +++ b/apps/emqx_gateway_exproto/src/emqx_gateway_exproto.erl @@ -14,12 +14,28 @@ %% limitations under the License. %%-------------------------------------------------------------------- -%% @doc The ExProto Gateway Implement interface --module(emqx_exproto_impl). - --behaviour(emqx_gateway_impl). +%% @doc The ExProto Gateway implement +-module(emqx_gateway_exproto). -include_lib("emqx/include/logger.hrl"). +-include_lib("emqx_gateway/include/emqx_gateway.hrl"). + +%% define a gateway named stomp +-gateway(#{ + name => exproto, + callback_module => ?MODULE, + config_schema_module => emqx_exproto_schema +}). + +%% callback_module must implement the emqx_gateway_impl behaviour +-behaviour(emqx_gateway_impl). + +%% callback for emqx_gateway_impl +-export([ + on_gateway_load/2, + on_gateway_update/3, + on_gateway_unload/2 +]). -import( emqx_gateway_utils, @@ -30,31 +46,8 @@ ] ). -%% APIs --export([ - reg/0, - unreg/0 -]). - --export([ - on_gateway_load/2, - on_gateway_update/3, - on_gateway_unload/2 -]). - %%-------------------------------------------------------------------- -%% APIs -%%-------------------------------------------------------------------- - -reg() -> - RegistryOptions = [{cbkmod, ?MODULE}], - emqx_gateway_registry:reg(exproto, RegistryOptions). - -unreg() -> - emqx_gateway_registry:unreg(exproto). - -%%-------------------------------------------------------------------- -%% emqx_gateway_registry callbacks +%% emqx_gateway_impl callbacks %%-------------------------------------------------------------------- on_gateway_load( @@ -156,7 +149,7 @@ start_grpc_server(GwName, Options = #{bind := ListenOn}) -> } }, SvrOptions = - case emqx_map_lib:deep_get([ssl, enable], Options, false) of + case emqx_utils_maps:deep_get([ssl, enable], Options, false) of false -> []; true -> @@ -208,7 +201,7 @@ start_grpc_client_channel(GwName, Options = #{address := Address}) -> }} ) end, - case emqx_map_lib:deep_get([ssl, enable], Options, false) of + case emqx_utils_maps:deep_get([ssl, enable], Options, false) of false -> SvrAddr = compose_http_uri(http, Host, Port), grpc_client_sup:create_channel_pool(GwName, SvrAddr, #{}); diff --git a/apps/emqx_gateway/test/emqx_exproto_SUITE.erl b/apps/emqx_gateway_exproto/test/emqx_exproto_SUITE.erl similarity index 99% rename from apps/emqx_gateway/test/emqx_exproto_SUITE.erl rename to apps/emqx_gateway_exproto/test/emqx_exproto_SUITE.erl index b476a40cb..264f6af95 100644 --- a/apps/emqx_gateway/test/emqx_exproto_SUITE.erl +++ b/apps/emqx_gateway_exproto/test/emqx_exproto_SUITE.erl @@ -76,6 +76,7 @@ metrics() -> [tcp, ssl, udp, dtls]. init_per_group(GrpName, Cfg) -> + application:load(emqx_gateway_exproto), put(grpname, GrpName), Svrs = emqx_exproto_echo_svr:start(), emqx_common_test_helpers:start_apps([emqx_authn, emqx_gateway], fun set_special_cfg/1), diff --git a/apps/emqx_gateway/test/emqx_exproto_echo_svr.erl b/apps/emqx_gateway_exproto/test/emqx_exproto_echo_svr.erl similarity index 93% rename from apps/emqx_gateway/test/emqx_exproto_echo_svr.erl rename to apps/emqx_gateway_exproto/test/emqx_exproto_echo_svr.erl index b2e3ad4a7..e04990f5f 100644 --- a/apps/emqx_gateway/test/emqx_exproto_echo_svr.erl +++ b/apps/emqx_gateway_exproto/test/emqx_exproto_echo_svr.erl @@ -148,7 +148,7 @@ on_received_bytes(Stream, _Md) -> fun(Reqs) -> lists:foreach( fun(#{conn := Conn, bytes := Bytes}) -> - #{<<"type">> := Type} = Params = emqx_json:decode(Bytes, [return_maps]), + #{<<"type">> := Type} = Params = emqx_utils_json:decode(Bytes, [return_maps]), _ = handle_in(Conn, Type, Params) end, Reqs @@ -284,16 +284,16 @@ handle_out(Conn, ?TYPE_DISCONNECT) -> %% Frame frame_connect(ClientInfo, Password) -> - emqx_json:encode(#{ + emqx_utils_json:encode(#{ type => ?TYPE_CONNECT, clientinfo => ClientInfo, password => Password }). frame_connack(Code) -> - emqx_json:encode(#{type => ?TYPE_CONNACK, code => Code}). + emqx_utils_json:encode(#{type => ?TYPE_CONNACK, code => Code}). frame_publish(Topic, Qos, Payload) -> - emqx_json:encode(#{ + emqx_utils_json:encode(#{ type => ?TYPE_PUBLISH, topic => Topic, qos => Qos, @@ -301,19 +301,19 @@ frame_publish(Topic, Qos, Payload) -> }). frame_puback(Code) -> - emqx_json:encode(#{type => ?TYPE_PUBACK, code => Code}). + emqx_utils_json:encode(#{type => ?TYPE_PUBACK, code => Code}). frame_subscribe(Topic, Qos) -> - emqx_json:encode(#{type => ?TYPE_SUBSCRIBE, topic => Topic, qos => Qos}). + emqx_utils_json:encode(#{type => ?TYPE_SUBSCRIBE, topic => Topic, qos => Qos}). frame_suback(Code) -> - emqx_json:encode(#{type => ?TYPE_SUBACK, code => Code}). + emqx_utils_json:encode(#{type => ?TYPE_SUBACK, code => Code}). frame_unsubscribe(Topic) -> - emqx_json:encode(#{type => ?TYPE_UNSUBSCRIBE, topic => Topic}). + emqx_utils_json:encode(#{type => ?TYPE_UNSUBSCRIBE, topic => Topic}). frame_unsuback(Code) -> - emqx_json:encode(#{type => ?TYPE_UNSUBACK, code => Code}). + emqx_utils_json:encode(#{type => ?TYPE_UNSUBACK, code => Code}). frame_disconnect() -> - emqx_json:encode(#{type => ?TYPE_DISCONNECT}). + emqx_utils_json:encode(#{type => ?TYPE_DISCONNECT}). diff --git a/apps/emqx_gateway_lwm2m/.gitignore b/apps/emqx_gateway_lwm2m/.gitignore new file mode 100644 index 000000000..f1c455451 --- /dev/null +++ b/apps/emqx_gateway_lwm2m/.gitignore @@ -0,0 +1,19 @@ +.rebar3 +_* +.eunit +*.o +*.beam +*.plt +*.swp +*.swo +.erlang.cookie +ebin +log +erl_crash.dump +.rebar +logs +_build +.idea +*.iml +rebar3.crashdump +*~ diff --git a/apps/emqx_gateway_lwm2m/README.md b/apps/emqx_gateway_lwm2m/README.md new file mode 100644 index 000000000..678d74dcf --- /dev/null +++ b/apps/emqx_gateway_lwm2m/README.md @@ -0,0 +1,61 @@ +# emqx_lwm2m + +[LwM2M (Lightweight Machine-to-Machine)](https://lwm2m.openmobilealliance.org/) +is a protocol designed for IoT devices and machine-to-machine communication. +It is a lightweight protocol that supports devices with limited processing power and memory. + + +The **LwM2M Gateway** in EMQX can accept LwM2M clients and translate theirevents +and messages into MQTT Publish messages. + +In the current implementation, it has the following limitations: +- Based UDP/DTLS transport. +- Only supports v1.0.2. The v1.1.x and v1.2.x is not supported yet. +- Not included LwM2M Bootstrap services. + +## Quick Start + +In EMQX 5.0, LwM2M gateways can be configured and enabled through the Dashboard. + +It can also be enabled via the HTTP API, and emqx.conf e.g, In emqx.conf: + +```properties +gateway.lwm2m { + xml_dir = "etc/lwm2m_xml/" + auto_observe = true + enable_stats = true + idle_timeout = "30s" + lifetime_max = "86400s" + lifetime_min = "1s" + mountpoint = "lwm2m/${endpoint_namea}/" + qmode_time_window = "22s" + update_msg_publish_condition = "contains_object_list" + translators { + command {qos = 0, topic = "dn/#"} + notify {qos = 0, topic = "up/notify"} + register {qos = 0, topic = "up/resp"} + response {qos = 0, topic = "up/resp"} + update {qos = 0, topic = "up/update"} + } + listeners { + udp { + default { + bind = "5783" + max_conn_rate = 1000 + max_connections = 1024000 + } + } + } +} +``` + +> Note: +> Configuring the gateway via emqx.conf requires changes on a per-node basis, +> but configuring it via Dashboard or the HTTP API will take effect across the cluster. +::: + +## Object definations + +emqx_lwm2m needs object definitions to parse data from lwm2m devices. Object definitions are declared by organizations in XML format, you could find those XMLs from [LwM2MRegistry](http://www.openmobilealliance.org/wp/OMNA/LwM2M/LwM2MRegistry.html), download and put them into the directory specified by `lwm2m.xml_dir`. If no associated object definition is found, response from device will be discarded and report an error message in log. + +More documentations: [LwM2M Gateway](https://www.emqx.io/docs/en/v5.0/gateway/lwm2m.html) diff --git a/apps/emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl b/apps/emqx_gateway_lwm2m/include/emqx_lwm2m.hrl similarity index 100% rename from apps/emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl rename to apps/emqx_gateway_lwm2m/include/emqx_lwm2m.hrl diff --git a/apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Access_Control-v1_0_1.xml b/apps/emqx_gateway_lwm2m/lwm2m_xml/LWM2M_Access_Control-v1_0_1.xml similarity index 100% rename from apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Access_Control-v1_0_1.xml rename to apps/emqx_gateway_lwm2m/lwm2m_xml/LWM2M_Access_Control-v1_0_1.xml diff --git a/apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Connectivity_Statistics-v1_0_1.xml b/apps/emqx_gateway_lwm2m/lwm2m_xml/LWM2M_Connectivity_Statistics-v1_0_1.xml similarity index 100% rename from apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Connectivity_Statistics-v1_0_1.xml rename to apps/emqx_gateway_lwm2m/lwm2m_xml/LWM2M_Connectivity_Statistics-v1_0_1.xml diff --git a/apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Device-v1_0_1.xml b/apps/emqx_gateway_lwm2m/lwm2m_xml/LWM2M_Device-v1_0_1.xml similarity index 100% rename from apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Device-v1_0_1.xml rename to apps/emqx_gateway_lwm2m/lwm2m_xml/LWM2M_Device-v1_0_1.xml diff --git a/apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Firmware_Update-v1_0_1.xml b/apps/emqx_gateway_lwm2m/lwm2m_xml/LWM2M_Firmware_Update-v1_0_1.xml similarity index 100% rename from apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Firmware_Update-v1_0_1.xml rename to apps/emqx_gateway_lwm2m/lwm2m_xml/LWM2M_Firmware_Update-v1_0_1.xml diff --git a/apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Location-v1_0.xml b/apps/emqx_gateway_lwm2m/lwm2m_xml/LWM2M_Location-v1_0.xml similarity index 100% rename from apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Location-v1_0.xml rename to apps/emqx_gateway_lwm2m/lwm2m_xml/LWM2M_Location-v1_0.xml diff --git a/apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Security-v1_0.xml b/apps/emqx_gateway_lwm2m/lwm2m_xml/LWM2M_Security-v1_0.xml similarity index 100% rename from apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Security-v1_0.xml rename to apps/emqx_gateway_lwm2m/lwm2m_xml/LWM2M_Security-v1_0.xml diff --git a/apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Server-v1_0.xml b/apps/emqx_gateway_lwm2m/lwm2m_xml/LWM2M_Server-v1_0.xml similarity index 100% rename from apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Server-v1_0.xml rename to apps/emqx_gateway_lwm2m/lwm2m_xml/LWM2M_Server-v1_0.xml diff --git a/apps/emqx_gateway_lwm2m/rebar.config b/apps/emqx_gateway_lwm2m/rebar.config new file mode 100644 index 000000000..c8675c3ba --- /dev/null +++ b/apps/emqx_gateway_lwm2m/rebar.config @@ -0,0 +1,4 @@ +{erl_opts, [debug_info]}. +{deps, [ {emqx, {path, "../../apps/emqx"}}, + {emqx_gateway, {path, "../../apps/emqx_gateway"}} + ]}. diff --git a/apps/emqx_gateway_lwm2m/src/emqx_gateway_lwm2m.app.src b/apps/emqx_gateway_lwm2m/src/emqx_gateway_lwm2m.app.src new file mode 100644 index 000000000..83a707395 --- /dev/null +++ b/apps/emqx_gateway_lwm2m/src/emqx_gateway_lwm2m.app.src @@ -0,0 +1,10 @@ +{application, emqx_gateway_lwm2m, [ + {description, "LwM2M Gateway"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [kernel, stdlib, emqx, emqx_gateway, emqx_gateway_coap]}, + {env, []}, + {modules, []}, + {licenses, ["Apache 2.0"]}, + {links, []} +]}. diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl b/apps/emqx_gateway_lwm2m/src/emqx_gateway_lwm2m.erl similarity index 87% rename from apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl rename to apps/emqx_gateway_lwm2m/src/emqx_gateway_lwm2m.erl index fa4537315..1c8f67863 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl +++ b/apps/emqx_gateway_lwm2m/src/emqx_gateway_lwm2m.erl @@ -14,35 +14,37 @@ %% limitations under the License. %%-------------------------------------------------------------------- -%% @doc The LwM2M Gateway Implement interface --module(emqx_lwm2m_impl). - --behaviour(emqx_gateway_impl). +%% @doc The LwM2M Gateway implement +-module(emqx_gateway_lwm2m). -include_lib("emqx/include/logger.hrl"). +-include_lib("emqx_gateway/include/emqx_gateway.hrl"). -%% APIs --export([ - reg/0, - unreg/0 -]). +%% define a gateway named stomp +-gateway(#{ + name => lwm2m, + callback_module => ?MODULE, + config_schema_module => emqx_lwm2m_schema +}). +%% callback_module must implement the emqx_gateway_impl behaviour +-behaviour(emqx_gateway_impl). + +%% callback for emqx_gateway_impl -export([ on_gateway_load/2, on_gateway_update/3, on_gateway_unload/2 ]). -%%-------------------------------------------------------------------- -%% APIs -%%-------------------------------------------------------------------- - -reg() -> - RegistryOptions = [{cbkmod, ?MODULE}], - emqx_gateway_registry:reg(lwm2m, RegistryOptions). - -unreg() -> - emqx_gateway_registry:unreg(lwm2m). +-import( + emqx_gateway_utils, + [ + normalize_config/1, + start_listeners/4, + stop_listeners/2 + ] +). %%-------------------------------------------------------------------- %% emqx_gateway_registry callbacks diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_api.erl b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_api.erl similarity index 98% rename from apps/emqx_gateway/src/lwm2m/emqx_lwm2m_api.erl rename to apps/emqx_gateway_lwm2m/src/emqx_lwm2m_api.erl index 2cd53d6eb..ca32d03db 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_api.erl +++ b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_api.erl @@ -32,6 +32,8 @@ -import(hoconsc, [mk/2, ref/1, ref/2]). -import(emqx_dashboard_swagger, [error_codes/2]). +-elvis([{elvis_style, atom_naming_convention, disable}]). + namespace() -> "lwm2m". api_spec() -> @@ -225,7 +227,7 @@ to_operations(Obj, ObjDefinition) -> }. path_list(Path) -> - case binary:split(binary_util:trim(Path, $/), [<<$/>>], [global]) of + case binary:split(emqx_utils_binary:trim(Path, $/), [<<$/>>], [global]) of [ObjId, ObjInsId, ResId, ResInstId] -> [ObjId, ObjInsId, ResId, ResInstId]; [ObjId, ObjInsId, ResId] -> [ObjId, ObjInsId, ResId]; [ObjId, ObjInsId] -> [ObjId, ObjInsId]; diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_channel.erl similarity index 98% rename from apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl rename to apps/emqx_gateway_lwm2m/src/emqx_lwm2m_channel.erl index 16d0f9630..77652744a 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl +++ b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_channel.erl @@ -16,9 +16,9 @@ -module(emqx_lwm2m_channel). +-include("emqx_lwm2m.hrl"). -include_lib("emqx/include/logger.hrl"). --include("src/coap/include/emqx_coap.hrl"). --include("src/lwm2m/include/emqx_lwm2m.hrl"). +-include_lib("emqx_gateway_coap/include/emqx_coap.hrl"). %% API -export([ @@ -105,7 +105,7 @@ info(conn_state, #channel{conn_state = ConnState}) -> info(clientinfo, #channel{clientinfo = ClientInfo}) -> ClientInfo; info(session, #channel{session = Session}) -> - emqx_misc:maybe_apply(fun emqx_lwm2m_session:info/1, Session); + emqx_utils:maybe_apply(fun emqx_lwm2m_session:info/1, Session); info(clientid, #channel{clientinfo = #{clientid := ClientId}}) -> ClientId; info(ctx, #channel{ctx = Ctx}) -> @@ -286,7 +286,7 @@ handle_call(discard, _From, Channel) -> % pendings = Pendings}) -> % ok = emqx_session:takeover(Session), % %% TODO: Should not drain deliver here (side effect) -% Delivers = emqx_misc:drain_deliver(), +% Delivers = emqx_utils:drain_deliver(), % AllPendings = lists:append(Delivers, Pendings), % shutdown_and_reply(takenover, AllPendings, Channel); @@ -390,7 +390,7 @@ set_peercert_infos(Peercert, ClientInfo) -> ClientInfo#{dn => DN, cn => CN}. make_timer(Name, Time, Msg, Channel = #channel{timers = Timers}) -> - TRef = emqx_misc:start_timer(Time, Msg), + TRef = emqx_utils:start_timer(Time, Msg), Channel#channel{timers = Timers#{Name => TRef}}. update_life_timer(#channel{session = Session, timers = Timers} = Channel) -> @@ -413,7 +413,7 @@ do_takeover(_DesireId, Msg, Channel) -> do_connect(Req, Result, Channel, Iter) -> case - emqx_misc:pipeline( + emqx_utils:pipeline( [ fun check_lwm2m_version/2, fun enrich_conninfo/2, @@ -464,14 +464,14 @@ check_lwm2m_version( _ -> false end, - if - IsValid -> + case IsValid of + true -> NConnInfo = ConnInfo#{ connected_at => erlang:system_time(millisecond), proto_ver => Ver }, {ok, Channel#channel{conninfo = NConnInfo}}; - true -> + _ -> ?SLOG(error, #{ msg => "reject_REGISTRE_request", reason => {unsupported_version, Ver} diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_cmd.erl similarity index 98% rename from apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl rename to apps/emqx_gateway_lwm2m/src/emqx_lwm2m_cmd.erl index 090af3e87..8e4286343 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl +++ b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_cmd.erl @@ -16,9 +16,9 @@ -module(emqx_lwm2m_cmd). +-include("emqx_lwm2m.hrl"). -include_lib("emqx/include/logger.hrl"). --include("src/coap/include/emqx_coap.hrl"). --include("src/lwm2m/include/emqx_lwm2m.hrl"). +-include_lib("emqx_gateway_coap/include/emqx_coap.hrl"). -export([ mqtt_to_coap/2, @@ -292,9 +292,9 @@ make_response(Code, Ref = #{}) -> BaseRsp = make_base_response(Ref), make_data_response(BaseRsp, Code). -make_response(Code, Ref = #{}, _Format, Result) -> +make_response(Code, Ref = #{}, Format, Result) -> BaseRsp = make_base_response(Ref), - make_data_response(BaseRsp, Code, _Format, Result). + make_data_response(BaseRsp, Code, Format, Result). %% The base response format is what included in the request: %% @@ -335,7 +335,7 @@ remove_tmp_fields(Ref) -> -spec path_list(Path :: binary()) -> {[PathWord :: binary()], [Query :: binary()]}. path_list(Path) -> - case binary:split(binary_util:trim(Path, $/), [<<$/>>], [global]) of + case binary:split(emqx_utils_binary:trim(Path, $/), [<<$/>>], [global]) of [ObjId, ObjInsId, ResId, LastPart] -> {ResInstId, QueryList} = query_list(LastPart), {[ObjId, ObjInsId, ResId, ResInstId], QueryList}; @@ -389,7 +389,7 @@ observe_seq(Options) -> add_alternate_path_prefix(<<"/">>, PathList) -> PathList; add_alternate_path_prefix(AlternatePath, PathList) -> - [binary_util:trim(AlternatePath, $/) | PathList]. + [emqx_utils_binary:trim(AlternatePath, $/) | PathList]. extract_path(Ref = #{}) -> drop_query( diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_message.erl b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_message.erl similarity index 95% rename from apps/emqx_gateway/src/lwm2m/emqx_lwm2m_message.erl rename to apps/emqx_gateway_lwm2m/src/emqx_lwm2m_message.erl index f09a8ea3d..8b9ba2491 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_message.erl +++ b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_message.erl @@ -24,7 +24,7 @@ translate_json/1 ]). --include("src/lwm2m/include/emqx_lwm2m.hrl"). +-include("emqx_lwm2m.hrl"). tlv_to_json(BaseName, TlvData) -> DecodedTlv = emqx_lwm2m_tlv:parse(TlvData), @@ -97,7 +97,7 @@ tlv_single_resource(BaseName, Id, Value, ObjDefinition) -> [#{path => BaseName, value => Val}]. basename(OldBaseName, _ObjectId, ObjectInstanceId, ResourceId, 3) -> - case binary:split(binary_util:trim(OldBaseName, $/), [<<$/>>], [global]) of + case binary:split(emqx_utils_binary:trim(OldBaseName, $/), [<<$/>>], [global]) of [ObjId, ObjInsId, ResId] -> <<$/, ObjId/binary, $/, ObjInsId/binary, $/, ResId/binary>>; [ObjId, ObjInsId] -> @@ -113,13 +113,13 @@ basename(OldBaseName, _ObjectId, ObjectInstanceId, ResourceId, 3) -> >> end; basename(OldBaseName, _ObjectId, ObjectInstanceId, _ResourceId, 2) -> - case binary:split(binary_util:trim(OldBaseName, $/), [<<$/>>], [global]) of + case binary:split(emqx_utils_binary:trim(OldBaseName, $/), [<<$/>>], [global]) of [ObjId, ObjInsId, _ResId] -> <<$/, ObjId/binary, $/, ObjInsId/binary>>; [ObjId, ObjInsId] -> <<$/, ObjId/binary, $/, ObjInsId/binary>>; [ObjId] -> <<$/, ObjId/binary, $/, (integer_to_binary(ObjectInstanceId))/binary>> end. % basename(OldBaseName, _ObjectId, _ObjectInstanceId, _ResourceId, 1) -> -% case binary:split(binary_util:trim(OldBaseName, $/), [<<$/>>], [global]) of +% case binary:split(emqx_utils_binary:trim(OldBaseName, $/), [<<$/>>], [global]) of % [ObjId, _ObjInsId, _ResId] -> <<$/, ObjId/binary>>; % [ObjId, _ObjInsId] -> <<$/, ObjId/binary>>; % [ObjId] -> <<$/, ObjId/binary>> @@ -129,7 +129,7 @@ make_path(RelativePath, Id) -> <>. object_id(BaseName) -> - case binary:split(binary_util:trim(BaseName, $/), [<<$/>>], [global]) of + case binary:split(emqx_utils_binary:trim(BaseName, $/), [<<$/>>], [global]) of [ObjId] -> binary_to_integer(ObjId); [ObjId, _] -> binary_to_integer(ObjId); [ObjId, _, _] -> binary_to_integer(ObjId); @@ -137,7 +137,7 @@ object_id(BaseName) -> end. object_resource_id(BaseName) -> - case binary:split(binary_util:trim(BaseName, $/), [<<$/>>], [global]) of + case binary:split(emqx_utils_binary:trim(BaseName, $/), [<<$/>>], [global]) of [_ObjIdBin1] -> error({invalid_basename, BaseName}); [_ObjIdBin2, _] -> @@ -152,7 +152,7 @@ object_resource_id(BaseName) -> value(Value, ResourceId, ObjDefinition) -> case emqx_lwm2m_xml_object:get_resource_type(ResourceId, ObjDefinition) of "String" -> - % keep binary type since it is same as a string for jsx + % keep binary type since it is same as a string for emqx_utils_json Value; "Integer" -> Size = byte_size(Value) * 8, @@ -351,7 +351,7 @@ opaque_to_json(BaseName, Binary) -> [#{path => BaseName, value => base64:encode(Binary)}]. translate_json(JSONBin) -> - JSONTerm = emqx_json:decode(JSONBin, [return_maps]), + JSONTerm = emqx_utils_json:decode(JSONBin, [return_maps]), BaseName = maps:get(<<"bn">>, JSONTerm, <<>>), ElementList = maps:get(<<"e">>, JSONTerm, []), translate_element(BaseName, ElementList, []). @@ -371,8 +371,8 @@ translate_element(BaseName, [Element | ElementList], Acc) -> translate_element(BaseName, ElementList, NewAcc). full_path(BaseName, RelativePath) -> - Prefix = binary_util:rtrim(BaseName, $/), - Path = binary_util:ltrim(RelativePath, $/), + Prefix = emqx_utils_binary:rtrim(BaseName, $/), + Path = emqx_utils_binary:ltrim(RelativePath, $/), <>. get_element_value(#{<<"t">> := Value}) -> Value; @@ -412,9 +412,11 @@ byte_size_of_signed(UInt) -> byte_size_of_signed(UInt, N) -> BitSize = (8 * N - 1), Max = (1 bsl BitSize), - if - UInt =< Max -> N; - UInt > Max -> byte_size_of_signed(UInt, N + 1) + case UInt =< Max of + true -> + N; + false -> + byte_size_of_signed(UInt, N + 1) end. binary_to_number(NumStr) -> diff --git a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_schema.erl b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_schema.erl new file mode 100644 index 000000000..b674c3260 --- /dev/null +++ b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_schema.erl @@ -0,0 +1,184 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_lwm2m_schema). + +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("typerefl/include/types.hrl"). + +-type duration() :: non_neg_integer(). +-type duration_s() :: non_neg_integer(). + +-typerefl_from_string({duration/0, emqx_schema, to_duration}). +-typerefl_from_string({duration_s/0, emqx_schema, to_duration_s}). + +-reflect_type([duration/0, duration_s/0]). + +%% config schema provides +-export([fields/1, desc/1]). + +fields(lwm2m) -> + [ + {xml_dir, + sc( + binary(), + #{ + %% since this is not packaged with emqx, nor + %% present in the packages, we must let the user + %% specify it rather than creating a dynamic + %% default (especially difficult to handle when + %% generating docs). + example => <<"/etc/emqx/lwm2m_xml">>, + required => true, + desc => ?DESC(lwm2m_xml_dir) + } + )}, + {lifetime_min, + sc( + duration(), + #{ + default => <<"15s">>, + desc => ?DESC(lwm2m_lifetime_min) + } + )}, + {lifetime_max, + sc( + duration(), + #{ + default => <<"86400s">>, + desc => ?DESC(lwm2m_lifetime_max) + } + )}, + {qmode_time_window, + sc( + duration_s(), + #{ + default => <<"22s">>, + desc => ?DESC(lwm2m_qmode_time_window) + } + )}, + %% TODO: Support config resource path + {auto_observe, + sc( + boolean(), + #{ + default => false, + desc => ?DESC(lwm2m_auto_observe) + } + )}, + %% FIXME: not working now + {update_msg_publish_condition, + sc( + hoconsc:enum([always, contains_object_list]), + #{ + default => contains_object_list, + desc => ?DESC(lwm2m_update_msg_publish_condition) + } + )}, + {translators, + sc( + ref(lwm2m_translators), + #{ + required => true, + desc => ?DESC(lwm2m_translators) + } + )}, + {mountpoint, emqx_gateway_schema:mountpoint("lwm2m/${endpoint_name}/")}, + {listeners, sc(ref(emqx_gateway_schema, udp_listeners), #{desc => ?DESC(udp_listeners)})} + ] ++ emqx_gateway_schema:gateway_common_options(); +fields(lwm2m_translators) -> + [ + {command, + sc( + ref(translator), + #{ + desc => ?DESC(lwm2m_translators_command), + required => true + } + )}, + {response, + sc( + ref(translator), + #{ + desc => ?DESC(lwm2m_translators_response), + required => true + } + )}, + {notify, + sc( + ref(translator), + #{ + desc => ?DESC(lwm2m_translators_notify), + required => true + } + )}, + {register, + sc( + ref(translator), + #{ + desc => ?DESC(lwm2m_translators_register), + required => true + } + )}, + {update, + sc( + ref(translator), + #{ + desc => ?DESC(lwm2m_translators_update), + required => true + } + )} + ]; +fields(translator) -> + [ + {topic, + sc( + binary(), + #{ + required => true, + desc => ?DESC(translator_topic) + } + )}, + {qos, + sc( + emqx_schema:qos(), + #{ + default => 0, + desc => ?DESC(translator_qos) + } + )} + ]. + +desc(lwm2m) -> + "The LwM2M protocol gateway."; +desc(lwm2m_translators) -> + "MQTT topics that correspond to LwM2M events."; +desc(translator) -> + "MQTT topic that corresponds to a particular type of event."; +desc(_) -> + undefined. + +%%-------------------------------------------------------------------- +%% helpers + +sc(Type, Meta) -> + hoconsc:mk(Type, Meta). + +ref(StructName) -> + ref(?MODULE, StructName). + +ref(Mod, Field) -> + hoconsc:ref(Mod, Field). diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_session.erl b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_session.erl similarity index 98% rename from apps/emqx_gateway/src/lwm2m/emqx_lwm2m_session.erl rename to apps/emqx_gateway_lwm2m/src/emqx_lwm2m_session.erl index 8634280e3..e267692a6 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_session.erl +++ b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_session.erl @@ -15,12 +15,12 @@ %%-------------------------------------------------------------------- -module(emqx_lwm2m_session). --include("src/coap/include/emqx_coap.hrl"). --include("src/lwm2m/include/emqx_lwm2m.hrl"). +-include("emqx_lwm2m.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). +-include_lib("emqx_gateway_coap/include/emqx_coap.hrl"). %% API -export([ @@ -379,18 +379,18 @@ is_alternate_path(LinkAttrs) -> true; [AttrKey, _] when AttrKey =/= <<>> -> false; - _BadAttr -> - throw({bad_attr, _BadAttr}) + BadAttr -> + throw({bad_attr, BadAttr}) end end, LinkAttrs ). -trim(Str) -> binary_util:trim(Str, $\s). +trim(Str) -> emqx_utils_binary:trim(Str, $\s). delink(Str) -> - Ltrim = binary_util:ltrim(Str, $<), - binary_util:rtrim(Ltrim, $>). + Ltrim = emqx_utils_binary:ltrim(Str, $<), + emqx_utils_binary:rtrim(Ltrim, $>). get_lifetime(#{<<"lt">> := LT}) -> case LT of @@ -679,10 +679,10 @@ send_to_coap(#session{queue = Queue} = Session) -> case queue:out(Queue) of {{value, {Timestamp, Ctx, Req}}, Q2} -> Now = ?NOW, - if - Timestamp =:= 0 orelse Timestamp > Now -> - send_to_coap(Ctx, Req, Session#session{queue = Q2}); + case Timestamp =:= 0 orelse Timestamp > Now of true -> + send_to_coap(Ctx, Req, Session#session{queue = Q2}); + false -> send_to_coap(Session#session{queue = Q2}) end; {empty, _} -> @@ -737,7 +737,7 @@ proto_publish( Epn, Qos, MountedTopic, - emqx_json:encode(Payload), + emqx_utils_json:encode(Payload), #{}, Headers ), @@ -786,7 +786,7 @@ deliver_to_coap(AlternatePath, JsonData, MQTT, CacheMode, WithContext, Session) is_binary(JsonData) -> try - TermData = emqx_json:decode(JsonData, [return_maps]), + TermData = emqx_utils_json:decode(JsonData, [return_maps]), deliver_to_coap(AlternatePath, TermData, MQTT, CacheMode, WithContext, Session) catch ExClass:Error:ST -> diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_tlv.erl b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_tlv.erl similarity index 90% rename from apps/emqx_gateway/src/lwm2m/emqx_lwm2m_tlv.erl rename to apps/emqx_gateway_lwm2m/src/emqx_lwm2m_tlv.erl index 782bbec5e..314666638 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_tlv.erl +++ b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_tlv.erl @@ -25,7 +25,7 @@ -export([binary_to_hex_string/1]). -endif. --include("src/lwm2m/include/emqx_lwm2m.hrl"). +-include("emqx_lwm2m.hrl"). -define(TLV_TYPE_OBJECT_INSTANCE, 0). -define(TLV_TYPE_RESOURCE_INSTANCE, 1). @@ -37,13 +37,18 @@ -define(TLV_LEGNTH_16_BIT, 2). -define(TLV_LEGNTH_24_BIT, 3). -%---------------------------------------------------------------------------------------------------------------------------------------- -% [#{tlv_object_instance := Id11, value := Value11}, #{tlv_object_instance := Id12, value := Value12}, ...] +-elvis([{elvis_style, no_if_expression, disable}]). + +%%-------------------------------------------------------------------- +% [#{tlv_object_instance := Id11, value := Value11}, +% #{tlv_object_instance := Id12, value := Value12}, ...] % where Value11 and Value12 is a list: -% [#{tlv_resource_with_value => Id21, value => Value21}, #{tlv_multiple_resource => Id22, value = Value22}, ...] +% [#{tlv_resource_with_value => Id21, value => Value21}, +% #{tlv_multiple_resource => Id22, value = Value22}, ...] % where Value21 is a binary % Value22 is a list: -% [#{tlv_resource_instance => Id31, value => Value31}, #{tlv_resource_instance => Id32, value => Value32}, ...] +% [#{tlv_resource_instance => Id31, value => Value31}, +% #{tlv_resource_instance => Id32, value => Value32}, ...] % where Value31 and Value32 is a binary % % correspond to three levels: @@ -51,8 +56,9 @@ % 2) Resource Level % 3) Resource Instance Level % -% NOTE: TLV does not has object level, only has object instance level. It implies TLV can not represent multiple objects -%---------------------------------------------------------------------------------------------------------------------------------------- +% NOTE: TLV does not has object level, only has object instance level. +% It implies TLV can not represent multiple objects +%%-------------------------------------------------------------------- parse(Data) -> parse_loop(Data, []). diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object.erl b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_xml_object.erl similarity index 98% rename from apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object.erl rename to apps/emqx_gateway_lwm2m/src/emqx_lwm2m_xml_object.erl index a4dc44f2c..3525f72aa 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object.erl +++ b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_xml_object.erl @@ -16,7 +16,7 @@ -module(emqx_lwm2m_xml_object). --include("src/lwm2m/include/emqx_lwm2m.hrl"). +-include("emqx_lwm2m.hrl"). -include_lib("xmerl/include/xmerl.hrl"). -export([ diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_xml_object_db.erl similarity index 97% rename from apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl rename to apps/emqx_gateway_lwm2m/src/emqx_lwm2m_xml_object_db.erl index 58373e114..2908a65e0 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl +++ b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_xml_object_db.erl @@ -16,7 +16,7 @@ -module(emqx_lwm2m_xml_object_db). --include("src/lwm2m/include/emqx_lwm2m.hrl"). +-include("emqx_lwm2m.hrl"). -include_lib("xmerl/include/xmerl.hrl"). -include_lib("emqx/include/logger.hrl"). @@ -45,6 +45,8 @@ -record(state, {}). +-elvis([{elvis_style, atom_naming_convention, disable}]). + %% ------------------------------------------------------------------ %% API Function Definitions %% ------------------------------------------------------------------ @@ -124,10 +126,10 @@ code_change(_OldVsn, State, _Extra) -> load(BaseDir) -> Wild = filename:join(BaseDir, "*.xml"), Wild2 = - if - is_binary(Wild) -> - erlang:binary_to_list(Wild); + case is_binary(Wild) of true -> + erlang:binary_to_list(Wild); + false -> Wild end, case filelib:wildcard(Wild2) of diff --git a/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl b/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_SUITE.erl similarity index 97% rename from apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl rename to apps/emqx_gateway_lwm2m/test/emqx_lwm2m_SUITE.erl index fc852709c..9f388b07c 100644 --- a/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl +++ b/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_SUITE.erl @@ -31,8 +31,8 @@ -define(LOGT(Format, Args), ct:pal("TEST_SUITE: " ++ Format, Args)). --include("src/lwm2m/include/emqx_lwm2m.hrl"). --include("src/coap/include/emqx_coap.hrl"). +-include("emqx_lwm2m.hrl"). +-include_lib("emqx_gateway_coap/include/emqx_coap.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). @@ -134,6 +134,7 @@ groups() -> init_per_suite(Config) -> %% load application first for minirest api searching application:load(emqx_gateway), + application:load(emqx_gateway_lwm2m), emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_authn]), Config. @@ -176,11 +177,19 @@ default_config() -> default_config(#{}). default_config(Overrides) -> + XmlDir = filename:join( + [ + emqx_common_test_helpers:proj_root(), + "apps", + "emqx_gateway_lwm2m", + "lwm2m_xml" + ] + ), iolist_to_binary( io_lib:format( "\n" "gateway.lwm2m {\n" - " xml_dir = \"../../lib/emqx_gateway/src/lwm2m/lwm2m_xml\"\n" + " xml_dir = \"~s\"\n" " lifetime_min = 1s\n" " lifetime_max = 86400s\n" " qmode_time_window = 22\n" @@ -199,6 +208,7 @@ default_config(Overrides) -> " }\n" "}\n", [ + XmlDir, maps:get(auto_observe, Overrides, false), maps:get(bind, Overrides, ?PORT) ] @@ -392,7 +402,7 @@ case01_register_report(Config) -> timer:sleep(50), true = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"msgType">> => <<"register">>, <<"data">> => #{ @@ -468,7 +478,7 @@ case02_update_deregister(Config) -> ?LOGT("Options got: ~p", [Opts]), Location = maps:get(location_path, Opts), - Register = emqx_json:encode( + Register = emqx_utils_json:encode( #{ <<"msgType">> => <<"register">>, <<"data">> => #{ @@ -511,7 +521,7 @@ case02_update_deregister(Config) -> } = test_recv_coap_response(UdpSock), {ok, changed} = Method2, MsgId2 = RspId2, - Update = emqx_json:encode( + Update = emqx_utils_json:encode( #{ <<"msgType">> => <<"update">>, <<"data">> => #{ @@ -744,7 +754,7 @@ case08_reregister(Config) -> timer:sleep(50), true = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"msgType">> => <<"register">>, <<"data">> => #{ @@ -861,7 +871,7 @@ case10_read(Config) -> <<"path">> => <<"/3/0/0">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), ?LOGT("CommandJson=~p", [CommandJson]), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), @@ -892,7 +902,7 @@ case10_read(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -947,7 +957,7 @@ case10_read_bad_request(Config) -> <<"path">> => <<"/3333/0/0">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), ?LOGT("CommandJson=~p", [CommandJson]), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), @@ -969,7 +979,7 @@ case10_read_bad_request(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode(#{ + ReadResult = emqx_utils_json:encode(#{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, <<"msgType">> => <<"read">>, @@ -1005,7 +1015,7 @@ case10_read_separate_ack(Config) -> <<"path">> => <<"/3/0/0">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), ?LOGT("CommandJson=~p", [CommandJson]), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), @@ -1022,7 +1032,7 @@ case10_read_separate_ack(Config) -> ?assertEqual(<<>>, Payload2), test_send_empty_ack(UdpSock, "127.0.0.1", ?PORT, Request2), - ReadResultACK = emqx_json:encode( + ReadResultACK = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1047,7 +1057,7 @@ case10_read_separate_ack(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1090,7 +1100,7 @@ case11_read_object_tlv(Config) -> <<"path">> => <<"/3/0">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), ?LOGT("CommandJson=~p", [CommandJson]), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), @@ -1122,7 +1132,7 @@ case11_read_object_tlv(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1175,7 +1185,7 @@ case11_read_object_json(Config) -> <<"path">> => <<"/3/0">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), ?LOGT("CommandJson=~p", [CommandJson]), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), @@ -1205,7 +1215,7 @@ case11_read_object_json(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1257,7 +1267,7 @@ case12_read_resource_opaque(Config) -> <<"path">> => <<"/3/0/8">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), ?LOGT("CommandJson=~p", [CommandJson]), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), @@ -1283,7 +1293,7 @@ case12_read_resource_opaque(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1325,7 +1335,7 @@ case13_read_no_xml(Config) -> <<"msgType">> => <<"read">>, <<"data">> => #{<<"path">> => <<"/9723/0/0">>} }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), ?LOGT("CommandJson=~p", [CommandJson]), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), @@ -1350,7 +1360,7 @@ case13_read_no_xml(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1389,7 +1399,7 @@ case20_single_write(Config) -> <<"value">> => <<"12345">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), Request2 = test_recv_coap_request(UdpSock), @@ -1416,7 +1426,7 @@ case20_single_write(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1460,7 +1470,7 @@ case20_write(Config) -> ] } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), Request2 = test_recv_coap_request(UdpSock), @@ -1487,7 +1497,7 @@ case20_write(Config) -> ), timer:sleep(100), - WriteResult = emqx_json:encode( + WriteResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1537,7 +1547,7 @@ case21_write_object(Config) -> ] } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), Request2 = test_recv_coap_request(UdpSock), @@ -1564,7 +1574,7 @@ case21_write_object(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1608,7 +1618,7 @@ case22_write_error(Config) -> ] } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), Request2 = test_recv_coap_request(UdpSock), @@ -1629,7 +1639,7 @@ case22_write_error(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1667,7 +1677,7 @@ case_create_basic(Config) -> <<"basePath">> => <<"/5">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), Request2 = test_recv_coap_request(UdpSock), @@ -1693,7 +1703,7 @@ case_create_basic(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1728,7 +1738,7 @@ case_delete_basic(Config) -> <<"msgType">> => <<"delete">>, <<"data">> => #{<<"path">> => <<"/5/0">>} }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), Request2 = test_recv_coap_request(UdpSock), @@ -1754,7 +1764,7 @@ case_delete_basic(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1794,7 +1804,7 @@ case30_execute(Config) -> <<"args">> => <<"2,7">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), Request2 = test_recv_coap_request(UdpSock), @@ -1820,7 +1830,7 @@ case30_execute(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1858,7 +1868,7 @@ case31_execute_error(Config) -> <<"args">> => <<"2,7">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), Request2 = test_recv_coap_request(UdpSock), @@ -1884,7 +1894,7 @@ case31_execute_error(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1921,7 +1931,7 @@ case40_discover(Config) -> <<"path">> => <<"/3/0/7">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), Request2 = test_recv_coap_request(UdpSock), @@ -1951,7 +1961,7 @@ case40_discover(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1996,7 +2006,7 @@ case50_write_attribute(Config) -> <<"lt">> => <<"5">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(100), Request2 = test_recv_coap_request(UdpSock), @@ -2032,7 +2042,7 @@ case50_write_attribute(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -2069,7 +2079,7 @@ case60_observe(Config) -> <<"msgType">> => <<"observe">>, <<"data">> => #{<<"path">> => <<"/3/0/10">>} }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), Request2 = test_recv_coap_request(UdpSock), @@ -2096,7 +2106,7 @@ case60_observe(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -2131,7 +2141,7 @@ case60_observe(Config) -> timer:sleep(100), #coap_message{} = test_recv_coap_response(UdpSock), - ReadResult2 = emqx_json:encode( + ReadResult2 = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -2163,7 +2173,7 @@ case60_observe(Config) -> <<"path">> => <<"/3/0/10">> } }, - CommandJson3 = emqx_json:encode(Command3), + CommandJson3 = emqx_utils_json:encode(Command3), test_mqtt_broker:publish(CommandTopic, CommandJson3, 0), timer:sleep(50), Request3 = test_recv_coap_request(UdpSock), @@ -2190,7 +2200,7 @@ case60_observe(Config) -> ), timer:sleep(100), - ReadResult3 = emqx_json:encode( + ReadResult3 = emqx_utils_json:encode( #{ <<"requestID">> => CmdId3, <<"cacheID">> => CmdId3, @@ -2232,7 +2242,7 @@ case60_observe(Config) -> %% MsgId1), %% #coap_message{method = Method1} = test_recv_coap_response(UdpSock), %% ?assertEqual({ok,created}, Method1), -%% ReadResult = emqx_json:encode( +%% ReadResult = emqx_utils_json:encode( %% #{<<"msgType">> => <<"register">>, %% <<"data">> => #{ %% <<"alternatePath">> => <<"/">>, @@ -2258,7 +2268,7 @@ case60_observe(Config) -> %% <<"path">> => <<"/19/0/0">> %% } %% }, -%% CommandJson = emqx_json:encode(Command), +%% CommandJson = emqx_utils_json:encode(Command), %% test_mqtt_broker:publish(CommandTopic, CommandJson, 0), %% timer:sleep(50), %% Request2 = test_recv_coap_request(UdpSock), @@ -2315,7 +2325,7 @@ case60_observe(Config) -> %% <<"value">> => base64:encode(<<12345:32>>) %% }}, %% -%% CommandJson = emqx_json:encode(Command), +%% CommandJson = emqx_utils_json:encode(Command), %% test_mqtt_broker:publish(CommandTopic, CommandJson, 0), %% timer:sleep(50), %% Request2 = test_recv_coap_request(UdpSock), @@ -2332,7 +2342,7 @@ case60_observe(Config) -> %% {ok, changed}, #coap_content{}, Request2, true), %% timer:sleep(100), %% -%% ReadResult = emqx_json:encode( +%% ReadResult = emqx_utils_json:encode( %% #{<<"requestID">> => CmdId, %% <<"cacheID">> => CmdId, %% <<"data">> => #{ @@ -2492,7 +2502,7 @@ send_read_command_1(CmdId, _UdpSock) -> <<"msgType">> => <<"read">>, <<"data">> => #{<<"path">> => <<"/3/0/0">>} }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50). @@ -2518,7 +2528,7 @@ verify_read_response_1(CmdId, UdpSock) -> true ), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, diff --git a/apps/emqx_gateway/test/emqx_lwm2m_api_SUITE.erl b/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_api_SUITE.erl similarity index 91% rename from apps/emqx_gateway/test/emqx_lwm2m_api_SUITE.erl rename to apps/emqx_gateway_lwm2m/test/emqx_lwm2m_api_SUITE.erl index c40d1af55..6fa46ebbc 100644 --- a/apps/emqx_gateway/test/emqx_lwm2m_api_SUITE.erl +++ b/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_api_SUITE.erl @@ -23,34 +23,11 @@ -define(LOGT(Format, Args), ct:pal("TEST_SUITE: " ++ Format, Args)). --include("src/lwm2m/include/emqx_lwm2m.hrl"). --include("src/coap/include/emqx_coap.hrl"). +-include("emqx_lwm2m.hrl"). +-include("emqx_gateway_coap/include/emqx_coap.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). --define(CONF_DEFAULT, << - "\n" - "gateway.lwm2m {\n" - " xml_dir = \"../../lib/emqx_gateway/src/lwm2m/lwm2m_xml\"\n" - " lifetime_min = 100s\n" - " lifetime_max = 86400s\n" - " qmode_time_window = 200\n" - " auto_observe = false\n" - " mountpoint = \"lwm2m/${username}\"\n" - " update_msg_publish_condition = contains_object_list\n" - " translators {\n" - " command = {topic = \"/dn/#\", qos = 0}\n" - " response = {topic = \"/up/resp\", qos = 0}\n" - " notify = {topic = \"/up/notify\", qos = 0}\n" - " register = {topic = \"/up/resp\", qos = 0}\n" - " update = {topic = \"/up/resp\", qos = 0}\n" - " }\n" - " listeners.udp.default {\n" - " bind = 5783\n" - " }\n" - "}\n" ->>). - -define(assertExists(Map, Key), ?assertNotEqual(maps:get(Key, Map, undefined), undefined) ). @@ -81,8 +58,10 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), application:load(emqx_gateway), + application:load(emqx_gateway_lwm2m), + DefaultConfig = emqx_lwm2m_SUITE:default_config(), + ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, DefaultConfig), emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_authn]), Config. @@ -93,7 +72,8 @@ end_per_suite(Config) -> Config. init_per_testcase(_AllTestCase, Config) -> - ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), + DefaultConfig = emqx_lwm2m_SUITE:default_config(), + ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, DefaultConfig), {ok, _} = application:ensure_all_started(emqx_gateway), {ok, ClientUdpSock} = gen_udp:open(0, [binary, {active, false}]), @@ -151,7 +131,7 @@ t_lookup_read(Config) -> <<"path">> => <<"/3/0/0">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), ?LOGT("CommandJson=~p", [CommandJson]), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), @@ -198,7 +178,7 @@ t_lookup_discover(Config) -> <<"path">> => <<"/3/0/7">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(200), @@ -370,10 +350,10 @@ no_received_request(ClientId, Path, Action) -> <<"codeMsg">> => <<"reply_not_received">>, <<"path">> => Path }, - ?assertEqual(NotReceived, emqx_json:decode(Response, [return_maps])). + ?assertEqual(NotReceived, emqx_utils_json:decode(Response, [return_maps])). normal_received_request(ClientId, Path, Action) -> Response = call_lookup_api(ClientId, Path, Action), - RCont = emqx_json:decode(Response, [return_maps]), + RCont = emqx_utils_json:decode(Response, [return_maps]), ?assertEqual(list_to_binary(ClientId), maps:get(<<"clientid">>, RCont, undefined)), ?assertEqual(Path, maps:get(<<"path">>, RCont, undefined)), ?assertEqual(Action, maps:get(<<"action">>, RCont, undefined)), diff --git a/apps/emqx_gateway/test/emqx_tlv_SUITE.erl b/apps/emqx_gateway_lwm2m/test/emqx_tlv_SUITE.erl similarity index 99% rename from apps/emqx_gateway/test/emqx_tlv_SUITE.erl rename to apps/emqx_gateway_lwm2m/test/emqx_tlv_SUITE.erl index 5dcef7e72..c413469ea 100644 --- a/apps/emqx_gateway/test/emqx_tlv_SUITE.erl +++ b/apps/emqx_gateway_lwm2m/test/emqx_tlv_SUITE.erl @@ -21,8 +21,8 @@ -define(LOGT(Format, Args), logger:debug("TEST_SUITE: " ++ Format, Args)). --include("src/lwm2m/include/emqx_lwm2m.hrl"). --include("src/coap/include/emqx_coap.hrl"). +-include("emqx_lwm2m.hrl"). +-include("emqx_gateway_coap/include/emqx_coap.hrl"). -include_lib("eunit/include/eunit.hrl"). %%-------------------------------------------------------------------- diff --git a/apps/emqx_gateway_mqttsn/.gitignore b/apps/emqx_gateway_mqttsn/.gitignore new file mode 100644 index 000000000..f1c455451 --- /dev/null +++ b/apps/emqx_gateway_mqttsn/.gitignore @@ -0,0 +1,19 @@ +.rebar3 +_* +.eunit +*.o +*.beam +*.plt +*.swp +*.swo +.erlang.cookie +ebin +log +erl_crash.dump +.rebar +logs +_build +.idea +*.iml +rebar3.crashdump +*~ diff --git a/apps/emqx_gateway_mqttsn/README.md b/apps/emqx_gateway_mqttsn/README.md new file mode 100644 index 000000000..dd72a86a5 --- /dev/null +++ b/apps/emqx_gateway_mqttsn/README.md @@ -0,0 +1,34 @@ +# emqx_mqttsn + +The MQTT-SN gateway is based on the +[MQTT-SN v1.2](https://www.oasis-open.org/committees/download.php/66091/MQTT-SN_spec_v1.2.pdf). + +## Quick Start + +In EMQX 5.0, MQTT-SN gateway can be configured and enabled through the Dashboard. + +It can also be enabled via the HTTP API or emqx.conf, e.g. In emqx.conf: + +```properties +gateway.mqttsn { + + mountpoint = "mqtt/sn" + + gateway_id = 1 + + broadcast = true + + enable_qos3 = true + + listeners.udp.default { + bind = 1884 + max_connections = 10240000 max_conn_rate = 1000 + } +} +``` + +> Note: +> Configuring the gateway via emqx.conf requires changes on a per-node basis, +> but configuring it via Dashboard or the HTTP API will take effect across the cluster. + +More documentations: [MQTT-SN Gateway](https://www.emqx.io/docs/en/v5.0/gateway/mqttsn.html) diff --git a/apps/emqx_gateway/src/mqttsn/include/emqx_sn.hrl b/apps/emqx_gateway_mqttsn/include/emqx_mqttsn.hrl similarity index 100% rename from apps/emqx_gateway/src/mqttsn/include/emqx_sn.hrl rename to apps/emqx_gateway_mqttsn/include/emqx_mqttsn.hrl diff --git a/apps/emqx_gateway_mqttsn/rebar.config b/apps/emqx_gateway_mqttsn/rebar.config new file mode 100644 index 000000000..c8675c3ba --- /dev/null +++ b/apps/emqx_gateway_mqttsn/rebar.config @@ -0,0 +1,4 @@ +{erl_opts, [debug_info]}. +{deps, [ {emqx, {path, "../../apps/emqx"}}, + {emqx_gateway, {path, "../../apps/emqx_gateway"}} + ]}. diff --git a/apps/emqx_gateway_mqttsn/src/emqx_gateway_mqttsn.app.src b/apps/emqx_gateway_mqttsn/src/emqx_gateway_mqttsn.app.src new file mode 100644 index 000000000..dd48b2723 --- /dev/null +++ b/apps/emqx_gateway_mqttsn/src/emqx_gateway_mqttsn.app.src @@ -0,0 +1,10 @@ +{application, emqx_gateway_mqttsn, [ + {description, "MQTT-SN Gateway"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [kernel, stdlib, emqx, emqx_gateway]}, + {env, []}, + {modules, []}, + {licenses, ["Apache 2.0"]}, + {links, []} +]}. diff --git a/apps/emqx_gateway/src/mqttsn/emqx_sn_impl.erl b/apps/emqx_gateway_mqttsn/src/emqx_gateway_mqttsn.erl similarity index 76% rename from apps/emqx_gateway/src/mqttsn/emqx_sn_impl.erl rename to apps/emqx_gateway_mqttsn/src/emqx_gateway_mqttsn.erl index db730aee1..167ee465c 100644 --- a/apps/emqx_gateway/src/mqttsn/emqx_sn_impl.erl +++ b/apps/emqx_gateway_mqttsn/src/emqx_gateway_mqttsn.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -14,13 +14,28 @@ %% limitations under the License. %%-------------------------------------------------------------------- -%% @doc The MQTT-SN Gateway Implement interface --module(emqx_sn_impl). - --behaviour(emqx_gateway_impl). +%% @doc The MQTT-SN Gateway implement interface +-module(emqx_gateway_mqttsn). -include_lib("emqx/include/logger.hrl"). +%% define a gateway named stomp +-gateway(#{ + name => mqttsn, + callback_module => ?MODULE, + config_schema_module => emqx_mqttsn_schema +}). + +%% callback_module must implement the emqx_gateway_impl behaviour +-behaviour(emqx_gateway_impl). + +%% callback for emqx_gateway_impl +-export([ + on_gateway_load/2, + on_gateway_update/3, + on_gateway_unload/2 +]). + -import( emqx_gateway_utils, [ @@ -30,31 +45,8 @@ ] ). -%% APIs --export([ - reg/0, - unreg/0 -]). - --export([ - on_gateway_load/2, - on_gateway_update/3, - on_gateway_unload/2 -]). - %%-------------------------------------------------------------------- -%% APIs -%%-------------------------------------------------------------------- - -reg() -> - RegistryOptions = [{cbkmod, ?MODULE}], - emqx_gateway_registry:reg(mqttsn, RegistryOptions). - -unreg() -> - emqx_gateway_registry:unreg(mqttsn). - -%%-------------------------------------------------------------------- -%% emqx_gateway_registry callbacks +%% emqx_gateway_impl callbacks %%-------------------------------------------------------------------- on_gateway_load( @@ -64,8 +56,8 @@ on_gateway_load( }, Ctx ) -> - %% We Also need to start `emqx_sn_broadcast` & - %% `emqx_sn_registry` process + %% We Also need to start `emqx_mqttsn_broadcast` & + %% `emqx_mqttsn_registry` process case maps:get(broadcast, Config, false) of false -> ok; @@ -73,23 +65,23 @@ on_gateway_load( %% FIXME: Port = 1884, SnGwId = maps:get(gateway_id, Config, undefined), - _ = emqx_sn_broadcast:start_link(SnGwId, Port), + _ = emqx_mqttsn_broadcast:start_link(SnGwId, Port), ok end, PredefTopics = maps:get(predefined, Config, []), - {ok, RegistrySvr} = emqx_sn_registry:start_link(GwName, PredefTopics), + {ok, RegistrySvr} = emqx_mqttsn_registry:start_link(GwName, PredefTopics), NConfig = maps:without( [broadcast, predefined], - Config#{registry => emqx_sn_registry:lookup_name(RegistrySvr)} + Config#{registry => emqx_mqttsn_registry:lookup_name(RegistrySvr)} ), Listeners = emqx_gateway_utils:normalize_config(NConfig), ModCfg = #{ - frame_mod => emqx_sn_frame, - chann_mod => emqx_sn_channel + frame_mod => emqx_mqttsn_frame, + chann_mod => emqx_mqttsn_channel }, case diff --git a/apps/emqx_gateway/src/mqttsn/emqx_sn_broadcast.erl b/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_broadcast.erl similarity index 89% rename from apps/emqx_gateway/src/mqttsn/emqx_sn_broadcast.erl rename to apps/emqx_gateway_mqttsn/src/emqx_mqttsn_broadcast.erl index 5fc08ad7f..be0122e0e 100644 --- a/apps/emqx_gateway/src/mqttsn/emqx_sn_broadcast.erl +++ b/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_broadcast.erl @@ -14,17 +14,11 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_sn_broadcast). +-module(emqx_mqttsn_broadcast). -behaviour(gen_server). --ifdef(TEST). -%% make rebar3 ct happy when testing with --suite path/to/module_SUITE.erl --include_lib("emqx_gateway/src/mqttsn/include/emqx_sn.hrl"). --else. -%% make mix happy --include("src/mqttsn/include/emqx_sn.hrl"). --endif. +-include("emqx_mqttsn.hrl"). -include_lib("emqx/include/logger.hrl"). -export([ @@ -65,7 +59,7 @@ stop() -> init([GwId, Port]) -> %% FIXME: - Duration = application:get_env(emqx_sn, advertise_duration, ?DEFAULT_DURATION), + Duration = application:get_env(emqx_mqttsn, advertise_duration, ?DEFAULT_DURATION), {ok, Sock} = gen_udp:open(0, [binary, {broadcast, true}]), {ok, ensure_advertise(#state{ @@ -121,7 +115,7 @@ send_advertise(#state{ addrs = Addrs, duration = Duration }) -> - Data = emqx_sn_frame:serialize_pkt(?SN_ADVERTISE_MSG(GwId, Duration), #{}), + Data = emqx_mqttsn_frame:serialize_pkt(?SN_ADVERTISE_MSG(GwId, Duration), #{}), lists:foreach( fun(Addr) -> ?SLOG(debug, #{ diff --git a/apps/emqx_gateway/src/mqttsn/emqx_sn_channel.erl b/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl similarity index 98% rename from apps/emqx_gateway/src/mqttsn/emqx_sn_channel.erl rename to apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl index 23d07113c..1ccc8b95a 100644 --- a/apps/emqx_gateway/src/mqttsn/emqx_sn_channel.erl +++ b/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl @@ -14,11 +14,11 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_sn_channel). +-module(emqx_mqttsn_channel). -behaviour(emqx_gateway_channel). --include("src/mqttsn/include/emqx_sn.hrl"). +-include("emqx_mqttsn.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/types.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). @@ -51,7 +51,7 @@ %% Context ctx :: emqx_gateway_ctx:context(), %% Registry - registry :: emqx_sn_registry:registry(), + registry :: emqx_mqttsn_registry:registry(), %% Gateway Id gateway_id :: integer(), %% Enable QoS3 @@ -218,7 +218,7 @@ info(conn_state, #channel{conn_state = ConnState}) -> info(clientinfo, #channel{clientinfo = ClientInfo}) -> ClientInfo; info(session, #channel{session = Session}) -> - emqx_misc:maybe_apply(fun emqx_session:info/1, Session); + emqx_utils:maybe_apply(fun emqx_session:info/1, Session); info(will_msg, #channel{will_msg = WillMsg}) -> WillMsg; info(clientid, #channel{clientinfo = #{clientid := ClientId}}) -> @@ -282,7 +282,7 @@ enrich_clientinfo( feedvar(Override, Packet, ConnInfo, ClientInfo0), ClientInfo0 ), - {ok, NPacket, NClientInfo} = emqx_misc:pipeline( + {ok, NPacket, NClientInfo} = emqx_utils:pipeline( [ fun maybe_assign_clientid/2, %% FIXME: CALL After authentication successfully @@ -414,7 +414,7 @@ process_connect( Channel#channel{session = Session} ); {ok, #{session := Session, present := true, pendings := Pendings}} -> - Pendings1 = lists:usort(lists:append(Pendings, emqx_misc:drain_deliver())), + Pendings1 = lists:usort(lists:append(Pendings, emqx_utils:drain_deliver())), NChannel = Channel#channel{ session = Session, resuming = true, @@ -478,7 +478,7 @@ handle_in( true -> <>; false -> - emqx_sn_registry:lookup_topic( + emqx_mqttsn_registry:lookup_topic( Registry, ?NEG_QOS_CLIENT_ID, TopicId @@ -595,7 +595,7 @@ handle_in( Channel = #channel{conn_state = idle} ) -> case - emqx_misc:pipeline( + emqx_utils:pipeline( [ fun enrich_conninfo/2, fun run_conn_hooks/2, @@ -624,7 +624,7 @@ handle_in( clientinfo = #{clientid := ClientId} } ) -> - case emqx_sn_registry:register_topic(Registry, ClientId, TopicName) of + case emqx_mqttsn_registry:register_topic(Registry, ClientId, TopicName) of TopicId when is_integer(TopicId) -> ?SLOG(debug, #{ msg => "registered_topic_name", @@ -718,7 +718,7 @@ handle_in(PubPkt = ?SN_PUBLISH_MSG(_Flags, TopicId0, MsgId, _Data), Channel) -> Id end, case - emqx_misc:pipeline( + emqx_utils:pipeline( [ fun check_qos3_enable/2, fun preproc_pub_pkt/2, @@ -778,7 +778,7 @@ handle_in( {ok, Channel} end; ?SN_RC_INVALID_TOPIC_ID -> - case emqx_sn_registry:lookup_topic(Registry, ClientId, TopicId) of + case emqx_mqttsn_registry:lookup_topic(Registry, ClientId, TopicId) of undefined -> {ok, Channel}; TopicName -> @@ -877,7 +877,7 @@ handle_in( end; handle_in(SubPkt = ?SN_SUBSCRIBE_MSG(_, MsgId, _), Channel) -> case - emqx_misc:pipeline( + emqx_utils:pipeline( [ fun preproc_subs_type/2, fun check_subscribe_authz/2, @@ -911,7 +911,7 @@ handle_in( Channel ) -> case - emqx_misc:pipeline( + emqx_utils:pipeline( [ fun preproc_unsub_type/2, fun run_client_unsub_hook/2, @@ -1093,7 +1093,7 @@ convert_topic_id_to_name( clientinfo = #{clientid := ClientId} } ) -> - case emqx_sn_registry:lookup_topic(Registry, ClientId, TopicId) of + case emqx_mqttsn_registry:lookup_topic(Registry, ClientId, TopicId) of undefined -> {error, ?SN_RC_INVALID_TOPIC_ID}; TopicName -> @@ -1202,7 +1202,7 @@ preproc_subs_type( %% If the gateway is able accept the subscription, %% it assigns a topic id to the received topic name %% and returns it within a SUBACK message - case emqx_sn_registry:register_topic(Registry, ClientId, TopicName) of + case emqx_mqttsn_registry:register_topic(Registry, ClientId, TopicName) of {error, too_large} -> {error, ?SN_RC2_EXCEED_LIMITATION}; {error, wildcard_topic} -> @@ -1228,7 +1228,7 @@ preproc_subs_type( } ) -> case - emqx_sn_registry:lookup_topic( + emqx_mqttsn_registry:lookup_topic( Registry, ClientId, TopicId @@ -1344,7 +1344,7 @@ preproc_unsub_type( } ) -> case - emqx_sn_registry:lookup_topic( + emqx_mqttsn_registry:lookup_topic( Registry, ClientId, TopicId @@ -1765,7 +1765,7 @@ message_to_packet( ?QOS_0 -> 0; _ -> MsgId end, - case emqx_sn_registry:lookup_topic_id(Registry, ClientId, Topic) of + case emqx_mqttsn_registry:lookup_topic_id(Registry, ClientId, Topic) of {predef, PredefTopicId} -> Flags = #mqtt_sn_flags{qos = QoS, topic_id_type = ?SN_PREDEFINED_TOPIC}, ?SN_PUBLISH_MSG(Flags, PredefTopicId, NMsgId, Payload); @@ -1823,7 +1823,7 @@ handle_call( ) -> ok = emqx_session:takeover(Session), %% TODO: Should not drain deliver here (side effect) - Delivers = emqx_misc:drain_deliver(), + Delivers = emqx_utils:drain_deliver(), AllPendings = lists:append(Delivers, Pendings), shutdown_and_reply(takenover, AllPendings, Channel); %handle_call(list_authz_cache, _From, Channel) -> @@ -1932,9 +1932,9 @@ ensure_registered_topic_name( Channel = #channel{registry = Registry} ) -> ClientId = clientid(Channel), - case emqx_sn_registry:lookup_topic_id(Registry, ClientId, TopicName) of + case emqx_mqttsn_registry:lookup_topic_id(Registry, ClientId, TopicName) of undefined -> - case emqx_sn_registry:register_topic(Registry, ClientId, TopicName) of + case emqx_mqttsn_registry:register_topic(Registry, ClientId, TopicName) of {error, Reason} -> {error, Reason}; TopicId -> {ok, TopicId} end; @@ -2247,7 +2247,7 @@ ensure_register_timer(Channel) -> ensure_register_timer(RetryTimes, Channel = #channel{timers = Timers}) -> Msg = maps:get(register_timer, ?TIMER_TABLE), - TRef = emqx_misc:start_timer(?REGISTER_TIMEOUT, {Msg, RetryTimes}), + TRef = emqx_utils:start_timer(?REGISTER_TIMEOUT, {Msg, RetryTimes}), Channel#channel{timers = Timers#{register_timer => TRef}}. cancel_timer(Name, Channel = #channel{timers = Timers}) -> @@ -2255,7 +2255,7 @@ cancel_timer(Name, Channel = #channel{timers = Timers}) -> undefined -> Channel; TRef -> - emqx_misc:cancel_timer(TRef), + emqx_utils:cancel_timer(TRef), Channel#channel{timers = maps:without([Name], Timers)} end. @@ -2270,7 +2270,7 @@ ensure_timer(Name, Channel = #channel{timers = Timers}) -> ensure_timer(Name, Time, Channel = #channel{timers = Timers}) -> Msg = maps:get(Name, ?TIMER_TABLE), - TRef = emqx_misc:start_timer(Time, Msg), + TRef = emqx_utils:start_timer(Time, Msg), Channel#channel{timers = Timers#{Name => TRef}}. reset_timer(Name, Channel) -> diff --git a/apps/emqx_gateway/src/mqttsn/emqx_sn_frame.erl b/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_frame.erl similarity index 96% rename from apps/emqx_gateway/src/mqttsn/emqx_sn_frame.erl rename to apps/emqx_gateway_mqttsn/src/emqx_mqttsn_frame.erl index 39bd9e889..3be2f1dc2 100644 --- a/apps/emqx_gateway/src/mqttsn/emqx_sn_frame.erl +++ b/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_frame.erl @@ -16,11 +16,11 @@ %%-------------------------------------------------------------------- %% @doc The frame parser for MQTT-SN protocol --module(emqx_sn_frame). +-module(emqx_mqttsn_frame). -behaviour(emqx_gateway_frame). --include("src/mqttsn/include/emqx_sn.hrl"). +-include("emqx_mqttsn.hrl"). -export([ initial_parse_state/1, @@ -58,10 +58,10 @@ serialize_opts() -> %% Parse MQTT-SN Message %%-------------------------------------------------------------------- -parse(<<16#01:?byte, Len:?short, Type:?byte, Var/binary>>, _State) -> - {ok, parse(Type, Len - 4, Var), <<>>, _State}; -parse(<>, _State) -> - {ok, parse(Type, Len - 2, Var), <<>>, _State}. +parse(<<16#01:?byte, Len:?short, Type:?byte, Var/binary>>, State) -> + {ok, parse(Type, Len - 4, Var), <<>>, State}; +parse(<>, State) -> + {ok, parse(Type, Len - 2, Var), <<>>, State}. parse(Type, Len, Var) when Len =:= size(Var) -> #mqtt_sn_message{type = Type, variable = parse_var(Type, Var)}; @@ -160,9 +160,11 @@ parse_topic(2#11, Topic) -> Topic. serialize_pkt(#mqtt_sn_message{type = Type, variable = Var}, Opts) -> VarBin = serialize(Type, Var, Opts), VarLen = size(VarBin), - if - VarLen < 254 -> <<(VarLen + 2), Type, VarBin/binary>>; - true -> <<16#01, (VarLen + 4):?short, Type, VarBin/binary>> + case VarLen < 254 of + true -> + <<(VarLen + 2), Type, VarBin/binary>>; + false -> + <<16#01, (VarLen + 4):?short, Type, VarBin/binary>> end. serialize(?SN_ADVERTISE, {GwId, Duration}, _Opts) -> @@ -438,7 +440,7 @@ format(?SN_DISCONNECT_MSG(Duration)) -> format(#mqtt_sn_message{type = Type, variable = Var}) -> io_lib:format( "mqtt_sn_message(type=~s, Var=~w)", - [emqx_sn_frame:message_type(Type), Var] + [emqx_mqttsn_frame:message_type(Type), Var] ). is_message(#mqtt_sn_message{type = Type}) when diff --git a/apps/emqx_gateway/src/mqttsn/emqx_sn_registry.erl b/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_registry.erl similarity index 90% rename from apps/emqx_gateway/src/mqttsn/emqx_sn_registry.erl rename to apps/emqx_gateway_mqttsn/src/emqx_mqttsn_registry.erl index 689aab8ce..9db355a9b 100644 --- a/apps/emqx_gateway/src/mqttsn/emqx_sn_registry.erl +++ b/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_registry.erl @@ -15,13 +15,11 @@ %%-------------------------------------------------------------------- %% @doc The MQTT-SN Topic Registry -%% -%% XXX: --module(emqx_sn_registry). +-module(emqx_mqttsn_registry). -behaviour(gen_server). --include("src/mqttsn/include/emqx_sn.hrl"). +-include("emqx_mqttsn.hrl"). -include_lib("emqx/include/logger.hrl"). -export([start_link/2]). @@ -53,11 +51,11 @@ -export([lookup_name/1]). --define(SN_SHARD, emqx_sn_shard). +-define(SN_SHARD, emqx_mqttsn_shard). -record(state, {tabname, max_predef_topic_id = 0}). --record(emqx_sn_registry, {key, value}). +-record(emqx_mqttsn_registry, {key, value}). -type registry() :: {Tab :: atom(), RegistryPid :: pid()}. @@ -126,7 +124,7 @@ lookup_name(Pid) -> %%----------------------------------------------------------------------------- name(InstaId) -> - list_to_atom(lists:concat([emqx_sn_, InstaId, '_registry'])). + list_to_atom(lists:concat([emqx_mqttsn_, InstaId, '_registry'])). init([InstaId, PredefTopics]) -> %% {predef, TopicId} -> TopicName @@ -136,8 +134,8 @@ init([InstaId, PredefTopics]) -> Tab = name(InstaId), ok = mria:create_table(Tab, [ {storage, ram_copies}, - {record_name, emqx_sn_registry}, - {attributes, record_info(fields, emqx_sn_registry)}, + {record_name, emqx_mqttsn_registry}, + {attributes, record_info(fields, emqx_mqttsn_registry)}, {storage_properties, [{ets, [{read_concurrency, true}]}]}, {rlog_shard, ?SN_SHARD} ]), @@ -145,17 +143,17 @@ init([InstaId, PredefTopics]) -> MaxPredefId = lists:foldl( fun(#{id := TopicId, topic := TopicName0}, AccId) -> TopicName = iolist_to_binary(TopicName0), - mria:dirty_write(Tab, #emqx_sn_registry{ + mria:dirty_write(Tab, #emqx_mqttsn_registry{ key = {predef, TopicId}, value = TopicName }), - mria:dirty_write(Tab, #emqx_sn_registry{ + mria:dirty_write(Tab, #emqx_mqttsn_registry{ key = {predef, TopicName}, value = TopicId }), - if - TopicId > AccId -> TopicId; - true -> AccId + case TopicId > AccId of + true -> TopicId; + false -> AccId end end, 0, @@ -193,7 +191,7 @@ handle_call( handle_call({unregister, ClientId}, _From, State = #state{tabname = Tab}) -> Registry = mnesia:dirty_match_object( Tab, - {emqx_sn_registry, {ClientId, '_'}, '_'} + {emqx_mqttsn_registry, {ClientId, '_'}, '_'} ), lists:foreach( fun(R) -> @@ -234,7 +232,7 @@ code_change(_OldVsn, State, _Extra) -> do_register(Tab, ClientId, TopicId, TopicName) -> mnesia:write( Tab, - #emqx_sn_registry{ + #emqx_mqttsn_registry{ key = {ClientId, next_topic_id}, value = TopicId + 1 }, @@ -242,7 +240,7 @@ do_register(Tab, ClientId, TopicId, TopicName) -> ), mnesia:write( Tab, - #emqx_sn_registry{ + #emqx_mqttsn_registry{ key = {ClientId, TopicName}, value = TopicId }, @@ -250,7 +248,7 @@ do_register(Tab, ClientId, TopicId, TopicName) -> ), mnesia:write( Tab, - #emqx_sn_registry{ + #emqx_mqttsn_registry{ key = {ClientId, TopicId}, value = TopicName }, @@ -261,6 +259,6 @@ do_register(Tab, ClientId, TopicId, TopicName) -> next_topic_id(Tab, PredefId, ClientId) -> case mnesia:dirty_read(Tab, {ClientId, next_topic_id}) of - [#emqx_sn_registry{value = Id}] -> Id; + [#emqx_mqttsn_registry{value = Id}] -> Id; [] -> PredefId + 1 end. diff --git a/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_schema.erl b/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_schema.erl new file mode 100644 index 000000000..cb33cbe95 --- /dev/null +++ b/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_schema.erl @@ -0,0 +1,107 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_mqttsn_schema). + +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("typerefl/include/types.hrl"). + +%% config schema provides +-export([fields/1, desc/1]). + +fields(mqttsn) -> + [ + {gateway_id, + sc( + integer(), + #{ + default => 1, + required => true, + desc => ?DESC(mqttsn_gateway_id) + } + )}, + {broadcast, + sc( + boolean(), + #{ + default => false, + desc => ?DESC(mqttsn_broadcast) + } + )}, + %% TODO: rename + {enable_qos3, + sc( + boolean(), + #{ + default => true, + desc => ?DESC(mqttsn_enable_qos3) + } + )}, + {subs_resume, + sc( + boolean(), + #{ + default => false, + desc => ?DESC(mqttsn_subs_resume) + } + )}, + {predefined, + sc( + hoconsc:array(ref(mqttsn_predefined)), + #{ + default => [], + required => {false, recursively}, + desc => ?DESC(mqttsn_predefined) + } + )}, + {mountpoint, emqx_gateway_schema:mountpoint()}, + {listeners, sc(ref(emqx_gateway_schema, udp_listeners), #{desc => ?DESC(udp_listeners)})} + ] ++ emqx_gateway_schema:gateway_common_options(); +fields(mqttsn_predefined) -> + [ + {id, + sc(integer(), #{ + required => true, + desc => ?DESC(mqttsn_predefined_id) + })}, + + {topic, + sc(binary(), #{ + required => true, + desc => ?DESC(mqttsn_predefined_topic) + })} + ]. + +desc(mqttsn) -> + "The MQTT-SN (MQTT for Sensor Networks) protocol gateway."; +desc(mqttsn_predefined) -> + "The pre-defined topic name corresponding to the pre-defined topic\n" + "ID of N.\n\n" + "Note: the pre-defined topic ID of 0 is reserved."; +desc(_) -> + undefined. + +%%-------------------------------------------------------------------- +%% internal functions + +sc(Type, Meta) -> + hoconsc:mk(Type, Meta). + +ref(StructName) -> + ref(?MODULE, StructName). + +ref(Mod, Field) -> + hoconsc:ref(Mod, Field). diff --git a/apps/emqx_gateway/test/broadcast_test.py b/apps/emqx_gateway_mqttsn/test/broadcast_test.py similarity index 100% rename from apps/emqx_gateway/test/broadcast_test.py rename to apps/emqx_gateway_mqttsn/test/broadcast_test.py diff --git a/apps/emqx_gateway/test/emqx_sn_frame_SUITE.erl b/apps/emqx_gateway_mqttsn/test/emqx_sn_frame_SUITE.erl similarity index 97% rename from apps/emqx_gateway/test/emqx_sn_frame_SUITE.erl rename to apps/emqx_gateway_mqttsn/test/emqx_sn_frame_SUITE.erl index aa3fed707..86cc0cf7e 100644 --- a/apps/emqx_gateway/test/emqx_sn_frame_SUITE.erl +++ b/apps/emqx_gateway_mqttsn/test/emqx_sn_frame_SUITE.erl @@ -19,7 +19,7 @@ -compile(export_all). -compile(nowarn_export_all). --include("src/mqttsn/include/emqx_sn.hrl"). +-include("emqx_mqttsn.hrl"). -include_lib("eunit/include/eunit.hrl"). %%-------------------------------------------------------------------- @@ -30,11 +30,11 @@ all() -> emqx_common_test_helpers:all(?MODULE). parse(D) -> - {ok, P, _Rest, _State} = emqx_sn_frame:parse(D, #{}), + {ok, P, _Rest, _State} = emqx_mqttsn_frame:parse(D, #{}), P. serialize_pkt(P) -> - emqx_sn_frame:serialize_pkt(P, #{}). + emqx_mqttsn_frame:serialize_pkt(P, #{}). %%-------------------------------------------------------------------- %% Test cases diff --git a/apps/emqx_gateway/test/emqx_sn_protocol_SUITE.erl b/apps/emqx_gateway_mqttsn/test/emqx_sn_protocol_SUITE.erl similarity index 99% rename from apps/emqx_gateway/test/emqx_sn_protocol_SUITE.erl rename to apps/emqx_gateway_mqttsn/test/emqx_sn_protocol_SUITE.erl index adc1e7382..04b1b5fb2 100644 --- a/apps/emqx_gateway/test/emqx_sn_protocol_SUITE.erl +++ b/apps/emqx_gateway_mqttsn/test/emqx_sn_protocol_SUITE.erl @@ -27,7 +27,7 @@ ] ). --include("src/mqttsn/include/emqx_sn.hrl"). +-include("emqx_mqttsn.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). @@ -97,6 +97,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> + application:load(emqx_gateway_mqttsn), ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_authn, emqx_gateway]), Config. @@ -270,7 +271,7 @@ t_subscribe_case03(_) -> %% In this case We use predefined topic name to register and subscribe, %% and expect to receive the corresponding predefined topic id but not a new %% generated topic id from broker. We design this case to illustrate -%% emqx_sn_gateway's compatibility of dealing with predefined and normal +%% MQTT-SN Gateway's compatibility of dealing with predefined and normal %% topics. %% %% Once we give more restrictions to different topic id type, this case diff --git a/apps/emqx_gateway/test/emqx_sn_registry_SUITE.erl b/apps/emqx_gateway_mqttsn/test/emqx_sn_registry_SUITE.erl similarity index 98% rename from apps/emqx_gateway/test/emqx_sn_registry_SUITE.erl rename to apps/emqx_gateway_mqttsn/test/emqx_sn_registry_SUITE.erl index 739255e71..4d89a802d 100644 --- a/apps/emqx_gateway/test/emqx_sn_registry_SUITE.erl +++ b/apps/emqx_gateway_mqttsn/test/emqx_sn_registry_SUITE.erl @@ -21,7 +21,7 @@ -include_lib("eunit/include/eunit.hrl"). --define(REGISTRY, emqx_sn_registry). +-define(REGISTRY, emqx_mqttsn_registry). -define(MAX_PREDEF_ID, 2). -define(PREDEF_TOPICS, [ #{id => 1, topic => <<"/predefined/topic/name/hello">>}, @@ -66,7 +66,7 @@ t_register(Config) -> ?assertEqual(<<"Topic2">>, ?REGISTRY:lookup_topic(Reg, <<"ClientId">>, ?MAX_PREDEF_ID + 2)), ?assertEqual(?MAX_PREDEF_ID + 1, ?REGISTRY:lookup_topic_id(Reg, <<"ClientId">>, <<"Topic1">>)), ?assertEqual(?MAX_PREDEF_ID + 2, ?REGISTRY:lookup_topic_id(Reg, <<"ClientId">>, <<"Topic2">>)), - emqx_sn_registry:unregister_topic(Reg, <<"ClientId">>), + emqx_mqttsn_registry:unregister_topic(Reg, <<"ClientId">>), ?assertEqual(undefined, ?REGISTRY:lookup_topic(Reg, <<"ClientId">>, ?MAX_PREDEF_ID + 1)), ?assertEqual(undefined, ?REGISTRY:lookup_topic(Reg, <<"ClientId">>, ?MAX_PREDEF_ID + 2)), ?assertEqual(undefined, ?REGISTRY:lookup_topic_id(Reg, <<"ClientId">>, <<"Topic1">>)), diff --git a/apps/emqx_gateway/test/intergration_test/Makefile b/apps/emqx_gateway_mqttsn/test/intergration_test/Makefile similarity index 100% rename from apps/emqx_gateway/test/intergration_test/Makefile rename to apps/emqx_gateway_mqttsn/test/intergration_test/Makefile diff --git a/apps/emqx_gateway/test/intergration_test/README.md b/apps/emqx_gateway_mqttsn/test/intergration_test/README.md similarity index 100% rename from apps/emqx_gateway/test/intergration_test/README.md rename to apps/emqx_gateway_mqttsn/test/intergration_test/README.md diff --git a/apps/emqx_gateway/test/intergration_test/add_emqx_sn_to_project.py b/apps/emqx_gateway_mqttsn/test/intergration_test/add_emqx_sn_to_project.py similarity index 100% rename from apps/emqx_gateway/test/intergration_test/add_emqx_sn_to_project.py rename to apps/emqx_gateway_mqttsn/test/intergration_test/add_emqx_sn_to_project.py diff --git a/apps/emqx_gateway/test/intergration_test/client/case1_qos0pub.c b/apps/emqx_gateway_mqttsn/test/intergration_test/client/case1_qos0pub.c similarity index 100% rename from apps/emqx_gateway/test/intergration_test/client/case1_qos0pub.c rename to apps/emqx_gateway_mqttsn/test/intergration_test/client/case1_qos0pub.c diff --git a/apps/emqx_gateway/test/intergration_test/client/case1_qos0sub.c b/apps/emqx_gateway_mqttsn/test/intergration_test/client/case1_qos0sub.c similarity index 100% rename from apps/emqx_gateway/test/intergration_test/client/case1_qos0sub.c rename to apps/emqx_gateway_mqttsn/test/intergration_test/client/case1_qos0sub.c diff --git a/apps/emqx_gateway/test/intergration_test/client/case2_qos0pub.c b/apps/emqx_gateway_mqttsn/test/intergration_test/client/case2_qos0pub.c similarity index 100% rename from apps/emqx_gateway/test/intergration_test/client/case2_qos0pub.c rename to apps/emqx_gateway_mqttsn/test/intergration_test/client/case2_qos0pub.c diff --git a/apps/emqx_gateway/test/intergration_test/client/case2_qos0sub.c b/apps/emqx_gateway_mqttsn/test/intergration_test/client/case2_qos0sub.c similarity index 100% rename from apps/emqx_gateway/test/intergration_test/client/case2_qos0sub.c rename to apps/emqx_gateway_mqttsn/test/intergration_test/client/case2_qos0sub.c diff --git a/apps/emqx_gateway/test/intergration_test/client/case3_qos0pub.c b/apps/emqx_gateway_mqttsn/test/intergration_test/client/case3_qos0pub.c similarity index 100% rename from apps/emqx_gateway/test/intergration_test/client/case3_qos0pub.c rename to apps/emqx_gateway_mqttsn/test/intergration_test/client/case3_qos0pub.c diff --git a/apps/emqx_gateway/test/intergration_test/client/case3_qos0sub.c b/apps/emqx_gateway_mqttsn/test/intergration_test/client/case3_qos0sub.c similarity index 100% rename from apps/emqx_gateway/test/intergration_test/client/case3_qos0sub.c rename to apps/emqx_gateway_mqttsn/test/intergration_test/client/case3_qos0sub.c diff --git a/apps/emqx_gateway/test/intergration_test/client/case4_qos3pub.c b/apps/emqx_gateway_mqttsn/test/intergration_test/client/case4_qos3pub.c similarity index 100% rename from apps/emqx_gateway/test/intergration_test/client/case4_qos3pub.c rename to apps/emqx_gateway_mqttsn/test/intergration_test/client/case4_qos3pub.c diff --git a/apps/emqx_gateway/test/intergration_test/client/case4_qos3sub.c b/apps/emqx_gateway_mqttsn/test/intergration_test/client/case4_qos3sub.c similarity index 100% rename from apps/emqx_gateway/test/intergration_test/client/case4_qos3sub.c rename to apps/emqx_gateway_mqttsn/test/intergration_test/client/case4_qos3sub.c diff --git a/apps/emqx_gateway/test/intergration_test/client/case5_qos3pub.c b/apps/emqx_gateway_mqttsn/test/intergration_test/client/case5_qos3pub.c similarity index 100% rename from apps/emqx_gateway/test/intergration_test/client/case5_qos3pub.c rename to apps/emqx_gateway_mqttsn/test/intergration_test/client/case5_qos3pub.c diff --git a/apps/emqx_gateway/test/intergration_test/client/case5_qos3sub.c b/apps/emqx_gateway_mqttsn/test/intergration_test/client/case5_qos3sub.c similarity index 100% rename from apps/emqx_gateway/test/intergration_test/client/case5_qos3sub.c rename to apps/emqx_gateway_mqttsn/test/intergration_test/client/case5_qos3sub.c diff --git a/apps/emqx_gateway/test/intergration_test/client/case6_sleep.c b/apps/emqx_gateway_mqttsn/test/intergration_test/client/case6_sleep.c similarity index 100% rename from apps/emqx_gateway/test/intergration_test/client/case6_sleep.c rename to apps/emqx_gateway_mqttsn/test/intergration_test/client/case6_sleep.c diff --git a/apps/emqx_gateway/test/intergration_test/client/case7_double_connect.c b/apps/emqx_gateway_mqttsn/test/intergration_test/client/case7_double_connect.c similarity index 100% rename from apps/emqx_gateway/test/intergration_test/client/case7_double_connect.c rename to apps/emqx_gateway_mqttsn/test/intergration_test/client/case7_double_connect.c diff --git a/apps/emqx_gateway/test/intergration_test/client/int_test_result.c b/apps/emqx_gateway_mqttsn/test/intergration_test/client/int_test_result.c similarity index 100% rename from apps/emqx_gateway/test/intergration_test/client/int_test_result.c rename to apps/emqx_gateway_mqttsn/test/intergration_test/client/int_test_result.c diff --git a/apps/emqx_gateway/test/intergration_test/client/int_test_result.h b/apps/emqx_gateway_mqttsn/test/intergration_test/client/int_test_result.h similarity index 100% rename from apps/emqx_gateway/test/intergration_test/client/int_test_result.h rename to apps/emqx_gateway_mqttsn/test/intergration_test/client/int_test_result.h diff --git a/apps/emqx_gateway/test/intergration_test/disable_qos3.py b/apps/emqx_gateway_mqttsn/test/intergration_test/disable_qos3.py similarity index 100% rename from apps/emqx_gateway/test/intergration_test/disable_qos3.py rename to apps/emqx_gateway_mqttsn/test/intergration_test/disable_qos3.py diff --git a/apps/emqx_gateway/test/intergration_test/enable_qos3.py b/apps/emqx_gateway_mqttsn/test/intergration_test/enable_qos3.py similarity index 100% rename from apps/emqx_gateway/test/intergration_test/enable_qos3.py rename to apps/emqx_gateway_mqttsn/test/intergration_test/enable_qos3.py diff --git a/apps/emqx_gateway/test/props/emqx_sn_proper_types.erl b/apps/emqx_gateway_mqttsn/test/props/emqx_sn_proper_types.erl similarity index 99% rename from apps/emqx_gateway/test/props/emqx_sn_proper_types.erl rename to apps/emqx_gateway_mqttsn/test/props/emqx_sn_proper_types.erl index 2869a8958..70b13ef8f 100644 --- a/apps/emqx_gateway/test/props/emqx_sn_proper_types.erl +++ b/apps/emqx_gateway_mqttsn/test/props/emqx_sn_proper_types.erl @@ -16,7 +16,7 @@ -module(emqx_sn_proper_types). --include("src/mqttsn/include/emqx_sn.hrl"). +-include("emqx_mqttsn.hrl"). -include_lib("proper/include/proper.hrl"). -compile({no_auto_import, [register/1]}). diff --git a/apps/emqx_gateway/test/props/prop_emqx_sn_frame.erl b/apps/emqx_gateway_mqttsn/test/props/prop_emqx_sn_frame.erl similarity index 94% rename from apps/emqx_gateway/test/props/prop_emqx_sn_frame.erl rename to apps/emqx_gateway_mqttsn/test/props/prop_emqx_sn_frame.erl index f2dfbb8e9..0abe2485c 100644 --- a/apps/emqx_gateway/test/props/prop_emqx_sn_frame.erl +++ b/apps/emqx_gateway_mqttsn/test/props/prop_emqx_sn_frame.erl @@ -16,7 +16,7 @@ -module(prop_emqx_sn_frame). --include("src/mqttsn/include/emqx_sn.hrl"). +-include("emqx_mqttsn.hrl"). -include_lib("proper/include/proper.hrl"). -compile({no_auto_import, [register/1]}). @@ -32,11 +32,11 @@ ). parse(D) -> - {ok, P, _Rest, _State} = emqx_sn_frame:parse(D, #{}), + {ok, P, _Rest, _State} = emqx_mqttsn_frame:parse(D, #{}), P. serialize(P) -> - emqx_sn_frame:serialize_pkt(P, #{}). + emqx_mqttsn_frame:serialize_pkt(P, #{}). %%-------------------------------------------------------------------- %% Properties diff --git a/apps/emqx_gateway_stomp/.gitignore b/apps/emqx_gateway_stomp/.gitignore new file mode 100644 index 000000000..f1c455451 --- /dev/null +++ b/apps/emqx_gateway_stomp/.gitignore @@ -0,0 +1,19 @@ +.rebar3 +_* +.eunit +*.o +*.beam +*.plt +*.swp +*.swo +.erlang.cookie +ebin +log +erl_crash.dump +.rebar +logs +_build +.idea +*.iml +rebar3.crashdump +*~ diff --git a/apps/emqx_gateway_stomp/README.md b/apps/emqx_gateway_stomp/README.md new file mode 100644 index 000000000..0c41ff520 --- /dev/null +++ b/apps/emqx_gateway_stomp/README.md @@ -0,0 +1,31 @@ +# emqx_stomp + +The Stomp Gateway is based on the +[Stomp v1.2](https://stomp.github.io/stomp-specification-1.2.html) and is +compatible with the Stomp v1.0 and v1.1 specification. + +## Quick Start + +In EMQX 5.0, Stomp gateway can be configured and enabled through the Dashboard. + +It can also be enabled via the HTTP API or emqx.conf, e.g. In emqx.conf: + +```properties +gateway.stomp { + + mountpoint = "stomp/" + + listeners.tcp.default { + bind = 61613 + acceptors = 16 + max_connections = 1024000 + max_conn_rate = 1000 + } +} +``` + +> Note: +> Configuring the gateway via emqx.conf requires changes on a per-node basis, +> but configuring it via Dashboard or the HTTP API will take effect across the cluster. + +More documentations: [Stomp Gateway](https://www.emqx.io/docs/en/v5.0/gateway/stomp.html) diff --git a/apps/emqx_gateway/src/stomp/include/emqx_stomp.hrl b/apps/emqx_gateway_stomp/include/emqx_stomp.hrl similarity index 100% rename from apps/emqx_gateway/src/stomp/include/emqx_stomp.hrl rename to apps/emqx_gateway_stomp/include/emqx_stomp.hrl diff --git a/apps/emqx_gateway_stomp/rebar.config b/apps/emqx_gateway_stomp/rebar.config new file mode 100644 index 000000000..cfeb0a195 --- /dev/null +++ b/apps/emqx_gateway_stomp/rebar.config @@ -0,0 +1,6 @@ +{erl_opts, [debug_info]}. +{deps, [ + {emqx, {path, "../../apps/emqx"}}, + {emqx_utils, {path, "../emqx_utils"}}, + {emqx_gateway, {path, "../../apps/emqx_gateway"}} +]}. diff --git a/apps/emqx_gateway_stomp/src/emqx_gateway_stomp.app.src b/apps/emqx_gateway_stomp/src/emqx_gateway_stomp.app.src new file mode 100644 index 000000000..38da1e18b --- /dev/null +++ b/apps/emqx_gateway_stomp/src/emqx_gateway_stomp.app.src @@ -0,0 +1,10 @@ +{application, emqx_gateway_stomp, [ + {description, "Stomp Gateway"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [kernel, stdlib, emqx, emqx_gateway]}, + {env, []}, + {modules, []}, + {licenses, ["Apache 2.0"]}, + {links, []} +]}. diff --git a/apps/emqx_gateway/src/stomp/emqx_stomp_impl.erl b/apps/emqx_gateway_stomp/src/emqx_gateway_stomp.erl similarity index 83% rename from apps/emqx_gateway/src/stomp/emqx_stomp_impl.erl rename to apps/emqx_gateway_stomp/src/emqx_gateway_stomp.erl index c2907c262..b8c2f0166 100644 --- a/apps/emqx_gateway/src/stomp/emqx_stomp_impl.erl +++ b/apps/emqx_gateway_stomp/src/emqx_gateway_stomp.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -14,13 +14,29 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_stomp_impl). - --behaviour(emqx_gateway_impl). +%% @doc The Stomp Gateway implement +-module(emqx_gateway_stomp). -include_lib("emqx/include/logger.hrl"). -include_lib("emqx_gateway/include/emqx_gateway.hrl"). +%% define a gateway named stomp +-gateway(#{ + name => stomp, + callback_module => ?MODULE, + config_schema_module => emqx_stomp_schema +}). + +%% callback_module must implement the emqx_gateway_impl behaviour +-behaviour(emqx_gateway_impl). + +%% callback for emqx_gateway_impl +-export([ + on_gateway_load/2, + on_gateway_update/3, + on_gateway_unload/2 +]). + -import( emqx_gateway_utils, [ @@ -30,33 +46,8 @@ ] ). -%% APIs --export([ - reg/0, - unreg/0 -]). - --export([ - on_gateway_load/2, - on_gateway_update/3, - on_gateway_unload/2 -]). - %%-------------------------------------------------------------------- -%% APIs -%%-------------------------------------------------------------------- - --spec reg() -> ok | {error, any()}. -reg() -> - RegistryOptions = [{cbkmod, ?MODULE}], - emqx_gateway_registry:reg(stomp, RegistryOptions). - --spec unreg() -> ok | {error, any()}. -unreg() -> - emqx_gateway_registry:unreg(stomp). - -%%-------------------------------------------------------------------- -%% emqx_gateway_registry callbacks +%% emqx_gateway_impl callbacks %%-------------------------------------------------------------------- on_gateway_load( diff --git a/apps/emqx_gateway/src/stomp/emqx_stomp_channel.erl b/apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl similarity index 99% rename from apps/emqx_gateway/src/stomp/emqx_stomp_channel.erl rename to apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl index b95bb827c..316432dea 100644 --- a/apps/emqx_gateway/src/stomp/emqx_stomp_channel.erl +++ b/apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl @@ -18,7 +18,7 @@ -behaviour(emqx_gateway_channel). --include("src/stomp/include/emqx_stomp.hrl"). +-include("emqx_stomp.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/logger.hrl"). @@ -252,7 +252,7 @@ enrich_clientinfo( feedvar(Override, Packet, ConnInfo, ClientInfo0), ClientInfo0 ), - {ok, NPacket, NClientInfo} = emqx_misc:pipeline( + {ok, NPacket, NClientInfo} = emqx_utils:pipeline( [ fun maybe_assign_clientid/2, fun parse_heartbeat/2, @@ -416,7 +416,7 @@ handle_in( {error, unexpected_connect, Channel}; handle_in(Packet = ?PACKET(?CMD_CONNECT), Channel) -> case - emqx_misc:pipeline( + emqx_utils:pipeline( [ fun enrich_conninfo/2, fun negotiate_version/2, @@ -474,7 +474,7 @@ handle_in( Topic = header(<<"destination">>, Headers), Ack = header(<<"ack">>, Headers, <<"auto">>), case - emqx_misc:pipeline( + emqx_utils:pipeline( [ fun parse_topic_filter/2, fun check_subscribed_status/2, @@ -796,7 +796,7 @@ handle_call( reply({error, no_subid}, Channel); SubId -> case - emqx_misc:pipeline( + emqx_utils:pipeline( [ fun parse_topic_filter/2, fun check_subscribed_status/2 @@ -869,7 +869,7 @@ handle_call(discard, _From, Channel) -> % pendings = Pendings}) -> % ok = emqx_session:takeover(Session), % %% TODO: Should not drain deliver here (side effect) -% Delivers = emqx_misc:drain_deliver(), +% Delivers = emqx_utils:drain_deliver(), % AllPendings = lists:append(Delivers, Pendings), % shutdown_and_reply(takenover, AllPendings, Channel); @@ -1289,7 +1289,7 @@ ensure_timer(Name, Channel = #channel{timers = Timers}) -> ensure_timer(Name, Time, Channel = #channel{timers = Timers}) -> Msg = maps:get(Name, ?TIMER_TABLE), - TRef = emqx_misc:start_timer(Time, Msg), + TRef = emqx_utils:start_timer(Time, Msg), Channel#channel{timers = Timers#{Name => TRef}}. reset_timer(Name, Channel) -> diff --git a/apps/emqx_gateway/src/stomp/emqx_stomp_frame.erl b/apps/emqx_gateway_stomp/src/emqx_stomp_frame.erl similarity index 99% rename from apps/emqx_gateway/src/stomp/emqx_stomp_frame.erl rename to apps/emqx_gateway_stomp/src/emqx_stomp_frame.erl index 47e045412..4913d6b2a 100644 --- a/apps/emqx_gateway/src/stomp/emqx_stomp_frame.erl +++ b/apps/emqx_gateway_stomp/src/emqx_stomp_frame.erl @@ -70,7 +70,7 @@ -behaviour(emqx_gateway_frame). --include("src/stomp/include/emqx_stomp.hrl"). +-include("emqx_stomp.hrl"). -export([ initial_parse_state/1, diff --git a/apps/emqx_gateway/src/stomp/emqx_stomp_heartbeat.erl b/apps/emqx_gateway_stomp/src/emqx_stomp_heartbeat.erl similarity index 97% rename from apps/emqx_gateway/src/stomp/emqx_stomp_heartbeat.erl rename to apps/emqx_gateway_stomp/src/emqx_stomp_heartbeat.erl index 88720c513..2e4239bdc 100644 --- a/apps/emqx_gateway/src/stomp/emqx_stomp_heartbeat.erl +++ b/apps/emqx_gateway_stomp/src/emqx_stomp_heartbeat.erl @@ -17,7 +17,7 @@ %% @doc Stomp heartbeat. -module(emqx_stomp_heartbeat). --include("src/stomp/include/emqx_stomp.hrl"). +-include("emqx_stomp.hrl"). -export([ init/1, @@ -36,6 +36,8 @@ outgoing => #heartbeater{} }. +-elvis([{elvis_style, no_if_expression, disable}]). + %%-------------------------------------------------------------------- %% APIs %%-------------------------------------------------------------------- diff --git a/apps/emqx_gateway_stomp/src/emqx_stomp_schema.erl b/apps/emqx_gateway_stomp/src/emqx_stomp_schema.erl new file mode 100644 index 000000000..b1c6a92e2 --- /dev/null +++ b/apps/emqx_gateway_stomp/src/emqx_stomp_schema.erl @@ -0,0 +1,80 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_stomp_schema). + +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("typerefl/include/types.hrl"). + +%% config schema provides +-export([fields/1, desc/1]). + +fields(stomp) -> + [ + {frame, sc(ref(stomp_frame))}, + {mountpoint, emqx_gateway_schema:mountpoint()}, + {listeners, sc(ref(emqx_gateway_schema, tcp_listeners), #{desc => ?DESC(tcp_listeners)})} + ] ++ emqx_gateway_schema:gateway_common_options(); +fields(stomp_frame) -> + [ + {max_headers, + sc( + non_neg_integer(), + #{ + default => 10, + desc => ?DESC(stom_frame_max_headers) + } + )}, + {max_headers_length, + sc( + non_neg_integer(), + #{ + default => 1024, + desc => ?DESC(stomp_frame_max_headers_length) + } + )}, + {max_body_length, + sc( + integer(), + #{ + default => 65536, + desc => ?DESC(stom_frame_max_body_length) + } + )} + ]. + +desc(stomp) -> + "The STOMP protocol gateway provides EMQX with the ability to access STOMP\n" + "(Simple (or Streaming) Text Orientated Messaging Protocol) protocol."; +desc(stomp_frame) -> + "Size limits for the STOMP frames."; +desc(_) -> + undefined. + +%%-------------------------------------------------------------------- +%% internal functions + +sc(Type) -> + sc(Type, #{}). + +sc(Type, Meta) -> + hoconsc:mk(Type, Meta). + +ref(StructName) -> + ref(?MODULE, StructName). + +ref(Mod, Field) -> + hoconsc:ref(Mod, Field). diff --git a/apps/emqx_gateway/test/emqx_stomp_SUITE.erl b/apps/emqx_gateway_stomp/test/emqx_stomp_SUITE.erl similarity index 99% rename from apps/emqx_gateway/test/emqx_stomp_SUITE.erl rename to apps/emqx_gateway_stomp/test/emqx_stomp_SUITE.erl index 2cf245ce2..4323cf32f 100644 --- a/apps/emqx_gateway/test/emqx_stomp_SUITE.erl +++ b/apps/emqx_gateway_stomp/test/emqx_stomp_SUITE.erl @@ -17,7 +17,7 @@ -module(emqx_stomp_SUITE). -include_lib("eunit/include/eunit.hrl"). --include("src/stomp/include/emqx_stomp.hrl"). +-include("emqx_stomp.hrl"). -compile(export_all). -compile(nowarn_export_all). @@ -53,6 +53,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). %%-------------------------------------------------------------------- init_per_suite(Cfg) -> + application:load(emqx_gateway_stomp), ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), emqx_mgmt_api_test_util:init_suite([emqx_authn, emqx_gateway]), Cfg. diff --git a/apps/emqx_gateway/test/emqx_stomp_heartbeat_SUITE.erl b/apps/emqx_gateway_stomp/test/emqx_stomp_heartbeat_SUITE.erl similarity index 100% rename from apps/emqx_gateway/test/emqx_stomp_heartbeat_SUITE.erl rename to apps/emqx_gateway_stomp/test/emqx_stomp_heartbeat_SUITE.erl diff --git a/apps/emqx_machine/README.md b/apps/emqx_machine/README.md new file mode 100644 index 000000000..a8221ba73 --- /dev/null +++ b/apps/emqx_machine/README.md @@ -0,0 +1,48 @@ +# EMQX machine + +This application manages other OTP applications in EMQX and serves as the entry point when BEAM VM starts up. +It prepares the node before starting mnesia/mria, as well as EMQX business logic. +It keeps track of the business applications storing data in Mnesia, which need to be restarted when the node joins the cluster by registering `ekka` callbacks. +Also it kicks off autoclustering (EMQX cluster discovery) on the core nodes. + +`emqx_machine` application doesn't do much on its own, but it facilitates the environment for running other EMQX applications. + +# Features + +## Global GC + +`emqx_global_gc` is a gen_server that forces garbage collection of all Erlang processes running in the BEAM VM. +This is meant to save the RAM. + +## Restricted shell + +`emqx_restricted_shell` module prevents user from accidentally issuing Erlang shell commands that can stop the remote node. + +## Signal handler + +`emqx_machine_signal_handler` handles POSIX signals sent to BEAM VM process. +It helps to shut down EMQX broker gracefully when it receives `SIGTERM` signal. + +## Cover + +`emqx_cover` is a helper module that helps to collect coverage reports during testing. + +# Limitation + +Currently `emqx_machine` boots the business apps before starting autocluster, so a fresh node joining the cluster actually starts business application twice: first in the singleton mode, and then in clustered mode. + +# Documentation links + +Configuration: [node.global_gc_interval](https://www.emqx.io/docs/en/v5.0/configuration/configuration-manual.html#node-and-cookie) + +# Configurations + +The following application environment variables are used: + +- `emqx_machine.global_gc_interval`: interval at which global GC is run +- `emqx_machine.custom_shard_transports`: contains a map that allows to fine tune transport (`rpc` or `gen_rpc`) used to send Mria transactions from the core node to the replicant +- `emqx_machine.backtrace_depth`: set maximum depth of Erlang stacktraces in crash reports + + +# Contributing +Please see our [contributing.md](../../CONTRIBUTING.md). diff --git a/apps/emqx_machine/rebar.config b/apps/emqx_machine/rebar.config index 9f17b7657..dee2902a5 100644 --- a/apps/emqx_machine/rebar.config +++ b/apps/emqx_machine/rebar.config @@ -1,5 +1,8 @@ %% -*- mode: erlang -*- -{deps, [{emqx, {path, "../emqx"}}]}. +{deps, [ + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}} +]}. {project_plugins, [erlfmt]}. diff --git a/apps/emqx_machine/src/emqx_global_gc.erl b/apps/emqx_machine/src/emqx_global_gc.erl index f17ab2d16..121855e68 100644 --- a/apps/emqx_machine/src/emqx_global_gc.erl +++ b/apps/emqx_machine/src/emqx_global_gc.erl @@ -86,7 +86,7 @@ ensure_timer(State) -> disabled -> State; Interval when is_integer(Interval) -> - TRef = emqx_misc:start_timer(Interval, run), + TRef = emqx_utils:start_timer(Interval, run), State#{timer := TRef} end. diff --git a/apps/emqx_machine/src/emqx_machine.erl b/apps/emqx_machine/src/emqx_machine.erl index 243c4bb8c..6872b150c 100644 --- a/apps/emqx_machine/src/emqx_machine.erl +++ b/apps/emqx_machine/src/emqx_machine.erl @@ -88,7 +88,7 @@ start_sysmon() -> end. node_status() -> - emqx_json:encode(#{ + emqx_utils_json:encode(#{ backend => mria_rlog:backend(), role => mria_rlog:role() }). diff --git a/apps/emqx_management/rebar.config b/apps/emqx_management/rebar.config index 73cbf471f..b2f5a40af 100644 --- a/apps/emqx_management/rebar.config +++ b/apps/emqx_management/rebar.config @@ -1,6 +1,9 @@ %% -*- mode: erlang -*- -{deps, [{emqx, {path, "../emqx"}}]}. +{deps, [ + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}} +]}. {edoc_opts, [{preprocess, true}]}. {erl_opts, [ diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index 9863f5cf6..f423213af 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -2,7 +2,7 @@ {application, emqx_management, [ {description, "EMQX Management API and CLI"}, % strict semver, bump manually! - {vsn, "5.0.17"}, + {vsn, "5.0.19"}, {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel, stdlib, emqx_plugins, minirest, emqx, emqx_ctl]}, diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl index 5ba12646f..0b91817f0 100644 --- a/apps/emqx_management/src/emqx_mgmt.erl +++ b/apps/emqx_management/src/emqx_mgmt.erl @@ -112,8 +112,8 @@ %%-------------------------------------------------------------------- list_nodes() -> - Running = mria:cluster_nodes(running), - Stopped = mria:cluster_nodes(stopped), + Running = emqx:cluster_nodes(running), + Stopped = emqx:cluster_nodes(stopped), DownNodes = lists:map(fun stopped_node_info/1, Stopped), [{Node, Info} || #{node := Node} = Info <- node_info(Running)] ++ DownNodes. @@ -199,7 +199,7 @@ vm_stats() -> %%-------------------------------------------------------------------- list_brokers() -> - Running = mria:running_nodes(), + Running = emqx:running_nodes(), [{Node, Broker} || #{node := Node} = Broker <- broker_info(Running)]. lookup_broker(Node) -> @@ -223,7 +223,7 @@ broker_info(Nodes) -> %%-------------------------------------------------------------------- get_metrics() -> - nodes_info_count([get_metrics(Node) || Node <- mria:running_nodes()]). + nodes_info_count([get_metrics(Node) || Node <- emqx:running_nodes()]). get_metrics(Node) -> unwrap_rpc(emqx_proto_v1:get_metrics(Node)). @@ -238,13 +238,20 @@ get_stats() -> 'subscriptions.shared.count', 'subscriptions.shared.max' ], - CountStats = nodes_info_count([ - begin - Stats = get_stats(Node), - delete_keys(Stats, GlobalStatsKeys) - end - || Node <- mria:running_nodes() - ]), + CountStats = nodes_info_count( + lists:foldl( + fun(Node, Acc) -> + case get_stats(Node) of + {error, _} -> + Acc; + Stats -> + [delete_keys(Stats, GlobalStatsKeys) | Acc] + end + end, + [], + emqx:running_nodes() + ) + ), GlobalStats = maps:with(GlobalStatsKeys, maps:from_list(get_stats(node()))), maps:merge(CountStats, GlobalStats). @@ -275,12 +282,12 @@ nodes_info_count(PropList) -> lookup_client({clientid, ClientId}, FormatFun) -> lists:append([ lookup_client(Node, {clientid, ClientId}, FormatFun) - || Node <- mria:running_nodes() + || Node <- emqx:running_nodes() ]); lookup_client({username, Username}, FormatFun) -> lists:append([ lookup_client(Node, {username, Username}, FormatFun) - || Node <- mria:running_nodes() + || Node <- emqx:running_nodes() ]). lookup_client(Node, Key, FormatFun) -> @@ -307,7 +314,7 @@ kickout_client(ClientId) -> [] -> {error, not_found}; _ -> - Results = [kickout_client(Node, ClientId) || Node <- mria:running_nodes()], + Results = [kickout_client(Node, ClientId) || Node <- emqx:running_nodes()], check_results(Results) end. @@ -322,7 +329,7 @@ list_client_subscriptions(ClientId) -> [] -> {error, not_found}; _ -> - Results = [client_subscriptions(Node, ClientId) || Node <- mria:running_nodes()], + Results = [client_subscriptions(Node, ClientId) || Node <- emqx:running_nodes()], Filter = fun ({error, _}) -> @@ -340,18 +347,18 @@ client_subscriptions(Node, ClientId) -> {Node, unwrap_rpc(emqx_broker_proto_v1:list_client_subscriptions(Node, ClientId))}. clean_authz_cache(ClientId) -> - Results = [clean_authz_cache(Node, ClientId) || Node <- mria:running_nodes()], + Results = [clean_authz_cache(Node, ClientId) || Node <- emqx:running_nodes()], check_results(Results). clean_authz_cache(Node, ClientId) -> unwrap_rpc(emqx_proto_v1:clean_authz_cache(Node, ClientId)). clean_authz_cache_all() -> - Results = [{Node, clean_authz_cache_all(Node)} || Node <- mria:running_nodes()], + Results = [{Node, clean_authz_cache_all(Node)} || Node <- emqx:running_nodes()], wrap_results(Results). clean_pem_cache_all() -> - Results = [{Node, clean_pem_cache_all(Node)} || Node <- mria:running_nodes()], + Results = [{Node, clean_pem_cache_all(Node)} || Node <- emqx:running_nodes()], wrap_results(Results). wrap_results(Results) -> @@ -379,7 +386,7 @@ set_keepalive(_ClientId, _Interval) -> %% @private call_client(ClientId, Req) -> - Results = [call_client(Node, ClientId, Req) || Node <- mria:running_nodes()], + Results = [call_client(Node, ClientId, Req) || Node <- emqx:running_nodes()], Expected = lists:filter( fun ({error, _}) -> false; @@ -428,7 +435,7 @@ list_subscriptions(Node) -> list_subscriptions_via_topic(Topic, FormatFun) -> lists:append([ list_subscriptions_via_topic(Node, Topic, FormatFun) - || Node <- mria:running_nodes() + || Node <- emqx:running_nodes() ]). list_subscriptions_via_topic(Node, Topic, _FormatFun = {M, F}) -> @@ -442,7 +449,7 @@ list_subscriptions_via_topic(Node, Topic, _FormatFun = {M, F}) -> %%-------------------------------------------------------------------- subscribe(ClientId, TopicTables) -> - subscribe(mria:running_nodes(), ClientId, TopicTables). + subscribe(emqx:running_nodes(), ClientId, TopicTables). subscribe([Node | Nodes], ClientId, TopicTables) -> case unwrap_rpc(emqx_management_proto_v3:subscribe(Node, ClientId, TopicTables)) of @@ -467,7 +474,7 @@ publish(Msg) -> -spec unsubscribe(emqx_types:clientid(), emqx_types:topic()) -> {unsubscribe, _} | {error, channel_not_found}. unsubscribe(ClientId, Topic) -> - unsubscribe(mria:running_nodes(), ClientId, Topic). + unsubscribe(emqx:running_nodes(), ClientId, Topic). -spec unsubscribe([node()], emqx_types:clientid(), emqx_types:topic()) -> {unsubscribe, _} | {error, channel_not_found}. @@ -490,7 +497,7 @@ do_unsubscribe(ClientId, Topic) -> -spec unsubscribe_batch(emqx_types:clientid(), [emqx_types:topic()]) -> {unsubscribe, _} | {error, channel_not_found}. unsubscribe_batch(ClientId, Topics) -> - unsubscribe_batch(mria:running_nodes(), ClientId, Topics). + unsubscribe_batch(emqx:running_nodes(), ClientId, Topics). -spec unsubscribe_batch([node()], emqx_types:clientid(), [emqx_types:topic()]) -> {unsubscribe_batch, _} | {error, channel_not_found}. @@ -515,7 +522,7 @@ do_unsubscribe_batch(ClientId, Topics) -> %%-------------------------------------------------------------------- get_alarms(Type) -> - [{Node, get_alarms(Node, Type)} || Node <- mria:running_nodes()]. + [{Node, get_alarms(Node, Type)} || Node <- emqx:running_nodes()]. get_alarms(Node, Type) -> add_duration_field(unwrap_rpc(emqx_proto_v1:get_alarms(Node, Type))). @@ -524,7 +531,7 @@ deactivate(Node, Name) -> unwrap_rpc(emqx_proto_v1:deactivate_alarm(Node, Name)). delete_all_deactivated_alarms() -> - [delete_all_deactivated_alarms(Node) || Node <- mria:running_nodes()]. + [delete_all_deactivated_alarms(Node) || Node <- emqx:running_nodes()]. delete_all_deactivated_alarms(Node) -> unwrap_rpc(emqx_proto_v1:delete_all_deactivated_alarms(Node)). diff --git a/apps/emqx_management/src/emqx_mgmt_api.erl b/apps/emqx_management/src/emqx_mgmt_api.erl index c77752f7d..8365b983c 100644 --- a/apps/emqx_management/src/emqx_mgmt_api.erl +++ b/apps/emqx_management/src/emqx_mgmt_api.erl @@ -163,7 +163,7 @@ cluster_query(Tab, QString, QSchema, MsFun, FmtFun) -> {error, page_limit_invalid}; Meta -> {_CodCnt, NQString} = parse_qstring(QString, QSchema), - Nodes = mria:running_nodes(), + Nodes = emqx:running_nodes(), ResultAcc = init_query_result(), QueryState = init_query_state(Tab, NQString, MsFun, Meta), NResultAcc = do_cluster_query( diff --git a/apps/emqx_management/src/emqx_mgmt_api_banned.erl b/apps/emqx_management/src/emqx_mgmt_api_banned.erl index 0a5cc3afe..508cf7d07 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_banned.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_banned.erl @@ -165,7 +165,7 @@ banned(post, #{body := Body}) -> {ok, Banned} -> {200, format(Banned)}; {error, {already_exist, Old}} -> - OldBannedFormat = emqx_json:encode(format(Old)), + OldBannedFormat = emqx_utils_json:encode(format(Old)), {400, 'ALREADY_EXISTS', OldBannedFormat} end end. diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl index cac3edaed..681c851bf 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl @@ -644,7 +644,7 @@ list_clients(QString) -> fun ?MODULE:format_channel_info/2 ); Node0 -> - case emqx_misc:safe_to_existing_atom(Node0) of + case emqx_utils:safe_to_existing_atom(Node0) of {ok, Node1} -> QStringWithoutNode = maps:without([<<"node">>], QString), emqx_mgmt_api:node_query( @@ -860,8 +860,8 @@ format_channel_info(ChannInfo = {_, _ClientInfo, _ClientStats}) -> format_channel_info(WhichNode, {_, ClientInfo0, ClientStats}) -> Node = maps:get(node, ClientInfo0, WhichNode), - ClientInfo1 = emqx_map_lib:deep_remove([conninfo, clientid], ClientInfo0), - ClientInfo2 = emqx_map_lib:deep_remove([conninfo, username], ClientInfo1), + ClientInfo1 = emqx_utils_maps:deep_remove([conninfo, clientid], ClientInfo0), + ClientInfo2 = emqx_utils_maps:deep_remove([conninfo, username], ClientInfo1), StatsMap = maps:without( [memory, next_pkt_id, total_heap_size], maps:from_list(ClientStats) @@ -958,4 +958,4 @@ format_authz_cache({{PubSub, Topic}, {AuthzResult, Timestamp}}) -> to_topic_info(Data) -> M = maps:with([<<"topic">>, <<"qos">>, <<"nl">>, <<"rap">>, <<"rh">>], Data), - emqx_map_lib:safe_atom_key_map(M). + emqx_utils_maps:safe_atom_key_map(M). diff --git a/apps/emqx_management/src/emqx_mgmt_api_cluster.erl b/apps/emqx_management/src/emqx_mgmt_api_cluster.erl index 68bb6c81d..e74b6c362 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_cluster.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_cluster.erl @@ -101,7 +101,7 @@ cluster_info(get, _) -> ClusterName = application:get_env(ekka, cluster_name, emqxcl), Info = #{ name => ClusterName, - nodes => mria:running_nodes(), + nodes => emqx:running_nodes(), self => node() }, {200, Info}. diff --git a/apps/emqx_management/src/emqx_mgmt_api_configs.erl b/apps/emqx_management/src/emqx_mgmt_api_configs.erl index 55cc50597..af203dfe9 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_configs.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_configs.erl @@ -31,41 +31,21 @@ global_zone_configs/3 ]). --export([gen_schema/1]). - -define(PREFIX, "/configs/"). -define(PREFIX_RESET, "/configs_reset/"). -define(ERR_MSG(MSG), list_to_binary(io_lib:format("~p", [MSG]))). -define(OPTS, #{rawconf_with_defaults => true, override_to => cluster}). -define(TAGS, ["Configs"]). --define(EXCLUDES, - [ - <<"exhook">>, - <<"gateway">>, - <<"plugins">>, - <<"bridges">>, - <<"rule_engine">>, - <<"authorization">>, - <<"authentication">>, - <<"rpc">>, - <<"connectors">>, - <<"slow_subs">>, - <<"psk_authentication">>, - <<"topic_metrics">>, - <<"rewrite">>, - <<"auto_subscribe">>, - <<"retainer">>, - <<"statsd">>, - <<"delayed">>, - <<"event_message">>, - <<"prometheus">>, - <<"telemetry">>, - <<"listeners">>, - <<"license">>, - <<"api_key">> - ] ++ global_zone_roots() -). +-define(ROOT_KEYS, [ + <<"dashboard">>, + <<"alarm">>, + <<"sys_topics">>, + <<"sysmon">>, + <<"log">>, + <<"persistent_session_store">>, + <<"zones">> +]). api_spec() -> emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). @@ -119,7 +99,6 @@ schema("/configs_reset/:rootname") -> "- For a config entry that has default value, this resets it to the default value;\n" "- For a config entry that has no default value, an error 400 will be returned" >>, - summary => <<"Reset config entry">>, %% We only return "200" rather than the new configs that has been changed, as %% the schema of the changed configs is depends on the request parameter %% `conf_path`, it cannot be defined here. @@ -214,12 +193,11 @@ fields(Field) -> %%%============================================================================================== %% HTTP API Callbacks config(get, _Params, Req) -> + [Path] = conf_path(Req), + {200, get_raw_config(Path)}; +config(put, #{body := NewConf}, Req) -> Path = conf_path(Req), - {ok, Conf} = emqx_map_lib:deep_find(Path, get_full_config()), - {200, Conf}; -config(put, #{body := Body}, Req) -> - Path = conf_path(Req), - case emqx_conf:update(Path, Body, ?OPTS) of + case emqx_conf:update(Path, NewConf, ?OPTS) of {ok, #{raw_config := RawConf}} -> {200, RawConf}; {error, {permission_denied, Reason}} -> @@ -229,28 +207,29 @@ config(put, #{body := Body}, Req) -> end. global_zone_configs(get, _Params, _Req) -> - Paths = global_zone_roots(), - Zones = lists:foldl( - fun(Path, Acc) -> maps:merge(Acc, get_config_with_default(Path)) end, - #{}, - Paths - ), - {200, Zones}; + {200, get_zones()}; global_zone_configs(put, #{body := Body}, _Req) -> + PrevZones = get_zones(), Res = maps:fold( fun(Path, Value, Acc) -> - case emqx_conf:update([Path], Value, ?OPTS) of - {ok, #{raw_config := RawConf}} -> - Acc#{Path => RawConf}; - {error, Reason} -> - ?SLOG(error, #{ - msg => "update global zone failed", - reason => Reason, - path => Path, - value => Value - }), - Acc + PrevValue = maps:get(Path, PrevZones), + case Value =/= PrevValue of + true -> + case emqx_conf:update([Path], Value, ?OPTS) of + {ok, #{raw_config := RawConf}} -> + Acc#{Path => RawConf}; + {error, Reason} -> + ?SLOG(error, #{ + msg => "update global zone failed", + reason => Reason, + path => Path, + value => Value + }), + Acc + end; + false -> + Acc#{Path => Value} end end, #{}, @@ -279,7 +258,7 @@ configs(get, Params, _Req) -> QS = maps:get(query_string, Params, #{}), Node = maps:get(<<"node">>, QS, node()), case - lists:member(Node, mria:running_nodes()) andalso + lists:member(Node, emqx:running_nodes()) andalso emqx_management_proto_v2:get_full_config(Node) of false -> @@ -298,13 +277,30 @@ conf_path_reset(Req) -> get_full_config() -> emqx_config:fill_defaults( - maps:without( - ?EXCLUDES, + maps:with( + ?ROOT_KEYS, emqx:get_raw_config([]) ), #{obfuscate_sensitive_values => true} ). +get_raw_config(Path) -> + #{Path := Conf} = + emqx_config:fill_defaults( + #{Path => emqx:get_raw_config([Path])}, + #{obfuscate_sensitive_values => true} + ), + Conf. + +get_zones() -> + lists:foldl( + fun(Path, Acc) -> + maps:merge(Acc, get_config_with_default(Path)) + end, + #{}, + global_zone_roots() + ). + get_config_with_default(Path) -> emqx_config:fill_defaults(#{Path => emqx:get_raw_config([Path])}). @@ -317,43 +313,14 @@ conf_path_from_querystr(Req) -> config_list() -> Mod = emqx_conf:schema_module(), Roots = hocon_schema:roots(Mod), - lists:foldl(fun(Key, Acc) -> lists:keydelete(Key, 1, Acc) end, Roots, ?EXCLUDES). + lists:foldl(fun(Key, Acc) -> [lists:keyfind(Key, 1, Roots) | Acc] end, [], ?ROOT_KEYS). conf_path(Req) -> <<"/api/v5", ?PREFIX, Path/binary>> = cowboy_req:path(Req), string:lexemes(Path, "/ "). -%% TODO: generate from hocon schema -gen_schema(Conf) when is_boolean(Conf) -> - with_default_value(#{type => boolean}, Conf); -gen_schema(Conf) when is_binary(Conf); is_atom(Conf) -> - with_default_value(#{type => string}, Conf); -gen_schema(Conf) when is_number(Conf) -> - with_default_value(#{type => number}, Conf); -gen_schema(Conf) when is_list(Conf) -> - case io_lib:printable_unicode_list(Conf) of - true -> - gen_schema(unicode:characters_to_binary(Conf)); - false -> - #{type => array, items => gen_schema(hd(Conf))} - end; -gen_schema(Conf) when is_map(Conf) -> - #{ - type => object, - properties => - maps:map(fun(_K, V) -> gen_schema(V) end, Conf) - }; -gen_schema(_Conf) -> - %% the conf is not of JSON supported type, it may have been converted - %% by the hocon schema - #{type => string}. - -with_default_value(Type, Value) -> - Type#{example => emqx_map_lib:binary_string(Value)}. - global_zone_roots() -> - lists:map(fun({K, _}) -> K end, global_zone_schema()). + lists:map(fun({K, _}) -> list_to_binary(K) end, global_zone_schema()). global_zone_schema() -> - Roots = hocon_schema:roots(emqx_zone_schema), - lists:map(fun({RootKey, {_Root, Schema}}) -> {RootKey, Schema} end, Roots). + emqx_zone_schema:zone_without_hidden(). diff --git a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl index c126cfe19..de86700ef 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl @@ -390,7 +390,7 @@ crud_listeners_by_id(put, #{bindings := #{id := Id}, body := Body0}) -> undefined -> {404, #{code => 'BAD_LISTENER_ID', message => ?LISTENER_NOT_FOUND}}; PrevConf -> - MergeConfT = emqx_map_lib:deep_merge(PrevConf, Conf), + MergeConfT = emqx_utils_maps:deep_merge(PrevConf, Conf), MergeConf = emqx_listeners:ensure_override_limiter_conf(MergeConfT, Conf), case update(Path, MergeConf) of {ok, #{raw_config := _RawConf}} -> @@ -483,7 +483,7 @@ err_msg_str(Reason) -> io_lib:format("~p", [Reason]). list_listeners() -> - [list_listeners(Node) || Node <- mria:running_nodes()]. + [list_listeners(Node) || Node <- emqx:running_nodes()]. list_listeners(Node) -> wrap_rpc(emqx_management_proto_v2:list_listeners(Node)). diff --git a/apps/emqx_management/src/emqx_mgmt_api_metrics.erl b/apps/emqx_management/src/emqx_mgmt_api_metrics.erl index 1c5c8f62a..0fcc45d8e 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_metrics.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_metrics.erl @@ -59,7 +59,7 @@ metrics(get, #{query_string := Qs}) -> maps:from_list( emqx_mgmt:get_metrics(Node) ++ [{node, Node}] ) - || Node <- mria:running_nodes() + || Node <- emqx:running_nodes() ], {200, Data} end. diff --git a/apps/emqx_management/src/emqx_mgmt_api_nodes.erl b/apps/emqx_management/src/emqx_mgmt_api_nodes.erl index a4173f5b0..ecf465f43 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_nodes.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_nodes.erl @@ -247,13 +247,13 @@ nodes(get, _Params) -> list_nodes(#{}). node(get, #{bindings := #{node := NodeName}}) -> - emqx_api_lib:with_node(NodeName, to_ok_result_fun(fun get_node/1)). + emqx_utils_api:with_node(NodeName, to_ok_result_fun(fun get_node/1)). node_metrics(get, #{bindings := #{node := NodeName}}) -> - emqx_api_lib:with_node(NodeName, to_ok_result_fun(fun emqx_mgmt:get_metrics/1)). + emqx_utils_api:with_node(NodeName, to_ok_result_fun(fun emqx_mgmt:get_metrics/1)). node_stats(get, #{bindings := #{node := NodeName}}) -> - emqx_api_lib:with_node(NodeName, to_ok_result_fun(fun emqx_mgmt:get_stats/1)). + emqx_utils_api:with_node(NodeName, to_ok_result_fun(fun emqx_mgmt:get_stats/1)). %%-------------------------------------------------------------------- %% api apply diff --git a/apps/emqx_management/src/emqx_mgmt_api_stats.erl b/apps/emqx_management/src/emqx_mgmt_api_stats.erl index 1d3c0e21b..5f4bbce65 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_stats.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_stats.erl @@ -127,9 +127,21 @@ list(get, #{query_string := Qs}) -> true -> {200, emqx_mgmt:get_stats()}; _ -> - Data = [ - maps:from_list(emqx_mgmt:get_stats(Node) ++ [{node, Node}]) - || Node <- mria:running_nodes() - ], + Data = lists:foldl( + fun(Node, Acc) -> + case emqx_mgmt:get_stats(Node) of + {error, _Err} -> + Acc; + Stats when is_list(Stats) -> + Data = maps:from_list([{node, Node} | Stats]), + [Data | Acc] + end + end, + [], + emqx:running_nodes() + ), {200, Data} end. + +%%%============================================================================================== +%% Internal diff --git a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl index 409af4d95..1b69835f9 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl @@ -149,7 +149,7 @@ subscriptions(get, #{query_string := QString}) -> fun ?MODULE:format/2 ); Node0 -> - case emqx_misc:safe_to_existing_atom(Node0) of + case emqx_utils:safe_to_existing_atom(Node0) of {ok, Node1} -> emqx_mgmt_api:node_query( Node1, diff --git a/apps/emqx_management/src/emqx_mgmt_api_trace.erl b/apps/emqx_management/src/emqx_mgmt_api_trace.erl index b93839b0b..25cc2734f 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_trace.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_trace.erl @@ -94,7 +94,8 @@ schema("/trace") -> 409 => emqx_dashboard_swagger:error_codes( [ 'ALREADY_EXISTS', - 'DUPLICATE_CONDITION' + 'DUPLICATE_CONDITION', + 'BAD_TYPE' ], <<"trace already exists">> ) @@ -265,6 +266,19 @@ fields(trace) -> example => running } )}, + {payload_encode, + hoconsc:mk(hoconsc:enum([hex, text, hidden]), #{ + desc => + "" + "Determine the format of the payload format in the trace file.
\n" + "`text`: Text-based protocol or plain text protocol.\n" + " It is recommended when payload is JSON encoded.
\n" + "`hex`: Binary hexadecimal encode." + "It is recommended when payload is a custom binary protocol.
\n" + "`hidden`: payload is obfuscated as `******`" + "", + default => text + })}, {start_at, hoconsc:mk( emqx_datetime:epoch_second(), @@ -376,7 +390,7 @@ trace(get, _Params) -> fun(#{start_at := A}, #{start_at := B}) -> A > B end, emqx_trace:format(List0) ), - Nodes = mria:running_nodes(), + Nodes = emqx:running_nodes(), TraceSize = wrap_rpc(emqx_mgmt_trace_proto_v2:get_trace_size(Nodes)), AllFileSize = lists:foldl(fun(F, Acc) -> maps:merge(Acc, F) end, #{}, TraceSize), Now = erlang:system_time(second), @@ -421,6 +435,11 @@ trace(post, #{body := Param}) -> code => 'DUPLICATE_CONDITION', message => ?TO_BIN([Name, " Duplication Condition"]) }}; + {error, {bad_type, _}} -> + {409, #{ + code => 'BAD_TYPE', + message => <<"Rolling upgrade in progress, create failed">> + }}; {error, Reason} -> {400, #{ code => 'INVALID_PARAMS', @@ -445,7 +464,7 @@ format_trace(Trace0) -> LogSize = lists:foldl( fun(Node, Acc) -> Acc#{Node => 0} end, #{}, - mria:running_nodes() + emqx:running_nodes() ), Trace2 = maps:without([enable, filter], Trace1), Trace2#{ @@ -479,7 +498,7 @@ download_trace_log(get, #{bindings := #{name := Name}, query_string := Query}) - %% We generate a session ID so that we name files %% with unique names. Then we won't cause %% overwrites for concurrent requests. - SessionId = emqx_misc:gen_id(), + SessionId = emqx_utils:gen_id(), ZipDir = filename:join([emqx_trace:zip_dir(), SessionId]), ok = file:make_dir(ZipDir), %% Write files to ZipDir and create an in-memory zip file @@ -541,13 +560,13 @@ group_trace_file(ZipDir, TraceLog, TraceFiles) -> ). collect_trace_file(undefined, TraceLog) -> - Nodes = mria:running_nodes(), + Nodes = emqx:running_nodes(), wrap_rpc(emqx_mgmt_trace_proto_v2:trace_file(Nodes, TraceLog)); collect_trace_file(Node, TraceLog) -> wrap_rpc(emqx_mgmt_trace_proto_v2:trace_file([Node], TraceLog)). collect_trace_file_detail(TraceLog) -> - Nodes = mria:running_nodes(), + Nodes = emqx:running_nodes(), wrap_rpc(emqx_mgmt_trace_proto_v2:trace_file_detail(Nodes, TraceLog)). wrap_rpc({GoodRes, BadNodes}) -> @@ -677,7 +696,7 @@ parse_node(Query, Default) -> {ok, Default}; {ok, NodeBin} -> Node = binary_to_existing_atom(NodeBin), - true = lists:member(Node, mria:running_nodes()), + true = lists:member(Node, emqx:running_nodes()), {ok, Node} end catch diff --git a/apps/emqx_management/src/emqx_mgmt_auth.erl b/apps/emqx_management/src/emqx_mgmt_auth.erl index 6f2a27414..12a7a6641 100644 --- a/apps/emqx_management/src/emqx_mgmt_auth.erl +++ b/apps/emqx_management/src/emqx_mgmt_auth.erl @@ -174,7 +174,7 @@ create_app(Name, ApiSecret, Enable, ExpiredAt, Desc) -> desc = Desc, created_at = erlang:system_time(second), api_secret_hash = emqx_dashboard_admin:hash(ApiSecret), - api_key = list_to_binary(emqx_misc:gen_id(16)) + api_key = list_to_binary(emqx_utils:gen_id(16)) }, case create_app(App) of {ok, Res} -> @@ -213,7 +213,7 @@ do_force_create_app(App, ApiKey, NamePrefix) -> end. generate_unique_name(NamePrefix) -> - New = list_to_binary(NamePrefix ++ emqx_misc:gen_id(16)), + New = list_to_binary(NamePrefix ++ emqx_utils:gen_id(16)), case mnesia:read(?APP, New) of [] -> New; _ -> generate_unique_name(NamePrefix) @@ -246,7 +246,7 @@ init_bootstrap_file(File) -> {ok, MP} = re:compile(<<"(\.+):(\.+$)">>, [ungreedy]), init_bootstrap_file(File, Dev, MP); {error, Reason0} -> - Reason = emqx_misc:explain_posix(Reason0), + Reason = emqx_utils:explain_posix(Reason0), ?SLOG( error, #{ diff --git a/apps/emqx_management/src/emqx_mgmt_cli.erl b/apps/emqx_management/src/emqx_mgmt_cli.erl index 442d5c7de..253d527ac 100644 --- a/apps/emqx_management/src/emqx_mgmt_cli.erl +++ b/apps/emqx_management/src/emqx_mgmt_cli.erl @@ -356,7 +356,7 @@ mnesia(_) -> %% @doc Logger Command log(["set-level", Level]) -> - case emqx_misc:safe_to_existing_atom(Level) of + case emqx_utils:safe_to_existing_atom(Level) of {ok, Level1} -> case emqx_logger:set_log_level(Level1) of ok -> emqx_ctl:print("~ts~n", [Level]); @@ -369,7 +369,7 @@ log(["primary-level"]) -> Level = emqx_logger:get_primary_log_level(), emqx_ctl:print("~ts~n", [Level]); log(["primary-level", Level]) -> - case emqx_misc:safe_to_existing_atom(Level) of + case emqx_utils:safe_to_existing_atom(Level) of {ok, Level1} -> _ = emqx_logger:set_primary_log_level(Level1), ok; @@ -392,7 +392,7 @@ log(["handlers", "list"]) -> ], ok; log(["handlers", "start", HandlerId]) -> - case emqx_misc:safe_to_existing_atom(HandlerId) of + case emqx_utils:safe_to_existing_atom(HandlerId) of {ok, HandlerId1} -> case emqx_logger:start_log_handler(HandlerId1) of ok -> @@ -406,7 +406,7 @@ log(["handlers", "start", HandlerId]) -> emqx_ctl:print("[error] invalid handler:~ts~n", [HandlerId]) end; log(["handlers", "stop", HandlerId]) -> - case emqx_misc:safe_to_existing_atom(HandlerId) of + case emqx_utils:safe_to_existing_atom(HandlerId) of {ok, HandlerId1} -> case emqx_logger:stop_log_handler(HandlerId1) of ok -> @@ -420,9 +420,9 @@ log(["handlers", "stop", HandlerId]) -> emqx_ctl:print("[error] invalid handler:~ts~n", [HandlerId]) end; log(["handlers", "set-level", HandlerId, Level]) -> - case emqx_misc:safe_to_existing_atom(HandlerId) of + case emqx_utils:safe_to_existing_atom(HandlerId) of {ok, HandlerId1} -> - case emqx_misc:safe_to_existing_atom(Level) of + case emqx_utils:safe_to_existing_atom(Level) of {ok, Level1} -> case emqx_logger:set_log_handler_level(HandlerId1, Level1) of ok -> @@ -628,7 +628,7 @@ listeners([]) -> emqx_listeners:list() ); listeners(["stop", ListenerId]) -> - case emqx_misc:safe_to_existing_atom(ListenerId) of + case emqx_utils:safe_to_existing_atom(ListenerId) of {ok, ListenerId1} -> case emqx_listeners:stop_listener(ListenerId1) of ok -> @@ -640,7 +640,7 @@ listeners(["stop", ListenerId]) -> emqx_ctl:print("Invalid listener: ~0p~n", [ListenerId]) end; listeners(["start", ListenerId]) -> - case emqx_misc:safe_to_existing_atom(ListenerId) of + case emqx_utils:safe_to_existing_atom(ListenerId) of {ok, ListenerId1} -> case emqx_listeners:start_listener(ListenerId1) of ok -> @@ -652,7 +652,7 @@ listeners(["start", ListenerId]) -> emqx_ctl:print("Invalid listener: ~0p~n", [ListenerId]) end; listeners(["restart", ListenerId]) -> - case emqx_misc:safe_to_existing_atom(ListenerId) of + case emqx_utils:safe_to_existing_atom(ListenerId) of {ok, ListenerId1} -> case emqx_listeners:restart_listener(ListenerId1) of ok -> diff --git a/apps/emqx_management/test/emqx_mgmt_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_SUITE.erl index 71b51b67c..3eb37060e 100644 --- a/apps/emqx_management/test/emqx_mgmt_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_SUITE.erl @@ -36,16 +36,16 @@ end_per_suite(_) -> emqx_mgmt_api_test_util:end_suite([emqx_management, emqx_conf]). init_per_testcase(TestCase, Config) -> - meck:expect(mria, running_nodes, 0, [node()]), + meck:expect(emqx, running_nodes, 0, [node()]), emqx_common_test_helpers:init_per_testcase(?MODULE, TestCase, Config). end_per_testcase(TestCase, Config) -> - meck:unload(mria), + meck:unload(emqx), emqx_common_test_helpers:end_per_testcase(?MODULE, TestCase, Config). t_list_nodes(init, Config) -> meck:expect( - mria, + emqx, cluster_nodes, fun (running) -> [node()]; @@ -125,7 +125,7 @@ t_lookup_client(_Config) -> emqx_mgmt:lookup_client({username, <<"user1">>}, ?FORMATFUN) ), ?assertEqual([], emqx_mgmt:lookup_client({clientid, <<"notfound">>}, ?FORMATFUN)), - meck:expect(mria, running_nodes, 0, [node(), 'fake@nonode']), + meck:expect(emqx, running_nodes, 0, [node(), 'fake@nonode']), ?assertMatch( [_ | {error, nodedown}], emqx_mgmt:lookup_client({clientid, <<"client1">>}, ?FORMATFUN) ). @@ -188,7 +188,7 @@ t_clean_cache(_Config) -> {error, _}, emqx_mgmt:clean_pem_cache_all() ), - meck:expect(mria, running_nodes, 0, [node(), 'fake@nonode']), + meck:expect(emqx, running_nodes, 0, [node(), 'fake@nonode']), ?assertMatch( {error, [{'fake@nonode', {error, _}}]}, emqx_mgmt:clean_authz_cache_all() diff --git a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl index bacec718d..a53ffc9c4 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl @@ -179,14 +179,14 @@ t_bad_rpc(_) -> ClientLs1 = [start_emqtt_client(node(), I, 1883) || I <- lists:seq(1, 10)], Path = emqx_mgmt_api_test_util:api_path(["clients?limit=2&page=2"]), try - meck:expect(mria, running_nodes, 0, ['fake@nohost']), + meck:expect(emqx, running_nodes, 0, ['fake@nohost']), {error, {_, 500, _}} = emqx_mgmt_api_test_util:request_api(get, Path), %% good cop, bad cop - meck:expect(mria, running_nodes, 0, [node(), 'fake@nohost']), + meck:expect(emqx, running_nodes, 0, [node(), 'fake@nohost']), {error, {_, 500, _}} = emqx_mgmt_api_test_util:request_api(get, Path) after _ = lists:foreach(fun(C) -> emqtt:disconnect(C) end, ClientLs1), - meck:unload(mria), + meck:unload(emqx), emqx_mgmt_api_test_util:end_suite() end. diff --git a/apps/emqx_management/test/emqx_mgmt_api_alarms_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_alarms_SUITE.erl index 69ace16e8..dc7be7671 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_alarms_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_alarms_SUITE.erl @@ -56,7 +56,7 @@ get_alarms(AssertCount, Activated) -> Qs = "activated=" ++ Activated, Headers = emqx_mgmt_api_test_util:auth_header_(), {ok, Response} = emqx_mgmt_api_test_util:request_api(get, Path, Qs, Headers), - Data = emqx_json:decode(Response, [return_maps]), + Data = emqx_utils_json:decode(Response, [return_maps]), Meta = maps:get(<<"meta">>, Data), Page = maps:get(<<"page">>, Meta), Limit = maps:get(<<"limit">>, Meta), diff --git a/apps/emqx_management/test/emqx_mgmt_api_api_keys_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_api_keys_SUITE.erl index 241a73dc4..1a396d795 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_api_keys_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_api_keys_SUITE.erl @@ -228,7 +228,7 @@ list_app() -> AuthHeader = emqx_dashboard_SUITE:auth_header_(), Path = emqx_mgmt_api_test_util:api_path(["api_key"]), case emqx_mgmt_api_test_util:request_api(get, Path, AuthHeader) of - {ok, Apps} -> {ok, emqx_json:decode(Apps, [return_maps])}; + {ok, Apps} -> {ok, emqx_utils_json:decode(Apps, [return_maps])}; Error -> Error end. @@ -236,7 +236,7 @@ read_app(Name) -> AuthHeader = emqx_dashboard_SUITE:auth_header_(), Path = emqx_mgmt_api_test_util:api_path(["api_key", Name]), case emqx_mgmt_api_test_util:request_api(get, Path, AuthHeader) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. @@ -251,7 +251,7 @@ create_app(Name) -> enable => true }, case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, App) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. @@ -260,7 +260,7 @@ create_unexpired_app(Name, Params) -> Path = emqx_mgmt_api_test_util:api_path(["api_key"]), App = maps:merge(#{name => Name, desc => <<"Note"/utf8>>, enable => true}, Params), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, App) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. @@ -273,7 +273,7 @@ update_app(Name, Change) -> AuthHeader = emqx_dashboard_SUITE:auth_header_(), UpdatePath = emqx_mgmt_api_test_util:api_path(["api_key", Name]), case emqx_mgmt_api_test_util:request_api(put, UpdatePath, "", AuthHeader, Change) of - {ok, Update} -> {ok, emqx_json:decode(Update, [return_maps])}; + {ok, Update} -> {ok, emqx_utils_json:decode(Update, [return_maps])}; Error -> Error end. diff --git a/apps/emqx_management/test/emqx_mgmt_api_banned_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_banned_SUITE.erl index c765f00bc..9f1b560f7 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_banned_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_banned_SUITE.erl @@ -160,7 +160,7 @@ t_delete(_Config) -> list_banned() -> Path = emqx_mgmt_api_test_util:api_path(["banned"]), case emqx_mgmt_api_test_util:request_api(get, Path) of - {ok, Apps} -> {ok, emqx_json:decode(Apps, [return_maps])}; + {ok, Apps} -> {ok, emqx_utils_json:decode(Apps, [return_maps])}; Error -> Error end. @@ -168,7 +168,7 @@ create_banned(Banned) -> AuthHeader = emqx_mgmt_api_test_util:auth_header_(), Path = emqx_mgmt_api_test_util:api_path(["banned"]), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Banned) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. diff --git a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl index 9f26f8542..6d7733b22 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl @@ -58,7 +58,7 @@ t_clients(_) -> %% get /clients ClientsPath = emqx_mgmt_api_test_util:api_path(["clients"]), {ok, Clients} = emqx_mgmt_api_test_util:request_api(get, ClientsPath), - ClientsResponse = emqx_json:decode(Clients, [return_maps]), + ClientsResponse = emqx_utils_json:decode(Clients, [return_maps]), ClientsMeta = maps:get(<<"meta">>, ClientsResponse), ClientsPage = maps:get(<<"page">>, ClientsMeta), ClientsLimit = maps:get(<<"limit">>, ClientsMeta), @@ -70,7 +70,7 @@ t_clients(_) -> %% get /clients/:clientid Client1Path = emqx_mgmt_api_test_util:api_path(["clients", binary_to_list(ClientId1)]), {ok, Client1} = emqx_mgmt_api_test_util:request_api(get, Client1Path), - Client1Response = emqx_json:decode(Client1, [return_maps]), + Client1Response = emqx_utils_json:decode(Client1, [return_maps]), ?assertEqual(Username1, maps:get(<<"username">>, Client1Response)), ?assertEqual(ClientId1, maps:get(<<"clientid">>, Client1Response)), ?assertEqual(120, maps:get(<<"expiry_interval">>, Client1Response)), @@ -130,7 +130,7 @@ t_clients(_) -> "", AuthHeader ), - [SubscriptionsData] = emqx_json:decode(SubscriptionsRes, [return_maps]), + [SubscriptionsData] = emqx_utils_json:decode(SubscriptionsRes, [return_maps]), ?assertMatch( #{ <<"clientid">> := ClientId1, @@ -210,7 +210,7 @@ t_query_clients_with_time(_) -> GteParamRfc3339 ++ GteParamStamp ], DecodedResults = [ - emqx_json:decode(Response, [return_maps]) + emqx_utils_json:decode(Response, [return_maps]) || {ok, Response} <- RequestResults ], {LteResponseDecodeds, GteResponseDecodeds} = lists:split(4, DecodedResults), @@ -247,7 +247,7 @@ t_keepalive(_Config) -> {ok, C1} = emqtt:start_link(#{username => Username, clientid => ClientId}), {ok, _} = emqtt:connect(C1), {ok, NewClient} = emqx_mgmt_api_test_util:request_api(put, Path, <<"">>, AuthHeader, Body), - #{<<"keepalive">> := 11} = emqx_json:decode(NewClient, [return_maps]), + #{<<"keepalive">> := 11} = emqx_utils_json:decode(NewClient, [return_maps]), [Pid] = emqx_cm:lookup_channels(list_to_binary(ClientId)), #{conninfo := #{keepalive := Keepalive}} = emqx_connection:info(Pid), ?assertEqual(11, Keepalive), diff --git a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl index 2d24bce99..5a0116a4d 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl @@ -55,15 +55,20 @@ t_update(_Config) -> %% update ok {ok, SysMon} = get_config(<<"sysmon">>), #{<<"vm">> := #{<<"busy_port">> := BusyPort}} = SysMon, - NewSysMon = emqx_map_lib:deep_put([<<"vm">>, <<"busy_port">>], SysMon, not BusyPort), + NewSysMon = #{<<"vm">> => #{<<"busy_port">> => not BusyPort}}, {ok, #{}} = update_config(<<"sysmon">>, NewSysMon), {ok, SysMon1} = get_config(<<"sysmon">>), #{<<"vm">> := #{<<"busy_port">> := BusyPort1}} = SysMon1, ?assertEqual(BusyPort, not BusyPort1), assert_busy_port(BusyPort1), + %% Make sure the override config is updated, and remove the default value. + ?assertMatch( + #{<<"vm">> := #{<<"busy_port">> := BusyPort1}}, + maps:get(<<"sysmon">>, emqx_config:read_override_conf(#{override_to => cluster})) + ), %% update failed - ErrorSysMon = emqx_map_lib:deep_put([<<"vm">>, <<"busy_port">>], SysMon, "123"), + ErrorSysMon = emqx_utils_maps:deep_put([<<"vm">>, <<"busy_port">>], SysMon, "123"), ?assertMatch( {error, {"HTTP/1.1", 400, _}}, update_config(<<"sysmon">>, ErrorSysMon) @@ -78,7 +83,7 @@ t_update(_Config) -> assert_busy_port(true), %% reset no_default_value config - NewSysMon1 = emqx_map_lib:deep_put([<<"vm">>, <<"busy_port">>], SysMon, false), + NewSysMon1 = emqx_utils_maps:deep_put([<<"vm">>, <<"busy_port">>], SysMon, false), {ok, #{}} = update_config(<<"sysmon">>, NewSysMon1), ?assertMatch({error, {"HTTP/1.1", 400, _}}, reset_config(<<"sysmon">>, "")), {ok, SysMon4} = get_config(<<"sysmon">>), @@ -94,57 +99,80 @@ t_log(_Config) -> {ok, Log} = get_config("log"), File = "log/emqx-test.log", %% update handler - Log1 = emqx_map_lib:deep_put([<<"file_handlers">>, <<"default">>, <<"enable">>], Log, true), - Log2 = emqx_map_lib:deep_put([<<"file_handlers">>, <<"default">>, <<"file">>], Log1, File), + Log1 = emqx_utils_maps:deep_put([<<"file_handlers">>, <<"default">>, <<"enable">>], Log, true), + Log2 = emqx_utils_maps:deep_put([<<"file_handlers">>, <<"default">>, <<"file">>], Log1, File), {ok, #{}} = update_config(<<"log">>, Log2), {ok, Log3} = logger:get_handler_config(default), ?assertMatch(#{config := #{file := File}}, Log3), - ErrLog1 = emqx_map_lib:deep_put([<<"file_handlers">>, <<"default">>, <<"enable">>], Log, 1), + ErrLog1 = emqx_utils_maps:deep_put([<<"file_handlers">>, <<"default">>, <<"enable">>], Log, 1), ?assertMatch({error, {"HTTP/1.1", 400, _}}, update_config(<<"log">>, ErrLog1)), - ErrLog2 = emqx_map_lib:deep_put([<<"file_handlers">>, <<"default">>, <<"enabfe">>], Log, true), + ErrLog2 = emqx_utils_maps:deep_put( + [<<"file_handlers">>, <<"default">>, <<"enabfe">>], Log, true + ), ?assertMatch({error, {"HTTP/1.1", 400, _}}, update_config(<<"log">>, ErrLog2)), %% add new handler File1 = "log/emqx-test1.log", - Handler = emqx_map_lib:deep_get([<<"file_handlers">>, <<"default">>], Log2), - NewLog1 = emqx_map_lib:deep_put([<<"file_handlers">>, <<"new">>], Log2, Handler), - NewLog2 = emqx_map_lib:deep_put([<<"file_handlers">>, <<"new">>, <<"file">>], NewLog1, File1), + Handler = emqx_utils_maps:deep_get([<<"file_handlers">>, <<"default">>], Log2), + NewLog1 = emqx_utils_maps:deep_put([<<"file_handlers">>, <<"new">>], Log2, Handler), + NewLog2 = emqx_utils_maps:deep_put( + [<<"file_handlers">>, <<"new">>, <<"file">>], NewLog1, File1 + ), {ok, #{}} = update_config(<<"log">>, NewLog2), {ok, Log4} = logger:get_handler_config(new), ?assertMatch(#{config := #{file := File1}}, Log4), %% disable new handler - Disable = emqx_map_lib:deep_put([<<"file_handlers">>, <<"new">>, <<"enable">>], NewLog2, false), + Disable = emqx_utils_maps:deep_put( + [<<"file_handlers">>, <<"new">>, <<"enable">>], NewLog2, false + ), {ok, #{}} = update_config(<<"log">>, Disable), ?assertEqual({error, {not_found, new}}, logger:get_handler_config(new)), ok. t_global_zone(_Config) -> {ok, Zones} = get_global_zone(), - ZonesKeys = lists:map(fun({K, _}) -> K end, hocon_schema:roots(emqx_zone_schema)), + ZonesKeys = lists:map( + fun({K, _}) -> list_to_binary(K) end, emqx_zone_schema:zone_without_hidden() + ), ?assertEqual(lists:usort(ZonesKeys), lists:usort(maps:keys(Zones))), ?assertEqual( emqx_config:get_zone_conf(no_default, [mqtt, max_qos_allowed]), - emqx_map_lib:deep_get([<<"mqtt">>, <<"max_qos_allowed">>], Zones) + emqx_utils_maps:deep_get([<<"mqtt">>, <<"max_qos_allowed">>], Zones) ), - NewZones = emqx_map_lib:deep_put([<<"mqtt">>, <<"max_qos_allowed">>], Zones, 1), + NewZones = emqx_utils_maps:deep_put([<<"mqtt">>, <<"max_qos_allowed">>], Zones, 1), {ok, #{}} = update_global_zone(NewZones), ?assertEqual(1, emqx_config:get_zone_conf(no_default, [mqtt, max_qos_allowed])), + %% Make sure the override config is updated, and remove the default value. + ?assertMatch(#{<<"max_qos_allowed">> := 1}, read_conf(<<"mqtt">>)), - BadZones = emqx_map_lib:deep_put([<<"mqtt">>, <<"max_qos_allowed">>], Zones, 3), + BadZones = emqx_utils_maps:deep_put([<<"mqtt">>, <<"max_qos_allowed">>], Zones, 3), ?assertMatch({error, {"HTTP/1.1", 400, _}}, update_global_zone(BadZones)), %% Remove max_qos_allowed from raw config, but we still get default value(2). Mqtt0 = emqx_conf:get_raw([<<"mqtt">>]), - ?assertEqual(1, emqx_map_lib:deep_get([<<"max_qos_allowed">>], Mqtt0)), + ?assertEqual(1, emqx_utils_maps:deep_get([<<"max_qos_allowed">>], Mqtt0)), Mqtt1 = maps:remove(<<"max_qos_allowed">>, Mqtt0), ok = emqx_config:put_raw([<<"mqtt">>], Mqtt1), Mqtt2 = emqx_conf:get_raw([<<"mqtt">>]), ?assertNot(maps:is_key(<<"max_qos_allowed">>, Mqtt2), Mqtt2), {ok, #{<<"mqtt">> := Mqtt3}} = get_global_zone(), %% the default value is 2 - ?assertEqual(2, emqx_map_lib:deep_get([<<"max_qos_allowed">>], Mqtt3)), + ?assertEqual(2, emqx_utils_maps:deep_get([<<"max_qos_allowed">>], Mqtt3)), ok = emqx_config:put_raw([<<"mqtt">>], Mqtt0), + + DefaultZones = emqx_utils_maps:deep_put([<<"mqtt">>, <<"max_qos_allowed">>], Zones, 2), + {ok, #{}} = update_global_zone(DefaultZones), + #{<<"mqtt">> := Mqtt} = emqx_config:fill_defaults(emqx_schema, #{<<"mqtt">> => #{}}, #{}), + Default = maps:map( + fun + (_, V) when is_boolean(V) -> V; + (_, V) when is_atom(V) -> atom_to_binary(V); + (_, V) -> V + end, + Mqtt + ), + ?assertEqual(Default, read_conf(<<"mqtt">>)), ok. get_global_zone() -> @@ -169,7 +197,7 @@ t_dashboard(_Config) -> Https1 = #{enable => true, bind => 18084}, ?assertMatch( {error, {"HTTP/1.1", 400, _}}, - update_config("dashboard", Dashboard#{<<"https">> => Https1}) + update_config("dashboard", Dashboard#{<<"listeners">> => Listeners#{<<"https">> => Https1}}) ), Https2 = #{ @@ -179,40 +207,46 @@ t_dashboard(_Config) -> cacertfile => "etc/certs/badcacert.pem", certfile => "etc/certs/badcert.pem" }, - Dashboard2 = Dashboard#{listeners => Listeners#{https => Https2}}, + Dashboard2 = Dashboard#{<<"listeners">> => Listeners#{<<"https">> => Https2}}, ?assertMatch( {error, {"HTTP/1.1", 400, _}}, update_config("dashboard", Dashboard2) ), - Keyfile = emqx_common_test_helpers:app_path(emqx, filename:join(["etc", "certs", "key.pem"])), - Certfile = emqx_common_test_helpers:app_path(emqx, filename:join(["etc", "certs", "cert.pem"])), - Cacertfile = emqx_common_test_helpers:app_path( + KeyFile = emqx_common_test_helpers:app_path(emqx, filename:join(["etc", "certs", "key.pem"])), + CertFile = emqx_common_test_helpers:app_path(emqx, filename:join(["etc", "certs", "cert.pem"])), + CacertFile = emqx_common_test_helpers:app_path( emqx, filename:join(["etc", "certs", "cacert.pem"]) ), Https3 = #{ - enable => true, - bind => 18084, - keyfile => Keyfile, - cacertfile => Cacertfile, - certfile => Certfile + <<"enable">> => true, + <<"bind">> => 18084, + <<"keyfile">> => list_to_binary(KeyFile), + <<"cacertfile">> => list_to_binary(CacertFile), + <<"certfile">> => list_to_binary(CertFile) }, - Dashboard3 = Dashboard#{listeners => Listeners#{https => Https3}}, + Dashboard3 = Dashboard#{<<"listeners">> => Listeners#{<<"https">> => Https3}}, ?assertMatch({ok, _}, update_config("dashboard", Dashboard3)), - Dashboard4 = Dashboard#{listeners => Listeners#{https => #{enable => false}}}, + Dashboard4 = Dashboard#{<<"listeners">> => Listeners#{<<"https">> => #{<<"enable">> => false}}}, ?assertMatch({ok, _}, update_config("dashboard", Dashboard4)), + {ok, Dashboard41} = get_config("dashboard"), + ?assertEqual( + Https3#{<<"enable">> => false}, + read_conf([<<"dashboard">>, <<"listeners">>, <<"https">>]), + Dashboard41 + ), ?assertMatch({ok, _}, update_config("dashboard", Dashboard)), {ok, Dashboard1} = get_config("dashboard"), ?assertNotEqual(Dashboard, Dashboard1), - timer:sleep(1000), + timer:sleep(1500), ok. t_configs_node({'init', Config}) -> Node = node(), - meck:expect(mria, running_nodes, fun() -> [Node, bad_node, other_node] end), + meck:expect(emqx, running_nodes, fun() -> [Node, bad_node, other_node] end), meck:expect( emqx_management_proto_v2, get_full_config, @@ -224,7 +258,7 @@ t_configs_node({'init', Config}) -> ), Config; t_configs_node({'end', _}) -> - meck:unload([mria, emqx_management_proto_v2]); + meck:unload([emqx, emqx_management_proto_v2]); t_configs_node(_) -> Node = atom_to_list(node()), @@ -235,7 +269,7 @@ t_configs_node(_) -> ?assertEqual(error, ExpType), ?assertMatch({{_, 404, _}, _, _}, ExpRes), {_, _, Body} = ExpRes, - ?assertMatch(#{<<"code">> := <<"NOT_FOUND">>}, emqx_json:decode(Body, [return_maps])), + ?assertMatch(#{<<"code">> := <<"NOT_FOUND">>}, emqx_utils_json:decode(Body, [return_maps])), ?assertMatch({error, {_, 500, _}}, get_configs("bad_node")). @@ -245,7 +279,7 @@ get_config(Name) -> Path = emqx_mgmt_api_test_util:api_path(["configs", Name]), case emqx_mgmt_api_test_util:request_api(get, Path) of {ok, Res} -> - {ok, emqx_json:decode(Res, [return_maps])}; + {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. @@ -264,8 +298,8 @@ get_configs(Node, Opts) -> end, URI = emqx_mgmt_api_test_util:api_path(Path), case emqx_mgmt_api_test_util:request_api(get, URI, [], [], [], Opts) of - {ok, {_, _, Res}} -> {ok, emqx_json:decode(Res, [return_maps])}; - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, {_, _, Res}} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. @@ -273,7 +307,7 @@ update_config(Name, Change) -> AuthHeader = emqx_mgmt_api_test_util:auth_header_(), UpdatePath = emqx_mgmt_api_test_util:api_path(["configs", Name]), case emqx_mgmt_api_test_util:request_api(put, UpdatePath, "", AuthHeader, Change) of - {ok, Update} -> {ok, emqx_json:decode(Update, [return_maps])}; + {ok, Update} -> {ok, emqx_utils_json:decode(Update, [return_maps])}; Error -> Error end. @@ -288,3 +322,11 @@ reset_config(Name, Key) -> {ok, []} -> ok; Error -> Error end. + +read_conf(RootKeys) when is_list(RootKeys) -> + case emqx_config:read_override_conf(#{override_to => cluster}) of + undefined -> undefined; + Conf -> emqx_utils_maps:deep_get(RootKeys, Conf, undefined) + end; +read_conf(RootKey) -> + read_conf([RootKey]). diff --git a/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl index 3238588e2..33cb66eb2 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl @@ -168,8 +168,8 @@ t_api_listeners_list_not_ready(Config) when is_list(Config) -> L3 = get_tcp_listeners(Node2), Comment = #{ - node1 => rpc:call(Node1, mria, running_nodes, []), - node2 => rpc:call(Node2, mria, running_nodes, []) + node1 => rpc:call(Node1, emqx, running_nodes, []), + node2 => rpc:call(Node2, emqx, running_nodes, []) }, ?assert(length(L1) > length(L2), Comment), @@ -193,10 +193,10 @@ t_clear_certs(Config) when is_list(Config) -> }, %% create, make sure the cert files are created - NewConf = emqx_map_lib:deep_put( + NewConf = emqx_utils_maps:deep_put( [<<"ssl_options">>, <<"certfile">>], ConfTemp, cert_file("certfile") ), - NewConf2 = emqx_map_lib:deep_put( + NewConf2 = emqx_utils_maps:deep_put( [<<"ssl_options">>, <<"keyfile">>], NewConf, cert_file("keyfile") ), @@ -205,7 +205,7 @@ t_clear_certs(Config) when is_list(Config) -> ?assertMatch({ok, [_, _]}, ListResult1), %% update - UpdateConf = emqx_map_lib:deep_put( + UpdateConf = emqx_utils_maps:deep_put( [<<"ssl_options">>, <<"keyfile">>], NewConf2, cert_file("keyfile2") ), _ = request(put, NewPath, [], UpdateConf), @@ -385,7 +385,7 @@ action_listener(ID, Action, Running) -> request(Method, Url, QueryParams, Body) -> AuthHeader = emqx_mgmt_api_test_util:auth_header_(), case emqx_mgmt_api_test_util:request_api(Method, Url, QueryParams, AuthHeader, Body) of - {ok, Res} -> emqx_json:decode(Res, [return_maps]); + {ok, Res} -> emqx_utils_json:decode(Res, [return_maps]); Error -> Error end. diff --git a/apps/emqx_management/test/emqx_mgmt_api_metrics_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_metrics_SUITE.erl index 93cb69f0a..7ecfe9817 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_metrics_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_metrics_SUITE.erl @@ -32,13 +32,13 @@ end_per_suite(_) -> t_metrics_api(_) -> {ok, MetricsResponse} = request_helper("metrics?aggregate=true"), - MetricsFromAPI = emqx_json:decode(MetricsResponse, [return_maps]), + MetricsFromAPI = emqx_utils_json:decode(MetricsResponse, [return_maps]), AggregateMetrics = emqx_mgmt:get_metrics(), match_helper(AggregateMetrics, MetricsFromAPI). t_single_node_metrics_api(_) -> {ok, MetricsResponse} = request_helper("metrics"), - [MetricsFromAPI] = emqx_json:decode(MetricsResponse, [return_maps]), + [MetricsFromAPI] = emqx_utils_json:decode(MetricsResponse, [return_maps]), LocalNodeMetrics = maps:from_list( emqx_mgmt:get_metrics(node()) ++ [{node, to_bin(node())}] ), diff --git a/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl index 30313e555..1f14d075e 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl @@ -34,8 +34,8 @@ init_per_testcase(t_log_path, Config) -> emqx_config_logger:add_handler(), Log = emqx_conf:get_raw([log], #{}), File = "log/emqx-test.log", - Log1 = emqx_map_lib:deep_put([<<"file_handlers">>, <<"default">>, <<"enable">>], Log, true), - Log2 = emqx_map_lib:deep_put([<<"file_handlers">>, <<"default">>, <<"file">>], Log1, File), + Log1 = emqx_utils_maps:deep_put([<<"file_handlers">>, <<"default">>, <<"enable">>], Log, true), + Log2 = emqx_utils_maps:deep_put([<<"file_handlers">>, <<"default">>, <<"file">>], Log1, File), {ok, #{}} = emqx_conf:update([log], Log2, #{rawconf_with_defaults => true}), Config; init_per_testcase(_, Config) -> @@ -43,7 +43,7 @@ init_per_testcase(_, Config) -> end_per_testcase(t_log_path, Config) -> Log = emqx_conf:get_raw([log], #{}), - Log1 = emqx_map_lib:deep_put([<<"file_handlers">>, <<"default">>, <<"enable">>], Log, false), + Log1 = emqx_utils_maps:deep_put([<<"file_handlers">>, <<"default">>, <<"enable">>], Log, false), {ok, #{}} = emqx_conf:update([log], Log1, #{rawconf_with_defaults => true}), emqx_config_logger:remove_handler(), Config; @@ -53,7 +53,7 @@ end_per_testcase(_, Config) -> t_nodes_api(_) -> NodesPath = emqx_mgmt_api_test_util:api_path(["nodes"]), {ok, Nodes} = emqx_mgmt_api_test_util:request_api(get, NodesPath), - NodesResponse = emqx_json:decode(Nodes, [return_maps]), + NodesResponse = emqx_utils_json:decode(Nodes, [return_maps]), LocalNodeInfo = hd(NodesResponse), Node = binary_to_atom(maps:get(<<"node">>, LocalNodeInfo), utf8), ?assertEqual(Node, node()), @@ -63,7 +63,7 @@ t_nodes_api(_) -> NodePath = emqx_mgmt_api_test_util:api_path(["nodes", atom_to_list(node())]), {ok, NodeInfo} = emqx_mgmt_api_test_util:request_api(get, NodePath), NodeNameResponse = - binary_to_atom(maps:get(<<"node">>, emqx_json:decode(NodeInfo, [return_maps])), utf8), + binary_to_atom(maps:get(<<"node">>, emqx_utils_json:decode(NodeInfo, [return_maps])), utf8), ?assertEqual(node(), NodeNameResponse), BadNodePath = emqx_mgmt_api_test_util:api_path(["nodes", "badnode"]), @@ -75,7 +75,7 @@ t_nodes_api(_) -> t_log_path(_) -> NodePath = emqx_mgmt_api_test_util:api_path(["nodes", atom_to_list(node())]), {ok, NodeInfo} = emqx_mgmt_api_test_util:request_api(get, NodePath), - #{<<"log_path">> := Path} = emqx_json:decode(NodeInfo, [return_maps]), + #{<<"log_path">> := Path} = emqx_utils_json:decode(NodeInfo, [return_maps]), ?assertEqual( <<"log">>, filename:basename(Path) @@ -85,7 +85,7 @@ t_node_stats_api(_) -> StatsPath = emqx_mgmt_api_test_util:api_path(["nodes", atom_to_binary(node(), utf8), "stats"]), SystemStats = emqx_mgmt:get_stats(), {ok, StatsResponse} = emqx_mgmt_api_test_util:request_api(get, StatsPath), - Stats = emqx_json:decode(StatsResponse, [return_maps]), + Stats = emqx_utils_json:decode(StatsResponse, [return_maps]), Fun = fun(Key) -> ?assertEqual(maps:get(Key, SystemStats), maps:get(atom_to_binary(Key, utf8), Stats)) @@ -103,7 +103,7 @@ t_node_metrics_api(_) -> emqx_mgmt_api_test_util:api_path(["nodes", atom_to_binary(node(), utf8), "metrics"]), SystemMetrics = emqx_mgmt:get_metrics(), {ok, MetricsResponse} = emqx_mgmt_api_test_util:request_api(get, MetricsPath), - Metrics = emqx_json:decode(MetricsResponse, [return_maps]), + Metrics = emqx_utils_json:decode(MetricsResponse, [return_maps]), Fun = fun(Key) -> ?assertEqual(maps:get(Key, SystemMetrics), maps:get(atom_to_binary(Key, utf8), Metrics)) diff --git a/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl index 24e55494d..62fed8211 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl @@ -136,14 +136,14 @@ t_bad_plugin(Config) -> list_plugins() -> Path = emqx_mgmt_api_test_util:api_path(["plugins"]), case emqx_mgmt_api_test_util:request_api(get, Path) of - {ok, Apps} -> {ok, emqx_json:decode(Apps, [return_maps])}; + {ok, Apps} -> {ok, emqx_utils_json:decode(Apps, [return_maps])}; Error -> Error end. describe_plugins(Name) -> Path = emqx_mgmt_api_test_util:api_path(["plugins", Name]), case emqx_mgmt_api_test_util:request_api(get, Path) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. @@ -172,7 +172,7 @@ update_boot_order(Name, MoveBody) -> Auth = emqx_mgmt_api_test_util:auth_header_(), Path = emqx_mgmt_api_test_util:api_path(["plugins", Name, "move"]), case emqx_mgmt_api_test_util:request_api(post, Path, "", Auth, MoveBody) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. @@ -206,7 +206,7 @@ create_renamed_package(PackagePath, NewNameVsn) -> NewPackagePath. update_release_json(["release.json"], FileContent, NewName) -> - ContentMap = emqx_json:decode(FileContent, [return_maps]), - emqx_json:encode(ContentMap#{<<"name">> => NewName}); + ContentMap = emqx_utils_json:decode(FileContent, [return_maps]), + emqx_utils_json:encode(ContentMap#{<<"name">> => NewName}); update_release_json(_FileName, FileContent, _NewName) -> FileContent. diff --git a/apps/emqx_management/test/emqx_mgmt_api_publish_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_publish_SUITE.erl index 44b8069f5..303a73b41 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_publish_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_publish_SUITE.erl @@ -352,4 +352,4 @@ receive_assert(Topic, Qos, Payload) -> end. decode_json(In) -> - emqx_json:decode(In, [return_maps]). + emqx_utils_json:decode(In, [return_maps]). diff --git a/apps/emqx_management/test/emqx_mgmt_api_stats_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_stats_SUITE.erl index 7236ac9e4..2550bdbe2 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_stats_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_stats_SUITE.erl @@ -24,16 +24,18 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> + meck:expect(emqx, running_nodes, 0, [node(), 'fake@node']), emqx_mgmt_api_test_util:init_suite(), Config. end_per_suite(_) -> + meck:unload(emqx), emqx_mgmt_api_test_util:end_suite(). t_stats_api(_) -> S = emqx_mgmt_api_test_util:api_path(["stats?aggregate=false"]), {ok, S1} = emqx_mgmt_api_test_util:request_api(get, S), - [Stats1] = emqx_json:decode(S1, [return_maps]), + [Stats1] = emqx_utils_json:decode(S1, [return_maps]), SystemStats1 = emqx_mgmt:get_stats(), Fun1 = fun(Key) -> @@ -43,7 +45,7 @@ t_stats_api(_) -> StatsPath = emqx_mgmt_api_test_util:api_path(["stats?aggregate=true"]), SystemStats = emqx_mgmt:get_stats(), {ok, StatsResponse} = emqx_mgmt_api_test_util:request_api(get, StatsPath), - Stats = emqx_json:decode(StatsResponse, [return_maps]), + Stats = emqx_utils_json:decode(StatsResponse, [return_maps]), ?assertEqual(erlang:length(maps:keys(SystemStats)), erlang:length(maps:keys(Stats))), Fun = fun(Key) -> diff --git a/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl index b9e9fffd8..a23d70f2f 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl @@ -55,7 +55,7 @@ t_subscription_api(Config) -> {ok, _, _} = emqtt:subscribe(Client, ?TOPIC2), Path = emqx_mgmt_api_test_util:api_path(["subscriptions"]), {ok, Response} = emqx_mgmt_api_test_util:request_api(get, Path), - Data = emqx_json:decode(Response, [return_maps]), + Data = emqx_utils_json:decode(Response, [return_maps]), Meta = maps:get(<<"meta">>, Data), ?assertEqual(1, maps:get(<<"page">>, Meta)), ?assertEqual(emqx_mgmt:default_row_limit(), maps:get(<<"limit">>, Meta)), @@ -158,7 +158,7 @@ t_list_with_internal_subscription(_Config) -> request_json(Method, Query, Headers) when is_list(Query) -> Qs = uri_string:compose_query(Query), {ok, MatchRes} = emqx_mgmt_api_test_util:request_api(Method, path(), Qs, Headers), - emqx_json:decode(MatchRes, [return_maps]). + emqx_utils_json:decode(MatchRes, [return_maps]). path() -> emqx_mgmt_api_test_util:api_path(["subscriptions"]). diff --git a/apps/emqx_management/test/emqx_mgmt_api_test_util.erl b/apps/emqx_management/test/emqx_mgmt_api_test_util.erl index 782f493fe..985b95d5b 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_test_util.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_test_util.erl @@ -108,7 +108,8 @@ request_api(Method, Url, QueryParams, AuthOrHeaders, Body, Opts) when end, do_request_api( Method, - {NewUrl, build_http_header(AuthOrHeaders), "application/json", emqx_json:encode(Body)}, + {NewUrl, build_http_header(AuthOrHeaders), "application/json", + emqx_utils_json:encode(Body)}, Opts ). @@ -156,7 +157,7 @@ api_path_without_base_path(Parts) -> %% %% Usage with RequestData: %% Payload = [{upload_type, <<"user_picture">>}], -%% PayloadContent = jsx:encode(Payload), +%% PayloadContent = emqx_utils_json:encode(Payload), %% RequestData = [ %% {<<"payload">>, PayloadContent} %% ] diff --git a/apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl index 0c2e684b4..659ae0d44 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl @@ -49,7 +49,7 @@ t_nodes_api(Config) -> %% list all Path = emqx_mgmt_api_test_util:api_path(["topics"]), {ok, Response} = emqx_mgmt_api_test_util:request_api(get, Path), - RoutesData = emqx_json:decode(Response, [return_maps]), + RoutesData = emqx_utils_json:decode(Response, [return_maps]), Meta = maps:get(<<"meta">>, RoutesData), ?assertEqual(1, maps:get(<<"page">>, Meta)), ?assertEqual(emqx_mgmt:default_row_limit(), maps:get(<<"limit">>, Meta)), @@ -68,7 +68,7 @@ t_nodes_api(Config) -> ]), Headers = emqx_mgmt_api_test_util:auth_header_(), {ok, MatchResponse} = emqx_mgmt_api_test_util:request_api(get, Path, QS, Headers), - MatchData = emqx_json:decode(MatchResponse, [return_maps]), + MatchData = emqx_utils_json:decode(MatchResponse, [return_maps]), ?assertMatch( #{<<"count">> := 1, <<"page">> := 1, <<"limit">> := 100}, maps:get(<<"meta">>, MatchData) @@ -90,6 +90,6 @@ t_nodes_api(Config) -> [ #{<<"topic">> := Topic, <<"node">> := Node1}, #{<<"topic">> := Topic, <<"node">> := Node2} - ] = emqx_json:decode(RouteResponse, [return_maps]), + ] = emqx_utils_json:decode(RouteResponse, [return_maps]), ?assertEqual(lists:usort([Node, atom_to_binary(Slave)]), lists:usort([Node1, Node2])). diff --git a/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl index 162d07aaa..0102eb56c 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl @@ -174,6 +174,13 @@ t_create_failed(_Config) -> {error, {"HTTP/1.1", 409, _}}, request_api(post, api_path("trace"), [GoodName2 | Trace]) ), + %% new name but bad payload-encode + GoodName3 = {<<"name">>, <<"test-name-2">>}, + PayloadEncode = {<<"payload_encode">>, <<"bad">>}, + ?assertMatch( + {error, {"HTTP/1.1", 400, _}}, + request_api(post, api_path("trace"), [GoodName3, PayloadEncode | Trace]) + ), unload(), emqx_trace:clear(), @@ -377,7 +384,7 @@ api_path(Path) -> emqx_mgmt_api_test_util:api_path([Path]). json(Data) -> - {ok, Jsx} = emqx_json:safe_decode(Data, [return_maps]), + {ok, Jsx} = emqx_utils_json:safe_decode(Data, [return_maps]), Jsx. load() -> diff --git a/apps/emqx_modules/README.md b/apps/emqx_modules/README.md new file mode 100644 index 000000000..dfa349514 --- /dev/null +++ b/apps/emqx_modules/README.md @@ -0,0 +1,53 @@ +# EMQX Modules + +The application provides some minor functional modules that are not included in the MQTT +protocol standard, including "Delayed Publish", "Topic Rewrite", "Topic Metrics" and "Telemetry". + + +## Delayed Publish + +After enabling this module, messages sent by the clients with the topic prefixed with +`$delayed/{Interval}/{Topic}` will be delayed by `{Interval}` seconds before +being published to the `{Topic}`. + +More introduction about [Delayed Publish](https://www.emqx.io/docs/en/v5.0/mqtt/mqtt-delayed-publish.html). + +See [Enabling/Disabling Delayed Publish via HTTP API](https://www.emqx.io/docs/en/v5.0/admin/api-docs.html#tag/MQTT/paths/~1mqtt~1delayed/put). + + +## Topic Rewrite + +Topic Rewrite allows users to configure rules to change the topic strings that +the client requests to subscribe or publish. + +This feature is very useful when there's a need to transform between different topic structures. +For example, an old device that has already been issued and cannot +be upgraded may use old topic designs, but for some reason, we adjusted the format of topics. We can use this feature to rewrite the old topics as the new format to eliminate these differences. + +More introduction about [Topic Rewrite](https://www.emqx.io/docs/en/v5.0/mqtt/mqtt-topic-rewrite.html). + +See [List all rewrite rules](https://www.emqx.io/docs/en/v5.0/admin/api-docs.html#tag/MQTT/paths/~1mqtt~1topic_rewrite/get) +and [Create or Update rewrite rules](https://www.emqx.io/docs/en/v5.0/admin/api-docs.html#tag/MQTT/paths/~1mqtt~1topic_rewrite/put). + + +## Topic Metrics + +Topic Metrics is used for users to specify monitoring of certain topics and to +count the number of messages, QoS distribution, and rate for all messages on that topic. + +More introduction about [Topic Metrics](https://www.emqx.io/docs/en/v5.0/dashboard/diagnose.html#topic-metrics). + +See HTTP API docs to [List all monitored topics](https://www.emqx.io/docs/en/v5.0/admin/api-docs.html#tag/MQTT/paths/~1mqtt~1topic_metrics/get), +[Create topic metrics](https://www.emqx.io/docs/en/v5.0/admin/api-docs.html#tag/MQTT/paths/~1mqtt~1topic_metrics/post) +and [Get the monitored result](https://www.emqx.io/docs/en/v5.0/admin/api-docs.html#tag/MQTT/paths/~1mqtt~1topic_metrics~1%7Btopic%7D/get). + + +## Telemetry + +Telemetry is used for collecting non-sensitive information about the EMQX cluster. + +More introduction about [Telemetry](https://www.emqx.io/docs/en/v5.0/telemetry/telemetry.html#telemetry). + +See HTTP API docs to [Enable/Disable telemetry](https://www.emqx.io/docs/en/v5.0/admin/api-docs.html#tag/Telemetry/paths/~1telemetry~1status/put), +[Get the enabled status](https://www.emqx.io/docs/en/v5.0/admin/api-docs.html#tag/Telemetry/paths/~1telemetry~1status/get) +and [Get the data of the module collected](https://www.emqx.io/docs/en/v5.0/admin/api-docs.html#tag/Telemetry/paths/~1telemetry~1data/get). diff --git a/apps/emqx_modules/rebar.config b/apps/emqx_modules/rebar.config index 9688d5043..ff542aed7 100644 --- a/apps/emqx_modules/rebar.config +++ b/apps/emqx_modules/rebar.config @@ -2,6 +2,7 @@ {deps, [ {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}}, {emqx_conf, {path, "../emqx_conf"}} ]}. {project_plugins, [erlfmt]}. diff --git a/apps/emqx_modules/src/emqx_delayed.erl b/apps/emqx_modules/src/emqx_delayed.erl index 91cdbedd3..85313c181 100644 --- a/apps/emqx_modules/src/emqx_delayed.erl +++ b/apps/emqx_modules/src/emqx_delayed.erl @@ -328,7 +328,7 @@ handle_info(Info, State) -> terminate(_Reason, #{stats_timer := StatsTimer} = State) -> emqx_conf:remove_handler([delayed]), - emqx_misc:cancel_timer(StatsTimer), + emqx_utils:cancel_timer(StatsTimer), do_load_or_unload(false, State). code_change(_Vsn, State, _Extra) -> @@ -370,14 +370,14 @@ ensure_publish_timer({Ts, _Id}, State = #{publish_timer := undefined}) -> ensure_publish_timer({Ts, _Id}, State = #{publish_timer := TRef, publish_at := PubAt}) when Ts < PubAt -> - ok = emqx_misc:cancel_timer(TRef), + ok = emqx_utils:cancel_timer(TRef), ensure_publish_timer(Ts, ?NOW, State); ensure_publish_timer(_Key, State) -> State. ensure_publish_timer(Ts, Now, State) -> Interval = max(1, Ts - Now), - TRef = emqx_misc:start_timer(Interval, do_publish), + TRef = emqx_utils:start_timer(Interval, do_publish), State#{publish_timer := TRef, publish_at := Now + Interval}. do_publish(Key, Now) -> @@ -418,7 +418,7 @@ do_load_or_unload(true, State) -> State; do_load_or_unload(false, #{publish_timer := PubTimer} = State) -> emqx_hooks:del('message.publish', {?MODULE, on_message_publish}), - emqx_misc:cancel_timer(PubTimer), + emqx_utils:cancel_timer(PubTimer), ets:delete_all_objects(?TAB), State#{publish_timer := undefined, publish_at := 0}; do_load_or_unload(_, State) -> diff --git a/apps/emqx_modules/src/emqx_modules.app.src b/apps/emqx_modules/src/emqx_modules.app.src index fdc13f354..1e38e25d1 100644 --- a/apps/emqx_modules/src/emqx_modules.app.src +++ b/apps/emqx_modules/src/emqx_modules.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_modules, [ {description, "EMQX Modules"}, - {vsn, "5.0.12"}, + {vsn, "5.0.13"}, {modules, []}, {applications, [kernel, stdlib, emqx, emqx_ctl]}, {mod, {emqx_modules_app, []}}, diff --git a/apps/emqx_modules/src/emqx_modules_schema.erl b/apps/emqx_modules/src/emqx_modules_schema.erl index ddb8f37a3..36a08de60 100644 --- a/apps/emqx_modules/src/emqx_modules_schema.erl +++ b/apps/emqx_modules/src/emqx_modules_schema.erl @@ -34,8 +34,14 @@ roots() -> [ "delayed", "telemetry", - array("rewrite", #{desc => "List of topic rewrite rules."}), - array("topic_metrics", #{desc => "List of topics whose metrics are reported."}) + array("rewrite", #{ + desc => "List of topic rewrite rules.", + importance => ?IMPORTANCE_HIDDEN + }), + array("topic_metrics", #{ + desc => "List of topics whose metrics are reported.", + importance => ?IMPORTANCE_HIDDEN + }) ]. fields("telemetry") -> diff --git a/apps/emqx_modules/src/emqx_telemetry.erl b/apps/emqx_modules/src/emqx_telemetry.erl index 6d5c772f0..3b27302df 100644 --- a/apps/emqx_modules/src/emqx_telemetry.erl +++ b/apps/emqx_modules/src/emqx_telemetry.erl @@ -161,7 +161,7 @@ handle_call(enable, _From, State) -> FirstReportTimeoutMS = timer:seconds(10), {reply, ok, ensure_report_timer(FirstReportTimeoutMS, State)}; handle_call(disable, _From, State = #state{timer = Timer}) -> - emqx_misc:cancel_timer(Timer), + emqx_utils:cancel_timer(Timer), {reply, ok, State#state{timer = undefined}}; handle_call(get_node_uuid, _From, State = #state{node_uuid = UUID}) -> {reply, {ok, UUID}, State}; @@ -205,7 +205,7 @@ ensure_report_timer(State = #state{report_interval = ReportInterval}) -> ensure_report_timer(ReportInterval, State). ensure_report_timer(ReportInterval, State) -> - State#state{timer = emqx_misc:start_timer(ReportInterval, time_to_report_telemetry_data)}. + State#state{timer = emqx_utils:start_timer(ReportInterval, time_to_report_telemetry_data)}. os_info() -> case erlang:system_info(os_type) of @@ -356,7 +356,7 @@ get_telemetry(State0 = #state{node_uuid = NodeUUID, cluster_uuid = ClusterUUID}) report_telemetry(State0 = #state{url = URL}) -> {State, Data} = get_telemetry(State0), - case emqx_json:safe_encode(Data) of + case emqx_utils_json:safe_encode(Data) of {ok, Bin} -> httpc_request(post, URL, [], Bin), ?tp(debug, telemetry_data_reported, #{}); diff --git a/apps/emqx_modules/src/emqx_telemetry_api.erl b/apps/emqx_modules/src/emqx_telemetry_api.erl index 798c3ad17..b7209d146 100644 --- a/apps/emqx_modules/src/emqx_telemetry_api.erl +++ b/apps/emqx_modules/src/emqx_telemetry_api.erl @@ -243,7 +243,7 @@ status(put, #{body := Body}) -> data(get, _Request) -> case emqx_modules_conf:is_telemetry_enabled() of true -> - {200, emqx_json:encode(get_telemetry_data())}; + {200, emqx_utils_json:encode(get_telemetry_data())}; false -> {404, #{ code => ?NOT_FOUND, diff --git a/apps/emqx_modules/src/emqx_topic_metrics.erl b/apps/emqx_modules/src/emqx_topic_metrics.erl index dfc6b07ab..de09e568f 100644 --- a/apps/emqx_modules/src/emqx_topic_metrics.erl +++ b/apps/emqx_modules/src/emqx_topic_metrics.erl @@ -201,7 +201,7 @@ reset() -> init([Opts]) -> erlang:process_flag(trap_exit, true), - ok = emqx_tables:new(?TAB, [{read_concurrency, true}]), + ok = emqx_utils_ets:new(?TAB, [{read_concurrency, true}]), erlang:send_after(timer:seconds(?TICKING_INTERVAL), self(), ticking), Fun = fun(#{topic := Topic}, CurrentSpeeds) -> diff --git a/apps/emqx_modules/test/emqx_delayed_api_SUITE.erl b/apps/emqx_modules/test/emqx_delayed_api_SUITE.erl index ed3cd9292..4ae3dec88 100644 --- a/apps/emqx_modules/test/emqx_delayed_api_SUITE.erl +++ b/apps/emqx_modules/test/emqx_delayed_api_SUITE.erl @@ -229,8 +229,8 @@ t_large_payload(_) -> %%-------------------------------------------------------------------- decode_json(Data) -> - BinJson = emqx_json:decode(Data, [return_maps]), - emqx_map_lib:unsafe_atom_key_map(BinJson). + BinJson = emqx_utils_json:decode(Data, [return_maps]), + emqx_utils_maps:unsafe_atom_key_map(BinJson). clear_all_record() -> ets:delete_all_objects(emqx_delayed). diff --git a/apps/emqx_modules/test/emqx_rewrite_api_SUITE.erl b/apps/emqx_modules/test/emqx_rewrite_api_SUITE.erl index ddb136f1e..68d12a2c1 100644 --- a/apps/emqx_modules/test/emqx_rewrite_api_SUITE.erl +++ b/apps/emqx_modules/test/emqx_rewrite_api_SUITE.erl @@ -75,7 +75,7 @@ t_mqtt_topic_rewrite(_) -> ?assertEqual( Rules, - emqx_json:decode(Result, [return_maps]) + emqx_utils_json:decode(Result, [return_maps]) ). t_mqtt_topic_rewrite_limit(_) -> diff --git a/apps/emqx_modules/test/emqx_telemetry_SUITE.erl b/apps/emqx_modules/test/emqx_telemetry_SUITE.erl index cee255e77..bb5f39c1f 100644 --- a/apps/emqx_modules/test/emqx_telemetry_SUITE.erl +++ b/apps/emqx_modules/test/emqx_telemetry_SUITE.erl @@ -45,6 +45,7 @@ init_per_suite(Config) -> ok = emqx_common_test_helpers:load_config(emqx_modules_schema, ?BASE_CONF, #{ raw_with_default => true }), + emqx_gateway_test_utils:load_all_gateway_apps(), emqx_common_test_helpers:start_apps( [emqx_conf, emqx_authn, emqx_authz, emqx_modules], fun set_special_configs/1 @@ -511,7 +512,7 @@ t_send_after_enable(_) -> ), receive {request, post, _URL, _Headers, Body} -> - {ok, Decoded} = emqx_json:safe_decode(Body, [return_maps]), + {ok, Decoded} = emqx_utils_json:safe_decode(Body, [return_maps]), ?assertMatch( #{ <<"uuid">> := _, diff --git a/apps/emqx_modules/test/emqx_telemetry_api_SUITE.erl b/apps/emqx_modules/test/emqx_telemetry_api_SUITE.erl index 16f942bc0..ac6d12039 100644 --- a/apps/emqx_modules/test/emqx_telemetry_api_SUITE.erl +++ b/apps/emqx_modules/test/emqx_telemetry_api_SUITE.erl @@ -113,7 +113,7 @@ t_status(_) -> ?assertEqual( #{<<"enable">> => false}, - jsx:decode(Result0) + emqx_utils_json:decode(Result0) ), ?assertMatch( @@ -139,7 +139,7 @@ t_status(_) -> ?assertEqual( #{<<"enable">> => true}, - jsx:decode(Result1) + emqx_utils_json:decode(Result1) ), ?assertMatch( @@ -180,7 +180,7 @@ t_data(_) -> <<"uuid">> := _, <<"vm_specs">> := _ }, - jsx:decode(Result) + emqx_utils_json:decode(Result) ), {ok, 200, _} = diff --git a/apps/emqx_modules/test/emqx_topic_metrics_api_SUITE.erl b/apps/emqx_modules/test/emqx_topic_metrics_api_SUITE.erl index ea85d1fe9..5d64f123d 100644 --- a/apps/emqx_modules/test/emqx_topic_metrics_api_SUITE.erl +++ b/apps/emqx_modules/test/emqx_topic_metrics_api_SUITE.erl @@ -74,7 +74,7 @@ t_mqtt_topic_metrics_collection(_) -> ?assertEqual( [], - jsx:decode(Result0) + emqx_utils_json:decode(Result0) ), {ok, 200, _} = request( @@ -95,7 +95,7 @@ t_mqtt_topic_metrics_collection(_) -> <<"metrics">> := #{} } ], - jsx:decode(Result1) + emqx_utils_json:decode(Result1) ), ?assertMatch( @@ -150,7 +150,7 @@ t_mqtt_topic_metrics(_) -> uri(["mqtt", "topic_metrics"]) ), - ?assertMatch([_], jsx:decode(Result0)), + ?assertMatch([_], emqx_utils_json:decode(Result0)), {ok, 200, Result1} = request( get, @@ -162,7 +162,7 @@ t_mqtt_topic_metrics(_) -> <<"topic">> := <<"topic/1/2">>, <<"metrics">> := #{} }, - jsx:decode(Result1) + emqx_utils_json:decode(Result1) ), ?assertMatch( @@ -288,7 +288,7 @@ t_node_aggregation(_) -> <<"topic">> := <<"topic/1/2">>, <<"metrics">> := #{<<"messages.dropped.count">> := 3} }, - jsx:decode(Result) + emqx_utils_json:decode(Result) ), meck:unload(emqx_topic_metrics_proto_v1). diff --git a/apps/emqx_plugin_libs/rebar.config b/apps/emqx_plugin_libs/rebar.config index 9f17b7657..dee2902a5 100644 --- a/apps/emqx_plugin_libs/rebar.config +++ b/apps/emqx_plugin_libs/rebar.config @@ -1,5 +1,8 @@ %% -*- mode: erlang -*- -{deps, [{emqx, {path, "../emqx"}}]}. +{deps, [ + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}} +]}. {project_plugins, [erlfmt]}. diff --git a/apps/emqx_plugin_libs/src/emqx_placeholder.erl b/apps/emqx_plugin_libs/src/emqx_placeholder.erl index 1f93c1d3e..18ef9e8fb 100644 --- a/apps/emqx_plugin_libs/src/emqx_placeholder.erl +++ b/apps/emqx_plugin_libs/src/emqx_placeholder.erl @@ -240,7 +240,7 @@ sql_data(Bin) when is_binary(Bin) -> Bin; sql_data(Num) when is_number(Num) -> Num; sql_data(Bool) when is_boolean(Bool) -> Bool; sql_data(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8); -sql_data(Map) when is_map(Map) -> emqx_json:encode(Map). +sql_data(Map) when is_map(Map) -> emqx_utils_json:encode(Map). -spec bin(term()) -> binary(). bin(Val) -> emqx_plugin_libs_rule:bin(Val). diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src index dcb330df4..24b5a3240 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_plugin_libs, [ {description, "EMQX Plugin utility libs"}, - {vsn, "4.3.8"}, + {vsn, "4.3.9"}, {modules, []}, {applications, [kernel, stdlib]}, {env, []} diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl b/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl index d1a821895..8844fe586 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl @@ -225,7 +225,7 @@ tcp_connectivity(Host, Port) -> ) -> ok | {error, Reason :: term()}. tcp_connectivity(Host, Port, Timeout) -> - case gen_tcp:connect(Host, Port, emqx_misc:ipv6_probe([]), Timeout) of + case gen_tcp:connect(Host, Port, emqx_utils:ipv6_probe([]), Timeout) of {ok, Sock} -> gen_tcp:close(Sock), ok; @@ -236,11 +236,11 @@ tcp_connectivity(Host, Port, Timeout) -> str(Bin) when is_binary(Bin) -> binary_to_list(Bin); str(Num) when is_number(Num) -> number_to_list(Num); str(Atom) when is_atom(Atom) -> atom_to_list(Atom); -str(Map) when is_map(Map) -> binary_to_list(emqx_json:encode(Map)); +str(Map) when is_map(Map) -> binary_to_list(emqx_utils_json:encode(Map)); str(List) when is_list(List) -> case io_lib:printable_list(List) of true -> List; - false -> binary_to_list(emqx_json:encode(List)) + false -> binary_to_list(emqx_utils_json:encode(List)) end; str(Data) -> error({invalid_str, Data}). @@ -258,11 +258,11 @@ utf8_str(Str) -> bin(Bin) when is_binary(Bin) -> Bin; bin(Num) when is_number(Num) -> number_to_binary(Num); bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8); -bin(Map) when is_map(Map) -> emqx_json:encode(Map); +bin(Map) when is_map(Map) -> emqx_utils_json:encode(Map); bin(List) when is_list(List) -> case io_lib:printable_list(List) of true -> list_to_binary(List); - false -> emqx_json:encode(List) + false -> emqx_utils_json:encode(List) end; bin(Data) -> error({invalid_bin, Data}). @@ -312,7 +312,7 @@ float2str(Float, Precision) when is_float(Float) and is_integer(Precision) -> float_to_binary(Float, [{decimals, Precision}, compact]). map(Bin) when is_binary(Bin) -> - case emqx_json:decode(Bin, [return_maps]) of + case emqx_utils_json:decode(Bin, [return_maps]) of Map = #{} -> Map; _ -> error({invalid_map, Bin}) end; diff --git a/apps/emqx_plugins/src/emqx_plugins.app.src b/apps/emqx_plugins/src/emqx_plugins.app.src index c0372c003..d5c16ea59 100644 --- a/apps/emqx_plugins/src/emqx_plugins.app.src +++ b/apps/emqx_plugins/src/emqx_plugins.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_plugins, [ {description, "EMQX Plugin Management"}, - {vsn, "0.1.3"}, + {vsn, "0.1.4"}, {modules, []}, {mod, {emqx_plugins_app, []}}, {applications, [kernel, stdlib, emqx]}, diff --git a/apps/emqx_plugins/src/emqx_plugins.erl b/apps/emqx_plugins/src/emqx_plugins.erl index 264247086..04faa44e9 100644 --- a/apps/emqx_plugins/src/emqx_plugins.erl +++ b/apps/emqx_plugins/src/emqx_plugins.erl @@ -479,22 +479,39 @@ ensure_exists_and_installed(NameVsn) -> case filelib:is_dir(dir(NameVsn)) of true -> ok; - _ -> - Nodes = [N || N <- mria:running_nodes(), N /= node()], - case get_from_any_node(Nodes, NameVsn, []) of + false -> + %% Do we have the package, but it's not extracted yet? + case get_tar(NameVsn) of {ok, TarContent} -> ok = file:write_file(pkg_file(NameVsn), TarContent), ok = do_ensure_installed(NameVsn); - {error, NodeErrors} -> - ?SLOG(error, #{ - msg => "failed_to_copy_plugin_from_other_nodes", - name_vsn => NameVsn, - node_errors => NodeErrors - }), - {error, plugin_not_found} + _ -> + %% If not, try to get it from the cluster. + do_get_from_cluster(NameVsn) end end. +do_get_from_cluster(NameVsn) -> + Nodes = [N || N <- mria:running_nodes(), N /= node()], + case get_from_any_node(Nodes, NameVsn, []) of + {ok, TarContent} -> + ok = file:write_file(pkg_file(NameVsn), TarContent), + ok = do_ensure_installed(NameVsn); + {error, NodeErrors} when Nodes =/= [] -> + ?SLOG(error, #{ + msg => "failed_to_copy_plugin_from_other_nodes", + name_vsn => NameVsn, + node_errors => NodeErrors + }), + {error, plugin_not_found}; + {error, _} -> + ?SLOG(error, #{ + msg => "no_nodes_to_copy_plugin_from", + name_vsn => NameVsn + }), + {error, plugin_not_found} + end. + get_from_any_node([], _NameVsn, Errors) -> {error, Errors}; get_from_any_node([Node | T], NameVsn, Errors) -> diff --git a/apps/emqx_plugins/src/emqx_plugins_schema.erl b/apps/emqx_plugins/src/emqx_plugins_schema.erl index 9d9d045de..b86f6b6c1 100644 --- a/apps/emqx_plugins/src/emqx_plugins_schema.erl +++ b/apps/emqx_plugins/src/emqx_plugins_schema.erl @@ -29,7 +29,7 @@ namespace() -> "plugin". -roots() -> [?CONF_ROOT]. +roots() -> [{?CONF_ROOT, ?HOCON(?R_REF(?CONF_ROOT), #{importance => ?IMPORTANCE_LOW})}]. fields(?CONF_ROOT) -> #{ @@ -73,16 +73,19 @@ states(type) -> ?ARRAY(?R_REF(state)); states(required) -> false; states(default) -> []; states(desc) -> ?DESC(states); +states(importance) -> ?IMPORTANCE_HIGH; states(_) -> undefined. install_dir(type) -> string(); install_dir(required) -> false; -%% runner's root dir +%% runner's root dir todo move to data dir in 5.1 install_dir(default) -> <<"plugins">>; -install_dir(T) when T =/= desc -> undefined; -install_dir(desc) -> ?DESC(install_dir). +install_dir(desc) -> ?DESC(install_dir); +install_dir(importance) -> ?IMPORTANCE_LOW; +install_dir(_) -> undefined. check_interval(type) -> emqx_schema:duration(); check_interval(default) -> <<"5s">>; -check_interval(T) when T =/= desc -> undefined; -check_interval(desc) -> ?DESC(check_interval). +check_interval(desc) -> ?DESC(check_interval); +check_interval(deprecated) -> {since, "5.0.24"}; +check_interval(_) -> undefined. diff --git a/apps/emqx_plugins/test/emqx_plugins_SUITE.erl b/apps/emqx_plugins/test/emqx_plugins_SUITE.erl index 260ad1681..14d6d06fc 100644 --- a/apps/emqx_plugins/test/emqx_plugins_SUITE.erl +++ b/apps/emqx_plugins/test/emqx_plugins_SUITE.erl @@ -45,7 +45,10 @@ all() -> groups() -> [ - {copy_plugin, [sequence], [group_t_copy_plugin_to_a_new_node]}, + {copy_plugin, [sequence], [ + group_t_copy_plugin_to_a_new_node, + group_t_copy_plugin_to_a_new_node_single_node + ]}, {create_tar_copy_plugin, [sequence], [group_t_copy_plugin_to_a_new_node]} ]. @@ -601,6 +604,78 @@ group_t_copy_plugin_to_a_new_node(Config) -> rpc:call(CopyToNode, emqx_plugins, describe, [NameVsn]) ). +%% checks that we can start a cluster with a lone node. +group_t_copy_plugin_to_a_new_node_single_node({init, Config}) -> + PrivDataDir = ?config(priv_dir, Config), + ToInstallDir = filename:join(PrivDataDir, "plugins_copy_to"), + file:del_dir_r(ToInstallDir), + ok = filelib:ensure_path(ToInstallDir), + #{package := Package, release_name := PluginName} = get_demo_plugin_package(ToInstallDir), + NameVsn = filename:basename(Package, ?PACKAGE_SUFFIX), + [{CopyTo, CopyToOpts}] = + emqx_common_test_helpers:emqx_cluster( + [ + {core, plugins_copy_to} + ], + #{ + apps => [emqx_conf, emqx_plugins], + env => [ + {emqx, init_config_load_done, false}, + {emqx, boot_modules, []} + ], + env_handler => fun + (emqx_plugins) -> + ok = emqx_plugins:put_config(install_dir, ToInstallDir), + %% this is to simulate an user setting the state + %% via environment variables before starting the node + ok = emqx_plugins:put_config( + states, + [#{name_vsn => NameVsn, enable => true}] + ), + ok; + (_) -> + ok + end, + priv_data_dir => PrivDataDir, + schema_mod => emqx_conf_schema, + peer_mod => slave, + load_schema => true + } + ), + [ + {to_install_dir, ToInstallDir}, + {copy_to_node_name, CopyTo}, + {copy_to_opts, CopyToOpts}, + {name_vsn, NameVsn}, + {plugin_name, PluginName} + | Config + ]; +group_t_copy_plugin_to_a_new_node_single_node({'end', Config}) -> + CopyToNode = proplists:get_value(copy_to_node, Config), + ok = emqx_common_test_helpers:stop_slave(CopyToNode), + ok = file:del_dir_r(proplists:get_value(to_install_dir, Config)), + ok; +group_t_copy_plugin_to_a_new_node_single_node(Config) -> + CopyTo = ?config(copy_to_node_name, Config), + CopyToOpts = ?config(copy_to_opts, Config), + ToInstallDir = ?config(to_install_dir, Config), + NameVsn = proplists:get_value(name_vsn, Config), + %% Start the node for the first time. The plugin should start + %% successfully even if it's not extracted yet. Simply starting + %% the node would crash if not working properly. + CopyToNode = emqx_common_test_helpers:start_slave(CopyTo, CopyToOpts), + ct:pal("~p config:\n ~p", [ + CopyToNode, erpc:call(CopyToNode, emqx_plugins, get_config, [[], #{}]) + ]), + ct:pal("~p install_dir:\n ~p", [ + CopyToNode, erpc:call(CopyToNode, file, list_dir, [ToInstallDir]) + ]), + ?assertMatch( + {ok, #{running_status := running, config_status := enabled}}, + rpc:call(CopyToNode, emqx_plugins, describe, [NameVsn]) + ), + ok. + make_tar(Cwd, NameWithVsn) -> make_tar(Cwd, NameWithVsn, NameWithVsn). diff --git a/apps/emqx_prometheus/README.md b/apps/emqx_prometheus/README.md index ddff9774c..c008adc81 100644 --- a/apps/emqx_prometheus/README.md +++ b/apps/emqx_prometheus/README.md @@ -1,279 +1,16 @@ -# emqx-prometheus +# EMQX Prometheus Agent -EMQX Prometheus Agent - -## push emqx stats/metrics to prometheus PushGateway - -``` -prometheus.push.gateway.server = http://127.0.0.1:9091 - -prometheus.interval = 15000 -``` - -## pull emqx stats/metrics - -``` -Method: GET -Path: api/v4/emqx_prometheus?type=prometheus -params: type: [prometheus| json] - -prometheus data - -# TYPE erlang_vm_ets_limit gauge -erlang_vm_ets_limit 256000 -# TYPE erlang_vm_logical_processors gauge -erlang_vm_logical_processors 4 -# TYPE erlang_vm_logical_processors_available gauge -erlang_vm_logical_processors_available NaN -# TYPE erlang_vm_logical_processors_online gauge -erlang_vm_logical_processors_online 4 -# TYPE erlang_vm_port_count gauge -erlang_vm_port_count 17 -# TYPE erlang_vm_port_limit gauge -erlang_vm_port_limit 1048576 +This application provides the ability to integrate with Prometheus. It provides +an HTTP API for collecting metrics of the current node +and also supports configuring a Push Gateway URL address for pushing these metrics. -json data +More introduction about [Integrate with Prometheus](https://www.emqx.io/docs/en/v5.0/observability/prometheus.html#integrate-with-prometheus) -{ - "stats": {key:value}, - "metrics": {key:value}, - "packets": {key:value}, - "messages": {key:value}, - "delivery": {key:value}, - "client": {key:value}, - "session": {key:value} -} - -``` +See HTTP API docs to learn how to +[Update Prometheus config](https://www.emqx.io/docs/en/v5.0/admin/api-docs.html#tag/Monitor/paths/~1prometheus/put) +and [Get all metrics data](https://www.emqx.io/docs/en/v5.0/admin/api-docs.html#tag/Monitor/paths/~1prometheus~1stats/get). -## Before EMQX v4.0.0 -The prometheus data simple is: - - -```bash -# TYPE erlang_vm_ets_limit gauge -erlang_vm_ets_limit 2097152 -# TYPE erlang_vm_logical_processors gauge -erlang_vm_logical_processors 2 -# TYPE erlang_vm_logical_processors_available gauge -erlang_vm_logical_processors_available 2 -# TYPE erlang_vm_logical_processors_online gauge -erlang_vm_logical_processors_online 2 -# TYPE erlang_vm_port_count gauge -erlang_vm_port_count 19 -# TYPE erlang_vm_port_limit gauge -erlang_vm_port_limit 1048576 -# TYPE erlang_vm_process_count gauge -erlang_vm_process_count 460 -# TYPE erlang_vm_process_limit gauge -erlang_vm_process_limit 2097152 -# TYPE erlang_vm_schedulers gauge -erlang_vm_schedulers 2 -# TYPE erlang_vm_schedulers_online gauge -erlang_vm_schedulers_online 2 -# TYPE erlang_vm_smp_support untyped -erlang_vm_smp_support 1 -# TYPE erlang_vm_threads untyped -erlang_vm_threads 1 -# TYPE erlang_vm_thread_pool_size gauge -erlang_vm_thread_pool_size 32 -# TYPE erlang_vm_time_correction untyped -erlang_vm_time_correction 1 -# TYPE erlang_vm_statistics_context_switches counter -erlang_vm_statistics_context_switches 39850 -# TYPE erlang_vm_statistics_garbage_collection_number_of_gcs counter -erlang_vm_statistics_garbage_collection_number_of_gcs 17116 -# TYPE erlang_vm_statistics_garbage_collection_words_reclaimed counter -erlang_vm_statistics_garbage_collection_words_reclaimed 55711819 -# TYPE erlang_vm_statistics_garbage_collection_bytes_reclaimed counter -erlang_vm_statistics_garbage_collection_bytes_reclaimed 445694552 -# TYPE erlang_vm_statistics_bytes_received_total counter -erlang_vm_statistics_bytes_received_total 400746 -# TYPE erlang_vm_statistics_bytes_output_total counter -erlang_vm_statistics_bytes_output_total 337197 -# TYPE erlang_vm_statistics_reductions_total counter -erlang_vm_statistics_reductions_total 21157980 -# TYPE erlang_vm_statistics_run_queues_length_total gauge -erlang_vm_statistics_run_queues_length_total 0 -# TYPE erlang_vm_statistics_runtime_milliseconds counter -erlang_vm_statistics_runtime_milliseconds 6559 -# TYPE erlang_vm_statistics_wallclock_time_milliseconds counter -erlang_vm_statistics_wallclock_time_milliseconds 261243 -# TYPE erlang_vm_memory_atom_bytes_total gauge -erlang_vm_memory_atom_bytes_total{usage="used"} 1814822 -erlang_vm_memory_atom_bytes_total{usage="free"} 22459 -# TYPE erlang_vm_memory_bytes_total gauge -erlang_vm_memory_bytes_total{kind="system"} 109820104 -erlang_vm_memory_bytes_total{kind="processes"} 44983656 -# TYPE erlang_vm_dets_tables gauge -erlang_vm_dets_tables 1 -# TYPE erlang_vm_ets_tables gauge -erlang_vm_ets_tables 139 -# TYPE erlang_vm_memory_processes_bytes_total gauge -erlang_vm_memory_processes_bytes_total{usage="used"} 44983656 -erlang_vm_memory_processes_bytes_total{usage="free"} 0 -# TYPE erlang_vm_memory_system_bytes_total gauge -erlang_vm_memory_system_bytes_total{usage="atom"} 1837281 -erlang_vm_memory_system_bytes_total{usage="binary"} 595872 -erlang_vm_memory_system_bytes_total{usage="code"} 40790577 -erlang_vm_memory_system_bytes_total{usage="ets"} 37426896 -erlang_vm_memory_system_bytes_total{usage="other"} 29169478 -# TYPE erlang_mnesia_held_locks gauge -erlang_mnesia_held_locks 0 -# TYPE erlang_mnesia_lock_queue gauge -erlang_mnesia_lock_queue 0 -# TYPE erlang_mnesia_transaction_participants gauge -erlang_mnesia_transaction_participants 0 -# TYPE erlang_mnesia_transaction_coordinators gauge -erlang_mnesia_transaction_coordinators 0 -# TYPE erlang_mnesia_failed_transactions counter -erlang_mnesia_failed_transactions 2 -# TYPE erlang_mnesia_committed_transactions counter -erlang_mnesia_committed_transactions 239 -# TYPE erlang_mnesia_logged_transactions counter -erlang_mnesia_logged_transactions 60 -# TYPE erlang_mnesia_restarted_transactions counter -erlang_mnesia_restarted_transactions 0 -# TYPE emqx_packets_auth_received counter -emqx_packets_auth_received 0 -# TYPE emqx_packets_auth_sent counter -emqx_packets_auth_sent 0 -# TYPE emqx_packets_received counter -emqx_packets_received 0 -# TYPE emqx_packets_sent counter -emqx_packets_sent 0 -# TYPE emqx_packets_connect counter -emqx_packets_connect 0 -# TYPE emqx_packets_connack_sent counter -emqx_packets_connack_sent 0 -# TYPE emqx_packets_connack_error counter -emqx_packets_connack_error 0 -# TYPE emqx_packets_connack_auth_error counter -emqx_packets_connack_auth_error 0 -# TYPE emqx_packets_disconnect_received counter -emqx_packets_disconnect_received 0 -# TYPE emqx_packets_disconnect_sent counter -emqx_packets_disconnect_sent 0 -# TYPE emqx_packets_subscribe counter -emqx_packets_subscribe 0 -# TYPE emqx_packets_subscribe_error counter -emqx_packets_subscribe_error 0 -# TYPE emqx_packets_subscribe_auth_error counter -emqx_packets_subscribe_auth_error 0 -# TYPE emqx_packets_suback counter -emqx_packets_suback 0 -# TYPE emqx_packets_unsubscribe counter -emqx_packets_unsubscribe 0 -# TYPE emqx_packets_unsubscribe_error counter -emqx_packets_unsubscribe_error 0 -# TYPE emqx_packets_unsuback counter -emqx_packets_unsuback 0 -# TYPE emqx_packets_publish_received counter -emqx_packets_publish_received 0 -# TYPE emqx_packets_publish_sent counter -emqx_packets_publish_sent 0 -# TYPE emqx_packets_publish_auth_error counter -emqx_packets_publish_auth_error 0 -# TYPE emqx_packets_publish_error counter -emqx_packets_publish_error 0 -# TYPE emqx_packets_puback_received counter -emqx_packets_puback_received 0 -# TYPE emqx_packets_puback_sent counter -emqx_packets_puback_sent 0 -# TYPE emqx_packets_puback_missed counter -emqx_packets_puback_missed 0 -# TYPE emqx_packets_pubrec_received counter -emqx_packets_pubrec_received 0 -# TYPE emqx_packets_pubrec_sent counter -emqx_packets_pubrec_sent 0 -# TYPE emqx_packets_pubrec_missed counter -emqx_packets_pubrec_missed 0 -# TYPE emqx_packets_pubrel_received counter -emqx_packets_pubrel_received 0 -# TYPE emqx_packets_pubrel_sent counter -emqx_packets_pubrel_sent 0 -# TYPE emqx_packets_pubrel_missed counter -emqx_packets_pubrel_missed 0 -# TYPE emqx_packets_pubcomp_received counter -emqx_packets_pubcomp_received 0 -# TYPE emqx_packets_pubcomp_sent counter -emqx_packets_pubcomp_sent 0 -# TYPE emqx_packets_pubcomp_missed counter -emqx_packets_pubcomp_missed 0 -# TYPE emqx_packets_pingreq counter -emqx_packets_pingreq 0 -# TYPE emqx_packets_pingresp counter -emqx_packets_pingresp 0 -# TYPE emqx_bytes_received counter -emqx_bytes_received 0 -# TYPE emqx_bytes_sent counter -emqx_bytes_sent 0 -# TYPE emqx_connections_count gauge -emqx_connections_count 0 -# TYPE emqx_connections_max gauge -emqx_connections_max 0 -# TYPE emqx_retained_count gauge -emqx_retained_count 3 -# TYPE emqx_retained_max gauge -emqx_retained_max 3 -# TYPE emqx_sessions_count gauge -emqx_sessions_count 0 -# TYPE emqx_sessions_max gauge -emqx_sessions_max 0 -# TYPE emqx_subscriptions_count gauge -emqx_subscriptions_count 0 -# TYPE emqx_subscriptions_max gauge -emqx_subscriptions_max 0 -# TYPE emqx_topics_count gauge -emqx_topics_count 0 -# TYPE emqx_topics_max gauge -emqx_topics_max 0 -# TYPE emqx_vm_cpu_use gauge -emqx_vm_cpu_use 100.0 -# TYPE emqx_vm_cpu_idle gauge -emqx_vm_cpu_idle 0.0 -# TYPE emqx_vm_run_queue gauge -emqx_vm_run_queue 1 -# TYPE emqx_vm_process_messages_in_queues gauge -emqx_vm_process_messages_in_queues 0 -# TYPE emqx_messages_received counter -emqx_messages_received 0 -# TYPE emqx_messages_sent counter -emqx_messages_sent 0 -# TYPE emqx_messages_dropped counter -emqx_messages_dropped 0 -# TYPE emqx_messages_retained counter -emqx_messages_retained 3 -# TYPE emqx_messages_qos0_received counter -emqx_messages_qos0_received 0 -# TYPE emqx_messages_qos0_sent counter -emqx_messages_qos0_sent 0 -# TYPE emqx_messages_qos1_received counter -emqx_messages_qos1_received 0 -# TYPE emqx_messages_qos1_sent counter -emqx_messages_qos1_sent 0 -# TYPE emqx_messages_qos2_received counter -emqx_messages_qos2_received 0 -# TYPE emqx_messages_qos2_expired counter -emqx_messages_qos2_expired 0 -# TYPE emqx_messages_qos2_sent counter -emqx_messages_qos2_sent 0 -# TYPE emqx_messages_qos2_dropped counter -emqx_messages_qos2_dropped 0 -# TYPE emqx_messages_forward counter -emqx_messages_forward 0 -``` - - -License -------- - -Apache License Version 2.0 - -Author ------- - -EMQX Team. - +Correspondingly, we have also provided a [Grafana template](https://grafana.com/grafana/dashboards/17446-emqx/) +for visualizing these metrics. diff --git a/apps/emqx_prometheus/TODO b/apps/emqx_prometheus/TODO deleted file mode 100644 index a868fba7e..000000000 --- a/apps/emqx_prometheus/TODO +++ /dev/null @@ -1,2 +0,0 @@ -1. Add more VM Metrics -2. Add more emqx Metrics diff --git a/apps/emqx_prometheus/rebar.config b/apps/emqx_prometheus/rebar.config index 88b3d27a2..7b9a6cc48 100644 --- a/apps/emqx_prometheus/rebar.config +++ b/apps/emqx_prometheus/rebar.config @@ -2,6 +2,7 @@ {deps, [ {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}}, {prometheus, {git, "https://github.com/deadtrickster/prometheus.erl", {tag, "v4.8.1"}}} ]}. diff --git a/apps/emqx_prometheus/src/emqx_prometheus.app.src b/apps/emqx_prometheus/src/emqx_prometheus.app.src index 1e7e59f7a..f94b22d81 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.app.src +++ b/apps/emqx_prometheus/src/emqx_prometheus.app.src @@ -2,7 +2,7 @@ {application, emqx_prometheus, [ {description, "Prometheus for EMQX"}, % strict semver, bump manually! - {vsn, "5.0.8"}, + {vsn, "5.0.10"}, {modules, []}, {registered, [emqx_prometheus_sup]}, {applications, [kernel, stdlib, prometheus, emqx, emqx_management]}, diff --git a/apps/emqx_prometheus/src/emqx_prometheus.erl b/apps/emqx_prometheus/src/emqx_prometheus.erl index 60d52f58b..d999f294e 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus.erl @@ -144,7 +144,7 @@ terminate(_Reason, _State) -> ok. ensure_timer(Interval) -> - emqx_misc:start_timer(Interval, ?TIMER_MSG). + emqx_utils:start_timer(Interval, ?TIMER_MSG). %%-------------------------------------------------------------------- %% prometheus callbacks @@ -599,8 +599,8 @@ emqx_cluster() -> ]. emqx_cluster_data() -> - Running = mria:cluster_nodes(running), - Stopped = mria:cluster_nodes(stopped), + Running = emqx:cluster_nodes(running), + Stopped = emqx:cluster_nodes(stopped), [ {nodes_running, length(Running)}, {nodes_stopped, length(Stopped)} diff --git a/apps/emqx_prometheus/src/emqx_prometheus_api.erl b/apps/emqx_prometheus/src/emqx_prometheus_api.erl index 945c6eba9..d3bfc0224 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_api.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_api.erl @@ -122,13 +122,7 @@ prometheus_config_example() -> interval => "15s", push_gateway_server => <<"http://127.0.0.1:9091">>, headers => #{'header-name' => 'header-value'}, - job_name => <<"${name}/instance/${name}~${host}">>, - vm_dist_collector => enabled, - mnesia_collector => enabled, - vm_statistics_collector => enabled, - vm_system_info_collector => enabled, - vm_memory_collector => enabled, - vm_msacc_collector => enabled + job_name => <<"${name}/instance/${name}~${host}">> }. prometheus_data_schema() -> diff --git a/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl b/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl index c4867d9fd..e29d46720 100644 --- a/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl +++ b/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl @@ -66,7 +66,7 @@ t_prometheus_api(_) -> Auth = emqx_mgmt_api_test_util:auth_header_(), {ok, Response} = emqx_mgmt_api_test_util:request_api(get, Path, "", Auth), - Conf = emqx_json:decode(Response, [return_maps]), + Conf = emqx_utils_json:decode(Response, [return_maps]), ?assertMatch( #{ <<"push_gateway_server">> := _, @@ -84,7 +84,7 @@ t_prometheus_api(_) -> NewConf = Conf#{<<"interval">> => <<"2s">>, <<"vm_statistics_collector">> => <<"disabled">>}, {ok, Response2} = emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, NewConf), - Conf2 = emqx_json:decode(Response2, [return_maps]), + Conf2 = emqx_utils_json:decode(Response2, [return_maps]), ?assertMatch(NewConf, Conf2), ?assertEqual({ok, []}, application:get_env(prometheus, vm_statistics_collector_metrics)), ?assertEqual({ok, all}, application:get_env(prometheus, vm_memory_collector_metrics)), @@ -106,7 +106,7 @@ t_stats_api(_) -> Headers = [{"accept", "application/json"}, Auth], {ok, Response} = emqx_mgmt_api_test_util:request_api(get, Path, "", Headers), - Data = emqx_json:decode(Response, [return_maps]), + Data = emqx_utils_json:decode(Response, [return_maps]), ?assertMatch(#{<<"client">> := _, <<"delivery">> := _}, Data), {ok, _} = emqx_mgmt_api_test_util:request_api(get, Path, "", Auth), diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl index d799e7d93..91572eac3 100644 --- a/apps/emqx_resource/include/emqx_resource.hrl +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -43,8 +43,7 @@ config := resource_config(), error := term(), state := resource_state(), - status := resource_status(), - metrics => emqx_metrics_worker:metrics() + status := resource_status() }. -type resource_group() :: binary(). -type creation_opts() :: #{ @@ -71,7 +70,7 @@ auto_restart_interval => pos_integer(), batch_size => pos_integer(), batch_time => pos_integer(), - max_queue_bytes => pos_integer(), + max_buffer_bytes => pos_integer(), query_mode => query_mode(), resume_interval => pos_integer(), inflight_window => pos_integer() @@ -86,11 +85,8 @@ -define(WORKER_POOL_SIZE, 16). --define(DEFAULT_QUEUE_SEG_SIZE, 10 * 1024 * 1024). --define(DEFAULT_QUEUE_SEG_SIZE_RAW, <<"10MB">>). - --define(DEFAULT_QUEUE_SIZE, 100 * 1024 * 1024). --define(DEFAULT_QUEUE_SIZE_RAW, <<"100MB">>). +-define(DEFAULT_BUFFER_BYTES, 256 * 1024 * 1024). +-define(DEFAULT_BUFFER_BYTES_RAW, <<"256MB">>). -define(DEFAULT_REQUEST_TIMEOUT, timer:seconds(15)). diff --git a/apps/emqx_resource/src/emqx_resource.app.src b/apps/emqx_resource/src/emqx_resource.app.src index fbfe8c1fa..2553e6dd8 100644 --- a/apps/emqx_resource/src/emqx_resource.app.src +++ b/apps/emqx_resource/src/emqx_resource.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_resource, [ {description, "Manager for all external resources"}, - {vsn, "0.1.11"}, + {vsn, "0.1.14"}, {registered, []}, {mod, {emqx_resource_app, []}}, {applications, [ diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index 0f7e93a9b..d8b91942b 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -103,6 +103,7 @@ list_instances_verbose/0, %% return the data of the instance get_instance/1, + get_metrics/1, fetch_creation_opts/1, %% return all the instances of the same resource type list_instances_by_type/1, @@ -311,7 +312,12 @@ set_resource_status_connecting(ResId) -> -spec get_instance(resource_id()) -> {ok, resource_group(), resource_data()} | {error, Reason :: term()}. get_instance(ResId) -> - emqx_resource_manager:lookup_cached(ResId, [metrics]). + emqx_resource_manager:lookup_cached(ResId). + +-spec get_metrics(resource_id()) -> + emqx_metrics_worker:metrics(). +get_metrics(ResId) -> + emqx_resource_manager:get_metrics(ResId). -spec fetch_creation_opts(map()) -> creation_opts(). fetch_creation_opts(Opts) -> @@ -321,9 +327,12 @@ fetch_creation_opts(Opts) -> list_instances() -> [Id || #{id := Id} <- list_instances_verbose()]. --spec list_instances_verbose() -> [resource_data()]. +-spec list_instances_verbose() -> [_ResourceDataWithMetrics :: map()]. list_instances_verbose() -> - emqx_resource_manager:list_all(). + [ + Res#{metrics => get_metrics(ResId)} + || #{id := ResId} = Res <- emqx_resource_manager:list_all() + ]. -spec list_instances_by_type(module()) -> [resource_id()]. list_instances_by_type(ResourceType) -> diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl index 0fa4c0bd8..2e2cd5631 100644 --- a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl @@ -52,7 +52,7 @@ -export([queue_item_marshaller/1, estimate_size/1]). --export([handle_async_reply/2, handle_async_batch_reply/2]). +-export([handle_async_reply/2, handle_async_batch_reply/2, reply_call/2]). -export([clear_disk_queue_dir/2]). @@ -178,20 +178,7 @@ init({Id, Index, Opts}) -> process_flag(trap_exit, true), true = gproc_pool:connect_worker(Id, {Id, Index}), BatchSize = maps:get(batch_size, Opts, ?DEFAULT_BATCH_SIZE), - SegBytes0 = maps:get(queue_seg_bytes, Opts, ?DEFAULT_QUEUE_SEG_SIZE), - TotalBytes = maps:get(max_queue_bytes, Opts, ?DEFAULT_QUEUE_SIZE), - SegBytes = min(SegBytes0, TotalBytes), - QueueOpts = - #{ - dir => disk_queue_dir(Id, Index), - marshaller => fun ?MODULE:queue_item_marshaller/1, - max_total_bytes => TotalBytes, - %% we don't want to retain the queue after - %% resource restarts. - offload => {true, volatile}, - seg_bytes => SegBytes, - sizer => fun ?MODULE:estimate_size/1 - }, + QueueOpts = replayq_opts(Id, Index, Opts), Queue = replayq:open(QueueOpts), emqx_resource_metrics:queuing_set(Id, Index, queue_count(Queue)), emqx_resource_metrics:inflight_set(Id, Index, 0), @@ -214,7 +201,7 @@ init({Id, Index, Opts}) -> resume_interval => ResumeInterval, tref => undefined }, - ?tp(buffer_worker_init, #{id => Id, index => Index}), + ?tp(buffer_worker_init, #{id => Id, index => Index, queue_opts => QueueOpts}), {ok, running, Data}. running(enter, _, #{tref := _Tref} = Data) -> @@ -306,10 +293,8 @@ code_change(_OldVsn, State, _Extra) -> pick_call(Id, Key, Query, Timeout) -> ?PICK(Id, Key, Pid, begin - Caller = self(), MRef = erlang:monitor(process, Pid, [{alias, reply_demonitor}]), - From = {Caller, MRef}, - ReplyTo = {fun gen_statem:reply/2, [From]}, + ReplyTo = {fun ?MODULE:reply_call/2, [MRef]}, erlang:send(Pid, ?SEND_REQ(ReplyTo, Query)), receive {MRef, Response} -> @@ -1535,7 +1520,7 @@ queue_count(Q) -> disk_queue_dir(Id, Index) -> QDir0 = binary_to_list(Id) ++ ":" ++ integer_to_list(Index), QDir = filename:join([emqx:data_dir(), "bufs", node(), QDir0]), - emqx_misc:safe_filename(QDir). + emqx_utils:safe_filename(QDir). clear_disk_queue_dir(Id, Index) -> ReplayQDir = disk_queue_dir(Id, Index), @@ -1679,6 +1664,32 @@ adjust_batch_time(Id, RequestTimeout, BatchTime0) -> end, BatchTime. +replayq_opts(Id, Index, Opts) -> + BufferMode = maps:get(buffer_mode, Opts, memory_only), + TotalBytes = maps:get(max_buffer_bytes, Opts, ?DEFAULT_BUFFER_BYTES), + case BufferMode of + memory_only -> + #{ + mem_only => true, + marshaller => fun ?MODULE:queue_item_marshaller/1, + max_total_bytes => TotalBytes, + sizer => fun ?MODULE:estimate_size/1 + }; + volatile_offload -> + SegBytes0 = maps:get(buffer_seg_bytes, Opts, TotalBytes), + SegBytes = min(SegBytes0, TotalBytes), + #{ + dir => disk_queue_dir(Id, Index), + marshaller => fun ?MODULE:queue_item_marshaller/1, + max_total_bytes => TotalBytes, + %% we don't want to retain the queue after + %% resource restarts. + offload => {true, volatile}, + seg_bytes => SegBytes, + sizer => fun ?MODULE:estimate_size/1 + } + end. + %% The request timeout should be greater than the resume interval, as %% it defines how often the buffer worker tries to unblock. If request %% timeout is <= resume interval and the buffer worker is ever @@ -1690,6 +1701,17 @@ default_resume_interval(_RequestTimeout = infinity, HealthCheckInterval) -> default_resume_interval(RequestTimeout, HealthCheckInterval) -> max(1, min(HealthCheckInterval, RequestTimeout div 3)). +-spec reply_call(reference(), term()) -> ok. +reply_call(Alias, Response) -> + %% Since we use a reference created with `{alias, + %% reply_demonitor}', after we `demonitor' it in case of a + %% timeout, we won't send any more messages that the caller is not + %% expecting anymore. Using `gen_statem:reply({pid(), + %% reference()}, _)' would still send a late reply even after the + %% demonitor. + erlang:send(Alias, {Alias, Response}), + ok. + -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). adjust_batch_time_test_() -> diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index 6a4919b41..877b35fff 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -37,7 +37,6 @@ list_all/0, list_group/1, lookup_cached/1, - lookup_cached/2, get_metrics/1, reset_metrics/1 ]). @@ -231,25 +230,14 @@ set_resource_status_connecting(ResId) -> -spec lookup(resource_id()) -> {ok, resource_group(), resource_data()} | {error, not_found}. lookup(ResId) -> case safe_call(ResId, lookup, ?T_LOOKUP) of - {error, timeout} -> lookup_cached(ResId, [metrics]); + {error, timeout} -> lookup_cached(ResId); Result -> Result end. %% @doc Lookup the group and data of a resource from the cache -spec lookup_cached(resource_id()) -> {ok, resource_group(), resource_data()} | {error, not_found}. lookup_cached(ResId) -> - lookup_cached(ResId, []). - -%% @doc Lookup the group and data of a resource from the cache --spec lookup_cached(resource_id(), [Option]) -> - {ok, resource_group(), resource_data()} | {error, not_found} -when - Option :: metrics. -lookup_cached(ResId, Options) -> - NeedMetrics = lists:member(metrics, Options), case read_cache(ResId) of - {Group, Data} when NeedMetrics -> - {ok, Group, data_record_to_external_map_with_metrics(Data)}; {Group, Data} -> {ok, Group, data_record_to_external_map(Data)}; not_found -> @@ -270,7 +258,7 @@ reset_metrics(ResId) -> list_all() -> try [ - data_record_to_external_map_with_metrics(Data) + data_record_to_external_map(Data) || {_Id, _Group, Data} <- ets:tab2list(?ETS_TABLE) ] catch @@ -366,7 +354,7 @@ handle_event({call, From}, {remove, ClearMetrics}, _State, Data) -> handle_remove_event(From, ClearMetrics, Data); % Called when the state-data of the resource is being looked up. handle_event({call, From}, lookup, _State, #data{group = Group} = Data) -> - Reply = {ok, Group, data_record_to_external_map_with_metrics(Data)}, + Reply = {ok, Group, data_record_to_external_map(Data)}, {keep_state_and_data, [{reply, From, Reply}]}; % Called when doing a manually health check. handle_event({call, From}, health_check, stopped, _Data) -> @@ -387,7 +375,7 @@ handle_event(state_timeout, health_check, connecting, Data) -> %% and successful health_checks handle_event(enter, _OldState, connected = State, Data) -> ok = log_state_consistency(State, Data), - _ = emqx_alarm:deactivate(Data#data.id), + _ = emqx_alarm:safe_deactivate(Data#data.id), ?tp(resource_connected_enter, #{}), {keep_state_and_data, health_check_actions(Data)}; handle_event(state_timeout, health_check, connected, Data) -> @@ -523,10 +511,10 @@ start_resource(Data, From) -> id => Data#data.id, reason => Reason }), - _ = maybe_alarm(disconnected, Data#data.id, Data#data.error), + _ = maybe_alarm(disconnected, Data#data.id, Err, Data#data.error), %% Keep track of the error reason why the connection did not work %% so that the Reason can be returned when the verification call is made. - UpdatedData = Data#data{status = disconnected, error = Reason}, + UpdatedData = Data#data{status = disconnected, error = Err}, Actions = maybe_reply(retry_actions(UpdatedData), From, Err), {next_state, disconnected, update_state(UpdatedData, Data), Actions} end. @@ -551,7 +539,7 @@ stop_resource(#data{state = ResState, id = ResId} = Data) -> Data#data{status = stopped}. make_test_id() -> - RandId = iolist_to_binary(emqx_misc:gen_id(16)), + RandId = iolist_to_binary(emqx_utils:gen_id(16)), <>. handle_manually_health_check(From, Data) -> @@ -594,11 +582,11 @@ handle_connected_health_check(Data) -> with_health_check(#data{state = undefined} = Data, Func) -> Func(disconnected, Data); -with_health_check(Data, Func) -> +with_health_check(#data{error = PrevError} = Data, Func) -> ResId = Data#data.id, HCRes = emqx_resource:call_health_check(Data#data.manager_id, Data#data.mod, Data#data.state), {Status, NewState, Err} = parse_health_check_result(HCRes, Data), - _ = maybe_alarm(Status, ResId, Err), + _ = maybe_alarm(Status, ResId, Err, PrevError), ok = maybe_resume_resource_workers(ResId, Status), UpdatedData = Data#data{ state = NewState, status = Status, error = Err @@ -617,21 +605,25 @@ update_state(Data, _DataWas) -> health_check_interval(Opts) -> maps:get(health_check_interval, Opts, ?HEALTHCHECK_INTERVAL). -maybe_alarm(connected, _ResId, _Error) -> +maybe_alarm(connected, _ResId, _Error, _PrevError) -> ok; -maybe_alarm(_Status, <>, _Error) -> +maybe_alarm(_Status, <>, _Error, _PrevError) -> ok; -maybe_alarm(_Status, ResId, Error) -> +%% Assume that alarm is already active +maybe_alarm(_Status, _ResId, Error, Error) -> + ok; +maybe_alarm(_Status, ResId, Error, _PrevError) -> HrError = case Error of - undefined -> <<"Unknown reason">>; - _Else -> emqx_misc:readable_error_msg(Error) + {error, undefined} -> <<"Unknown reason">>; + {error, Reason} -> emqx_utils:readable_error_msg(Reason) end, - emqx_alarm:activate( + emqx_alarm:safe_activate( ResId, #{resource_id => ResId, reason => resource_down}, <<"resource down: ", HrError/binary>> - ). + ), + ?tp(resource_activate_alarm, #{resource_id => ResId}). maybe_resume_resource_workers(ResId, connected) -> lists:foreach( @@ -644,14 +636,14 @@ maybe_resume_resource_workers(_, _) -> maybe_clear_alarm(<>) -> ok; maybe_clear_alarm(ResId) -> - emqx_alarm:deactivate(ResId). + emqx_alarm:safe_deactivate(ResId). parse_health_check_result(Status, Data) when ?IS_STATUS(Status) -> - {Status, Data#data.state, undefined}; + {Status, Data#data.state, status_to_error(Status)}; parse_health_check_result({Status, NewState}, _Data) when ?IS_STATUS(Status) -> - {Status, NewState, undefined}; + {Status, NewState, status_to_error(Status)}; parse_health_check_result({Status, NewState, Error}, _Data) when ?IS_STATUS(Status) -> - {Status, NewState, Error}; + {Status, NewState, {error, Error}}; parse_health_check_result({error, Error}, Data) -> ?SLOG( error, @@ -661,7 +653,16 @@ parse_health_check_result({error, Error}, Data) -> reason => Error } ), - {disconnected, Data#data.state, Error}. + {disconnected, Data#data.state, {error, Error}}. + +status_to_error(connected) -> + undefined; +status_to_error(_) -> + {error, undefined}. + +%% Compatibility +external_error({error, Reason}) -> Reason; +external_error(Other) -> Other. maybe_reply(Actions, undefined, _Reply) -> Actions; @@ -672,7 +673,7 @@ maybe_reply(Actions, From, Reply) -> data_record_to_external_map(Data) -> #{ id => Data#data.id, - error => Data#data.error, + error => external_error(Data#data.error), mod => Data#data.mod, callback_mode => Data#data.callback_mode, query_mode => Data#data.query_mode, @@ -681,11 +682,6 @@ data_record_to_external_map(Data) -> state => Data#data.state }. --spec data_record_to_external_map_with_metrics(data()) -> resource_data(). -data_record_to_external_map_with_metrics(Data) -> - DataMap = data_record_to_external_map(Data), - DataMap#{metrics => get_metrics(Data#data.id)}. - -spec wait_for_ready(resource_id(), integer()) -> ok | timeout | {error, term()}. wait_for_ready(ResId, WaitTime) -> do_wait_for_ready(ResId, WaitTime div ?WAIT_FOR_RESOURCE_DELAY). @@ -696,8 +692,8 @@ do_wait_for_ready(ResId, Retry) -> case read_cache(ResId) of {_Group, #data{status = connected}} -> ok; - {_Group, #data{status = disconnected, error = Reason}} -> - {error, Reason}; + {_Group, #data{status = disconnected, error = Err}} -> + {error, external_error(Err)}; _ -> timer:sleep(?WAIT_FOR_RESOURCE_DELAY), do_wait_for_ready(ResId, Retry - 1) diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl b/apps/emqx_resource/src/emqx_resource_pool.erl similarity index 80% rename from apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl rename to apps/emqx_resource/src/emqx_resource_pool.erl index 9b286f360..913b29c86 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl +++ b/apps/emqx_resource/src/emqx_resource_pool.erl @@ -14,31 +14,27 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_plugin_libs_pool). +-module(emqx_resource_pool). -export([ - start_pool/3, - stop_pool/1, - pool_name/1, - health_check_ecpool_workers/2, - health_check_ecpool_workers/3 + start/3, + stop/1, + health_check_workers/2, + health_check_workers/3 ]). -include_lib("emqx/include/logger.hrl"). -define(HEALTH_CHECK_TIMEOUT, 15000). -pool_name(ID) when is_binary(ID) -> - list_to_atom(binary_to_list(ID)). - -start_pool(Name, Mod, Options) -> +start(Name, Mod, Options) -> case ecpool:start_sup_pool(Name, Mod, Options) of {ok, _} -> ?SLOG(info, #{msg => "start_ecpool_ok", pool_name => Name}), ok; {error, {already_started, _Pid}} -> - stop_pool(Name), - start_pool(Name, Mod, Options); + stop(Name), + start(Name, Mod, Options); {error, Reason} -> NReason = parse_reason(Reason), ?SLOG(error, #{ @@ -49,7 +45,7 @@ start_pool(Name, Mod, Options) -> {error, {start_pool_failed, Name, NReason}} end. -stop_pool(Name) -> +stop(Name) -> case ecpool:stop_sup_pool(Name) of ok -> ?SLOG(info, #{msg => "stop_ecpool_ok", pool_name => Name}); @@ -64,10 +60,10 @@ stop_pool(Name) -> error({stop_pool_failed, Name, Reason}) end. -health_check_ecpool_workers(PoolName, CheckFunc) -> - health_check_ecpool_workers(PoolName, CheckFunc, ?HEALTH_CHECK_TIMEOUT). +health_check_workers(PoolName, CheckFunc) -> + health_check_workers(PoolName, CheckFunc, ?HEALTH_CHECK_TIMEOUT). -health_check_ecpool_workers(PoolName, CheckFunc, Timeout) -> +health_check_workers(PoolName, CheckFunc, Timeout) -> Workers = [Worker || {_WorkerName, Worker} <- ecpool:workers(PoolName)], DoPerWorker = fun(Worker) -> @@ -79,7 +75,7 @@ health_check_ecpool_workers(PoolName, CheckFunc, Timeout) -> false end end, - try emqx_misc:pmap(DoPerWorker, Workers, Timeout) of + try emqx_utils:pmap(DoPerWorker, Workers, Timeout) of [_ | _] = Status -> lists:all(fun(St) -> St =:= true end, Status); [] -> diff --git a/apps/emqx_resource/src/schema/emqx_resource_schema.erl b/apps/emqx_resource/src/schema/emqx_resource_schema.erl index 647a40fed..3b4fb66e5 100644 --- a/apps/emqx_resource/src/schema/emqx_resource_schema.erl +++ b/apps/emqx_resource/src/schema/emqx_resource_schema.erl @@ -40,6 +40,7 @@ fields("resource_opts") -> ]; fields("creation_opts") -> [ + {buffer_mode, fun buffer_mode/1}, {worker_pool_size, fun worker_pool_size/1}, {health_check_interval, fun health_check_interval/1}, {resume_interval, fun resume_interval/1}, @@ -53,7 +54,8 @@ fields("creation_opts") -> {batch_size, fun batch_size/1}, {batch_time, fun batch_time/1}, {enable_queue, fun enable_queue/1}, - {max_queue_bytes, fun max_queue_bytes/1} + {max_buffer_bytes, fun max_buffer_bytes/1}, + {buffer_seg_bytes, fun buffer_seg_bytes/1} ]. resource_opts_meta() -> @@ -143,10 +145,24 @@ batch_time(default) -> ?DEFAULT_BATCH_TIME_RAW; batch_time(required) -> false; batch_time(_) -> undefined. -max_queue_bytes(type) -> emqx_schema:bytesize(); -max_queue_bytes(desc) -> ?DESC("max_queue_bytes"); -max_queue_bytes(default) -> ?DEFAULT_QUEUE_SIZE_RAW; -max_queue_bytes(required) -> false; -max_queue_bytes(_) -> undefined. +max_buffer_bytes(type) -> emqx_schema:bytesize(); +max_buffer_bytes(aliases) -> [max_queue_bytes]; +max_buffer_bytes(desc) -> ?DESC("max_buffer_bytes"); +max_buffer_bytes(default) -> ?DEFAULT_BUFFER_BYTES_RAW; +max_buffer_bytes(required) -> false; +max_buffer_bytes(_) -> undefined. + +buffer_mode(type) -> enum([memory_only, volatile_offload]); +buffer_mode(desc) -> ?DESC("buffer_mode"); +buffer_mode(default) -> memory_only; +buffer_mode(required) -> false; +buffer_mode(importance) -> ?IMPORTANCE_HIDDEN; +buffer_mode(_) -> undefined. + +buffer_seg_bytes(type) -> emqx_schema:bytesize(); +buffer_seg_bytes(desc) -> ?DESC("buffer_seg_bytes"); +buffer_seg_bytes(required) -> false; +buffer_seg_bytes(importance) -> ?IMPORTANCE_HIDDEN; +buffer_seg_bytes(_) -> undefined. desc("creation_opts") -> ?DESC("creation_opts"). diff --git a/apps/emqx_resource/test/emqx_connector_demo.erl b/apps/emqx_resource/test/emqx_connector_demo.erl index a1393c574..96e22c6b6 100644 --- a/apps/emqx_resource/test/emqx_connector_demo.erl +++ b/apps/emqx_resource/test/emqx_connector_demo.erl @@ -62,6 +62,7 @@ set_callback_mode(Mode) -> persistent_term:put(?CM_KEY, Mode). on_start(_InstId, #{create_error := true}) -> + ?tp(connector_demo_start_error, #{}), error("some error"); on_start(InstId, #{name := Name} = Opts) -> Register = maps:get(register, Opts, false), @@ -144,7 +145,11 @@ on_query(_InstId, {sleep_before_reply, For}, #{pid := Pid}) -> Result after 1000 -> {error, timeout} - end. + end; +on_query(_InstId, {sync_sleep_before_reply, SleepFor}, _State) -> + %% This simulates a slow sync call + timer:sleep(SleepFor), + {ok, slept}. on_query_async(_InstId, block, ReplyFun, #{pid := Pid}) -> Pid ! {block, ReplyFun}, @@ -239,6 +244,7 @@ batch_big_payload({async, ReplyFunAndArgs}, InstId, Batch, State = #{pid := Pid} {ok, Pid}. on_get_status(_InstId, #{health_check_error := true}) -> + ?tp(connector_demo_health_check_error, #{}), disconnected; on_get_status(_InstId, #{pid := Pid}) -> timer:sleep(300), diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index 8638c381f..f8ddd56b5 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -349,7 +349,7 @@ t_query_counter_async_query(_) -> ?assertMatch([#{query := {query, _, get_counter, _, _}}], QueryTrace) end ), - {ok, _, #{metrics := #{counters := C}}} = emqx_resource:get_instance(?ID), + #{counters := C} = emqx_resource:get_metrics(?ID), ?assertMatch(#{matched := 1002, 'success' := 1002, 'failed' := 0}, C), ok = emqx_resource:remove_local(?ID). @@ -402,7 +402,7 @@ t_query_counter_async_callback(_) -> ?assertMatch([#{query := {query, _, get_counter, _, _}}], QueryTrace) end ), - {ok, _, #{metrics := #{counters := C}}} = emqx_resource:get_instance(?ID), + #{counters := C} = emqx_resource:get_metrics(?ID), ?assertMatch(#{matched := 1002, 'success' := 1002, 'failed' := 0}, C), ?assertMatch(1000, ets:info(Tab0, size)), ?assert( @@ -1314,7 +1314,8 @@ t_delete_and_re_create_with_same_name(_Config) -> query_mode => sync, batch_size => 1, worker_pool_size => NumBufferWorkers, - queue_seg_bytes => 100, + buffer_mode => volatile_offload, + buffer_seg_bytes => 100, resume_interval => 1_000 } ), @@ -1373,7 +1374,7 @@ t_delete_and_re_create_with_same_name(_Config) -> query_mode => async, batch_size => 1, worker_pool_size => 2, - queue_seg_bytes => 100, + buffer_seg_bytes => 100, resume_interval => 1_000 } ), @@ -1405,7 +1406,7 @@ t_always_overflow(_Config) -> query_mode => sync, batch_size => 1, worker_pool_size => 1, - max_queue_bytes => 1, + max_buffer_bytes => 1, resume_interval => 1_000 } ), @@ -2634,8 +2635,199 @@ t_call_mode_uncoupled_from_query_mode(_Config) -> Trace2 ) ), + ok + end + ). + +%% The default mode is currently `memory_only'. +t_volatile_offload_mode(_Config) -> + MaxBufferBytes = 1_000, + DefaultOpts = #{ + max_buffer_bytes => MaxBufferBytes, + worker_pool_size => 1 + }, + ?check_trace( + begin + emqx_connector_demo:set_callback_mode(async_if_possible), + %% Create without any specified segment bytes; should + %% default to equal max bytes. + ?assertMatch( + {ok, _}, + emqx_resource:create( + ?ID, + ?DEFAULT_RESOURCE_GROUP, + ?TEST_RESOURCE, + #{name => test_resource}, + DefaultOpts#{buffer_mode => volatile_offload} + ) + ), + ?assertEqual(ok, emqx_resource:remove_local(?ID)), + + %% Create with segment bytes < max bytes + ?assertMatch( + {ok, _}, + emqx_resource:create( + ?ID, + ?DEFAULT_RESOURCE_GROUP, + ?TEST_RESOURCE, + #{name => test_resource}, + DefaultOpts#{ + buffer_mode => volatile_offload, + buffer_seg_bytes => MaxBufferBytes div 2 + } + ) + ), + ?assertEqual(ok, emqx_resource:remove_local(?ID)), + %% Create with segment bytes = max bytes + ?assertMatch( + {ok, _}, + emqx_resource:create( + ?ID, + ?DEFAULT_RESOURCE_GROUP, + ?TEST_RESOURCE, + #{name => test_resource}, + DefaultOpts#{ + buffer_mode => volatile_offload, + buffer_seg_bytes => MaxBufferBytes + } + ) + ), + ?assertEqual(ok, emqx_resource:remove_local(?ID)), + + %% Create with segment bytes > max bytes; should normalize + %% to max bytes. + ?assertMatch( + {ok, _}, + emqx_resource:create( + ?ID, + ?DEFAULT_RESOURCE_GROUP, + ?TEST_RESOURCE, + #{name => test_resource}, + DefaultOpts#{ + buffer_mode => volatile_offload, + buffer_seg_bytes => 2 * MaxBufferBytes + } + ) + ), + ?assertEqual(ok, emqx_resource:remove_local(?ID)), ok + end, + fun(Trace) -> + HalfMaxBufferBytes = MaxBufferBytes div 2, + ?assertMatch( + [ + #{ + dir := _, + max_total_bytes := MaxTotalBytes, + seg_bytes := MaxTotalBytes, + offload := {true, volatile} + }, + #{ + dir := _, + max_total_bytes := MaxTotalBytes, + %% uses the specified value since it's smaller + %% than max bytes. + seg_bytes := HalfMaxBufferBytes, + offload := {true, volatile} + }, + #{ + dir := _, + max_total_bytes := MaxTotalBytes, + seg_bytes := MaxTotalBytes, + offload := {true, volatile} + }, + #{ + dir := _, + max_total_bytes := MaxTotalBytes, + seg_bytes := MaxTotalBytes, + offload := {true, volatile} + } + ], + ?projection(queue_opts, ?of_kind(buffer_worker_init, Trace)) + ), + ok + end + ). + +t_late_call_reply(_Config) -> + emqx_connector_demo:set_callback_mode(always_sync), + RequestTimeout = 500, + ?assertMatch( + {ok, _}, + emqx_resource:create( + ?ID, + ?DEFAULT_RESOURCE_GROUP, + ?TEST_RESOURCE, + #{name => test_resource}, + #{ + buffer_mode => memory_only, + request_timeout => RequestTimeout, + query_mode => sync + } + ) + ), + ?check_trace( + begin + %% Sleep for longer than the request timeout; the call reply will + %% have been already returned (a timeout), but the resource will + %% still send a message with the reply. + %% The demo connector will reply with `{error, timeout}' after 1 s. + SleepFor = RequestTimeout + 500, + ?assertMatch( + {error, {resource_error, #{reason := timeout}}}, + emqx_resource:query( + ?ID, + {sync_sleep_before_reply, SleepFor}, + #{timeout => RequestTimeout} + ) + ), + %% Our process shouldn't receive any late messages. + receive + LateReply -> + ct:fail("received late reply: ~p", [LateReply]) + after SleepFor -> + ok + end, + ok + end, + [] + ), + ok. + +t_resource_create_error_activate_alarm_once(_) -> + do_t_resource_activate_alarm_once( + #{name => test_resource, create_error => true}, + connector_demo_start_error + ). + +t_resource_health_check_error_activate_alarm_once(_) -> + do_t_resource_activate_alarm_once( + #{name => test_resource, health_check_error => true}, + connector_demo_health_check_error + ). + +do_t_resource_activate_alarm_once(ResourceConfig, SubscribeEvent) -> + ?check_trace( + begin + ?wait_async_action( + emqx_resource:create_local( + ?ID, + ?DEFAULT_RESOURCE_GROUP, + ?TEST_RESOURCE, + ResourceConfig, + #{auto_restart_interval => 100, health_check_interval => 100} + ), + #{?snk_kind := resource_activate_alarm, resource_id := ?ID} + ), + ?assertMatch([#{activated := true, name := ?ID}], emqx_alarm:get_alarms(activated)), + {ok, SubRef} = snabbkaffe:subscribe( + ?match_event(#{?snk_kind := SubscribeEvent}), 4, 7000 + ), + ?assertMatch({ok, [_, _, _, _]}, snabbkaffe:receive_events(SubRef)) + end, + fun(Trace) -> + ?assertMatch([_], ?of_kind(resource_activate_alarm, Trace)) end ). @@ -2702,7 +2894,7 @@ config() -> Config. tap_metrics(Line) -> - {ok, _, #{metrics := #{counters := C, gauges := G}}} = emqx_resource:get_instance(?ID), + #{counters := C, gauges := G} = emqx_resource:get_metrics(?ID), ct:pal("metrics (l. ~b): ~p", [Line, #{counters => C, gauges => G}]), #{counters => C, gauges => G}. diff --git a/apps/emqx_retainer/rebar.config b/apps/emqx_retainer/rebar.config index 65de71fdd..a178e10a1 100644 --- a/apps/emqx_retainer/rebar.config +++ b/apps/emqx_retainer/rebar.config @@ -1,6 +1,9 @@ %% -*- mode: erlang -*- -{deps, [{emqx, {path, "../emqx"}}]}. +{deps, [ + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}} +]}. {edoc_opts, [{preprocess, true}]}. {erl_opts, [ diff --git a/apps/emqx_retainer/src/emqx_retainer.app.src b/apps/emqx_retainer/src/emqx_retainer.app.src index 11013cdd3..7bfc8ee4e 100644 --- a/apps/emqx_retainer/src/emqx_retainer.app.src +++ b/apps/emqx_retainer/src/emqx_retainer.app.src @@ -2,7 +2,7 @@ {application, emqx_retainer, [ {description, "EMQX Retainer"}, % strict semver, bump manually! - {vsn, "5.0.11"}, + {vsn, "5.0.12"}, {modules, []}, {registered, [emqx_retainer_sup]}, {applications, [kernel, stdlib, emqx, emqx_ctl]}, diff --git a/apps/emqx_retainer/src/emqx_retainer_dispatcher.erl b/apps/emqx_retainer/src/emqx_retainer_dispatcher.erl index 454a65eb3..7b0a0dc2a 100644 --- a/apps/emqx_retainer/src/emqx_retainer_dispatcher.erl +++ b/apps/emqx_retainer/src/emqx_retainer_dispatcher.erl @@ -91,7 +91,7 @@ worker() -> | ignore. start_link(Pool, Id) -> gen_server:start_link( - {local, emqx_misc:proc_name(?MODULE, Id)}, + {local, emqx_utils:proc_name(?MODULE, Id)}, ?MODULE, [Pool, Id], [{hibernate_after, 1000}] @@ -156,7 +156,7 @@ handle_cast({dispatch, Context, Pid, Topic}, #{limiter := Limiter} = State) -> {ok, Limiter2} = dispatch(Context, Pid, Topic, undefined, Limiter), {noreply, State#{limiter := Limiter2}}; handle_cast({refresh_limiter, Conf}, State) -> - BucketCfg = emqx_map_lib:deep_get([flow_control, batch_deliver_limiter], Conf, undefined), + BucketCfg = emqx_utils_maps:deep_get([flow_control, batch_deliver_limiter], Conf, undefined), {ok, Limiter} = emqx_limiter_server:connect(?APP, internal, BucketCfg), {noreply, State#{limiter := Limiter}}; handle_cast(Msg, State) -> diff --git a/apps/emqx_retainer/src/emqx_retainer_schema.erl b/apps/emqx_retainer/src/emqx_retainer_schema.erl index dbe1ad9d5..74f1e2371 100644 --- a/apps/emqx_retainer/src/emqx_retainer_schema.erl +++ b/apps/emqx_retainer/src/emqx_retainer_schema.erl @@ -53,7 +53,8 @@ fields("retainer") -> sc( ?R_REF(flow_control), flow_control, - #{} + #{}, + ?IMPORTANCE_HIDDEN )}, {max_payload_size, sc( @@ -125,7 +126,9 @@ desc(_) -> %% hoconsc:mk(Type, #{desc => ?DESC(DescId)}). sc(Type, DescId, Default) -> - hoconsc:mk(Type, #{default => Default, desc => ?DESC(DescId)}). + sc(Type, DescId, Default, ?DEFAULT_IMPORTANCE). +sc(Type, DescId, Default, Importance) -> + hoconsc:mk(Type, #{default => Default, desc => ?DESC(DescId), importance => Importance}). backend_config() -> hoconsc:mk(hoconsc:ref(?MODULE, mnesia_config), #{desc => ?DESC(backend)}). diff --git a/apps/emqx_retainer/test/emqx_retainer_SUITE.erl b/apps/emqx_retainer/test/emqx_retainer_SUITE.erl index 09d1f77da..c90ec6b2b 100644 --- a/apps/emqx_retainer/test/emqx_retainer_SUITE.erl +++ b/apps/emqx_retainer/test/emqx_retainer_SUITE.erl @@ -758,23 +758,22 @@ with_conf(ConfMod, Case) -> end. make_limiter_cfg(Rate) -> - Infinity = emqx_limiter_schema:infinity_value(), Client = #{ rate => Rate, initial => 0, - capacity => Infinity, + burst => 0, low_watermark => 1, divisible => false, max_retry_time => timer:seconds(5), failure_strategy => force }, - #{client => Client, rate => Infinity, initial => 0, capacity => Infinity}. + #{client => Client, rate => Rate, initial => 0, burst => 0}. make_limiter_json(Rate) -> Client = #{ <<"rate">> => Rate, <<"initial">> => 0, - <<"capacity">> => <<"infinity">>, + <<"burst">> => <<"0">>, <<"low_watermark">> => 0, <<"divisible">> => <<"false">>, <<"max_retry_time">> => <<"5s">>, @@ -784,5 +783,5 @@ make_limiter_json(Rate) -> <<"client">> => Client, <<"rate">> => <<"infinity">>, <<"initial">> => 0, - <<"capacity">> => <<"infinity">> + <<"burst">> => <<"0">> }. diff --git a/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl b/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl index ba96887a2..61eee0510 100644 --- a/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl +++ b/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl @@ -72,7 +72,7 @@ t_config(_Config) -> ), UpdateConf = fun(Enable) -> - RawConf = emqx_json:decode(ConfJson, [return_maps]), + RawConf = emqx_utils_json:decode(ConfJson, [return_maps]), UpdateJson = RawConf#{<<"enable">> := Enable}, {ok, UpdateResJson} = request_api( put, @@ -81,7 +81,7 @@ t_config(_Config) -> auth_header_(), UpdateJson ), - UpdateRawConf = emqx_json:decode(UpdateResJson, [return_maps]), + UpdateRawConf = emqx_utils_json:decode(UpdateResJson, [return_maps]), ?assertEqual(Enable, maps:get(<<"enable">>, UpdateRawConf)) end, @@ -224,7 +224,7 @@ t_lookup_and_delete(_) -> t_change_storage_type(_Config) -> Path = api_path(["mqtt", "retainer"]), {ok, ConfJson} = request_api(get, Path), - RawConf = emqx_json:decode(ConfJson, [return_maps]), + RawConf = emqx_utils_json:decode(ConfJson, [return_maps]), %% pre-conditions ?assertMatch( #{ @@ -257,7 +257,7 @@ t_change_storage_type(_Config) -> #{data := Msgs0, meta := _} = decode_json(MsgsJson0), ?assertEqual(1, length(Msgs0)), - ChangedConf = emqx_map_lib:deep_merge( + ChangedConf = emqx_utils_maps:deep_merge( RawConf, #{ <<"backend">> => @@ -271,7 +271,7 @@ t_change_storage_type(_Config) -> auth_header_(), ChangedConf ), - UpdatedRawConf = emqx_json:decode(UpdateResJson, [return_maps]), + UpdatedRawConf = emqx_utils_json:decode(UpdateResJson, [return_maps]), ?assertMatch( #{ <<"backend">> := #{ @@ -311,8 +311,8 @@ t_change_storage_type(_Config) -> %% HTTP Request %%-------------------------------------------------------------------- decode_json(Data) -> - BinJson = emqx_json:decode(Data, [return_maps]), - emqx_map_lib:unsafe_atom_key_map(BinJson). + BinJson = emqx_utils_json:decode(Data, [return_maps]), + emqx_utils_maps:unsafe_atom_key_map(BinJson). %%-------------------------------------------------------------------- %% Internal funcs diff --git a/apps/emqx_rule_engine/rebar.config b/apps/emqx_rule_engine/rebar.config index 110caa33d..07c53d3e3 100644 --- a/apps/emqx_rule_engine/rebar.config +++ b/apps/emqx_rule_engine/rebar.config @@ -1,7 +1,8 @@ %% -*- mode: erlang -*- {deps, [ - {emqx, {path, "../emqx"}} + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}} ]}. {erl_opts, [ diff --git a/apps/emqx_rule_engine/src/emqx_rule_actions.erl b/apps/emqx_rule_engine/src/emqx_rule_actions.erl index c4a6e2e73..a9f24ddcd 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_actions.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_actions.erl @@ -175,7 +175,7 @@ safe_publish(RuleId, Topic, QoS, Flags, Payload, PubProps) -> flags = Flags, headers = #{ republish_by => RuleId, - properties => emqx_misc:pub_props_to_packet(PubProps) + properties => emqx_utils:pub_props_to_packet(PubProps) }, topic = Topic, payload = Payload, @@ -213,7 +213,7 @@ replace_simple_var(Val, _Data, _Default) -> Val. format_msg([], Selected) -> - emqx_json:encode(Selected); + emqx_utils_json:encode(Selected); format_msg(Tokens, Selected) -> emqx_plugin_libs_rule:proc_tmpl(Tokens, Selected). diff --git a/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl index 23c2aab50..c9926f56f 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl @@ -26,7 +26,7 @@ -export([roots/0, fields/1]). --type tag() :: rule_creation | rule_test. +-type tag() :: rule_creation | rule_test | rule_engine. -spec check_params(map(), tag()) -> {ok, map()} | {error, term()}. check_params(Params, Tag) -> @@ -48,12 +48,15 @@ check_params(Params, Tag) -> roots() -> [ + {"rule_engine", sc(ref("rule_engine"), #{desc => ?DESC("root_rule_engine")})}, {"rule_creation", sc(ref("rule_creation"), #{desc => ?DESC("root_rule_creation")})}, {"rule_info", sc(ref("rule_info"), #{desc => ?DESC("root_rule_info")})}, {"rule_events", sc(ref("rule_events"), #{desc => ?DESC("root_rule_events")})}, {"rule_test", sc(ref("rule_test"), #{desc => ?DESC("root_rule_test")})} ]. +fields("rule_engine") -> + emqx_rule_engine_schema:rule_engine_settings(); fields("rule_creation") -> emqx_rule_engine_schema:fields("rules"); fields("rule_info") -> diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index 8d50f60e3..932ebc5ed 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -2,7 +2,7 @@ {application, emqx_rule_engine, [ {description, "EMQX Rule Engine"}, % strict semver, bump manually! - {vsn, "5.0.12"}, + {vsn, "5.0.15"}, {modules, []}, {registered, [emqx_rule_engine_sup, emqx_rule_engine]}, {applications, [kernel, stdlib, rulesql, getopt, emqx_ctl]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.erl b/apps/emqx_rule_engine/src/emqx_rule_engine.erl index 44b49a75b..ada52c5aa 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.erl @@ -115,7 +115,7 @@ start_link() -> %%------------------------------------------------------------------------------ post_config_update(_, _Req, NewRules, OldRules, _AppEnvs) -> #{added := Added, removed := Removed, changed := Updated} = - emqx_map_lib:diff_maps(NewRules, OldRules), + emqx_utils_maps:diff_maps(NewRules, OldRules), maps_foreach( fun({Id, {_Old, New}}) -> {ok, _} = update_rule(New#{id => bin(Id)}) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index 106693a0a..d66f2c1c9 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -32,6 +32,7 @@ %% API callbacks -export([ + '/rule_engine'/2, '/rule_events'/2, '/rule_test'/2, '/rules'/2, @@ -41,7 +42,7 @@ ]). %% query callback --export([qs2ms/2, run_fuzzy_match/2, format_rule_resp/1]). +-export([qs2ms/2, run_fuzzy_match/2, format_rule_info_resp/1]). -define(ERR_BADARGS(REASON), begin R0 = err_msg(REASON), @@ -134,6 +135,7 @@ api_spec() -> paths() -> [ + "/rule_engine", "/rule_events", "/rule_test", "/rules", @@ -145,6 +147,9 @@ paths() -> error_schema(Code, Message) when is_atom(Code) -> emqx_dashboard_swagger:error_codes([Code], list_to_binary(Message)). +rule_engine_schema() -> + ref(emqx_rule_api_schema, "rule_engine"). + rule_creation_schema() -> ref(emqx_rule_api_schema, "rule_creation"). @@ -184,7 +189,7 @@ schema("/rules") -> responses => #{ 200 => [ - {data, mk(array(rule_info_schema()), #{desc => ?DESC("desc9")})}, + {data, mk(array(rule_info_schema()), #{desc => ?DESC("api1_resp")})}, {meta, mk(ref(emqx_dashboard_swagger, meta), #{})} ], 400 => error_schema('BAD_REQUEST', "Invalid Parameters") @@ -289,6 +294,26 @@ schema("/rule_test") -> 200 => <<"Rule Test Pass">> } } + }; +schema("/rule_engine") -> + #{ + 'operationId' => '/rule_engine', + get => #{ + tags => [<<"rules">>], + description => ?DESC("api9"), + responses => #{ + 200 => rule_engine_schema() + } + }, + put => #{ + tags => [<<"rules">>], + description => ?DESC("api10"), + 'requestBody' => rule_engine_schema(), + responses => #{ + 200 => rule_engine_schema(), + 400 => error_schema('BAD_REQUEST', "Invalid request") + } + } }. param_path_id() -> @@ -309,7 +334,7 @@ param_path_id() -> QueryString, ?RULE_QS_SCHEMA, fun ?MODULE:qs2ms/2, - fun ?MODULE:format_rule_resp/1 + fun ?MODULE:format_rule_info_resp/1 ) of {error, page_limit_invalid} -> @@ -318,7 +343,7 @@ param_path_id() -> {200, Result} end; '/rules'(post, #{body := Params0}) -> - case maps:get(<<"id">>, Params0, list_to_binary(emqx_misc:gen_id(8))) of + case maps:get(<<"id">>, Params0, list_to_binary(emqx_utils:gen_id(8))) of <<>> -> {400, #{code => 'BAD_REQUEST', message => <<"empty rule id is not allowed">>}}; Id -> @@ -331,7 +356,7 @@ param_path_id() -> case emqx_conf:update(ConfPath, Params, #{override_to => cluster}) of {ok, #{post_config_update := #{emqx_rule_engine := AllRules}}} -> [Rule] = get_one_rule(AllRules, Id), - {201, format_rule_resp(Rule)}; + {201, format_rule_info_resp(Rule)}; {error, Reason} -> ?SLOG(error, #{ msg => "create_rule_failed", @@ -362,7 +387,7 @@ param_path_id() -> '/rules/:id'(get, #{bindings := #{id := Id}}) -> case emqx_rule_engine:get_rule(Id) of {ok, Rule} -> - {200, format_rule_resp(Rule)}; + {200, format_rule_info_resp(Rule)}; not_found -> {404, #{code => 'NOT_FOUND', message => <<"Rule Id Not Found">>}} end; @@ -372,7 +397,7 @@ param_path_id() -> case emqx_conf:update(ConfPath, Params, #{override_to => cluster}) of {ok, #{post_config_update := #{emqx_rule_engine := AllRules}}} -> [Rule] = get_one_rule(AllRules, Id), - {200, format_rule_resp(Rule)}; + {200, format_rule_info_resp(Rule)}; {error, Reason} -> ?SLOG(error, #{ msg => "update_rule_failed", @@ -419,32 +444,40 @@ param_path_id() -> {404, #{code => 'NOT_FOUND', message => <<"Rule Id Not Found">>}} end. +'/rule_engine'(get, _Params) -> + {200, format_rule_engine_resp(emqx_conf:get([rule_engine]))}; +'/rule_engine'(put, #{body := Params}) -> + case rule_engine_update(Params) of + {ok, Config} -> + {200, format_rule_engine_resp(Config)}; + {error, Reason} -> + {400, #{code => 'BAD_REQUEST', message => ?ERR_BADARGS(Reason)}} + end. + %%------------------------------------------------------------------------------ %% Internal functions %%------------------------------------------------------------------------------ err_msg({RuleError, {_E, Reason, _S}}) -> - emqx_misc:readable_error_msg(encode_nested_error(RuleError, Reason)); + emqx_utils:readable_error_msg(encode_nested_error(RuleError, Reason)); err_msg({Reason, _Details}) -> - emqx_misc:readable_error_msg(Reason); + emqx_utils:readable_error_msg(Reason); err_msg(Msg) -> - emqx_misc:readable_error_msg(Msg). + emqx_utils:readable_error_msg(Msg). encode_nested_error(RuleError, Reason) when is_tuple(Reason) -> encode_nested_error(RuleError, element(1, Reason)); encode_nested_error(RuleError, Reason) -> - case emqx_json:safe_encode([{RuleError, Reason}]) of + case emqx_utils_json:safe_encode([{RuleError, Reason}]) of {ok, Json} -> Json; _ -> {RuleError, Reason} end. -format_rule_resp(Rules) when is_list(Rules) -> - [format_rule_resp(R) || R <- Rules]; -format_rule_resp({Id, Rule}) -> - format_rule_resp(Rule#{id => Id}); -format_rule_resp(#{ +format_rule_info_resp({Id, Rule}) -> + format_rule_info_resp(Rule#{id => Id}); +format_rule_info_resp(#{ id := Id, name := Name, created_at := CreatedAt, @@ -465,6 +498,9 @@ format_rule_resp(#{ description => Descr }. +format_rule_engine_resp(Config) -> + maps:remove(rules, Config). + format_datetime(Timestamp, Unit) -> list_to_binary(calendar:system_time_to_rfc3339(Timestamp, [{unit, Unit}])). @@ -661,3 +697,14 @@ run_fuzzy_match(E = {_Id, #{from := Topics}}, [{from, like, Pattern} | Fuzzy]) - run_fuzzy_match(E, Fuzzy); run_fuzzy_match(E, [_ | Fuzzy]) -> run_fuzzy_match(E, Fuzzy). + +rule_engine_update(Params) -> + case emqx_rule_api_schema:check_params(Params, rule_engine) of + {ok, _CheckedParams} -> + {ok, #{config := Config}} = emqx_conf:update([rule_engine], Params, #{ + override_to => cluster + }), + {ok, Config}; + {error, Reason} -> + {error, Reason} + end. diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl index 2281eea53..bc8cae07a 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl @@ -27,7 +27,8 @@ roots/0, fields/1, desc/1, - post_config_update/5 + post_config_update/5, + rule_engine_settings/0 ]). -export([validate_sql/1]). @@ -37,34 +38,16 @@ namespace() -> rule_engine. tags() -> [<<"Rule Engine">>]. -roots() -> ["rule_engine"]. +roots() -> [{"rule_engine", ?HOCON(?R_REF("rule_engine"), #{importance => ?IMPORTANCE_LOW})}]. fields("rule_engine") -> - [ - {ignore_sys_message, - ?HOCON(boolean(), #{default => true, desc => ?DESC("rule_engine_ignore_sys_message")})}, - {rules, - ?HOCON(hoconsc:map("id", ?R_REF("rules")), #{ - desc => ?DESC("rule_engine_rules"), default => #{} - })}, - {jq_function_default_timeout, - ?HOCON( - emqx_schema:duration_ms(), - #{ - default => <<"10s">>, - desc => ?DESC("rule_engine_jq_function_default_timeout") - } - )}, - {jq_implementation_module, - ?HOCON( - hoconsc:enum([jq_nif, jq_port]), - #{ - default => jq_nif, - mapping => "jq.jq_implementation_module", - desc => ?DESC("rule_engine_jq_implementation_module") - } - )} - ]; + rule_engine_settings() ++ + [ + {rules, + ?HOCON(hoconsc:map("id", ?R_REF("rules")), #{ + desc => ?DESC("rule_engine_rules"), default => #{} + })} + ]; fields("rules") -> [ rule_name(), @@ -227,6 +210,31 @@ actions() -> qos() -> ?UNION([emqx_schema:qos(), binary()]). +rule_engine_settings() -> + [ + {ignore_sys_message, + ?HOCON(boolean(), #{default => true, desc => ?DESC("rule_engine_ignore_sys_message")})}, + {jq_function_default_timeout, + ?HOCON( + emqx_schema:duration_ms(), + #{ + default => <<"10s">>, + desc => ?DESC("rule_engine_jq_function_default_timeout") + } + )}, + {jq_implementation_module, + ?HOCON( + hoconsc:enum([jq_nif, jq_port]), + #{ + default => jq_nif, + mapping => "jq.jq_implementation_module", + desc => ?DESC("rule_engine_jq_implementation_module"), + deprecated => {since, "v5.0.22"}, + importance => ?IMPORTANCE_HIDDEN + } + )} + ]. + validate_sql(Sql) -> case emqx_rule_sqlparser:parse(Sql) of {ok, _Result} -> ok; diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index b8bfeb84c..02163f95b 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -227,9 +227,20 @@ now_timestamp/1, format_date/3, format_date/4, + date_to_unix_ts/3, date_to_unix_ts/4 ]). +%% MongoDB specific date functions. These functions return a date tuple. The +%% MongoDB bridge converts such date tuples to a MongoDB date type. The +%% following functions are therefore only useful for rules with at least one +%% MongoDB action. +-export([ + mongo_date/0, + mongo_date/1, + mongo_date/2 +]). + %% Proc Dict Func -export([ proc_dict_get/1, @@ -643,10 +654,10 @@ map(Data) -> emqx_plugin_libs_rule:map(Data). bin2hexstr(Bin) when is_binary(Bin) -> - emqx_misc:bin_to_hexstr(Bin, upper). + emqx_utils:bin_to_hexstr(Bin, upper). hexstr2bin(Str) when is_binary(Str) -> - emqx_misc:hexstr_to_bin(Str). + emqx_utils:hexstr_to_bin(Str). %%------------------------------------------------------------------------------ %% NULL Funcs @@ -944,7 +955,7 @@ sha256(S) when is_binary(S) -> hash(sha256, S). hash(Type, Data) -> - emqx_misc:bin_to_hexstr(crypto:hash(Type, Data), lower). + emqx_utils:bin_to_hexstr(crypto:hash(Type, Data), lower). %%------------------------------------------------------------------------------ %% gzip Funcs @@ -987,10 +998,10 @@ base64_decode(Data) when is_binary(Data) -> base64:decode(Data). json_encode(Data) -> - emqx_json:encode(Data). + emqx_utils_json:encode(Data). json_decode(Data) -> - emqx_json:decode(Data, [return_maps]). + emqx_utils_json:decode(Data, [return_maps]). term_encode(Term) -> erlang:term_to_binary(Term). @@ -1085,6 +1096,9 @@ format_date(TimeUnit, Offset, FormatString, TimeEpoch) -> ) ). +date_to_unix_ts(TimeUnit, FormatString, InputString) -> + date_to_unix_ts(TimeUnit, "Z", FormatString, InputString). + date_to_unix_ts(TimeUnit, Offset, FormatString, InputString) -> emqx_rule_date:parse_date( time_unit(TimeUnit), @@ -1097,26 +1111,27 @@ date_to_unix_ts(TimeUnit, Offset, FormatString, InputString) -> %% Here the emqx_rule_funcs module acts as a proxy, forwarding %% the function handling to the worker module. %% @end -% '$handle_undefined_function'(schema_decode, [SchemaId, Data|MoreArgs]) -> -% emqx_schema_parser:decode(SchemaId, Data, MoreArgs); -% '$handle_undefined_function'(schema_decode, Args) -> -% error({args_count_error, {schema_decode, Args}}); - -% '$handle_undefined_function'(schema_encode, [SchemaId, Term|MoreArgs]) -> -% emqx_schema_parser:encode(SchemaId, Term, MoreArgs); -% '$handle_undefined_function'(schema_encode, Args) -> -% error({args_count_error, {schema_encode, Args}}); - -% '$handle_undefined_function'(sprintf, [Format|Args]) -> -% erlang:apply(fun sprintf_s/2, [Format, Args]); - -% '$handle_undefined_function'(Fun, Args) -> -% error({sql_function_not_supported, function_literal(Fun, Args)}). - +-if(?EMQX_RELEASE_EDITION == ee). +%% EE +'$handle_undefined_function'(schema_decode, [SchemaId, Data | MoreArgs]) -> + emqx_ee_schema_registry_serde:decode(SchemaId, Data, MoreArgs); +'$handle_undefined_function'(schema_decode, Args) -> + error({args_count_error, {schema_decode, Args}}); +'$handle_undefined_function'(schema_encode, [SchemaId, Term | MoreArgs]) -> + emqx_ee_schema_registry_serde:encode(SchemaId, Term, MoreArgs); +'$handle_undefined_function'(schema_encode, Args) -> + error({args_count_error, {schema_encode, Args}}); '$handle_undefined_function'(sprintf, [Format | Args]) -> erlang:apply(fun sprintf_s/2, [Format, Args]); '$handle_undefined_function'(Fun, Args) -> error({sql_function_not_supported, function_literal(Fun, Args)}). +-else. +%% CE +'$handle_undefined_function'(sprintf, [Format | Args]) -> + erlang:apply(fun sprintf_s/2, [Format, Args]); +'$handle_undefined_function'(Fun, Args) -> + error({sql_function_not_supported, function_literal(Fun, Args)}). +-endif. map_path(Key) -> {path, [{key, P} || P <- string:split(Key, ".", all)]}. @@ -1134,3 +1149,21 @@ function_literal(Fun, [FArg | Args]) when is_atom(Fun), is_list(Args) -> ) ++ ")"; function_literal(Fun, Args) -> {invalid_func, {Fun, Args}}. + +mongo_date() -> + erlang:timestamp(). + +mongo_date(MillisecondsTimestamp) -> + convert_timestamp(MillisecondsTimestamp). + +mongo_date(Timestamp, Unit) -> + InsertedTimeUnit = time_unit(Unit), + ScaledEpoch = erlang:convert_time_unit(Timestamp, InsertedTimeUnit, millisecond), + convert_timestamp(ScaledEpoch). + +convert_timestamp(MillisecondsTimestamp) -> + MicroTimestamp = MillisecondsTimestamp * 1000, + MegaSecs = MicroTimestamp div 1000_000_000_000, + Secs = MicroTimestamp div 1000_000 - MegaSecs * 1000_000, + MicroSecs = MicroTimestamp rem 1000_000, + {MegaSecs, Secs, MicroSecs}. diff --git a/apps/emqx_rule_engine/src/emqx_rule_maps.erl b/apps/emqx_rule_engine/src/emqx_rule_maps.erl index 13f99c88b..3dfffca46 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_maps.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_maps.erl @@ -86,7 +86,7 @@ general_map_put(Key, Val, Map, OrgData) -> ). general_find(KeyOrIndex, Data, OrgData, Handler) when is_binary(Data) -> - try emqx_json:decode(Data, [return_maps]) of + try emqx_utils_json:decode(Data, [return_maps]) of Json -> general_find(KeyOrIndex, Json, OrgData, Handler) catch _:_ -> Handler(not_found) diff --git a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl index e8d807d38..d7412d03c 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl @@ -495,7 +495,7 @@ cache_payload(DecodedP) -> safe_decode_and_cache(MaybeJson) -> try - cache_payload(emqx_json:decode(MaybeJson, [return_maps])) + cache_payload(emqx_utils_json:decode(MaybeJson, [return_maps])) catch _:_ -> error({decode_json_failed, MaybeJson}) end. @@ -525,6 +525,8 @@ inc_action_metrics(R, RuleId) -> is_ok_result(ok) -> true; +is_ok_result({async_return, R}) -> + is_ok_result(R); is_ok_result(R) when is_tuple(R) -> ok == erlang:element(1, R); is_ok_result(_) -> diff --git a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl index e97c45b35..455efe389 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl @@ -48,7 +48,7 @@ test(#{sql := Sql, context := Context}) -> end. test_rule(Sql, Select, Context, EventTopics) -> - RuleId = iolist_to_binary(["sql_tester:", emqx_misc:gen_id(16)]), + RuleId = iolist_to_binary(["sql_tester:", emqx_utils:gen_id(16)]), ok = emqx_rule_engine:maybe_add_metrics_for_rule(RuleId), Rule = #{ id => RuleId, diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index 93d7c7352..eb253e516 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -614,7 +614,9 @@ t_event_client_disconnected_normal(_Config) -> receive {publish, #{topic := T, payload := Payload}} -> ?assertEqual(RepubT, T), - ?assertMatch(#{<<"reason">> := <<"normal">>}, emqx_json:decode(Payload, [return_maps])) + ?assertMatch( + #{<<"reason">> := <<"normal">>}, emqx_utils_json:decode(Payload, [return_maps]) + ) after 1000 -> ct:fail(wait_for_repub_disconnected_normal) end, @@ -651,7 +653,9 @@ t_event_client_disconnected_kicked(_Config) -> receive {publish, #{topic := T, payload := Payload}} -> ?assertEqual(RepubT, T), - ?assertMatch(#{<<"reason">> := <<"kicked">>}, emqx_json:decode(Payload, [return_maps])) + ?assertMatch( + #{<<"reason">> := <<"kicked">>}, emqx_utils_json:decode(Payload, [return_maps]) + ) after 1000 -> ct:fail(wait_for_repub_disconnected_kicked) end, @@ -692,7 +696,7 @@ t_event_client_disconnected_discarded(_Config) -> {publish, #{topic := T, payload := Payload}} -> ?assertEqual(RepubT, T), ?assertMatch( - #{<<"reason">> := <<"discarded">>}, emqx_json:decode(Payload, [return_maps]) + #{<<"reason">> := <<"discarded">>}, emqx_utils_json:decode(Payload, [return_maps]) ) after 1000 -> ct:fail(wait_for_repub_disconnected_discarded) @@ -737,7 +741,7 @@ t_event_client_disconnected_takenover(_Config) -> {publish, #{topic := T, payload := Payload}} -> ?assertEqual(RepubT, T), ?assertMatch( - #{<<"reason">> := <<"takenover">>}, emqx_json:decode(Payload, [return_maps]) + #{<<"reason">> := <<"takenover">>}, emqx_utils_json:decode(Payload, [return_maps]) ) after 1000 -> ct:fail(wait_for_repub_disconnected_discarded) @@ -2800,7 +2804,7 @@ verify_event(EventName) -> [ begin %% verify fields can be formatted to JSON string - _ = emqx_json:encode(Fields), + _ = emqx_utils_json:encode(Fields), %% verify metadata fields verify_metadata_fields(EventName, Fields), %% verify available fields for each event name diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl index d89bc2651..8d7546fca 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl @@ -23,6 +23,14 @@ -include_lib("common_test/include/ct.hrl"). -define(CONF_DEFAULT, <<"rule_engine {rules {}}">>). +-define(SIMPLE_RULE(NAME_SUFFIX), #{ + <<"description">> => <<"A simple rule">>, + <<"enable">> => true, + <<"actions">> => [#{<<"function">> => <<"console">>}], + <<"sql">> => <<"SELECT * from \"t/1\"">>, + <<"name">> => <<"test_rule", NAME_SUFFIX/binary>> +}). +-define(SIMPLE_RULE(ID, NAME_SUFFIX), ?SIMPLE_RULE(NAME_SUFFIX)#{<<"id">> => ID}). all() -> emqx_common_test_helpers:all(?MODULE). @@ -37,18 +45,21 @@ end_per_suite(_Config) -> emqx_common_test_helpers:stop_apps([emqx_conf, emqx_rule_engine]), ok. +init_per_testcase(t_crud_rule_api, Config) -> + meck:new(emqx_utils_json, [passthrough]), + init_per_testcase(common, Config); init_per_testcase(_, Config) -> Config. end_per_testcase(t_crud_rule_api, Config) -> - meck:unload(emqx_json), + meck:unload(emqx_utils_json), end_per_testcase(common, Config); end_per_testcase(_, _Config) -> {200, #{data := Rules}} = emqx_rule_engine_api:'/rules'(get, #{query_string => #{}}), lists:foreach( fun(#{id := Id}) -> - emqx_rule_engine_api:'/rules/:id'( + {204} = emqx_rule_engine_api:'/rules/:id'( delete, #{bindings => #{id => Id}} ) @@ -57,45 +68,38 @@ end_per_testcase(_, _Config) -> ). t_crud_rule_api(_Config) -> - RuleID = <<"my_rule">>, - Params0 = #{ - <<"description">> => <<"A simple rule">>, - <<"enable">> => true, - <<"id">> => RuleID, - <<"actions">> => [#{<<"function">> => <<"console">>}], - <<"sql">> => <<"SELECT * from \"t/1\"">>, - <<"name">> => <<"test_rule">> - }, - {201, Rule} = emqx_rule_engine_api:'/rules'(post, #{body => Params0}), - %% if we post again with the same params, it return with 400 "rule id already exists" - ?assertMatch( - {400, #{code := _, message := _Message}}, - emqx_rule_engine_api:'/rules'(post, #{body => Params0}) - ), + RuleId = <<"my_rule">>, + Rule = simple_rule_fixture(RuleId, <<>>), + ?assertEqual(RuleId, maps:get(id, Rule)), - ?assertEqual(RuleID, maps:get(id, Rule)), {200, #{data := Rules}} = emqx_rule_engine_api:'/rules'(get, #{query_string => #{}}), ct:pal("RList : ~p", [Rules]), ?assert(length(Rules) > 0), + %% if we post again with the same id, it return with 400 "rule id already exists" + ?assertMatch( + {400, #{code := _, message := _Message}}, + emqx_rule_engine_api:'/rules'(post, #{body => ?SIMPLE_RULE(RuleId, <<"some_other">>)}) + ), + {204} = emqx_rule_engine_api:'/rules/:id/metrics/reset'(put, #{ - bindings => #{id => RuleID} + bindings => #{id => RuleId} }), - {200, Rule1} = emqx_rule_engine_api:'/rules/:id'(get, #{bindings => #{id => RuleID}}), + {200, Rule1} = emqx_rule_engine_api:'/rules/:id'(get, #{bindings => #{id => RuleId}}), ct:pal("RShow : ~p", [Rule1]), ?assertEqual(Rule, Rule1), - {200, Metrics} = emqx_rule_engine_api:'/rules/:id/metrics'(get, #{bindings => #{id => RuleID}}), + {200, Metrics} = emqx_rule_engine_api:'/rules/:id/metrics'(get, #{bindings => #{id => RuleId}}), ct:pal("RMetrics : ~p", [Metrics]), - ?assertMatch(#{id := RuleID, metrics := _, node_metrics := _}, Metrics), + ?assertMatch(#{id := RuleId, metrics := _, node_metrics := _}, Metrics), {200, Rule2} = emqx_rule_engine_api:'/rules/:id'(put, #{ - bindings => #{id => RuleID}, - body => Params0#{<<"sql">> => <<"select * from \"t/b\"">>} + bindings => #{id => RuleId}, + body => ?SIMPLE_RULE(RuleId)#{<<"sql">> => <<"select * from \"t/b\"">>} }), - {200, Rule3} = emqx_rule_engine_api:'/rules/:id'(get, #{bindings => #{id => RuleID}}), + {200, Rule3} = emqx_rule_engine_api:'/rules/:id'(get, #{bindings => #{id => RuleId}}), %ct:pal("RShow : ~p", [Rule3]), ?assertEqual(Rule3, Rule2), ?assertEqual(<<"select * from \"t/b\"">>, maps:get(sql, Rule3)), @@ -112,14 +116,14 @@ t_crud_rule_api(_Config) -> {204}, emqx_rule_engine_api:'/rules/:id'( delete, - #{bindings => #{id => RuleID}} + #{bindings => #{id => RuleId}} ) ), %ct:pal("Show After Deleted: ~p", [NotFound]), ?assertMatch( {404, #{code := _, message := _Message}}, - emqx_rule_engine_api:'/rules/:id'(get, #{bindings => #{id => RuleID}}) + emqx_rule_engine_api:'/rules/:id'(get, #{bindings => #{id => RuleId}}) ), {400, #{ @@ -132,7 +136,7 @@ t_crud_rule_api(_Config) -> ), ?assertMatch( #{<<"select_and_transform_error">> := <<"decode_json_failed">>}, - emqx_json:decode(SelectAndTransformJsonError, [return_maps]) + emqx_utils_json:decode(SelectAndTransformJsonError, [return_maps]) ), {400, #{ code := 'BAD_REQUEST', @@ -146,7 +150,7 @@ t_crud_rule_api(_Config) -> ), ?assertMatch( #{<<"select_and_transform_error">> := <<"badarg">>}, - emqx_json:decode(SelectAndTransformBadArgError, [return_maps]) + emqx_utils_json:decode(SelectAndTransformBadArgError, [return_maps]) ), {400, #{ code := 'BAD_REQUEST', @@ -158,7 +162,7 @@ t_crud_rule_api(_Config) -> ) ), ?assertMatch({match, _}, re:run(BadSqlMessage, "syntax error")), - meck:expect(emqx_json, safe_encode, 1, {error, foo}), + meck:expect(emqx_utils_json, safe_encode, 1, {error, foo}), ?assertMatch( {400, #{ code := 'BAD_REQUEST', @@ -174,30 +178,15 @@ t_crud_rule_api(_Config) -> ok. t_list_rule_api(_Config) -> - AddIds = - lists:map( - fun(Seq0) -> - Seq = integer_to_binary(Seq0), - Params = #{ - <<"description">> => <<"A simple rule">>, - <<"enable">> => true, - <<"actions">> => [#{<<"function">> => <<"console">>}], - <<"sql">> => <<"SELECT * from \"t/1\"">>, - <<"name">> => <<"test_rule", Seq/binary>> - }, - {201, #{id := Id}} = emqx_rule_engine_api:'/rules'(post, #{body => Params}), - Id - end, - lists:seq(1, 20) - ), - + AddIds = rules_fixture(20), + ct:pal("rule ids: ~p", [AddIds]), {200, #{data := Rules, meta := #{count := Count}}} = emqx_rule_engine_api:'/rules'(get, #{query_string => #{}}), ?assertEqual(20, length(AddIds)), ?assertEqual(20, length(Rules)), ?assertEqual(20, Count), - [RuleID | _] = AddIds, + [RuleId | _] = AddIds, UpdateParams = #{ <<"description">> => <<"中文的描述也能搜索"/utf8>>, <<"enable">> => false, @@ -206,7 +195,7 @@ t_list_rule_api(_Config) -> <<"name">> => <<"test_rule_update1">> }, {200, _Rule2} = emqx_rule_engine_api:'/rules/:id'(put, #{ - bindings => #{id => RuleID}, + bindings => #{id => RuleId}, body => UpdateParams }), QueryStr1 = #{query_string => #{<<"enable">> => false}}, @@ -229,20 +218,13 @@ t_list_rule_api(_Config) -> {200, Result5} = emqx_rule_engine_api:'/rules'(get, QueryStr5), ?assertEqual(maps:get(data, Result1), maps:get(data, Result5)), - QueryStr6 = #{query_string => #{<<"like_id">> => RuleID}}, + QueryStr6 = #{query_string => #{<<"like_id">> => RuleId}}, {200, Result6} = emqx_rule_engine_api:'/rules'(get, QueryStr6), ?assertEqual(maps:get(data, Result1), maps:get(data, Result6)), ok. t_reset_metrics_on_disable(_Config) -> - Params = #{ - <<"description">> => <<"A simple rule">>, - <<"enable">> => true, - <<"actions">> => [#{<<"function">> => <<"console">>}], - <<"sql">> => <<"SELECT * from \"t/1\"">>, - <<"name">> => atom_to_binary(?FUNCTION_NAME) - }, - {201, #{id := RuleId}} = emqx_rule_engine_api:'/rules'(post, #{body => Params}), + #{id := RuleId} = simple_rule_fixture(), %% generate some fake metrics emqx_metrics_worker:inc(rule_metrics, RuleId, 'matched', 10), @@ -256,7 +238,7 @@ t_reset_metrics_on_disable(_Config) -> %% disable the rule; metrics should be reset {200, _Rule2} = emqx_rule_engine_api:'/rules/:id'(put, #{ bindings => #{id => RuleId}, - body => Params#{<<"enable">> := false} + body => #{<<"enable">> => false} }), {200, #{metrics := Metrics1}} = emqx_rule_engine_api:'/rules/:id/metrics'( @@ -281,3 +263,45 @@ test_rule_params(Sql, Payload) -> <<"sql">> => Sql } }. + +t_rule_engine(_) -> + _ = simple_rule_fixture(), + {200, Config} = emqx_rule_engine_api:'/rule_engine'(get, #{}), + ?assert(not maps:is_key(rules, Config)), + {200, #{ + jq_function_default_timeout := 12000 + % hidden! jq_implementation_module := jq_port + }} = emqx_rule_engine_api:'/rule_engine'(put, #{ + body => #{ + <<"jq_function_default_timeout">> => <<"12s">>, + <<"jq_implementation_module">> => <<"jq_port">> + } + }), + SomeRule = #{<<"sql">> => <<"SELECT * FROM \"t/#\"">>}, + {400, _} = emqx_rule_engine_api:'/rule_engine'(put, #{ + body => #{<<"rules">> => #{<<"some_rule">> => SomeRule}} + }), + {400, _} = emqx_rule_engine_api:'/rule_engine'(put, #{body => #{<<"something">> => <<"weird">>}}). + +rules_fixture(N) -> + lists:map( + fun(Seq0) -> + Seq = integer_to_binary(Seq0), + #{id := Id} = simple_rule_fixture(Seq), + Id + end, + lists:seq(1, N) + ). + +simple_rule_fixture() -> + simple_rule_fixture(<<>>). + +simple_rule_fixture(NameSuffix) -> + create_rule(?SIMPLE_RULE(NameSuffix)). + +simple_rule_fixture(Id, NameSuffix) -> + create_rule(?SIMPLE_RULE(Id, NameSuffix)). + +create_rule(Params) -> + {201, Rule} = emqx_rule_engine_api:'/rules'(post, #{body => Params}), + Rule. diff --git a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl index 94adb3506..d88637312 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl @@ -686,7 +686,6 @@ t_jq(_) -> %% Got timeout as expected got_timeout end, - ConfigRootKey = emqx_rule_engine_schema:namespace(), ?assertThrow( {jq_exception, {timeout, _}}, apply_func(jq, [TOProgram, <<"-2">>]) @@ -959,7 +958,7 @@ prop_format_date_fun() -> Args1 = [<<"second">>, <<"+07:00">>, <<"%m--%d--%y---%H:%M:%S%Z">>], ?FORALL( S, - erlang:system_time(second), + range(0, 4000000000), S == apply_func( date_to_unix_ts, @@ -975,7 +974,7 @@ prop_format_date_fun() -> Args2 = [<<"millisecond">>, <<"+04:00">>, <<"--%m--%d--%y---%H:%M:%S%Z">>], ?FORALL( S, - erlang:system_time(millisecond), + range(0, 4000000000), S == apply_func( date_to_unix_ts, @@ -991,7 +990,7 @@ prop_format_date_fun() -> Args = [<<"second">>, <<"+08:00">>, <<"%y-%m-%d-%H:%M:%S%Z">>], ?FORALL( S, - erlang:system_time(second), + range(0, 4000000000), S == apply_func( date_to_unix_ts, @@ -1003,6 +1002,24 @@ prop_format_date_fun() -> ) ] ) + ), + %% When no offset is specified, the offset should be taken from the formatted time string + ArgsNoOffset = [<<"second">>, <<"%y-%m-%d-%H:%M:%S%Z">>], + ArgsOffset = [<<"second">>, <<"+08:00">>, <<"%y-%m-%d-%H:%M:%S%Z">>], + ?FORALL( + S, + range(0, 4000000000), + S == + apply_func( + date_to_unix_ts, + ArgsNoOffset ++ + [ + apply_func( + format_date, + ArgsOffset ++ [S] + ) + ] + ) ). %%------------------------------------------------------------------------------ diff --git a/apps/emqx_slow_subs/README.md b/apps/emqx_slow_subs/README.md new file mode 100644 index 000000000..8b83508c2 --- /dev/null +++ b/apps/emqx_slow_subs/README.md @@ -0,0 +1,47 @@ +# EMQX Slow Subscriptions + +This application can calculate the latency (time spent) of the message to be processed and transmitted since it arrives at EMQX. + +If the latency exceeds a specified threshold, this application will add the subscriber and topic information to a slow subscriptions list or update the existing record. + +More introduction: [Slow Subscriptions](https://www.emqx.io/docs/en/v5.0/observability/slow-subscribers-statistics.html) + +# Usage + +You can add the below section into `emqx.conf` to enable this application + +```yaml +slow_subs { + enable = true + threshold = "500ms" + expire_interval = "300s" + top_k_num = 10 + stats_type = whole +} +``` + +# Configurations + +**threshold**: Latency threshold for statistics, only messages with latency exceeding this value will be collected. + +Minimum value: 100ms +Default value: 500ms + +**expire_interval**: Eviction time of the record, will start the counting since the creation of the record, and the records that are not triggered again within this specified period will be removed from the rank list. + +Default: 300s + +**top_k_num**: Maximum number of records in the slow subscription statistics record table. + +Maximum value: 1,000 +Default value: 10 + +**stats_type**: Calculation methods of the latency, which are +- **whole**: From the time the message arrives at EMQX until the message transmission completes +- **internal**: From when the message arrives at EMQX until when EMQX starts delivering the message +- **response**: From the time EMQX starts delivering the message until the message transmission completes + +Default value: whole + +# Contributing +Please see our [contributing.md](../../CONTRIBUTING.md). diff --git a/apps/emqx_slow_subs/rebar.config b/apps/emqx_slow_subs/rebar.config index 9f17b7657..dee2902a5 100644 --- a/apps/emqx_slow_subs/rebar.config +++ b/apps/emqx_slow_subs/rebar.config @@ -1,5 +1,8 @@ %% -*- mode: erlang -*- -{deps, [{emqx, {path, "../emqx"}}]}. +{deps, [ + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}} +]}. {project_plugins, [erlfmt]}. diff --git a/apps/emqx_slow_subs/src/emqx_slow_subs.app.src b/apps/emqx_slow_subs/src/emqx_slow_subs.app.src index 7d3fc341d..a06ff2595 100644 --- a/apps/emqx_slow_subs/src/emqx_slow_subs.app.src +++ b/apps/emqx_slow_subs/src/emqx_slow_subs.app.src @@ -1,7 +1,7 @@ {application, emqx_slow_subs, [ {description, "EMQX Slow Subscribers Statistics"}, % strict semver, bump manually! - {vsn, "1.0.4"}, + {vsn, "1.0.5"}, {modules, []}, {registered, [emqx_slow_subs_sup]}, {applications, [kernel, stdlib, emqx]}, diff --git a/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl b/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl index 9e9e6488a..47ea18c3c 100644 --- a/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl +++ b/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl @@ -22,7 +22,8 @@ namespace() -> "slow_subs". -roots() -> ["slow_subs"]. +roots() -> + [{"slow_subs", ?HOCON(?R_REF("slow_subs"), #{importance => ?IMPORTANCE_HIDDEN})}]. fields("slow_subs") -> [ diff --git a/apps/emqx_slow_subs/test/emqx_slow_subs_api_SUITE.erl b/apps/emqx_slow_subs/test/emqx_slow_subs_api_SUITE.erl index 6b0721e3d..5196868c7 100644 --- a/apps/emqx_slow_subs/test/emqx_slow_subs_api_SUITE.erl +++ b/apps/emqx_slow_subs/test/emqx_slow_subs_api_SUITE.erl @@ -108,7 +108,7 @@ t_get_history(_) -> "page=1&limit=10", auth_header_() ), - #{<<"data">> := [First | _]} = emqx_json:decode(Data, [return_maps]), + #{<<"data">> := [First | _]} = emqx_utils_json:decode(Data, [return_maps]), ?assertMatch( #{ @@ -165,8 +165,8 @@ t_settting(_) -> ?assertEqual(Conf2#{stats_type := <<"internal">>}, GetReturn). decode_json(Data) -> - BinJosn = emqx_json:decode(Data, [return_maps]), - emqx_map_lib:unsafe_atom_key_map(BinJosn). + BinJosn = emqx_utils_json:decode(Data, [return_maps]), + emqx_utils_maps:unsafe_atom_key_map(BinJosn). request_api(Method, Url, Auth) -> request_api(Method, Url, [], Auth, []). @@ -187,7 +187,7 @@ request_api(Method, Url, QueryParams, Auth, Body) -> "" -> Url; _ -> Url ++ "?" ++ QueryParams end, - do_request_api(Method, {NewUrl, [Auth], "application/json", emqx_json:encode(Body)}). + do_request_api(Method, {NewUrl, [Auth], "application/json", emqx_utils_json:encode(Body)}). do_request_api(Method, Request) -> ct:pal("Method: ~p, Request: ~p", [Method, Request]), diff --git a/apps/emqx_statsd/rebar.config b/apps/emqx_statsd/rebar.config index bb9a14272..a1383d920 100644 --- a/apps/emqx_statsd/rebar.config +++ b/apps/emqx_statsd/rebar.config @@ -3,6 +3,7 @@ {erl_opts, [debug_info]}. {deps, [ {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}}, {estatsd, {git, "https://github.com/emqx/estatsd", {tag, "0.1.0"}}} ]}. diff --git a/apps/emqx_statsd/src/emqx_statsd.app.src b/apps/emqx_statsd/src/emqx_statsd.app.src index 9d40c6857..87fc8c596 100644 --- a/apps/emqx_statsd/src/emqx_statsd.app.src +++ b/apps/emqx_statsd/src/emqx_statsd.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_statsd, [ {description, "EMQX Statsd"}, - {vsn, "5.0.7"}, + {vsn, "5.0.9"}, {registered, []}, {mod, {emqx_statsd_app, []}}, {applications, [ diff --git a/apps/emqx_statsd/src/emqx_statsd.erl b/apps/emqx_statsd/src/emqx_statsd.erl index c2e1819ac..c5a7fc1c8 100644 --- a/apps/emqx_statsd/src/emqx_statsd.erl +++ b/apps/emqx_statsd/src/emqx_statsd.erl @@ -144,7 +144,7 @@ flush_interval(_FlushInterval, SampleInterval) -> SampleInterval. ensure_timer(State = #{sample_time_interval := SampleTimeInterval}) -> - State#{timer => emqx_misc:start_timer(SampleTimeInterval, ?SAMPLE_TIMEOUT)}. + State#{timer => emqx_utils:start_timer(SampleTimeInterval, ?SAMPLE_TIMEOUT)}. check_multicall_result({Results, []}) -> case diff --git a/apps/emqx_statsd/src/emqx_statsd_api.erl b/apps/emqx_statsd/src/emqx_statsd_api.erl index e65c93432..4ee144d57 100644 --- a/apps/emqx_statsd/src/emqx_statsd_api.erl +++ b/apps/emqx_statsd/src/emqx_statsd_api.erl @@ -49,6 +49,7 @@ schema("/statsd") -> 'operationId' => statsd, get => #{ + deprecated => true, description => ?DESC(get_statsd_config_api), tags => ?API_TAG_STATSD, responses => @@ -56,6 +57,7 @@ schema("/statsd") -> }, put => #{ + deprecated => true, description => ?DESC(update_statsd_config_api), tags => ?API_TAG_STATSD, 'requestBody' => statsd_config_schema(), diff --git a/apps/emqx_statsd/src/emqx_statsd_schema.erl b/apps/emqx_statsd/src/emqx_statsd_schema.erl index e44f94954..01decc6f7 100644 --- a/apps/emqx_statsd/src/emqx_statsd_schema.erl +++ b/apps/emqx_statsd/src/emqx_statsd_schema.erl @@ -32,7 +32,8 @@ namespace() -> "statsd". -roots() -> ["statsd"]. +roots() -> + [{"statsd", hoconsc:mk(hoconsc:ref(?MODULE, "statsd"), #{importance => ?IMPORTANCE_HIDDEN})}]. fields("statsd") -> [ diff --git a/apps/emqx_statsd/test/emqx_statsd_SUITE.erl b/apps/emqx_statsd/test/emqx_statsd_SUITE.erl index bcc710050..b5669e4b9 100644 --- a/apps/emqx_statsd/test/emqx_statsd_SUITE.erl +++ b/apps/emqx_statsd/test/emqx_statsd_SUITE.erl @@ -200,7 +200,7 @@ request(Method) -> request(Method, []). request(Method, Body) -> case request(Method, uri(["statsd"]), Body) of {ok, 200, Res} -> - {ok, emqx_json:decode(Res, [return_maps])}; + {ok, emqx_utils_json:decode(Res, [return_maps])}; {ok, _Status, _} -> error end. diff --git a/apps/emqx_utils/README.md b/apps/emqx_utils/README.md new file mode 100644 index 000000000..f8c386f3d --- /dev/null +++ b/apps/emqx_utils/README.md @@ -0,0 +1,24 @@ +# Erlang utility library for EMQX + +## Overview + +`emqx_utils` is a collection of utility functions for EMQX, organized into +several modules. It provides various functionalities to make it easier to work +with EMQX, such as binary manipulations, maps, JSON en- and decoding, ets table +handling, data conversions, and more. + +## Features + +- `emqx_utils`: unsorted helper functions, formerly known as `emqx_misc` - NEEDS WORK +- `emqx_utils_api`: collection of helper functions for API responses +- `emqx_utils_binary`: binary reverse, join, trim etc +- `emqx_utils_ets`: convenience functions for creating and looking up data in ets tables. +- `emqx_utils_json`: JSON encoding and decoding +- `emqx_utils_maps`: convenience functions for map lookup and manipulation like + deep_get etc. + +## Contributing + +Please see our [contributing guidelines](../../CONTRIBUTING.md) for information +on how to contribute to `emqx_utils`. We welcome bug reports, feature requests, +and pull requests. diff --git a/apps/emqx/include/emqx_api_lib.hrl b/apps/emqx_utils/include/emqx_utils_api.hrl similarity index 97% rename from apps/emqx/include/emqx_api_lib.hrl rename to apps/emqx_utils/include/emqx_utils_api.hrl index 549b0f94c..bfc8e0a53 100644 --- a/apps/emqx/include/emqx_api_lib.hrl +++ b/apps/emqx_utils/include/emqx_utils_api.hrl @@ -17,7 +17,7 @@ -ifndef(EMQX_API_LIB_HRL). -define(EMQX_API_LIB_HRL, true). --define(ERROR_MSG(CODE, REASON), #{code => CODE, message => emqx_misc:readable_error_msg(REASON)}). +-define(ERROR_MSG(CODE, REASON), #{code => CODE, message => emqx_utils:readable_error_msg(REASON)}). -define(OK(CONTENT), {200, CONTENT}). diff --git a/apps/emqx_utils/rebar.config b/apps/emqx_utils/rebar.config new file mode 100644 index 000000000..4c39cfe64 --- /dev/null +++ b/apps/emqx_utils/rebar.config @@ -0,0 +1,11 @@ +%% -*- mode: erlang -*- + +{erl_opts, [ + debug_info +]}. + +{deps, [ + {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} +]}. + +{project_plugins, [erlfmt]}. diff --git a/apps/emqx_utils/src/emqx_utils.app.src b/apps/emqx_utils/src/emqx_utils.app.src new file mode 100644 index 000000000..eb6371411 --- /dev/null +++ b/apps/emqx_utils/src/emqx_utils.app.src @@ -0,0 +1,27 @@ +%% -*- mode: erlang -*- +{application, emqx_utils, [ + {description, "Miscellaneous utilities for EMQX apps"}, + % strict semver, bump manually! + {vsn, "5.0.0"}, + {modules, [ + emqx_utils, + emqx_utils_api, + emqx_utils_binary, + emqx_utils_ets, + emqx_utils_json, + emqx_utils_maps + ]}, + {registered, []}, + {applications, [ + kernel, + stdlib, + jiffy + ]}, + {env, []}, + {licenses, ["Apache-2.0"]}, + {maintainers, ["EMQX Team "]}, + {links, [ + {"Homepage", "https://emqx.io/"}, + {"Github", "https://github.com/emqx/emqx"} + ]} +]}. diff --git a/apps/emqx/src/emqx_misc.erl b/apps/emqx_utils/src/emqx_utils.erl similarity index 98% rename from apps/emqx/src/emqx_misc.erl rename to apps/emqx_utils/src/emqx_utils.erl index cdd62df11..21dbd339d 100644 --- a/apps/emqx/src/emqx_misc.erl +++ b/apps/emqx_utils/src/emqx_utils.erl @@ -14,14 +14,12 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_misc). +-module(emqx_utils). -compile(inline). +%% [TODO] Cleanup so the instruction below is not necessary. -elvis([{elvis_style, god_modules, disable}]). --include("types.hrl"). --include("logger.hrl"). - -export([ merge_opts/2, maybe_apply/2, @@ -71,6 +69,8 @@ -export([clamp/3, redact/1, redact/2, is_redacted/2, is_redacted/3]). +-type maybe(T) :: undefined | T. + -dialyzer({nowarn_function, [nolink_apply/2]}). -define(SHORT, 8). @@ -221,6 +221,7 @@ drain_down(Cnt, Acc) -> %% `ok': There is nothing out of the ordinary. %% `shutdown': Some numbers (message queue length hit the limit), %% hence shutdown for greater good (system stability). +%% [FIXME] cross-dependency on `emqx_types`. -spec check_oom(emqx_types:oom_policy()) -> ok | {shutdown, term()}. check_oom(Policy) -> check_oom(self(), Policy). @@ -246,7 +247,7 @@ do_check_oom([]) -> ok; do_check_oom([{Val, Max, Reason} | Rest]) -> case is_integer(Max) andalso (0 < Max) andalso (Max < Val) of - true -> {shutdown, Reason}; + true -> {shutdown, #{reason => Reason, value => Val, max => Max}}; false -> do_check_oom(Rest) end. @@ -279,6 +280,7 @@ proc_name(Mod, Id) -> list_to_atom(lists:concat([Mod, "_", Id])). %% Get Proc's Stats. +%% [FIXME] cross-dependency on `emqx_types`. -spec proc_stats() -> emqx_types:stats(). proc_stats() -> proc_stats(self()). diff --git a/apps/emqx/src/emqx_api_lib.erl b/apps/emqx_utils/src/emqx_utils_api.erl similarity index 67% rename from apps/emqx/src/emqx_api_lib.erl rename to apps/emqx_utils/src/emqx_utils_api.erl index 8c49c57c3..e6bd07272 100644 --- a/apps/emqx/src/emqx_api_lib.erl +++ b/apps/emqx_utils/src/emqx_utils_api.erl @@ -14,31 +14,31 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_api_lib). +-module(emqx_utils_api). -export([ with_node/2, with_node_or_cluster/2 ]). --include("emqx_api_lib.hrl"). +-include("emqx_utils_api.hrl"). -define(NODE_NOT_FOUND(NODE), ?NOT_FOUND(<<"Node not found: ", NODE/binary>>)). %%-------------------------------------------------------------------- %% exported API %%-------------------------------------------------------------------- --spec with_node(binary(), fun((atom()) -> {ok, term()} | {error, term()})) -> +-spec with_node(binary() | atom(), fun((atom()) -> {ok, term()} | {error, term()})) -> ?OK(term()) | ?NOT_FOUND(binary()) | ?BAD_REQUEST(term()). -with_node(BinNode, Fun) -> - case lookup_node(BinNode) of +with_node(Node0, Fun) -> + case lookup_node(Node0) of {ok, Node} -> handle_result(Fun(Node)); not_found -> - ?NODE_NOT_FOUND(BinNode) + ?NODE_NOT_FOUND(Node0) end. --spec with_node_or_cluster(binary(), fun((atom()) -> {ok, term()} | {error, term()})) -> +-spec with_node_or_cluster(binary() | atom(), fun((atom()) -> {ok, term()} | {error, term()})) -> ?OK(term()) | ?NOT_FOUND(iolist()) | ?BAD_REQUEST(term()). with_node_or_cluster(<<"all">>, Fun) -> handle_result(Fun(all)); @@ -49,18 +49,24 @@ with_node_or_cluster(Node, Fun) -> %% Internal %%-------------------------------------------------------------------- --spec lookup_node(binary()) -> {ok, atom()} | not_found. -lookup_node(BinNode) -> - case emqx_misc:safe_to_existing_atom(BinNode, utf8) of +-spec lookup_node(atom() | binary()) -> {ok, atom()} | not_found. +lookup_node(BinNode) when is_binary(BinNode) -> + case emqx_utils:safe_to_existing_atom(BinNode, utf8) of {ok, Node} -> - case lists:member(Node, mria:running_nodes()) of - true -> - {ok, Node}; - false -> - not_found - end; + is_running_node(Node); _Error -> not_found + end; +lookup_node(Node) when is_atom(Node) -> + is_running_node(Node). + +-spec is_running_node(atom()) -> {ok, atom()} | not_found. +is_running_node(Node) -> + case lists:member(Node, mria:running_nodes()) of + true -> + {ok, Node}; + false -> + not_found end. handle_result({ok, Result}) -> diff --git a/apps/emqx_gateway/src/lwm2m/binary_util.erl b/apps/emqx_utils/src/emqx_utils_binary.erl similarity index 97% rename from apps/emqx_gateway/src/lwm2m/binary_util.erl rename to apps/emqx_utils/src/emqx_utils_binary.erl index 68ac7a0d7..26976496d 100644 --- a/apps/emqx_gateway/src/lwm2m/binary_util.erl +++ b/apps/emqx_utils/src/emqx_utils_binary.erl @@ -1,4 +1,6 @@ %%-------------------------------------------------------------------- +%% Original file taken from https://github.com/arcusfelis/binary2 +%% Copyright (c) 2016 Michael Uvarov %% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,9 +16,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(binary_util). - -%% copied from https://github.com/arcusfelis/binary2 +-module(emqx_utils_binary). %% Bytes -export([ diff --git a/apps/emqx/src/emqx_tables.erl b/apps/emqx_utils/src/emqx_utils_ets.erl similarity index 98% rename from apps/emqx/src/emqx_tables.erl rename to apps/emqx_utils/src/emqx_utils_ets.erl index ffdf7d891..099152675 100644 --- a/apps/emqx/src/emqx_tables.erl +++ b/apps/emqx_utils/src/emqx_utils_ets.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_tables). +-module(emqx_utils_ets). -export([ new/1, diff --git a/apps/emqx/src/emqx_json.erl b/apps/emqx_utils/src/emqx_utils_json.erl similarity index 91% rename from apps/emqx/src/emqx_json.erl rename to apps/emqx_utils/src/emqx_utils_json.erl index 7827b98c9..df7388c94 100644 --- a/apps/emqx/src/emqx_json.erl +++ b/apps/emqx_utils/src/emqx_utils_json.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_json). +-module(emqx_utils_json). -compile(inline). @@ -46,6 +46,10 @@ ]} ). +-export([is_json/1]). + +-compile({inline, [is_json/1]}). + -type encode_options() :: jiffy:encode_options(). -type decode_options() :: jiffy:decode_options(). @@ -79,7 +83,7 @@ safe_encode(Term, Opts) -> end. -spec decode(json_text()) -> json_term(). -decode(Json) -> decode(Json, []). +decode(Json) -> decode(Json, [return_maps]). -spec decode(json_text(), decode_options()) -> json_term(). decode(Json, Opts) -> @@ -100,6 +104,10 @@ safe_decode(Json, Opts) -> {error, Reason} end. +-spec is_json(json_text()) -> boolean(). +is_json(Json) -> + element(1, safe_decode(Json)) =:= ok. + %%-------------------------------------------------------------------- %% Helpers %%-------------------------------------------------------------------- @@ -117,6 +125,8 @@ to_ejson([{_, _} | _] = L) -> {[{K, to_ejson(V)} || {K, V} <- L]}; to_ejson(L) when is_list(L) -> [to_ejson(E) || E <- L]; +to_ejson(M) when is_map(M) -> + maps:map(fun(_K, V) -> to_ejson(V) end, M); to_ejson(T) -> T. diff --git a/apps/emqx/src/emqx_map_lib.erl b/apps/emqx_utils/src/emqx_utils_maps.erl similarity index 99% rename from apps/emqx/src/emqx_map_lib.erl rename to apps/emqx_utils/src/emqx_utils_maps.erl index 631c3914d..6bec32ae3 100644 --- a/apps/emqx/src/emqx_map_lib.erl +++ b/apps/emqx_utils/src/emqx_utils_maps.erl @@ -13,7 +13,7 @@ %% See the License for the specific language governing permissions and %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_map_lib). +-module(emqx_utils_maps). -export([ deep_get/2, @@ -210,6 +210,7 @@ binary_string_kv(K, V, JsonableFun) -> {K1, V1} -> {binary_string(K1), V1} end. +%% [FIXME] this doesn't belong here binary_string([]) -> []; binary_string(Val) when is_list(Val) -> @@ -332,7 +333,7 @@ deep_filter(M, F) when is_map(M) -> if_only_to_toggle_enable(OldConf, Conf) -> #{added := Added, removed := Removed, changed := Updated} = - emqx_map_lib:diff_maps(OldConf, Conf), + emqx_utils_maps:diff_maps(OldConf, Conf), case {Added, Removed, Updated} of {Added, Removed, #{enable := _} = Updated} when map_size(Added) =:= 0, diff --git a/apps/emqx/test/emqx_misc_SUITE.erl b/apps/emqx_utils/test/emqx_utils_SUITE.erl similarity index 66% rename from apps/emqx/test/emqx_misc_SUITE.erl rename to apps/emqx_utils/test/emqx_utils_SUITE.erl index c2bd751fa..99516b0eb 100644 --- a/apps/emqx/test/emqx_misc_SUITE.erl +++ b/apps/emqx_utils/test/emqx_utils_SUITE.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_misc_SUITE). +-module(emqx_utils_SUITE). -compile(export_all). -compile(nowarn_export_all). @@ -32,7 +32,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). t_merge_opts(_) -> - Opts = emqx_misc:merge_opts(?SOCKOPTS, [ + Opts = emqx_utils:merge_opts(?SOCKOPTS, [ raw, binary, {backlog, 1024}, @@ -57,58 +57,59 @@ t_merge_opts(_) -> ). t_maybe_apply(_) -> - ?assertEqual(undefined, emqx_misc:maybe_apply(fun(A) -> A end, undefined)), - ?assertEqual(a, emqx_misc:maybe_apply(fun(A) -> A end, a)). + ?assertEqual(undefined, emqx_utils:maybe_apply(fun(A) -> A end, undefined)), + ?assertEqual(a, emqx_utils:maybe_apply(fun(A) -> A end, a)). t_run_fold(_) -> - ?assertEqual(1, emqx_misc:run_fold([], 1, state)), + ?assertEqual(1, emqx_utils:run_fold([], 1, state)), Add = fun(I, St) -> I + St end, Mul = fun(I, St) -> I * St end, - ?assertEqual(6, emqx_misc:run_fold([Add, Mul], 1, 2)). + ?assertEqual(6, emqx_utils:run_fold([Add, Mul], 1, 2)). t_pipeline(_) -> - ?assertEqual({ok, input, state}, emqx_misc:pipeline([], input, state)), + ?assertEqual({ok, input, state}, emqx_utils:pipeline([], input, state)), Funs = [ fun(_I, _St) -> ok end, fun(_I, St) -> {ok, St + 1} end, fun(I, St) -> {ok, I + 1, St + 1} end, fun(I, St) -> {ok, I * 2, St * 2} end ], - ?assertEqual({ok, 4, 6}, emqx_misc:pipeline(Funs, 1, 1)), + ?assertEqual({ok, 4, 6}, emqx_utils:pipeline(Funs, 1, 1)), ?assertEqual( - {error, undefined, 1}, emqx_misc:pipeline([fun(_I) -> {error, undefined} end], 1, 1) + {error, undefined, 1}, emqx_utils:pipeline([fun(_I) -> {error, undefined} end], 1, 1) ), ?assertEqual( - {error, undefined, 2}, emqx_misc:pipeline([fun(_I, _St) -> {error, undefined, 2} end], 1, 1) + {error, undefined, 2}, + emqx_utils:pipeline([fun(_I, _St) -> {error, undefined, 2} end], 1, 1) ). t_start_timer(_) -> - TRef = emqx_misc:start_timer(1, tmsg), + TRef = emqx_utils:start_timer(1, tmsg), timer:sleep(2), ?assertEqual([{timeout, TRef, tmsg}], drain()), - ok = emqx_misc:cancel_timer(TRef). + ok = emqx_utils:cancel_timer(TRef). t_cancel_timer(_) -> - Timer = emqx_misc:start_timer(0, foo), - ok = emqx_misc:cancel_timer(Timer), + Timer = emqx_utils:start_timer(0, foo), + ok = emqx_utils:cancel_timer(Timer), ?assertEqual([], drain()), - ok = emqx_misc:cancel_timer(undefined). + ok = emqx_utils:cancel_timer(undefined). t_proc_name(_) -> - ?assertEqual(emqx_pool_1, emqx_misc:proc_name(emqx_pool, 1)). + ?assertEqual(emqx_pool_1, emqx_utils:proc_name(emqx_pool, 1)). t_proc_stats(_) -> Pid1 = spawn(fun() -> exit(normal) end), timer:sleep(10), - ?assertEqual([], emqx_misc:proc_stats(Pid1)), + ?assertEqual([], emqx_utils:proc_stats(Pid1)), Pid2 = spawn(fun() -> - ?assertMatch([{mailbox_len, 0} | _], emqx_misc:proc_stats()), + ?assertMatch([{mailbox_len, 0} | _], emqx_utils:proc_stats()), timer:sleep(200) end), timer:sleep(10), Pid2 ! msg, timer:sleep(10), - ?assertMatch([{mailbox_len, 1} | _], emqx_misc:proc_stats(Pid2)). + ?assertMatch([{mailbox_len, 1} | _], emqx_utils:proc_stats(Pid2)). t_drain_deliver(_) -> self() ! {deliver, t1, m1}, @@ -118,24 +119,24 @@ t_drain_deliver(_) -> {deliver, t1, m1}, {deliver, t2, m2} ], - emqx_misc:drain_deliver(2) + emqx_utils:drain_deliver(2) ). t_drain_down(_) -> {Pid1, _Ref1} = erlang:spawn_monitor(fun() -> ok end), {Pid2, _Ref2} = erlang:spawn_monitor(fun() -> ok end), timer:sleep(100), - ?assertEqual([Pid1, Pid2], lists:sort(emqx_misc:drain_down(2))), - ?assertEqual([], emqx_misc:drain_down(1)). + ?assertEqual([Pid1, Pid2], lists:sort(emqx_utils:drain_down(2))), + ?assertEqual([], emqx_utils:drain_down(1)). t_index_of(_) -> - try emqx_misc:index_of(a, []) of + try emqx_utils:index_of(a, []) of _ -> ct:fail(should_throw_error) catch error:Reason -> ?assertEqual(badarg, Reason) end, - ?assertEqual(3, emqx_misc:index_of(a, [b, c, a, e, f])). + ?assertEqual(3, emqx_utils:index_of(a, [b, c, a, e, f])). t_check(_) -> Policy = #{ @@ -144,9 +145,12 @@ t_check(_) -> enable => true }, [self() ! {msg, I} || I <- lists:seq(1, 5)], - ?assertEqual(ok, emqx_misc:check_oom(Policy)), + ?assertEqual(ok, emqx_utils:check_oom(Policy)), [self() ! {msg, I} || I <- lists:seq(1, 6)], - ?assertEqual({shutdown, message_queue_too_long}, emqx_misc:check_oom(Policy)). + ?assertEqual( + {shutdown, #{reason => message_queue_too_long, value => 11, max => 10}}, + emqx_utils:check_oom(Policy) + ). drain() -> drain([]). @@ -159,22 +163,22 @@ drain(Acc) -> end. t_rand_seed(_) -> - ?assert(is_tuple(emqx_misc:rand_seed())). + ?assert(is_tuple(emqx_utils:rand_seed())). t_now_to_secs(_) -> - ?assert(is_integer(emqx_misc:now_to_secs(os:timestamp()))). + ?assert(is_integer(emqx_utils:now_to_secs(os:timestamp()))). t_now_to_ms(_) -> - ?assert(is_integer(emqx_misc:now_to_ms(os:timestamp()))). + ?assert(is_integer(emqx_utils:now_to_ms(os:timestamp()))). t_gen_id(_) -> - ?assertEqual(10, length(emqx_misc:gen_id(10))), - ?assertEqual(20, length(emqx_misc:gen_id(20))). + ?assertEqual(10, length(emqx_utils:gen_id(10))), + ?assertEqual(20, length(emqx_utils:gen_id(20))). t_pmap_normal(_) -> ?assertEqual( [5, 7, 9], - emqx_misc:pmap( + emqx_utils:pmap( fun({A, B}) -> A + B end, [{2, 3}, {3, 4}, {4, 5}] ) @@ -183,7 +187,7 @@ t_pmap_normal(_) -> t_pmap_timeout(_) -> ?assertExit( timeout, - emqx_misc:pmap( + emqx_utils:pmap( fun (timeout) -> ct:sleep(1000); ({A, B}) -> A + B @@ -196,7 +200,7 @@ t_pmap_timeout(_) -> t_pmap_exception(_) -> ?assertError( foobar, - emqx_misc:pmap( + emqx_utils:pmap( fun (error) -> error(foobar); ({A, B}) -> A + B diff --git a/apps/emqx/test/emqx_api_lib_SUITE.erl b/apps/emqx_utils/test/emqx_utils_api_SUITE.erl similarity index 92% rename from apps/emqx/test/emqx_api_lib_SUITE.erl rename to apps/emqx_utils/test/emqx_utils_api_SUITE.erl index 29f5c6095..3ed3cd250 100644 --- a/apps/emqx/test/emqx_api_lib_SUITE.erl +++ b/apps/emqx_utils/test/emqx_utils_api_SUITE.erl @@ -14,12 +14,12 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_api_lib_SUITE). +-module(emqx_utils_api_SUITE). -compile(export_all). -compile(nowarn_export_all). --include("emqx_api_lib.hrl"). +-include("emqx_utils_api.hrl"). -include_lib("eunit/include/eunit.hrl"). -define(DUMMY, dummy_module). @@ -45,14 +45,14 @@ end_per_testcase(_Case, _Config) -> meck:unload(?DUMMY). t_with_node(_) -> - test_with(fun emqx_api_lib:with_node/2, [<<"all">>]). + test_with(fun emqx_utils_api:with_node/2, [<<"all">>]). t_with_node_or_cluster(_) -> - test_with(fun emqx_api_lib:with_node_or_cluster/2, []), + test_with(fun emqx_utils_api:with_node_or_cluster/2, []), meck:reset(?DUMMY), ?assertEqual( ?OK(success), - emqx_api_lib:with_node_or_cluster( + emqx_utils_api:with_node_or_cluster( <<"all">>, fun ?DUMMY:expect_success/1 ) diff --git a/apps/emqx_utils/test/emqx_utils_binary_tests.erl b/apps/emqx_utils/test/emqx_utils_binary_tests.erl new file mode 100644 index 000000000..79851dca5 --- /dev/null +++ b/apps/emqx_utils/test/emqx_utils_binary_tests.erl @@ -0,0 +1,213 @@ +%%-------------------------------------------------------------------- +%% Original file taken from https://github.com/arcusfelis/binary2 +%% Copyright (c) 2016 Michael Uvarov +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +-module(emqx_utils_binary_tests). + +-import(emqx_utils_binary, [ + trim/1, + ltrim/1, + rtrim/1, + trim/2, + ltrim/2, + rtrim/2, + reverse/1, + inverse/1, + join/2, + suffix/2, + prefix/2, + duplicate/2, + union/2, + intersection/2, + subtract/2, + optimize_patterns/1 +]). + +-include_lib("eunit/include/eunit.hrl"). + +trim1_test_() -> + [ + ?_assertEqual(trim(<<>>), <<>>), + ?_assertEqual(trim(<<0, 0, 0>>), <<>>), + ?_assertEqual(trim(<<1, 2, 3>>), <<1, 2, 3>>), + ?_assertEqual(trim(<<0, 1, 2>>), <<1, 2>>), + ?_assertEqual(trim(<<0, 0, 1, 2>>), <<1, 2>>), + ?_assertEqual(trim(<<1, 2, 0, 0>>), <<1, 2>>), + ?_assertEqual(trim(<<0, 1, 2, 0>>), <<1, 2>>), + ?_assertEqual(trim(<<0, 0, 0, 1, 2, 0, 0, 0>>), <<1, 2>>) + ]. + +ltrim1_test_() -> + [ + ?_assertEqual(ltrim(<<>>), <<>>), + ?_assertEqual(ltrim(<<0, 0, 0>>), <<>>), + ?_assertEqual(ltrim(<<1, 2, 3>>), <<1, 2, 3>>), + ?_assertEqual(ltrim(<<0, 1, 2>>), <<1, 2>>), + ?_assertEqual(ltrim(<<0, 0, 1, 2>>), <<1, 2>>), + ?_assertEqual(ltrim(<<1, 2, 0, 0>>), <<1, 2, 0, 0>>), + ?_assertEqual(ltrim(<<0, 1, 2, 0>>), <<1, 2, 0>>), + ?_assertEqual(ltrim(<<0, 0, 0, 1, 2, 0, 0, 0>>), <<1, 2, 0, 0, 0>>) + ]. + +rtrim1_test_() -> + [ + ?_assertEqual(rtrim(<<>>), <<>>), + ?_assertEqual(rtrim(<<1, 2, 3>>), <<1, 2, 3>>), + ?_assertEqual(rtrim(<<0, 0, 0>>), <<>>), + ?_assertEqual(rtrim(<<0, 1, 2>>), <<0, 1, 2>>), + ?_assertEqual(rtrim(<<0, 0, 1, 2>>), <<0, 0, 1, 2>>), + ?_assertEqual(rtrim(<<1, 2, 0, 0>>), <<1, 2>>), + ?_assertEqual(rtrim(<<0, 1, 2, 0>>), <<0, 1, 2>>), + ?_assertEqual(rtrim(<<0, 0, 0, 1, 2, 0, 0, 0>>), <<0, 0, 0, 1, 2>>) + ]. + +trim2_test_() -> + [ + ?_assertEqual(trim(<<5>>, 5), <<>>), + ?_assertEqual(trim(<<5, 1, 2, 5>>, 5), <<1, 2>>), + ?_assertEqual(trim(<<5, 5, 5, 1, 2, 0, 0, 0>>, 5), <<1, 2, 0, 0, 0>>) + ]. + +ltrim2_test_() -> + [ + ?_assertEqual(ltrim(<<5>>, 5), <<>>), + ?_assertEqual(ltrim(<<5, 1, 2, 5>>, 5), <<1, 2, 5>>), + ?_assertEqual(ltrim(<<5, 5, 5, 1, 2, 0, 0, 0>>, 5), <<1, 2, 0, 0, 0>>) + ]. + +rtrim2_test_() -> + [ + ?_assertEqual(rtrim(<<5>>, 5), <<>>), + ?_assertEqual(rtrim(<<5, 1, 2, 5>>, 5), <<5, 1, 2>>), + ?_assertEqual(rtrim(<<5, 5, 5, 1, 2, 0, 0, 0>>, 5), <<5, 5, 5, 1, 2, 0, 0, 0>>) + ]. + +mtrim2_test_() -> + [ + ?_assertEqual(trim(<<5>>, [1, 5]), <<>>), + ?_assertEqual(trim(<<5, 1, 2, 5>>, [1, 5]), <<2>>), + ?_assertEqual(trim(<<5, 1, 2, 5>>, [1, 2, 5]), <<>>), + ?_assertEqual(trim(<<5, 5, 5, 1, 2, 0, 0, 0>>, [1, 5]), <<2, 0, 0, 0>>) + ]. + +mltrim2_test_() -> + [ + ?_assertEqual(ltrim(<<5>>, [1, 5]), <<>>), + ?_assertEqual(ltrim(<<5, 1, 2, 5>>, [1, 5]), <<2, 5>>), + ?_assertEqual(ltrim(<<5, 1, 2, 5>>, [2, 5]), <<1, 2, 5>>), + ?_assertEqual(ltrim(<<5, 5, 5, 1, 2, 0, 0, 0>>, [1, 5]), <<2, 0, 0, 0>>) + ]. + +mrtrim2_test_() -> + [ + ?_assertEqual(rtrim(<<5>>, [1, 5]), <<>>), + ?_assertEqual(rtrim(<<5, 1, 2, 5>>, [1, 5]), <<5, 1, 2>>), + ?_assertEqual(rtrim(<<5, 1, 2, 5>>, [2, 5]), <<5, 1>>), + ?_assertEqual(rtrim(<<5, 5, 5, 1, 2, 0, 0, 0>>, [1, 5]), <<5, 5, 5, 1, 2, 0, 0, 0>>), + ?_assertEqual(rtrim(<<5, 5, 5, 1, 2, 0, 0, 0>>, [0, 5]), <<5, 5, 5, 1, 2>>) + ]. + +reverse_test_() -> + [?_assertEqual(reverse(<<0, 1, 2>>), <<2, 1, 0>>)]. + +join_test_() -> + [ + ?_assertEqual(join([<<1, 2>>, <<3, 4>>, <<5, 6>>], <<0>>), <<1, 2, 0, 3, 4, 0, 5, 6>>), + ?_assertEqual( + join([<<"abc">>, <<"def">>, <<"xyz">>], <<"|">>), + <<"abc|def|xyz">> + ), + ?_assertEqual( + join([<<>>, <<"|">>, <<"x|z">>], <<"|">>), + <<"|||x|z">> + ), + ?_assertEqual( + join([<<"abc">>, <<"def">>, <<"xyz">>], <<>>), + <<"abcdefxyz">> + ), + ?_assertEqual(join([], <<"|">>), <<>>) + ]. + +duplicate_test_() -> + [ + ?_assertEqual(duplicate(5, <<1, 2>>), <<1, 2, 1, 2, 1, 2, 1, 2, 1, 2>>), + ?_assertEqual(duplicate(50, <<0>>), <<0:400>>) + ]. + +suffix_test_() -> + [ + ?_assertEqual(suffix(<<1, 2, 3, 4, 5>>, 2), <<4, 5>>), + ?_assertError(badarg, prefix(<<1, 2, 3, 4, 5>>, 25)) + ]. + +prefix_test_() -> + [ + ?_assertEqual(prefix(<<1, 2, 3, 4, 5>>, 2), <<1, 2>>), + ?_assertError(badarg, prefix(<<1, 2, 3, 4, 5>>, 25)) + ]. + +union_test_() -> + [ + ?_assertEqual( + union( + <<2#0011011:7>>, + <<2#1011110:7>> + ), + <<2#1011111:7>> + ) + ]. + +inverse_test_() -> + [ + ?_assertEqual(inverse(inverse(<<0, 1, 2>>)), <<0, 1, 2>>), + ?_assertEqual(inverse(<<0>>), <<255>>), + ?_assertEqual(inverse(<<2#1:1>>), <<2#0:1>>), + ?_assertEqual(inverse(<<2#0:1>>), <<2#1:1>>), + ?_assertEqual( + inverse(<<2#01:2>>), + <<2#10:2>> + ), + ?_assertEqual( + inverse(<<2#0011011:7>>), + <<2#1100100:7>> + ) + ]. + +intersection_test_() -> + [ + ?_assertEqual( + intersection( + <<2#0011011>>, + <<2#1011110>> + ), + <<2#0011010>> + ) + ]. + +subtract_test_() -> + [ + ?_assertEqual( + subtract( + <<2#0011011>>, + <<2#1011110>> + ), + <<2#0000001>> + ) + ]. + +optimize_patterns_test_() -> + [ + ?_assertEqual( + [<<"t">>], + optimize_patterns([<<"t">>, <<"test">>]) + ), + ?_assertEqual( + [<<"t">>], + optimize_patterns([<<"t">>, <<"t">>, <<"test">>]) + ), + ?_assertEqual( + [<<"t">>], + optimize_patterns([<<"test">>, <<"t">>, <<"t">>]) + ) + ]. diff --git a/apps/emqx/test/emqx_tables_SUITE.erl b/apps/emqx_utils/test/emqx_utils_ets_SUITE.erl similarity index 73% rename from apps/emqx/test/emqx_tables_SUITE.erl rename to apps/emqx_utils/test/emqx_utils_ets_SUITE.erl index ad53e7139..13bf427fd 100644 --- a/apps/emqx/test/emqx_tables_SUITE.erl +++ b/apps/emqx_utils/test/emqx_utils_ets_SUITE.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_tables_SUITE). +-module(emqx_utils_ets_SUITE). -compile(export_all). -compile(nowarn_export_all). @@ -26,19 +26,19 @@ all() -> emqx_common_test_helpers:all(?MODULE). t_new(_) -> - ok = emqx_tables:new(?TAB), - ok = emqx_tables:new(?TAB, [{read_concurrency, true}]), + ok = emqx_utils_ets:new(?TAB), + ok = emqx_utils_ets:new(?TAB, [{read_concurrency, true}]), ?assertEqual(?TAB, ets:info(?TAB, name)). t_lookup_value(_) -> - ok = emqx_tables:new(?TAB, []), + ok = emqx_utils_ets:new(?TAB, []), true = ets:insert(?TAB, {key, val}), - ?assertEqual(val, emqx_tables:lookup_value(?TAB, key)), - ?assertEqual(undefined, emqx_tables:lookup_value(?TAB, badkey)). + ?assertEqual(val, emqx_utils_ets:lookup_value(?TAB, key)), + ?assertEqual(undefined, emqx_utils_ets:lookup_value(?TAB, badkey)). t_delete(_) -> - ok = emqx_tables:new(?TAB, []), + ok = emqx_utils_ets:new(?TAB, []), ?assertEqual(?TAB, ets:info(?TAB, name)), - ok = emqx_tables:delete(?TAB), - ok = emqx_tables:delete(?TAB), + ok = emqx_utils_ets:delete(?TAB), + ok = emqx_utils_ets:delete(?TAB), ?assertEqual(undefined, ets:info(?TAB, name)). diff --git a/apps/emqx/test/emqx_json_SUITE.erl b/apps/emqx_utils/test/emqx_utils_json_SUITE.erl similarity index 83% rename from apps/emqx/test/emqx_json_SUITE.erl rename to apps/emqx_utils/test/emqx_utils_json_SUITE.erl index a0bf48e4e..daf31b440 100644 --- a/apps/emqx/test/emqx_json_SUITE.erl +++ b/apps/emqx_utils/test/emqx_utils_json_SUITE.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_json_SUITE). +-module(emqx_utils_json_SUITE). -compile(export_all). -compile(nowarn_export_all). @@ -22,7 +22,7 @@ -include_lib("eunit/include/eunit.hrl"). -import( - emqx_json, + emqx_utils_json, [ encode/1, decode/1, @@ -51,7 +51,7 @@ %% #{<<"foo">> => <<"bar">>} -> {"foo": "bar"} -> #{<<"foo">> => <<"bar">>} %%-------------------------------------------------------------------- -%% but in emqx_json, we use the jsx style for it: +%% but in emqx_utils_json, we use the jsx style for it: %%-------------------------------------------------------------------- %% Erlang JSON Erlang %% ------------------------------------------------------------------- @@ -84,10 +84,10 @@ t_decode_encode(_) -> 1.25 = decode(encode(1.25)), [] = decode(encode([])), [true, 1] = decode(encode([true, 1])), - [{}] = decode(encode([{}])), - [{<<"foo">>, <<"bar">>}] = decode(encode([{foo, bar}])), - [{<<"foo">>, <<"bar">>}] = decode(encode([{<<"foo">>, <<"bar">>}])), - [[{<<"foo">>, <<"bar">>}]] = decode(encode([[{<<"foo">>, <<"bar">>}]])), + [{}] = decode(encode([{}]), []), + [{<<"foo">>, <<"bar">>}] = decode(encode([{foo, bar}]), []), + [{<<"foo">>, <<"bar">>}] = decode(encode([{<<"foo">>, <<"bar">>}]), []), + [[{<<"foo">>, <<"bar">>}]] = decode(encode([[{<<"foo">>, <<"bar">>}]]), []), [ [ {<<"foo">>, <<"bar">>}, @@ -101,7 +101,8 @@ t_decode_encode(_) -> {<<"a">>, <<"b">>} ], [{<<"x">>, <<"y">>}] - ]) + ]), + [] ), #{<<"foo">> := <<"bar">>} = decode(encode(#{<<"foo">> => <<"bar">>}), [return_maps]), JsonText = <<"{\"bool\":true,\"int\":10,\"foo\":\"bar\"}">>, @@ -110,8 +111,12 @@ t_decode_encode(_) -> <<"int">> => 10, <<"foo">> => <<"bar">> }, - ?assertEqual(JsonText, encode({decode(JsonText)})), - ?assertEqual(JsonMaps, decode(JsonText, [return_maps])). + ?assertEqual(JsonText, encode({decode(JsonText, [])})), + ?assertEqual(JsonMaps, decode(JsonText, [return_maps])), + ?assertEqual( + #{<<"foo">> => #{<<"bar">> => <<"baz">>}}, + decode(encode(#{<<"foo">> => [{<<"bar">>, <<"baz">>}]})) + ). t_safe_decode_encode(_) -> safe_encode_decode(null), @@ -123,16 +128,20 @@ t_safe_decode_encode(_) -> 1.25 = safe_encode_decode(1.25), [] = safe_encode_decode([]), [true, 1] = safe_encode_decode([true, 1]), - [{}] = decode(encode([{}])), + [{}] = decode(encode([{}]), []), [{<<"foo">>, <<"bar">>}] = safe_encode_decode([{foo, bar}]), [{<<"foo">>, <<"bar">>}] = safe_encode_decode([{<<"foo">>, <<"bar">>}]), [[{<<"foo">>, <<"bar">>}]] = safe_encode_decode([[{<<"foo">>, <<"bar">>}]]), - {ok, Json} = emqx_json:safe_encode(#{<<"foo">> => <<"bar">>}), - {ok, #{<<"foo">> := <<"bar">>}} = emqx_json:safe_decode(Json, [return_maps]). + {ok, Json} = emqx_utils_json:safe_encode(#{<<"foo">> => <<"bar">>}), + {ok, #{<<"foo">> := <<"bar">>}} = emqx_utils_json:safe_decode(Json, [return_maps]). safe_encode_decode(Term) -> - {ok, Json} = emqx_json:safe_encode(Term), - case emqx_json:safe_decode(Json) of + {ok, Json} = emqx_utils_json:safe_encode(Term), + case emqx_utils_json:safe_decode(Json, []) of {ok, {NTerm}} -> NTerm; {ok, NTerm} -> NTerm end. + +t_is_json(_) -> + ?assert(emqx_utils_json:is_json(<<"{}">>)), + ?assert(not emqx_utils_json:is_json(<<"foo">>)). diff --git a/apps/emqx/test/emqx_map_lib_tests.erl b/apps/emqx_utils/test/emqx_utils_maps_tests.erl similarity index 76% rename from apps/emqx/test/emqx_map_lib_tests.erl rename to apps/emqx_utils/test/emqx_utils_maps_tests.erl index 894811d7c..506851f0a 100644 --- a/apps/emqx/test/emqx_map_lib_tests.erl +++ b/apps/emqx_utils/test/emqx_utils_maps_tests.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_map_lib_tests). +-module(emqx_utils_maps_tests). -include_lib("eunit/include/eunit.hrl"). best_effort_recursive_sum_test_() -> @@ -22,21 +22,21 @@ best_effort_recursive_sum_test_() -> [ ?_assertEqual( #{foo => 3}, - emqx_map_lib:best_effort_recursive_sum(#{foo => 1}, #{foo => 2}, DummyLogger) + emqx_utils_maps:best_effort_recursive_sum(#{foo => 1}, #{foo => 2}, DummyLogger) ), ?_assertEqual( #{foo => 3, bar => 6.0}, - emqx_map_lib:best_effort_recursive_sum( + emqx_utils_maps:best_effort_recursive_sum( #{foo => 1, bar => 2.0}, #{foo => 2, bar => 4.0}, DummyLogger ) ), ?_assertEqual( #{foo => 1, bar => 2}, - emqx_map_lib:best_effort_recursive_sum(#{foo => 1}, #{bar => 2}, DummyLogger) + emqx_utils_maps:best_effort_recursive_sum(#{foo => 1}, #{bar => 2}, DummyLogger) ), ?_assertEqual( #{foo => #{bar => 42}}, - emqx_map_lib:best_effort_recursive_sum( + emqx_utils_maps:best_effort_recursive_sum( #{foo => #{bar => 2}}, #{foo => #{bar => 40}}, DummyLogger ) ), @@ -45,7 +45,9 @@ best_effort_recursive_sum_test_() -> Logger = fun(What) -> Self ! {log, What} end, ?assertEqual( #{foo => 1, bar => 2}, - emqx_map_lib:best_effort_recursive_sum(#{foo => 1, bar => 2}, #{bar => bar}, Logger) + emqx_utils_maps:best_effort_recursive_sum( + #{foo => 1, bar => 2}, #{bar => bar}, Logger + ) ), receive {log, Log} -> @@ -55,55 +57,55 @@ best_effort_recursive_sum_test_() -> end, ?_assertEqual( #{}, - emqx_map_lib:best_effort_recursive_sum( + emqx_utils_maps:best_effort_recursive_sum( #{foo => foo}, #{foo => bar}, DummyLogger ) ), ?_assertEqual( #{foo => 1}, - emqx_map_lib:best_effort_recursive_sum( + emqx_utils_maps:best_effort_recursive_sum( #{foo => 1}, #{foo => bar}, DummyLogger ) ), ?_assertEqual( #{foo => 1}, - emqx_map_lib:best_effort_recursive_sum( + emqx_utils_maps:best_effort_recursive_sum( #{foo => bar}, #{foo => 1}, DummyLogger ) ), ?_assertEqual( #{foo => #{bar => 1}}, - emqx_map_lib:best_effort_recursive_sum( + emqx_utils_maps:best_effort_recursive_sum( #{foo => #{bar => 1}}, #{foo => 1}, DummyLogger ) ), ?_assertEqual( #{foo => #{bar => 1}}, - emqx_map_lib:best_effort_recursive_sum( + emqx_utils_maps:best_effort_recursive_sum( #{foo => 1}, #{foo => #{bar => 1}}, DummyLogger ) ), ?_assertEqual( #{foo => #{bar => 1}}, - emqx_map_lib:best_effort_recursive_sum( + emqx_utils_maps:best_effort_recursive_sum( #{foo => 1, bar => ignored}, #{foo => #{bar => 1}}, DummyLogger ) ), ?_assertEqual( #{foo => #{bar => 2}, bar => #{foo => 1}}, - emqx_map_lib:best_effort_recursive_sum( + emqx_utils_maps:best_effort_recursive_sum( #{foo => 1, bar => #{foo => 1}}, #{foo => #{bar => 2}, bar => 2}, DummyLogger ) ), ?_assertEqual( #{foo => #{bar => 2}, bar => #{foo => 1}}, - emqx_map_lib:best_effort_recursive_sum( + emqx_utils_maps:best_effort_recursive_sum( #{foo => #{bar => 2}, bar => 2}, #{foo => 1, bar => #{foo => 1}}, DummyLogger ) ), ?_assertEqual( #{foo => #{bar => #{}}}, - emqx_map_lib:best_effort_recursive_sum( + emqx_utils_maps:best_effort_recursive_sum( #{foo => #{bar => #{foo => []}}}, #{foo => 1}, DummyLogger ) ) diff --git a/apps/emqx/test/props/prop_emqx_json.erl b/apps/emqx_utils/test/props/prop_emqx_utils_json.erl similarity index 97% rename from apps/emqx/test/props/prop_emqx_json.erl rename to apps/emqx_utils/test/props/prop_emqx_utils_json.erl index 2bc079634..0be1508da 100644 --- a/apps/emqx/test/props/prop_emqx_json.erl +++ b/apps/emqx_utils/test/props/prop_emqx_utils_json.erl @@ -14,10 +14,10 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(prop_emqx_json). +-module(prop_emqx_utils_json). -import( - emqx_json, + emqx_utils_json, [ decode/1, decode/2, @@ -66,7 +66,7 @@ prop_object_proplist_to_proplist() -> begin {ok, J} = safe_encode(T), {ok, T} = safe_decode(J), - T = decode(encode(T)), + T = decode(encode(T), []), true end ). @@ -108,7 +108,7 @@ prop_object_map_to_proplist() -> T = to_list(T0), {ok, J} = safe_encode(T0), {ok, T} = safe_decode(J), - T = decode(encode(T0)), + T = decode(encode(T0), []), true end ). diff --git a/bin/nodetool b/bin/nodetool index 9a5d5e069..8170e68e2 100755 --- a/bin/nodetool +++ b/bin/nodetool @@ -272,7 +272,7 @@ chkconfig(File) -> end. check_license(Config) -> - ok = application:load(emqx_license), + ok = ensure_application_load(emqx_license), %% This checks formal license validity to ensure %% that the node can successfully start with the given license. @@ -362,6 +362,27 @@ add_libs_dir() -> add_lib_dir(RootDir, Name, Vsn) -> LibDir = filename:join([RootDir, lib, atom_to_list(Name) ++ "-" ++ Vsn, ebin]), case code:add_patha(LibDir) of - true -> ok; + true -> + %% load all applications into application controller, before performing + %% the configuration check of HOCON + %% + %% It helps to implement the feature of dynamically searching schema. + %% See `emqx_gateway_schema:fields(gateway)` + is_emqx_application(Name) andalso ensure_application_load(Name), + ok; {error, _} -> error(LibDir) end. + +is_emqx_application(Name) when is_atom(Name) -> + is_emqx_application(atom_to_list(Name)); +is_emqx_application("emqx_" ++ _Rest) -> + true; +is_emqx_application(_) -> + false. + +ensure_application_load(Name) -> + case application:load(Name) of + ok -> ok; + {error, {already_loaded, _}} -> ok; + {error, Reason} -> error({failed_to_load_application, Name, Reason}) + end. diff --git a/build b/build index 3c558c19a..77d4dbfc8 100755 --- a/build +++ b/build @@ -117,18 +117,14 @@ make_docs() { mkdir -p "$docdir" "$dashboard_www_static" # shellcheck disable=SC2086 erl -noshell -pa $libs_dir1 $libs_dir2 $libs_dir3 -eval \ - "I18nFile = filename:join([apps, emqx_dashboard, priv, 'i18n.conf']), \ - ok = emqx_conf:dump_schema('$docdir', $SCHEMA_MODULE, I18nFile), \ + "ok = emqx_conf:dump_schema('$docdir', $SCHEMA_MODULE), \ halt(0)." cp "$docdir"/bridge-api-*.json "$dashboard_www_static" cp "$docdir"/hot-config-schema-*.json "$dashboard_www_static" } assert_no_compile_time_only_deps() { - if [ "$("$FIND" "_build/$PROFILE/rel/emqx/lib/" -maxdepth 1 -name 'gpb-*' -type d)" != "" ]; then - echo "gpb should not be included in the release" - exit 1 - fi + : } make_rel() { diff --git a/changes/ce/feat-10156.en.md b/changes/ce/feat-10156.en.md new file mode 100644 index 000000000..589578f26 --- /dev/null +++ b/changes/ce/feat-10156.en.md @@ -0,0 +1,7 @@ +Change the priority of the configuration: +1. If it is a new installation of EMQX, the priority of configuration is `ENV > emqx.conf > HTTP API`. +2. If EMQX is upgraded from an old version (i.e., the cluster-override.conf file still exists in EMQX's data directory), then the configuration priority remains the same as before. That is, `HTTP API > ENV > emqx.conf`. + +Deprecated data/configs/local-override.conf. + +Stabilizing the HTTP API for hot updates. diff --git a/changes/ce/feat-10278.en.md b/changes/ce/feat-10278.en.md new file mode 100644 index 000000000..d029c1420 --- /dev/null +++ b/changes/ce/feat-10278.en.md @@ -0,0 +1 @@ +Refactor the directory structure of all gateways. diff --git a/changes/ce/feat-10278.zh.md b/changes/ce/feat-10278.zh.md new file mode 100644 index 000000000..d2e738ec1 --- /dev/null +++ b/changes/ce/feat-10278.zh.md @@ -0,0 +1 @@ +重构所有网关的源码目录结构。 diff --git a/changes/ce/feat-10336.en.md b/changes/ce/feat-10336.en.md new file mode 100644 index 000000000..5e6039f9b --- /dev/null +++ b/changes/ce/feat-10336.en.md @@ -0,0 +1 @@ +Add `/rule_engine` API endpoint to manage configuration of rule engine. diff --git a/changes/ce/feat-10354.en.md b/changes/ce/feat-10354.en.md new file mode 100644 index 000000000..d728a1b20 --- /dev/null +++ b/changes/ce/feat-10354.en.md @@ -0,0 +1,2 @@ +More specific error messages when configure with bad max_heap_size value. +Log current value and the max value when the `message_queue_too_long` error is thrown. diff --git a/changes/ce/feat-10359.en.md b/changes/ce/feat-10359.en.md new file mode 100644 index 000000000..524b6dbdd --- /dev/null +++ b/changes/ce/feat-10359.en.md @@ -0,0 +1 @@ +Metrics now are not implicitly collected in places where API handlers don't make any use of them. Instead, a separate backplane RPC gathers cluster-wide metrics. diff --git a/changes/ce/feat-10373.en.md b/changes/ce/feat-10373.en.md new file mode 100644 index 000000000..7609e2a1d --- /dev/null +++ b/changes/ce/feat-10373.en.md @@ -0,0 +1,2 @@ +Deprecate the trace.payload_encode configuration. +Add payload_encode=[text,hidden,hex] option when creating a trace via HTTP API. diff --git a/changes/ce/feat-10389.en.md b/changes/ce/feat-10389.en.md new file mode 100644 index 000000000..3789e80ae --- /dev/null +++ b/changes/ce/feat-10389.en.md @@ -0,0 +1,2 @@ +Unify the config formats for `cluster.core_nodes` and `cluster.statics.seeds`. +Now they both support formats in array `["emqx1@127.0.0.1", "emqx2@127.0.0.1"]` or semicolon-separated string `"emqx1@127.0.0.1,emqx2@127.0.0.1"`. diff --git a/changes/ce/feat-10391.en.md b/changes/ce/feat-10391.en.md new file mode 100644 index 000000000..f33757404 --- /dev/null +++ b/changes/ce/feat-10391.en.md @@ -0,0 +1,7 @@ +Hide a large number of advanced options to simplify the configuration file. + +That includes `rewrite`, `topic_metric`, `persistent_session_store`, `overload_protection`, +`flapping_detect`, `conn_congestion`, `stats,auto_subscribe`, `broker_perf`, +`shared_subscription_group`, `slow_subs`, `ssl_options.user_lookup_fun` and some advance items +in `node` and `dashboard` section, [#10358](https://github.com/emqx/emqx/pull/10358), +[#10381](https://github.com/emqx/emqx/pull/10381), [#10385](https://github.com/emqx/emqx/pull/10385). diff --git a/changes/ce/feat-10392.en.md b/changes/ce/feat-10392.en.md new file mode 100644 index 000000000..04c6c85cc --- /dev/null +++ b/changes/ce/feat-10392.en.md @@ -0,0 +1 @@ +A new function to convert a formatted date to an integer timestamp has been added: date_to_unix_ts/3 diff --git a/changes/ce/feat-10404.en.md b/changes/ce/feat-10404.en.md new file mode 100644 index 000000000..ad216336e --- /dev/null +++ b/changes/ce/feat-10404.en.md @@ -0,0 +1,2 @@ +Change the default queue mode for buffer workers to `memory_only`. +Before this change, the default queue mode was `volatile_offload`. When under high message rate pressure and when the resource is not keeping up with such rate, the buffer performance degraded a lot due to the constant disk operations. diff --git a/changes/ce/feat-10426.en.md b/changes/ce/feat-10426.en.md new file mode 100644 index 000000000..8575347d6 --- /dev/null +++ b/changes/ce/feat-10426.en.md @@ -0,0 +1,4 @@ +Optimize the configuration priority mechanism to fix the issue where the configuration +changes made to `etc/emqx.conf` do not take effect after restarting EMQX. + +More introduction about the new mechanism: [Configure Override Rules](https://www.emqx.io/docs/en/v5.0/configuration/configuration.html#configure-override-rules) diff --git a/changes/ce/feat-10457.en.md b/changes/ce/feat-10457.en.md new file mode 100644 index 000000000..d6a44bd53 --- /dev/null +++ b/changes/ce/feat-10457.en.md @@ -0,0 +1,4 @@ +Deprecates the integration with StatsD. + +Since StatsD is not used a lot. So we will deprecate it in the next release +and plan to remove it in 5.1 diff --git a/changes/ce/feat-10458.en.md b/changes/ce/feat-10458.en.md new file mode 100644 index 000000000..655885145 --- /dev/null +++ b/changes/ce/feat-10458.en.md @@ -0,0 +1,3 @@ +Set the level of plugin configuration options to low level, +in most cases, users only need to manage plugins on the dashboard +without the need for manual modification, so we lowered the level. diff --git a/changes/ce/feat-9986.en.md b/changes/ce/feat-9986.en.md new file mode 100644 index 000000000..7a63c2e74 --- /dev/null +++ b/changes/ce/feat-9986.en.md @@ -0,0 +1 @@ +Add MQTT ingress to helm charts and update helm charts documentation diff --git a/changes/ce/fix-10014.en.md b/changes/ce/fix-10014.en.md new file mode 100644 index 000000000..2a3674772 --- /dev/null +++ b/changes/ce/fix-10014.en.md @@ -0,0 +1 @@ +Ensure Monitor API `/monitor(_current)/nodes/:node` returns `404` instead of `400` if node does not exist. diff --git a/changes/ce/fix-10014.zh.md b/changes/ce/fix-10014.zh.md new file mode 100644 index 000000000..1ce2a3c43 --- /dev/null +++ b/changes/ce/fix-10014.zh.md @@ -0,0 +1 @@ +如果 API 查询的节点不存在，将会返回 `404` 而不再是 `400`。 diff --git a/changes/ce/fix-10055.en.md b/changes/ce/fix-10055.en.md index 15237c135..b22a319c6 100644 --- a/changes/ce/fix-10055.en.md +++ b/changes/ce/fix-10055.en.md @@ -1 +1 @@ -Fix: configuration parameter `mqtt.max_awaiting_rel` had no effect. +The configuration parameter `mqtt.max_awaiting_rel` was not functional and has now been corrected. diff --git a/changes/ce/fix-10107.en.md b/changes/ce/fix-10107.en.md new file mode 100644 index 000000000..6f76bc083 --- /dev/null +++ b/changes/ce/fix-10107.en.md @@ -0,0 +1,9 @@ +For operations on Bridges API if `bridge-id` is unknown we now return `404` +instead of `400`. Also a bug was fixed that caused a crash if that was a node +operation. Additionally we now also check if the given bridge is enabled when +doing the cluster operation `start` . Affected endpoints: + * [cluster] `/bridges/:id/:operation`, + * [node] `/nodes/:node/bridges/:id/:operation`, where `operation` is one of +`[start|stop|restart]`. +Moreover, for a node operation, EMQX checks if node name is in our cluster and +return `404` instead of `501`. diff --git a/changes/ce/fix-10154.zh.md b/changes/ce/fix-10154.zh.md new file mode 100644 index 000000000..8adf365ae --- /dev/null +++ b/changes/ce/fix-10154.zh.md @@ -0,0 +1 @@ +将数据桥接和连接器的 resume_interval 参数值设为 health_check_interval 和 request_timeout / 3 中的较小值，以解决请求超时的问题。 diff --git a/changes/ce/fix-10237.zh.md b/changes/ce/fix-10237.zh.md new file mode 100644 index 000000000..c4bae060f --- /dev/null +++ b/changes/ce/fix-10237.zh.md @@ -0,0 +1 @@ +当调用 `/nodes/:node[/metrics|/stats]` API ，若节点不存在则返回 `404` 状态码。 diff --git a/changes/ce/fix-10251.zh.md b/changes/ce/fix-10251.zh.md new file mode 100644 index 000000000..0c2e0ba09 --- /dev/null +++ b/changes/ce/fix-10251.zh.md @@ -0,0 +1 @@ +修复了当删除一个使用中的 ingress 类型的桥接时，未提示存在规则依赖的问题。 diff --git a/changes/ce/fix-10314.en.md b/changes/ce/fix-10314.en.md new file mode 100644 index 000000000..0557e714e --- /dev/null +++ b/changes/ce/fix-10314.en.md @@ -0,0 +1,2 @@ +Fix /monitor_current API so that it only looks at the current node. +Fix /stats API to not crash when one or more nodes in the cluster are down. diff --git a/changes/ce/fix-10314.zh.md b/changes/ce/fix-10314.zh.md new file mode 100644 index 000000000..8b99197ca --- /dev/null +++ b/changes/ce/fix-10314.zh.md @@ -0,0 +1 @@ +修复 `/monitor_current` API ，使其仅查看当前 节点。修复了 `/stats` API，以防止当集群中的一个或多个节点关闭时出现崩溃。 diff --git a/changes/ce/fix-10327.zh.md b/changes/ce/fix-10327.zh.md new file mode 100644 index 000000000..e99bda40a --- /dev/null +++ b/changes/ce/fix-10327.zh.md @@ -0,0 +1 @@ +在收到不可恢复的错误时，不要增加 'actions.failed.unknown' 规则指标计数。 diff --git a/changes/ce/fix-10369.en.md b/changes/ce/fix-10369.en.md new file mode 100644 index 000000000..3c32d33f3 --- /dev/null +++ b/changes/ce/fix-10369.en.md @@ -0,0 +1,6 @@ +Fix error in `/api/v5/monitor_current` API endpoint that happens when some EMQX nodes are down. + +Prior to this fix, sometimes the request returned HTTP code 500 and the following message: +``` +{"code":"INTERNAL_ERROR","message":"error, badarg, [{erlang,'++',[{error,nodedown},[{node,'emqx@10.42.0.150'}]], ... +``` diff --git a/changes/ce/fix-10407.en.md b/changes/ce/fix-10407.en.md new file mode 100644 index 000000000..d9df9ce69 --- /dev/null +++ b/changes/ce/fix-10407.en.md @@ -0,0 +1,7 @@ +Improve 'emqx_alarm' performance by using Mnesia dirty operations and avoiding +unnecessary calls from 'emqx_resource_manager' to reactivate alarms that have been already activated. +Use new safe 'emqx_alarm' API to activate/deactivate alarms to ensure that emqx_resource_manager +doesn't crash because of alarm timeouts. +The crashes were possible when the following conditions co-occurred: + - a relatively high number of failing resources, e.g. bridges tried to activate alarms on re-occurring errors; + - the system experienced a very high load. diff --git a/changes/ce/fix-10410.en.md b/changes/ce/fix-10410.en.md new file mode 100644 index 000000000..aa219c96e --- /dev/null +++ b/changes/ce/fix-10410.en.md @@ -0,0 +1,2 @@ +Fix config check failed when gateways are configured in emqx.conf. +This issue was first introduced in v5.0.22 via [#10278](https://github.com/emqx/emqx/pull/10278), the boot-time config check was missing. diff --git a/changes/ce/fix-10420.en.md b/changes/ce/fix-10420.en.md new file mode 100644 index 000000000..70afd8b45 --- /dev/null +++ b/changes/ce/fix-10420.en.md @@ -0,0 +1,3 @@ +Fix HTTP path handling when composing the URL for the HTTP requests in authentication and authorization modules. +* Avoid unnecessary URL normalization since we cannot assume that external servers treat original and normalized URLs equally. This led to bugs like [#10411](https://github.com/emqx/emqx/issues/10411). +* Fix the issue that path segments could be HTTP encoded twice. diff --git a/changes/ce/fix-10420.zh.md b/changes/ce/fix-10420.zh.md new file mode 100644 index 000000000..e69de29bb diff --git a/changes/ce/fix-10422.en.md b/changes/ce/fix-10422.en.md new file mode 100644 index 000000000..7c18ccf32 --- /dev/null +++ b/changes/ce/fix-10422.en.md @@ -0,0 +1 @@ +Fixed a bug where external plugins could not be configured via environment variables in a lone-node cluster. diff --git a/changes/ce/fix-10448.en.md b/changes/ce/fix-10448.en.md new file mode 100644 index 000000000..c35ecdd8b --- /dev/null +++ b/changes/ce/fix-10448.en.md @@ -0,0 +1,3 @@ +Fix a compatibility issue of limiter configuration introduced by v5.0.23 which broke the upgrade from previous versions if the `capacity` is `infinity`. + +In v5.0.23 we have replaced `capacity` with `burst`. After this fix, a `capacity = infinity` config will be automatically converted to equivalent `burst = 0`. diff --git a/changes/ce/fix-10449.en.md b/changes/ce/fix-10449.en.md new file mode 100644 index 000000000..005dea73c --- /dev/null +++ b/changes/ce/fix-10449.en.md @@ -0,0 +1,2 @@ +Validate the ssl_options and header configurations when creating authentication http (`authn_http`). +Prior to this, incorrect `ssl` configuration could result in successful creation but the entire authn being unusable. diff --git a/changes/ce/fix-10455.en.md b/changes/ce/fix-10455.en.md new file mode 100644 index 000000000..07d8c71db --- /dev/null +++ b/changes/ce/fix-10455.en.md @@ -0,0 +1,9 @@ +Fixed an issue that could cause (otherwise harmless) noise in the logs. + +During some particularly slow synchronous calls to bridges, some late replies could be sent to connections processes that were no longer expecting a reply, and then emit an error log like: + +``` +2023-04-19T18:24:35.350233+00:00 [error] msg: unexpected_info, mfa: emqx_channel:handle_info/2, line: 1278, peername: 172.22.0.1:36384, clientid: caribdis_bench_sub_1137967633_4788, info: {#Ref<0.408802983.1941504010.189402>,{ok,200,[{<<"cache-control">>,<<"max-age=0, ...">>}} +``` + +Those logs are harmless, but they could flood and worry the users without need. diff --git a/changes/ce/fix-10463.en.md b/changes/ce/fix-10463.en.md new file mode 100644 index 000000000..9d57bc1b0 --- /dev/null +++ b/changes/ce/fix-10463.en.md @@ -0,0 +1,2 @@ +Improve bridges API error handling. +If Webhook bridge URL is not valid, bridges API will return '400' error instead of '500'. diff --git a/changes/ce/perf-10376.en.md b/changes/ce/perf-10376.en.md new file mode 100644 index 000000000..d585ad5b2 --- /dev/null +++ b/changes/ce/perf-10376.en.md @@ -0,0 +1,6 @@ +Simplify the configuration of the limiter feature and optimize some codes +- Rename `message_in` to `messages` +- Rename `bytes_in` to `bytes` +- Use `burst` instead of `capacity` +- Hide non-importance fields +- Optimize limiter instances in different rate settings diff --git a/changes/ce/perf-10430.en.md b/changes/ce/perf-10430.en.md new file mode 100644 index 000000000..e01d8ba64 --- /dev/null +++ b/changes/ce/perf-10430.en.md @@ -0,0 +1,2 @@ +Simplify the configuration of the `retainer` feature. +- Mark `flow_control` as non-importance field. diff --git a/changes/e5.0.2.en.md b/changes/e5.0.2.en.md new file mode 100644 index 000000000..83eecdfe9 --- /dev/null +++ b/changes/e5.0.2.en.md @@ -0,0 +1,87 @@ +# e5.0.2 + +## Enhancements + +- [#10022](https://github.com/emqx/emqx/pull/10022) Release installation packages for Rocky Linux 9 (compatible with Red Hat Enterprise Linux 9) and macOS 12 for Intel platform. + +- [#10139](https://github.com/emqx/emqx/pull/10139) Add `extraVolumeMounts` to EMQX Helm Chart, you can mount user's own files to EMQX instance, such as ACL rule files mentioned in [#9052](https://github.com/emqx/emqx/issues/9052). + +- [#9893](https://github.com/emqx/emqx/pull/9893) When connecting with the flag `clean_start=false`, EMQX will filter out messages that published by clients banned by the blacklist feature in the session. +Previously, messages sent by clients banned by the blacklist feature could still be delivered to subscribers in this case. + +- [#9986](https://github.com/emqx/emqx/pull/9986) Add MQTT ingress to helm charts and remove obsolete mgmt references. + +- [#9564](https://github.com/emqx/emqx/pull/9564) Implement Kafka Consumer Bridge, which supports consuming messages from Kafka and publishing them to MQTT topics. + +- [#9881](https://github.com/emqx/emqx/pull/9881) Improve error logging related to health checks for InfluxDB connections. + +- [#10123](https://github.com/emqx/emqx/pull/10123) Improve the performance of `/bridges` API. +Earlier, when the number of nodes in the cluster was large or the node was busy, the API may had a request timeout. + +- [#9998](https://github.com/emqx/emqx/pull/9998) Obfuscate request body in error log when using HTTP service for client authentication for security reasons. + +- [#10026](https://github.com/emqx/emqx/pull/10026) Metrics are now only exposed via the `/bridges/:id/metrics` endpoint, and no longer returned in other API operations. + +- [#10052](https://github.com/emqx/emqx/pull/10052) Improve startup failure logs in daemon mode. + + +## Bug Fixes + +- [#10013](https://github.com/emqx/emqx/pull/10013) Fix return type structure for error case in API schema for `/gateways/:name/clients`. + +- [#10014](https://github.com/emqx/emqx/pull/10014) Ensure Monitor API `/monitor(_current)/nodes/:node` returns `404` instead of `400` if node does not exist. + +- [#10027](https://github.com/emqx/emqx/pull/10027) Allow setting node name via environment variable `EMQX_NODE__NAME` in Docker. + +- [#10050](https://github.com/emqx/emqx/pull/10050) Ensure Bridge API returns `404` status code consistently for resources that don't exist. + +- [#10055](https://github.com/emqx/emqx/pull/10055) The configuration parameter `mqtt.max_awaiting_rel` was not functional and has now been corrected. + +- [#10056](https://github.com/emqx/emqx/pull/10056) Fix `/bridges` API status code. + Return `400` instead of `403` in case of removing a data bridge that is dependent on an active rule. + Return `400` instead of `403` in case of calling operations (start|stop|restart) when Data-Bridging is not enabled. + +- [#10066](https://github.com/emqx/emqx/pull/10066) Improve error messages for `/briges_probe` and `[/node/:node]/bridges/:id/:operation` API calls to make them more readable. And set HTTP status code to `400` instead of `500`. + +- [#10074](https://github.com/emqx/emqx/pull/10074) Check if type in `PUT /authorization/sources/:type` matches `type` given in the request body. + +- [#10079](https://github.com/emqx/emqx/pull/10079) Fix wrong description about `shared_subscription_strategy`. + +- [#10085](https://github.com/emqx/emqx/pull/10085) Consistently return `404` for all requests on non-existent source in `/authorization/sources/:source[/*]`. + +- [#10098](https://github.com/emqx/emqx/pull/10098) Fix an issue where the MongoDB connector crashed when MongoDB authorization was configured. + +- [#10100](https://github.com/emqx/emqx/pull/10100) Fix channel crash for slow clients with enhanced authentication. +Previously, when the client was using enhanced authentication, but the Auth message was sent slowly or the Auth message was lost, the client process would crash. + +- [#10107](https://github.com/emqx/emqx/pull/10107) For operations on Bridges API if `bridge-id` is unknown we now return `404` instead of `400`. + +- [#10117](https://github.com/emqx/emqx/pull/10117) Fix an error occurring when a joining node doesn't have plugins that are installed on other nodes in the cluster. +After this fix, the joining node will copy all the necessary plugins from other nodes. + +- [#10118](https://github.com/emqx/emqx/pull/10118) Fix problems related to manual joining of EMQX replicant nodes to the cluster. + +- [#10119](https://github.com/emqx/emqx/pull/10119) Fix crash when `statsd.server` is set to an empty string. + +- [#10124](https://github.com/emqx/emqx/pull/10124) The default heartbeat period for MongoDB has been increased to reduce the risk of too excessive logging to the MongoDB log file. + +- [#10130](https://github.com/emqx/emqx/pull/10130) Fix garbled config display in dashboard when the value is originally from environment variables. + +- [#10132](https://github.com/emqx/emqx/pull/10132) Fix some error logs generated by `systemctl stop emqx` command. +Prior to the fix, the command was not stopping `jq` and `os_mon` applications properly. + +- [#10144](https://github.com/emqx/emqx/pull/10144) Fix an issue where emqx cli failed to set the Erlang cookie when the emqx directory was read-only. + +- [#10154](https://github.com/emqx/emqx/pull/10154) Change the default `resume_interval` for bridges and connectors to be the minimum of `health_check_interval` and `request_timeout / 3` to resolve issue of request timeout. + +- [#10157](https://github.com/emqx/emqx/pull/10157) Fix default rate limit configuration not being applied correctly when creating a new listener. + +- [#10237](https://github.com/emqx/emqx/pull/10237) Ensure we return `404` status code for unknown node names in `/nodes/:node[/metrics|/stats]` API. + +- [#10251](https://github.com/emqx/emqx/pull/10251) Fix an issue where rule dependencies were not prompted when deleting an ingress-type bridge in use. + +- [#10313](https://github.com/emqx/emqx/pull/10313) Ensure that when the core or replicant node starting, the `cluster-override.conf` file is only copied from the core node. + +- [#10327](https://github.com/emqx/emqx/pull/10327) Don't increase “actions.failed.unknown” rule metrics counter upon receiving unrecoverable data bridge errors. + +- [#10095](https://github.com/emqx/emqx/pull/10095) Fix an issue where when the MySQL connector was in batch mode, clients would keep querying the server with unnecessary `PREPARE` statements on each batch, possibly causing server resource exhaustion. diff --git a/changes/e5.0.2.zh.md b/changes/e5.0.2.zh.md new file mode 100644 index 000000000..f6dcc3e72 --- /dev/null +++ b/changes/e5.0.2.zh.md @@ -0,0 +1,79 @@ +# e5.0.2 + +## 优化 + +- [#10022](https://github.com/emqx/emqx/pull/10022) 发布 Rocky Linux 9 (兼容Red Hat Enterprise Linux 9) 以及 macOS 12 Intel 平台的安装包。 + +- [#10139](https://github.com/emqx/emqx/pull/10139) 在 EMQX Helm Chart 中添加 `extraVolumeMounts`，可以挂载用户自己的文件到 EMQX 实例中，例如在 [#9052](https://github.com/emqx/emqx/issues/9052) 中提到的 ACL 规则文件。 + +- [#9893](https://github.com/emqx/emqx/pull/9893) 当使用 `clean_start=false` 标志连接时，EMQX 将过滤掉会话中被被黑名单功能禁止的客户端发布的消息。以前，在这种情况下，被黑名单功能禁止的客户端发送的消息仍可能被传递给订阅者。 + +- [#9986](https://github.com/emqx/emqx/pull/9986) 在 helm charts 中增加 MQTT ingress 并删除过时的 `mgmt` 引用。 + +- [#9564](https://github.com/emqx/emqx/pull/9564) 数据桥接新增 Kafka Consumer，支持从 Kafka 消费消息并将它们发布到 MQTT 主题。 + +- [#9881](https://github.com/emqx/emqx/pull/9881) 改进了与 InfluxDB 连接的健康检查相关的错误日志。 + +- [#10123](https://github.com/emqx/emqx/pull/10123) 改进了 `/bridges` API 的性能。避免了当集群节点数量较多时可能出现的请求响应超时。 + +- [#9998](https://github.com/emqx/emqx/pull/9998) 出于安全原因，在使用 HTTP 服务进行客户端认证时，对错误日志中的请求体进行脱敏处理。 + +- [#10026](https://github.com/emqx/emqx/pull/10026) 仅在 `/bridges/:id/metrics` API 中返回指标数据。 + +- [#10052](https://github.com/emqx/emqx/pull/10052) 改进守护进程模式下启动失败后的日志。 + +## 修复 + +- [#10013](https://github.com/emqx/emqx/pull/10013) 修复 `/gateways/:name/clients` API 在错误情况下的返回类型结构。 + +- [#10014](https://github.com/emqx/emqx/pull/10014) 当节点不存在时，`/monitor(_current)/nodes/:node` API 返回 `404` 错误代码而不是 `400` 。 + +- [#10027](https://github.com/emqx/emqx/pull/10027) 允许在 Docker 中通过 `EMQX_NODE__NAME` 设置节点名称。 + +- [#10050](https://github.com/emqx/emqx/pull/10050) 当调用 Bridge API 时若资源不能存在则返回 `404` 状态码。 + +- [#10055](https://github.com/emqx/emqx/pull/10055) 修复配置项 `mqtt.max_awaiting_rel` 设置无效的问题。 + +- [#10056](https://github.com/emqx/emqx/pull/10056) 修复 `/bridges` API 状态码返回错误。当被删除的 Bridge 存在依赖，或 Bridge 未启用时进行启用、停止、重启等操作时返回 `400` 状态码。 + +- [#10066](https://github.com/emqx/emqx/pull/10066) 优化 `/briges_probe` 和 `[/node/:node]/bridges/:id/:operation` API 调用的错误消息，使其更易理解。并修正错误状态码为`400`。 + +- [#10074](https://github.com/emqx/emqx/pull/10074) 增加 `PUT /authorization/sources/:type` 请求参数 `type` 的值与请求体中的实际类型的一致性检查。 + +- [#10079](https://github.com/emqx/emqx/pull/10079) 修复文档中关于 `shared_subscription_strategy` 的描述错误。 + +- [#10085](https://github.com/emqx/emqx/pull/10085) 对于 `/authorization/sources/:source[/*]` API 中不存在的资源的所有请求，始终返回 `404`。 + +- [#10098](https://github.com/emqx/emqx/pull/10098) 修复了当配置 MongoDB 授权时 MongoDB 连接器崩溃的问题。 + +- [#10100](https://github.com/emqx/emqx/pull/10100) 修复了当客户端使用增强认证时，认证消息发送缓慢或者消息丢失时客户端进程崩溃的问题。 + +- [#10107](https://github.com/emqx/emqx/pull/10107) 当调用 `bridges API` 时如果 `bridge-id` 不存在则返回 `404` 状态码。 + +- [#10117](https://github.com/emqx/emqx/pull/10117) 修复当加入的节点没有安装在集群其他节点上的插件时发生的错误。修复后，加入的节点将从其他节点复制所有必要的插件。 + +- [#10118](https://github.com/emqx/emqx/pull/10118) 修复手动添加 EMQX 副本类型节点到集群的相关问题。 + +- [#10119](https://github.com/emqx/emqx/pull/10119) 修复了当 `statsd.server` 设置为空字符串时会崩溃的问题。 + +- [#10124](https://github.com/emqx/emqx/pull/10124) 调整 MongoDB 的默认心跳周期，以降低日志文件记录过多的风险。 + +- [#10130](https://github.com/emqx/emqx/pull/10130) 修复了通过环境变量设置的值，在 Dashboard 中显示乱码的问题。 + +- [#10132](https://github.com/emqx/emqx/pull/10132) 修复了 `systemctl stop emqx` 命令无法正确停止 `jq` 和 `os_mon` 应用的问题。 + +- [#10144](https://github.com/emqx/emqx/pull/10144) 修复了当 emqx 目录只读时， emqx cli 设置 Erlang cookie 失败的问题。 + +- [#10154](https://github.com/emqx/emqx/pull/10154) 将数据桥接和连接器的 `resume_interval` 参数值设为 `health_check_interval` 和 `request_timeout / 3` 中的较小值，以解决请求超时的问题。 + +- [#10157](https://github.com/emqx/emqx/pull/10157) 修复在创建新的监听器时默认速率限制不生效的问题。 + +- [#10237](https://github.com/emqx/emqx/pull/10237) 当调用 `/nodes/:node[/metrics|/stats]` API，若节点不存在则返回 `404` 状态码。 + +- [#10251](https://github.com/emqx/emqx/pull/10251) 修复了当删除一个使用中的 ingress 类型的桥接时，未提示存在规则依赖的问题。 + +- [#10313](https://github.com/emqx/emqx/pull/10313) 确保在核心节点或副本节点启动时，仅从核心节点复制 `cluster-override.conf` 文件。 + +- [#10327](https://github.com/emqx/emqx/pull/10327) 数据桥接出现不可恢复的错误不再计入到 `actions.failed.unknown` 指标中。 + +- [#10095](https://github.com/emqx/emqx/pull/10095) 修复当 MySQL 连接器处于批处理模式时，会发生客户端在每个批次上不断使用不必要的 `PREPARE` 语句查询服务器，可能会导致服务器资源耗尽的问题。 diff --git a/changes/ee/feat-10140.en.md b/changes/ee/feat-10140.en.md index f2616fda1..42238a21f 100644 --- a/changes/ee/feat-10140.en.md +++ b/changes/ee/feat-10140.en.md @@ -1,4 +1,2 @@ Integrate `Cassandra` into `bridges` as a new backend. At the current stage: - Only support Cassandra version 3.x, not yet 4.x. -- Only support storing data in synchronously, not yet asynchronous and batch - method to store data to Cassandra. diff --git a/changes/ee/feat-10140.zh.md b/changes/ee/feat-10140.zh.md index 5b070133a..0d0ece3a0 100644 --- a/changes/ee/feat-10140.zh.md +++ b/changes/ee/feat-10140.zh.md @@ -1,3 +1,2 @@ 支持 Cassandra 数据桥接。在当前阶段： - 仅支持 Cassandra 3.x 版本，暂不支持 4.x。 -- 仅支持以同步的方式存储数据，暂不支持异步和批量的方式来存储数据到 Cassandra。 diff --git a/changes/ee/feat-10337.en.md b/changes/ee/feat-10337.en.md new file mode 100644 index 000000000..299933351 --- /dev/null +++ b/changes/ee/feat-10337.en.md @@ -0,0 +1,3 @@ +Add schema registry feature. + +With schema registry, one can encode and decode special serialization formats in payloads when transforming messages in Rule Engine. Currently, only [Apache Avro](https://avro.apache.org/) is supported. diff --git a/changes/ee/feat-10363.en.md b/changes/ee/feat-10363.en.md new file mode 100644 index 000000000..84816031d --- /dev/null +++ b/changes/ee/feat-10363.en.md @@ -0,0 +1 @@ +Implement Microsoft SQL Server bridge. diff --git a/changes/ee/feat-10408.en.md b/changes/ee/feat-10408.en.md new file mode 100644 index 000000000..75cc4b945 --- /dev/null +++ b/changes/ee/feat-10408.en.md @@ -0,0 +1 @@ +The rule engine SQL-like language has got three more built-in functions for creating values of the MongoDB date type. These functions are useful for rules with MongoDB bridge actions only and not supported in other actions. diff --git a/changes/ee/feat-10409.en.md b/changes/ee/feat-10409.en.md new file mode 100644 index 000000000..dfa9bfa76 --- /dev/null +++ b/changes/ee/feat-10409.en.md @@ -0,0 +1 @@ +Add support for [Protocol Buffers](https://protobuf.dev/) schemas in Schema Registry. diff --git a/changes/ee/feat-9564.en.md b/changes/ee/feat-9564.en.md new file mode 100644 index 000000000..93cbcdb0d --- /dev/null +++ b/changes/ee/feat-9564.en.md @@ -0,0 +1,2 @@ +Implement Kafka Consumer bridge. +Now it's possible to consume messages from Kafka and publish them to MQTT topics. diff --git a/changes/ee/feat-9881.en.md b/changes/ee/feat-9881.en.md new file mode 100644 index 000000000..b5c6de484 --- /dev/null +++ b/changes/ee/feat-9881.en.md @@ -0,0 +1,4 @@ +Enhance the error logs related to InfluxDB connectivity health checks. +Previously, if InfluxDB failed to pass the health checks using the specified parameters, the only message provided was "timed out waiting for it to become healthy". +With the updated implementation, the error message will be displayed in both the logs and the dashboard, enabling easier identification and resolution of the issue. + diff --git a/changes/ee/fix-10324.en.md b/changes/ee/fix-10324.en.md new file mode 100644 index 000000000..2d4c323da --- /dev/null +++ b/changes/ee/fix-10324.en.md @@ -0,0 +1 @@ +Previously, when attempting to reconnect to a misconfigured Clickhouse bridge through the dashboard, users would not receive an error message. This issue is now resolved, and error messages will now be displayed diff --git a/changes/ee/fix-10324.zh.md b/changes/ee/fix-10324.zh.md new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/changes/ee/fix-10324.zh.md @@ -0,0 +1 @@ + diff --git a/changes/ee/fix-10438.en.md b/changes/ee/fix-10438.en.md new file mode 100644 index 000000000..6394bc3cf --- /dev/null +++ b/changes/ee/fix-10438.en.md @@ -0,0 +1,5 @@ +Fix some configuration item terminology errors in the DynamoDB data bridge: + +- Changed `database` to `table` +- Changed `username` to `aws_access_key_id` +- Changed `password` to `aws_secret_access_key` diff --git a/changes/v5.0.22.en.md b/changes/v5.0.22.en.md new file mode 100644 index 000000000..853f781cc --- /dev/null +++ b/changes/v5.0.22.en.md @@ -0,0 +1,131 @@ +# v5.0.22 + +## Enhancements + +- [#10077](https://github.com/emqx/emqx/pull/10077) Add support for QUIC TLS password protected certificate file. + + +- [#10128](https://github.com/emqx/emqx/pull/10128) Add support for OCSP stapling for SSL MQTT listeners. + +- [#10164](https://github.com/emqx/emqx/pull/10164) Add CRL check support for TLS MQTT listeners. + +- [#10206](https://github.com/emqx/emqx/pull/10206) Decouple the query mode from the underlying call mode for buffer + workers. + + Prior to this change, setting the query mode of a resource + such as a bridge to `sync` would force the buffer to call the + underlying connector in a synchronous way, even if it supports async + calls. + +- [#10207](https://github.com/emqx/emqx/pull/10207) Use 'label' from i18n file as 'summary' in OpenAPI spec. + +- [#10210](https://github.com/emqx/emqx/pull/10210) Unregister Mnesia post commit hook when Mria is being stopped. + This fixes hook failures occasionally occurring on stopping/restarting Mria. + + [Mria PR](https://github.com/emqx/mria/pull/133) + +- [#10224](https://github.com/emqx/emqx/pull/10224) Add the option to customize `clusterIP` in Helm chart, so that a user may set it to a fixed IP. + +- [#10263](https://github.com/emqx/emqx/pull/10263) Add command 'eval-ex' for Elixir expression evaluation. + +- [#10278](https://github.com/emqx/emqx/pull/10278) Refactor the directory structure of all gateways. + +- [#10306](https://github.com/emqx/emqx/pull/10306) Add support for `async` query mode for most bridges. + + Before this change, some bridges (Cassandra, MongoDB, MySQL, Postgres, Redis, RocketMQ, TDengine) were only allowed to be created with a `sync` query mode. + +- [#10318](https://github.com/emqx/emqx/pull/10318) Now, the rule engine language's FROM clause supports both strings enclosed in double quotes (") and single quotes ('). + +- [#10336](https://github.com/emqx/emqx/pull/10336) Add `/rule_engine` API endpoint to manage configuration of rule engine. + + + +## Bug Fixes + +- [#10145](https://github.com/emqx/emqx/pull/10145) Fix `bridges` API to report error conditions for a failing bridge as + `status_reason`. Also when creating an alarm for a failing resource we include + this error condition with the alarm's message. + +- [#10154](https://github.com/emqx/emqx/pull/10154) Change the default `resume_interval` for bridges and connectors to be + the minimum of `health_check_interval` and `request_timeout / 3`. + Also exposes it as a hidden configuration to allow fine tuning. + + Before this change, the default values for `resume_interval` meant + that, if a buffer ever got blocked due to resource errors or high + message volumes, then, by the time the buffer would try to resume its + normal operations, almost all requests would have timed out. + +- [#10172](https://github.com/emqx/emqx/pull/10172) Fix the incorrect default ACL rule, which was: + ``` + {allow, {username, "^dashboard?"}, subscribe, ["$SYS/#"]}. + ``` + + However, it should use `{re, "^dashboard$"}` to perform a regular expression match: + ``` + {allow, {username, {re,"^dashboard$"}}, subscribe, ["$SYS/#"]}. + ``` + +- [#10174](https://github.com/emqx/emqx/pull/10174) Upgrade library `esockd` from 5.9.4 to 5.9.6. + Fix an unnecessary error level logging when a connection is closed before proxy protocol header is sent by the proxy. + +- [#10195](https://github.com/emqx/emqx/pull/10195) Add labels to API schemas where description contains HTML and breaks formatting of generated documentation otherwise. + +- [#10196](https://github.com/emqx/emqx/pull/10196) Use lower-case for schema summaries and descritptions to be used in menu of generated online documentation. + +- [#10209](https://github.com/emqx/emqx/pull/10209) Fix bug where a last will testament (LWT) message could be published + when kicking out a banned client. + +- [#10211](https://github.com/emqx/emqx/pull/10211) Hide `broker.broker_perf` config and API documents. + The two configs `route_lock_type` and `trie_compaction` are rarely used and requires a full cluster restart to take effect. They are not suitable for being exposed to users. + Detailed changes can be found here: https://gist.github.com/zmstone/01ad5754b9beaeaf3f5b86d14d49a0b7/revisions + +- [#10225](https://github.com/emqx/emqx/pull/10225) Allow installing a plugin if its name matches the beginning of another (already installed) plugin name. + For example: if plugin "emqx_plugin_template_a" is installed, it must not block installing plugin "emqx_plugin_template". + +- [#10226](https://github.com/emqx/emqx/pull/10226) Don't crash on validation error in `/bridges` API, return `400` instead. + +- [#10237](https://github.com/emqx/emqx/pull/10237) Ensure we return `404` status code for unknown node names in `/nodes/:node[/metrics|/stats]` API. + +- [#10242](https://github.com/emqx/emqx/pull/10242) Fixed a log data field name clash. + Piror to this fix, some debug logs may report a wrong Erlang PID which may affect troubleshooting session takeover issues. + +- [#10251](https://github.com/emqx/emqx/pull/10251) Consider bridges referenced in `FROM` rule clauses as dependencies. + + Before this fix, when one tried to delete an ingress rule referenced in an action like `select * from "$bridges/mqtt:ingress"`, the UI would not trigger a warning about dependent rule actions. + +- [#10257](https://github.com/emqx/emqx/pull/10257) Fixed the issue where `auto_observe` was not working in LwM2M Gateway. + + Before the fix, OBSERVE requests were sent without a token, causing failures + that LwM2M clients could not handle. + + After the fix, LwM2M Gateway can correctly observe the resource list carried by + client, furthermore, unknown resources will be ignored and printing the following + warning log: + ``` + 2023-03-28T18:50:27.771123+08:00 [warning] msg: ignore_observer_resource, mfa: emqx_lwm2m_session:observe_object_list/3, line: 522, peername: 127.0.0.1:56830, clientid: testlwm2mclient, object_id: 31024, reason: no_xml_definition + ``` + +- [#10286](https://github.com/emqx/emqx/pull/10286) Enhance logging behaviour during boot failure. + When EMQX fails to start due to corrupted configuration files, excessive logging is eliminated and no crash dump file is generated. + +- [#10297](https://github.com/emqx/emqx/pull/10297) Keeps `eval` command backward compatible with v4 by evaluating only Erlang expressions, even on Elixir node. For Elixir expressions, use `eval-ex` command. + +- [#10300](https://github.com/emqx/emqx/pull/10300) Fixed an issue where a build made with Elixir could not receive uploaded plugins until the `plugins` folder was created manually to receive the uploaded files. + +- [#10313](https://github.com/emqx/emqx/pull/10313) Ensure that when the core or replicant node starting, the `cluster-override.conf` file is only copied from the core node. + Previously, when sorting nodes by startup time, the core node may have copied this file from the replicant node. + +- [#10314](https://github.com/emqx/emqx/pull/10314) Fix /monitor_current API so that it only looks at the current node. + Fix /stats API to not crash when one or more nodes in the cluster are down. + +- [#10315](https://github.com/emqx/emqx/pull/10315) Fix crash checking `limit` and `page` parameters in `/mqtt/delayed/messages` API call. + +- [#10317](https://github.com/emqx/emqx/pull/10317) Do not expose listener level authentications before extensive verification. + +- [#10323](https://github.com/emqx/emqx/pull/10323) For security reasons, the value of the `password` field in the API examples is replaced with `******`. + + +- [#10327](https://github.com/emqx/emqx/pull/10327) Don't increment 'actions.failed.unknown' rule metrics counter upon receiving unrecoverable bridge errors. + This counter is displayed on the dashboard's rule overview tab ('Action statistics' - 'Unknown'). + The fix is only applicable for synchronous bridges, as all rule actions for asynchronous bridges + are counted as successful (they increment 'actions.success' which is displayed as 'Action statistics' - 'Success'). diff --git a/changes/v5.0.23.en.md b/changes/v5.0.23.en.md new file mode 100644 index 000000000..6d016d2da --- /dev/null +++ b/changes/v5.0.23.en.md @@ -0,0 +1,62 @@ +# v5.0.23 + +## Enhancements + +- [#10156](https://github.com/emqx/emqx/pull/10156) Change the priority of the configuration: + 1. If it is a new installation of EMQX, the priority of configuration is `ENV > emqx.conf > HTTP API`. + 2. If EMQX is upgraded from an old version (i.e., the cluster-override.conf file still exists in EMQX's data directory), then the configuration priority remains the same as before. That is, `HTTP API > ENV > emqx.conf`. + + Deprecated data/configs/local-override.conf. + + Stabilizing the HTTP API for hot updates. + +- [#10354](https://github.com/emqx/emqx/pull/10354) More specific error messages when configure with bad max_heap_size value. + Log current value and the max value when the `message_queue_too_long` error is thrown. + +- [#10359](https://github.com/emqx/emqx/pull/10359) Metrics now are not implicitly collected in places where API handlers don't make any use of them. Instead, a separate backplane RPC gathers cluster-wide metrics. + +- [#10373](https://github.com/emqx/emqx/pull/10373) Deprecate the trace.payload_encode configuration. + Add payload_encode=[text,hidden,hex] option when creating a trace via HTTP API. + +- [#10389](https://github.com/emqx/emqx/pull/10389) Unify the config formats for `cluster.core_nodes` and `cluster.statics.seeds`. + Now they both support formats in array `["emqx1@127.0.0.1", "emqx2@127.0.0.1"]` or semicolon-separated string `"emqx1@127.0.0.1,emqx2@127.0.0.1"`. + +- [#10391](https://github.com/emqx/emqx/pull/10391) Hide a large number of advanced options to simplify the configuration file. + + That includes `rewrite`, `topic_metric`, `persistent_session_store`, `overload_protection`, + `flapping_detect`, `conn_congestion`, `stats,auto_subscribe`, `broker_perf`, + `shared_subscription_group`, `slow_subs`, `ssl_options.user_lookup_fun` and some advance items + in `node` and `dashboard` section, [#10358](https://github.com/emqx/emqx/pull/10358), + [#10381](https://github.com/emqx/emqx/pull/10381), [#10385](https://github.com/emqx/emqx/pull/10385). + +- [#10392](https://github.com/emqx/emqx/pull/10392) A new function to convert a formatted date to an integer timestamp has been added: date_to_unix_ts/3 + +- [#10404](https://github.com/emqx/emqx/pull/10404) Change the default queue mode for buffer workers to `memory_only`. + Before this change, the default queue mode was `volatile_offload`. When under high message rate pressure and when the resource is not keeping up with such rate, the buffer performance degraded a lot due to the constant disk operations. + +- [#10426](https://github.com/emqx/emqx/pull/10426) Optimize the configuration priority mechanism to fix the issue where the configuration + changes made to `etc/emqx.conf` do not take effect after restarting EMQX. + + More introduction about the new mechanism: [Configure Override Rules](https://www.emqx.io/docs/en/v5.0/configuration/configuration.html#configure-override-rules) + +- [#10376](https://github.com/emqx/emqx/pull/10376) Simplify the configuration of the limiter feature and optimize some codes + - Rename `message_in` to `messages` + - Rename `bytes_in` to `bytes` + - Use `burst` instead of `capacity` + - Hide non-importance fields + - Optimize limiter instances in different rate settings + +- [#10430](https://github.com/emqx/emqx/pull/10430) Simplify the configuration of the `retainer` feature. + - Mark `flow_control` as non-importance field. + +## Bug Fixes + +- [#10369](https://github.com/emqx/emqx/pull/10369) Fix error in `/api/v5/monitor_current` API endpoint that happens when some EMQX nodes are down. + + Prior to this fix, sometimes the request returned HTTP code 500 and the following message: + ``` + {"code":"INTERNAL_ERROR","message":"error, badarg, [{erlang,'++',[{error,nodedown},[{node,'emqx@10.42.0.150'}]], ... + ``` + +- [#10410](https://github.com/emqx/emqx/pull/10410) Fix config check failed when gateways are configured in emqx.conf. + This issue was first introduced in v5.0.22 via [#10278](https://github.com/emqx/emqx/pull/10278), the boot-time config check was missing. diff --git a/deploy/charts/emqx-enterprise/Chart.yaml b/deploy/charts/emqx-enterprise/Chart.yaml index 4b5382090..85bd20f6e 100644 --- a/deploy/charts/emqx-enterprise/Chart.yaml +++ b/deploy/charts/emqx-enterprise/Chart.yaml @@ -14,8 +14,8 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 5.0.2 +version: 5.0.3 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 5.0.2 +appVersion: 5.0.3 diff --git a/deploy/charts/emqx/Chart.yaml b/deploy/charts/emqx/Chart.yaml index a0662c9cd..312a9dfbe 100644 --- a/deploy/charts/emqx/Chart.yaml +++ b/deploy/charts/emqx/Chart.yaml @@ -14,8 +14,8 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 5.0.21 +version: 5.0.23 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 5.0.21 +appVersion: 5.0.23 diff --git a/lib-ee/emqx_ee_bridge/docker-ct b/lib-ee/emqx_ee_bridge/docker-ct index 963122082..aa19a495f 100644 --- a/lib-ee/emqx_ee_bridge/docker-ct +++ b/lib-ee/emqx_ee_bridge/docker-ct @@ -1,6 +1,5 @@ toxiproxy influxdb -kafka mongo mongo_rs_sharded mysql @@ -11,4 +10,4 @@ tdengine clickhouse dynamo rocketmq -cassandra +sqlserver diff --git a/lib-ee/emqx_ee_bridge/rebar.config b/lib-ee/emqx_ee_bridge/rebar.config index 985b387d4..358ff3bc8 100644 --- a/lib-ee/emqx_ee_bridge/rebar.config +++ b/lib-ee/emqx_ee_bridge/rebar.config @@ -1,12 +1,8 @@ {erl_opts, [debug_info]}. -{deps, [ {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "1.7.5"}}} - , {kafka_protocol, {git, "https://github.com/kafka4beam/kafka_protocol.git", {tag, "4.1.2"}}} - , {brod_gssapi, {git, "https://github.com/kafka4beam/brod_gssapi.git", {tag, "v0.1.0-rc1"}}} - , {brod, {git, "https://github.com/kafka4beam/brod.git", {tag, "3.16.8"}}} - , {ecql, {git, "https://github.com/emqx/ecql.git", {tag, "v0.4.2"}}} - , {emqx_connector, {path, "../../apps/emqx_connector"}} +{deps, [ {emqx_connector, {path, "../../apps/emqx_connector"}} , {emqx_resource, {path, "../../apps/emqx_resource"}} , {emqx_bridge, {path, "../../apps/emqx_bridge"}} + , {emqx_utils, {path, "../emqx_utils"}} ]}. {shell, [ diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src index 156c3eeac..440889d02 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src @@ -1,12 +1,14 @@ {application, emqx_ee_bridge, [ {description, "EMQX Enterprise data bridges"}, - {vsn, "0.1.8"}, - {registered, [emqx_ee_bridge_kafka_consumer_sup]}, + {vsn, "0.1.11"}, + {registered, []}, {applications, [ kernel, stdlib, emqx_ee_connector, - telemetry + telemetry, + emqx_bridge_kafka, + emqx_bridge_gcp_pubsub ]}, {env, []}, {modules, []}, diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl index 1ddc1a110..7fdfbba99 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl @@ -14,9 +14,10 @@ api_schemas(Method) -> [ - ref(emqx_ee_bridge_gcp_pubsub, Method), - ref(emqx_ee_bridge_kafka, Method ++ "_consumer"), - ref(emqx_ee_bridge_kafka, Method ++ "_producer"), + ref(emqx_bridge_gcp_pubsub, Method), + ref(emqx_bridge_kafka, Method ++ "_consumer"), + ref(emqx_bridge_kafka, Method ++ "_producer"), + ref(emqx_bridge_cassandra, Method), ref(emqx_ee_bridge_mysql, Method), ref(emqx_ee_bridge_pgsql, Method), ref(emqx_ee_bridge_mongodb, Method ++ "_rs"), @@ -34,14 +35,15 @@ api_schemas(Method) -> ref(emqx_ee_bridge_clickhouse, Method), ref(emqx_ee_bridge_dynamo, Method), ref(emqx_ee_bridge_rocketmq, Method), - ref(emqx_ee_bridge_cassa, Method) + ref(emqx_ee_bridge_sqlserver, Method) ]. schema_modules() -> [ - emqx_ee_bridge_kafka, + emqx_bridge_kafka, + emqx_bridge_cassandra, emqx_ee_bridge_hstreamdb, - emqx_ee_bridge_gcp_pubsub, + emqx_bridge_gcp_pubsub, emqx_ee_bridge_influxdb, emqx_ee_bridge_mongodb, emqx_ee_bridge_mysql, @@ -53,7 +55,7 @@ schema_modules() -> emqx_ee_bridge_clickhouse, emqx_ee_bridge_dynamo, emqx_ee_bridge_rocketmq, - emqx_ee_bridge_cassa + emqx_ee_bridge_sqlserver ]. examples(Method) -> @@ -69,12 +71,13 @@ examples(Method) -> lists:foldl(Fun, #{}, schema_modules()). resource_type(Type) when is_binary(Type) -> resource_type(binary_to_atom(Type, utf8)); -resource_type(kafka_consumer) -> emqx_bridge_impl_kafka_consumer; +resource_type(kafka_consumer) -> emqx_bridge_kafka_impl_consumer; %% TODO: rename this to `kafka_producer' after alias support is added %% to hocon; keeping this as just `kafka' for backwards compatibility. -resource_type(kafka) -> emqx_bridge_impl_kafka_producer; +resource_type(kafka) -> emqx_bridge_kafka_impl_producer; +resource_type(cassandra) -> emqx_bridge_cassandra_connector; resource_type(hstreamdb) -> emqx_ee_connector_hstreamdb; -resource_type(gcp_pubsub) -> emqx_ee_connector_gcp_pubsub; +resource_type(gcp_pubsub) -> emqx_bridge_gcp_pubsub_connector; resource_type(mongodb_rs) -> emqx_ee_connector_mongodb; resource_type(mongodb_sharded) -> emqx_ee_connector_mongodb; resource_type(mongodb_single) -> emqx_ee_connector_mongodb; @@ -91,7 +94,7 @@ resource_type(tdengine) -> emqx_ee_connector_tdengine; resource_type(clickhouse) -> emqx_ee_connector_clickhouse; resource_type(dynamo) -> emqx_ee_connector_dynamo; resource_type(rocketmq) -> emqx_ee_connector_rocketmq; -resource_type(cassandra) -> emqx_ee_connector_cassa. +resource_type(sqlserver) -> emqx_ee_connector_sqlserver. fields(bridges) -> [ @@ -105,7 +108,7 @@ fields(bridges) -> )}, {gcp_pubsub, mk( - hoconsc:map(name, ref(emqx_ee_bridge_gcp_pubsub, "config")), + hoconsc:map(name, ref(emqx_bridge_gcp_pubsub, "config")), #{ desc => <<"EMQX Enterprise Config">>, required => false @@ -145,14 +148,14 @@ fields(bridges) -> )}, {cassandra, mk( - hoconsc:map(name, ref(emqx_ee_bridge_cassa, "config")), + hoconsc:map(name, ref(emqx_bridge_cassandra, "config")), #{ desc => <<"Cassandra Bridge Config">>, required => false } )} ] ++ kafka_structs() ++ mongodb_structs() ++ influxdb_structs() ++ redis_structs() ++ - pgsql_structs() ++ clickhouse_structs(). + pgsql_structs() ++ clickhouse_structs() ++ sqlserver_structs(). mongodb_structs() -> [ @@ -174,16 +177,16 @@ kafka_structs() -> %% backwards compatibility. {kafka, mk( - hoconsc:map(name, ref(emqx_ee_bridge_kafka, kafka_producer)), + hoconsc:map(name, ref(emqx_bridge_kafka, kafka_producer)), #{ desc => <<"Kafka Producer Bridge Config">>, required => false, - converter => fun emqx_ee_bridge_kafka:kafka_producer_converter/2 + converter => fun emqx_bridge_kafka:kafka_producer_converter/2 } )}, {kafka_consumer, mk( - hoconsc:map(name, ref(emqx_ee_bridge_kafka, kafka_consumer)), + hoconsc:map(name, ref(emqx_bridge_kafka, kafka_consumer)), #{desc => <<"Kafka Consumer Bridge Config">>, required => false} )} ]. @@ -249,3 +252,15 @@ clickhouse_structs() -> } )} ]. + +sqlserver_structs() -> + [ + {sqlserver, + mk( + hoconsc:map(name, ref(emqx_ee_bridge_sqlserver, "config")), + #{ + desc => <<"Microsoft SQL Server Bridge Config">>, + required => false + } + )} + ]. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_clickhouse.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_clickhouse.erl index 0b611c142..56671c586 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_clickhouse.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_clickhouse.erl @@ -61,7 +61,7 @@ values(_Method, Type) -> batch_size => ?DEFAULT_BATCH_SIZE, batch_time => ?DEFAULT_BATCH_TIME, query_mode => async, - max_queue_bytes => ?DEFAULT_QUEUE_SIZE + max_buffer_bytes => ?DEFAULT_BUFFER_BYTES } }. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_dynamo.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_dynamo.erl index e6a3d1a58..cbfa5b6b1 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_dynamo.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_dynamo.erl @@ -43,10 +43,10 @@ values(_Method) -> type => dynamo, name => <<"foo">>, url => <<"http://127.0.0.1:8000">>, - database => <<"mqtt">>, + table => <<"mqtt">>, pool_size => 8, - username => <<"root">>, - password => <<"******">>, + aws_access_key_id => <<"root">>, + aws_secret_access_key => <<"******">>, template => ?DEFAULT_TEMPLATE, local_topic => <<"local/topic/#">>, resource_opts => #{ @@ -56,7 +56,7 @@ values(_Method) -> batch_size => ?DEFAULT_BATCH_SIZE, batch_time => ?DEFAULT_BATCH_TIME, query_mode => sync, - max_queue_bytes => ?DEFAULT_QUEUE_SIZE + max_buffer_bytes => ?DEFAULT_BUFFER_BYTES } }. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl index f3ed44247..7914c77e2 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl @@ -57,7 +57,7 @@ values(_Method) -> batch_size => ?DEFAULT_BATCH_SIZE, batch_time => ?DEFAULT_BATCH_TIME, query_mode => async, - max_queue_bytes => ?DEFAULT_QUEUE_SIZE + max_buffer_bytes => ?DEFAULT_BUFFER_BYTES } }. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl index 958bc3449..a5dcb19e6 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl @@ -59,7 +59,7 @@ values(_Method, Type) -> batch_size => ?DEFAULT_BATCH_SIZE, batch_time => ?DEFAULT_BATCH_TIME, query_mode => async, - max_queue_bytes => ?DEFAULT_QUEUE_SIZE + max_buffer_bytes => ?DEFAULT_BUFFER_BYTES } }. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_rocketmq.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_rocketmq.erl index 78fd527d3..28b94a1a4 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_rocketmq.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_rocketmq.erl @@ -56,7 +56,7 @@ values(post) -> batch_size => ?DEFAULT_BATCH_SIZE, batch_time => ?DEFAULT_BATCH_TIME, query_mode => sync, - max_queue_bytes => ?DEFAULT_QUEUE_SIZE + max_buffer_bytes => ?DEFAULT_BUFFER_BYTES } }; values(put) -> diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_sqlserver.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_sqlserver.erl new file mode 100644 index 000000000..e216299c2 --- /dev/null +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_sqlserver.erl @@ -0,0 +1,128 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_ee_bridge_sqlserver). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("emqx_bridge/include/emqx_bridge.hrl"). +-include_lib("emqx_resource/include/emqx_resource.hrl"). + +-import(hoconsc, [mk/2, enum/1, ref/2]). + +-export([ + conn_bridge_examples/1 +]). + +-export([ + namespace/0, + roots/0, + fields/1, + desc/1 +]). + +-define(DEFAULT_SQL, << + "insert into t_mqtt_msg(msgid, topic, qos, payload)" + "values (${id}, ${topic}, ${qos}, ${payload})" +>>). + +-define(DEFAULT_DRIVER, <<"ms-sqlserver-18">>). + +conn_bridge_examples(Method) -> + [ + #{ + <<"sqlserver">> => #{ + summary => <<"Microsoft SQL Server Bridge">>, + value => values(Method) + } + } + ]. + +values(get) -> + values(post); +values(post) -> + #{ + enable => true, + type => sqlserver, + name => <<"bar">>, + server => <<"127.0.0.1:1433">>, + database => <<"test">>, + pool_size => 8, + username => <<"sa">>, + password => <<"******">>, + sql => ?DEFAULT_SQL, + driver => ?DEFAULT_DRIVER, + local_topic => <<"local/topic/#">>, + resource_opts => #{ + worker_pool_size => 1, + health_check_interval => ?HEALTHCHECK_INTERVAL_RAW, + auto_restart_interval => ?AUTO_RESTART_INTERVAL_RAW, + batch_size => ?DEFAULT_BATCH_SIZE, + batch_time => ?DEFAULT_BATCH_TIME, + query_mode => async, + max_buffer_bytes => ?DEFAULT_BUFFER_BYTES + } + }; +values(put) -> + values(post). + +%% ------------------------------------------------------------------------------------------------- +%% Hocon Schema Definitions +namespace() -> "bridge_sqlserver". + +roots() -> []. + +fields("config") -> + [ + {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})}, + {sql, + mk( + binary(), + #{desc => ?DESC("sql_template"), default => ?DEFAULT_SQL, format => <<"sql">>} + )}, + {driver, mk(binary(), #{desc => ?DESC("driver"), default => ?DEFAULT_DRIVER})}, + {local_topic, + mk( + binary(), + #{desc => ?DESC("local_topic"), default => undefined} + )}, + {resource_opts, + mk( + ref(?MODULE, "creation_opts"), + #{ + required => false, + default => #{}, + desc => ?DESC(emqx_resource_schema, <<"resource_opts">>) + } + )} + ] ++ + (emqx_ee_connector_sqlserver:fields(config) -- + emqx_connector_schema_lib:prepare_statement_fields()); +fields("creation_opts") -> + emqx_resource_schema:fields("creation_opts"); +fields("post") -> + fields("post", sqlserver); +fields("put") -> + fields("config"); +fields("get") -> + emqx_bridge_schema:status_fields() ++ fields("post"). + +fields("post", Type) -> + [type_field(Type), name_field() | fields("config")]. + +desc("config") -> + ?DESC("desc_config"); +desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> + ["Configuration for Microsoft SQL Server using `", string:to_upper(Method), "` method."]; +desc("creation_opts" = Name) -> + emqx_resource_schema:desc(Name); +desc(_) -> + undefined. + +%% ------------------------------------------------------------------------------------------------- + +type_field(Type) -> + {type, mk(enum([Type]), #{required => true, desc => ?DESC("desc_type")})}. + +name_field() -> + {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})}. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_tdengine.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_tdengine.erl index 7a958d45f..54406541d 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_tdengine.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_tdengine.erl @@ -58,7 +58,7 @@ values(_Method) -> batch_size => ?DEFAULT_BATCH_SIZE, batch_time => ?DEFAULT_BATCH_TIME, query_mode => sync, - max_queue_bytes => ?DEFAULT_QUEUE_SIZE + max_buffer_bytes => ?DEFAULT_BUFFER_BYTES } }. diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl index c0d58c4f7..9cf7eb8f4 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl @@ -14,8 +14,8 @@ % DB defaults -define(TABLE, "mqtt"). -define(TABLE_BIN, to_bin(?TABLE)). --define(USERNAME, "root"). --define(PASSWORD, "public"). +-define(ACCESS_KEY_ID, "root"). +-define(SECRET_ACCESS_KEY, "public"). -define(HOST, "dynamo"). -define(PORT, 8000). -define(SCHEMA, "http://"). @@ -40,7 +40,7 @@ groups() -> %% due to the poorly implemented driver or other reasons %% if we mix these cases with others, this suite will become flaky. - Flaky = [t_get_status, t_write_failure, t_write_timeout], + Flaky = [t_get_status, t_write_failure], TCs = TCs0 -- Flaky, [ @@ -158,9 +158,9 @@ dynamo_config(BridgeType, Config) -> "bridges.~s.~s {\n" " enable = true\n" " url = ~p\n" - " database = ~p\n" - " username = ~p\n" - " password = ~p\n" + " table = ~p\n" + " aws_access_key_id = ~p\n" + " aws_secret_access_key = ~p\n" " resource_opts = {\n" " request_timeout = 500ms\n" " batch_size = ~b\n" @@ -172,8 +172,8 @@ dynamo_config(BridgeType, Config) -> Name, Url, ?TABLE, - ?USERNAME, - ?PASSWORD, + ?ACCESS_KEY_ID, + ?SECRET_ACCESS_KEY, BatchSize, QueryMode ] @@ -193,7 +193,7 @@ create_bridge(Config, Overrides) -> BridgeType = ?config(dynamo_bridge_type, Config), Name = ?config(dynamo_name, Config), DynamoConfig0 = ?config(dynamo_config, Config), - DynamoConfig = emqx_map_lib:deep_merge(DynamoConfig0, Overrides), + DynamoConfig = emqx_utils_maps:deep_merge(DynamoConfig0, Overrides), emqx_bridge:create(BridgeType, Name, DynamoConfig). delete_all_bridges() -> @@ -208,7 +208,7 @@ create_bridge_http(Params) -> Path = emqx_mgmt_api_test_util:api_path(["bridges"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. @@ -244,10 +244,10 @@ delete_table(_Config) -> setup_dynamo(Config) -> Host = ?GET_CONFIG(host, Config), Port = ?GET_CONFIG(port, Config), - erlcloud_ddb2:configure(?USERNAME, ?PASSWORD, Host, Port, ?SCHEMA). + erlcloud_ddb2:configure(?ACCESS_KEY_ID, ?SECRET_ACCESS_KEY, Host, Port, ?SCHEMA). directly_setup_dynamo() -> - erlcloud_ddb2:configure(?USERNAME, ?PASSWORD, ?HOST, ?PORT, ?SCHEMA). + erlcloud_ddb2:configure(?ACCESS_KEY_ID, ?SECRET_ACCESS_KEY, ?HOST, ?PORT, ?SCHEMA). directly_query(Query) -> directly_setup_dynamo(), @@ -272,7 +272,7 @@ t_setup_via_config_and_publish(Config) -> {ok, _}, create_bridge(Config) ), - MsgId = emqx_misc:gen_id(), + MsgId = emqx_utils:gen_id(), SentData = #{id => MsgId, payload => ?PAYLOAD}, ?check_trace( begin @@ -309,7 +309,7 @@ t_setup_via_http_api_and_publish(Config) -> {ok, _}, create_bridge_http(PgsqlConfig) ), - MsgId = emqx_misc:gen_id(), + MsgId = emqx_utils:gen_id(), SentData = #{id => MsgId, payload => ?PAYLOAD}, ?check_trace( begin @@ -375,7 +375,7 @@ t_write_failure(Config) -> #{?snk_kind := resource_connected_enter}, 20_000 ), - SentData = #{id => emqx_misc:gen_id(), payload => ?PAYLOAD}, + SentData = #{id => emqx_utils:gen_id(), payload => ?PAYLOAD}, emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() -> ?assertMatch( {error, {resource_error, #{reason := timeout}}}, send_message(Config, SentData) @@ -383,25 +383,6 @@ t_write_failure(Config) -> end), ok. -t_write_timeout(Config) -> - ProxyName = ?config(proxy_name, Config), - ProxyPort = ?config(proxy_port, Config), - ProxyHost = ?config(proxy_host, Config), - {{ok, _}, {ok, _}} = - ?wait_async_action( - create_bridge(Config), - #{?snk_kind := resource_connected_enter}, - 20_000 - ), - SentData = #{id => emqx_misc:gen_id(), payload => ?PAYLOAD}, - emqx_common_test_helpers:with_failure(timeout, ProxyName, ProxyHost, ProxyPort, fun() -> - ?assertMatch( - {error, {resource_error, #{reason := timeout}}}, - query_resource(Config, {send_message, SentData}) - ) - end), - ok. - t_simple_query(Config) -> ?assertMatch( {ok, _}, diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl index 1b4b4aeb2..3def35920 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl @@ -354,7 +354,7 @@ create_bridge(Config, Overrides) -> Type = influxdb_type_bin(?config(influxdb_type, Config)), Name = ?config(influxdb_name, Config), InfluxDBConfig0 = ?config(influxdb_config, Config), - InfluxDBConfig = emqx_map_lib:deep_merge(InfluxDBConfig0, Overrides), + InfluxDBConfig = emqx_utils_maps:deep_merge(InfluxDBConfig0, Overrides), emqx_bridge:create(Type, Name, InfluxDBConfig). delete_bridge(Config) -> @@ -390,11 +390,11 @@ create_rule_and_action_http(Config, Overrides) -> sql => <<"SELECT * FROM \"t/topic\"">>, actions => [BridgeId] }, - Params = emqx_map_lib:deep_merge(Params0, Overrides), + Params = emqx_utils_maps:deep_merge(Params0, Overrides), Path = emqx_mgmt_api_test_util:api_path(["rules"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. @@ -435,7 +435,7 @@ query_by_clientid(ClientId, Config) -> {"Content-Type", "application/json"} ], Body = - emqx_json:encode(#{ + emqx_utils_json:encode(#{ query => Query, dialect => #{ header => true, @@ -545,7 +545,7 @@ t_start_ok(Config) -> int_value => <<"-123">>, uint_value => <<"123">>, float_value => <<"24.5">>, - payload => emqx_json:encode(Payload) + payload => emqx_utils_json:encode(Payload) }, assert_persisted_data(ClientId, Expected, PersistedData), ok @@ -764,7 +764,7 @@ t_boolean_variants(Config) -> bool => atom_to_binary(Translation), int_value => <<"-123">>, uint_value => <<"123">>, - payload => emqx_json:encode(Payload) + payload => emqx_utils_json:encode(Payload) }, assert_persisted_data(ClientId, Expected, PersistedData), ok @@ -1024,9 +1024,9 @@ t_missing_field(Config) -> ClientId0 = emqx_guid:to_hexstr(emqx_guid:gen()), ClientId1 = emqx_guid:to_hexstr(emqx_guid:gen()), %% Message with the field that we "forgot" to select in the rule - Msg0 = emqx_message:make(ClientId0, <<"t/topic">>, emqx_json:encode(#{foo => 123})), + Msg0 = emqx_message:make(ClientId0, <<"t/topic">>, emqx_utils_json:encode(#{foo => 123})), %% Message without any fields - Msg1 = emqx_message:make(ClientId1, <<"t/topic">>, emqx_json:encode(#{})), + Msg1 = emqx_message:make(ClientId1, <<"t/topic">>, emqx_utils_json:encode(#{})), ?check_trace( begin emqx:publish(Msg0), diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_tests.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_tests.erl index ce3a0b06f..1e065f6c8 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_tests.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_tests.erl @@ -5,8 +5,6 @@ -include_lib("eunit/include/eunit.hrl"). --import(emqx_ee_bridge_influxdb, [to_influx_lines/1]). - -define(INVALID_LINES, [ " ", " \n", @@ -326,3 +324,13 @@ test_pairs(PairsList) -> join(Sep, LinesList) -> lists:flatten(lists:join(Sep, LinesList)). + +to_influx_lines(RawLines) -> + OldLevel = emqx_logger:get_primary_log_level(), + try + %% mute error logs from this call + emqx_logger:set_primary_log_level(none), + emqx_ee_bridge_influxdb:to_influx_lines(RawLines) + after + emqx_logger:set_primary_log_level(OldLevel) + end. diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl index 116dcc729..105f1fe75 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl @@ -27,7 +27,8 @@ group_tests() -> t_setup_via_config_and_publish, t_setup_via_http_api_and_publish, t_payload_template, - t_collection_template + t_collection_template, + t_mongo_date_rule_engine_functions ]. groups() -> @@ -140,13 +141,24 @@ start_apps() -> %% we want to make sure they are loaded before %% ekka start in emqx_common_test_helpers:start_apps/1 emqx_common_test_helpers:render_and_load_app_config(emqx_conf), - ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]). + ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_rule_engine, emqx_bridge]). ensure_loaded() -> _ = application:load(emqx_ee_bridge), + _ = application:load(emqtt), _ = emqx_ee_bridge:module_info(), ok. +mongo_type(Config) -> + case ?config(mongo_type, Config) of + rs -> + {rs, maps:get(<<"replica_set_name">>, ?config(mongo_config, Config))}; + sharded -> + sharded; + single -> + single + end. + mongo_type_bin(rs) -> <<"mongodb_rs">>; mongo_type_bin(sharded) -> @@ -245,7 +257,7 @@ create_bridge(Config, Overrides) -> Type = mongo_type_bin(?config(mongo_type, Config)), Name = ?config(mongo_name, Config), MongoConfig0 = ?config(mongo_config, Config), - MongoConfig = emqx_map_lib:deep_merge(MongoConfig0, Overrides), + MongoConfig = emqx_utils_maps:deep_merge(MongoConfig0, Overrides), ct:pal("creating ~p bridge with config:\n ~p", [Type, MongoConfig]), emqx_bridge:create(Type, Name, MongoConfig). @@ -258,22 +270,19 @@ create_bridge_http(Params) -> Path = emqx_mgmt_api_test_util:api_path(["bridges"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. clear_db(Config) -> - Type = mongo_type_bin(?config(mongo_type, Config)), - Name = ?config(mongo_name, Config), - #{<<"collection">> := Collection} = ?config(mongo_config, Config), - ResourceID = emqx_bridge_resource:resource_id(Type, Name), - {ok, _, #{state := #{connector_state := #{poolname := PoolName}}}} = - emqx_resource:get_instance(ResourceID), - Selector = #{}, - {true, _} = ecpool:pick_and_do( - PoolName, {mongo_api, delete, [Collection, Selector]}, no_handover - ), - ok. + Type = mongo_type(Config), + Host = ?config(mongo_host, Config), + Port = ?config(mongo_port, Config), + Server = Host ++ ":" ++ integer_to_list(Port), + #{<<"database">> := Db, <<"collection">> := Collection} = ?config(mongo_config, Config), + {ok, Client} = mongo_api:connect(Type, [Server], [], [{database, Db}, {w_mode, unsafe}]), + {true, _} = mongo_api:delete(Client, Collection, _Selector = #{}), + mongo_api:disconnect(Client). find_all(Config) -> Type = mongo_type_bin(?config(mongo_type, Config)), @@ -282,6 +291,27 @@ find_all(Config) -> ResourceID = emqx_bridge_resource:resource_id(Type, Name), emqx_resource:simple_sync_query(ResourceID, {find, Collection, #{}, #{}}). +find_all_wait_until_non_empty(Config) -> + wait_until( + fun() -> + case find_all(Config) of + {ok, []} -> false; + _ -> true + end + end, + 5_000 + ), + find_all(Config). + +wait_until(Fun, Timeout) when Timeout >= 0 -> + case Fun() of + true -> + ok; + false -> + timer:sleep(100), + wait_until(Fun, Timeout - 100) + end. + send_message(Config, Payload) -> Name = ?config(mongo_name, Config), Type = mongo_type_bin(?config(mongo_type, Config)), @@ -376,3 +406,51 @@ t_collection_template(Config) -> find_all(Config) ), ok. + +t_mongo_date_rule_engine_functions(Config) -> + {ok, _} = + create_bridge( + Config, + #{ + <<"payload_template">> => + <<"{\"date_0\": ${date_0}, \"date_1\": ${date_1}, \"date_2\": ${date_2}}">> + } + ), + Type = mongo_type_bin(?config(mongo_type, Config)), + Name = ?config(mongo_name, Config), + SQL = + "SELECT mongo_date() as date_0, mongo_date(1000) as date_1, mongo_date(1, 'second') as date_2 FROM " + "\"t_mongo_date_rule_engine_functions/topic\"", + %% Remove rule if it already exists + RuleId = <<"rule:t_mongo_date_rule_engine_functions">>, + emqx_rule_engine:delete_rule(RuleId), + BridgeId = emqx_bridge_resource:bridge_id(Type, Name), + {ok, Rule} = emqx_rule_engine:create_rule( + #{ + id => <<"rule:t_mongo_date_rule_engine_functions">>, + sql => SQL, + actions => [ + BridgeId, + #{function => console} + ], + description => <<"to mongo bridge">> + } + ), + %% Send a message to topic + {ok, Client} = emqtt:start_link([{clientid, <<"pub-02">>}, {proto_ver, v5}]), + {ok, _} = emqtt:connect(Client), + emqtt:publish(Client, <<"t_mongo_date_rule_engine_functions/topic">>, #{}, <<"{\"x\":1}">>, [ + {qos, 2} + ]), + emqtt:stop(Client), + ?assertMatch( + {ok, [ + #{ + <<"date_0">> := {_, _, _}, + <<"date_1">> := {0, 1, 0}, + <<"date_2">> := {0, 1, 0} + } + ]}, + find_all_wait_until_non_empty(Config) + ), + ok. diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl index 38e31c7ae..e497e0a47 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl @@ -226,7 +226,7 @@ create_bridge_http(Params) -> Path = emqx_mgmt_api_test_util:api_path(["bridges"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. @@ -265,7 +265,7 @@ unprepare(Config, Key) -> Name = ?config(mysql_name, Config), BridgeType = ?config(mysql_bridge_type, Config), ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), - {ok, _, #{state := #{poolname := PoolName}}} = emqx_resource:get_instance(ResourceID), + {ok, _, #{state := #{pool_name := PoolName}}} = emqx_resource:get_instance(ResourceID), [ begin {ok, Conn} = ecpool_worker:client(Worker), @@ -615,7 +615,7 @@ t_workload_fits_prepared_statement_limit(Config) -> create_bridge(Config) ), Results = lists:append( - emqx_misc:pmap( + emqx_utils:pmap( fun(_) -> [ begin diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_pgsql_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_pgsql_SUITE.erl index 83cb8b1f3..d76149b16 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_pgsql_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_pgsql_SUITE.erl @@ -229,7 +229,7 @@ create_bridge(Config, Overrides) -> BridgeType = ?config(pgsql_bridge_type, Config), Name = ?config(pgsql_name, Config), PGConfig0 = ?config(pgsql_config, Config), - PGConfig = emqx_map_lib:deep_merge(PGConfig0, Overrides), + PGConfig = emqx_utils_maps:deep_merge(PGConfig0, Overrides), emqx_bridge:create(BridgeType, Name, PGConfig). delete_bridge(Config) -> @@ -241,7 +241,7 @@ create_bridge_http(Params) -> Path = emqx_mgmt_api_test_util:api_path(["bridges"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_rocketmq_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_rocketmq_SUITE.erl index 95ec47e7f..0cb14e5c3 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_rocketmq_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_rocketmq_SUITE.erl @@ -176,7 +176,7 @@ create_bridge_http(Params) -> Path = emqx_mgmt_api_test_util:api_path(["bridges"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl new file mode 100644 index 000000000..68bf7a057 --- /dev/null +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl @@ -0,0 +1,682 @@ +%%-------------------------------------------------------------------- +% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_ee_bridge_sqlserver_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +% SQL definitions +-define(SQL_BRIDGE, + "insert into t_mqtt_msg(msgid, topic, qos, payload) values ( ${id}, ${topic}, ${qos}, ${payload})" +). +-define(SQL_SERVER_DRIVER, "ms-sql"). + +-define(SQL_CREATE_DATABASE_IF_NOT_EXISTS, + " IF NOT EXISTS(SELECT name FROM sys.databases WHERE name = 'mqtt')" + " BEGIN" + " CREATE DATABASE mqtt;" + " END" +). + +-define(SQL_CREATE_TABLE_IN_DB_MQTT, + " CREATE TABLE mqtt.dbo.t_mqtt_msg" + " (id int PRIMARY KEY IDENTITY(1000000001,1) NOT NULL," + " msgid VARCHAR(64) NULL," + " topic VARCHAR(100) NULL," + " qos tinyint NOT NULL DEFAULT 0," + %% use VARCHAR to use utf8 encoding + %% for default, sqlserver use utf16 encoding NVARCHAR() + " payload VARCHAR(100) NULL," + " arrived DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP)" +). + +-define(SQL_DROP_DB_MQTT, "DROP DATABASE mqtt"). +-define(SQL_DROP_TABLE, "DROP TABLE mqtt.dbo.t_mqtt_msg"). +-define(SQL_DELETE, "DELETE from mqtt.dbo.t_mqtt_msg"). +-define(SQL_SELECT, "SELECT payload FROM mqtt.dbo.t_mqtt_msg"). +-define(SQL_SELECT_COUNT, "SELECT COUNT(*) FROM mqtt.dbo.t_mqtt_msg"). +% DB defaults +-define(SQL_SERVER_DATABASE, "mqtt"). +-define(SQL_SERVER_USERNAME, "sa"). +-define(SQL_SERVER_PASSWORD, "mqtt_public1"). +-define(BATCH_SIZE, 10). +-define(REQUEST_TIMEOUT_MS, 500). + +-define(WORKER_POOL_SIZE, 4). + +-define(WITH_CON(Process), + Con = connect_direct_sqlserver(Config), + Process, + ok = disconnect(Con) +). + +%% How to run it locally (all commands are run in $PROJ_ROOT dir): +%% A: run ct on host +%% 1. Start all deps services +%% sudo docker compose -f .ci/docker-compose-file/docker-compose.yaml \ +%% -f .ci/docker-compose-file/docker-compose-sqlserver.yaml \ +%% -f .ci/docker-compose-file/docker-compose-toxiproxy.yaml \ +%% up --build +%% +%% 2. Run use cases with special environment variables +%% 11433 is toxiproxy exported port. +%% Local: +%% ``` +%% SQLSERVER_HOST=toxiproxy SQLSERVER_PORT=11433 \ +%% PROXY_HOST=toxiproxy PROXY_PORT=1433 \ +%% ./rebar3 as test ct -c -v --readable true --name ct@127.0.0.1 --suite lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl +%% ``` +%% +%% B: run ct in docker container +%% run script: +%% ./scripts/ct/run.sh --ci --app lib-ee/emqx_ee_bridge/ \ +%% -- --name 'test@127.0.0.1' -c -v --readable true --suite lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl + +%%------------------------------------------------------------------------------ +%% CT boilerplate +%%------------------------------------------------------------------------------ + +all() -> + [ + {group, async}, + {group, sync} + ]. + +groups() -> + TCs = emqx_common_test_helpers:all(?MODULE), + NonBatchCases = [t_write_timeout], + BatchingGroups = [{group, with_batch}, {group, without_batch}], + [ + {async, BatchingGroups}, + {sync, BatchingGroups}, + {with_batch, TCs -- NonBatchCases}, + {without_batch, TCs} + ]. + +init_per_group(async, Config) -> + [{query_mode, async} | Config]; +init_per_group(sync, Config) -> + [{query_mode, sync} | Config]; +init_per_group(with_batch, Config0) -> + Config = [{enable_batch, true} | Config0], + common_init(Config); +init_per_group(without_batch, Config0) -> + Config = [{enable_batch, false} | Config0], + common_init(Config); +init_per_group(_Group, Config) -> + Config. + +end_per_group(Group, Config) when Group =:= with_batch; Group =:= without_batch -> + connect_and_drop_table(Config), + connect_and_drop_db(Config), + ProxyHost = ?config(proxy_host, Config), + ProxyPort = ?config(proxy_port, Config), + emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), + ok; +end_per_group(_Group, _Config) -> + ok. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + emqx_mgmt_api_test_util:end_suite(), + ok = emqx_common_test_helpers:stop_apps([emqx_bridge, emqx_conf]), + ok. + +init_per_testcase(_Testcase, Config) -> + %% drop database and table + %% connect_and_clear_table(Config), + %% create a new one + %% TODO: create a new database for each test case + delete_bridge(Config), + snabbkaffe:start_trace(), + Config. + +end_per_testcase(_Testcase, Config) -> + ProxyHost = ?config(proxy_host, Config), + ProxyPort = ?config(proxy_port, Config), + emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), + connect_and_clear_table(Config), + ok = snabbkaffe:stop(), + delete_bridge(Config), + ok. + +%%------------------------------------------------------------------------------ +%% Testcases +%%------------------------------------------------------------------------------ + +t_setup_via_config_and_publish(Config) -> + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + Val = str(erlang:unique_integer()), + SentData = sent_data(Val), + ?check_trace( + begin + ?wait_async_action( + ?assertEqual(ok, send_message(Config, SentData)), + #{?snk_kind := sqlserver_connector_query_return}, + 10_000 + ), + ?assertMatch( + [{Val}], + connect_and_get_payload(Config) + ), + ok + end, + fun(Trace0) -> + Trace = ?of_kind(sqlserver_connector_query_return, Trace0), + ?assertMatch([#{result := ok}], Trace), + ok + end + ), + ok. + +t_setup_via_http_api_and_publish(Config) -> + BridgeType = ?config(sqlserver_bridge_type, Config), + Name = ?config(sqlserver_name, Config), + SQLServerConfig0 = ?config(sqlserver_config, Config), + SQLServerConfig = SQLServerConfig0#{ + <<"name">> => Name, + <<"type">> => BridgeType + }, + ?assertMatch( + {ok, _}, + create_bridge_http(SQLServerConfig) + ), + Val = str(erlang:unique_integer()), + SentData = sent_data(Val), + ?check_trace( + begin + ?wait_async_action( + ?assertEqual(ok, send_message(Config, SentData)), + #{?snk_kind := sqlserver_connector_query_return}, + 10_000 + ), + ?assertMatch( + [{Val}], + connect_and_get_payload(Config) + ), + ok + end, + fun(Trace0) -> + Trace = ?of_kind(sqlserver_connector_query_return, Trace0), + ?assertMatch([#{result := ok}], Trace), + ok + end + ), + ok. + +t_get_status(Config) -> + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + ProxyPort = ?config(proxy_port, Config), + ProxyHost = ?config(proxy_host, Config), + ProxyName = ?config(proxy_name, Config), + + health_check_resource_ok(Config), + emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() -> + health_check_resource_down(Config) + end), + ok. + +t_create_disconnected(Config) -> + ProxyPort = ?config(proxy_port, Config), + ProxyHost = ?config(proxy_host, Config), + ProxyName = ?config(proxy_name, Config), + emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() -> + ?assertMatch({ok, _}, create_bridge(Config)), + health_check_resource_down(Config) + end), + health_check_resource_ok(Config), + ok. + +t_create_with_invalid_password(Config) -> + BridgeType = ?config(sqlserver_bridge_type, Config), + Name = ?config(sqlserver_name, Config), + SQLServerConfig0 = ?config(sqlserver_config, Config), + SQLServerConfig = SQLServerConfig0#{ + <<"name">> => Name, + <<"type">> => BridgeType, + <<"password">> => <<"wrong_password">> + }, + ?check_trace( + begin + ?assertMatch( + {ok, _}, + create_bridge_http(SQLServerConfig) + ) + end, + fun(Trace) -> + ?assertMatch( + [#{error := {start_pool_failed, _, _}}], + ?of_kind(sqlserver_connector_start_failed, Trace) + ), + ok + end + ), + ok. + +t_write_failure(Config) -> + ProxyName = ?config(proxy_name, Config), + ProxyPort = ?config(proxy_port, Config), + ProxyHost = ?config(proxy_host, Config), + QueryMode = ?config(query_mode, Config), + Val = str(erlang:unique_integer()), + SentData = sent_data(Val), + {{ok, _}, {ok, _}} = + ?wait_async_action( + create_bridge(Config), + #{?snk_kind := resource_connected_enter}, + 20_000 + ), + emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() -> + case QueryMode of + sync -> + ?assertMatch( + {error, {resource_error, #{reason := timeout}}}, + send_message(Config, SentData) + ); + async -> + ?assertMatch( + ok, send_message(Config, SentData) + ) + end + end), + ok. + +t_write_timeout(_Config) -> + %% msodbc driver handled all connection exceptions + %% the case is same as t_write_failure/1 + ok. + +t_simple_query(Config) -> + BatchSize = batch_size(Config), + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + {Requests, Vals} = gen_batch_req(BatchSize), + ?check_trace( + begin + ?wait_async_action( + begin + [?assertEqual(ok, query_resource(Config, Request)) || Request <- Requests] + end, + #{?snk_kind := sqlserver_connector_query_return}, + 10_000 + ), + %% just assert the data count is correct + ?assertMatch( + BatchSize, + connect_and_get_count(Config) + ), + %% assert the data order is correct + ?assertMatch( + Vals, + connect_and_get_payload(Config) + ) + end, + fun(Trace0) -> + Trace = ?of_kind(sqlserver_connector_query_return, Trace0), + case BatchSize of + 1 -> + ?assertMatch([#{result := ok}], Trace); + _ -> + [?assertMatch(#{result := ok}, Trace1) || Trace1 <- Trace] + end, + ok + end + ), + ok. + +-define(MISSING_TINYINT_ERROR, + "[Microsoft][ODBC Driver 17 for SQL Server][SQL Server]" + "Conversion failed when converting the varchar value 'undefined' to data type tinyint. SQLSTATE IS: 22018" +). + +t_missing_data(Config) -> + QueryMode = ?config(query_mode, Config), + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + Result = send_message(Config, #{}), + case QueryMode of + sync -> + ?assertMatch( + {error, {unrecoverable_error, {invalid_request, ?MISSING_TINYINT_ERROR}}}, + Result + ); + async -> + ?assertMatch( + ok, send_message(Config, #{}) + ) + end, + ok. + +t_bad_parameter(Config) -> + QueryMode = ?config(query_mode, Config), + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + Result = send_message(Config, #{}), + case QueryMode of + sync -> + ?assertMatch( + {error, {unrecoverable_error, {invalid_request, ?MISSING_TINYINT_ERROR}}}, + Result + ); + async -> + ?assertMatch( + ok, send_message(Config, #{}) + ) + end, + ok. + +%%------------------------------------------------------------------------------ +%% Helper fns +%%------------------------------------------------------------------------------ + +common_init(ConfigT) -> + Host = os:getenv("SQLSERVER_HOST", "toxiproxy"), + Port = list_to_integer(os:getenv("SQLSERVER_PORT", "1433")), + + Config0 = [ + {sqlserver_host, Host}, + {sqlserver_port, Port}, + %% see also for `proxy_name` : $PROJ_ROOT/.ci/docker-compose-file/toxiproxy.json + {proxy_name, "sqlserver"}, + {batch_size, batch_size(ConfigT)} + | ConfigT + ], + + BridgeType = proplists:get_value(bridge_type, Config0, <<"sqlserver">>), + case emqx_common_test_helpers:is_tcp_server_available(Host, Port) of + true -> + % Setup toxiproxy + ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"), + ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")), + emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), + % Ensure EE bridge module is loaded + _ = application:load(emqx_ee_bridge), + _ = emqx_ee_bridge:module_info(), + ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]), + emqx_mgmt_api_test_util:init_suite(), + % Connect to sqlserver directly + % drop old db and table, and then create new ones + connect_and_create_db_and_table(Config0), + {Name, SQLServerConf} = sqlserver_config(BridgeType, Config0), + Config = + [ + {sqlserver_config, SQLServerConf}, + {sqlserver_bridge_type, BridgeType}, + {sqlserver_name, Name}, + {proxy_host, ProxyHost}, + {proxy_port, ProxyPort} + | Config0 + ], + Config; + false -> + case os:getenv("IS_CI") of + "yes" -> + throw(no_sqlserver); + _ -> + {skip, no_sqlserver} + end + end. + +sqlserver_config(BridgeType, Config) -> + Port = integer_to_list(?config(sqlserver_port, Config)), + Server = ?config(sqlserver_host, Config) ++ ":" ++ Port, + Name = atom_to_binary(?MODULE), + BatchSize = batch_size(Config), + QueryMode = ?config(query_mode, Config), + ConfigString = + io_lib:format( + "bridges.~s.~s {\n" + " enable = true\n" + " server = ~p\n" + " database = ~p\n" + " username = ~p\n" + " password = ~p\n" + " sql = ~p\n" + " driver = ~p\n" + " resource_opts = {\n" + " request_timeout = 500ms\n" + " batch_size = ~b\n" + " query_mode = ~s\n" + " worker_pool_size = ~b\n" + " }\n" + "}", + [ + BridgeType, + Name, + Server, + ?SQL_SERVER_DATABASE, + ?SQL_SERVER_USERNAME, + ?SQL_SERVER_PASSWORD, + ?SQL_BRIDGE, + ?SQL_SERVER_DRIVER, + BatchSize, + QueryMode, + ?WORKER_POOL_SIZE + ] + ), + {Name, parse_and_check(ConfigString, BridgeType, Name)}. + +parse_and_check(ConfigString, BridgeType, Name) -> + {ok, RawConf} = hocon:binary(ConfigString, #{format => map}), + hocon_tconf:check_plain(emqx_bridge_schema, RawConf, #{required => false, atom_key => false}), + #{<<"bridges">> := #{BridgeType := #{Name := Config}}} = RawConf, + Config. + +create_bridge(Config) -> + create_bridge(Config, _Overrides = #{}). + +create_bridge(Config, Overrides) -> + BridgeType = ?config(sqlserver_bridge_type, Config), + Name = ?config(sqlserver_name, Config), + SSConfig0 = ?config(sqlserver_config, Config), + SSConfig = emqx_utils_maps:deep_merge(SSConfig0, Overrides), + emqx_bridge:create(BridgeType, Name, SSConfig). + +delete_bridge(Config) -> + BridgeType = ?config(sqlserver_bridge_type, Config), + Name = ?config(sqlserver_name, Config), + emqx_bridge:remove(BridgeType, Name). + +create_bridge_http(Params) -> + Path = emqx_mgmt_api_test_util:api_path(["bridges"]), + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; + Error -> Error + end. + +send_message(Config, Payload) -> + Name = ?config(sqlserver_name, Config), + BridgeType = ?config(sqlserver_bridge_type, Config), + BridgeID = emqx_bridge_resource:bridge_id(BridgeType, Name), + emqx_bridge:send_message(BridgeID, Payload). + +query_resource(Config, Request) -> + Name = ?config(sqlserver_name, Config), + BridgeType = ?config(sqlserver_bridge_type, Config), + ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), + emqx_resource:query(ResourceID, Request, #{timeout => 1_000}). + +query_resource_async(Config, Request) -> + Name = ?config(sqlserver_name, Config), + BridgeType = ?config(sqlserver_bridge_type, Config), + Ref = alias([reply]), + AsyncReplyFun = fun(Result) -> Ref ! {result, Ref, Result} end, + ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), + Return = emqx_resource:query(ResourceID, Request, #{ + timeout => 500, async_reply_fun => {AsyncReplyFun, []} + }), + {Return, Ref}. + +resource_id(Config) -> + Name = ?config(sqlserver_name, Config), + BridgeType = ?config(sqlserver_bridge_type, Config), + _ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name). + +health_check_resource_ok(Config) -> + ?assertEqual({ok, connected}, emqx_resource_manager:health_check(resource_id(Config))). + +health_check_resource_down(Config) -> + case emqx_resource_manager:health_check(resource_id(Config)) of + {ok, Status} when Status =:= disconnected orelse Status =:= connecting -> + ok; + {error, timeout} -> + ok; + Other -> + ?assert( + false, lists:flatten(io_lib:format("invalid health check result:~p~n", [Other])) + ) + end. + +receive_result(Ref, Timeout) -> + receive + {result, Ref, Result} -> + {ok, Result}; + {Ref, Result} -> + {ok, Result} + after Timeout -> + timeout + end. + +connect_direct_sqlserver(Config) -> + Opts = [ + {host, ?config(sqlserver_host, Config)}, + {port, ?config(sqlserver_port, Config)}, + {username, ?SQL_SERVER_USERNAME}, + {password, ?SQL_SERVER_PASSWORD}, + {driver, ?SQL_SERVER_DRIVER}, + {pool_size, 8} + ], + {ok, Con} = connect(Opts), + Con. + +connect(Options) -> + ConnectStr = lists:concat(conn_str(Options, [])), + Opts = proplists:get_value(options, Options, []), + odbc:connect(ConnectStr, Opts). + +disconnect(Ref) -> + odbc:disconnect(Ref). + +% These funs connect and then stop the sqlserver connection +connect_and_create_db_and_table(Config) -> + ?WITH_CON(begin + {updated, undefined} = directly_query(Con, ?SQL_CREATE_DATABASE_IF_NOT_EXISTS), + {updated, undefined} = directly_query(Con, ?SQL_CREATE_TABLE_IN_DB_MQTT) + end). + +connect_and_drop_db(Config) -> + ?WITH_CON({updated, undefined} = directly_query(Con, ?SQL_DROP_DB_MQTT)). + +connect_and_drop_table(Config) -> + ?WITH_CON({updated, undefined} = directly_query(Con, ?SQL_DROP_TABLE)). + +connect_and_clear_table(Config) -> + ?WITH_CON({updated, _} = directly_query(Con, ?SQL_DELETE)). + +connect_and_get_payload(Config) -> + ?WITH_CON( + {selected, ["payload"], Rows} = directly_query(Con, ?SQL_SELECT) + ), + Rows. + +connect_and_get_count(Config) -> + ?WITH_CON( + {selected, [[]], [{Count}]} = directly_query(Con, ?SQL_SELECT_COUNT) + ), + Count. + +directly_query(Con, Query) -> + directly_query(Con, Query, ?REQUEST_TIMEOUT_MS). + +directly_query(Con, Query, Timeout) -> + odbc:sql_query(Con, Query, Timeout). + +%%-------------------------------------------------------------------- +%% help functions +%%-------------------------------------------------------------------- + +batch_size(Config) -> + case ?config(enable_batch, Config) of + true -> ?BATCH_SIZE; + false -> 1 + end. + +conn_str([], Acc) -> + %% TODO: for msodbc 18+, we need to add "Encrypt=YES;TrustServerCertificate=YES" + %% but havn't tested now + %% we should use this for msodbcsql 18+ + %% lists:join(";", ["Encrypt=YES", "TrustServerCertificate=YES" | Acc]); + lists:join(";", Acc); +conn_str([{driver, Driver} | Opts], Acc) -> + conn_str(Opts, ["Driver=" ++ str(Driver) | Acc]); +conn_str([{host, Host} | Opts], Acc) -> + Port = proplists:get_value(port, Opts, "1433"), + NOpts = proplists:delete(port, Opts), + conn_str(NOpts, ["Server=" ++ str(Host) ++ "," ++ str(Port) | Acc]); +conn_str([{port, Port} | Opts], Acc) -> + Host = proplists:get_value(host, Opts, "localhost"), + NOpts = proplists:delete(host, Opts), + conn_str(NOpts, ["Server=" ++ str(Host) ++ "," ++ str(Port) | Acc]); +conn_str([{database, Database} | Opts], Acc) -> + conn_str(Opts, ["Database=" ++ str(Database) | Acc]); +conn_str([{username, Username} | Opts], Acc) -> + conn_str(Opts, ["UID=" ++ str(Username) | Acc]); +conn_str([{password, Password} | Opts], Acc) -> + conn_str(Opts, ["PWD=" ++ str(Password) | Acc]); +conn_str([{_, _} | Opts], Acc) -> + conn_str(Opts, Acc). + +sent_data(Payload) -> + #{ + payload => to_bin(Payload), + id => <<"0005F8F84FFFAFB9F44200000D810002">>, + topic => <<"test/topic">>, + qos => 0 + }. + +gen_batch_req(Count) when + is_integer(Count) andalso Count > 0 +-> + Vals = [{str(erlang:unique_integer())} || _Seq <- lists:seq(1, Count)], + Requests = [{send_message, sent_data(Payload)} || {Payload} <- Vals], + {Requests, Vals}; +gen_batch_req(Count) -> + ct:pal("Gen batch requests failed with unexpected Count: ~p", [Count]). + +str(List) when is_list(List) -> + unicode:characters_to_list(List, utf8); +str(Bin) when is_binary(Bin) -> + unicode:characters_to_list(Bin, utf8); +str(Num) when is_number(Num) -> + number_to_list(Num). + +number_to_list(Int) when is_integer(Int) -> + integer_to_list(Int); +number_to_list(Float) when is_float(Float) -> + float_to_list(Float, [{decimals, 10}, compact]). + +to_bin(List) when is_list(List) -> + unicode:characters_to_binary(List, utf8); +to_bin(Bin) when is_binary(Bin) -> + Bin. diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_tdengine_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_tdengine_SUITE.erl index c956a93c6..36ed10f38 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_tdengine_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_tdengine_SUITE.erl @@ -207,7 +207,7 @@ create_bridge(Config, Overrides) -> BridgeType = ?config(tdengine_bridge_type, Config), Name = ?config(tdengine_name, Config), TDConfig0 = ?config(tdengine_config, Config), - TDConfig = emqx_map_lib:deep_merge(TDConfig0, Overrides), + TDConfig = emqx_utils_maps:deep_merge(TDConfig0, Overrides), emqx_bridge:create(BridgeType, Name, TDConfig). delete_bridge(Config) -> @@ -219,7 +219,7 @@ create_bridge_http(Params) -> Path = emqx_mgmt_api_test_util:api_path(["bridges"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. diff --git a/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl b/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl index df3382df1..2ae627f35 100644 --- a/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl +++ b/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl @@ -10,6 +10,7 @@ -define(EE_SCHEMA_MODULES, [ emqx_license_schema, + emqx_ee_schema_registry_schema, emqx_ft_schema ]). diff --git a/lib-ee/emqx_ee_connector/docker-ct b/lib-ee/emqx_ee_connector/docker-ct index 446c8ee4e..cb2f6f028 100644 --- a/lib-ee/emqx_ee_connector/docker-ct +++ b/lib-ee/emqx_ee_connector/docker-ct @@ -1,4 +1,4 @@ toxiproxy influxdb clickhouse -cassandra +sqlserver diff --git a/lib-ee/emqx_ee_connector/include/emqx_ee_connector.hrl b/lib-ee/emqx_ee_connector/include/emqx_ee_connector.hrl index 2a91d2524..4b6fbbd92 100644 --- a/lib-ee/emqx_ee_connector/include/emqx_ee_connector.hrl +++ b/lib-ee/emqx_ee_connector/include/emqx_ee_connector.hrl @@ -3,4 +3,3 @@ %%------------------------------------------------------------------- -define(INFLUXDB_DEFAULT_PORT, 8086). --define(CASSANDRA_DEFAULT_PORT, 9042). diff --git a/lib-ee/emqx_ee_connector/rebar.config b/lib-ee/emqx_ee_connector/rebar.config index 654b1bf54..03b4e0aad 100644 --- a/lib-ee/emqx_ee_connector/rebar.config +++ b/lib-ee/emqx_ee_connector/rebar.config @@ -1,12 +1,13 @@ {erl_opts, [debug_info]}. {deps, [ - {hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.2.5"}}}, - {influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.9"}}}, - {tdengine, {git, "https://github.com/emqx/tdengine-client-erl", {tag, "0.1.6"}}}, - {clickhouse, {git, "https://github.com/emqx/clickhouse-client-erl", {tag, "0.3"}}}, - {erlcloud, {git, "https://github.com/emqx/erlcloud", {tag, "3.6.8-emqx-1"}}}, - {rocketmq, {git, "https://github.com/emqx/rocketmq-client-erl.git", {tag, "v0.5.1"}}}, - {emqx, {path, "../../apps/emqx"}} + {hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.2.5"}}}, + {influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.9"}}}, + {tdengine, {git, "https://github.com/emqx/tdengine-client-erl", {tag, "0.1.6"}}}, + {clickhouse, {git, "https://github.com/emqx/clickhouse-client-erl", {tag, "0.3"}}}, + {erlcloud, {git, "https://github.com/emqx/erlcloud", {tag, "3.6.8-emqx-1"}}}, + {rocketmq, {git, "https://github.com/emqx/rocketmq-client-erl.git", {tag, "v0.5.1"}}}, + {emqx, {path, "../../apps/emqx"}}, + {emqx_utils, {path, "../../apps/emqx_utils"}} ]}. {shell, [ diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src index 7a1da983f..82f556bdb 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src @@ -1,19 +1,18 @@ {application, emqx_ee_connector, [ {description, "EMQX Enterprise connectors"}, - {vsn, "0.1.8"}, + {vsn, "0.1.11"}, {registered, []}, {applications, [ kernel, stdlib, + ecpool, hstreamdb_erl, influxdb, tdengine, - wolff, - brod, clickhouse, erlcloud, rocketmq, - ecql + odbc ]}, {env, []}, {modules, []}, diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_clickhouse.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_clickhouse.erl index 8f2fdc042..a7afcd6d5 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_clickhouse.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_clickhouse.erl @@ -1,17 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_ee_connector_clickhouse). @@ -74,7 +62,8 @@ -type state() :: #{ templates := templates(), - poolname := atom() + pool_name := binary(), + connect_timeout := pos_integer() }. -type clickhouse_config() :: map(). @@ -151,9 +140,8 @@ on_start( ?SLOG(info, #{ msg => "starting_clickhouse_connector", connector => InstanceID, - config => emqx_misc:redact(Config) + config => emqx_utils:redact(Config) }), - PoolName = emqx_plugin_libs_pool:pool_name(InstanceID), Options = [ {url, URL}, {user, maps:get(username, Config, "default")}, @@ -161,46 +149,43 @@ on_start( {database, DB}, {auto_reconnect, ?AUTO_RECONNECT_INTERVAL}, {pool_size, PoolSize}, - {pool, PoolName} + {pool, InstanceID} ], - InitState = #{ - poolname => PoolName, - connect_timeout => ConnectTimeout - }, try Templates = prepare_sql_templates(Config), - State = maps:merge(InitState, #{templates => Templates}), - case emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Options) of + State = #{ + pool_name => InstanceID, + templates => Templates, + connect_timeout => ConnectTimeout + }, + case emqx_resource_pool:start(InstanceID, ?MODULE, Options) of ok -> {ok, State}; {error, Reason} -> - log_start_error(Config, Reason, none), + ?tp( + info, + "clickhouse_connector_start_failed", + #{ + error => Reason, + config => emqx_utils:redact(Config) + } + ), {error, Reason} end catch _:CatchReason:Stacktrace -> - log_start_error(Config, CatchReason, Stacktrace), + ?tp( + info, + "clickhouse_connector_start_failed", + #{ + error => CatchReason, + stacktrace => Stacktrace, + config => emqx_utils:redact(Config) + } + ), {error, CatchReason} end. -log_start_error(Config, Reason, Stacktrace) -> - StacktraceMap = - case Stacktrace of - none -> #{}; - _ -> #{stacktrace => Stacktrace} - end, - LogMessage = - #{ - msg => "clickhouse_connector_start_failed", - error_reason => Reason, - config => emqx_misc:redact(Config) - }, - ?SLOG(info, maps:merge(LogMessage, StacktraceMap)), - ?tp( - clickhouse_connector_start_failed, - #{error => Reason} - ). - %% Helper functions to prepare SQL tempaltes prepare_sql_templates(#{ @@ -252,7 +237,7 @@ split_clickhouse_insert_sql(SQL) -> end. % This is a callback for ecpool which is triggered by the call to -% emqx_plugin_libs_pool:start_pool in on_start/2 +% emqx_resource_pool:start in on_start/2 connect(Options) -> URL = iolist_to_binary(emqx_http_lib:normalize(proplists:get_value(url, Options))), @@ -270,8 +255,15 @@ connect(Options) -> {pool_size, PoolSize} ], case clickhouse:start_link(FixedOptions) of - {ok, _Conn} = Ok -> - Ok; + {ok, Connection} -> + %% Check if we can connect and send a query + case clickhouse:detailed_status(Connection) of + ok -> + {ok, Connection}; + Error -> + ok = clickhouse:stop(Connection), + Error + end; {error, Reason} -> {error, Reason} end. @@ -282,23 +274,20 @@ connect(Options) -> -spec on_stop(resource_id(), resource_state()) -> term(). -on_stop(ResourceID, #{poolname := PoolName}) -> +on_stop(InstanceID, #{pool_name := PoolName}) -> ?SLOG(info, #{ msg => "stopping clickouse connector", - connector => ResourceID + connector => InstanceID }), - emqx_plugin_libs_pool:stop_pool(PoolName). + emqx_resource_pool:stop(PoolName). %% ------------------------------------------------------------------- %% on_get_status emqx_resouce callback and related functions %% ------------------------------------------------------------------- on_get_status( - _InstId, - #{ - poolname := PoolName, - connect_timeout := Timeout - } = State + _InstanceID, + #{pool_name := PoolName, connect_timeout := Timeout} = State ) -> case do_get_status(PoolName, Timeout) of ok -> @@ -323,7 +312,7 @@ do_get_status(PoolName, Timeout) -> Error end end, - try emqx_misc:pmap(DoPerWorker, Workers, Timeout) of + try emqx_utils:pmap(DoPerWorker, Workers, Timeout) of Results -> case [E || {error, _} = E <- Results] of [] -> @@ -357,7 +346,7 @@ do_get_status(PoolName, Timeout) -> on_query( ResourceID, {RequestType, DataOrSQL}, - #{poolname := PoolName} = State + #{pool_name := PoolName} = State ) -> ?SLOG(debug, #{ msg => "clickhouse connector received sql query", @@ -396,16 +385,11 @@ query_type(send_message) -> on_batch_query( ResourceID, BatchReq, - State + #{pool_name := PoolName, templates := Templates} = _State ) -> %% Currently we only support batch requests with the send_message key {Keys, ObjectsToInsert} = lists:unzip(BatchReq), ensure_keys_are_of_type_send_message(Keys), - %% Pick out the SQL template - #{ - templates := Templates, - poolname := PoolName - } = State, %% Create batch insert SQL statement SQL = objects_to_sql(ObjectsToInsert, Templates), %% Do the actual query in the database diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl index 4703e0a21..ebb86f577 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl @@ -51,29 +51,22 @@ roots() -> fields(config) -> [ - {url, mk(binary(), #{required => true, desc => ?DESC("url")})} - | add_default_username( - emqx_connector_schema_lib:relational_db_fields() - ) + {url, mk(binary(), #{required => true, desc => ?DESC("url")})}, + {table, mk(binary(), #{required => true, desc => ?DESC("table")})}, + {aws_access_key_id, + mk( + binary(), + #{required => true, desc => ?DESC("aws_access_key_id")} + )}, + {aws_secret_access_key, + mk( + binary(), + #{required => true, desc => ?DESC("aws_secret_access_key")} + )}, + {pool_size, fun emqx_connector_schema_lib:pool_size/1}, + {auto_reconnect, fun emqx_connector_schema_lib:auto_reconnect/1} ]. -add_default_username(Fields) -> - lists:map( - fun - ({username, OrigUsernameFn}) -> - {username, add_default_fn(OrigUsernameFn, <<"root">>)}; - (Field) -> - Field - end, - Fields - ). - -add_default_fn(OrigFn, Default) -> - fun - (default) -> Default; - (Field) -> OrigFn(Field) - end. - %%======================================================================================== %% `emqx_resource' API %%======================================================================================== @@ -86,16 +79,16 @@ on_start( InstanceId, #{ url := Url, - username := Username, - password := Password, - database := Database, + aws_access_key_id := AccessKeyID, + aws_secret_access_key := SecretAccessKey, + table := Table, pool_size := PoolSize } = Config ) -> ?SLOG(info, #{ msg => "starting_dynamo_connector", connector => InstanceId, - config => emqx_misc:redact(Config) + config => redact(Config) }), {Schema, Server} = get_host_schema(to_str(Url)), @@ -105,8 +98,8 @@ on_start( {config, #{ host => Host, port => Port, - username => to_str(Username), - password => to_str(Password), + aws_access_key_id => to_str(AccessKeyID), + aws_secret_access_key => to_str(SecretAccessKey), schema => Schema }}, {pool_size, PoolSize} @@ -114,23 +107,23 @@ on_start( Templates = parse_template(Config), State = #{ - poolname => InstanceId, - database => Database, + pool_name => InstanceId, + table => Table, templates => Templates }, - case emqx_plugin_libs_pool:start_pool(InstanceId, ?MODULE, Options) of + case emqx_resource_pool:start(InstanceId, ?MODULE, Options) of ok -> {ok, State}; Error -> Error end. -on_stop(InstanceId, #{poolname := PoolName} = _State) -> +on_stop(InstanceId, #{pool_name := PoolName}) -> ?SLOG(info, #{ msg => "stopping_dynamo_connector", connector => InstanceId }), - emqx_plugin_libs_pool:stop_pool(PoolName). + emqx_resource_pool:stop(PoolName). on_query(InstanceId, Query, State) -> do_query(InstanceId, Query, handover, State). @@ -160,8 +153,8 @@ on_batch_query_async(InstanceId, [{send_message, _} | _] = Query, Reply, State) on_batch_query_async(_InstanceId, Query, _Reply, _State) -> {error, {unrecoverable_error, {invalid_request, Query}}}. -on_get_status(_InstanceId, #{poolname := Pool}) -> - Health = emqx_plugin_libs_pool:health_check_ecpool_workers(Pool, fun ?MODULE:do_get_status/1), +on_get_status(_InstanceId, #{pool_name := PoolName}) -> + Health = emqx_resource_pool:health_check_workers(PoolName, fun ?MODULE:do_get_status/1), status_result(Health). do_get_status(_Conn) -> @@ -183,7 +176,7 @@ do_query( InstanceId, Query, ApplyMode, - #{poolname := PoolName, templates := Templates, database := Database} = State + #{pool_name := PoolName, templates := Templates, table := Table} = State ) -> ?TRACE( "QUERY", @@ -192,7 +185,7 @@ do_query( ), Result = ecpool:pick_and_do( PoolName, - {?MODULE, worker_do_query, [Database, Query, Templates]}, + {?MODULE, worker_do_query, [Table, Query, Templates]}, ApplyMode ), @@ -217,43 +210,43 @@ do_query( Result end. -worker_do_query(_Client, Database, Query0, Templates) -> +worker_do_query(_Client, Table, Query0, Templates) -> try Query = apply_template(Query0, Templates), - execute(Query, Database) + execute(Query, Table) catch _Type:Reason -> {error, {unrecoverable_error, {invalid_request, Reason}}} end. %% some simple query commands for authn/authz or test -execute({insert_item, Msg}, Database) -> +execute({insert_item, Msg}, Table) -> Item = convert_to_item(Msg), - erlcloud_ddb2:put_item(Database, Item); -execute({delete_item, Key}, Database) -> - erlcloud_ddb2:delete_item(Database, Key); -execute({get_item, Key}, Database) -> - erlcloud_ddb2:get_item(Database, Key); + erlcloud_ddb2:put_item(Table, Item); +execute({delete_item, Key}, Table) -> + erlcloud_ddb2:delete_item(Table, Key); +execute({get_item, Key}, Table) -> + erlcloud_ddb2:get_item(Table, Key); %% commands for data bridge query or batch query -execute({send_message, Msg}, Database) -> +execute({send_message, Msg}, Table) -> Item = convert_to_item(Msg), - erlcloud_ddb2:put_item(Database, Item); -execute([{put, _} | _] = Msgs, Database) -> + erlcloud_ddb2:put_item(Table, Item); +execute([{put, _} | _] = Msgs, Table) -> %% type of batch_write_item argument :: batch_write_item_request_items() %% batch_write_item_request_items() :: maybe_list(batch_write_item_request_item()) %% batch_write_item_request_item() :: {table_name(), list(batch_write_item_request())} %% batch_write_item_request() :: {put, item()} | {delete, key()} - erlcloud_ddb2:batch_write_item({Database, Msgs}). + erlcloud_ddb2:batch_write_item({Table, Msgs}). connect(Opts) -> #{ - username := Username, - password := Password, + aws_access_key_id := AccessKeyID, + aws_secret_access_key := SecretAccessKey, host := Host, port := Port, schema := Schema } = proplists:get_value(config, Opts), - erlcloud_ddb2:configure(Username, Password, Host, Port, Schema), + erlcloud_ddb2:configure(AccessKeyID, SecretAccessKey, Host, Port, Schema), %% The dynamodb driver uses caller process as its connection process %% so at here, the connection process is the ecpool worker self @@ -322,7 +315,7 @@ convert_to_item(Msg) when is_map(Msg), map_size(Msg) > 0 -> Msg ); convert_to_item(MsgBin) when is_binary(MsgBin) -> - Msg = emqx_json:decode(MsgBin), + Msg = emqx_utils_json:decode(MsgBin), convert_to_item(Msg); convert_to_item(Item) -> erlang:throw({invalid_item, Item}). @@ -334,7 +327,10 @@ convert2binary(Value) when is_binary(Value); is_number(Value) -> convert2binary(Value) when is_list(Value) -> unicode:characters_to_binary(Value); convert2binary(Value) when is_map(Value) -> - emqx_json:encode(Value). + emqx_utils_json:encode(Value). do_async_reply(Result, {ReplyFun, [Context]}) -> ReplyFun(Context, Result). + +redact(Data) -> + emqx_utils:redact(Data, fun(Any) -> Any =:= aws_secret_access_key end). diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl index afe17cae6..700eb2a81 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl @@ -216,8 +216,8 @@ start_client(InstId, Config) -> ?SLOG(info, #{ msg => "starting influxdb connector", connector => InstId, - config => emqx_misc:redact(Config), - client_config => emqx_misc:redact(ClientConfig) + config => emqx_utils:redact(Config), + client_config => emqx_utils:redact(ClientConfig) }), try do_start_client(InstId, ClientConfig, Config) @@ -353,7 +353,7 @@ password(_) -> []. redact_auth(Term) -> - emqx_misc:redact(Term, fun is_auth_key/1). + emqx_utils:redact(Term, fun is_auth_key/1). is_auth_key(Key) when is_binary(Key) -> string:equal("authorization", Key, true); diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl index aa03863b0..59f763904 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl @@ -83,5 +83,89 @@ render_message(undefined = _PayloadTemplate, Message) -> render_message(PayloadTemplate, Message) -> %% Note: mongo expects a map as a document, so the rendered result %% must be JSON-serializable - Rendered = emqx_plugin_libs_rule:proc_tmpl(PayloadTemplate, Message), - emqx_json:decode(Rendered, [return_maps]). + format_data(PayloadTemplate, Message). + +%% The following function was originally copied over from +%% https://github.com/emqx/emqx-enterprise/commit/50e3628129720f13f544053600ca1502731e29e0. +%% The rule engine has support for producing fields that are date tuples +%% (produced by the SQL language's built-in functions mongo_date/0, +%% mongo_date/1 and mongo_date/2) which the MongoDB driver recognizes and +%% converts to the MongoDB ISODate type +%% (https://www.compose.com/articles/understanding-dates-in-compose-mongodb/). +%% For this to work we have to replace the tuple values with references, make +%% an instance of the template, convert the instance to map with the help of +%% emqx_utils_json:decode and then finally replace the references with the +%% corresponding tuples in the resulting map. +format_data(PayloadTks, Msg) -> + % Check the Message for any tuples that need to be extracted before running the template though a json parser + PreparedTupleMap = create_mapping_of_references_to_tuple_values(Msg), + case maps:size(PreparedTupleMap) of + % If no tuples were found simply proceed with the json decoding and be done with it + 0 -> + emqx_utils_json:decode(emqx_plugin_libs_rule:proc_tmpl(PayloadTks, Msg), [return_maps]); + _ -> + % If tuples were found, replace the tuple values with the references created, run + % the modified message through the json parser, and then at the end replace the + % references with the actual tuple values. + ProcessedMessage = replace_message_values_with_references(Msg, PreparedTupleMap), + DecodedMap = emqx_utils_json:decode( + emqx_plugin_libs_rule:proc_tmpl(PayloadTks, ProcessedMessage), [return_maps] + ), + populate_map_with_tuple_values(PreparedTupleMap, DecodedMap) + end. + +replace_message_values_with_references(RawMessage, TupleMap) -> + % Iterate over every created reference/value pair and inject the reference into the message + maps:fold( + fun(Reference, OriginalValue, OriginalMessage) -> + % Iterate over the Message, which is a map, and look for the element which + % matches the Value in the map which holds the references/original values and replace + % with the reference + maps:fold( + fun(Key, Value, NewMap) -> + case Value == OriginalValue of + true -> + %% Wrap the reference in a string to make it JSON-serializable + StringRef = io_lib:format("\"~s\"", [Reference]), + WrappedRef = erlang:iolist_to_binary(StringRef), + maps:put(Key, WrappedRef, NewMap); + false -> + maps:put(Key, Value, NewMap) + end + end, + #{}, + OriginalMessage + ) + end, + RawMessage, + TupleMap + ). + +create_mapping_of_references_to_tuple_values(Message) -> + maps:fold( + fun + (_Key, Value, TupleMap) when is_tuple(Value) -> + Ref0 = emqx_guid:to_hexstr(emqx_guid:gen()), + Ref = <<"MONGO_DATE_REF_", Ref0/binary>>, + maps:put(Ref, Value, TupleMap); + (_Key, _Value, TupleMap) -> + TupleMap + end, + #{}, + Message + ). + +populate_map_with_tuple_values(TupleMap, MapToMap) -> + MappingFun = + fun + (_Key, Value) when is_map(Value) -> + populate_map_with_tuple_values(TupleMap, Value); + (_Key, Value) -> + case maps:is_key(Value, TupleMap) of + true -> + maps:get(Value, TupleMap); + false -> + Value + end + end, + maps:map(MappingFun, MapToMap). diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl index 84f2e2a89..205359bb8 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl @@ -248,14 +248,14 @@ get_topic_key([Query | _], RawTopic, TopicTks) -> apply_template({Key, Msg} = _Req, Templates) -> case maps:get(Key, Templates, undefined) of undefined -> - emqx_json:encode(Msg); + emqx_utils_json:encode(Msg); Template -> emqx_plugin_libs_rule:proc_tmpl(Template, Msg) end; apply_template([{Key, _} | _] = Reqs, Templates) -> case maps:get(Key, Templates, undefined) of undefined -> - [emqx_json:encode(Msg) || {_, Msg} <- Reqs]; + [emqx_utils_json:encode(Msg) || {_, Msg} <- Reqs]; Template -> [emqx_plugin_libs_rule:proc_tmpl(Template, Msg) || {_, Msg} <- Reqs] end. @@ -265,7 +265,7 @@ client_id(InstanceId) -> erlang:binary_to_atom(Name, utf8). redact(Msg) -> - emqx_misc:redact(Msg, fun is_sensitive_key/1). + emqx_utils:redact(Msg, fun is_sensitive_key/1). is_sensitive_key(security_token) -> true; diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl new file mode 100644 index 000000000..70bd76d14 --- /dev/null +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl @@ -0,0 +1,528 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_ee_connector_sqlserver). + +-behaviour(emqx_resource). + +-include_lib("kernel/include/file.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx_resource/include/emqx_resource.hrl"). +-include_lib("emqx_ee_connector/include/emqx_ee_connector.hrl"). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). + +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +%%==================================================================== +%% Exports +%%==================================================================== + +%% Hocon config schema exports +-export([ + roots/0, + fields/1 +]). + +%% callbacks for behaviour emqx_resource +-export([ + callback_mode/0, + is_buffer_supported/0, + on_start/2, + on_stop/2, + on_query/3, + on_batch_query/3, + on_query_async/4, + on_batch_query_async/4, + on_get_status/2 +]). + +%% callbacks for ecpool +-export([connect/1]). + +%% Internal exports used to execute code with ecpool worker +-export([do_get_status/2, worker_do_insert/3, do_async_reply/2]). + +-import(emqx_plugin_libs_rule, [str/1]). +-import(hoconsc, [mk/2, enum/1, ref/2]). + +-define(ACTION_SEND_MESSAGE, send_message). + +-define(SYNC_QUERY_MODE, handover). +-define(ASYNC_QUERY_MODE(REPLY), {handover_async, {?MODULE, do_async_reply, [REPLY]}}). + +-define(SQLSERVER_HOST_OPTIONS, #{ + default_port => 1433 +}). + +-define(REQUEST_TIMEOUT(RESOURCE_OPTS), + maps:get(request_timeout, RESOURCE_OPTS, ?DEFAULT_REQUEST_TIMEOUT) +). + +-define(BATCH_INSERT_TEMP, batch_insert_temp). + +-define(BATCH_INSERT_PART, batch_insert_part). +-define(BATCH_PARAMS_TOKENS, batch_insert_tks). + +-define(FILE_MODE_755, 33261). +%% 32768 + 8#00400 + 8#00200 + 8#00100 + 8#00040 + 8#00010 + 8#00004 + 8#00001 +%% See also +%% https://www.erlang.org/doc/man/file.html#read_file_info-2 + +%% Copied from odbc reference page +%% https://www.erlang.org/doc/man/odbc.html + +%% as returned by connect/2 +-type connection_reference() :: pid(). +-type time_out() :: milliseconds() | infinity. +-type sql() :: string() | binary(). +-type milliseconds() :: pos_integer(). +%% Tuple of column values e.g. one row of the result set. +%% it's a variable size tuple of column values. +-type row() :: tuple(). +%% Some kind of explanation of what went wrong +-type common_reason() :: connection_closed | extended_error() | term(). +%% extended error type with ODBC +%% and native database error codes, as well as the base reason that would have been +%% returned had extended_errors not been enabled. +-type extended_error() :: {string(), integer(), _Reason :: term()}. +%% Name of column in the result set +-type col_name() :: string(). +%% e.g. a list of the names of the selected columns in the result set. +-type col_names() :: [col_name()]. +%% A list of rows from the result set. +-type rows() :: list(row()). + +%% -type result_tuple() :: {updated, n_rows()} | {selected, col_names(), rows()}. +-type updated_tuple() :: {updated, n_rows()}. +-type selected_tuple() :: {selected, col_names(), rows()}. +%% The number of affected rows for UPDATE, +%% INSERT, or DELETE queries. For other query types the value +%% is driver defined, and hence should be ignored. +-type n_rows() :: integer(). + +%% These type was not used in this module, but we may use it later +%% -type odbc_data_type() :: +%% sql_integer +%% | sql_smallint +%% | sql_tinyint +%% | {sql_decimal, precision(), scale()} +%% | {sql_numeric, precision(), scale()} +%% | {sql_char, size()} +%% | {sql_wchar, size()} +%% | {sql_varchar, size()} +%% | {sql_wvarchar, size()} +%% | {sql_float, precision()} +%% | {sql_wlongvarchar, size()} +%% | {sql_float, precision()} +%% | sql_real +%% | sql_double +%% | sql_bit +%% | atom(). +%% -type precision() :: integer(). +%% -type scale() :: integer(). +%% -type size() :: integer(). + +-type state() :: #{ + pool_name := binary(), + resource_opts := map(), + sql_templates := map() +}. + +%%==================================================================== +%% Configuration and default values +%%==================================================================== + +roots() -> + [{config, #{type => hoconsc:ref(?MODULE, config)}}]. + +fields(config) -> + [ + {server, server()} + | add_default_username(emqx_connector_schema_lib:relational_db_fields()) + ]. + +add_default_username(Fields) -> + lists:map( + fun + ({username, OrigUsernameFn}) -> + {username, add_default_fn(OrigUsernameFn, <<"sa">>)}; + (Field) -> + Field + end, + Fields + ). + +add_default_fn(OrigFn, Default) -> + fun + (default) -> Default; + (Field) -> OrigFn(Field) + end. + +server() -> + Meta = #{desc => ?DESC("server")}, + emqx_schema:servers_sc(Meta, ?SQLSERVER_HOST_OPTIONS). + +%%==================================================================== +%% Callbacks defined in emqx_resource +%%==================================================================== + +callback_mode() -> async_if_possible. + +is_buffer_supported() -> false. + +on_start( + InstanceId = PoolName, + #{ + server := Server, + username := Username, + password := Password, + driver := Driver, + database := Database, + pool_size := PoolSize, + resource_opts := ResourceOpts + } = Config +) -> + ?SLOG(info, #{ + msg => "starting_sqlserver_connector", + connector => InstanceId, + config => emqx_utils:redact(Config) + }), + + ODBCDir = code:priv_dir(odbc), + OdbcserverDir = filename:join(ODBCDir, "bin/odbcserver"), + {ok, Info = #file_info{mode = Mode}} = file:read_file_info(OdbcserverDir), + case ?FILE_MODE_755 =:= Mode of + true -> + ok; + false -> + _ = file:write_file_info(OdbcserverDir, Info#file_info{mode = ?FILE_MODE_755}), + ok + end, + + Options = [ + {server, to_bin(Server)}, + {username, Username}, + {password, Password}, + {driver, Driver}, + {database, Database}, + {pool_size, PoolSize} + ], + + State = #{ + %% also InstanceId + pool_name => PoolName, + sql_templates => parse_sql_template(Config), + resource_opts => ResourceOpts + }, + case emqx_resource_pool:start(PoolName, ?MODULE, Options) of + ok -> + {ok, State}; + {error, Reason} -> + ?tp( + sqlserver_connector_start_failed, + #{error => Reason} + ), + {error, Reason} + end. + +on_stop(InstanceId, #{pool_name := PoolName} = _State) -> + ?SLOG(info, #{ + msg => "stopping_sqlserver_connector", + connector => InstanceId + }), + emqx_resource_pool:stop(PoolName). + +-spec on_query( + manager_id(), + {?ACTION_SEND_MESSAGE, map()}, + state() +) -> + ok + | {ok, list()} + | {error, {recoverable_error, term()}} + | {error, term()}. +on_query(InstanceId, {?ACTION_SEND_MESSAGE, _Msg} = Query, State) -> + ?TRACE( + "SINGLE_QUERY_SYNC", + "bridge_sqlserver_received", + #{requests => Query, connector => InstanceId, state => State} + ), + do_query(InstanceId, Query, ?SYNC_QUERY_MODE, State). + +-spec on_query_async( + manager_id(), + {?ACTION_SEND_MESSAGE, map()}, + {ReplyFun :: function(), Args :: list()}, + state() +) -> + {ok, any()} + | {error, term()}. +on_query_async( + InstanceId, + {?ACTION_SEND_MESSAGE, _Msg} = Query, + ReplyFunAndArgs, + State +) -> + ?TRACE( + "SINGLE_QUERY_ASYNC", + "bridge_sqlserver_received", + #{requests => Query, connector => InstanceId, state => State} + ), + do_query(InstanceId, Query, ?ASYNC_QUERY_MODE(ReplyFunAndArgs), State). + +-spec on_batch_query( + manager_id(), + [{?ACTION_SEND_MESSAGE, map()}], + state() +) -> + ok + | {ok, list()} + | {error, {recoverable_error, term()}} + | {error, term()}. +on_batch_query(InstanceId, BatchRequests, State) -> + ?TRACE( + "BATCH_QUERY_SYNC", + "bridge_sqlserver_received", + #{requests => BatchRequests, connector => InstanceId, state => State} + ), + do_query(InstanceId, BatchRequests, ?SYNC_QUERY_MODE, State). + +-spec on_batch_query_async( + manager_id(), + [{?ACTION_SEND_MESSAGE, map()}], + {ReplyFun :: function(), Args :: list()}, + state() +) -> {ok, any()}. +on_batch_query_async(InstanceId, Requests, ReplyFunAndArgs, State) -> + ?TRACE( + "BATCH_QUERY_ASYNC", + "bridge_sqlserver_received", + #{requests => Requests, connector => InstanceId, state => State} + ), + do_query(InstanceId, Requests, ?ASYNC_QUERY_MODE(ReplyFunAndArgs), State). + +on_get_status(_InstanceId, #{pool_name := PoolName, resource_opts := ResourceOpts} = _State) -> + RequestTimeout = ?REQUEST_TIMEOUT(ResourceOpts), + Health = emqx_resource_pool:health_check_workers( + PoolName, + {?MODULE, do_get_status, [RequestTimeout]}, + RequestTimeout + ), + status_result(Health). + +status_result(_Status = true) -> connected; +status_result(_Status = false) -> connecting. +%% TODO: +%% case for disconnected + +%%==================================================================== +%% ecpool callback fns +%%==================================================================== + +-spec connect(Options :: list()) -> {ok, connection_reference()} | {error, term()}. +connect(Options) -> + ConnectStr = lists:concat(conn_str(Options, [])), + Opts = proplists:get_value(options, Options, []), + odbc:connect(ConnectStr, Opts). + +-spec do_get_status(connection_reference(), time_out()) -> Result :: boolean(). +do_get_status(Conn, RequestTimeout) -> + case execute(Conn, <<"SELECT 1">>, RequestTimeout) of + {selected, [[]], [{1}]} -> true; + _ -> false + end. + +%%==================================================================== +%% Internal Helper fns +%%==================================================================== + +%% TODO && FIXME: +%% About the connection string attribute `Encrypt`: +%% The default value is `yes` in odbc version 18.0+ and `no` in previous versions. +%% And encrypted connections always verify the server's certificate. +%% So `Encrypt=YES;TrustServerCertificate=YES` must be set in the connection string +%% when connecting to a server that has a self-signed certificate. +%% See also: +%% 'https://learn.microsoft.com/en-us/sql/connect/odbc/ +%% dsn-connection-string-attribute?source=recommendations&view=sql-server-ver16#encrypt' +conn_str([], Acc) -> + %% we should use this for msodbcsql 18+ + %% lists:join(";", ["Encrypt=YES", "TrustServerCertificate=YES" | Acc]); + lists:join(";", Acc); +conn_str([{driver, Driver} | Opts], Acc) -> + conn_str(Opts, ["Driver=" ++ str(Driver) | Acc]); +conn_str([{server, Server} | Opts], Acc) -> + {Host, Port} = emqx_schema:parse_server(Server, ?SQLSERVER_HOST_OPTIONS), + conn_str(Opts, ["Server=" ++ str(Host) ++ "," ++ str(Port) | Acc]); +conn_str([{database, Database} | Opts], Acc) -> + conn_str(Opts, ["Database=" ++ str(Database) | Acc]); +conn_str([{username, Username} | Opts], Acc) -> + conn_str(Opts, ["UID=" ++ str(Username) | Acc]); +conn_str([{password, Password} | Opts], Acc) -> + conn_str(Opts, ["PWD=" ++ str(Password) | Acc]); +conn_str([{_, _} | Opts], Acc) -> + conn_str(Opts, Acc). + +%% Sync & Async query with singe & batch sql statement +-spec do_query( + manager_id(), + Query :: {?ACTION_SEND_MESSAGE, map()} | [{?ACTION_SEND_MESSAGE, map()}], + ApplyMode :: + handover + | {handover_async, {?MODULE, do_async_reply, [{ReplyFun :: function(), Args :: list()}]}}, + state() +) -> + {ok, list()} + | {error, {recoverable_error, term()}} + | {error, term()}. +do_query( + InstanceId, + Query, + ApplyMode, + #{pool_name := PoolName, sql_templates := Templates} = State +) -> + ?TRACE( + "SINGLE_QUERY_SYNC", + "sqlserver_connector_received", + #{query => Query, connector => InstanceId, state => State} + ), + + %% only insert sql statement for single query and batch query + case apply_template(Query, Templates) of + {?ACTION_SEND_MESSAGE, SQL} -> + Result = ecpool:pick_and_do( + PoolName, + {?MODULE, worker_do_insert, [SQL, State]}, + ApplyMode + ); + Query -> + Result = {error, {unrecoverable_error, invalid_query}}; + _ -> + Result = {error, {unrecoverable_error, failed_to_apply_sql_template}} + end, + case Result of + {error, Reason} -> + ?tp( + sqlserver_connector_query_return, + #{error => Reason} + ), + ?SLOG(error, #{ + msg => "sqlserver_connector_do_query_failed", + connector => InstanceId, + query => Query, + reason => Reason + }), + Result; + _ -> + ?tp( + sqlserver_connector_query_return, + #{result => Result} + ), + Result + end. + +worker_do_insert( + Conn, SQL, #{resource_opts := ResourceOpts, pool_name := InstanceId} = State +) -> + LogMeta = #{connector => InstanceId, state => State}, + try + case execute(Conn, SQL, ?REQUEST_TIMEOUT(ResourceOpts)) of + {selected, Rows, _} -> + {ok, Rows}; + {updated, _} -> + ok; + {error, ErrStr} -> + ?SLOG(error, LogMeta#{msg => "invalid_request", reason => ErrStr}), + {error, {unrecoverable_error, {invalid_request, ErrStr}}} + end + catch + _Type:Reason -> + ?SLOG(error, LogMeta#{msg => "invalid_request", reason => Reason}), + {error, {unrecoverable_error, {invalid_request, Reason}}} + end. + +-spec execute(pid(), sql(), time_out()) -> + updated_tuple() + | selected_tuple() + | [updated_tuple()] + | [selected_tuple()] + | {error, common_reason()}. +execute(Conn, SQL, Timeout) -> + odbc:sql_query(Conn, str(SQL), Timeout). + +to_bin(List) when is_list(List) -> + unicode:characters_to_binary(List, utf8). + +%% for bridge data to sql server +parse_sql_template(Config) -> + RawSQLTemplates = + case maps:get(sql, Config, undefined) of + undefined -> #{}; + <<>> -> #{}; + SQLTemplate -> #{?ACTION_SEND_MESSAGE => SQLTemplate} + end, + + BatchInsertTks = #{}, + parse_sql_template(maps:to_list(RawSQLTemplates), BatchInsertTks). + +parse_sql_template([{Key, H} | T], BatchInsertTks) -> + case emqx_plugin_libs_rule:detect_sql_type(H) of + {ok, select} -> + parse_sql_template(T, BatchInsertTks); + {ok, insert} -> + case emqx_plugin_libs_rule:split_insert_sql(H) of + {ok, {InsertSQL, Params}} -> + parse_sql_template( + T, + BatchInsertTks#{ + Key => + #{ + ?BATCH_INSERT_PART => InsertSQL, + ?BATCH_PARAMS_TOKENS => emqx_plugin_libs_rule:preproc_tmpl( + Params + ) + } + } + ); + {error, Reason} -> + ?SLOG(error, #{msg => "split sql failed", sql => H, reason => Reason}), + parse_sql_template(T, BatchInsertTks) + end; + {error, Reason} -> + ?SLOG(error, #{msg => "detect sql type failed", sql => H, reason => Reason}), + parse_sql_template(T, BatchInsertTks) + end; +parse_sql_template([], BatchInsertTks) -> + #{ + ?BATCH_INSERT_TEMP => BatchInsertTks + }. + +%% single insert +apply_template( + {?ACTION_SEND_MESSAGE = _Key, _Msg} = Query, Templates +) -> + %% TODO: fix emqx_plugin_libs_rule:proc_tmpl/2 + %% it won't add single quotes for string + apply_template([Query], Templates); +%% batch inserts +apply_template( + [{?ACTION_SEND_MESSAGE = Key, _Msg} | _T] = BatchReqs, + #{?BATCH_INSERT_TEMP := BatchInsertsTks} = _Templates +) -> + case maps:get(Key, BatchInsertsTks, undefined) of + undefined -> + BatchReqs; + #{?BATCH_INSERT_PART := BatchInserts, ?BATCH_PARAMS_TOKENS := BatchParamsTks} -> + SQL = emqx_plugin_libs_rule:proc_batch_sql(BatchReqs, BatchInserts, BatchParamsTks), + {Key, SQL} + end; +apply_template(Query, Templates) -> + %% TODO: more detail infomatoin + ?SLOG(error, #{msg => "apply sql template failed", query => Query, templates => Templates}), + {error, failed_to_apply_sql_template}. + +do_async_reply(Result, {ReplyFun, Args}) -> + erlang:apply(ReplyFun, Args ++ [Result]). diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_tdengine.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_tdengine.erl index 746ab5485..f9ca21ad7 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_tdengine.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_tdengine.erl @@ -93,7 +93,7 @@ on_start( ?SLOG(info, #{ msg => "starting_tdengine_connector", connector => InstanceId, - config => emqx_misc:redact(Config) + config => emqx_utils:redact(Config) }), {Host, Port} = emqx_schema:parse_server(Server, ?TD_HOST_OPTIONS), @@ -107,20 +107,20 @@ on_start( ], Prepares = parse_prepare_sql(Config), - State = maps:merge(Prepares, #{poolname => InstanceId, query_opts => query_opts(Config)}), - case emqx_plugin_libs_pool:start_pool(InstanceId, ?MODULE, Options) of + State = Prepares#{pool_name => InstanceId, query_opts => query_opts(Config)}, + case emqx_resource_pool:start(InstanceId, ?MODULE, Options) of ok -> {ok, State}; Error -> Error end. -on_stop(InstanceId, #{poolname := PoolName} = _State) -> +on_stop(InstanceId, #{pool_name := PoolName}) -> ?SLOG(info, #{ msg => "stopping_tdengine_connector", connector => InstanceId }), - emqx_plugin_libs_pool:stop_pool(PoolName). + emqx_resource_pool:stop(PoolName). on_query(InstanceId, {query, SQL}, State) -> do_query(InstanceId, SQL, State); @@ -150,8 +150,8 @@ on_batch_query( {error, {unrecoverable_error, invalid_request}} end. -on_get_status(_InstanceId, #{poolname := Pool}) -> - Health = emqx_plugin_libs_pool:health_check_ecpool_workers(Pool, fun ?MODULE:do_get_status/1), +on_get_status(_InstanceId, #{pool_name := PoolName}) -> + Health = emqx_resource_pool:health_check_workers(PoolName, fun ?MODULE:do_get_status/1), status_result(Health). do_get_status(Conn) -> @@ -171,7 +171,7 @@ do_batch_insert(InstanceId, BatchReqs, InsertPart, Tokens, State) -> SQL = emqx_plugin_libs_rule:proc_batch_sql(BatchReqs, InsertPart, Tokens), do_query(InstanceId, SQL, State). -do_query(InstanceId, Query, #{poolname := PoolName, query_opts := Opts} = State) -> +do_query(InstanceId, Query, #{pool_name := PoolName, query_opts := Opts} = State) -> ?TRACE( "QUERY", "tdengine_connector_received", diff --git a/lib-ee/emqx_ee_connector/test/ee_connector_clickhouse_SUITE.erl b/lib-ee/emqx_ee_connector/test/emqx_ee_connector_clickhouse_SUITE.erl similarity index 75% rename from lib-ee/emqx_ee_connector/test/ee_connector_clickhouse_SUITE.erl rename to lib-ee/emqx_ee_connector/test/emqx_ee_connector_clickhouse_SUITE.erl index 111deba06..e704a2c0c 100644 --- a/lib-ee/emqx_ee_connector/test/ee_connector_clickhouse_SUITE.erl +++ b/lib-ee/emqx_ee_connector/test/emqx_ee_connector_clickhouse_SUITE.erl @@ -1,19 +1,8 @@ -% %%-------------------------------------------------------------------- -% %% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -% %% -% %% Licensed under the Apache License, Version 2.0 (the "License"); -% %% you may not use this file except in compliance with the License. -% %% You may obtain a copy of the License at -% %% http://www.apache.org/licenses/LICENSE-2.0 -% %% -% %% Unless required by applicable law or agreed to in writing, software -% %% distributed under the License is distributed on an "AS IS" BASIS, -% %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -% %% See the License for the specific language governing permissions and -% %% limitations under the License. -% %%-------------------------------------------------------------------- +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- --module(ee_connector_clickhouse_SUITE). +-module(emqx_ee_connector_clickhouse_SUITE). -compile(nowarn_export_all). -compile(export_all). @@ -106,15 +95,15 @@ show(Label, What) -> erlang:display({Label, What}), What. -perform_lifecycle_check(PoolName, InitialConfig) -> +perform_lifecycle_check(ResourceID, InitialConfig) -> {ok, #{config := CheckedConfig}} = emqx_resource:check_config(?CLICKHOUSE_RESOURCE_MOD, InitialConfig), {ok, #{ - state := #{poolname := ReturnedPoolName} = State, + state := #{pool_name := PoolName} = State, status := InitialStatus }} = emqx_resource:create_local( - PoolName, + ResourceID, ?CONNECTOR_RESOURCE_GROUP, ?CLICKHOUSE_RESOURCE_MOD, CheckedConfig, @@ -126,49 +115,49 @@ perform_lifecycle_check(PoolName, InitialConfig) -> state := State, status := InitialStatus }} = - emqx_resource:get_instance(PoolName), - ?assertEqual({ok, connected}, emqx_resource:health_check(PoolName)), + emqx_resource:get_instance(ResourceID), + ?assertEqual({ok, connected}, emqx_resource:health_check(ResourceID)), % % Perform query as further check that the resource is working as expected (fun() -> - erlang:display({pool_name, PoolName}), - QueryNoParamsResWrapper = emqx_resource:query(PoolName, test_query_no_params()), + erlang:display({pool_name, ResourceID}), + QueryNoParamsResWrapper = emqx_resource:query(ResourceID, test_query_no_params()), ?assertMatch({ok, _}, QueryNoParamsResWrapper), {_, QueryNoParamsRes} = QueryNoParamsResWrapper, ?assertMatch(<<"1">>, string:trim(QueryNoParamsRes)) end)(), - ?assertEqual(ok, emqx_resource:stop(PoolName)), + ?assertEqual(ok, emqx_resource:stop(ResourceID)), % Resource will be listed still, but state will be changed and healthcheck will fail % as the worker no longer exists. {ok, ?CONNECTOR_RESOURCE_GROUP, #{ state := State, status := StoppedStatus }} = - emqx_resource:get_instance(PoolName), + emqx_resource:get_instance(ResourceID), ?assertEqual(stopped, StoppedStatus), - ?assertEqual({error, resource_is_stopped}, emqx_resource:health_check(PoolName)), + ?assertEqual({error, resource_is_stopped}, emqx_resource:health_check(ResourceID)), % Resource healthcheck shortcuts things by checking ets. Go deeper by checking pool itself. - ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)), + ?assertEqual({error, not_found}, ecpool:stop_sup_pool(PoolName)), % Can call stop/1 again on an already stopped instance - ?assertEqual(ok, emqx_resource:stop(PoolName)), + ?assertEqual(ok, emqx_resource:stop(ResourceID)), % Make sure it can be restarted and the healthchecks and queries work properly - ?assertEqual(ok, emqx_resource:restart(PoolName)), + ?assertEqual(ok, emqx_resource:restart(ResourceID)), % async restart, need to wait resource timer:sleep(500), {ok, ?CONNECTOR_RESOURCE_GROUP, #{status := InitialStatus}} = - emqx_resource:get_instance(PoolName), - ?assertEqual({ok, connected}, emqx_resource:health_check(PoolName)), + emqx_resource:get_instance(ResourceID), + ?assertEqual({ok, connected}, emqx_resource:health_check(ResourceID)), (fun() -> QueryNoParamsResWrapper = - emqx_resource:query(PoolName, test_query_no_params()), + emqx_resource:query(ResourceID, test_query_no_params()), ?assertMatch({ok, _}, QueryNoParamsResWrapper), {_, QueryNoParamsRes} = QueryNoParamsResWrapper, ?assertMatch(<<"1">>, string:trim(QueryNoParamsRes)) end)(), % Stop and remove the resource in one go. - ?assertEqual(ok, emqx_resource:remove_local(PoolName)), - ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)), + ?assertEqual(ok, emqx_resource:remove_local(ResourceID)), + ?assertEqual({error, not_found}, ecpool:stop_sup_pool(PoolName)), % Should not even be able to get the resource data out of ets now unlike just stopping. - ?assertEqual({error, not_found}, emqx_resource:get_instance(PoolName)). + ?assertEqual({error, not_found}, emqx_resource:get_instance(ResourceID)). % %%------------------------------------------------------------------------------ % %% Helpers diff --git a/lib-ee/emqx_ee_connector/test/ee_connector_hstreamdb_SUITE.erl b/lib-ee/emqx_ee_connector/test/emqx_ee_connector_hstreamdb_SUITE.erl similarity index 90% rename from lib-ee/emqx_ee_connector/test/ee_connector_hstreamdb_SUITE.erl rename to lib-ee/emqx_ee_connector/test/emqx_ee_connector_hstreamdb_SUITE.erl index 8acabbef4..ad49d9f62 100644 --- a/lib-ee/emqx_ee_connector/test/ee_connector_hstreamdb_SUITE.erl +++ b/lib-ee/emqx_ee_connector/test/emqx_ee_connector_hstreamdb_SUITE.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(ee_connector_hstreamdb_SUITE). +-module(emqx_ee_connector_hstreamdb_SUITE). -compile(nowarn_export_all). -compile(export_all). diff --git a/lib-ee/emqx_ee_schema_registry/.gitignore b/lib-ee/emqx_ee_schema_registry/.gitignore new file mode 100644 index 000000000..f1c455451 --- /dev/null +++ b/lib-ee/emqx_ee_schema_registry/.gitignore @@ -0,0 +1,19 @@ +.rebar3 +_* +.eunit +*.o +*.beam +*.plt +*.swp +*.swo +.erlang.cookie +ebin +log +erl_crash.dump +.rebar +logs +_build +.idea +*.iml +rebar3.crashdump +*~ diff --git a/lib-ee/emqx_ee_schema_registry/README.md b/lib-ee/emqx_ee_schema_registry/README.md new file mode 100644 index 000000000..c1c409c7d --- /dev/null +++ b/lib-ee/emqx_ee_schema_registry/README.md @@ -0,0 +1,73 @@ +# EMQX Schema Registry + +EMQX Schema Registry for managing various of schemas for +decoding/encoding messages. + +To use schema in rule engine, a schema name should be passed to the +SQL functions that decode/encode data, like: + +```sql +SELECT + schema_decode('sensor_notify', payload) as payload +FROM + "message.publish" +WHERE + topic = 't/1' +``` + +## Using schema registry with rule engine + +``` + +---------------------------+ + | | + Events/Msgs | | Events/Msgs + --------------------> EMQX |------------------> + | | + | | + +-------------|-------------+ + | + HOOK | + | + +-------------v-------------+ +----------+ + | | Data | | + | Rule Engine ------------- Backends | + | | | | + +------|-------------|------+ +----------+ + |^ |^ + Decode|| ||Encode + || || + +------v|------------v|-----+ + | | + | Schema Registry | + | | + +---------------------------+ +``` + +## Architecture + +``` + | | + Decode | [APIs] | Encode + | | + | | [Registry] + +------v--------------v------+ + REGISTER SCHEMA | | + INSTANCE | | +--------+ + -------------------> | | | +[Management APIs] | Schema Registry ------ Schema | + | | | | + | | +--------+ + | | + +----------------------------+ + / | \ + +---/---+ +---|----+ +---\---+ + | | | | | | + [Decoders] | Avro | |ProtoBuf| |Others | + | | | | | | + +-------+ +--------+ +-------+ + +``` + +- Register schema instance: adds a new instance of a schema of a + certain type. For example, when the user may have several Avro or + Protobuf schemas that they wish to use with different data flows. diff --git a/lib-ee/emqx_ee_schema_registry/etc/emqx_ee_schema_registry.conf b/lib-ee/emqx_ee_schema_registry/etc/emqx_ee_schema_registry.conf new file mode 100644 index 000000000..e69de29bb diff --git a/lib-ee/emqx_ee_schema_registry/include/emqx_ee_schema_registry.hrl b/lib-ee/emqx_ee_schema_registry/include/emqx_ee_schema_registry.hrl new file mode 100644 index 000000000..058abf007 --- /dev/null +++ b/lib-ee/emqx_ee_schema_registry/include/emqx_ee_schema_registry.hrl @@ -0,0 +1,56 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-ifndef(EMQX_EE_SCHEMA_REGISTRY_HRL). +-define(EMQX_EE_SCHEMA_REGISTRY_HRL, true). + +-define(CONF_KEY_ROOT, schema_registry). +-define(CONF_KEY_PATH, [?CONF_KEY_ROOT]). + +-define(SCHEMA_REGISTRY_SHARD, emqx_ee_schema_registry_shard). +-define(SERDE_TAB, emqx_ee_schema_registry_serde_tab). +-define(PROTOBUF_CACHE_TAB, emqx_ee_schema_registry_protobuf_cache_tab). + +-type schema_name() :: binary(). +-type schema_source() :: binary(). + +-type encoded_data() :: iodata(). +-type decoded_data() :: map(). +-type serializer() :: + fun((decoded_data()) -> encoded_data()) + | fun((decoded_data(), term()) -> encoded_data()). +-type deserializer() :: + fun((encoded_data()) -> decoded_data()) + | fun((encoded_data(), term()) -> decoded_data()). +-type destructor() :: fun(() -> ok). +-type serde_type() :: avro. +-type serde_opts() :: map(). + +-record(serde, { + name :: schema_name(), + serializer :: serializer(), + deserializer :: deserializer(), + destructor :: destructor() +}). +-type serde() :: #serde{}. + +-record(protobuf_cache, { + fingerprint, + module, + module_binary +}). +-type protobuf_cache() :: #protobuf_cache{ + fingerprint :: binary(), + module :: module(), + module_binary :: binary() +}. + +-type serde_map() :: #{ + name := schema_name(), + serializer := serializer(), + deserializer := deserializer(), + destructor := destructor() +}. + +-endif. diff --git a/lib-ee/emqx_ee_schema_registry/rebar.config b/lib-ee/emqx_ee_schema_registry/rebar.config new file mode 100644 index 000000000..e42ff7278 --- /dev/null +++ b/lib-ee/emqx_ee_schema_registry/rebar.config @@ -0,0 +1,14 @@ +%% -*- mode: erlang -*- + +{erl_opts, [debug_info]}. +{deps, [ + {emqx, {path, "../../apps/emqx"}}, + {emqx_utils, {path, "../../apps/emqx_utils"}}, + {erlavro, {git, "https://github.com/klarna/erlavro.git", {tag, "2.9.8"}}}, + {gpb, "4.19.7"} +]}. + +{shell, [ + % {config, "config/sys.config"}, + {apps, [emqx_ee_schema_registry]} +]}. diff --git a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.app.src b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.app.src new file mode 100644 index 000000000..87f6e53d0 --- /dev/null +++ b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.app.src @@ -0,0 +1,16 @@ +{application, emqx_ee_schema_registry, [ + {description, "EMQX Schema Registry"}, + {vsn, "0.1.1"}, + {registered, [emqx_ee_schema_registry_sup]}, + {mod, {emqx_ee_schema_registry_app, []}}, + {applications, [ + kernel, + stdlib, + erlavro, + gpb + ]}, + {env, []}, + {modules, []}, + + {links, []} +]}. diff --git a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.erl b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.erl new file mode 100644 index 000000000..59a224fc7 --- /dev/null +++ b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.erl @@ -0,0 +1,260 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_ee_schema_registry). + +-behaviour(gen_server). +-behaviour(emqx_config_handler). + +-include("emqx_ee_schema_registry.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +%% API +-export([ + start_link/0, + + get_serde/1, + + add_schema/2, + get_schema/1, + delete_schema/1, + list_schemas/0 +]). + +%% `gen_server' API +-export([ + init/1, + handle_call/3, + handle_cast/2, + handle_continue/2, + terminate/2 +]). + +%% `emqx_config_handler' API +-export([post_config_update/5]). + +-type schema() :: #{ + type := serde_type(), + source := binary(), + description => binary() +}. + +%%------------------------------------------------------------------------------------------------- +%% API +%%------------------------------------------------------------------------------------------------- + +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +-spec get_serde(schema_name()) -> {ok, serde_map()} | {error, not_found}. +get_serde(SchemaName) -> + case ets:lookup(?SERDE_TAB, to_bin(SchemaName)) of + [] -> + {error, not_found}; + [Serde] -> + {ok, serde_to_map(Serde)} + end. + +-spec get_schema(schema_name()) -> {ok, map()} | {error, not_found}. +get_schema(SchemaName) -> + case emqx_config:get([?CONF_KEY_ROOT, schemas, SchemaName], undefined) of + undefined -> + {error, not_found}; + Config -> + {ok, Config} + end. + +-spec add_schema(schema_name(), schema()) -> ok | {error, term()}. +add_schema(Name, Schema) -> + RawSchema = emqx_utils_maps:binary_key_map(Schema), + Res = emqx_conf:update( + [?CONF_KEY_ROOT, schemas, Name], + RawSchema, + #{override_to => cluster} + ), + case Res of + {ok, _} -> + ok; + Error -> + Error + end. + +-spec delete_schema(schema_name()) -> ok | {error, term()}. +delete_schema(Name) -> + Res = emqx_conf:remove( + [?CONF_KEY_ROOT, schemas, Name], + #{override_to => cluster} + ), + case Res of + {ok, _} -> + ok; + Error -> + Error + end. + +-spec list_schemas() -> #{schema_name() => schema()}. +list_schemas() -> + emqx_config:get([?CONF_KEY_ROOT, schemas], #{}). + +%%------------------------------------------------------------------------------------------------- +%% `emqx_config_handler' API +%%------------------------------------------------------------------------------------------------- + +post_config_update( + [?CONF_KEY_ROOT, schemas] = _Path, + _Cmd, + NewConf = #{schemas := NewSchemas}, + OldConf = #{}, + _AppEnvs +) -> + OldSchemas = maps:get(schemas, OldConf, #{}), + #{ + added := Added, + changed := Changed0, + removed := Removed + } = emqx_utils_maps:diff_maps(NewSchemas, OldSchemas), + Changed = maps:map(fun(_N, {_Old, New}) -> New end, Changed0), + RemovedNames = maps:keys(Removed), + case RemovedNames of + [] -> + ok; + _ -> + async_delete_serdes(RemovedNames) + end, + SchemasToBuild = maps:to_list(maps:merge(Changed, Added)), + case build_serdes(SchemasToBuild) of + ok -> + {ok, NewConf}; + {error, Reason, SerdesToRollback} -> + lists:foreach(fun ensure_serde_absent/1, SerdesToRollback), + {error, Reason} + end; +post_config_update(_Path, _Cmd, NewConf, _OldConf, _AppEnvs) -> + {ok, NewConf}. + +%%------------------------------------------------------------------------------------------------- +%% `gen_server' API +%%------------------------------------------------------------------------------------------------- + +init(_) -> + process_flag(trap_exit, true), + create_tables(), + Schemas = emqx_conf:get([?CONF_KEY_ROOT, schemas], #{}), + State = #{}, + {ok, State, {continue, {build_serdes, Schemas}}}. + +handle_continue({build_serdes, Schemas}, State) -> + do_build_serdes(Schemas), + {noreply, State}. + +handle_call(_Call, _From, State) -> + {reply, {error, unknown_call}, State}. + +handle_cast({delete_serdes, Names}, State) -> + lists:foreach(fun ensure_serde_absent/1, Names), + ?tp(schema_registry_serdes_deleted, #{}), + {noreply, State}; +handle_cast({build_serdes, Schemas}, State) -> + do_build_serdes(Schemas), + {noreply, State}; +handle_cast(_Cast, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +%%------------------------------------------------------------------------------------------------- +%% Internal fns +%%------------------------------------------------------------------------------------------------- + +create_tables() -> + ok = mria:create_table(?SERDE_TAB, [ + {type, ordered_set}, + {rlog_shard, ?SCHEMA_REGISTRY_SHARD}, + {storage, ram_copies}, + {record_name, serde}, + {attributes, record_info(fields, serde)} + ]), + ok = mria:create_table(?PROTOBUF_CACHE_TAB, [ + {type, set}, + {rlog_shard, ?SCHEMA_REGISTRY_SHARD}, + {storage, disc_only_copies}, + {record_name, protobuf_cache}, + {attributes, record_info(fields, protobuf_cache)} + ]), + ok = mria:wait_for_tables([?SERDE_TAB, ?PROTOBUF_CACHE_TAB]), + ok. + +do_build_serdes(Schemas) -> + %% TODO: use some kind of mutex to make each core build a + %% different serde to avoid duplicate work. Maybe ekka_locker? + maps:foreach(fun do_build_serde/2, Schemas), + ?tp(schema_registry_serdes_built, #{}). + +build_serdes(Serdes) -> + build_serdes(Serdes, []). + +build_serdes([{Name, Params} | Rest], Acc0) -> + Acc = [Name | Acc0], + case do_build_serde(Name, Params) of + ok -> + build_serdes(Rest, Acc); + {error, Error} -> + {error, Error, Acc} + end; +build_serdes([], _Acc) -> + ok. + +do_build_serde(Name0, #{type := Type, source := Source}) -> + try + Name = to_bin(Name0), + {Serializer, Deserializer, Destructor} = + emqx_ee_schema_registry_serde:make_serde(Type, Name, Source), + Serde = #serde{ + name = Name, + serializer = Serializer, + deserializer = Deserializer, + destructor = Destructor + }, + ok = mria:dirty_write(?SERDE_TAB, Serde), + ok + catch + Kind:Error:Stacktrace -> + ?SLOG( + error, + #{ + msg => "error_building_serde", + name => Name0, + type => Type, + kind => Kind, + error => Error, + stacktrace => Stacktrace + } + ), + {error, Error} + end. + +ensure_serde_absent(Name) -> + case get_serde(Name) of + {ok, #{destructor := Destructor}} -> + Destructor(), + ok = mria:dirty_delete(?SERDE_TAB, to_bin(Name)); + {error, not_found} -> + ok + end. + +async_delete_serdes(Names) -> + gen_server:cast(?MODULE, {delete_serdes, Names}). + +to_bin(A) when is_atom(A) -> atom_to_binary(A); +to_bin(B) when is_binary(B) -> B. + +-spec serde_to_map(serde()) -> serde_map(). +serde_to_map(#serde{} = Serde) -> + #{ + name => Serde#serde.name, + serializer => Serde#serde.serializer, + deserializer => Serde#serde.deserializer, + destructor => Serde#serde.destructor + }. diff --git a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_app.erl b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_app.erl new file mode 100644 index 000000000..e82ed95bd --- /dev/null +++ b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_app.erl @@ -0,0 +1,19 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_ee_schema_registry_app). + +-behaviour(application). + +-include("emqx_ee_schema_registry.hrl"). + +-export([start/2, stop/1]). + +start(_StartType, _StartArgs) -> + ok = mria_rlog:wait_for_shards([?SCHEMA_REGISTRY_SHARD], infinity), + emqx_conf:add_handler(?CONF_KEY_PATH, emqx_ee_schema_registry), + emqx_ee_schema_registry_sup:start_link(). + +stop(_State) -> + emqx_conf:remove_handler(?CONF_KEY_PATH), + ok. diff --git a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_http_api.erl b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_http_api.erl new file mode 100644 index 000000000..c7cc6c99a --- /dev/null +++ b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_http_api.erl @@ -0,0 +1,251 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_ee_schema_registry_http_api). + +-behaviour(minirest_api). + +-include("emqx_ee_schema_registry.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx_utils/include/emqx_utils_api.hrl"). + +-export([ + namespace/0, + api_spec/0, + paths/0, + schema/1 +]). + +-export([ + '/schema_registry'/2, + '/schema_registry/:name'/2 +]). + +%%------------------------------------------------------------------------------------------------- +%% `minirest' and `minirest_trails' API +%%------------------------------------------------------------------------------------------------- + +namespace() -> "schema_registry_http_api". + +api_spec() -> + emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). + +paths() -> + [ + "/schema_registry", + "/schema_registry/:name" + ]. + +schema("/schema_registry") -> + #{ + 'operationId' => '/schema_registry', + get => #{ + tags => [<<"schema_registry">>], + summary => <<"List registered schemas">>, + description => ?DESC("desc_schema_registry_api_list"), + responses => + #{ + 200 => + emqx_dashboard_swagger:schema_with_examples( + hoconsc:array(emqx_ee_schema_registry_schema:api_schema("get")), + #{ + sample => + #{value => sample_list_schemas_response()} + } + ) + } + }, + post => #{ + tags => [<<"schema_registry">>], + summary => <<"Register a new schema">>, + description => ?DESC("desc_schema_registry_api_post"), + 'requestBody' => emqx_dashboard_swagger:schema_with_examples( + emqx_ee_schema_registry_schema:api_schema("post"), + post_examples() + ), + responses => + #{ + 201 => + emqx_dashboard_swagger:schema_with_examples( + emqx_ee_schema_registry_schema:api_schema("post"), + post_examples() + ), + 400 => error_schema('ALREADY_EXISTS', "Schema already exists") + } + } + }; +schema("/schema_registry/:name") -> + #{ + 'operationId' => '/schema_registry/:name', + get => #{ + tags => [<<"schema_registry">>], + summary => <<"Get registered schema">>, + description => ?DESC("desc_schema_registry_api_get"), + parameters => [param_path_schema_name()], + responses => + #{ + 200 => + emqx_dashboard_swagger:schema_with_examples( + emqx_ee_schema_registry_schema:api_schema("get"), + get_examples() + ), + 404 => error_schema('NOT_FOUND', "Schema not found") + } + }, + put => #{ + tags => [<<"schema_registry">>], + summary => <<"Update a schema">>, + description => ?DESC("desc_schema_registry_api_put"), + parameters => [param_path_schema_name()], + 'requestBody' => emqx_dashboard_swagger:schema_with_examples( + emqx_ee_schema_registry_schema:api_schema("put"), + post_examples() + ), + responses => + #{ + 200 => + emqx_dashboard_swagger:schema_with_examples( + emqx_ee_schema_registry_schema:api_schema("put"), + put_examples() + ), + 404 => error_schema('NOT_FOUND', "Schema not found") + } + }, + delete => #{ + tags => [<<"schema_registry">>], + summary => <<"Delete registered schema">>, + description => ?DESC("desc_schema_registry_api_delete"), + parameters => [param_path_schema_name()], + responses => + #{ + 204 => <<"Schema deleted">>, + 404 => error_schema('NOT_FOUND', "Schema not found") + } + } + }. + +%%------------------------------------------------------------------------------------------------- +%% API +%%------------------------------------------------------------------------------------------------- + +'/schema_registry'(get, _Params) -> + Schemas = emqx_ee_schema_registry:list_schemas(), + Response = + lists:map( + fun({Name, Params}) -> + Params#{name => Name} + end, + maps:to_list(Schemas) + ), + ?OK(Response); +'/schema_registry'(post, #{body := Params0 = #{<<"name">> := Name}}) -> + Params = maps:without([<<"name">>], Params0), + case emqx_ee_schema_registry:get_schema(Name) of + {error, not_found} -> + case emqx_ee_schema_registry:add_schema(Name, Params) of + ok -> + {ok, Res} = emqx_ee_schema_registry:get_schema(Name), + {201, Res#{name => Name}}; + {error, Error} -> + ?BAD_REQUEST(Error) + end; + {ok, _} -> + ?BAD_REQUEST('ALREADY_EXISTS', <<"Schema already exists">>) + end. + +'/schema_registry/:name'(get, #{bindings := #{name := Name}}) -> + case emqx_ee_schema_registry:get_schema(Name) of + {error, not_found} -> + ?NOT_FOUND(<<"Schema not found">>); + {ok, Schema} -> + ?OK(Schema#{name => Name}) + end; +'/schema_registry/:name'(put, #{bindings := #{name := Name}, body := Params}) -> + case emqx_ee_schema_registry:get_schema(Name) of + {error, not_found} -> + ?NOT_FOUND(<<"Schema not found">>); + {ok, _} -> + case emqx_ee_schema_registry:add_schema(Name, Params) of + ok -> + {ok, Res} = emqx_ee_schema_registry:get_schema(Name), + ?OK(Res#{name => Name}); + {error, Error} -> + ?BAD_REQUEST(Error) + end + end; +'/schema_registry/:name'(delete, #{bindings := #{name := Name}}) -> + case emqx_ee_schema_registry:get_schema(Name) of + {error, not_found} -> + ?NOT_FOUND(<<"Schema not found">>); + {ok, _} -> + case emqx_ee_schema_registry:delete_schema(Name) of + ok -> + ?NO_CONTENT; + {error, Error} -> + ?BAD_REQUEST(Error) + end + end. + +%%------------------------------------------------------------------------------------------------- +%% Examples +%%------------------------------------------------------------------------------------------------- + +sample_list_schemas_response() -> + [sample_get_schema_response(avro)]. + +sample_get_schema_response(avro) -> + #{ + type => <<"avro">>, + name => <<"my_avro_schema">>, + description => <<"My Avro Schema">>, + source => << + "{\"type\":\"record\"," + "\"fields\":[{\"type\":\"int\",\"name\":\"i\"}," + "{\"type\":\"string\",\"name\":\"s\"}]}" + >> + }. + +put_examples() -> + post_examples(). + +post_examples() -> + get_examples(). + +get_examples() -> + #{ + <<"avro_schema">> => + #{ + summary => <<"Avro">>, + value => sample_get_schema_response(avro) + } + }. + +%%------------------------------------------------------------------------------------------------- +%% Schemas and hocon types +%%------------------------------------------------------------------------------------------------- + +param_path_schema_name() -> + {name, + mk( + binary(), + #{ + in => path, + required => true, + example => <<"my_schema">>, + desc => ?DESC("desc_param_path_schema_name") + } + )}. + +%%------------------------------------------------------------------------------------------------- +%% Internal fns +%%------------------------------------------------------------------------------------------------- + +mk(Type, Meta) -> hoconsc:mk(Type, Meta). + +error_schema(Code, Message) when is_atom(Code) -> + error_schema([Code], Message); +error_schema(Codes, Message) when is_list(Message) -> + error_schema(Codes, list_to_binary(Message)); +error_schema(Codes, Message) when is_list(Codes) andalso is_binary(Message) -> + emqx_dashboard_swagger:error_codes(Codes, Message). diff --git a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_schema.erl b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_schema.erl new file mode 100644 index 000000000..237ec706f --- /dev/null +++ b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_schema.erl @@ -0,0 +1,143 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_ee_schema_registry_schema). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). +-include("emqx_ee_schema_registry.hrl"). + +%% `hocon_schema' API +-export([ + roots/0, + fields/1, + desc/1, + tags/0, + union_member_selector/1 +]). + +%% `minirest_trails' API +-export([ + api_schema/1 +]). + +%%------------------------------------------------------------------------------ +%% `hocon_schema' APIs +%%------------------------------------------------------------------------------ + +roots() -> + [{?CONF_KEY_ROOT, mk(ref(?CONF_KEY_ROOT), #{required => false})}]. + +tags() -> + [<<"Schema Registry">>]. + +fields(?CONF_KEY_ROOT) -> + [ + {schemas, + mk( + hoconsc:map( + name, + hoconsc:union(fun union_member_selector/1) + ), + #{ + default => #{}, + desc => ?DESC("schema_registry_schemas") + } + )} + ]; +fields(avro) -> + [ + {type, mk(avro, #{required => true, desc => ?DESC("schema_type")})}, + {source, + mk(emqx_schema:json_binary(), #{required => true, desc => ?DESC("schema_source")})}, + {description, mk(binary(), #{default => <<>>, desc => ?DESC("schema_description")})} + ]; +fields(protobuf) -> + [ + {type, mk(protobuf, #{required => true, desc => ?DESC("schema_type")})}, + {source, mk(binary(), #{required => true, desc => ?DESC("schema_source")})}, + {description, mk(binary(), #{default => <<>>, desc => ?DESC("schema_description")})} + ]; +fields("get_avro") -> + [{name, mk(binary(), #{required => true, desc => ?DESC("schema_name")})} | fields(avro)]; +fields("get_protobuf") -> + [{name, mk(binary(), #{required => true, desc => ?DESC("schema_name")})} | fields(protobuf)]; +fields("put_avro") -> + fields(avro); +fields("put_protobuf") -> + fields(protobuf); +fields("post_" ++ Type) -> + fields("get_" ++ Type). + +desc(?CONF_KEY_ROOT) -> + ?DESC("schema_registry_root"); +desc(avro) -> + ?DESC("avro_type"); +desc(protobuf) -> + ?DESC("protobuf_type"); +desc(_) -> + undefined. + +union_member_selector(all_union_members) -> + refs(); +union_member_selector({value, V}) -> + refs(V). + +union_member_selector_get_api(all_union_members) -> + refs_get_api(); +union_member_selector_get_api({value, V}) -> + refs_get_api(V). + +%%------------------------------------------------------------------------------ +%% `minirest_trails' "APIs" +%%------------------------------------------------------------------------------ + +api_schema("get") -> + hoconsc:union(fun union_member_selector_get_api/1); +api_schema("post") -> + api_schema("get"); +api_schema("put") -> + hoconsc:union(fun union_member_selector/1). + +%%------------------------------------------------------------------------------ +%% Internal fns +%%------------------------------------------------------------------------------ + +mk(Type, Meta) -> hoconsc:mk(Type, Meta). +ref(Name) -> hoconsc:ref(?MODULE, Name). + +supported_serde_types() -> + [avro, protobuf]. + +refs() -> + [ref(Type) || Type <- supported_serde_types()]. + +refs(#{<<"type">> := TypeAtom} = Value) when is_atom(TypeAtom) -> + refs(Value#{<<"type">> := atom_to_binary(TypeAtom)}); +refs(#{<<"type">> := <<"avro">>}) -> + [ref(avro)]; +refs(#{<<"type">> := <<"protobuf">>}) -> + [ref(protobuf)]; +refs(_) -> + Expected = lists:join(" | ", [atom_to_list(T) || T <- supported_serde_types()]), + throw(#{ + field_name => type, + expected => Expected + }). + +refs_get_api() -> + [ref("get_avro"), ref("get_protobuf")]. + +refs_get_api(#{<<"type">> := TypeAtom} = Value) when is_atom(TypeAtom) -> + refs(Value#{<<"type">> := atom_to_binary(TypeAtom)}); +refs_get_api(#{<<"type">> := <<"avro">>}) -> + [ref("get_avro")]; +refs_get_api(#{<<"type">> := <<"protobuf">>}) -> + [ref("get_protobuf")]; +refs_get_api(_) -> + Expected = lists:join(" | ", [atom_to_list(T) || T <- supported_serde_types()]), + throw(#{ + field_name => type, + expected => Expected + }). diff --git a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_serde.erl b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_serde.erl new file mode 100644 index 000000000..c65574032 --- /dev/null +++ b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_serde.erl @@ -0,0 +1,202 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_ee_schema_registry_serde). + +-include("emqx_ee_schema_registry.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-elvis([{elvis_style, invalid_dynamic_call, #{ignore => [emqx_ee_schema_registry_serde]}}]). + +%% API +-export([ + decode/2, + decode/3, + encode/2, + encode/3, + make_serde/3 +]). + +%%------------------------------------------------------------------------------ +%% API +%%------------------------------------------------------------------------------ + +-spec decode(schema_name(), encoded_data()) -> decoded_data(). +decode(SerdeName, RawData) -> + decode(SerdeName, RawData, []). + +-spec decode(schema_name(), encoded_data(), [term()]) -> decoded_data(). +decode(SerdeName, RawData, VarArgs) when is_list(VarArgs) -> + case emqx_ee_schema_registry:get_serde(SerdeName) of + {error, not_found} -> + error({serde_not_found, SerdeName}); + {ok, #{deserializer := Deserializer}} -> + apply(Deserializer, [RawData | VarArgs]) + end. + +-spec encode(schema_name(), decoded_data()) -> encoded_data(). +encode(SerdeName, RawData) -> + encode(SerdeName, RawData, []). + +-spec encode(schema_name(), decoded_data(), [term()]) -> encoded_data(). +encode(SerdeName, EncodedData, VarArgs) when is_list(VarArgs) -> + case emqx_ee_schema_registry:get_serde(SerdeName) of + {error, not_found} -> + error({serde_not_found, SerdeName}); + {ok, #{serializer := Serializer}} -> + apply(Serializer, [EncodedData | VarArgs]) + end. + +-spec make_serde(serde_type(), schema_name(), schema_source()) -> + {serializer(), deserializer(), destructor()}. +make_serde(avro, Name, Source0) -> + Source = inject_avro_name(Name, Source0), + Serializer = avro:make_simple_encoder(Source, _Opts = []), + Deserializer = avro:make_simple_decoder(Source, [{map_type, map}, {record_type, map}]), + Destructor = fun() -> + ?tp(serde_destroyed, #{type => avro, name => Name}), + ok + end, + {Serializer, Deserializer, Destructor}; +make_serde(protobuf, Name, Source) -> + SerdeMod = make_protobuf_serde_mod(Name, Source), + Serializer = + fun(DecodedData0, MessageName0) -> + DecodedData = emqx_utils_maps:safe_atom_key_map(DecodedData0), + MessageName = binary_to_existing_atom(MessageName0, utf8), + SerdeMod:encode_msg(DecodedData, MessageName) + end, + Deserializer = + fun(EncodedData, MessageName0) -> + MessageName = binary_to_existing_atom(MessageName0, utf8), + Decoded = SerdeMod:decode_msg(EncodedData, MessageName), + emqx_utils_maps:binary_key_map(Decoded) + end, + Destructor = + fun() -> + unload_code(SerdeMod), + ?tp(serde_destroyed, #{type => protobuf, name => Name}), + ok + end, + {Serializer, Deserializer, Destructor}. + +%%------------------------------------------------------------------------------ +%% Internal fns +%%------------------------------------------------------------------------------ + +-spec inject_avro_name(schema_name(), schema_source()) -> schema_source(). +inject_avro_name(Name, Source0) -> + %% The schema checks that the source is a valid JSON when + %% typechecking, so we shouldn't need to validate here. + Schema0 = emqx_utils_json:decode(Source0, [return_maps]), + Schema = Schema0#{<<"name">> => Name}, + emqx_utils_json:encode(Schema). + +-spec make_protobuf_serde_mod(schema_name(), schema_source()) -> module(). +make_protobuf_serde_mod(Name, Source) -> + {SerdeMod0, SerdeModFileName} = protobuf_serde_mod_name(Name), + case lazy_generate_protobuf_code(SerdeMod0, Source) of + {ok, SerdeMod, ModBinary} -> + load_code(SerdeMod, SerdeModFileName, ModBinary), + SerdeMod; + {error, #{error := Error, warnings := Warnings}} -> + ?SLOG( + warning, + #{ + msg => "error_generating_protobuf_code", + error => Error, + warnings => Warnings + } + ), + error({invalid_protobuf_schema, Error}) + end. + +-spec protobuf_serde_mod_name(schema_name()) -> {module(), string()}. +protobuf_serde_mod_name(Name) -> + %% must be a string (list) + SerdeModName = "$schema_parser_" ++ binary_to_list(Name), + SerdeMod = list_to_atom(SerdeModName), + %% the "path" to the module, for `code:load_binary'. + SerdeModFileName = SerdeModName ++ ".memory", + {SerdeMod, SerdeModFileName}. + +-spec lazy_generate_protobuf_code(module(), schema_source()) -> + {ok, module(), binary()} | {error, #{error := term(), warnings := [term()]}}. +lazy_generate_protobuf_code(SerdeMod0, Source) -> + %% We run this inside a transaction with locks to avoid running + %% the compile on all nodes; only one will get the lock, compile + %% the schema, and other nodes will simply read the final result. + {atomic, Res} = mria:transaction( + ?SCHEMA_REGISTRY_SHARD, + fun lazy_generate_protobuf_code_trans/2, + [SerdeMod0, Source] + ), + Res. + +-spec lazy_generate_protobuf_code_trans(module(), schema_source()) -> + {ok, module(), binary()} | {error, #{error := term(), warnings := [term()]}}. +lazy_generate_protobuf_code_trans(SerdeMod0, Source) -> + Fingerprint = erlang:md5(Source), + _ = mnesia:lock({record, ?PROTOBUF_CACHE_TAB, Fingerprint}, write), + case mnesia:read(?PROTOBUF_CACHE_TAB, Fingerprint) of + [#protobuf_cache{module = SerdeMod, module_binary = ModBinary}] -> + ?tp(schema_registry_protobuf_cache_hit, #{}), + {ok, SerdeMod, ModBinary}; + [] -> + ?tp(schema_registry_protobuf_cache_miss, #{}), + case generate_protobuf_code(SerdeMod0, Source) of + {ok, SerdeMod, ModBinary} -> + CacheEntry = #protobuf_cache{ + fingerprint = Fingerprint, + module = SerdeMod, + module_binary = ModBinary + }, + ok = mnesia:write(?PROTOBUF_CACHE_TAB, CacheEntry, write), + {ok, SerdeMod, ModBinary}; + {ok, SerdeMod, ModBinary, _Warnings} -> + CacheEntry = #protobuf_cache{ + fingerprint = Fingerprint, + module = SerdeMod, + module_binary = ModBinary + }, + ok = mnesia:write(?PROTOBUF_CACHE_TAB, CacheEntry, write), + {ok, SerdeMod, ModBinary}; + error -> + {error, #{error => undefined, warnings => []}}; + {error, Error} -> + {error, #{error => Error, warnings => []}}; + {error, Error, Warnings} -> + {error, #{error => Error, warnings => Warnings}} + end + end. + +generate_protobuf_code(SerdeMod, Source) -> + gpb_compile:string( + SerdeMod, + Source, + [ + binary, + strings_as_binaries, + {maps, true}, + %% Fixme: currently, some bug in `gpb' prevents this + %% option from working with `oneof' types... We're then + %% forced to use atom key maps. + %% {maps_key_type, binary}, + {maps_oneof, flat}, + {verify, always}, + {maps_unset_optional, omitted} + ] + ). + +-spec load_code(module(), string(), binary()) -> ok. +load_code(SerdeMod, SerdeModFileName, ModBinary) -> + _ = code:purge(SerdeMod), + {module, SerdeMod} = code:load_binary(SerdeMod, SerdeModFileName, ModBinary), + ok. + +-spec unload_code(module()) -> ok. +unload_code(SerdeMod) -> + _ = code:purge(SerdeMod), + _ = code:delete(SerdeMod), + ok. diff --git a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_sup.erl b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_sup.erl new file mode 100644 index 000000000..0dfc601d3 --- /dev/null +++ b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_sup.erl @@ -0,0 +1,43 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_ee_schema_registry_sup). + +-behaviour(supervisor). + +-export([start_link/0]). + +-export([init/1]). + +-define(SERVER, ?MODULE). + +start_link() -> + supervisor:start_link({local, ?SERVER}, ?MODULE, []). + +%% sup_flags() = #{strategy => strategy(), % optional +%% intensity => non_neg_integer(), % optional +%% period => pos_integer()} % optional +%% child_spec() = #{id => child_id(), % mandatory +%% start => mfargs(), % mandatory +%% restart => restart(), % optional +%% shutdown => shutdown(), % optional +%% type => worker(), % optional +%% modules => modules()} % optional +init([]) -> + SupFlags = #{ + strategy => one_for_one, + intensity => 10, + period => 100 + }, + ChildSpecs = [child_spec(emqx_ee_schema_registry)], + {ok, {SupFlags, ChildSpecs}}. + +child_spec(Mod) -> + #{ + id => Mod, + start => {Mod, start_link, []}, + restart => permanent, + shutdown => 5_000, + type => worker, + modules => [Mod] + }. diff --git a/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl new file mode 100644 index 000000000..7ad01fa06 --- /dev/null +++ b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl @@ -0,0 +1,689 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_ee_schema_registry_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-include("emqx_ee_schema_registry.hrl"). + +-import(emqx_common_test_helpers, [on_exit/1]). + +-define(APPS, [emqx_conf, emqx_rule_engine, emqx_ee_schema_registry]). + +%%------------------------------------------------------------------------------ +%% CT boilerplate +%%------------------------------------------------------------------------------ + +all() -> + [{group, avro}, {group, protobuf}]. + +groups() -> + AllTCs = emqx_common_test_helpers:all(?MODULE), + ProtobufOnlyTCs = protobuf_only_tcs(), + TCs = AllTCs -- ProtobufOnlyTCs, + [{avro, TCs}, {protobuf, AllTCs}]. + +protobuf_only_tcs() -> + [ + t_protobuf_union_encode, + t_protobuf_union_decode + ]. + +init_per_suite(Config) -> + emqx_config:save_schema_mod_and_names(emqx_ee_schema_registry_schema), + emqx_mgmt_api_test_util:init_suite(?APPS), + Config. + +end_per_suite(_Config) -> + emqx_mgmt_api_test_util:end_suite(lists:reverse(?APPS)), + ok. + +init_per_group(avro, Config) -> + [{serde_type, avro} | Config]; +init_per_group(protobuf, Config) -> + [{serde_type, protobuf} | Config]; +init_per_group(_Group, Config) -> + Config. + +end_per_group(_Group, _Config) -> + ok. + +init_per_testcase(_TestCase, Config) -> + ok = snabbkaffe:start_trace(), + Config. + +end_per_testcase(_TestCase, _Config) -> + ok = snabbkaffe:stop(), + emqx_common_test_helpers:call_janitor(), + clear_schemas(), + ok. + +%%------------------------------------------------------------------------------ +%% Helper fns +%%------------------------------------------------------------------------------ + +trace_rule(Data, Envs, _Args) -> + Now = erlang:monotonic_time(), + ets:insert(recorded_actions, {Now, #{data => Data, envs => Envs}}), + TestPid = persistent_term:get({?MODULE, test_pid}), + TestPid ! {action, #{data => Data, envs => Envs}}, + ok. + +make_trace_fn_action() -> + persistent_term:put({?MODULE, test_pid}, self()), + Fn = <<(atom_to_binary(?MODULE))/binary, ":trace_rule">>, + emqx_utils_ets:new(recorded_actions, [named_table, public, ordered_set]), + #{function => Fn, args => #{}}. + +create_rule_http(RuleParams) -> + RepublishTopic = <<"republish/schema_registry">>, + emqx:subscribe(RepublishTopic), + DefaultParams = #{ + enable => true, + actions => [ + make_trace_fn_action(), + #{ + <<"function">> => <<"republish">>, + <<"args">> => + #{ + <<"topic">> => RepublishTopic, + <<"payload">> => <<>>, + <<"qos">> => 0, + <<"retain">> => false, + <<"user_properties">> => <<>> + } + } + ] + }, + Params = maps:merge(DefaultParams, RuleParams), + Path = emqx_mgmt_api_test_util:api_path(["rules"]), + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of + {ok, Res0} -> + Res = #{<<"id">> := RuleId} = emqx_utils_json:decode(Res0, [return_maps]), + on_exit(fun() -> ok = emqx_rule_engine:delete_rule(RuleId) end), + {ok, Res}; + Error -> + Error + end. + +schema_params(avro) -> + Source = #{ + type => record, + fields => [ + #{name => <<"i">>, type => <<"int">>}, + #{name => <<"s">>, type => <<"string">>} + ] + }, + SourceBin = emqx_utils_json:encode(Source), + #{type => avro, source => SourceBin}; +schema_params(protobuf) -> + SourceBin = + << + "message Person {\n" + " required string name = 1;\n" + " required int32 id = 2;\n" + " optional string email = 3;\n" + " }\n" + "message UnionValue {\n" + " oneof u {\n" + " int32 a = 1;\n" + " string b = 2;\n" + " }\n" + "}\n" + >>, + #{type => protobuf, source => SourceBin}. + +create_serde(SerdeType, SerdeName) -> + Schema = schema_params(SerdeType), + ok = emqx_ee_schema_registry:add_schema(SerdeName, Schema), + ok. + +test_params_for(avro, encode_decode1) -> + SQL = + << + "select\n" + " schema_decode('my_serde',\n" + " schema_encode('my_serde', json_decode(payload))) as decoded,\n\n" + " decoded.i as decoded_int,\n" + " decoded.s as decoded_string\n" + "from t\n" + >>, + Payload = #{<<"i">> => 10, <<"s">> => <<"text">>}, + ExpectedRuleOutput = + #{ + <<"decoded">> => #{<<"i">> => 10, <<"s">> => <<"text">>}, + <<"decoded_int">> => 10, + <<"decoded_string">> => <<"text">> + }, + ExtraArgs = [], + #{ + sql => SQL, + payload => Payload, + expected_rule_output => ExpectedRuleOutput, + extra_args => ExtraArgs + }; +test_params_for(avro, encode1) -> + SQL = + << + "select\n" + " schema_encode('my_serde', json_decode(payload)) as encoded\n" + "from t\n" + >>, + Payload = #{<<"i">> => 10, <<"s">> => <<"text">>}, + ExtraArgs = [], + #{ + sql => SQL, + payload => Payload, + extra_args => ExtraArgs + }; +test_params_for(avro, decode1) -> + SQL = + << + "select\n" + " schema_decode('my_serde', payload) as decoded\n" + "from t\n" + >>, + Payload = #{<<"i">> => 10, <<"s">> => <<"text">>}, + ExtraArgs = [], + #{ + sql => SQL, + payload => Payload, + extra_args => ExtraArgs + }; +test_params_for(protobuf, encode_decode1) -> + SQL = + << + "select\n" + " schema_decode('my_serde',\n" + " schema_encode('my_serde', json_decode(payload), 'Person'),\n" + " 'Person') as decoded,\n" + " decoded.name as decoded_name,\n" + " decoded.email as decoded_email,\n" + " decoded.id as decoded_id\n" + "from t\n" + >>, + Payload = #{<<"name">> => <<"some name">>, <<"id">> => 10, <<"email">> => <<"emqx@emqx.io">>}, + ExpectedRuleOutput = + #{ + <<"decoded">> => + #{ + <<"email">> => <<"emqx@emqx.io">>, + <<"id">> => 10, + <<"name">> => <<"some name">> + }, + <<"decoded_email">> => <<"emqx@emqx.io">>, + <<"decoded_id">> => 10, + <<"decoded_name">> => <<"some name">> + }, + ExtraArgs = [<<"Person">>], + #{ + sql => SQL, + payload => Payload, + extra_args => ExtraArgs, + expected_rule_output => ExpectedRuleOutput + }; +test_params_for(protobuf, decode1) -> + SQL = + << + "select\n" + " schema_decode('my_serde', payload, 'Person') as decoded\n" + "from t\n" + >>, + Payload = #{<<"name">> => <<"some name">>, <<"id">> => 10, <<"email">> => <<"emqx@emqx.io">>}, + ExtraArgs = [<<"Person">>], + #{ + sql => SQL, + payload => Payload, + extra_args => ExtraArgs + }; +test_params_for(protobuf, encode1) -> + SQL = + << + "select\n" + " schema_encode('my_serde', json_decode(payload), 'Person') as encoded\n" + "from t\n" + >>, + Payload = #{<<"name">> => <<"some name">>, <<"id">> => 10, <<"email">> => <<"emqx@emqx.io">>}, + ExtraArgs = [<<"Person">>], + #{ + sql => SQL, + payload => Payload, + extra_args => ExtraArgs + }; +test_params_for(protobuf, union1) -> + SQL = + << + "select\n" + " schema_decode('my_serde', payload, 'UnionValue') as decoded,\n" + " decoded.a as decoded_a,\n" + " decoded.b as decoded_b\n" + "from t\n" + >>, + PayloadA = #{<<"a">> => 10}, + PayloadB = #{<<"b">> => <<"string">>}, + ExtraArgs = [<<"UnionValue">>], + #{ + sql => SQL, + payload => #{a => PayloadA, b => PayloadB}, + extra_args => ExtraArgs + }; +test_params_for(protobuf, union2) -> + SQL = + << + "select\n" + " schema_encode('my_serde', json_decode(payload), 'UnionValue') as encoded\n" + "from t\n" + >>, + PayloadA = #{<<"a">> => 10}, + PayloadB = #{<<"b">> => <<"string">>}, + ExtraArgs = [<<"UnionValue">>], + #{ + sql => SQL, + payload => #{a => PayloadA, b => PayloadB}, + extra_args => ExtraArgs + }; +test_params_for(Type, Name) -> + ct:fail("unimplemented: ~p", [{Type, Name}]). + +clear_schemas() -> + maps:foreach( + fun(Name, _Schema) -> + ok = emqx_ee_schema_registry:delete_schema(Name) + end, + emqx_ee_schema_registry:list_schemas() + ). + +receive_action_results() -> + receive + {action, #{data := _} = Res} -> + Res + after 1_000 -> + ct:fail("action didn't run") + end. + +receive_published(Line) -> + receive + {deliver, _Topic, Msg} -> + MsgMap = emqx_message:to_map(Msg), + maps:update_with( + payload, + fun(Raw) -> + case emqx_utils_json:safe_decode(Raw, [return_maps]) of + {ok, Decoded} -> Decoded; + {error, _} -> Raw + end + end, + MsgMap + ) + after 1_000 -> + ct:pal("mailbox: ~p", [process_info(self(), messages)]), + ct:fail("publish not received, line ~b", [Line]) + end. + +cluster(Config) -> + PrivDataDir = ?config(priv_dir, Config), + PeerModule = + case os:getenv("IS_CI") of + false -> + slave; + _ -> + ct_slave + end, + Cluster = emqx_common_test_helpers:emqx_cluster( + [core, core], + [ + {apps, ?APPS}, + {listener_ports, []}, + {peer_mod, PeerModule}, + {priv_data_dir, PrivDataDir}, + {load_schema, true}, + {start_autocluster, true}, + {schema_mod, emqx_ee_conf_schema}, + %% need to restart schema registry app in the tests so + %% that it re-registers the config handler that is lost + %% when emqx_conf restarts during join. + {env, [{emqx_machine, applications, [emqx_ee_schema_registry]}]}, + {load_apps, [emqx_machine | ?APPS]}, + {env_handler, fun + (emqx) -> + application:set_env(emqx, boot_modules, [broker, router]), + ok; + (emqx_conf) -> + ok; + (_) -> + ok + end} + ] + ), + ct:pal("cluster:\n ~p", [Cluster]), + Cluster. + +start_cluster(Cluster) -> + Nodes = [ + emqx_common_test_helpers:start_slave(Name, Opts) + || {Name, Opts} <- Cluster + ], + on_exit(fun() -> + emqx_utils:pmap( + fun(N) -> + ct:pal("stopping ~p", [N]), + ok = emqx_common_test_helpers:stop_slave(N) + end, + Nodes + ) + end), + erpc:multicall(Nodes, mria_rlog, wait_for_shards, [[?SCHEMA_REGISTRY_SHARD], 30_000]), + Nodes. + +wait_for_cluster_rpc(Node) -> + %% need to wait until the config handler is ready after + %% restarting during the cluster join. + ?retry( + _Sleep0 = 100, + _Attempts0 = 50, + true = is_pid(erpc:call(Node, erlang, whereis, [emqx_config_handler])) + ). + +serde_deletion_calls_destructor_spec(#{serde_type := SerdeType}, Trace) -> + ?assert( + ?strict_causality( + #{?snk_kind := will_delete_schema}, + #{?snk_kind := serde_destroyed, type := SerdeType}, + Trace + ) + ), + ok. + +protobuf_unique_cache_hit_spec(#{serde_type := protobuf} = Res, Trace) -> + #{nodes := Nodes} = Res, + CacheEvents = ?of_kind( + [ + schema_registry_protobuf_cache_hit, + schema_registry_protobuf_cache_miss + ], + Trace + ), + ?assertMatch( + [ + schema_registry_protobuf_cache_hit, + schema_registry_protobuf_cache_miss + ], + lists:sort(?projection(?snk_kind, CacheEvents)) + ), + ?assertEqual( + lists:usort(Nodes), + lists:usort([N || #{?snk_meta := #{node := N}} <- CacheEvents]) + ), + ok; +protobuf_unique_cache_hit_spec(_Res, _Trace) -> + ok. + +%%------------------------------------------------------------------------------ +%% Testcases +%%------------------------------------------------------------------------------ + +t_unknown_calls(_Config) -> + Ref = monitor(process, emqx_ee_schema_registry), + %% for coverage + emqx_ee_schema_registry ! unknown, + gen_server:cast(emqx_ee_schema_registry, unknown), + ?assertEqual({error, unknown_call}, gen_server:call(emqx_ee_schema_registry, unknown)), + receive + {'DOWN', Ref, process, _, _} -> + ct:fail("registry shouldn't have died") + after 500 -> + ok + end. + +t_encode_decode(Config) -> + SerdeType = ?config(serde_type, Config), + SerdeName = my_serde, + ok = create_serde(SerdeType, SerdeName), + #{ + sql := SQL, + payload := Payload, + expected_rule_output := ExpectedRuleOutput + } = test_params_for(SerdeType, encode_decode1), + {ok, _} = create_rule_http(#{sql => SQL}), + PayloadBin = emqx_utils_json:encode(Payload), + emqx:publish(emqx_message:make(<<"t">>, PayloadBin)), + Res = receive_action_results(), + ?assertMatch(#{data := ExpectedRuleOutput}, Res), + ok. + +t_delete_serde(Config) -> + SerdeType = ?config(serde_type, Config), + SerdeName = my_serde, + ?check_trace( + begin + ok = create_serde(SerdeType, SerdeName), + {ok, {ok, _}} = + ?wait_async_action( + emqx_ee_schema_registry:delete_schema(SerdeName), + #{?snk_kind := schema_registry_serdes_deleted}, + 1_000 + ), + ok + end, + fun(Trace) -> + ?assertMatch([_], ?of_kind(schema_registry_serdes_deleted, Trace)), + ?assertMatch([#{type := SerdeType}], ?of_kind(serde_destroyed, Trace)), + ok + end + ), + ok. + +t_encode(Config) -> + SerdeType = ?config(serde_type, Config), + SerdeName = my_serde, + ok = create_serde(SerdeType, SerdeName), + #{ + sql := SQL, + payload := Payload, + extra_args := ExtraArgs + } = test_params_for(SerdeType, encode1), + {ok, _} = create_rule_http(#{sql => SQL}), + PayloadBin = emqx_utils_json:encode(Payload), + emqx:publish(emqx_message:make(<<"t">>, PayloadBin)), + Published = receive_published(?LINE), + ?assertMatch( + #{payload := #{<<"encoded">> := _}}, + Published + ), + #{payload := #{<<"encoded">> := Encoded}} = Published, + {ok, #{deserializer := Deserializer}} = emqx_ee_schema_registry:get_serde(SerdeName), + ?assertEqual(Payload, apply(Deserializer, [Encoded | ExtraArgs])), + ok. + +t_decode(Config) -> + SerdeType = ?config(serde_type, Config), + SerdeName = my_serde, + ok = create_serde(SerdeType, SerdeName), + #{ + sql := SQL, + payload := Payload, + extra_args := ExtraArgs + } = test_params_for(SerdeType, decode1), + {ok, _} = create_rule_http(#{sql => SQL}), + {ok, #{serializer := Serializer}} = emqx_ee_schema_registry:get_serde(SerdeName), + EncodedBin = apply(Serializer, [Payload | ExtraArgs]), + emqx:publish(emqx_message:make(<<"t">>, EncodedBin)), + Published = receive_published(?LINE), + ?assertMatch( + #{payload := #{<<"decoded">> := _}}, + Published + ), + #{payload := #{<<"decoded">> := Decoded}} = Published, + ?assertEqual(Payload, Decoded), + ok. + +t_protobuf_union_encode(Config) -> + SerdeType = ?config(serde_type, Config), + ?assertEqual(protobuf, SerdeType), + SerdeName = my_serde, + ok = create_serde(SerdeType, SerdeName), + #{ + sql := SQL, + payload := #{a := PayloadA, b := PayloadB}, + extra_args := ExtraArgs + } = test_params_for(SerdeType, union1), + {ok, _} = create_rule_http(#{sql => SQL}), + {ok, #{serializer := Serializer}} = emqx_ee_schema_registry:get_serde(SerdeName), + + EncodedBinA = apply(Serializer, [PayloadA | ExtraArgs]), + emqx:publish(emqx_message:make(<<"t">>, EncodedBinA)), + PublishedA = receive_published(?LINE), + ?assertMatch( + #{payload := #{<<"decoded">> := _}}, + PublishedA + ), + #{payload := #{<<"decoded">> := DecodedA}} = PublishedA, + ?assertEqual(PayloadA, DecodedA), + + EncodedBinB = apply(Serializer, [PayloadB | ExtraArgs]), + emqx:publish(emqx_message:make(<<"t">>, EncodedBinB)), + PublishedB = receive_published(?LINE), + ?assertMatch( + #{payload := #{<<"decoded">> := _}}, + PublishedB + ), + #{payload := #{<<"decoded">> := DecodedB}} = PublishedB, + ?assertEqual(PayloadB, DecodedB), + + ok. + +t_protobuf_union_decode(Config) -> + SerdeType = ?config(serde_type, Config), + ?assertEqual(protobuf, SerdeType), + SerdeName = my_serde, + ok = create_serde(SerdeType, SerdeName), + #{ + sql := SQL, + payload := #{a := PayloadA, b := PayloadB}, + extra_args := ExtraArgs + } = test_params_for(SerdeType, union2), + {ok, _} = create_rule_http(#{sql => SQL}), + {ok, #{deserializer := Deserializer}} = emqx_ee_schema_registry:get_serde(SerdeName), + + EncodedBinA = emqx_utils_json:encode(PayloadA), + emqx:publish(emqx_message:make(<<"t">>, EncodedBinA)), + PublishedA = receive_published(?LINE), + ?assertMatch( + #{payload := #{<<"encoded">> := _}}, + PublishedA + ), + #{payload := #{<<"encoded">> := EncodedA}} = PublishedA, + ?assertEqual(PayloadA, apply(Deserializer, [EncodedA | ExtraArgs])), + + EncodedBinB = emqx_utils_json:encode(PayloadB), + emqx:publish(emqx_message:make(<<"t">>, EncodedBinB)), + PublishedB = receive_published(?LINE), + ?assertMatch( + #{payload := #{<<"encoded">> := _}}, + PublishedB + ), + #{payload := #{<<"encoded">> := EncodedB}} = PublishedB, + ?assertEqual(PayloadB, apply(Deserializer, [EncodedB | ExtraArgs])), + + ok. + +t_fail_rollback(Config) -> + SerdeType = ?config(serde_type, Config), + OkSchema = emqx_utils_maps:binary_key_map(schema_params(SerdeType)), + BrokenSchema = OkSchema#{<<"source">> := <<"{}">>}, + %% hopefully, for this small map, the key order is used. + Serdes = #{ + <<"a">> => OkSchema, + <<"z">> => BrokenSchema + }, + ?assertMatch( + {error, _}, + emqx_conf:update( + [?CONF_KEY_ROOT, schemas], + Serdes, + #{} + ) + ), + %% no serdes should be in the table + ?assertEqual({error, not_found}, emqx_ee_schema_registry:get_serde(<<"a">>)), + ?assertEqual({error, not_found}, emqx_ee_schema_registry:get_serde(<<"z">>)), + ok. + +t_cluster_serde_build(Config) -> + SerdeType = ?config(serde_type, Config), + Cluster = cluster(Config), + SerdeName = my_serde, + Schema = schema_params(SerdeType), + #{ + payload := Payload, + extra_args := ExtraArgs + } = test_params_for(SerdeType, encode_decode1), + ?check_trace( + begin + Nodes = [N1, N2 | _] = start_cluster(Cluster), + NumNodes = length(Nodes), + wait_for_cluster_rpc(N2), + ?assertEqual( + ok, + erpc:call(N2, emqx_ee_schema_registry, add_schema, [SerdeName, Schema]) + ), + %% check that we can serialize/deserialize in all nodes + lists:foreach( + fun(N) -> + erpc:call(N, fun() -> + Res0 = emqx_ee_schema_registry:get_serde(SerdeName), + ?assertMatch({ok, #{}}, Res0, #{node => N}), + {ok, #{serializer := Serializer, deserializer := Deserializer}} = Res0, + ?assertEqual( + Payload, + apply( + Deserializer, + [apply(Serializer, [Payload | ExtraArgs]) | ExtraArgs] + ), + #{node => N} + ), + ok + end) + end, + Nodes + ), + %% now we delete and check it's removed from the table + ?tp(will_delete_schema, #{}), + {ok, SRef1} = snabbkaffe:subscribe( + ?match_event(#{?snk_kind := schema_registry_serdes_deleted}), + NumNodes, + 5_000 + ), + ?assertEqual( + ok, + erpc:call(N1, emqx_ee_schema_registry, delete_schema, [SerdeName]) + ), + {ok, _} = snabbkaffe:receive_events(SRef1), + lists:foreach( + fun(N) -> + erpc:call(N, fun() -> + ?assertMatch( + {error, not_found}, + emqx_ee_schema_registry:get_serde(SerdeName), + #{node => N} + ), + ok + end) + end, + Nodes + ), + #{serde_type => SerdeType, nodes => Nodes} + end, + [ + {"destructor is always called", fun ?MODULE:serde_deletion_calls_destructor_spec/2}, + {"protobuf is only built on one node", fun ?MODULE:protobuf_unique_cache_hit_spec/2} + ] + ), + ok. diff --git a/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_http_api_SUITE.erl b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_http_api_SUITE.erl new file mode 100644 index 000000000..ee6a693db --- /dev/null +++ b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_http_api_SUITE.erl @@ -0,0 +1,298 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_ee_schema_registry_http_api_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-import(emqx_mgmt_api_test_util, [uri/1]). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-define(APPS, [emqx_conf, emqx_ee_schema_registry]). + +%%------------------------------------------------------------------------------ +%% CT boilerplate +%%------------------------------------------------------------------------------ + +all() -> + [ + {group, avro}, + {group, protobuf} + ]. + +groups() -> + AllTCs = emqx_common_test_helpers:all(?MODULE), + [ + {avro, AllTCs}, + {protobuf, AllTCs} + ]. + +init_per_suite(Config) -> + emqx_config:save_schema_mod_and_names(emqx_ee_schema_registry_schema), + emqx_mgmt_api_test_util:init_suite(?APPS), + Config. + +end_per_suite(_Config) -> + emqx_mgmt_api_test_util:end_suite(lists:reverse(?APPS)), + ok. + +init_per_group(avro, Config) -> + Source = #{ + type => record, + fields => [ + #{name => <<"i">>, type => <<"int">>}, + #{name => <<"s">>, type => <<"string">>} + ] + }, + SourceBin = emqx_utils_json:encode(Source), + InvalidSourceBin = <<"{}">>, + [ + {serde_type, avro}, + {schema_source, SourceBin}, + {invalid_schema_source, InvalidSourceBin} + | Config + ]; +init_per_group(protobuf, Config) -> + SourceBin = + << + "message Person {\n" + " required string name = 1;\n" + " required int32 id = 2;\n" + " optional string email = 3;\n" + " }\n" + "message UnionValue {\n" + " oneof u {\n" + " int32 a = 1;\n" + " string b = 2;\n" + " }\n" + "}\n" + >>, + InvalidSourceBin = <<"xxxx">>, + [ + {serde_type, protobuf}, + {schema_source, SourceBin}, + {invalid_schema_source, InvalidSourceBin} + | Config + ]. + +end_per_group(_Group, _Config) -> + ok. + +init_per_testcase(_TestCase, Config) -> + clear_schemas(), + ok = snabbkaffe:start_trace(), + Config. + +end_per_testcase(_TestCase, _Config) -> + clear_schemas(), + ok = snabbkaffe:stop(), + ok. + +%%------------------------------------------------------------------------------ +%% Helper fns +%%------------------------------------------------------------------------------ + +request(get) -> + do_request(get, _Parts = [], _Body = []); +request({get, Name}) -> + do_request(get, _Parts = [Name], _Body = []); +request({delete, Name}) -> + do_request(delete, _Parts = [Name], _Body = []); +request({put, Name, Params}) -> + do_request(put, _Parts = [Name], Params); +request({post, Params}) -> + do_request(post, _Parts = [], Params). + +do_request(Method, PathParts, Body) -> + Header = emqx_common_test_http:default_auth_header(), + URI = uri(["schema_registry" | PathParts]), + Opts = #{compatible_mode => true, httpc_req_opts => [{body_format, binary}]}, + Res0 = emqx_mgmt_api_test_util:request_api(Method, URI, [], Header, Body, Opts), + case Res0 of + {ok, Code, <<>>} -> + {ok, Code, <<>>}; + {ok, Code, Res1} -> + Res2 = emqx_utils_json:decode(Res1, [return_maps]), + Res3 = try_decode_error_message(Res2), + {ok, Code, Res3}; + Error -> + Error + end. + +try_decode_error_message(#{<<"message">> := Msg0} = Res0) -> + case emqx_utils_json:safe_decode(Msg0, [return_maps]) of + {ok, Msg} -> + Res0#{<<"message">> := Msg}; + {error, _} -> + Res0 + end; +try_decode_error_message(Res) -> + Res. + +clear_schemas() -> + maps:foreach( + fun(Name, _Schema) -> + ok = emqx_ee_schema_registry:delete_schema(Name) + end, + emqx_ee_schema_registry:list_schemas() + ). + +%%------------------------------------------------------------------------------ +%% Testcases +%%------------------------------------------------------------------------------ + +t_crud(Config) -> + SerdeType = ?config(serde_type, Config), + SourceBin = ?config(schema_source, Config), + InvalidSourceBin = ?config(invalid_schema_source, Config), + SerdeTypeBin = atom_to_binary(SerdeType), + SchemaName = <<"my_schema">>, + Params = #{ + <<"type">> => SerdeTypeBin, + <<"source">> => SourceBin, + <<"name">> => SchemaName, + <<"description">> => <<"My schema">> + }, + UpdateParams = maps:without([<<"name">>], Params), + + %% no schemas at first + ?assertMatch({ok, 200, []}, request(get)), + ?assertMatch( + {ok, 404, #{ + <<"code">> := <<"NOT_FOUND">>, + <<"message">> := <<"Schema not found">> + }}, + request({get, SchemaName}) + ), + ?assertMatch( + {ok, 404, #{ + <<"code">> := <<"NOT_FOUND">>, + <<"message">> := <<"Schema not found">> + }}, + request({put, SchemaName, UpdateParams}) + ), + ?assertMatch( + {ok, 404, #{ + <<"code">> := <<"NOT_FOUND">>, + <<"message">> := <<"Schema not found">> + }}, + request({delete, SchemaName}) + ), + + %% create a schema + ?assertMatch( + {ok, 201, #{ + <<"type">> := SerdeTypeBin, + <<"source">> := SourceBin, + <<"name">> := SchemaName, + <<"description">> := <<"My schema">> + }}, + request({post, Params}) + ), + ?assertMatch( + {ok, 200, #{ + <<"type">> := SerdeTypeBin, + <<"source">> := SourceBin, + <<"name">> := SchemaName, + <<"description">> := <<"My schema">> + }}, + request({get, SchemaName}) + ), + ?assertMatch( + {ok, 200, [ + #{ + <<"type">> := SerdeTypeBin, + <<"source">> := SourceBin, + <<"name">> := SchemaName, + <<"description">> := <<"My schema">> + } + ]}, + request(get) + ), + UpdateParams1 = UpdateParams#{<<"description">> := <<"My new schema">>}, + ?assertMatch( + {ok, 200, #{ + <<"type">> := SerdeTypeBin, + <<"source">> := SourceBin, + <<"name">> := SchemaName, + <<"description">> := <<"My new schema">> + }}, + request({put, SchemaName, UpdateParams1}) + ), + + ?assertMatch( + {ok, 400, #{ + <<"code">> := <<"ALREADY_EXISTS">>, + <<"message">> := <<"Schema already exists">> + }}, + request({post, Params}) + ), + %% typechecks, but is invalid + ?assertMatch( + {ok, 400, #{ + <<"code">> := <<"BAD_REQUEST">>, + <<"message">> := + <<"{post_config_update,emqx_ee_schema_registry,", _/binary>> + }}, + request({put, SchemaName, UpdateParams#{<<"source">> := InvalidSourceBin}}) + ), + + ?assertMatch( + {ok, 204, <<>>}, + request({delete, SchemaName}) + ), + + %% doesn't typecheck + lists:foreach( + fun(Field) -> + ?assertMatch( + {ok, 400, #{ + <<"code">> := <<"BAD_REQUEST">>, + <<"message">> := #{<<"reason">> := <<"required_field">>} + }}, + request({post, maps:without([Field], Params)}), + #{field => Field} + ) + end, + [<<"name">>, <<"source">>] + ), + ?assertMatch( + {ok, 400, #{ + <<"code">> := <<"BAD_REQUEST">>, + <<"message">> := + #{ + <<"expected">> := [_ | _], + <<"field_name">> := <<"type">> + } + }}, + request({post, maps:without([<<"type">>], Params)}), + #{field => <<"type">>} + ), + %% typechecks, but is invalid + ?assertMatch( + {ok, 400, #{ + <<"code">> := <<"BAD_REQUEST">>, + <<"message">> := + <<"{post_config_update,emqx_ee_schema_registry,", _/binary>> + }}, + request({post, Params#{<<"source">> := InvalidSourceBin}}) + ), + + %% unknown serde type + ?assertMatch( + {ok, 400, #{ + <<"code">> := <<"BAD_REQUEST">>, + <<"message">> := + #{ + <<"expected">> := [_ | _], + <<"field_name">> := <<"type">> + } + }}, + request({post, Params#{<<"type">> := <<"foo">>}}) + ), + + ok. diff --git a/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_serde_SUITE.erl b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_serde_SUITE.erl new file mode 100644 index 000000000..1ab5e3c01 --- /dev/null +++ b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_serde_SUITE.erl @@ -0,0 +1,172 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_ee_schema_registry_serde_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-include("emqx_ee_schema_registry.hrl"). + +-import(emqx_common_test_helpers, [on_exit/1]). + +-define(APPS, [emqx_conf, emqx_rule_engine, emqx_ee_schema_registry]). + +%%------------------------------------------------------------------------------ +%% CT boilerplate +%%------------------------------------------------------------------------------ + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + emqx_config:save_schema_mod_and_names(emqx_ee_schema_registry_schema), + emqx_mgmt_api_test_util:init_suite(?APPS), + Config. + +end_per_suite(_Config) -> + emqx_mgmt_api_test_util:end_suite(lists:reverse(?APPS)), + ok. +init_per_testcase(_TestCase, Config) -> + Config. + +end_per_testcase(_TestCase, _Config) -> + emqx_common_test_helpers:call_janitor(), + clear_schemas(), + ok. + +%%------------------------------------------------------------------------------ +%% Helper fns +%%------------------------------------------------------------------------------ + +clear_schemas() -> + maps:foreach( + fun(Name, _Schema) -> + ok = emqx_ee_schema_registry:delete_schema(Name) + end, + emqx_ee_schema_registry:list_schemas() + ). + +schema_params(avro) -> + Source = #{ + type => record, + fields => [ + #{name => <<"i">>, type => <<"int">>}, + #{name => <<"s">>, type => <<"string">>} + ] + }, + SourceBin = emqx_utils_json:encode(Source), + #{type => avro, source => SourceBin}; +schema_params(protobuf) -> + SourceBin = + << + "\n" + " message Person {\n" + " required string name = 1;\n" + " required int32 id = 2;\n" + " optional string email = 3;\n" + " }\n" + " message UnionValue {\n" + " oneof u {\n" + " int32 a = 1;\n" + " string b = 2;\n" + " }\n" + " }\n" + " " + >>, + #{type => protobuf, source => SourceBin}. + +assert_roundtrip(SerdeName, Original) -> + Encoded = emqx_ee_schema_registry_serde:encode(SerdeName, Original), + Decoded = emqx_ee_schema_registry_serde:decode(SerdeName, Encoded), + ?assertEqual(Original, Decoded, #{original => Original}). + +assert_roundtrip(SerdeName, Original, ArgsSerialize, ArgsDeserialize) -> + Encoded = emqx_ee_schema_registry_serde:encode(SerdeName, Original, ArgsSerialize), + Decoded = emqx_ee_schema_registry_serde:decode(SerdeName, Encoded, ArgsDeserialize), + ?assertEqual(Original, Decoded, #{original => Original}). + +%%------------------------------------------------------------------------------ +%% Testcases +%%------------------------------------------------------------------------------ + +t_roundtrip_avro(_Config) -> + SerdeName = my_serde, + Params = schema_params(avro), + ok = emqx_ee_schema_registry:add_schema(SerdeName, Params), + Original = #{<<"i">> => 10, <<"s">> => <<"hi">>}, + %% for coverage + assert_roundtrip(SerdeName, Original, _ArgsSerialize = [], _ArgsDeserialize = []), + assert_roundtrip(SerdeName, Original), + ok. + +t_avro_invalid_json_schema(_Config) -> + SerdeName = my_serde, + Params = schema_params(avro), + WrongParams = Params#{source := <<"{">>}, + ?assertMatch( + {error, #{reason := #{expected_type := _}}}, + emqx_ee_schema_registry:add_schema(SerdeName, WrongParams) + ), + ok. + +t_avro_invalid_schema(_Config) -> + SerdeName = my_serde, + Params = schema_params(avro), + WrongParams = Params#{source := <<"{}">>}, + ?assertMatch( + {error, {post_config_update, _, {not_found, <<"type">>}}}, + emqx_ee_schema_registry:add_schema(SerdeName, WrongParams) + ), + ok. + +t_serde_not_found(_Config) -> + %% for coverage + NonexistentSerde = <<"nonexistent">>, + Original = #{}, + ?assertError( + {serde_not_found, NonexistentSerde}, + emqx_ee_schema_registry_serde:encode(NonexistentSerde, Original) + ), + ?assertError( + {serde_not_found, NonexistentSerde}, + emqx_ee_schema_registry_serde:decode(NonexistentSerde, Original) + ), + ok. + +t_roundtrip_protobuf(_Config) -> + SerdeName = my_serde, + Params = schema_params(protobuf), + ok = emqx_ee_schema_registry:add_schema(SerdeName, Params), + ExtraArgsPerson = [<<"Person">>], + + Original0 = #{<<"name">> => <<"some name">>, <<"id">> => 10, <<"email">> => <<"emqx@emqx.io">>}, + assert_roundtrip(SerdeName, Original0, ExtraArgsPerson, ExtraArgsPerson), + + %% removing optional field + Original1 = #{<<"name">> => <<"some name">>, <<"id">> => 10}, + assert_roundtrip(SerdeName, Original1, ExtraArgsPerson, ExtraArgsPerson), + + %% `oneof' fields + ExtraArgsUnion = [<<"UnionValue">>], + Original2 = #{<<"a">> => 1}, + assert_roundtrip(SerdeName, Original2, ExtraArgsUnion, ExtraArgsUnion), + + Original3 = #{<<"b">> => <<"string">>}, + assert_roundtrip(SerdeName, Original3, ExtraArgsUnion, ExtraArgsUnion), + + ok. + +t_protobuf_invalid_schema(_Config) -> + SerdeName = my_serde, + Params = schema_params(protobuf), + WrongParams = Params#{source := <<"xxxx">>}, + ?assertMatch( + {error, {post_config_update, _, {invalid_protobuf_schema, _}}}, + emqx_ee_schema_registry:add_schema(SerdeName, WrongParams) + ), + ok. diff --git a/lib-ee/emqx_license/rebar.config b/lib-ee/emqx_license/rebar.config index 1cb5ace88..e7620387a 100644 --- a/lib-ee/emqx_license/rebar.config +++ b/lib-ee/emqx_license/rebar.config @@ -1,3 +1,6 @@ -{deps, [{emqx, {path, "../../apps/emqx"}}]}. +{deps, [ + {emqx, {path, "../../apps/emqx"}}, + {emqx_utils, {path, "../emqx_utils"}} +]}. {project_plugins, [erlfmt]}. diff --git a/lib-ee/emqx_license/src/emqx_license.app.src b/lib-ee/emqx_license/src/emqx_license.app.src index 0a97ee83b..fcdcbc05b 100644 --- a/lib-ee/emqx_license/src/emqx_license.app.src +++ b/lib-ee/emqx_license/src/emqx_license.app.src @@ -1,6 +1,6 @@ {application, emqx_license, [ {description, "EMQX License"}, - {vsn, "5.0.8"}, + {vsn, "5.0.9"}, {modules, []}, {registered, [emqx_license_sup]}, {applications, [kernel, stdlib, emqx_ctl]}, diff --git a/lib-ee/emqx_license/src/emqx_license_http_api.erl b/lib-ee/emqx_license/src/emqx_license_http_api.erl index 0133cd637..ef0e1ee52 100644 --- a/lib-ee/emqx_license/src/emqx_license_http_api.erl +++ b/lib-ee/emqx_license/src/emqx_license_http_api.erl @@ -95,7 +95,7 @@ sample_license_info_response() -> }. error_msg(Code, Msg) -> - #{code => Code, message => emqx_misc:readable_error_msg(Msg)}. + #{code => Code, message => emqx_utils:readable_error_msg(Msg)}. %% read license info '/license'(get, _Params) -> diff --git a/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl b/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl index 11ac1ca89..3de5ae121 100644 --- a/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl +++ b/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl @@ -103,7 +103,7 @@ t_license_info(_Config) -> <<"start_at">> => <<"2022-01-11">>, <<"type">> => <<"trial">> }, - emqx_json:decode(Payload, [return_maps]) + emqx_utils_json:decode(Payload, [return_maps]) ), ok. @@ -128,7 +128,7 @@ t_license_upload_key_success(_Config) -> <<"start_at">> => <<"2022-01-11">>, <<"type">> => <<"trial">> }, - emqx_json:decode(Payload, [return_maps]) + emqx_utils_json:decode(Payload, [return_maps]) ), ?assertMatch( #{max_connections := 999}, @@ -150,7 +150,7 @@ t_license_upload_key_bad_key(_Config) -> <<"code">> => <<"BAD_REQUEST">>, <<"message">> => <<"Bad license key">> }, - emqx_json:decode(Payload, [return_maps]) + emqx_utils_json:decode(Payload, [return_maps]) ), assert_untouched_license(), ok. @@ -168,7 +168,7 @@ t_license_upload_key_not_json(_Config) -> <<"code">> => <<"BAD_REQUEST">>, <<"message">> => <<"Invalid request params">> }, - emqx_json:decode(Payload, [return_maps]) + emqx_utils_json:decode(Payload, [return_maps]) ), assert_untouched_license(), ok. diff --git a/mix.exs b/mix.exs index cd6d08fd1..f94755b09 100644 --- a/mix.exs +++ b/mix.exs @@ -58,7 +58,7 @@ defmodule EMQXUmbrella.MixProject do {:ekka, github: "emqx/ekka", tag: "0.14.6", override: true}, {:gen_rpc, github: "emqx/gen_rpc", tag: "2.8.1", override: true}, {:grpc, github: "emqx/grpc-erl", tag: "0.6.7", override: true}, - {:minirest, github: "emqx/minirest", tag: "1.3.9", override: true}, + {:minirest, github: "emqx/minirest", tag: "1.3.8", override: true}, {:ecpool, github: "emqx/ecpool", tag: "0.5.3", override: true}, {:replayq, github: "emqx/replayq", tag: "0.3.7", override: true}, {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true}, @@ -72,7 +72,7 @@ defmodule EMQXUmbrella.MixProject do # in conflict by emqtt and hocon {:getopt, "1.0.2", override: true}, {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.7", override: true}, - {:hocon, github: "emqx/hocon", tag: "0.38.0", override: true}, + {:hocon, github: "emqx/hocon", tag: "0.39.2", override: true}, {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.2", override: true}, {:esasl, github: "emqx/esasl", tag: "0.2.0"}, {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"}, @@ -83,6 +83,8 @@ defmodule EMQXUmbrella.MixProject do # in conflict by emqx and observer_cli {:recon, github: "ferd/recon", tag: "2.5.1", override: true}, {:jsx, github: "talentdeficit/jsx", tag: "v3.1.0", override: true}, + # in conflict by erlavro and rocketmq + {:jsone, github: "emqx/jsone", tag: "1.7.1", override: true}, # dependencies of dependencies; we choose specific refs to match # what rebar3 chooses. # in conflict by gun and emqtt @@ -92,27 +94,23 @@ defmodule EMQXUmbrella.MixProject do {:ranch, github: "ninenines/ranch", ref: "a692f44567034dacf5efcaa24a24183788594eb7", override: true}, # in conflict by grpc and eetcd - {:gpb, "4.19.5", override: true, runtime: false}, - {:hackney, github: "emqx/hackney", tag: "1.18.1-1", override: true}, - {:erlcloud, github: "emqx/erlcloud", tag: "3.6.8-emqx-1", override: true}, - # erlcloud's rebar.config requires rebar3 and does not support Mix, - # so it tries to fetch deps from git. We need to override this. - {:lhttpc, github: "erlcloud/lhttpc", tag: "1.6.2", override: true}, - {:eini, "1.2.9", override: true}, - {:base16, "1.0.0", override: true} - # end of erlcloud's deps + {:gpb, "4.19.7", override: true, runtime: false}, + {:hackney, github: "emqx/hackney", tag: "1.18.1-1", override: true} ] ++ emqx_apps(profile_info, version) ++ enterprise_deps(profile_info) ++ bcrypt_dep() ++ jq_dep() ++ quicer_dep() end defp emqx_apps(profile_info, version) do - apps = community_apps() ++ enterprise_apps(profile_info) + apps = umbrella_apps() ++ enterprise_apps(profile_info) set_emqx_app_system_env(apps, profile_info, version) end - defp community_apps() do - community_app_paths() + defp umbrella_apps() do + enterprise_apps = enterprise_umbrella_apps() + + "apps/*" + |> Path.wildcard() |> Enum.map(fn path -> app = path @@ -121,10 +119,23 @@ defmodule EMQXUmbrella.MixProject do {app, path: path, manager: :rebar3, override: true} end) + |> Enum.reject(fn dep_spec -> + dep_spec + |> elem(0) + |> then(&MapSet.member?(enterprise_apps, &1)) + end) end defp enterprise_apps(_profile_info = %{edition_type: :enterprise}) do - enterprise_app_paths() + umbrella_apps = + Enum.map(enterprise_umbrella_apps(), fn app_name -> + path = "apps/#{app_name}" + {app_name, path: path, manager: :rebar3, override: true} + end) + + "lib-ee/*" + |> Path.wildcard() + |> Enum.filter(&File.dir?/1) |> Enum.map(fn path -> app = path @@ -133,32 +144,34 @@ defmodule EMQXUmbrella.MixProject do {app, path: path, manager: :rebar3, override: true} end) + |> Enum.concat(umbrella_apps) end defp enterprise_apps(_profile_info) do [] end - defp community_app_paths() do - "apps/*" - |> Path.wildcard() - |> Enum.filter(&File.dir?/1) - |> Enum.reject(&File.exists?(Path.join(&1, "BSL.txt"))) - end - - defp enterprise_app_paths() do - lib_ee_paths = - "lib-ee/*" - |> Path.wildcard() - |> Enum.filter(&File.dir?/1) - - apps_paths = - "apps/*" - |> Path.wildcard() - |> Enum.filter(&File.dir?/1) - |> Enum.filter(&File.exists?(Path.join(&1, "BSL.txt"))) - - lib_ee_paths ++ apps_paths + # need to remove those when listing `/apps/`... + defp enterprise_umbrella_apps() do + MapSet.new([ + :emqx_bridge_kafka, + :emqx_bridge_gcp_pubsub, + :emqx_bridge_cassandra, + :emqx_bridge_clickhouse, + :emqx_bridge_dynamo, + :emqx_bridge_hstreamdb, + :emqx_bridge_influxdb, + :emqx_bridge_matrix, + :emqx_bridge_mongodb, + :emqx_bridge_mysql, + :emqx_bridge_pgsql, + :emqx_bridge_redis, + :emqx_bridge_rocketmq, + :emqx_bridge_tdengine, + :emqx_bridge_timescale, + :emqx_ft, + :emqx_s3 + ]) end defp enterprise_deps(_profile_info = %{edition_type: :enterprise}) do @@ -170,7 +183,8 @@ defmodule EMQXUmbrella.MixProject do {:brod_gssapi, github: "kafka4beam/brod_gssapi", tag: "v0.1.0-rc1"}, {:brod, github: "kafka4beam/brod", tag: "3.16.8"}, {:snappyer, "1.2.8", override: true}, - {:supervisor3, "1.1.11", override: true} + {:crc32cer, "0.1.8", override: true}, + {:supervisor3, "1.1.12", override: true} ] end @@ -247,6 +261,11 @@ defmodule EMQXUmbrella.MixProject do applications: applications(edition_type), skip_mode_validation_for: [ :emqx_gateway, + :emqx_gateway_stomp, + :emqx_gateway_mqttsn, + :emqx_gateway_coap, + :emqx_gateway_lwm2m, + :emqx_gateway_exproto, :emqx_dashboard, :emqx_resource, :emqx_connector, @@ -300,6 +319,7 @@ defmodule EMQXUmbrella.MixProject do tools: :load, covertool: :load, system_monitor: :load, + emqx_utils: :load, emqx_http_lib: :permanent, emqx_resource: :permanent, emqx_connector: :permanent, @@ -307,6 +327,11 @@ defmodule EMQXUmbrella.MixProject do emqx_authz: :permanent, emqx_auto_subscribe: :permanent, emqx_gateway: :permanent, + emqx_gateway_stomp: :permanent, + emqx_gateway_mqttsn: :permanent, + emqx_gateway_coap: :permanent, + emqx_gateway_lwm2m: :permanent, + emqx_gateway_exproto: :permanent, emqx_exhook: :permanent, emqx_bridge: :permanent, emqx_rule_engine: :permanent, @@ -334,8 +359,22 @@ defmodule EMQXUmbrella.MixProject do emqx_ee_conf: :load, emqx_ee_connector: :permanent, emqx_ee_bridge: :permanent, - emqx_s3: :permanent, - emqx_ft: :permanent + emqx_bridge_kafka: :permanent, + emqx_bridge_gcp_pubsub: :permanent, + emqx_bridge_cassandra: :permanent, + emqx_bridge_clickhouse: :permanent, + emqx_bridge_dynamo: :permanent, + emqx_bridge_hstreamdb: :permanent, + emqx_bridge_influxdb: :permanent, + emqx_bridge_matrix: :permanent, + emqx_bridge_mongodb: :permanent, + emqx_bridge_mysql: :permanent, + emqx_bridge_pgsql: :permanent, + emqx_bridge_redis: :permanent, + emqx_bridge_rocketmq: :permanent, + emqx_bridge_tdengine: :permanent, + emqx_bridge_timescale: :permanent, + emqx_ee_schema_registry: :permanent ], else: [] ) @@ -453,7 +492,7 @@ defmodule EMQXUmbrella.MixProject do profile = System.get_env("MIX_ENV") Mix.Generator.copy_file( - "_build/docgen/#{profile}/emqx.conf.en.example", + "_build/docgen/#{profile}/emqx.conf.example", Path.join(etc, "emqx.conf.example"), force: overwrite? ) diff --git a/rebar.config b/rebar.config index 908adefdb..de520f124 100644 --- a/rebar.config +++ b/rebar.config @@ -7,35 +7,24 @@ %% with rebar.config.erl module. Final result is written to %% rebar.config.rendered if environment DEBUG is set. -{edoc_opts, [{preprocess, true}]}. -{erl_opts, [ - warn_unused_vars, - warn_shadow_vars, - warn_unused_import, - warn_obsolete_guard, - compressed, - nowarn_unused_import, - {d, snk_kind, msg} -]}. +{edoc_opts, [{preprocess,true}]}. +{erl_opts, [warn_unused_vars,warn_shadow_vars,warn_unused_import, + warn_obsolete_guard,compressed, nowarn_unused_import, + {d, snk_kind, msg} + ]}. -{xref_checks, [ - undefined_function_calls, - undefined_functions, - locals_not_used, - deprecated_function_calls, - warnings_as_errors, - deprecated_functions -]}. +{xref_checks,[undefined_function_calls,undefined_functions,locals_not_used, + deprecated_function_calls,warnings_as_errors,deprecated_functions]}. %% Check the forbidden mnesia calls: -{xref_queries, [ - {"E || \"mnesia\":\"dirty_delete.*\"/\".*\" : Fun", []}, - {"E || \"mnesia\":\"transaction\"/\".*\" : Fun", []}, - {"E || \"mnesia\":\"async_dirty\"/\".*\" : Fun", []}, - {"E || \"mnesia\":\"clear_table\"/\".*\" : Fun", []}, - {"E || \"mnesia\":\"create_table\"/\".*\" : Fun", []}, - {"E || \"mnesia\":\"delete_table\"/\".*\" : Fun", []} -]}. +{xref_queries, + [ {"E || \"mnesia\":\"dirty_delete.*\"/\".*\" : Fun", []} + , {"E || \"mnesia\":\"transaction\"/\".*\" : Fun", []} + , {"E || \"mnesia\":\"async_dirty\"/\".*\" : Fun", []} + , {"E || \"mnesia\":\"clear_table\"/\".*\" : Fun", []} + , {"E || \"mnesia\":\"create_table\"/\".*\" : Fun", []} + , {"E || \"mnesia\":\"delete_table\"/\".*\" : Fun", []} + ]}. {dialyzer, [ {warnings, [unmatched_returns, error_handling]}, @@ -43,76 +32,74 @@ {plt_prefix, "emqx_dialyzer"}, {plt_apps, all_apps}, {statistics, true} -]}. + ] +}. {cover_opts, [verbose]}. {cover_export_enabled, true}. {cover_excl_mods, - %% generated protobuf modules - [ - emqx_exproto_pb, - emqx_exhook_pb, - %% taken almost as-is from OTP - emqx_ssl_crl_cache - ]}. + [ %% generated protobuf modules + emqx_exproto_pb, + emqx_exhook_pb, + %% taken almost as-is from OTP + emqx_ssl_crl_cache + ]}. {provider_hooks, [{pre, [{release, {relup_helper, gen_appups}}]}]}. -{post_hooks, []}. +{post_hooks,[]}. -{deps, [ - {lc, {git, "https://github.com/emqx/lc.git", {tag, "0.3.2"}}}, - {redbug, "2.0.8"}, - {covertool, {git, "https://github.com/zmstone/covertool", {tag, "2.0.4.1"}}}, - %% gpb only used to build, but not for release, pin it here to avoid fetching a wrong version due to rebar plugins scattered in all the deps - {gpb, "4.19.5"}, - {typerefl, {git, "https://github.com/ieQu1/typerefl", {tag, "0.9.1"}}}, - {gun, {git, "https://github.com/emqx/gun", {tag, "1.3.9"}}}, - {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.4.7"}}}, - {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}}, - {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}, - {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}}, - {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}}, - {rocksdb, {git, "https://github.com/emqx/erlang-rocksdb", {tag, "1.7.2-emqx-9"}}}, - {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.14.6"}}}, - {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}, - {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.7"}}}, - {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.3"}}}, - {replayq, {git, "https://github.com/emqx/replayq.git", {tag, "0.3.7"}}}, - {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.9"}}}, - {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, - {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.8.5"}}}, - {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.5"}}}, - % NOTE: depends on recon 2.5.x - {observer_cli, "1.7.1"}, - {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}}, - {getopt, "1.0.2"}, - {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.7"}}}, - {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.38.0"}}}, - {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}}, - {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}}, - {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}}, - {telemetry, "1.1.0"}, - {hackney, {git, "https://github.com/emqx/hackney.git", {tag, "1.18.1-1"}}} -]}. - -{xref_ignores, - %% schema registry is for enterprise - [ - {emqx_schema_registry, get_all_schemas, 0}, - {emqx_schema_api, format_schema, 1}, - {emqx_schema_api, make_schema_params, 1}, - {emqx_schema_parser, decode, 3}, - {emqx_schema_parser, encode, 3}, - {emqx_schema_registry, add_schema, 1}, - % generated code for protobuf - emqx_exhook_pb, - % generated code for protobuf - emqx_exproto_pb +{deps, + [ {lc, {git, "https://github.com/emqx/lc.git", {tag, "0.3.2"}}} + , {redbug, "2.0.8"} + , {covertool, {git, "https://github.com/zmstone/covertool", {tag, "2.0.4.1"}}} + , {gpb, "4.19.7"} + , {typerefl, {git, "https://github.com/ieQu1/typerefl", {tag, "0.9.1"}}} + , {gun, {git, "https://github.com/emqx/gun", {tag, "1.3.9"}}} + , {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.4.7"}}} + , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} + , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} + , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}} + , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}} + , {rocksdb, {git, "https://github.com/emqx/erlang-rocksdb", {tag, "1.7.2-emqx-9"}}} + , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.14.6"}}} + , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}} + , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.7"}}} + , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.8"}}} + , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.3"}}} + , {replayq, {git, "https://github.com/emqx/replayq.git", {tag, "0.3.7"}}} + , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} + , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.8.5"}}} + , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.5"}}} + , {observer_cli, "1.7.1"} % NOTE: depends on recon 2.5.x + , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}} + , {getopt, "1.0.2"} + , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.7"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.2"}}} + , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}} + , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} + , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}} + , {telemetry, "1.1.0"} + , {hackney, {git, "https://github.com/emqx/hackney.git", {tag, "1.18.1-1"}}} + %% in conflict by erlavro and rocketmq + , {jsone, {git, "https://github.com/emqx/jsone.git", {tag, "1.7.1"}}} ]}. -{project_plugins, [ - erlfmt, - {rebar3_hex, "7.0.2"}, - {rebar3_sbom, {git, "https://github.com/emqx/rebar3_sbom.git", {tag, "v0.6.1-1"}}} +{xref_ignores, + [ %% schema registry is for enterprise + {emqx_schema_registry,get_all_schemas,0}, + {emqx_schema_api,format_schema,1}, + {emqx_schema_api,make_schema_params,1}, + {emqx_schema_parser,decode,3}, + {emqx_schema_parser,encode,3}, + {emqx_schema_registry,add_schema,1}, + emqx_exhook_pb, % generated code for protobuf + emqx_exproto_pb % generated code for protobuf +]}. + +{project_plugins, + [ erlfmt, + {rebar3_hex, "7.0.2"}, + {rebar3_sbom, + {git, "https://github.com/emqx/rebar3_sbom.git", {tag, "v0.6.1-1"}}} ]}. diff --git a/rebar.config.erl b/rebar.config.erl index 158f66cd6..19b3631d0 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -78,6 +78,25 @@ is_cover_enabled() -> is_enterprise(ce) -> false; is_enterprise(ee) -> true. +is_community_umbrella_app("apps/emqx_bridge_kafka") -> false; +is_community_umbrella_app("apps/emqx_bridge_gcp_pubsub") -> false; +is_community_umbrella_app("apps/emqx_bridge_cassandra") -> false; +is_community_umbrella_app("apps/emqx_bridge_clickhouse") -> false; +is_community_umbrella_app("apps/emqx_bridge_dynamo") -> false; +is_community_umbrella_app("apps/emqx_bridge_hstreamdb") -> false; +is_community_umbrella_app("apps/emqx_bridge_influxdb") -> false; +is_community_umbrella_app("apps/emqx_bridge_matrix") -> false; +is_community_umbrella_app("apps/emqx_bridge_mongodb") -> false; +is_community_umbrella_app("apps/emqx_bridge_mysql") -> false; +is_community_umbrella_app("apps/emqx_bridge_pgsql") -> false; +is_community_umbrella_app("apps/emqx_bridge_redis") -> false; +is_community_umbrella_app("apps/emqx_bridge_rocketmq") -> false; +is_community_umbrella_app("apps/emqx_bridge_tdengine") -> false; +is_community_umbrella_app("apps/emqx_bridge_timescale") -> false; +is_community_umbrella_app("apps/emqx_ft") -> false; +is_community_umbrella_app("apps/emqx_s3") -> false; +is_community_umbrella_app(_) -> true. + is_jq_supported() -> not (false =/= os:getenv("BUILD_WITHOUT_JQ") orelse is_win32()) orelse @@ -122,8 +141,14 @@ project_app_dirs() -> project_app_dirs(get_edition_from_profile_env()). project_app_dirs(Edition) -> - ["apps/*"] ++ - case is_enterprise(Edition) of + IsEnterprise = is_enterprise(Edition), + UmbrellaApps = [ + Path + || Path <- filelib:wildcard("apps/*"), + is_community_umbrella_app(Path) orelse IsEnterprise + ], + UmbrellaApps ++ + case IsEnterprise of true -> ["lib-ee/*"]; false -> [] end. @@ -382,6 +407,7 @@ relx_apps(ReleaseType, Edition) -> {covertool, load}, % started by emqx_machine {system_monitor, load}, + {emqx_utils, load}, emqx_http_lib, emqx_resource, emqx_connector, @@ -389,6 +415,11 @@ relx_apps(ReleaseType, Edition) -> emqx_authz, emqx_auto_subscribe, emqx_gateway, + emqx_gateway_stomp, + emqx_gateway_mqttsn, + emqx_gateway_coap, + emqx_gateway_lwm2m, + emqx_gateway_exproto, emqx_exhook, emqx_bridge, emqx_rule_engine, @@ -423,7 +454,22 @@ relx_apps_per_edition(ee) -> {emqx_ee_conf, load}, emqx_ee_connector, emqx_ee_bridge, - emqx_s3, + emqx_bridge_kafka, + emqx_bridge_gcp_pubsub, + emqx_bridge_cassandra, + emqx_bridge_clickhouse, + emqx_bridge_dynamo, + emqx_bridge_hstreamdb, + emqx_bridge_influxdb, + emqx_bridge_matrix, + emqx_bridge_mongodb, + emqx_bridge_mysql, + emqx_bridge_pgsql, + emqx_bridge_redis, + emqx_bridge_rocketmq, + emqx_bridge_tdengine, + emqx_bridge_timescale, + emqx_ee_schema_registry, emqx_ft ]; relx_apps_per_edition(ce) -> @@ -451,7 +497,7 @@ relx_overlay(ReleaseType, Edition) -> {copy, "bin/emqx_ctl", "bin/emqx_ctl-{{release_version}}"}, %% for relup {copy, "bin/install_upgrade.escript", "bin/install_upgrade.escript-{{release_version}}"}, - {copy, "apps/emqx_gateway/src/lwm2m/lwm2m_xml", "etc/lwm2m_xml"}, + {copy, "apps/emqx_gateway_lwm2m/lwm2m_xml", "etc/lwm2m_xml"}, {copy, "apps/emqx_authz/etc/acl.conf", "etc/acl.conf"}, {template, "bin/emqx.cmd", "bin/emqx.cmd"}, {template, "bin/emqx_ctl.cmd", "bin/emqx_ctl.cmd"}, @@ -464,8 +510,7 @@ etc_overlay(ReleaseType, Edition) -> [ {mkdir, "etc/"}, {copy, "{{base_dir}}/lib/emqx/etc/certs", "etc/"}, - {copy, "_build/docgen/" ++ name(Edition) ++ "/emqx.conf.en.example", - "etc/emqx.conf.example"} + {copy, "_build/docgen/" ++ name(Edition) ++ "/emqx.conf.example", "etc/emqx.conf.example"} ] ++ lists:map( fun diff --git a/rel/emqx_conf.template.en.md b/rel/emqx_conf.template.en.md index 8740e4319..c1259869c 100644 --- a/rel/emqx_conf.template.en.md +++ b/rel/emqx_conf.template.en.md @@ -7,21 +7,27 @@ and a superset of JSON. EMQX configuration consists of two layers. From bottom up: -1. Immutable base: `emqx.conf` + `EMQX_` prefixed environment variables.
- Changes in this layer require a full node restart to take effect. -1. Cluster overrides: `$EMQX_NODE__DATA_DIR/configs/cluster-override.conf` +1. Cluster-synced configs: `$EMQX_NODE__DATA_DIR/configs/cluster.hocon`. +2. Local node configs: `emqx.conf` + `EMQX_` prefixed environment variables. + +:::tip Tip +Prior to v5.0.23 and e5.0.3, the cluster-synced configs are stored in +`cluster-override.conf` which is applied on top of the local configs. + +If upgraded from an earlier version, as long as `cluster-override.conf` exists, +`cluster.hocon` will not be created, and `cluster-override.conf` will stay on +top of the overriding layers. +::: When environment variable `$EMQX_NODE__DATA_DIR` is not set, config `node.data_dir` is used. -The `cluster-override.conf` file is overwritten at runtime when changes -are made from dashboard UI, management HTTP API, or CLI. When clustered, +The `cluster.hocon` file is overwritten at runtime when changes +are made from Dashboard, management HTTP API, or CLI. When clustered, after EMQX restarts, it copies the file from the node which has the greatest `uptime`. :::tip Tip -Some of the configs (such as `node.name`) are boot-only configs and not overridable. -Config values from `*-override.conf` are **not** mapped to boot configs for -the config fields attributed with `mapping: path.to.boot.config.key` +To avoid confusion, don't add the same keys in both `cluster.hocon` and `emqx.conf`. ::: For detailed override rules, see [Config Overlay Rules](#config-overlay-rules). @@ -144,11 +150,10 @@ For example, this environment variable sets an array value. ``` export EMQX_LISTENERS__SSL__L1__AUTHENTICATION__SSL__CIPHERS='["TLS_AES_256_GCM_SHA384"]' ``` - -However this also means a string value should be quoted if it happens to contain special +However, this also means a string value should be quoted if it happens to contain special characters such as `=` and `:`. -For example, a string value `"localhost:1883"` would be +For example, a string value `"localhost:1883"` would be parsed into object (struct): `{"localhost": 1883}`. To keep it as a string, one should quote the value like below: @@ -226,7 +231,7 @@ Arrays in EMQX config have two different representations Dot-separated paths with number in it are parsed to indexed-maps e.g. `authentication.1={...}` is parsed as `authentication={"1": {...}}` -This feature makes it easy to override array elment values. For example: +This feature makes it easy to override array element values. For example: ``` authentication=[{enable=true, backend="built_in_database", mechanism="password_based"}] @@ -322,4 +327,3 @@ ciphers = "PSK-AES128-CBC-SHA" ] ``` - diff --git a/rel/emqx_conf.template.zh.md b/rel/emqx_conf.template.zh.md index 9402760a2..a9df27f63 100644 --- a/rel/emqx_conf.template.zh.md +++ b/rel/emqx_conf.template.zh.md @@ -5,20 +5,24 @@ HOCON（Human-Optimized Config Object Notation）是一个JSON的超集，非常 EMQX的配置文件可分为二层，自底向上依次是： -1. 不可变的基础层 `emqx.conf` 加上 `EMQX_` 前缀的环境变量。
- 修改这一层的配置之后，需要重启节点来使之生效。 -1. 集群范围重载层：`$EMQX_NODE__DATA_DIR/configs/cluster-override.conf` +1. 集群同步配置：`$EMQX_NODE__DATA_DIR/configs/cluster.hocon`。 +2. 本地节点配置：`emqx.conf` 加上 `EMQX_` 前缀的环境变量。 + +:::tip Tip +在 v5.0.23 或 e5.0.3 之前，集群同步配置保存在文件 `cluster-override.conf` 中，并且它覆盖在配置的最上层。 + +如果从之前的版本升级上来，只要 `cluster-override.conf` 文件存在， +EMQX 就不会创建 `cluster.hocon`，并且 `cluster-override.conf` 会继续覆盖在配置的最上层。 +::: 如果环境变量 `$EMQX_NODE__DATA_DIR` 没有设置，那么该目录会从 `emqx.conf` 的 `node.data_dir` 配置中读取。 -配置文件 `cluster-override.conf` 的内容会在运行时被EMQX重写。 +配置文件 `cluster.hocon` 的内容会在运行时被EMQX重写。 这些重写发生在 dashboard UI，管理HTTP API，或者CLI对集群配置进行修改时。 当EMQX运行在集群中时，一个EMQX节点重启之后，会从集群中其他节点复制该文件内容到本地。 :::tip Tip -有些配置项是不能被重载的（例如 `node.name`）。 -配置项如果有 `mapping: path.to.boot.config.key` 这个属性， -则不能被添加到重载文件 `*-override.conf` 中。 +为避免歧义，应尽量避免让 `cluster.hocon` 和 `emqx.conf` 出现配置交集。 ::: 更多的重载规则，请参考下文 [配置重载规则](#配置重载规则)。 diff --git a/rel/i18n/emqx_authn_api.hocon b/rel/i18n/emqx_authn_api.hocon index a068edee2..07f9c6c3e 100644 --- a/rel/i18n/emqx_authn_api.hocon +++ b/rel/i18n/emqx_authn_api.hocon @@ -1,217 +1,96 @@ emqx_authn_api { - authentication_get { - desc { - en: """List authenticators for global authentication.""" - zh: """列出全局认证链上的认证器。""" - } - } +authentication_get.desc: +"""List authenticators for global authentication.""" - authentication_post { - desc { - en: """Create authenticator for global authentication.""" - zh: """为全局认证链创建认证器。""" - } - } +authentication_id_delete.desc: +"""Delete authenticator from global authentication chain.""" - authentication_id_get { - desc { - en: """Get authenticator from global authentication chain.""" - zh: """获取全局认证链上的指定认证器。""" - } - } +authentication_id_get.desc: +"""Get authenticator from global authentication chain.""" - authentication_id_put { - desc { - en: """Update authenticator from global authentication chain.""" - zh: """更新全局认证链上的指定认证器。""" - } - } +authentication_id_position_put.desc: +"""Move authenticator in global authentication chain.""" - authentication_id_delete { - desc { - en: """Delete authenticator from global authentication chain.""" - zh: """删除全局认证链上的指定认证器。""" - } - } +authentication_id_put.desc: +"""Update authenticator from global authentication chain.""" - authentication_id_status_get { - desc { - en: """Get authenticator status from global authentication chain.""" - zh: """获取全局认证链上指定认证器的状态。""" - } - } +authentication_id_status_get.desc: +"""Get authenticator status from global authentication chain.""" - listeners_listener_id_authentication_get { - desc { - en: """List authenticators for listener authentication.""" - zh: """列出监听器认证链上的认证器。""" - } - } +authentication_id_users_get.desc: +"""List users in authenticator in global authentication chain.""" - listeners_listener_id_authentication_post { - desc { - en: """Create authenticator for listener authentication.""" - zh: """在监听器认证链上创建认证器。""" - } - } +authentication_id_users_post.desc: +"""Create users for authenticator in global authentication chain.""" - listeners_listener_id_authentication_id_get { - desc { - en: """Get authenticator from listener authentication chain.""" - zh: """获取监听器认证链上的指定认证器。""" - } - } +authentication_id_users_user_id_delete.desc: +"""Delete user in authenticator in global authentication chain.""" - listeners_listener_id_authentication_id_put { - desc { - en: """Update authenticator from listener authentication chain.""" - zh: """更新监听器认证链上的指定认证器。""" - } - } +authentication_id_users_user_id_get.desc: +"""Get user from authenticator in global authentication chain.""" - listeners_listener_id_authentication_id_delete { - desc { - en: """Delete authenticator from listener authentication chain.""" - zh: """删除监听器认证链上的指定认证器。""" - } - } +authentication_id_users_user_id_put.desc: +"""Update user in authenticator in global authentication chain.""" - listeners_listener_id_authentication_id_status_get { - desc { - en: """Get authenticator status from listener authentication chain.""" - zh: """获取监听器认证链上指定认证器的状态。""" - } - } +authentication_post.desc: +"""Create authenticator for global authentication.""" - authentication_id_position_put { - desc { - en: """Move authenticator in global authentication chain.""" - zh: """更改全局认证链上指定认证器的顺序。""" - } - } +is_superuser.desc: +"""Is superuser""" - listeners_listener_id_authentication_id_position_put { - desc { - en: """Move authenticator in listener authentication chain.""" - zh: """更改监听器认证链上指定认证器的顺序。""" - } - } +like_user_id.desc: +"""Fuzzy search user_id (username or clientid).""" - authentication_id_users_post { - desc { - en: """Create users for authenticator in global authentication chain.""" - zh: """为全局认证链上的指定认证器创建用户数据。""" - } - } +like_user_id.label: +"""like_user_id""" - authentication_id_users_get { - desc { - en: """List users in authenticator in global authentication chain.""" - zh: """获取全局认证链上指定认证器中的用户数据。""" - } - } +listeners_listener_id_authentication_get.desc: +"""List authenticators for listener authentication.""" - listeners_listener_id_authentication_id_users_post { - desc { - en: """Create users for authenticator in listener authentication chain.""" - zh: """为监听器认证链上的指定认证器创建用户数据。""" - } - } +listeners_listener_id_authentication_id_delete.desc: +"""Delete authenticator from listener authentication chain.""" - listeners_listener_id_authentication_id_users_get { - desc { - en: """List users in authenticator in listener authentication chain.""" - zh: """列出监听器认证链上指定认证器中的用户数据。""" - } - } +listeners_listener_id_authentication_id_get.desc: +"""Get authenticator from listener authentication chain.""" - authentication_id_users_user_id_get { - desc { - en: """Get user from authenticator in global authentication chain.""" - zh: """获取全局认证链上指定认证器中的指定用户数据。""" - } - } +listeners_listener_id_authentication_id_position_put.desc: +"""Move authenticator in listener authentication chain.""" - authentication_id_users_user_id_put { - desc { - en: """Update user in authenticator in global authentication chain.""" - zh: """更新全局认证链上指定认证器中的指定用户数据。""" - } - } +listeners_listener_id_authentication_id_put.desc: +"""Update authenticator from listener authentication chain.""" - authentication_id_users_user_id_delete { - desc { - en: """Delete user in authenticator in global authentication chain.""" - zh: """删除全局认证链上指定认证器中的指定用户数据。""" - } - } +listeners_listener_id_authentication_id_status_get.desc: +"""Get authenticator status from listener authentication chain.""" - listeners_listener_id_authentication_id_users_user_id_get { - desc { - en: """Get user from authenticator in listener authentication chain.""" - zh: """获取监听器认证链上指定认证器中的指定用户数据。""" - } - } +listeners_listener_id_authentication_id_users_get.desc: +"""List users in authenticator in listener authentication chain.""" - listeners_listener_id_authentication_id_users_user_id_put { - desc { - en: """Update user in authenticator in listener authentication chain.""" - zh: """更新监听器认证链上指定认证器中的指定用户数据。""" - } - } +listeners_listener_id_authentication_id_users_post.desc: +"""Create users for authenticator in listener authentication chain.""" - listeners_listener_id_authentication_id_users_user_id_delete { - desc { - en: """Delete user in authenticator in listener authentication chain.""" - zh: """删除监听器认证链上指定认证器中的指定用户数据。""" - } - } +listeners_listener_id_authentication_id_users_user_id_delete.desc: +"""Delete user in authenticator in listener authentication chain.""" - param_auth_id { - desc { - en: """Authenticator ID.""" - zh: """认证器 ID。""" - } - } +listeners_listener_id_authentication_id_users_user_id_get.desc: +"""Get user from authenticator in listener authentication chain.""" - param_listener_id { - desc { - en: """Listener ID.""" - zh: """监听器 ID。""" - } - } +listeners_listener_id_authentication_id_users_user_id_put.desc: +"""Update user in authenticator in listener authentication chain.""" - param_user_id { - desc { - en: """User ID.""" - zh: """用户 ID。""" - } - } +listeners_listener_id_authentication_post.desc: +"""Create authenticator for listener authentication.""" - param_position { - desc { - en: """Position of authenticator in chain. Possible values are 'front', 'rear', 'before:{other_authenticator}', 'after:{other_authenticator}'.""" - zh: """认证者在链中的位置。可能的值是 'front', 'rear', 'before:{other_authenticator}', 'after:{other_authenticator}'""" - } - } +param_auth_id.desc: +"""Authenticator ID.""" - like_user_id { - desc { - en: """Fuzzy search user_id (username or clientid).""" - zh: """使用用户 ID （username 或 clientid）模糊查询。""" - } - label { - en: """like_user_id""" - zh: """like_user_id""" - } - } +param_listener_id.desc: +"""Listener ID.""" - is_superuser { - desc { - en: """Is superuser""" - zh: """是否是超级用户""" - } - } +param_position.desc: +"""Position of authenticator in chain. Possible values are 'front', 'rear', 'before:{other_authenticator}', 'after:{other_authenticator}'.""" + +param_user_id.desc: +"""User ID.""" } diff --git a/rel/i18n/emqx_authn_http.hocon b/rel/i18n/emqx_authn_http.hocon index 129db5054..582bacea8 100644 --- a/rel/i18n/emqx_authn_http.hocon +++ b/rel/i18n/emqx_authn_http.hocon @@ -1,81 +1,45 @@ emqx_authn_http { - get { - desc { - en: """Configuration of authenticator using HTTP Server as authentication service (Using GET request).""" - zh: """使用 HTTP Server 作为认证服务的认证器的配置项 (使用 GET 请求)。""" - } - } - post { - desc { - en: """Configuration of authenticator using HTTP Server as authentication service (Using POST request).""" - zh: """使用 HTTP Server 作为认证服务的认证器的配置项 (使用 POST 请求)。""" - } - } +body.desc: +"""HTTP request body.""" - method { - desc { - en: """HTTP request method.""" - zh: """HTTP 请求方法。""" - } - label { - en: """Request Method""" - zh: """请求方法""" - } - } +body.label: +"""Request Body""" - url { - desc { - en: """URL of the HTTP server.""" - zh: """认证 HTTP 服务器地址。""" - } - label { - en: """URL""" - zh: """URL""" - } - } +get.desc: +"""Configuration of authenticator using HTTP Server as authentication service (Using GET request).""" - headers { - desc { - en: """List of HTTP Headers.""" - zh: """HTTP Headers 列表""" - } - label { - en: """Headers""" - zh: """请求头""" - } - } +headers.desc: +"""List of HTTP Headers.""" - headers_no_content_type { - desc { - en: """List of HTTP headers (without content-type).""" - zh: """HTTP Headers 列表 (无 content-type) 。""" - } - label { - en: """headers_no_content_type""" - zh: """请求头(无 content-type)""" - } - } +headers.label: +"""Headers""" - body { - desc { - en: """HTTP request body.""" - zh: """HTTP request body。""" - } - label { - en: """Request Body""" - zh: """Request Body""" - } - } +headers_no_content_type.desc: +"""List of HTTP headers (without content-type).""" + +headers_no_content_type.label: +"""headers_no_content_type""" + +method.desc: +"""HTTP request method.""" + +method.label: +"""Request Method""" + +post.desc: +"""Configuration of authenticator using HTTP Server as authentication service (Using POST request).""" + +request_timeout.desc: +"""HTTP request timeout.""" + +request_timeout.label: +"""Request Timeout""" + +url.desc: +"""URL of the HTTP server.""" + +url.label: +"""URL""" - request_timeout { - desc { - en: """HTTP request timeout.""" - zh: """HTTP 请求超时时长。""" - } - label { - en: """Request Timeout""" - zh: """请求超时时间""" - } - } } diff --git a/rel/i18n/emqx_authn_jwt.hocon b/rel/i18n/emqx_authn_jwt.hocon index 6a4a1e2d4..f947dcf5d 100644 --- a/rel/i18n/emqx_authn_jwt.hocon +++ b/rel/i18n/emqx_authn_jwt.hocon @@ -1,219 +1,118 @@ emqx_authn_jwt { - use_jwks { - desc { - en: """Whether to use JWKS.""" - zh: """是否使用 JWKS。""" - } - label { - en: """Whether to Use JWKS""" - zh: """是否使用 JWKS""" - } - } - algorithm { - desc { - en: """JWT signing algorithm, Supports HMAC (configured as hmac-based) and RSA, ECDSA (configured as public-key).""" - zh: """JWT 签名算法，支持 HMAC (配置为 hmac-based）和 RSA、ECDSA (配置为 public-key)。""" - } - label { - en: """JWT Signing Algorithm""" - zh: """JWT 签名算法""" - } - } +acl_claim_name.desc: +"""JWT claim name to use for getting ACL rules.""" - public_key { - desc { - en: """The public key used to verify the JWT.""" - zh: """用于验证 JWT 的公钥。""" - } - label { - en: """Public Key""" - zh: """公钥""" - } - } +acl_claim_name.label: +"""ACL claim name""" - secret_base64_encoded { - desc { - en: """Whether secret is base64 encoded.""" - zh: """密钥是否为 Base64 编码。""" - } - label { - en: """Whether Secret is Base64 Encoded""" - zh: """密钥是否为 Base64 编码""" - } - } +algorithm.desc: +"""JWT signing algorithm, Supports HMAC (configured as hmac-based) and RSA, ECDSA (configured as public-key).""" - secret { - desc { - en: """The key to verify the JWT using HMAC algorithm.""" - zh: """使用 HMAC 算法时用于验证 JWT 的密钥""" - } - label { - en: """Secret""" - zh: """Secret""" - } - } +algorithm.label: +"""JWT Signing Algorithm""" - endpoint { - desc { - en: """JWKS endpoint, it's a read-only endpoint that returns the server's public key set in the JWKS format.""" - zh: """JWKS 端点， 它是一个以 JWKS 格式返回服务端的公钥集的只读端点。""" - } - label { - en: """JWKS Endpoint""" - zh: """JWKS Endpoint""" - } - } +cacertfile.desc: +"""Path to a file containing PEM-encoded CA certificates.""" - refresh_interval { - desc { - en: """JWKS refresh interval.""" - zh: """JWKS 刷新间隔。""" - } - label { - en: """JWKS Refresh Interval""" - zh: """JWKS 刷新间隔""" - } - } +cacertfile.label: +"""CA Certificate File""" - cacertfile { - desc { - en: """Path to a file containing PEM-encoded CA certificates.""" - zh: """包含 PEM 编码的 CA 证书的文件的路径。""" - } - label { - en: """CA Certificate File""" - zh: """CA 证书文件""" - } - } +certfile.desc: +"""Path to a file containing the user certificate.""" - certfile { - desc { - en: """Path to a file containing the user certificate.""" - zh: """包含用户证书的文件的路径。""" - } - label { - en: """Certificate File""" - zh: """证书文件""" - } - } +certfile.label: +"""Certificate File""" - keyfile { - desc { - en: """Path to a file containing the user's private PEM-encoded key.""" - zh: """包含 PEM 编码的用户私钥的文件的路径。""" - } - label { - en: """Key File""" - zh: """私钥文件""" - } - } +enable.desc: +"""Enable/disable SSL.""" - verify { - desc { - en: """Enable or disable SSL peer verification.""" - zh: """指定握手过程中是否校验对端证书。""" - } - label { - en: """Verify""" - zh: """Verify""" - } - } +enable.label: +"""Enable/disable SSL""" - server_name_indication { - desc { - en: """Server Name Indication (SNI).""" - zh: """服务器名称指示（SNI）。""" - } - label { - en: """Server Name Indication""" - zh: """服务器名称指示""" - } - } +endpoint.desc: +"""JWKS endpoint, it's a read-only endpoint that returns the server's public key set in the JWKS format.""" - verify_claims { - desc { - en: """A list of custom claims to validate, which is a list of name/value pairs. +endpoint.label: +"""JWKS Endpoint""" + +from.desc: +"""Field to take JWT from.""" + +from.label: +"""From Field""" + +jwt_hmac.desc: +"""Configuration when the JWT for authentication is issued using the HMAC algorithm.""" + +jwt_jwks.desc: +"""Configuration when JWTs used for authentication need to be fetched from the JWKS endpoint.""" + +keyfile.desc: +"""Path to a file containing the user's private PEM-encoded key.""" + +keyfile.label: +"""Key File""" + +jwt_public_key.desc: +"""Configuration when the JWT for authentication is issued using RSA or ECDSA algorithm.""" + +public_key.desc: +"""The public key used to verify the JWT.""" + +public_key.label: +"""Public Key""" + +refresh_interval.desc: +"""JWKS refresh interval.""" + +refresh_interval.label: +"""JWKS Refresh Interval""" + +secret.desc: +"""The key to verify the JWT using HMAC algorithm.""" + +secret.label: +"""Secret""" + +secret_base64_encoded.desc: +"""Whether secret is base64 encoded.""" + +secret_base64_encoded.label: +"""Whether Secret is Base64 Encoded""" + +server_name_indication.desc: +"""Server Name Indication (SNI).""" + +server_name_indication.label: +"""Server Name Indication""" + +ssl.desc: +"""SSL options.""" + +ssl.label: +"""SSL Options""" + +use_jwks.desc: +"""Whether to use JWKS.""" + +use_jwks.label: +"""Whether to Use JWKS""" + +verify.desc: +"""Enable or disable SSL peer verification.""" + +verify.label: +"""Verify""" + +verify_claims.desc: +"""A list of custom claims to validate, which is a list of name/value pairs. Values can use the following placeholders: - ${username}: Will be replaced at runtime with Username used by the client when connecting - ${clientid}: Will be replaced at runtime with Client ID used by the client when connecting Authentication will verify that the value of claims in the JWT (taken from the Password field) matches what is required in verify_claims.""" - zh: """需要验证的自定义声明列表，它是一个名称/值对列表。 -值可以使用以下占位符： -- ${username}: 将在运行时被替换为客户端连接时使用的用户名 -- ${clientid}: 将在运行时被替换为客户端连接时使用的客户端标识符 -认证时将验证 JWT（取自 Password 字段）中 claims 的值是否与 verify_claims 中要求的相匹配。""" - } - label { - en: """Verify Claims""" - zh: """Verify Claims""" - } - } - - ssl { - desc { - en: """SSL options.""" - zh: """SSL 选项。""" - } - label { - en: """SSL Options""" - zh: """SSL 选项""" - } - } - - enable { - desc { - en: """Enable/disable SSL.""" - zh: """启用/禁用 SSL。""" - } - label { - en: """Enable/disable SSL""" - zh: """启用/禁用 SSL""" - } - } - - hmac-based { - desc { - en: """Configuration when the JWT for authentication is issued using the HMAC algorithm.""" - zh: """用于认证的 JWT 使用 HMAC 算法签发时的配置。""" - } - } - - public-key { - desc { - en: """Configuration when the JWT for authentication is issued using RSA or ECDSA algorithm.""" - zh: """用于认证的 JWT 使用 RSA 或 ECDSA 算法签发时的配置。""" - } - } - - jwks { - desc { - en: """Configuration when JWTs used for authentication need to be fetched from the JWKS endpoint.""" - zh: """用于认证的 JWTs 需要从 JWKS 端点获取时的配置。""" - } - } - - acl_claim_name { - desc { - en: """JWT claim name to use for getting ACL rules.""" - zh: """JWT claim name to use for getting ACL rules.""" - } - label { - en: """ACL claim name""" - zh: """ACL claim name""" - } - } - - from { - desc { - en: """Field to take JWT from.""" - zh: """要从中获取 JWT 的字段。""" - } - label { - en: """From Field""" - zh: """源字段""" - } - } +verify_claims.label: +"""Verify Claims""" } diff --git a/rel/i18n/emqx_authn_mnesia.hocon b/rel/i18n/emqx_authn_mnesia.hocon index 0d07217d9..6770df090 100644 --- a/rel/i18n/emqx_authn_mnesia.hocon +++ b/rel/i18n/emqx_authn_mnesia.hocon @@ -1,21 +1,12 @@ emqx_authn_mnesia { - authentication { - desc { - en: """Configuration of authenticator using built-in database as data source.""" - zh: """使用内置数据库作为认证数据源的认证器的配置项。""" - } - } - user_id_type { - desc { - en: """Specify whether to use `clientid` or `username` for authentication.""" - zh: """指定使用客户端ID `clientid` 还是用户名 `username` 进行认证。""" - } +builtin_db.desc: +"""Configuration of authenticator using built-in database as data source.""" - label: { - en: """Authentication ID Type""" - zh: """认证 ID 类型""" - } - } +user_id_type.desc: +"""Specify whether to use `clientid` or `username` for authentication.""" + +user_id_type.label: +"""Authentication ID Type""" } diff --git a/rel/i18n/emqx_authn_mongodb.hocon b/rel/i18n/emqx_authn_mongodb.hocon index 80d6473ed..6d851f58f 100644 --- a/rel/i18n/emqx_authn_mongodb.hocon +++ b/rel/i18n/emqx_authn_mongodb.hocon @@ -1,83 +1,45 @@ emqx_authn_mongodb { - standalone { - desc { - en: """Configuration of authenticator using MongoDB (Standalone) as authentication data source.""" - zh: """使用 MongoDB (Standalone) 作为认证数据源的认证器的配置项。""" - } - } - replica-set { - desc { - en: """Configuration of authenticator using MongoDB (Replica Set) as authentication data source.""" - zh: """使用 MongoDB (Replica Set) 作为认证数据源的认证器的配置项。""" - } - } +collection.desc: +"""Collection used to store authentication data.""" - sharded-cluster { - desc { - en: """Configuration of authenticator using MongoDB (Sharded Cluster) as authentication data source.""" - zh: """使用 MongoDB (Sharded Cluster) 作为认证数据源的认证器的配置项。""" - } - } +collection.label: +"""Collection""" - collection { - desc { - en: """Collection used to store authentication data.""" - zh: """存储认证数据的集合。""" - } - label: { - en: """Collection""" - zh: """集合""" - } - } - - filter { - desc { - en: """Conditional expression that defines the filter condition in the query. +filter.desc: +"""Conditional expression that defines the filter condition in the query. Filter supports the following placeholders: - ${username}: Will be replaced at runtime with Username used by the client when connecting - ${clientid}: Will be replaced at runtime with Client ID used by the client when connecting""" - zh: """在查询中定义过滤条件的条件表达式。 -过滤器支持如下占位符： -- ${username}: 将在运行时被替换为客户端连接时使用的用户名 -- ${clientid}: 将在运行时被替换为客户端连接时使用的客户端标识符""" - } - label: { - en: """Filter""" - zh: """过滤器""" - } - } - password_hash_field { - desc { - en: """Document field that contains password hash.""" - zh: """文档中用于存放密码散列的字段。""" - } - label: { - en: """Password Hash Field""" - zh: """密码散列字段""" - } - } +filter.label: +"""Filter""" - salt_field { - desc { - en: """Document field that contains the password salt.""" - zh: """文档中用于存放盐值的字段。""" - } - label: { - en: """Salt Field""" - zh: """盐值字段""" - } - } +is_superuser_field.desc: +"""Document field that defines if the user has superuser privileges.""" + +is_superuser_field.label: +"""Is Superuser Field""" + +password_hash_field.desc: +"""Document field that contains password hash.""" + +password_hash_field.label: +"""Password Hash Field""" + +replica-set.desc: +"""Configuration of authenticator using MongoDB (Replica Set) as authentication data source.""" + +salt_field.desc: +"""Document field that contains the password salt.""" + +salt_field.label: +"""Salt Field""" + +sharded-cluster.desc: +"""Configuration of authenticator using MongoDB (Sharded Cluster) as authentication data source.""" + +single.desc: +"""Configuration of authenticator using MongoDB (Standalone) as authentication data source.""" - is_superuser_field { - desc { - en: """Document field that defines if the user has superuser privileges.""" - zh: """文档中用于定义用户是否具有超级用户权限的字段。""" - } - label: { - en: """Is Superuser Field""" - zh: """超级用户字段""" - } - } } diff --git a/rel/i18n/emqx_authn_mysql.hocon b/rel/i18n/emqx_authn_mysql.hocon index 4ddfd5701..5eb2d23e9 100644 --- a/rel/i18n/emqx_authn_mysql.hocon +++ b/rel/i18n/emqx_authn_mysql.hocon @@ -1,30 +1,18 @@ emqx_authn_mysql { - authentication { - desc { - en: """Configuration of authenticator using MySQL as authentication data source.""" - zh: """使用 MySQL 作为认证数据源的认证器的配置项。""" - } - } - query { - desc { - en: """SQL used to query data for authentication, such as password hash.""" - zh: """用于查询密码散列等用于认证的数据的 SQL 语句。""" - } - label: { - en: """Query""" - zh: """查询语句""" - } - } +mysql.desc: +"""Configuration of authenticator using MySQL as authentication data source.""" + +query.desc: +"""SQL used to query data for authentication, such as password hash.""" + +query.label: +"""Query""" + +query_timeout.desc: +"""Timeout for the SQL query.""" + +query_timeout.label: +"""Query Timeout""" - query_timeout { - desc { - en: """Timeout for the SQL query.""" - zh: """SQL 查询的超时时间。""" - } - label: { - en: """Query Timeout""" - zh: """查询超时""" - } - } } diff --git a/rel/i18n/emqx_authn_pgsql.hocon b/rel/i18n/emqx_authn_pgsql.hocon index 298e38774..696a17861 100644 --- a/rel/i18n/emqx_authn_pgsql.hocon +++ b/rel/i18n/emqx_authn_pgsql.hocon @@ -1,19 +1,12 @@ emqx_authn_pgsql { - authentication { - desc { - en: """Configuration of authenticator using PostgreSQL as authentication data source.""" - zh: """使用 PostgreSQL 作为认证数据源的认证器的配置项。""" - } - } - query { - desc { - en: """SQL used to query data for authentication, such as password hash.""" - zh: """用于查询密码散列等用于认证的数据的 SQL 语句。""" - } - label: { - en: """Query""" - zh: """查询语句""" - } - } +postgresql.desc: +"""Configuration of authenticator using PostgreSQL as authentication data source.""" + +query.desc: +"""SQL used to query data for authentication, such as password hash.""" + +query.label: +"""Query""" + } diff --git a/rel/i18n/emqx_authn_redis.hocon b/rel/i18n/emqx_authn_redis.hocon index a9cd4a414..e8d98b9a1 100644 --- a/rel/i18n/emqx_authn_redis.hocon +++ b/rel/i18n/emqx_authn_redis.hocon @@ -1,33 +1,18 @@ emqx_authn_redis { - standalone { - desc { - en: """Configuration of authenticator using Redis (Standalone) as authentication data source.""" - zh: """使用 Redis (Standalone) 作为认证数据源的认证器的配置项。""" - } - } - cluster { - desc { - en: """Configuration of authenticator using Redis (Cluster) as authentication data source.""" - zh: """使用 Redis (Cluster) 作为认证数据源的认证器的配置项。""" - } - } +cluster.desc: +"""Configuration of authenticator using Redis (Cluster) as authentication data source.""" - sentinel { - desc { - en: """Configuration of authenticator using Redis (Sentinel) as authentication data source.""" - zh: """使用 Redis (Sentinel) 作为认证数据源的认证器的配置项。""" - } - } +cmd.desc: +"""The Redis Command used to query data for authentication such as password hash, currently only supports HGET and HMGET.""" + +cmd.label: +"""Command""" + +sentinel.desc: +"""Configuration of authenticator using Redis (Sentinel) as authentication data source.""" + +single.desc: +"""Configuration of authenticator using Redis (Standalone) as authentication data source.""" - cmd { - desc { - en: """The Redis Command used to query data for authentication such as password hash, currently only supports HGET and HMGET.""" - zh: """用于查询密码散列等用于认证的数据的 Redis Command，目前仅支持 HGET 与 HMGET。""" - } - label: { - en: """Command""" - zh: """Command""" - } - } } diff --git a/rel/i18n/emqx_authn_schema.hocon b/rel/i18n/emqx_authn_schema.hocon index 917554af0..98263ca49 100644 --- a/rel/i18n/emqx_authn_schema.hocon +++ b/rel/i18n/emqx_authn_schema.hocon @@ -1,243 +1,135 @@ emqx_authn_schema { - enable { - desc { - en: """Set to true or false to disable this auth provider.""" - zh: """设为 true 或 false 以启用或禁用此认证数据源。""" - } - label: { - en: """Enable""" - zh: """启用""" - } - } - mechanism { - desc { - en: """Authentication mechanism.""" - zh: """认证机制。""" - } - label: { - en: """Authentication Mechanism""" - zh: """认证机制""" - } - } +backend.desc: +"""Backend type.""" - backend { - desc { - en: """Backend type.""" - zh: """后端类型。""" - } - label: { - en: """Backend Type""" - zh: """后端类型""" - } - } +backend.label: +"""Backend Type""" - metrics { - desc { - en: """The metrics of the resource.""" - zh: """资源统计指标。""" - } - label: { - en: """Metrics""" - zh: """指标""" - } - } +enable.desc: +"""Set to true or false to disable this auth provider.""" - node_metrics { - desc { - en: """The metrics of the resource for each node.""" - zh: """每个节点上资源的统计指标。""" - } - label: { - en: """Resource Metrics in Node""" - zh: """节点资源指标""" - } - } +enable.label: +"""Enable""" - status { - desc { - en: """The status of the resource.""" - zh: """资源状态。""" - } - label: { - en: """Status""" - zh: """状态""" - } - } +failed.desc: +"""Count of query failed.""" - node_status { - desc { - en: """The status of the resource for each node.""" - zh: """每个节点上资源的状态。""" - } - label: { - en: """Resource Status in Node""" - zh: """节点资源状态""" - } - } +failed.label: +"""Failed""" - node_error { - desc { - en: """The error of node.""" - zh: """节点上产生的错误。""" - } - label: { - en: """Error in Node""" - zh: """节点产生的错误""" - } - } +matched.desc: +"""Count of this resource is queried.""" - matched { - desc { - en: """Count of this resource is queried.""" - zh: """请求命中次数。""" - } - label: { - en: """Matched""" - zh: """已命中""" - } - } +matched.label: +"""Matched""" - success { - desc { - en: """Count of query success.""" - zh: """请求成功次数。""" - } - label: { - en: """Success""" - zh: """成功""" - } - } +mechanism.desc: +"""Authentication mechanism.""" - failed { - desc { - en: """Count of query failed.""" - zh: """请求失败次数。""" - } - label: { - en: """Failed""" - zh: """失败""" - } - } +mechanism.label: +"""Authentication Mechanism""" - rate { - desc { - en: """The rate of matched, times/second.""" - zh: """命中速率，单位：次/秒。""" - } - label: { - en: """Rate""" - zh: """速率""" - } - } +metrics.desc: +"""The metrics of the resource.""" - rate_max { - desc { - en: """The max rate of matched, times/second.""" - zh: """最大命中速率，单位：次/秒。""" - } - label: { - en: """Max Rate""" - zh: """最大速率""" - } - } +metrics.label: +"""Metrics""" - rate_last5m { - desc { - en: """The average rate of matched in the last 5 minutes, times/second.""" - zh: """5分钟内平均命中速率，单位：次/秒。""" - } - label: { - en: """Rate in Last 5min""" - zh: """5分钟内速率""" - } - } +metrics_failed.desc: +"""The required authentication information is found in the current instance, and the instance returns authentication failure.""" - node { - desc { - en: """Node name.""" - zh: """节点名称。""" - } - label: { - en: """Node Name.""" - zh: """节点名称。""" - } - } +metrics_failed.label: +"""Authentication Failed Times""" - metrics_nomatch { - desc { - en: """The number of times the instance was ignored when the required authentication information was not found in the current instance.""" - zh: """在当前实例中没有找到需要的认证信息，实例被忽略的次数。""" - } - label: { - en: """Nomatch Times""" - zh: """实例被忽略的次数""" - } - } +metrics_nomatch.desc: +"""The number of times the instance was ignored when the required authentication information was not found in the current instance.""" - metrics_total { - desc { - en: """The total number of times the current instance was triggered.""" - zh: """当前实例被触发的总次数。""" - } - label: { - en: """Total Triggered Times""" - zh: """当前实例被触发的总次数""" - } - } +metrics_nomatch.label: +"""Nomatch Times""" - metrics_success { - desc { - en: """The required authentication information is found in the current instance, and the instance returns authentication success.""" - zh: """在当前实例中找到需要的认证信息，并且实例返回认证成功的次数。""" - } - label: { - en: """Authentication Success Times""" - zh: """实例认证成功的次数""" - } - } +metrics_rate.desc: +"""The total rate at which instances are triggered, times/second.""" - metrics_failed { - desc { - en: """The required authentication information is found in the current instance, and the instance returns authentication failure.""" - zh: """在当前实例中找到需要的认证信息，并且实例返回认证失败的次数。""" - } - label: { - en: """Authentication Failed Times""" - zh: """实例认证失败的次数""" - } - } +metrics_rate.label: +"""Total Triggered Rate""" - metrics_rate { - desc { - en: """The total rate at which instances are triggered, times/second.""" - zh: """实例被触发的速率。触发速率等于匹配速率 + 忽略速率，单位：次/秒。""" - } - label: { - en: """Total Triggered Rate""" - zh: """实例被触发的速率""" - } - } +metrics_rate_last5m.desc: +"""The average trigger rate of the instance within 5 minutes, times/second.""" - metrics_rate_max { - desc { - en: """The highest trigger rate the instance has ever reached, times/second.""" - zh: """实例曾经达到的最高触发速率，单位：次/秒。""" - } - label: { - en: """Highest Triggered Rate""" - zh: """实例曾经达到的最高触发速率""" - } - } +metrics_rate_last5m.label: +"""Average Triggered Rate in Last 5min""" + +metrics_rate_max.desc: +"""The highest trigger rate the instance has ever reached, times/second.""" + +metrics_rate_max.label: +"""Highest Triggered Rate""" + +metrics_success.desc: +"""The required authentication information is found in the current instance, and the instance returns authentication success.""" + +metrics_success.label: +"""Authentication Success Times""" + +metrics_total.desc: +"""The total number of times the current instance was triggered.""" + +metrics_total.label: +"""Total Triggered Times""" + +node.desc: +"""Node name.""" + +node.label: +"""Node Name.""" + +node_error.desc: +"""The error of node.""" + +node_error.label: +"""Error in Node""" + +node_metrics.desc: +"""The metrics of the resource for each node.""" + +node_metrics.label: +"""Resource Metrics in Node""" + +node_status.desc: +"""The status of the resource for each node.""" + +node_status.label: +"""Resource Status in Node""" + +rate.desc: +"""The rate of matched, times/second.""" + +rate.label: +"""Rate""" + +rate_last5m.desc: +"""The average rate of matched in the last 5 minutes, times/second.""" + +rate_last5m.label: +"""Rate in Last 5min""" + +rate_max.desc: +"""The max rate of matched, times/second.""" + +rate_max.label: +"""Max Rate""" + +status.desc: +"""The status of the resource.""" + +status.label: +"""Status""" + +success.desc: +"""Count of query success.""" + +success.label: +"""Success""" - metrics_rate_last5m { - desc { - en: """The average trigger rate of the instance within 5 minutes, times/second.""" - zh: """实例5分钟内平均触发速率，单位：次/秒。""" - } - label: { - en: """Average Triggered Rate in Last 5min""" - zh: """实例5分钟内平均触发速率""" - } - } } diff --git a/rel/i18n/emqx_authn_user_import_api.hocon b/rel/i18n/emqx_authn_user_import_api.hocon index 294897ec1..f8fb1757c 100644 --- a/rel/i18n/emqx_authn_user_import_api.hocon +++ b/rel/i18n/emqx_authn_user_import_api.hocon @@ -1,17 +1,9 @@ emqx_authn_user_import_api { - authentication_id_import_users_post { - desc { - en: """Import users into authenticator in global authentication chain.""" - zh: """为全局认证链上的指定认证器导入用户数据。""" - } - } +authentication_id_import_users_post.desc: +"""Import users into authenticator in global authentication chain.""" - listeners_listener_id_authentication_id_import_users_post { - desc { - en: """Import users into authenticator in listener authentication chain.""" - zh: """为监听器认证链上的指定认证器导入用户数据。""" - } - } +listeners_listener_id_authentication_id_import_users_post.desc: +"""Import users into authenticator in listener authentication chain.""" } diff --git a/rel/i18n/emqx_authz_api_cache.hocon b/rel/i18n/emqx_authz_api_cache.hocon index 9c620a22d..0789c1e71 100644 --- a/rel/i18n/emqx_authz_api_cache.hocon +++ b/rel/i18n/emqx_authz_api_cache.hocon @@ -1,8 +1,6 @@ emqx_authz_api_cache { - authorization_cache_delete { - desc { - en: """Clean all authorization cache in the cluster.""" - zh: """清除集群中所有授权数据缓存。""" - } - } + +authorization_cache_delete.desc: +"""Clean all authorization cache in the cluster.""" + } diff --git a/rel/i18n/emqx_authz_api_mnesia.hocon b/rel/i18n/emqx_authz_api_mnesia.hocon index 6d318d02b..4cfed2970 100644 --- a/rel/i18n/emqx_authz_api_mnesia.hocon +++ b/rel/i18n/emqx_authz_api_mnesia.hocon @@ -1,177 +1,87 @@ emqx_authz_api_mnesia { - users_username_get { - desc { - en: """Show the list of rules for users""" - zh: """获取内置数据库中所有用户名类型的规则记录""" - } - } - users_username_post { - desc { - en: """Add new rule for 'username'""" - zh: """添加内置数据库中用户名类型的规则记录""" - } - } +action.desc: +"""Authorized action (pub/sub/all)""" - users_clientid_get { - desc { - en: """Show the list of rules for clients""" - zh: """获取内置数据库中所有客户端标识符类型的规则记录""" - } - } +action.label: +"""action""" - users_clientid_post { - desc { - en: """Add new rule for 'clientid'""" - zh: """添加内置数据库中客户端标识符类型的规则记录""" - } - } +clientid.desc: +"""ClientID""" +clientid.label: +"""clientid""" - user_username_get { - desc { - en: """Get rule for 'username'""" - zh: """获取内置数据库中指定用户名类型的规则记录""" - } - } +fuzzy_clientid.desc: +"""Fuzzy search `clientid` as substring""" - user_username_put { - desc { - en: """Set rule for 'username'""" - zh: """更新内置数据库中指定用户名类型的规则记录""" - } - } +fuzzy_clientid.label: +"""fuzzy_clientid""" - user_username_delete { - desc { - en: """Delete rule for 'username'""" - zh: """删除内置数据库中指定用户名类型的规则记录""" - } - } +fuzzy_username.desc: +"""Fuzzy search `username` as substring""" - user_clientid_get { - desc { - en: """Get rule for 'clientid'""" - zh: """获取内置数据库中指定客户端标识符类型的规则记录""" - } - } +fuzzy_username.label: +"""fuzzy_username""" - user_clientid_put { - desc { - en: """Set rule for 'clientid'""" - zh: """更新内置数据库中指定客户端标识符类型的规则记录""" - } - } +permission.desc: +"""Permission""" - user_clientid_delete { - desc { - en: """Delete rule for 'clientid'""" - zh: """删除内置数据库中指定客户端标识符类型的规则记录""" - } - } +permission.label: +"""permission""" - rules_all_get { - desc { - en: """Show the list of rules for 'all'""" - zh: """列出为所有客户端启用的规则列表""" - } - } +rules_all_delete.desc: +"""Delete rules for 'all'""" - rules_all_post { - desc { - en: """Create/Update the list of rules for 'all'.""" - zh: """创建/更新 为所有客户端启用的规则列表。""" - } - } +rules_all_get.desc: +"""Show the list of rules for 'all'""" - rules_all_delete { - desc { - en: """Delete rules for 'all'""" - zh: """删除 `all` 规则""" - } - } +rules_all_post.desc: +"""Create/Update the list of rules for 'all'.""" - rules_delete { - desc { - en: """Delete all rules for all 'users', 'clients' and 'all'""" - zh: """清除内置数据库中的所有类型（'users' 、'clients' 、'all'）的所有规则""" - } - } +rules_delete.desc: +"""Delete all rules for all 'users', 'clients' and 'all'""" - fuzzy_username { - desc { - en: """Fuzzy search `username` as substring""" - zh: """使用字串匹配模糊搜索用户名""" - } - label { - en: """fuzzy_username""" - zh: """用户名子串""" - } - } +topic.desc: +"""Rule on specific topic""" - fuzzy_clientid { - desc { - en: """Fuzzy search `clientid` as substring""" - zh: """使用字串匹配模糊搜索客户端标识符""" - } - label { - en: """fuzzy_clientid""" - zh: """客户端标识符子串""" - } - } +topic.label: +"""topic""" - topic { - desc { - en: """Rule on specific topic""" - zh: """在指定主题上的规则""" - } - label { - en: """topic""" - zh: """主题""" - } - } +user_clientid_delete.desc: +"""Delete rule for 'clientid'""" - permission { - desc { - en: """Permission""" - zh: """权限""" - } - label { - en: """permission""" - zh: """权限""" - } - } +user_clientid_get.desc: +"""Get rule for 'clientid'""" - action { - desc { - en: """Authorized action (pub/sub/all)""" - zh: """被授权的行为 (发布/订阅/所有)""" - } - label { - en: """action""" - zh: """行为""" - } - } +user_clientid_put.desc: +"""Set rule for 'clientid'""" - clientid { - desc { - en: """ClientID""" - zh: """客户端标识符""" - } - label { - en: """clientid""" - zh: """客户端标识符""" - } - } +user_username_delete.desc: +"""Delete rule for 'username'""" + +user_username_get.desc: +"""Get rule for 'username'""" + +user_username_put.desc: +"""Set rule for 'username'""" + +username.desc: +"""Username""" + +username.label: +"""username""" + +users_clientid_get.desc: +"""Show the list of rules for clients""" + +users_clientid_post.desc: +"""Add new rule for 'clientid'""" + +users_username_get.desc: +"""Show the list of rules for users""" + +users_username_post.desc: +"""Add new rule for 'username'""" - username { - desc { - en: """Username""" - zh: """用户名""" - } - label { - en: """username""" - zh: """用户名""" - } - } } diff --git a/rel/i18n/emqx_authz_api_schema.hocon b/rel/i18n/emqx_authz_api_schema.hocon index d649dd5a0..2edfc8fcb 100644 --- a/rel/i18n/emqx_authz_api_schema.hocon +++ b/rel/i18n/emqx_authz_api_schema.hocon @@ -1,185 +1,90 @@ emqx_authz_api_schema { - enable { - desc { - en: """Set to true or false to disable this ACL provider.""" - zh: """设为 true 或 false 以启用或禁用此访问控制数据源。""" - } - label { - en: """enable""" - zh: """enable""" - } - } - type { - desc { - en: """Backend type.""" - zh: """数据后端类型。""" - } - label { - en: """type""" - zh: """type""" - } - } +body.desc: +"""HTTP request body.""" -#==== authz_file +body.label: +"""body""" - rules { - desc { - en: """Authorization static file rules.""" - zh: """静态授权文件规则。""" - } - label { - en: """rules""" - zh: """规则""" - } - } +cmd.desc: +"""Database query used to retrieve authorization data.""" -#==== authz_http +cmd.label: +"""cmd""" - method { - desc { - en: """HTTP method.""" - zh: """HTTP 请求方法。""" - } - label { - en: """method""" - zh: """method""" - } - } +collection.desc: +"""`MongoDB` collection containing the authorization data.""" - url { - desc { - en: """URL of the auth server.""" - zh: """认证服务器 URL。""" - } - label { - en: """url""" - zh: """url""" - } - } +collection.label: +"""collection""" - headers { - desc { - en: """List of HTTP Headers.""" - zh: """HTTP Headers 列表""" - } - label { - en: """Headers""" - zh: """请求头""" - } - } +enable.desc: +"""Set to true or false to disable this ACL provider.""" - headers_no_content_type { - desc { - en: """List of HTTP headers (without content-type).""" - zh: """HTTP Headers 列表（无 content-type）。""" - } - label { - en: """headers_no_content_type""" - zh: """请求头(无 content-type)""" - } - } +enable.label: +"""enable""" - body { - desc { - en: """HTTP request body.""" - zh: """HTTP 请求体。""" - } - label { - en: """body""" - zh: """请求体""" - } - } - - request_timeout { - desc { - en: """Request timeout.""" - zh: """请求超时时间。""" - } - label { - en: """request_timeout""" - zh: """请求超时""" - } - } - -#==== authz_mnesia - -# only common fields(`enable` and `type`) - -#==== authz_mongo - - collection { - desc { - en: """`MongoDB` collection containing the authorization data.""" - zh: """`MongoDB` 授权数据集。""" - } - label { - en: """collection""" - zh: """数据集""" - } - } - - filter { - desc { - en: """Conditional expression that defines the filter condition in the query. +filter.desc: +"""Conditional expression that defines the filter condition in the query. Filter supports the following placeholders: - ${username}: Will be replaced at runtime with Username used by the client when connecting; - ${clientid}: Will be replaced at runtime with Client ID used by the client when connecting.""" - zh: """在查询中定义过滤条件的条件表达式。 -过滤器支持如下占位符： -- ${username}: 将在运行时被替换为客户端连接时使用的用户名 -- ${clientid}: 将在运行时被替换为客户端连接时使用的客户端标识符""" - } - label { - en: """Filter""" - zh: """过滤器""" - } - } -#==== authz_mysql +filter.label: +"""Filter""" -# `query`, is common field +headers.desc: +"""List of HTTP Headers.""" -#==== authz_pgsql +headers.label: +"""Headers""" -# `query`, is common field +headers_no_content_type.desc: +"""List of HTTP headers (without content-type).""" -#==== authz_redis +headers_no_content_type.label: +"""headers_no_content_type""" - cmd { - desc { - en: """Database query used to retrieve authorization data.""" - zh: """访问控制数据查询命令。""" - } - label { - en: """cmd""" - zh: """查询命令""" - } - } +method.desc: +"""HTTP method.""" -#==== common field for DBs (except mongodb and redis) +method.label: +"""method""" - query { - desc { - en: """Database query used to retrieve authorization data.""" - zh: """访问控制数据查询语句。""" - } - label { - en: """query""" - zh: """查询语句""" - } - } +position.desc: +"""Where to place the source.""" -#==== fields +position.label: +"""position""" + +query.desc: +"""Database query used to retrieve authorization data.""" + +query.label: +"""query""" + +request_timeout.desc: +"""Request timeout.""" + +request_timeout.label: +"""request_timeout""" + +rules.desc: +"""Authorization static file rules.""" + +rules.label: +"""rules""" + +type.desc: +"""Backend type.""" + +type.label: +"""type""" + +url.desc: +"""URL of the auth server.""" + +url.label: +"""url""" - position { - desc { - en: """Where to place the source.""" - zh: """认证数据源位置。""" - } - label { - en: """position""" - zh: """位置""" - } - } } diff --git a/rel/i18n/emqx_authz_api_settings.hocon b/rel/i18n/emqx_authz_api_settings.hocon index b44580b34..1840848bd 100644 --- a/rel/i18n/emqx_authz_api_settings.hocon +++ b/rel/i18n/emqx_authz_api_settings.hocon @@ -1,15 +1,9 @@ emqx_authz_api_settings { - authorization_settings_get { - desc { - en: """Get authorization settings""" - zh: """获取授权配置""" - } - } - authorization_settings_put { - desc { - en: """Update authorization settings""" - zh: """更新授权配置""" - } - } +authorization_settings_get.desc: +"""Get authorization settings""" + +authorization_settings_put.desc: +"""Update authorization settings""" + } diff --git a/rel/i18n/emqx_authz_api_sources.hocon b/rel/i18n/emqx_authz_api_sources.hocon index c5f0eaad4..5d2dda69e 100644 --- a/rel/i18n/emqx_authz_api_sources.hocon +++ b/rel/i18n/emqx_authz_api_sources.hocon @@ -1,116 +1,48 @@ emqx_authz_api_sources { - authorization_sources_get { - desc { - en: """List all authorization sources""" - zh: """列出所有授权数据源""" - } - } - authorization_sources_post { - desc { - en: """Add a new source""" - zh: """添加授权数据源""" - } - } +authorization_sources_get.desc: +"""List all authorization sources""" - authorization_sources_type_get { - desc { - en: """Get a authorization source""" - zh: """获取指定类型的授权数据源""" - } - } +authorization_sources_post.desc: +"""Add a new source""" - authorization_sources_type_put { - desc { - en: """Update source""" - zh: """更新指定类型的授权数据源""" - } - } +authorization_sources_type_delete.desc: +"""Delete source""" - authorization_sources_type_delete { - desc { - en: """Delete source""" - zh: """删除指定类型的授权数据源""" - } - } +authorization_sources_type_get.desc: +"""Get a authorization source""" - authorization_sources_type_status_get { - desc { - en: """Get a authorization source""" - zh: """获取指定授权数据源的状态""" - } - } +authorization_sources_type_move_post.desc: +"""Change the exection order of sources""" - authorization_sources_type_move_post { - desc { - en: """Change the exection order of sources""" - zh: """更新授权数据源的优先执行顺序""" - } - } +authorization_sources_type_put.desc: +"""Update source""" - sources { - desc { - en: """Authorization source""" - zh: """授权数据源列表""" - } - label { - en: """sources""" - zh: """数据源列表""" - } - } +authorization_sources_type_status_get.desc: +"""Get a authorization source""" - sources { - desc { - en: """Authorization sources""" - zh: """授权数据源列表""" - } - label { - en: """sources""" - zh: """数据源列表""" - } - } +source.desc: +"""Authorization source""" - source_config { - desc { - en: """Source config""" - zh: """数据源配置""" - } - label { - en: """source_config""" - zh: """数据源配置""" - } - } +source.label: +"""source""" - source { - desc { - en: """Authorization source""" - zh: """授权数据源""" - } - label { - en: """source""" - zh: """数据源""" - } - } +source_config.desc: +"""Source config""" - source_config { - desc { - en: """Source config""" - zh: """数据源配置""" - } - label { - en: """source_config""" - zh: """数据源配置""" - } - } +source_config.label: +"""source_config""" + +source_type.desc: +"""Authorization type""" + +source_type.label: +"""source_type""" + +sources.desc: +"""Authorization sources""" + +sources.label: +"""sources""" - source_type { - desc { - en: """Authorization type""" - zh: """数据源类型""" - } - label { - en: """source_type""" - zh: """数据源类型""" - } - } } diff --git a/rel/i18n/emqx_authz_schema.hocon b/rel/i18n/emqx_authz_schema.hocon index e48c64b89..5318eb769 100644 --- a/rel/i18n/emqx_authz_schema.hocon +++ b/rel/i18n/emqx_authz_schema.hocon @@ -1,7 +1,251 @@ emqx_authz_schema { - sources { - desc { - en: """Authorization data sources.
+ +deny.desc: +"""The number of authentication failures.""" + +deny.label: +"""The Number of Authentication Failures""" + +redis_sentinel.desc: +"""Authorization using a Redis Sentinel.""" + +redis_sentinel.label: +"""redis_sentinel""" + +rate.desc: +"""The rate of matched, times/second.""" + +rate.label: +"""Rate""" + +status.desc: +"""The status of the resource.""" + +status.label: +"""Status""" + +method.desc: +"""HTTP method.""" + +method.label: +"""method""" + +query.desc: +"""Database query used to retrieve authorization data.""" + +query.label: +"""query""" + +metrics_total.desc: +"""The total number of times the authorization rule was triggered.""" + +metrics_total.label: +"""The Total Number of Times the Authorization Rule was Triggered""" + +redis_cluster.desc: +"""Authorization using a Redis cluster.""" + +redis_cluster.label: +"""redis_cluster""" + +mysql.desc: +"""Authorization using a MySQL database.""" + +mysql.label: +"""mysql""" + +postgresql.desc: +"""Authorization using a PostgreSQL database.""" + +postgresql.label: +"""postgresql""" + +mongo_rs.desc: +"""Authorization using a MongoDB replica set.""" + +mongo_rs.label: +"""mongo_rs""" + +type.desc: +"""Backend type.""" + +type.label: +"""type""" + +mongo_sharded.desc: +"""Authorization using a sharded MongoDB cluster.""" + +mongo_sharded.label: +"""mongo_sharded""" + +body.desc: +"""HTTP request body.""" + +body.label: +"""Request Body""" + +url.desc: +"""URL of the auth server.""" + +url.label: +"""URL""" + +node.desc: +"""Node name.""" + +node.label: +"""Node Name.""" + +headers.desc: +"""List of HTTP Headers.""" + +headers.label: +"""Headers""" + +rate_last5m.desc: +"""The average rate of matched in the last 5 minutes, times/second.""" + +rate_last5m.label: +"""Rate in Last 5min""" + +headers_no_content_type.desc: +"""List of HTTP headers (without content-type).""" + +headers_no_content_type.label: +"""headers_no_content_type""" + +node_error.desc: +"""The error of node.""" + +node_error.label: +"""Error in Node""" + +builtin_db.desc: +"""Authorization using a built-in database (mnesia).""" + +builtin_db.label: +"""Builtin Database""" + +enable.desc: +"""Set to true or false to disable this ACL provider""" + +enable.label: +"""enable""" + +matched.desc: +"""Count of this resource is queried.""" + +matched.label: +"""Matched""" + +node_status.desc: +"""The status of the resource for each node.""" + +node_status.label: +"""Resource Status in Node""" + +rate_max.desc: +"""The max rate of matched, times/second.""" + +rate_max.label: +"""Max Rate""" + +filter.desc: +"""Conditional expression that defines the filter condition in the query. +Filter supports the following placeholders
+ - ${username}: Will be replaced at runtime with Username used by the client when connecting
+ - ${clientid}: Will be replaced at runtime with Client ID used by the client when connecting""" + +filter.label: +"""Filter""" + +path.desc: +"""Path to the file which contains the ACL rules. +If the file provisioned before starting EMQX node, +it can be placed anywhere as long as EMQX has read access to it. +That is, EMQX will treat it as read only. + +In case the rule-set is created or updated from EMQX Dashboard or HTTP API, +a new file will be created and placed in `authz` subdirectory inside EMQX's `data_dir`, +and the old file will not be used anymore.""" + +path.label: +"""path""" + +redis_single.desc: +"""Authorization using a single Redis instance.""" + +redis_single.label: +"""redis_single""" + +failed.desc: +"""Count of query failed.""" + +failed.label: +"""Failed""" + +metrics.desc: +"""The metrics of the resource.""" + +metrics.label: +"""Metrics""" + +authorization.desc: +"""Configuration related to the client authorization.""" + +authorization.label: +"""authorization""" + +collection.desc: +"""`MongoDB` collection containing the authorization data.""" + +collection.label: +"""collection""" + +mongo_single.desc: +"""Authorization using a single MongoDB instance.""" + +mongo_single.label: +"""mongo_single""" + +file.desc: +"""Authorization using a static file.""" + +file.label: +"""file""" + +http_post.desc: +"""Authorization using an external HTTP server (via POST requests).""" + +http_post.label: +"""http_post""" + +request_timeout.desc: +"""HTTP request timeout.""" + +request_timeout.label: +"""Request Timeout""" + +allow.desc: +"""The number of times the authentication was successful.""" + +allow.label: +"""The Number of Times the Authentication was Successful""" + +cmd.desc: +"""Database query used to retrieve authorization data.""" + +cmd.label: +"""cmd""" + +nomatch.desc: +"""The number of times that no authorization rules were matched.""" + +nomatch.label: +"""The Number of Times that no Authorization Rules were Matched""" + +sources.desc: +"""Authorization data sources.
An array of authorization (ACL) data providers. It is designed as an array, not a hash-map, so the sources can be ordered to form a chain of access controls.
@@ -19,525 +263,25 @@ NOTE: The source elements are identified by their 'type'. It is NOT allowed to configure two or more sources of the same type.""" - zh: """授权数据源。
-授权（ACL）数据源的列表。 -它被设计为一个数组，而不是一个散列映射， -所以可以作为链式访问控制。
+sources.label: +"""sources""" -当授权一个 'publish' 或 'subscribe' 行为时， -该配置列表中的所有数据源将按顺序进行检查。 -如果在某个客户端未找到时(使用 ClientID 或 Username)。 -将会移动到下一个数据源。直至得到 'allow' 或 'deny' 的结果。
+node_metrics.desc: +"""The metrics of the resource for each node.""" -如果在任何数据源中都未找到对应的客户端信息。 -配置的默认行为 ('authorization.no_match') 将生效。
+node_metrics.label: +"""Resource Metrics in Node""" -注意： -数据源使用 'type' 进行标识。 -使用同一类型的数据源多于一次不被允许。""" - } - label { - en: """sources""" - zh: """数据源""" - } - } +success.desc: +"""Count of query success.""" - authorization { - desc { - en: """Configuration related to the client authorization.""" - zh: """客户端授权相关配置""" - } - label { - en: """authorization""" - zh: """授权""" - } - } +success.label: +"""Success""" - enable { - desc { - en: """Set to true or false to disable this ACL provider""" - zh: """设为 true 或 false 以启用或禁用此访问控制数据源""" - } - label { - en: """enable""" - zh: """enable""" - } - } +http_get.desc: +"""Authorization using an external HTTP server (via GET requests).""" - type { - desc { - en: """Backend type.""" - zh: """数据后端类型""" - } - label { - en: """type""" - zh: """type""" - } - } +http_get.label: +"""http_get""" -#==== authz_file - - file { - desc { - en: """Authorization using a static file.""" - zh: """使用静态文件授权""" - } - label { - en: """file""" - zh: """文件""" - } - } - - path { - desc { - en: """Path to the file which contains the ACL rules. -If the file provisioned before starting EMQX node, -it can be placed anywhere as long as EMQX has read access to it. -That is, EMQX will treat it as read only. - -In case the rule-set is created or updated from EMQX Dashboard or HTTP API, -a new file will be created and placed in `authz` subdirectory inside EMQX's `data_dir`, -and the old file will not be used anymore.""" - zh: """包含 ACL 规则的文件路径。 -如果在启动 EMQX 节点前预先配置该路径， -那么可以将该文件置于任何 EMQX 可以访问到的位置。 - -如果从 EMQX Dashboard 或 HTTP API 创建或修改了规则集， -那么EMQX将会生成一个新的文件并将它存放在 `data_dir` 下的 `authz` 子目录中， -并从此弃用旧的文件。""" - } - label { - en: """path""" - zh: """path""" - } - } - -#==== authz_http - - http_get { - desc { - en: """Authorization using an external HTTP server (via GET requests).""" - zh: """使用外部 HTTP 服务器授权(GET 请求)。""" - } - label { - en: """http_get""" - zh: """http_get""" - } - } - - http_post { - desc { - en: """Authorization using an external HTTP server (via POST requests).""" - zh: """使用外部 HTTP 服务器授权(POST 请求)。""" - } - label { - en: """http_post""" - zh: """http_post""" - } - } - - method { - desc { - en: """HTTP method.""" - zh: """HTTP 请求方法""" - } - label { - en: """method""" - zh: """method""" - } - } - - url { - desc { - en: """URL of the auth server.""" - zh: """授权 HTTP 服务器地址。""" - } - label { - en: """URL""" - zh: """URL""" - } - } - - headers { - desc { - en: """List of HTTP Headers.""" - zh: """HTTP Headers 列表""" - } - label { - en: """Headers""" - zh: """请求头""" - } - } - - headers_no_content_type { - desc { - en: """List of HTTP headers (without content-type).""" - zh: """HTTP Headers 列表 (无 content-type) 。""" - } - label { - en: """headers_no_content_type""" - zh: """请求头(无 content-type)""" - } - } - - body { - desc { - en: """HTTP request body.""" - zh: """HTTP request body。""" - } - label { - en: """Request Body""" - zh: """Request Body""" - } - } - - request_timeout { - desc { - en: """HTTP request timeout.""" - zh: """HTTP 请求超时时长。""" - } - label { - en: """Request Timeout""" - zh: """请求超时时间""" - } - } - -#==== authz_mnesia - - mnesia { - desc { - en: """Authorization using a built-in database (mnesia).""" - zh: """使用内部数据库授权（mnesia）。""" - } - label { - en: """mnesia""" - zh: """mnesia""" - } - } - -#==== authz_mongo - - mongo_single { - desc { - en: """Authorization using a single MongoDB instance.""" - zh: """使用 MongoDB 授权（单实例）。""" - } - label { - en: """mongo_single""" - zh: """mongo_single""" - } - } - - mongo_rs { - desc { - en: """Authorization using a MongoDB replica set.""" - zh: """使用 MongoDB 授权（副本集模式）""" - } - label { - en: """mongo_rs""" - zh: """mongo_rs""" - } - } - - mongo_sharded { - desc { - en: """Authorization using a sharded MongoDB cluster.""" - zh: """使用 MongoDB 授权（分片集群模式）。""" - } - label { - en: """mongo_sharded""" - zh: """mongo_sharded""" - } - } - - collection { - desc { - en: """`MongoDB` collection containing the authorization data.""" - zh: """`MongoDB` 授权数据集。""" - } - label { - en: """collection""" - zh: """数据集""" - } - } - - filter { - desc { - en: """Conditional expression that defines the filter condition in the query. -Filter supports the following placeholders
- - ${username}: Will be replaced at runtime with Username used by the client when connecting
- - ${clientid}: Will be replaced at runtime with Client ID used by the client when connecting""" - zh: """在查询中定义过滤条件的条件表达式。 -过滤器支持如下占位符：
-- ${username}：将在运行时被替换为客户端连接时使用的用户名
-- ${clientid}：将在运行时被替换为客户端连接时使用的客户端标识符""" - } - label { - en: """Filter""" - zh: """过滤器""" - } - } - -#==== authz_mysql - - mysql { - desc { - en: """Authorization using a MySQL database.""" - zh: """使用 MySOL 数据库授权""" - } - label { - en: """mysql""" - zh: """mysql""" - } - } - -#==== authz_pgsql - - postgresql { - desc { - en: """Authorization using a PostgreSQL database.""" - zh: """使用 PostgreSQL 数据库授权""" - } - label { - en: """postgresql""" - zh: """postgresql""" - } - } - -#==== authz_redis - - redis_single { - desc { - en: """Authorization using a single Redis instance.""" - zh: """使用 Redis 授权（单实例）。""" - } - label { - en: """redis_single""" - zh: """redis_single""" - } - } - - redis_sentinel { - desc { - en: """Authorization using a Redis Sentinel.""" - zh: """使用 Redis 授权（哨兵模式）。""" - } - label { - en: """redis_sentinel""" - zh: """redis_sentinel""" - } - } - - redis_cluster { - desc { - en: """Authorization using a Redis cluster.""" - zh: """使用 Redis 授权（集群模式）。""" - } - label { - en: """redis_cluster""" - zh: """redis_cluster""" - } - } - - cmd { - desc { - en: """Database query used to retrieve authorization data.""" - zh: """访问控制数据查查询命令""" - } - label { - en: """cmd""" - zh: """查询命令""" - } - } - -#==== common field for DBs (except redis) - - query { - desc { - en: """Database query used to retrieve authorization data.""" - zh: """访问控制数据查询语句/查询命令。""" - } - label { - en: """query""" - zh: """查询语句""" - } - } - -#==== metrics field - - metrics { - desc { - en: """The metrics of the resource.""" - zh: """资源统计指标。""" - } - label: { - en: """Metrics""" - zh: """指标""" - } - } - - node_metrics { - desc { - en: """The metrics of the resource for each node.""" - zh: """每个节点上资源的统计指标。""" - } - label: { - en: """Resource Metrics in Node""" - zh: """节点资源指标""" - } - } - - status { - desc { - en: """The status of the resource.""" - zh: """资源状态。""" - } - label: { - en: """Status""" - zh: """状态""" - } - } - - node_status { - desc { - en: """The status of the resource for each node.""" - zh: """每个节点上资源的状态。""" - } - label: { - en: """Resource Status in Node""" - zh: """节点资源状态""" - } - } - - node_error { - desc { - en: """The error of node.""" - zh: """节点上产生的错误。""" - } - label: { - en: """Error in Node""" - zh: """节点产生的错误""" - } - } - - matched { - desc { - en: """Count of this resource is queried.""" - zh: """请求命中次数。""" - } - label: { - en: """Matched""" - zh: """已命中""" - } - } - - success { - desc { - en: """Count of query success.""" - zh: """请求成功次数。""" - } - label: { - en: """Success""" - zh: """成功""" - } - } - - failed { - desc { - en: """Count of query failed.""" - zh: """请求失败次数。""" - } - label: { - en: """Failed""" - zh: """失败""" - } - } - - rate { - desc { - en: """The rate of matched, times/second.""" - zh: """命中速率，单位：次/秒。""" - } - label: { - en: """Rate""" - zh: """速率""" - } - } - - rate_max { - desc { - en: """The max rate of matched, times/second.""" - zh: """最大命中速率，单位：次/秒。""" - } - label: { - en: """Max Rate""" - zh: """最大速率""" - } - } - - rate_last5m { - desc { - en: """The average rate of matched in the last 5 minutes, times/second.""" - zh: """5分钟内平均命中速率，单位：次/秒。""" - } - label: { - en: """Rate in Last 5min""" - zh: """5分钟内速率""" - } - } - - node { - desc { - en: """Node name.""" - zh: """节点名称。""" - } - label: { - en: """Node Name.""" - zh: """节点名称。""" - } - } - - metrics_total { - desc { - en: """The total number of times the authorization rule was triggered.""" - zh: """授权实例被触发的总次数。""" - } - label: { - en: """The Total Number of Times the Authorization Rule was Triggered""" - zh: """授权实例被触发的总次数""" - } - } - - nomatch { - desc { - en: """The number of times that no authorization rules were matched.""" - zh: """没有匹配到任何授权规则的次数。""" - } - label: { - en: """The Number of Times that no Authorization Rules were Matched""" - zh: """没有匹配到任何授权规则的次数""" - } - } - - allow { - desc { - en: """The number of times the authentication was successful.""" - zh: """授权成功的次数。""" - } - label: { - en: """The Number of Times the Authentication was Successful""" - zh: """授权成功次数""" - } - } - - deny { - desc { - en: """The number of authentication failures.""" - zh: """授权失败的次数。""" - } - label: { - en: """The Number of Authentication Failures""" - zh: """授权失败次数""" - } - } } diff --git a/rel/i18n/emqx_auto_subscribe_api.hocon b/rel/i18n/emqx_auto_subscribe_api.hocon index b8d043e9a..df8e87e1a 100644 --- a/rel/i18n/emqx_auto_subscribe_api.hocon +++ b/rel/i18n/emqx_auto_subscribe_api.hocon @@ -1,23 +1,12 @@ emqx_auto_subscribe_api { - list_auto_subscribe_api { - desc { - en: """Get auto subscribe topic list""" - zh: """获取自动订阅主题列表""" - } - } - update_auto_subscribe_api { - desc { - en: """Update auto subscribe topic list""" - zh: """更新自动订阅主题列表""" - } - } +list_auto_subscribe_api.desc: +"""Get auto subscribe topic list""" - update_auto_subscribe_api_response409 { - desc { - en: """Auto Subscribe topics max limit""" - zh: """超出自定订阅主题列表长度限制""" - } - } +update_auto_subscribe_api.desc: +"""Update auto subscribe topic list""" + +update_auto_subscribe_api_response409.desc: +"""Auto Subscribe topics max limit""" } diff --git a/rel/i18n/emqx_auto_subscribe_schema.hocon b/rel/i18n/emqx_auto_subscribe_schema.hocon index ae8c40303..e26ed2546 100644 --- a/rel/i18n/emqx_auto_subscribe_schema.hocon +++ b/rel/i18n/emqx_auto_subscribe_schema.hocon @@ -1,85 +1,48 @@ emqx_auto_subscribe_schema { - auto_subscribe { - desc { - en: """After the device logs in successfully, the subscription is automatically completed for the device through the pre-defined subscription representation. Supports the use of placeholders.""" - zh: """设备登录成功之后，通过预设的订阅表示符，为设备自动完成订阅。支持使用占位符。""" - } - label { - en: """Auto Subscribe""" - zh: """自动订阅""" - } - } - topic { - desc { - en: """Topic name, placeholders are supported. For example: client/${clientid}/username/${username}/host/${host}/port/${port} -Required field, and cannot be empty string""" - zh: """订阅标识符，支持使用占位符，例如 client/${clientid}/username/${username}/host/${host}/port/${port} -必填，且不可为空字符串""" - } - label { - en: """Topic""" - zh: """订阅标识符""" - } - } +auto_subscribe.desc: +"""After the device logs in successfully, the subscription is automatically completed for the device through the pre-defined subscription representation. Supports the use of placeholders.""" - qos { - desc { - en: """Default value 0. Quality of service. +auto_subscribe.label: +"""Auto Subscribe""" + +nl.desc: +"""Default value 0. +MQTT v3.1.1： if you subscribe to the topic published by yourself, you will receive all messages that you published. +MQTT v5: if you set this option as 1 when subscribing, the server will not forward the message you published to you.""" + +nl.label: +"""No Local""" + +qos.desc: +"""Default value 0. Quality of service. At most once (0) At least once (1) Exactly once (2)""" - zh: """缺省值为 0，服务质量， -QoS 0：消息最多传递一次，如果当时客户端不可用，则会丢失该消息。 -QoS 1：消息传递至少 1 次。 -QoS 2：消息仅传送一次。""" - } - label { - en: """Quality of Service""" - zh: """服务质量""" - } - } - rh { - desc { - en: """Default value 0. This option is used to specify whether the server forwards the retained message to the client when establishing a subscription. +qos.label: +"""Quality of Service""" + +rap.desc: +"""Default value 0. This option is used to specify whether the server retains the RETAIN mark when forwarding messages to the client, and this option does not affect the RETAIN mark in the retained message. Therefore, when the option Retain As Publish is set to 0, the client will directly distinguish whether this is a normal forwarded message or a retained message according to the RETAIN mark in the message, instead of judging whether this message is the first received after subscribing(the forwarded message may be sent before the retained message, which depends on the specific implementation of different brokers).""" + +rap.label: +"""Retain As Publish""" + +rh.desc: +"""Default value 0. This option is used to specify whether the server forwards the retained message to the client when establishing a subscription. Retain Handling is equal to 0, as long as the client successfully subscribes, the server will send the retained message. Retain Handling is equal to 1, if the client successfully subscribes and this subscription does not exist previously, the server sends the retained message. After all, sometimes the client re-initiate the subscription just to change the QoS, but it does not mean that it wants to receive the reserved messages again. Retain Handling is equal to 2, even if the client successfully subscribes, the server does not send the retained message.""" - zh: """指定订阅建立时服务端是否向客户端发送保留消息， -可选值 0：只要客户端订阅成功，服务端就发送保留消息。 -可选值 1：客户端订阅成功且该订阅此前不存在，服务端才发送保留消息。毕竟有些时候客户端重新发起订阅可能只是为了改变一下 QoS，并不意味着它想再次接收保留消息。 -可选值 2：即便客户订阅成功，服务端也不会发送保留消息。""" - } - label { - en: """Retain Handling""" - zh: """Retain Handling""" - } - } - rap { - desc { - en: """Default value 0. This option is used to specify whether the server retains the RETAIN mark when forwarding messages to the client, and this option does not affect the RETAIN mark in the retained message. Therefore, when the option Retain As Publish is set to 0, the client will directly distinguish whether this is a normal forwarded message or a retained message according to the RETAIN mark in the message, instead of judging whether this message is the first received after subscribing(the forwarded message may be sent before the retained message, which depends on the specific implementation of different brokers).""" - zh: """缺省值为 0，这一选项用来指定服务端向客户端转发消息时是否要保留其中的 RETAIN 标识，注意这一选项不会影响保留消息中的 RETAIN 标识。因此当 Retain As Publish 选项被设置为 0 时，客户端直接依靠消息中的 RETAIN 标识来区分这是一个正常的转发消息还是一个保留消息，而不是去判断消息是否是自己订阅后收到的第一个消息（转发消息甚至可能会先于保留消息被发送，视不同 Broker 的具体实现而定）。""" - } - label { - en: """Retain As Publish""" - zh: """Retain As Publish""" - } - } +rh.label: +"""Retain Handling""" + +topic.desc: +"""Topic name, placeholders are supported. For example: client/${clientid}/username/${username}/host/${host}/port/${port} +Required field, and cannot be empty string""" + +topic.label: +"""Topic""" - nl { - desc { - en: """Default value 0. -MQTT v3.1.1： if you subscribe to the topic published by yourself, you will receive all messages that you published. -MQTT v5: if you set this option as 1 when subscribing, the server will not forward the message you published to you.""" - zh: """缺省值为0， -MQTT v3.1.1：如果设备订阅了自己发布消息的主题，那么将收到自己发布的所有消息。 -MQTT v5：如果设备在订阅时将此选项设置为 1，那么服务端将不会向设备转发自己发布的消息""" - } - label { - en: """No Local""" - zh: """No Local""" - } - } } diff --git a/rel/i18n/emqx_bridge_api.hocon b/rel/i18n/emqx_bridge_api.hocon index f5d372128..8b7950cdc 100644 --- a/rel/i18n/emqx_bridge_api.hocon +++ b/rel/i18n/emqx_bridge_api.hocon @@ -1,180 +1,100 @@ emqx_bridge_api { - desc_param_path_operation_cluster { - desc { - en: """Operations can be one of: stop, restart""" - zh: """集群可用操作：停止、重新启动""" - } - label: { - en: "Cluster Operation" - zh: "集群可用操作" - } - } +desc_api1.desc: +"""List all created bridges""" - desc_param_path_operation_on_node { - desc { - en: """Operations can be one of: stop, restart""" - zh: """节点可用操作：停止、重新启动""" - } - label: { - en: "Node Operation " - zh: "节点可用操作" - } - } +desc_api1.label: +"""List All Bridges""" - desc_param_path_node { - desc { - en: """The node name, e.g. emqx@127.0.0.1""" - zh: """节点名，比如 emqx@127.0.0.1""" - } - label: { - en: "The node name" - zh: "节点名" - } - } +desc_api2.desc: +"""Create a new bridge by type and name""" - desc_param_path_id { - desc { - en: """The bridge Id. Must be of format {type}:{name}""" - zh: """Bridge ID ， 格式为 {type}:{name}""" - } - label: { - en: "Bridge ID" - zh: "Bridge ID" - } - } +desc_api2.label: +"""Create Bridge""" - desc_param_path_enable { - desc { - en: """Whether to enable this bridge""" - zh: """是否启用桥接""" - } - label: { - en: "Enable bridge" - zh: "启用桥接" - } - } - desc_api1 { - desc { - en: """List all created bridges""" - zh: """列出所有 Birdge""" - } - label: { - en: "List All Bridges" - zh: "列出所有 Bridge" - } - } +desc_api3.desc: +"""Get a bridge by Id""" - desc_api2 { - desc { - en: """Create a new bridge by type and name""" - zh: """通过类型和名字创建 Bridge""" - } - label: { - en: "Create Bridge" - zh: "创建 Bridge" - } - } +desc_api3.label: +"""Get Bridge""" - desc_api3 { - desc { - en: """Get a bridge by Id""" - zh: """通过 ID 获取 Bridge""" - } - label: { - en: "Get Bridge" - zh: "获取 Bridge" - } - } +desc_api4.desc: +"""Update a bridge by Id""" - desc_api4 { - desc { - en: """Update a bridge by Id""" - zh: """通过 ID 更新 Bridge""" - } - label: { - en: "Update Bridge" - zh: "更新 Bridge" - } - } +desc_api4.label: +"""Update Bridge""" - desc_api5 { - desc { - en: """Delete a bridge by Id""" - zh: """通过 ID 删除 Bridge""" - } - label: { - en: "Delete Bridge" - zh: "删除 Bridge" - } - } +desc_api5.desc: +"""Delete a bridge by Id""" - desc_api6 { - desc { - en: """Reset a bridge metrics by Id""" - zh: """通过 ID 重置 Bridge 的计数""" - } - label: { - en: "Reset Bridge Metrics" - zh: "重置 Bridge 计数" - } - } +desc_api5.label: +"""Delete Bridge""" - desc_api7 { - desc { - en: """Stop/Restart bridges on all nodes in the cluster.""" - zh: """停止或启用所有节点上的桥接""" - } - label: { - en: "Cluster Bridge Operate" - zh: "集群 Bridge 操作" - } - } +desc_api6.desc: +"""Reset a bridge metrics by Id""" - desc_api8 { - desc { - en: """Stop/Restart bridges on a specific node.""" - zh: """在某个节点上停止/重新启动 Bridge。""" - } - label: { - en: "Node Bridge Operate" - zh: "单节点 Bridge 操作" - } - } +desc_api6.label: +"""Reset Bridge Metrics""" - desc_api9 { - desc { - en: """Test creating a new bridge by given ID
+desc_api7.desc: +"""Stop/Restart bridges on all nodes in the cluster.""" + +desc_api7.label: +"""Cluster Bridge Operate""" + +desc_api8.desc: +"""Stop/Restart bridges on a specific node.""" + +desc_api8.label: +"""Node Bridge Operate""" + +desc_api9.desc: +"""Test creating a new bridge by given ID
The ID must be of format '{type}:{name}'""" - zh: """通过给定的 ID 测试创建一个新的桥接。
-ID 的格式必须为 ’{type}:{name}”""" - } - label: { - en: "Test Bridge Creation" - zh: "测试桥接创建" - } - } - desc_bridge_metrics { - desc { - en: """Get bridge metrics by Id""" - zh: """通过 Id 来获取桥接的指标信息""" - } - label: { - en: "Get Bridge Metrics" - zh: "获取桥接的指标" - } - } +desc_api9.label: +"""Test Bridge Creation""" - desc_enable_bridge { - desc { - en: """Enable or Disable bridges on all nodes in the cluster.""" - zh: """启用或禁用所有节点上的桥接""" - } - label: { - en: "Cluster Bridge Enable" - zh: "是否启用集群内的桥接" - } - } +desc_bridge_metrics.desc: +"""Get bridge metrics by Id""" + +desc_bridge_metrics.label: +"""Get Bridge Metrics""" + +desc_enable_bridge.desc: +"""Enable or Disable bridges on all nodes in the cluster.""" + +desc_enable_bridge.label: +"""Cluster Bridge Enable""" + +desc_param_path_enable.desc: +"""Whether to enable this bridge""" + +desc_param_path_enable.label: +"""Enable bridge""" + +desc_param_path_id.desc: +"""The bridge Id. Must be of format {type}:{name}""" + +desc_param_path_id.label: +"""Bridge ID""" + +desc_param_path_node.desc: +"""The node name, e.g. emqx@127.0.0.1""" + +desc_param_path_node.label: +"""The node name""" + +desc_param_path_operation_cluster.desc: +"""Operations can be one of: stop, restart""" + +desc_param_path_operation_cluster.label: +"""Cluster Operation""" + +desc_param_path_operation_on_node.desc: +"""Operations can be one of: stop, restart""" + +desc_param_path_operation_on_node.label: +"""Node Operation """ } diff --git a/rel/i18n/emqx_bridge_cassandra.hocon b/rel/i18n/emqx_bridge_cassandra.hocon new file mode 100644 index 000000000..d598d3921 --- /dev/null +++ b/rel/i18n/emqx_bridge_cassandra.hocon @@ -0,0 +1,43 @@ +emqx_bridge_cassandra { + +config_enable.desc: +"""Enable or disable this bridge""" + +config_enable.label: +"""Enable Or Disable Bridge""" + +cql_template.desc: +"""CQL Template""" + +cql_template.label: +"""CQL Template""" + +desc_config.desc: +"""Configuration for a Cassandra bridge.""" + +desc_config.label: +"""Cassandra Bridge Configuration""" + +desc_name.desc: +"""Bridge name.""" + +desc_name.label: +"""Bridge Name""" + +desc_type.desc: +"""The Bridge Type""" + +desc_type.label: +"""Bridge Type""" + +local_topic.desc: +"""The MQTT topic filter to be forwarded to Cassandra. All MQTT 'PUBLISH' messages with the topic +matching the local_topic will be forwarded.
+NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is +configured, then both the data got from the rule and the MQTT messages that match local_topic +will be forwarded.""" + +local_topic.label: +"""Local Topic""" + +} diff --git a/rel/i18n/emqx_bridge_cassandra_connector.hocon b/rel/i18n/emqx_bridge_cassandra_connector.hocon new file mode 100644 index 000000000..b149cce8a --- /dev/null +++ b/rel/i18n/emqx_bridge_cassandra_connector.hocon @@ -0,0 +1,17 @@ +emqx_bridge_cassandra_connector { + +keyspace.desc: +"""Keyspace name to connect to.""" + +keyspace.label: +"""Keyspace""" + +servers.desc: +"""The IPv4 or IPv6 address or the hostname to connect to.
+A host entry has the following form: `Host[:Port][,Host2:Port]`.
+The Cassandra default port 9042 is used if `[:Port]` is not specified.""" + +servers.label: +"""Servers""" + +} diff --git a/rel/i18n/emqx_bridge_gcp_pubsub.hocon b/rel/i18n/emqx_bridge_gcp_pubsub.hocon new file mode 100644 index 000000000..cc255aec3 --- /dev/null +++ b/rel/i18n/emqx_bridge_gcp_pubsub.hocon @@ -0,0 +1,80 @@ +emqx_bridge_gcp_pubsub { + +connect_timeout.desc: +"""The timeout when connecting to the HTTP server.""" + +connect_timeout.label: +"""Connect Timeout""" + +desc_config.desc: +"""Configuration for a GCP PubSub bridge.""" + +desc_config.label: +"""GCP PubSub Bridge Configuration""" + +desc_name.desc: +"""Bridge name, used as a human-readable description of the bridge.""" + +desc_name.label: +"""Bridge Name""" + +desc_type.desc: +"""The Bridge Type""" + +desc_type.label: +"""Bridge Type""" + +local_topic.desc: +"""The MQTT topic filter to be forwarded to GCP PubSub. All MQTT 'PUBLISH' messages with the topic +matching `local_topic` will be forwarded.
+NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is +configured, then both the data got from the rule and the MQTT messages that match local_topic +will be forwarded.""" + +local_topic.label: +"""Local Topic""" + +max_retries.desc: +"""Max retry times if an error occurs when sending a request.""" + +max_retries.label: +"""Max Retries""" + +payload_template.desc: +"""The template for formatting the outgoing messages. If undefined, will send all the available context in JSON format.""" + +payload_template.label: +"""Payload template""" + +pipelining.desc: +"""A positive integer. Whether to send HTTP requests continuously, when set to 1, it means that after each HTTP request is sent, you need to wait for the server to return and then continue to send the next request.""" + +pipelining.label: +"""HTTP Pipelineing""" + +pool_size.desc: +"""The pool size.""" + +pool_size.label: +"""Pool Size""" + +pubsub_topic.desc: +"""The GCP PubSub topic to publish messages to.""" + +pubsub_topic.label: +"""GCP PubSub Topic""" + +request_timeout.desc: +"""Deprecated: Configure the request timeout in the buffer settings.""" + +request_timeout.label: +"""Request Timeout""" + +service_account_json.desc: +"""JSON containing the GCP Service Account credentials to be used with PubSub. +When a GCP Service Account is created (as described in https://developers.google.com/identity/protocols/oauth2/service-account#creatinganaccount), you have the option of downloading the credentials in JSON form. That's the file needed.""" + +service_account_json.label: +"""GCP Service Account Credentials""" + +} diff --git a/rel/i18n/emqx_bridge_kafka.hocon b/rel/i18n/emqx_bridge_kafka.hocon new file mode 100644 index 000000000..ef2e27972 --- /dev/null +++ b/rel/i18n/emqx_bridge_kafka.hocon @@ -0,0 +1,361 @@ +emqx_bridge_kafka { + +connect_timeout.desc: +"""Maximum wait time for TCP connection establishment (including authentication time if enabled).""" + +connect_timeout.label: +"""Connect Timeout""" + +producer_opts.desc: +"""Local MQTT data source and Kafka bridge configs.""" + +producer_opts.label: +"""MQTT to Kafka""" + +min_metadata_refresh_interval.desc: +"""Minimum time interval the client has to wait before refreshing Kafka broker and topic metadata. Setting too small value may add extra load on Kafka.""" + +min_metadata_refresh_interval.label: +"""Min Metadata Refresh Interval""" + +kafka_producer.desc: +"""Kafka Producer configuration.""" + +kafka_producer.label: +"""Kafka Producer""" + +producer_buffer.desc: +"""Configure producer message buffer. + +Tell Kafka producer how to buffer messages when EMQX has more messages to send than Kafka can keep up, or when Kafka is down.""" + +producer_buffer.label: +"""Message Buffer""" + +socket_send_buffer.desc: +"""Fine tune the socket send buffer. The default value is tuned for high throughput.""" + +socket_send_buffer.label: +"""Socket Send Buffer Size""" + +desc_name.desc: +"""Bridge name, used as a human-readable description of the bridge.""" + +desc_name.label: +"""Bridge Name""" + +consumer_offset_commit_interval_seconds.desc: +"""Defines the time interval between two offset commit requests sent for each consumer group.""" + +consumer_offset_commit_interval_seconds.label: +"""Offset Commit Interval""" + +consumer_max_batch_bytes.desc: +"""Set how many bytes to pull from Kafka in each fetch request. Please note that if the configured value is smaller than the message size in Kafka, it may negatively impact the fetch performance.""" + +consumer_max_batch_bytes.label: +"""Fetch Bytes""" + +socket_receive_buffer.desc: +"""Fine tune the socket receive buffer. The default value is tuned for high throughput.""" + +socket_receive_buffer.label: +"""Socket Receive Buffer Size""" + +consumer_topic_mapping.desc: +"""Defines the mapping between Kafka topics and MQTT topics. Must contain at least one item.""" + +consumer_topic_mapping.label: +"""Topic Mapping""" + +producer_kafka_opts.desc: +"""Kafka producer configs.""" + +producer_kafka_opts.label: +"""Kafka Producer""" + +kafka_topic.desc: +"""Kafka topic name""" + +kafka_topic.label: +"""Kafka Topic Name""" + +consumer_kafka_topic.desc: +"""Kafka topic to consume from.""" + +consumer_kafka_topic.label: +"""Kafka Topic""" + +auth_username_password.desc: +"""Username/password based authentication.""" + +auth_username_password.label: +"""Username/password Auth""" + +auth_sasl_password.desc: +"""SASL authentication password.""" + +auth_sasl_password.label: +"""Password""" + +kafka_message_timestamp.desc: +"""Which timestamp to use. The timestamp is expected to be a millisecond precision Unix epoch which can be in string format, e.g. 1661326462115 or '1661326462115'. When the desired data field for this template is not found, or if the found data is not a valid integer, the current system timestamp will be used.""" + +kafka_message_timestamp.label: +"""Message Timestamp""" + +buffer_mode.desc: +"""Message buffer mode. + +memory: Buffer all messages in memory. The messages will be lost in case of EMQX node restart +disk: Buffer all messages on disk. The messages on disk are able to survive EMQX node restart. +hybrid: Buffer message in memory first, when up to certain limit (see segment_bytes config for more information), then start offloading messages to disk, Like memory mode, the messages will be lost in case of EMQX node restart.""" + +buffer_mode.label: +"""Buffer Mode""" + +consumer_mqtt_qos.desc: +"""MQTT QoS used to publish messages consumed from Kafka.""" + +consumer_mqtt_qos.label: +"""QoS""" + +consumer_key_encoding_mode.desc: +"""Defines how the key from the Kafka message is encoded before being forwarded via MQTT. +none Uses the key from the Kafka message unchanged. Note: in this case, the key must be a valid UTF-8 string. +base64 Uses base-64 encoding on the received key.""" + +consumer_key_encoding_mode.label: +"""Key Encoding Mode""" + +auth_gssapi_kerberos.desc: +"""Use GSSAPI/Kerberos authentication.""" + +auth_gssapi_kerberos.label: +"""GSSAPI/Kerberos""" + +consumer_mqtt_opts.desc: +"""Local MQTT message publish.""" + +consumer_mqtt_opts.label: +"""MQTT publish""" + +auth_kerberos_principal.desc: +"""SASL GSSAPI authentication Kerberos principal. For example client_name@MY.KERBEROS.REALM.MYDOMAIN.COM, NOTE: The realm in use has to be configured in /etc/krb5.conf in EMQX nodes.""" + +auth_kerberos_principal.label: +"""Kerberos Principal""" + +socket_opts.desc: +"""Extra socket options.""" + +socket_opts.label: +"""Socket Options""" + +consumer_mqtt_topic.desc: +"""Local topic to which consumed Kafka messages should be published to.""" + +consumer_mqtt_topic.label: +"""MQTT Topic""" + +consumer_offset_reset_policy.desc: +"""Defines from which offset a consumer should start fetching when there is no commit history or when the commit history becomes invalid.""" + +consumer_offset_reset_policy.label: +"""Offset Reset Policy""" + +partition_count_refresh_interval.desc: +"""The time interval for Kafka producer to discover increased number of partitions. +After the number of partitions is increased in Kafka, EMQX will start taking the +discovered partitions into account when dispatching messages per partition_strategy.""" + +partition_count_refresh_interval.label: +"""Partition Count Refresh Interval""" + +max_batch_bytes.desc: +"""Maximum bytes to collect in a Kafka message batch. Most of the Kafka brokers default to a limit of 1 MB batch size. EMQX's default value is less than 1 MB in order to compensate Kafka message encoding overheads (especially when each individual message is very small). When a single message is over the limit, it is still sent (as a single element batch).""" + +max_batch_bytes.label: +"""Max Batch Bytes""" + +required_acks.desc: +"""Required acknowledgements for Kafka partition leader to wait for its followers before it sends back the acknowledgement to EMQX Kafka producer + +all_isr: Require all in-sync replicas to acknowledge. +leader_only: Require only the partition-leader's acknowledgement. +none: No need for Kafka to acknowledge at all.""" + +required_acks.label: +"""Required Acks""" + +metadata_request_timeout.desc: +"""Maximum wait time when fetching metadata from Kafka.""" + +metadata_request_timeout.label: +"""Metadata Request Timeout""" + +desc_type.desc: +"""The Bridge Type""" + +desc_type.label: +"""Bridge Type""" + +socket_nodelay.desc: +"""When set to 'true', TCP buffer is sent as soon as possible. Otherwise, the OS kernel may buffer small TCP packets for a while (40 ms by default).""" + +socket_nodelay.label: +"""No Delay""" + +authentication.desc: +"""Authentication configs.""" + +authentication.label: +"""Authentication""" + +buffer_memory_overload_protection.desc: +"""Applicable when buffer mode is set to memory +EMQX will drop old buffered messages under high memory pressure. The high memory threshold is defined in config sysmon.os.sysmem_high_watermark. NOTE: This config only works on Linux.""" + +buffer_memory_overload_protection.label: +"""Memory Overload Protection""" + +auth_sasl_mechanism.desc: +"""SASL authentication mechanism.""" + +auth_sasl_mechanism.label: +"""Mechanism""" + +config_enable.desc: +"""Enable (true) or disable (false) this Kafka bridge.""" + +config_enable.label: +"""Enable or Disable""" + +consumer_mqtt_payload.desc: +"""The template for transforming the incoming Kafka message. By default, it will use JSON format to serialize inputs from the Kafka message. Such fields are: +headers: an object containing string key-value pairs. +key: Kafka message key (uses the chosen key encoding). +offset: offset for the message. +topic: Kafka topic. +ts: message timestamp. +ts_type: message timestamp type, which is one of create, append or undefined. +value: Kafka message value (uses the chosen value encoding).""" + +consumer_mqtt_payload.label: +"""MQTT Payload Template""" + +consumer_opts.desc: +"""Local MQTT publish and Kafka consumer configs.""" + +consumer_opts.label: +"""MQTT to Kafka""" + +kafka_consumer.desc: +"""Kafka Consumer configuration.""" + +kafka_consumer.label: +"""Kafka Consumer""" + +desc_config.desc: +"""Configuration for a Kafka bridge.""" + +desc_config.label: +"""Kafka Bridge Configuration""" + +consumer_value_encoding_mode.desc: +"""Defines how the value from the Kafka message is encoded before being forwarded via MQTT. +none Uses the value from the Kafka message unchanged. Note: in this case, the value must be a valid UTF-8 string. +base64 Uses base-64 encoding on the received value.""" + +consumer_value_encoding_mode.label: +"""Value Encoding Mode""" + +buffer_per_partition_limit.desc: +"""Number of bytes allowed to buffer for each Kafka partition. When this limit is exceeded, old messages will be dropped in a trade for credits for new messages to be buffered.""" + +buffer_per_partition_limit.label: +"""Per-partition Buffer Limit""" + +bootstrap_hosts.desc: +"""A comma separated list of Kafka host[:port] endpoints to bootstrap the client. Default port number is 9092.""" + +bootstrap_hosts.label: +"""Bootstrap Hosts""" + +consumer_max_rejoin_attempts.desc: +"""Maximum number of times allowed for a member to re-join the group. If the consumer group can not reach balance after this configured number of attempts, the consumer group member will restart after a delay.""" + +consumer_max_rejoin_attempts.label: +"""Max Rejoin Attempts""" + +kafka_message_key.desc: +"""Template to render Kafka message key. If the template is rendered into a NULL value (i.e. there is no such data field in Rule Engine context) then Kafka's NULL (but not empty string) is used.""" + +kafka_message_key.label: +"""Message Key""" + +kafka_message.desc: +"""Template to render a Kafka message.""" + +kafka_message.label: +"""Kafka Message Template""" + +mqtt_topic.desc: +"""MQTT topic or topic filter as data source (bridge input). If rule action is used as data source, this config should be left empty, otherwise messages will be duplicated in Kafka.""" + +mqtt_topic.label: +"""Source MQTT Topic""" + +kafka_message_value.desc: +"""Template to render Kafka message value. If the template is rendered into a NULL value (i.e. there is no such data field in Rule Engine context) then Kafka's NULL (but not empty string) is used.""" + +kafka_message_value.label: +"""Message Value""" + +partition_strategy.desc: +"""Partition strategy is to tell the producer how to dispatch messages to Kafka partitions. + +random: Randomly pick a partition for each message +key_dispatch: Hash Kafka message key to a partition number""" + +partition_strategy.label: +"""Partition Strategy""" + +buffer_segment_bytes.desc: +"""Applicable when buffer mode is set to disk or hybrid. +This value is to specify the size of each on-disk buffer file.""" + +buffer_segment_bytes.label: +"""Segment File Bytes""" + +consumer_kafka_opts.desc: +"""Kafka consumer configs.""" + +consumer_kafka_opts.label: +"""Kafka Consumer""" + +max_inflight.desc: +"""Maximum number of batches allowed for Kafka producer (per-partition) to send before receiving acknowledgement from Kafka. Greater value typically means better throughput. However, there can be a risk of message reordering when this value is greater than 1.""" + +max_inflight.label: +"""Max Inflight""" + +auth_sasl_username.desc: +"""SASL authentication username.""" + +auth_sasl_username.label: +"""Username""" + +auth_kerberos_keytab_file.desc: +"""SASL GSSAPI authentication Kerberos keytab file path. NOTE: This file has to be placed in EMQX nodes, and the EMQX service runner user requires read permission.""" + +auth_kerberos_keytab_file.label: +"""Kerberos keytab file""" + +compression.desc: +"""Compression method.""" + +compression.label: +"""Compression""" + +} diff --git a/rel/i18n/emqx_bridge_mqtt_schema.hocon b/rel/i18n/emqx_bridge_mqtt_schema.hocon index b935b360c..e05c4fb0a 100644 --- a/rel/i18n/emqx_bridge_mqtt_schema.hocon +++ b/rel/i18n/emqx_bridge_mqtt_schema.hocon @@ -1,34 +1,21 @@ emqx_bridge_mqtt_schema { - config { - desc { - en: """The config for MQTT Bridges.""" - zh: """MQTT Bridge 的配置。""" - } - label: { - en: "Config" - zh: "配置" - } - } - desc_type { - desc { - en: """The bridge type.""" - zh: """Bridge 的类型""" - } - label: { - en: "Bridge Type" - zh: "Bridge 类型" - } - } - desc_name { - desc { - en: """Bridge name, used as a human-readable description of the bridge.""" - zh: """Bridge 名字，Bridge 的可读描述""" - } - label: { - en: "Bridge Name" - zh: "Bridge 名字" - } - } +config.desc: +"""The config for MQTT Bridges.""" + +config.label: +"""Config""" + +desc_name.desc: +"""Bridge name, used as a human-readable description of the bridge.""" + +desc_name.label: +"""Bridge Name""" + +desc_type.desc: +"""The bridge type.""" + +desc_type.label: +"""Bridge Type""" } diff --git a/rel/i18n/emqx_bridge_schema.hocon b/rel/i18n/emqx_bridge_schema.hocon index de4ceb0d5..1d3053f73 100644 --- a/rel/i18n/emqx_bridge_schema.hocon +++ b/rel/i18n/emqx_bridge_schema.hocon @@ -1,324 +1,158 @@ emqx_bridge_schema { - desc_enable { - desc { - en: """Enable or disable this bridge""" - zh: """启用/禁用 Bridge""" - } - label: { - en: "Enable Or Disable Bridge" - zh: "启用/禁用 Bridge" - } - } +bridges_mqtt.desc: +"""MQTT bridges to/from another MQTT broker""" - desc_metrics { - desc { - en: """The metrics of the bridge""" - zh: """Bridge 的计数""" - } - label: { - en: "Bridge Metrics" - zh: "Bridge 计数" - } - } +bridges_mqtt.label: +"""MQTT Bridge""" - desc_node_metrics { - desc { - en: """The metrics of the bridge for each node""" - zh: """每个节点的 Bridge 计数""" - } - label: { - en: "Each Node Bridge Metircs" - zh: "每个节点的 Bridge 计数" - } - } +bridges_webhook.desc: +"""WebHook to an HTTP server.""" - desc_status { - desc { - en: """The status of the bridge
+bridges_webhook.label: +"""WebHook""" + +desc_bridges.desc: +"""Configuration for MQTT bridges.""" + +desc_bridges.label: +"""MQTT Bridge Configuration""" + +desc_enable.desc: +"""Enable or disable this bridge""" + +desc_enable.label: +"""Enable Or Disable Bridge""" + +desc_metrics.desc: +"""Bridge metrics.""" + +desc_metrics.label: +"""Bridge Metrics""" + +desc_node_metrics.desc: +"""Node metrics.""" + +desc_node_metrics.label: +"""Node Metrics""" + +desc_node_name.desc: +"""The node name.""" + +desc_node_name.label: +"""Node Name""" + +desc_node_status.desc: +"""Node status.""" + +desc_node_status.label: +"""Node Status""" + +desc_status.desc: +"""The status of the bridge
- connecting: the initial state before any health probes were made.
- connected: when the bridge passes the health probes.
- disconnected: when the bridge can not pass health probes.
- stopped: when the bridge resource is requested to be stopped.
- inconsistent: When not all the nodes are at the same status.""" - zh: """Bridge 的连接状态
-- connecting: 启动时的初始状态。
-- connected: 桥接驱动健康检查正常。
-- disconnected: 当桥接无法通过健康检查。
-- stopped: 桥接处于停用状态。
-- inconsistent: 集群中有各节点汇报的状态不一致。""" - } - label: { - en: "Bridge Status" - zh: "Bridge 状态" - } - } - desc_status_reason { - desc { - en: "This is the reason given in case a bridge is failing to connect." - zh: "桥接连接失败的原因。" - } - label: { - en: "Failure reason" - zh: "失败原因" - } - } +desc_status.label: +"""Bridge Status""" - desc_node_status { - desc { - en: """The status of the bridge for each node. -- connecting: the initial state before any health probes were made.
-- connected: when the bridge passes the health probes.
-- disconnected: when the bridge can not pass health probes.
-- stopped: when the bridge resource is requested to be stopped.""" - zh: """每个节点的 Bridge 状态 -- connecting: 启动时的初始状态。
-- connected: 桥接驱动健康检查正常。
-- disconnected: 当桥接无法通过健康检查。
-- stopped: 桥接处于停用状态。""" - } - label: { - en: "Node Bridge Status" - zh: "每个节点的 Bridge 状态" - } - } +desc_status_reason.desc: +"""This is the reason given in case a bridge is failing to connect.""" - bridges_webhook { - desc { - en: """WebHook to an HTTP server.""" - zh: """转发消息到 HTTP 服务器的 WebHook""" - } - label: { - en: "WebHook" - zh: "WebHook" - } - } +desc_status_reason.label: +"""Failure reason""" +metric_dropped.desc: +"""Count of messages dropped.""" - bridges_mqtt { - desc { - en: """MQTT bridges to/from another MQTT broker""" - zh: """桥接到另一个 MQTT Broker 的 MQTT Bridge""" - } - label: { - en: "MQTT Bridge" - zh: "MQTT Bridge" - } - } +metric_dropped.label: +"""Dropped""" - metric_dropped { - desc { - en: """Count of messages dropped.""" - zh: """被丢弃的消息个数。""" - } - label: { - en: "Dropped" - zh: "丢弃" - } - } +metric_dropped_other.desc: +"""Count of messages dropped due to other reasons.""" - metric_dropped_other { - desc { - en: """Count of messages dropped due to other reasons.""" - zh: """因为其他原因被丢弃的消息个数。""" - } - label: { - en: "Dropped Other" - zh: "其他丢弃" - } - } - metric_dropped_queue_full { - desc { - en: """Count of messages dropped due to the queue is full.""" - zh: """因为队列已满被丢弃的消息个数。""" - } - label: { - en: "Dropped Queue Full" - zh: "队列已满被丢弃" - } - } - metric_dropped_resource_not_found { - desc { - en: """Count of messages dropped due to the resource is not found.""" - zh: """因为资源不存在被丢弃的消息个数。""" - } - label: { - en: "Dropped Resource NotFound" - zh: "资源不存在被丢弃" - } - } - metric_dropped_resource_stopped { - desc { - en: """Count of messages dropped due to the resource is stopped.""" - zh: """因为资源已停用被丢弃的消息个数。""" - } - label: { - en: "Dropped Resource Stopped" - zh: "资源停用被丢弃" - } - } - metric_matched { - desc { - en: """Count of this bridge is matched and queried.""" - zh: """Bridge 被匹配到（被请求）的次数。""" - } - label: { - en: "Matched" - zh: "匹配次数" - } - } +metric_dropped_other.label: +"""Dropped Other""" - metric_queuing { - desc { - en: """Count of messages that are currently queuing.""" - zh: """当前被缓存到磁盘队列的消息个数。""" - } - label: { - en: "Queued" - zh: "被缓存" - } - } - metric_retried { - desc { - en: """Times of retried.""" - zh: """重试的次数。""" - } - label: { - en: "Retried" - zh: "已重试" - } - } +metric_dropped_queue_full.desc: +"""Count of messages dropped due to the queue is full.""" - metric_sent_failed { - desc { - en: """Count of messages that sent failed.""" - zh: """发送失败的消息个数。""" - } - label: { - en: "Sent Failed" - zh: "发送失败" - } - } +metric_dropped_queue_full.label: +"""Dropped Queue Full""" - metric_inflight { - desc { - en: """Count of messages that were sent asynchronously but ACKs are not yet received.""" - zh: """已异步地发送但没有收到 ACK 的消息个数。""" - } - label: { - en: "Sent Inflight" - zh: "已发送未确认" - } - } - metric_sent_success { - desc { - en: """Count of messages that sent successfully.""" - zh: """已经发送成功的消息个数。""" - } - label: { - en: "Sent Success" - zh: "发送成功" - } - } +metric_dropped_resource_not_found.desc: +"""Count of messages dropped due to the resource is not found.""" - metric_rate { - desc { - en: """The rate of matched, times/second""" - zh: """执行操作的速率，次/秒""" - } - label: { - en: "Rate" - zh: "速率" - } - } +metric_dropped_resource_not_found.label: +"""Dropped Resource NotFound""" - metric_rate_max { - desc { - en: """The max rate of matched, times/second""" - zh: """执行操作的最大速率，次/秒""" - } - label: { - en: "Max Rate Of Matched" - zh: "执行操作的最大速率" - } - } +metric_dropped_resource_stopped.desc: +"""Count of messages dropped due to the resource is stopped.""" - metric_rate_last5m { - desc { - en: """The average rate of matched in the last 5 minutes, times/second""" - zh: """5 分钟平均速率，次/秒""" - } - label: { - en: "Last 5 Minutes Rate" - zh: "5 分钟平均速率" - } - } +metric_dropped_resource_stopped.label: +"""Dropped Resource Stopped""" - metric_received { - desc { - en: """Count of messages that is received from the remote system.""" - zh: """从远程系统收到的消息个数。""" - } - label: { - en: "Received" - zh: "已接收" - } - } +metric_inflight.desc: +"""Count of messages that were sent asynchronously but ACKs are not yet received.""" - desc_bridges { - desc { - en: """Configuration for MQTT bridges.""" - zh: """MQTT Bridge 配置""" - } - label: { - en: "MQTT Bridge Configuration" - zh: "MQTT Bridge 配置" - } - } +metric_inflight.label: +"""Sent Inflight""" - desc_metrics { - desc { - en: """Bridge metrics.""" - zh: """Bridge 计数""" - } - label: { - en: "Bridge Metrics" - zh: "Bridge 计数" - } - } +metric_matched.desc: +"""Count of this bridge is matched and queried.""" - desc_node_metrics { - desc { - en: """Node metrics.""" - zh: """节点的计数器""" - } - label: { - en: "Node Metrics" - zh: "节点的计数器" - } - } +metric_matched.label: +"""Matched""" - desc_node_status { - desc { - en: """Node status.""" - zh: """节点的状态""" - } - label: { - en: "Node Status" - zh: "节点的状态" - } - } +metric_queuing.desc: +"""Count of messages that are currently queuing.""" - desc_node_name { - desc { - en: """The node name.""" - zh: """节点的名字""" - } - label: { - en: "Node Name" - zh: "节点名字" - } - } +metric_queuing.label: +"""Queued""" + +metric_rate.desc: +"""The rate of matched, times/second""" + +metric_rate.label: +"""Rate""" + +metric_rate_last5m.desc: +"""The average rate of matched in the last 5 minutes, times/second""" + +metric_rate_last5m.label: +"""Last 5 Minutes Rate""" + +metric_rate_max.desc: +"""The max rate of matched, times/second""" + +metric_rate_max.label: +"""Max Rate Of Matched""" + +metric_received.desc: +"""Count of messages that is received from the remote system.""" + +metric_received.label: +"""Received""" + +metric_retried.desc: +"""Times of retried.""" + +metric_retried.label: +"""Retried""" + +metric_sent_failed.desc: +"""Count of messages that sent failed.""" + +metric_sent_failed.label: +"""Sent Failed""" + +metric_sent_success.desc: +"""Count of messages that sent successfully.""" + +metric_sent_success.label: +"""Sent Success""" } diff --git a/rel/i18n/emqx_bridge_webhook_schema.hocon b/rel/i18n/emqx_bridge_webhook_schema.hocon index cf86b63e0..4037893bd 100644 --- a/rel/i18n/emqx_bridge_webhook_schema.hocon +++ b/rel/i18n/emqx_bridge_webhook_schema.hocon @@ -1,162 +1,92 @@ emqx_bridge_webhook_schema { - config_enable { - desc { - en: """Enable or disable this bridge""" - zh: """启用/禁用 Bridge""" - } - label: { - en: "Enable Or Disable Bridge" - zh: "启用/禁用 Bridge" - } - } - config_direction { - desc { - en: """Deprecated, The direction of this bridge, MUST be 'egress'""" - zh: """已废弃，Bridge 的方向，必须是 egress""" - } - label: { - en: "Bridge Direction" - zh: "Bridge 方向" - } - } - config_url { - desc { - en: """The URL of the HTTP Bridge.
-Template with variables is allowed in the path, but variables cannot be used in the scheme, host, -or port part.
-For example, http://localhost:9901/${topic} is allowed, but - http://${host}:9901/message or http://localhost:${port}/message -is not allowed.""" - zh: """HTTP Bridge 的 URL。
-路径中允许使用带变量的模板，但是 host， port 不允许使用变量模板。
-例如， http://localhost:9901/${topic} 是允许的， -但是 http://${host}:9901/message -或 http://localhost:${port}/message -不允许。""" - } - label: { - en: "HTTP Bridge" - zh: "HTTP Bridge" - } - } - - config_local_topic { - desc { - en: """The MQTT topic filter to be forwarded to the HTTP server. All MQTT 'PUBLISH' messages with the topic -matching the local_topic will be forwarded.
-NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is -configured, then both the data got from the rule and the MQTT messages that match local_topic -will be forwarded.""" - zh: """发送到 'local_topic' 的消息都会转发到 HTTP 服务器。
-注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发到 HTTP 服务器。""" - } - label: { - en: "Local Topic" - zh: "本地 Topic" - } - } - - config_method { - desc { - en: """The method of the HTTP request. All the available methods are: post, put, get, delete.
-Template with variables is allowed.""" - zh: """HTTP 请求的方法。 所有可用的方法包括：post、put、get、delete。
-允许使用带有变量的模板。""" - } - label: { - en: "HTTP Method" - zh: "HTTP 请求方法" - } - } - - config_headers { - desc { - en: """The headers of the HTTP request.
-Template with variables is allowed.""" - zh: """HTTP 请求的标头。
-允许使用带有变量的模板。""" - } - label: { - en: "HTTP Header" - zh: "HTTP 请求标头" - } - } - - config_body { - desc { - en: """The body of the HTTP request.
+config_body.desc: +"""The body of the HTTP request.
If not provided, the body will be a JSON object of all the available fields.
There, 'all the available fields' means the context of a MQTT message when this webhook is triggered by receiving a MQTT message (the `local_topic` is set), or the context of the event when this webhook is triggered by a rule (i.e. this webhook is used as an action of a rule).
Template with variables is allowed.""" - zh: """HTTP 请求的正文。
-如果没有设置该字段，请求正文将是包含所有可用字段的 JSON object。
-如果该 webhook 是由于收到 MQTT 消息触发的，'所有可用字段' 将是 MQTT 消息的 -上下文信息；如果该 webhook 是由于规则触发的，'所有可用字段' 则为触发事件的上下文信息。
-允许使用带有变量的模板。""" - } - label: { - en: "HTTP Body" - zh: "HTTP 请求正文" - } - } - config_request_timeout { - desc { - en: """HTTP request timeout.""" - zh: """HTTP 请求超时""" - } - label: { - en: "HTTP Request Timeout" - zh: "HTTP 请求超时" - } - } +config_body.label: +"""HTTP Body""" - config_max_retries { - desc { - en: """HTTP request max retry times if failed.""" - zh: """HTTP 请求失败最大重试次数""" - } - label: { - en: "HTTP Request Max Retries" - zh: "HTTP 请求重试次数" - } - } +config_direction.desc: +"""Deprecated, The direction of this bridge, MUST be 'egress'""" - desc_type { - desc { - en: """The Bridge Type""" - zh: """Bridge 类型""" - } - label: { - en: "Bridge Type" - zh: "Bridge 类型" - } - } +config_direction.label: +"""Bridge Direction""" - desc_name { - desc { - en: """Bridge name, used as a human-readable description of the bridge.""" - zh: """Bridge 名字，Bridge 的可读描述""" - } - label: { - en: "Bridge Name" - zh: "Bridge 名字" - } - } +config_enable.desc: +"""Enable or disable this bridge""" - desc_config { - desc { - en: """Configuration for an HTTP bridge.""" - zh: """HTTP Bridge 配置""" - } - label: { - en: "HTTP Bridge Configuration" - zh: "HTTP Bridge 配置" - } - } +config_enable.label: +"""Enable Or Disable Bridge""" + +config_headers.desc: +"""The headers of the HTTP request.
+Template with variables is allowed.""" + +config_headers.label: +"""HTTP Header""" + +config_local_topic.desc: +"""The MQTT topic filter to be forwarded to the HTTP server. All MQTT 'PUBLISH' messages with the topic +matching the local_topic will be forwarded.
+NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is +configured, then both the data got from the rule and the MQTT messages that match local_topic +will be forwarded.""" + +config_local_topic.label: +"""Local Topic""" + +config_max_retries.desc: +"""HTTP request max retry times if failed.""" + +config_max_retries.label: +"""HTTP Request Max Retries""" + +config_method.desc: +"""The method of the HTTP request. All the available methods are: post, put, get, delete.
+Template with variables is allowed.""" + +config_method.label: +"""HTTP Method""" + +config_request_timeout.desc: +"""HTTP request timeout.""" + +config_request_timeout.label: +"""HTTP Request Timeout""" + +config_url.desc: +"""The URL of the HTTP Bridge.
+Template with variables is allowed in the path, but variables cannot be used in the scheme, host, +or port part.
+For example, http://localhost:9901/${topic} is allowed, but + http://${host}:9901/message or http://localhost:${port}/message +is not allowed.""" + +config_url.label: +"""HTTP Bridge""" + +desc_config.desc: +"""Configuration for an HTTP bridge.""" + +desc_config.label: +"""HTTP Bridge Configuration""" + +desc_name.desc: +"""Bridge name, used as a human-readable description of the bridge.""" + +desc_name.label: +"""Bridge Name""" + +desc_type.desc: +"""The Bridge Type""" + +desc_type.label: +"""Bridge Type""" } diff --git a/rel/i18n/emqx_coap_api.hocon b/rel/i18n/emqx_coap_api.hocon index 77ca40c00..14f644a87 100644 --- a/rel/i18n/emqx_coap_api.hocon +++ b/rel/i18n/emqx_coap_api.hocon @@ -1,58 +1,27 @@ emqx_coap_api { - send_coap_request { - desc { - en: """Send a CoAP request message to the client""" - zh: """发送 CoAP 消息到指定客户端""" - } - } +content_type.desc: +"""Payload type""" - token { - desc { - en: """Message token, can be empty""" - zh: """消息 Token, 可以为空""" - } - } +message_id.desc: +"""Message ID""" - method { - desc { - en: """Request method type""" - zh: """请求 Method 类型""" - } - } +method.desc: +"""Request method type""" - timeout { - desc { - en: """Timespan for response""" - zh: """请求超时时间""" - } - } +payload.desc: +"""The content of the payload""" - content_type { - desc { - en: """Payload type""" - zh: """Payload 类型""" - } - } +response_code.desc: +"""Response code""" - payload { - desc { - en: """The content of the payload""" - zh: """Payload 内容""" - } - } +send_coap_request.desc: +"""Send a CoAP request message to the client""" - message_id { - desc { - en: """Message ID""" - zh: """消息 ID""" - } - } +timeout.desc: +"""Timespan for response""" + +token.desc: +"""Message token, can be empty""" - response_code { - desc { - en: """Response code""" - zh: """应答码""" - } - } } diff --git a/rel/i18n/emqx_coap_schema.hocon b/rel/i18n/emqx_coap_schema.hocon new file mode 100644 index 000000000..322003a3f --- /dev/null +++ b/rel/i18n/emqx_coap_schema.hocon @@ -0,0 +1,38 @@ +emqx_coap_schema { + +coap.desc: +"""The CoAP Gateway configuration. +This gateway is implemented based on RFC-7252 and https://core-wg.github.io/coap-pubsub/draft-ietf-core-pubsub.html""" + +coap_connection_required.desc: +"""Enable or disable connection mode. +Connection mode is a feature of non-standard protocols. When connection mode is enabled, it is necessary to maintain the creation, authentication and alive of connection resources""" + +coap_heartbeat.desc: +"""The gateway server required minimum heartbeat interval. +When connection mode is enabled, this parameter is used to set the minimum heartbeat interval for the connection to be alive""" + +coap_notify_type.desc: +"""The Notification Message will be delivered to the CoAP client if a new message received on an observed topic. +The type of delivered coap message can be set to:
+ - non: Non-confirmable;
+ - con: Confirmable;
+ - qos: Mapping from QoS type of received message, QoS0 -> non, QoS1,2 -> con""" + +coap_publish_qos.desc: +"""The Default QoS Level indicator for publish request. +This option specifies the QoS level for the CoAP Client when publishing a message to EMQX PUB/SUB system, if the publish request is not carried `qos` option. The indicator can be set to:
+ - qos0, qos1, qos2: Fixed default QoS level
+ - coap: Dynamic QoS level by the message type of publish request
+ * qos0: If the publish request is non-confirmable
+ * qos1: If the publish request is confirmable""" + +coap_subscribe_qos.desc: +"""The Default QoS Level indicator for subscribe request. +This option specifies the QoS level for the CoAP Client when establishing a subscription membership, if the subscribe request is not carried `qos` option. The indicator can be set to:
+ - qos0, qos1, qos2: Fixed default QoS level
+ - coap: Dynamic QoS level by the message type of subscribe request
+ * qos0: If the subscribe request is non-confirmable
+ * qos1: If the subscribe request is confirmable""" + +} diff --git a/rel/i18n/emqx_conf_schema.hocon b/rel/i18n/emqx_conf_schema.hocon index 2f0c6b938..f4016d32e 100644 --- a/rel/i18n/emqx_conf_schema.hocon +++ b/rel/i18n/emqx_conf_schema.hocon @@ -1,411 +1,456 @@ emqx_conf_schema { - cluster_name { - desc { - en: """Human-friendly name of the EMQX cluster.""" - zh: """EMQX集群名称。每个集群都有一个唯一的名称。服务发现时会用于做路径的一部分。""" - } - label { - en: "Cluster Name" - zh: "集群名称" - } - } +common_handler_drop_mode_qlen.desc: +"""When the number of buffered log events is larger than this value, the new log events are dropped. +When drop mode is activated or deactivated, a message is printed in the logs.""" - process_limit { - desc { - en: """Maximum number of simultaneously existing processes for this Erlang system. -The actual maximum chosen may be much larger than the Number passed. -For more information, see: https://www.erlang.org/doc/man/erl.html""" +common_handler_drop_mode_qlen.label: +"""Queue Length before Entering Drop Mode""" - zh: """Erlang系统同时存在的最大进程数。 -实际选择的最大值可能比设置的数字大得多。 -参考: https://www.erlang.org/doc/man/erl.html""" - } - label { - en: "Erlang Process Limit" - zh: "Erlang 最大进程数" - } - } +cluster_mcast_addr.desc: +"""Multicast IPv4 address.""" - max_ports { - desc { - en: """Maximum number of simultaneously existing ports for this Erlang system. -The actual maximum chosen may be much larger than the Number passed. -For more information, see: https://www.erlang.org/doc/man/erl.html""" +cluster_mcast_addr.label: +"""Cluster Multicast Address""" - zh: """Erlang系统同时存在的最大端口数。 -实际选择的最大值可能比设置的数字大得多。 -参考: https://www.erlang.org/doc/man/erl.html""" - } - label { - en: "Erlang Port Limit" - zh: "Erlang 最大端口数" - } - } +desc_cluster_dns.desc: +"""Service discovery via DNS SRV records.""" - dist_buffer_size { - desc { - en: """Erlang's distribution buffer busy limit in kilobytes.""" - zh: """Erlang分布式缓冲区的繁忙阈值，单位是KB。""" - } - label { - en: "Erlang's dist buffer size(KB)" - zh: "Erlang分布式缓冲区的繁忙阈值(KB)" - } - } +desc_cluster_dns.label: +"""Cluster DNS""" - max_ets_tables { - desc { - en: """Max number of ETS tables""" - zh: """Erlang ETS 表的最大数量""" - } - label { - en: "Max number of ETS tables" - zh: "Erlang 表的最大数量" - } - } +cluster_dns_name.desc: +"""The domain name from which to discover peer EMQX nodes' IP addresses. +Applicable when cluster.discovery_strategy = dns""" - cluster_discovery_strategy { - desc { - en: """Service discovery method for the cluster nodes. Possible values are: -- manual: Use emqx ctl cluster command to manage cluster.
-- static: Configure static nodes list by setting seeds in config file.
-- dns: Use DNS A record to discover peer nodes.
-- etcd: Use etcd to discover peer nodes.
-- k8s: Use Kubernetes API to discover peer pods.""" +cluster_dns_name.label: +"""Cluster Dns Name""" - zh: """集群节点发现方式。可选值为: -- manual: 使用 emqx ctl cluster 命令管理集群。
-- static: 配置静态节点。配置几个固定的节点，新节点通过连接固定节点中的某一个来加入集群。
-- dns: 使用 DNS A 记录的方式发现节点。
-- etcd: 使用 etcd 发现节点。
-- k8s: 使用 Kubernetes API 发现节点。""" - } - label { - en: "Cluster Discovery Strategy" - zh: "集群服务发现策略" - } - } +rpc_keyfile.desc: +"""Path to the private key file for the rpc.certfile.
+Note: contents of this file are secret, so it's necessary to set permissions to 600.""" - cluster_autoclean { - desc { - en: """Remove disconnected nodes from the cluster after this interval.""" - zh: """指定多久之后从集群中删除离线节点。""" - } - label { - en: "Cluster Auto Clean" - zh: "自动删除离线节点时间" - } - } +rpc_keyfile.label: +"""RPC Keyfile""" - cluster_autoheal { - desc { - en: """If true, the node will try to heal network partitions automatically.""" - zh: """集群脑裂自动恢复机制开关。""" - } - label { - en: "Cluster Auto Heal" - zh: "节点脑裂自动修复机制" - } - } +cluster_mcast_recbuf.desc: +"""Size of the kernel-level buffer for incoming datagrams.""" - cluster_proto_dist { - desc { - en: """The Erlang distribution protocol for the cluster.
+cluster_mcast_recbuf.label: +"""Cluster Muticast Sendbuf""" + +cluster_autoheal.desc: +"""If true, the node will try to heal network partitions automatically.""" + +cluster_autoheal.label: +"""Cluster Auto Heal""" + +log_overload_kill_enable.desc: +"""Enable log handler overload kill feature.""" + +log_overload_kill_enable.label: +"""Log Handler Overload Kill""" + +node_etc_dir.desc: +"""etc dir for the node""" + +node_etc_dir.label: +"""Etc Dir""" + +cluster_proto_dist.desc: +"""The Erlang distribution protocol for the cluster.
- inet_tcp: IPv4 TCP
- inet_tls: IPv4 TLS, works together with etc/ssl_dist.conf""" - zh: """分布式 Erlang 集群协议类型。可选值为:
-- inet_tcp: 使用 IPv4
-- inet_tls: 使用 TLS，需要配合 etc/ssl_dist.conf 一起使用。""" - } - label { - en: "Cluster Protocol Distribution" - zh: "集群内部通信协议" - } - } - cluster_static_seeds { - desc { - en: """List EMQX node names in the static cluster. See node.name.""" - zh: """集群中的EMQX节点名称列表， -指定固定的节点列表，多个节点间使用逗号 , 分隔。 -当 cluster.discovery_strategy 为 static 时，此配置项才有效。 -适合于节点数量较少且固定的集群。""" - } - label { - en: "Cluster Static Seeds" - zh: "集群静态节点" - } - } +cluster_proto_dist.label: +"""Cluster Protocol Distribution""" - cluster_mcast_addr { - desc { - en: """Multicast IPv4 address.""" - zh: """指定多播 IPv4 地址。 -当 cluster.discovery_strategy 为 mcast 时，此配置项才有效。""" - } - label { - en: "Cluster Multicast Address" - zh: "多播地址" - } - } +log_burst_limit_enable.desc: +"""Enable log burst control feature.""" - cluster_mcast_ports { - desc { - en: """List of UDP ports used for service discovery.
-Note: probe messages are broadcast to all the specified ports.""" +log_burst_limit_enable.label: +"""Enable Burst""" - zh: """指定多播端口。如有多个端口使用逗号 , 分隔。 -当 cluster.discovery_strategy 为 mcast 时，此配置项才有效。""" - } - label { - en: "Cluster Multicast Ports" - zh: "多播端口" - } - } +dist_buffer_size.desc: +"""Erlang's distribution buffer busy limit in kilobytes.""" - cluster_mcast_iface { - desc { - en: """Local IP address the node discovery service needs to bind to.""" - zh: """指定节点发现服务需要绑定到本地 IP 地址。 -当 cluster.discovery_strategy 为 mcast 时，此配置项才有效。""" - } - label { - en: "Cluster Multicast Interface" - zh: "多播绑定地址" - } - } +dist_buffer_size.label: +"""Erlang's dist buffer size(KB)""" - cluster_mcast_ttl { - desc { - en: """Time-to-live (TTL) for the outgoing UDP datagrams.""" - zh: """指定多播的 Time-To-Live 值。 -当 cluster.discovery_strategy 为 mcast 时，此配置项才有效。""" - } - label { - en: "Cluster Multicast TTL" - zh: "多播TTL" - } - } +common_handler_max_depth.desc: +"""Maximum depth for Erlang term log formatting and Erlang process message queue inspection.""" - cluster_mcast_loop { - desc { - en: """If true, loop UDP datagrams back to the local socket.""" - zh: """设置多播的报文是否投递到本地回环地址。 -当 cluster.discovery_strategy 为 mcast 时，此配置项才有效。""" - } - label { - en: "Cluster Multicast Loop" - zh: "多播回环开关" - } - } +common_handler_max_depth.label: +"""Max Depth""" - cluster_mcast_sndbuf { - desc { - en: """Size of the kernel-level buffer for outgoing datagrams.""" - zh: """外发数据报的内核级缓冲区的大小。 -当 cluster.discovery_strategy 为 mcast 时，此配置项才有效。""" - } - label { - en: "Cluster Muticast Sendbuf" - zh: "多播发送缓存区" - } - } +desc_log.desc: +"""EMQX logging supports multiple sinks for the log events. +Each sink is represented by a _log handler_, which can be configured independently.""" - cluster_mcast_recbuf { - desc { - en: """Size of the kernel-level buffer for incoming datagrams.""" - zh: """接收数据报的内核级缓冲区的大小。 -当 cluster.discovery_strategy 为 mcast 时，此配置项才有效。""" - } - label { - en: "Cluster Muticast Sendbuf" - zh: "多播接收数据缓冲区" - } - } +desc_log.label: +"""Log""" - cluster_mcast_buffer { - desc { - en: """Size of the user-level buffer.""" - zh: """用户级缓冲区的大小。 -当 cluster.discovery_strategy 为 mcast 时，此配置项才有效。""" - } - label { - en: "Cluster Muticast Buffer" - zh: "多播用户级缓冲区" - } - } +common_handler_flush_qlen.desc: +"""If the number of buffered log events grows larger than this threshold, a flush (delete) operation takes place. +To flush events, the handler discards the buffered log messages without logging.""" - cluster_dns_name { - desc { - en: """The domain name from which to discover peer EMQX nodes' IP addresses. -Applicable when cluster.discovery_strategy = dns""" - zh: """指定 DNS A 记录的名字。emqx 会通过访问这个 DNS A 记录来获取 IP 地址列表。 -当cluster.discovery_strategy 为 dns 时有效。""" - } - label { - en: "Cluster Dns Name" - zh: "DNS名称" - } - } +common_handler_flush_qlen.label: +"""Flush Threshold""" - cluster_dns_record_type { - desc { - en: """DNS record type.""" - zh: """DNS 记录类型。""" - } - label { - en: "DNS Record Type" - zh: "DNS记录类型" - } - } +common_handler_chars_limit.desc: +"""Set the maximum length of a single log message. If this length is exceeded, the log message will be truncated. +NOTE: Restrict char limiter if formatter is JSON , it will get a truncated incomplete JSON data, which is not recommended.""" - cluster_etcd_server { - desc { - en: """List of endpoint URLs of the etcd cluster""" - zh: """指定 etcd 服务的地址。如有多个服务使用逗号 , 分隔。 -当 cluster.discovery_strategy 为 etcd 时，此配置项才有效。""" - } - label { - en: "Cluster Etcd Server" - zh: "Etcd 服务器地址" - } - } +common_handler_chars_limit.label: +"""Single Log Max Length""" - cluster_etcd_prefix { - desc { - en: """Key prefix used for EMQX service discovery.""" - zh: """指定 etcd 路径的前缀。每个节点在 etcd 中都会创建一个路径: -v2/keys///
-当 cluster.discovery_strategy 为 etcd 时，此配置项才有效。""" - } - label { - en: "Cluster Etcd Prefix" - zh: "Etcd 路径前缀" - } - } +cluster_k8s_namespace.desc: +"""Kubernetes namespace.""" - cluster_etcd_node_ttl { - desc { - en: """Expiration time of the etcd key associated with the node. -It is refreshed automatically, as long as the node is alive.""" +cluster_k8s_namespace.label: +"""K8s Namespace""" - zh: """指定 etcd 中节点信息的过期时间。 -当 cluster.discovery_strategy 为 etcd 时，此配置项才有效。""" - } - label { - en: "Cluster Etcd Node TTL" - zh: "Etcd 节点过期时间" - } - } +node_name.desc: +"""Unique name of the EMQX node. It must follow %name%@FQDN or +%name%@IPv4 format.""" - cluster_etcd_ssl { - desc { - en: """Options for the TLS connection to the etcd cluster.""" - zh: """当使用 TLS 连接 etcd 时的配置选项。 -当 cluster.discovery_strategy 为 etcd 时，此配置项才有效。""" - } - label { - en: "Cluster Etcd SSL Option" - zh: "Etcd SSL 选项" - } - } +node_name.label: +"""Node Name""" - cluster_k8s_apiserver { - desc { - en: """Kubernetes API endpoint URL.""" - zh: """指定 Kubernetes API Server。如有多个 Server 使用逗号 , 分隔。 -当 cluster.discovery_strategy 为 k8s 时，此配置项才有效。""" - } - label { - en: "Cluster k8s ApiServer" - zh: "K8s 服务地址" - } - } +rpc_port_discovery.desc: +"""manual: discover ports by tcp_server_port.
+stateless: discover ports in a stateless manner, using the following algorithm. +If node name is emqxN@127.0.0.1, where the N is an integer, +then the listening port will be 5370 + N.""" - cluster_k8s_service_name { - desc { - en: """EMQX broker service name.""" - zh: """指定 Kubernetes 中 EMQX 的服务名。 -当 cluster.discovery_strategy 为 k8s 时，此配置项才有效。""" - } - label { - en: "K8s Service Name" - zh: "K8s 服务别名" - } - } +rpc_port_discovery.label: +"""RRC Port Discovery""" - cluster_k8s_address_type { - desc { - en: """Address type used for connecting to the discovered nodes. +log_overload_kill_restart_after.desc: +"""The handler restarts automatically after a delay in the event of termination, unless the value `infinity` is set, which blocks any subsequent restarts.""" + +log_overload_kill_restart_after.label: +"""Handler Restart Timer""" + +log_file_handler_max_size.desc: +"""This parameter controls log file rotation. The value `infinity` means the log file will grow indefinitely, otherwise the log file will be rotated once it reaches `max_size` in bytes.""" + +log_file_handler_max_size.label: +"""Rotation Size""" + +desc_log_file_handler.desc: +"""Log handler that prints log events to files.""" + +desc_log_file_handler.label: +"""Files Log Handler""" + +rpc_socket_keepalive_count.desc: +"""How many times the keepalive probe message can fail to receive a reply +until the RPC connection is considered lost.""" + +rpc_socket_keepalive_count.label: +"""RPC Socket Keepalive Count""" + +cluster_etcd_server.desc: +"""List of endpoint URLs of the etcd cluster""" + +cluster_etcd_server.label: +"""Cluster Etcd Server""" + +db_backend.desc: +"""Select the backend for the embedded database.
+rlog is the default backend, +that is suitable for very large clusters.
+mnesia is a backend that offers decent performance in small clusters.""" + +db_backend.label: +"""DB Backend""" + +desc_authorization.desc: +"""Settings that control client authorization.""" + +desc_authorization.label: +"""Authorization""" + +cluster_etcd_ssl.desc: +"""Options for the TLS connection to the etcd cluster.""" + +cluster_etcd_ssl.label: +"""Cluster Etcd SSL Option""" + +rpc_insecure_fallback.desc: +"""Enable compatibility with old RPC authentication.""" + +rpc_insecure_fallback.label: +"""RPC insecure fallback""" + +cluster_mcast_buffer.desc: +"""Size of the user-level buffer.""" + +cluster_mcast_buffer.label: +"""Cluster Muticast Buffer""" + +rpc_authentication_timeout.desc: +"""Timeout for the remote node authentication.""" + +rpc_authentication_timeout.label: +"""RPC Authentication Timeout""" + +cluster_call_retry_interval.desc: +"""Time interval to retry after a failed call.""" + +cluster_call_retry_interval.label: +"""Cluster Call Retry Interval""" + +cluster_mcast_sndbuf.desc: +"""Size of the kernel-level buffer for outgoing datagrams.""" + +cluster_mcast_sndbuf.label: +"""Cluster Muticast Sendbuf""" + +rpc_driver.desc: +"""Transport protocol used for inter-broker communication""" + +rpc_driver.label: +"""RPC dirver""" + +max_ets_tables.desc: +"""Max number of ETS tables""" + +max_ets_tables.label: +"""Max number of ETS tables""" + +desc_db.desc: +"""Settings for the embedded database.""" + +desc_db.label: +"""Database""" + +desc_cluster_etcd.desc: +"""Service discovery using 'etcd' service.""" + +desc_cluster_etcd.label: +"""Cluster Etcd""" + +cluster_name.desc: +"""Human-friendly name of the EMQX cluster.""" + +cluster_name.label: +"""Cluster Name""" + +log_rotation_enable.desc: +"""Enable log rotation feature.""" + +log_rotation_enable.label: +"""Rotation Enable""" + +cluster_call_cleanup_interval.desc: +"""Time interval to clear completed but stale transactions. +Ensure that the number of completed transactions is less than the max_history.""" + +cluster_call_cleanup_interval.label: +"""Clean Up Interval""" + +desc_cluster_static.desc: +"""Service discovery via static nodes. +The new node joins the cluster by connecting to one of the bootstrap nodes.""" + +desc_cluster_static.label: +"""Cluster Static""" + +db_default_shard_transport.desc: +"""Defines the default transport for pushing transaction logs.
+This may be overridden on a per-shard basis in db.shard_transports. +gen_rpc uses the gen_rpc library, +distr uses the Erlang distribution.""" + +db_default_shard_transport.label: +"""Default Shard Transport""" + +cluster_static_seeds.desc: +"""List EMQX node names in the static cluster. See node.name.""" + +cluster_static_seeds.label: +"""Cluster Static Seeds""" + +log_overload_kill_qlen.desc: +"""Maximum allowed queue length.""" + +log_overload_kill_qlen.label: +"""Max Queue Length""" + +node_backtrace_depth.desc: +"""Maximum depth of the call stack printed in error messages and +process_info.""" + +node_backtrace_depth.label: +"""BackTrace Depth""" + +desc_log_burst_limit.desc: +"""Large bursts of log events produced in a short time can potentially cause problems, such as: + - Log files grow very large + - Log files are rotated too quickly, and useful information gets overwritten + - Overall performance impact on the system + +Log burst limit feature can temporarily disable logging to avoid these issues.""" + +desc_log_burst_limit.label: +"""Log Burst Limit""" + +common_handler_enable.desc: +"""Enable this log handler.""" + +common_handler_enable.label: +"""Enable Log Handler""" + +cluster_k8s_service_name.desc: +"""EMQX broker service name.""" + +cluster_k8s_service_name.label: +"""K8s Service Name""" + +log_rotation_count.desc: +"""Maximum number of log files.""" + +log_rotation_count.label: +"""Max Log Files Number""" + +node_cookie.desc: +"""Secret cookie is a random string that should be the same on all nodes in +the given EMQX cluster, but unique per EMQX cluster. It is used to prevent EMQX nodes that +belong to different clusters from accidentally connecting to each other.""" + +node_cookie.label: +"""Node Cookie""" + +db_role.desc: +"""Select a node role.
+core nodes provide durability of the data, and take care of writes. +It is recommended to place core nodes in different racks or different availability zones.
+replicant nodes are ephemeral worker nodes. Removing them from the cluster +doesn't affect database redundancy
+It is recommended to have more replicant nodes than core nodes.
+Note: this parameter only takes effect when the backend is set +to rlog.""" + +db_role.label: +"""DB Role""" + +rpc_tcp_server_port.desc: +"""Listening port used by RPC local service.
+Note that this config only takes effect when rpc.port_discovery is set to manual.""" + +rpc_tcp_server_port.label: +"""RPC TCP Server Port""" + +desc_console_handler.desc: +"""Log handler that prints log events to the EMQX console.""" + +desc_console_handler.label: +"""Console Handler""" + +node_applications.desc: +"""List of Erlang applications that shall be rebooted when the EMQX broker joins the cluster.""" + +node_applications.label: +"""Application""" + +log_burst_limit_max_count.desc: +"""Maximum number of log events to handle within a `window_time` interval. After the limit is reached, successive events are dropped until the end of the `window_time`.""" + +log_burst_limit_max_count.label: +"""Events Number""" + +rpc_tcp_client_num.desc: +"""Set the maximum number of RPC communication channels initiated by this node to each remote node.""" + +rpc_tcp_client_num.label: +"""RPC TCP Client Num""" + +cluster_k8s_address_type.desc: +"""Address type used for connecting to the discovered nodes. Setting cluster.k8s.address_type to ip will make EMQX to discover IP addresses of peer nodes from Kubernetes API.""" - zh: """当使用 k8s 方式集群时，address_type 用来从 Kubernetes 接口的应答里获取什么形式的 Host 列表。 -指定 cluster.k8s.address_type 为 ip，则将从 Kubernetes 接口中获取集群中其他节点 -的IP地址。""" - } - label { - en: "K8s Address Type" - zh: "K8s 地址类型" - } - } +cluster_k8s_address_type.label: +"""K8s Address Type""" - cluster_k8s_namespace { - desc { - en: """Kubernetes namespace.""" - zh: """当使用 k8s 方式并且 cluster.k8s.address_type 指定为 dns 类型时， -可设置 emqx 节点名的命名空间。与 cluster.k8s.suffix 一起使用用以拼接得到节点名列表。""" - } - label { - en: "K8s Namespace" - zh: "K8s 命名空间" - } - } +rpc_socket_sndbuf.desc: +"""TCP tuning parameters. TCP sending buffer size.""" - cluster_k8s_suffix { - desc { - en: """Node name suffix.
-Note: this parameter is only relevant when address_type is dns -or hostname.""" - zh: """当使用 k8s 方式并且 cluster.k8s.address_type 指定为 dns 类型时，可设置 emqx 节点名的后缀。 -与 cluster.k8s.namespace 一起使用用以拼接得到节点名列表。""" - } - label { - en: "K8s Suffix" - zh: "K8s 前缀" - } - } +rpc_socket_sndbuf.label: +"""RPC Socket Sndbuf""" - node_name { - desc { - en: """Unique name of the EMQX node. It must follow %name%@FQDN or -%name%@IPv4 format.""" - zh: """节点名。格式为 \@\。其中 可以是 IP 地址，也可以是 FQDN。 -详见 http://erlang.org/doc/reference_manual/distributed.html。""" - } - label { - en: "Node Name" - zh: "节点名" - } - } +cluster_mcast_ttl.desc: +"""Time-to-live (TTL) for the outgoing UDP datagrams.""" - node_cookie { - desc { - en: """Secret cookie is a random string that should be the same on all nodes in -the given EMQX cluster, but unique per EMQX cluster. It is used to prevent EMQX nodes that -belong to different clusters from accidentally connecting to each other.""" - zh: """分布式 Erlang 集群使用的 cookie 值。集群间保持一致""" - } - label { - en: "Node Cookie" - zh: "节点 Cookie" - } - } +cluster_mcast_ttl.label: +"""Cluster Multicast TTL""" - node_data_dir { - desc { - en: """Path to the persistent data directory.
+db_core_nodes.desc: +"""List of core nodes that the replicant will connect to.
+Note: this parameter only takes effect when the backend is set +to rlog and the role is set to replicant.
+This value needs to be defined for manual or static cluster discovery mechanisms.
+If an automatic cluster discovery mechanism is being used (such as etcd), +there is no need to set this value.""" + +db_core_nodes.label: +"""Db Core Node""" + +log_file_handler_file.desc: +"""Name the log file.""" + +log_file_handler_file.label: +"""Log File Name""" + +node_dist_net_ticktime.desc: +"""This is the approximate time an EMQX node may be unresponsive until it is considered down and thereby disconnected.""" + +node_dist_net_ticktime.label: +"""Dist Net TickTime""" + +desc_cluster_k8s.desc: +"""Service discovery via Kubernetes API server.""" + +desc_cluster_k8s.label: +"""Cluster Kubernetes""" + +desc_cluster_mcast.desc: +"""Service discovery via UDP multicast.""" + +desc_cluster_mcast.label: +"""Cluster Multicast""" + +rpc_cacertfile.desc: +"""Path to certification authority TLS certificate file used to validate rpc.certfile.
+Note: certificates of all nodes in the cluster must be signed by the same CA.""" + +rpc_cacertfile.label: +"""RPC Cacertfile""" + +desc_node.desc: +"""Node name, cookie, config & data directories and the Erlang virtual machine (BEAM) boot parameters.""" + +desc_node.label: +"""Node""" + +cluster_k8s_apiserver.desc: +"""Kubernetes API endpoint URL.""" + +cluster_k8s_apiserver.label: +"""Cluster k8s ApiServer""" + +common_handler_supervisor_reports.desc: +"""Type of supervisor reports that are logged. Defaults to error
+ - error: only log errors in the Erlang processes
. + - progress: log process startup.""" + +common_handler_supervisor_reports.label: +"""Report Type""" + +node_data_dir.desc: +"""Path to the persistent data directory.
Possible auto-created subdirectories are:
- `mnesia/`: EMQX's built-in database directory.
For example, `mnesia/emqx@127.0.0.1`.
@@ -418,822 +463,85 @@ the old dir should be deleted first.
**NOTE**: One data dir cannot be shared by two or more EMQX nodes.""" - zh: """节点数据存放目录，可能会自动创建的子目录如下：
-- `mnesia/`。EMQX的内置数据库目录。例如，`mnesia/emqx@127.0.0.1`。
-如果节点要被重新命名（例如，`emqx@10.0.1.1`）。旧目录应该首先被删除。
-- `configs`。在启动时生成的配置，以及集群/本地覆盖的配置。
-- `patches`: 热补丁文件将被放在这里。
-- `trace`: 日志跟踪文件。
+node_data_dir.label: +"""Node Data Dir""" -**注意**: 一个数据dir不能被两个或更多的EMQX节点同时使用。""" +cluster_k8s_suffix.desc: +"""Node name suffix.
+Note: this parameter is only relevant when address_type is dns +or hostname.""" - } - label { - en: "Node Data Dir" - zh: "节点数据目录" - } - } +cluster_k8s_suffix.label: +"""K8s Suffix""" - node_global_gc_interval { - desc { - en: """Periodic garbage collection interval. Set to disabled to have it disabled.""" - zh: """系统调优参数，设置节点运行多久强制进行一次全局垃圾回收。禁用设置为 disabled。""" - } - label { - en: "Global GC Interval" - zh: "全局垃圾回收" - } - } +db_rpc_module.desc: +"""Protocol used for pushing transaction logs to the replicant nodes.""" - node_crash_dump_file { - desc { - en: """Location of the crash dump file.""" - zh: """设置 Erlang crash_dump 文件的存储路径和文件名。""" - } - label { - en: "Crash Dump File" - zh: "节点崩溃时的Dump文件" - } - } +db_rpc_module.label: +"""RPC Module""" - node_crash_dump_seconds { - desc { - en: """This variable gives the number of seconds that the emulator is allowed to spend writing a crash dump. When the given number of seconds have elapsed, the emulator is terminated.
-- If setting to 0 seconds, the runtime system does not even attempt to write the crash dump file. It only terminates.
-- If setting to a positive value S, wait for S seconds to complete the crash dump file and then terminates the runtime system with a SIGALRM signal.
-- A negative value causes the termination of the runtime system to wait indefinitely until the crash dump file has been completely written.""" +cluster_etcd_prefix.desc: +"""Key prefix used for EMQX service discovery.""" - zh: """该配置给出了运行时系统允许花费的写入崩溃转储的秒数。当给定的秒数已经过去，运行时系统将被终止。
-- 如果设置为0秒，运行时会立即终止，不会尝试写入崩溃转储文件。
-- 如果设置为一个正数 S，节点会等待 S 秒来完成崩溃转储文件，然后用SIGALRM信号终止运行时系统。
-- 如果设置为一个负值导致运行时系统的终止等待无限期地直到崩溃转储文件已经完全写入。""" - } - label { - en: "Crash Dump Seconds" - zh: "保存崩溃文件最长时间" - } - } +cluster_etcd_prefix.label: +"""Cluster Etcd Prefix""" - node_crash_dump_bytes { - desc { - en: """This variable sets the maximum size of a crash dump file in bytes. -The crash dump will be truncated if this limit is exceeded. -If setting it to 0, the runtime system does not even attempt to write a crash dump file.""" +cluster_mcast_iface.desc: +"""Local IP address the node discovery service needs to bind to.""" - zh: """限制崩溃文件的大小，当崩溃时节点内存太大， -如果为了保存现场，需要全部存到崩溃文件中，此处限制最多能保存多大的文件。 -如果超过此限制，崩溃转储将被截断。如果设置为0，系统不会尝试写入崩溃转储文件。""" - } - label { - en: "Crash Dump Bytes" - zh: "崩溃文件最大容量" - } - } +cluster_mcast_iface.label: +"""Cluster Multicast Interface""" - node_dist_net_ticktime { - desc { - en: """This is the approximate time an EMQX node may be unresponsive until it is considered down and thereby disconnected.""" - zh: """系统调优参数，此配置将覆盖 vm.args 文件里的 -kernel net_ticktime 参数。当一个节点持续无响应多久之后，认为其已经宕机并断开连接。""" - } - label { - en: "Dist Net TickTime" - zh: "节点间心跳间隔" - } - } +log_burst_limit_window_time.desc: +"""See max_count.""" - node_backtrace_depth { - desc { - en: """Maximum depth of the call stack printed in error messages and -process_info.""" - zh: """错误信息中打印的最大堆栈层数""" - } - label { - en: "BackTrace Depth" - zh: "最大堆栈导数" - } - } +log_burst_limit_window_time.label: +"""Window Time""" - # TODO: check if this is still needed - node_applications { - desc { - en: """List of Erlang applications that shall be rebooted when the EMQX broker joins the cluster.""" - zh: """当新EMQX 加入集群时，应重启的Erlang应用程序的列表。""" - } - label { - en: "Application" - zh: "应用" - } - } +cluster_dns_record_type.desc: +"""DNS record type.""" - # deprecated, TODO: remove - node_etc_dir { - desc { - en: """etc dir for the node""" - zh: """etc 存放目录""" - } - label { - en: "Etc Dir" - zh: "Etc 目录" - } - } +cluster_dns_record_type.label: +"""DNS Record Type""" - db_backend { - desc { - en: """Select the backend for the embedded database.
-rlog is the default backend, -that is suitable for very large clusters.
-mnesia is a backend that offers decent performance in small clusters.""" - zh: """配置后端数据库驱动，默认值为 rlog 它适用于大规模的集群。 -mnesia 是备选数据库，适合中小集群。""" - } - label { - en: "DB Backend" - zh: "内置数据库" - } - } +cluster_autoclean.desc: +"""Remove disconnected nodes from the cluster after this interval.""" - db_role { - desc { - en: """Select a node role.
-core nodes provide durability of the data, and take care of writes. -It is recommended to place core nodes in different racks or different availability zones.
-replicant nodes are ephemeral worker nodes. Removing them from the cluster -doesn't affect database redundancy
-It is recommended to have more replicant nodes than core nodes.
-Note: this parameter only takes effect when the backend is set -to rlog.""" - zh: """选择节点的角色。
-core 节点提供数据的持久性，并负责写入。建议将核心节点放置在不同的机架或不同的可用区。
-repliant 节点是临时工作节点。 从集群中删除它们，不影响数据库冗余
-建议复制节点多于核心节点。
-注意：该参数仅在设置backend时生效到 rlog。""" - } - label { - en: "DB Role" - zh: "数据库角色" - } - } +cluster_autoclean.label: +"""Cluster Auto Clean""" - db_core_nodes { - desc { - en: """List of core nodes that the replicant will connect to.
-Note: this parameter only takes effect when the backend is set -to rlog and the role is set to replicant.
-This value needs to be defined for manual or static cluster discovery mechanisms.
-If an automatic cluster discovery mechanism is being used (such as etcd), -there is no need to set this value.""" - zh: """当前节点连接的核心节点列表。
-注意：该参数仅在设置backend时生效到 rlog -并且设置role为replicant时生效。
-该值需要在手动或静态集群发现机制下设置。
-如果使用了自动集群发现机制（如etcd），则不需要设置该值。""" - } - label { - en: "Db Core Node" - zh: "数据库核心节点" - } - } +process_limit.desc: +"""Maximum number of simultaneously existing processes for this Erlang system. +The actual maximum chosen may be much larger than the Number passed. +For more information, see: https://www.erlang.org/doc/man/erl.html""" - db_rpc_module { - desc { - en: """Protocol used for pushing transaction logs to the replicant nodes.""" - zh: """集群间推送事务日志到复制节点使用的协议。""" - } - label { - en: "RPC Module" - zh: "RPC协议" - } - } +process_limit.label: +"""Erlang Process Limit""" - db_tlog_push_mode { - desc { - en: """In sync mode the core node waits for an ack from the replicant nodes before sending the next -transaction log entry.""" - zh: """同步模式下，核心节点等待复制节点的确认信息，然后再发送下一条事务日志。""" - } - label { - en: "Tlog Push Mode" - zh: "Tlog推送模式" - } - } +max_ports.desc: +"""Maximum number of simultaneously existing ports for this Erlang system. +The actual maximum chosen may be much larger than the Number passed. +For more information, see: https://www.erlang.org/doc/man/erl.html""" - db_default_shard_transport { - desc { - en: """Defines the default transport for pushing transaction logs.
-This may be overridden on a per-shard basis in db.shard_transports. -gen_rpc uses the gen_rpc library, -distr uses the Erlang distribution.""" - zh: """定义用于推送事务日志的默认传输。
-这可以在 db.shard_transports 中基于每个分片被覆盖。 -gen_rpc 使用 gen_rpc 库， -distr 使用 Erlang 发行版。""" - } - label { - en: "Default Shard Transport" - zh: "事务日志传输默认协议" - } - } +max_ports.label: +"""Erlang Port Limit""" - db_shard_transports { - desc { - en: """Allows to tune the transport method used for transaction log replication, on a per-shard basis.
-gen_rpc uses the gen_rpc library, -distr uses the Erlang distribution.
If not specified, -the default is to use the value set in db.default_shard_transport.""" - zh: """允许为每个 shard 下的事务日志复制操作的传输方法进行调优。
-gen_rpc 使用 gen_rpc 库， -distr 使用 Erlang 自带的 rpc 库。
如果未指定， -默认是使用 db.default_shard_transport 中设置的值。""" - } - label { - en: "Shard Transports" - zh: "事务日志传输协议" - } - } +desc_log_rotation.desc: +"""By default, the logs are stored in `./log` directory (for installation from zip file) or in `/var/log/emqx` (for binary installation).
+This section of the configuration controls the number of files kept for each log handler.""" - cluster_call_retry_interval { - desc { - en: """Time interval to retry after a failed call.""" - zh: """当集群间调用出错时，多长时间重试一次。""" - } - label { - en: "Cluster Call Retry Interval" - zh: "重试时间间隔" - } - } +desc_log_rotation.label: +"""Log Rotation""" - cluster_call_max_history { - desc { - en: """Retain the maximum number of completed transactions (for queries).""" - zh: """集群间调用最多保留的历史记录数。只用于排错时查看。""" - } - label { - en: "Cluster Call Max History" - zh: "最大历史记录" - } - } +desc_log_overload_kill.desc: +"""Log overload kill features an overload protection that activates when the log handlers use too much memory or have too many buffered log messages.
+When the overload is detected, the log handler is terminated and restarted after a cooldown period.""" - cluster_call_cleanup_interval { - desc { - en: """Time interval to clear completed but stale transactions. -Ensure that the number of completed transactions is less than the max_history.""" - zh: """清理过期事务的时间间隔""" - } - label { - en: "Clean Up Interval" - zh: "清理间隔" - } - } +desc_log_overload_kill.label: +"""Log Overload Kill""" - rpc_mode { - desc { - en: """In sync mode the sending side waits for the ack from the receiving side.""" - zh: """在 sync 模式下，发送端等待接收端的 ack信号。""" - } - label { - en: "RPC Mode" - zh: "RPC 模式" - } - } - - rpc_driver { - desc { - en: """Transport protocol used for inter-broker communication""" - zh: """集群间通信使用的传输协议。""" - } - label { - en: "RPC dirver" - zh: "RPC 驱动" - } - } - - rpc_async_batch_size { - desc { - en: """The maximum number of batch messages sent in asynchronous mode. - Note that this configuration does not work in synchronous mode.""" - zh: """异步模式下，发送的批量消息的最大数量。""" - } - label { - en: "Async Batch Size" - zh: "异步模式下的批量消息数量" - } - } - - rpc_port_discovery { - desc { - en: """manual: discover ports by tcp_server_port.
-stateless: discover ports in a stateless manner, using the following algorithm. -If node name is emqxN@127.0.0.1, where the N is an integer, -then the listening port will be 5370 + N.""" - zh: """manual: 通过 tcp_server_port 来发现端口。 -
stateless: 使用无状态的方式来发现端口，使用如下算法。如果节点名称是 -emqxN@127.0.0.1, N 是一个数字，那么监听端口就是 5370 + N。""" - } - label { - en: "RRC Port Discovery" - zh: "RPC 端口发现策略" - } - } - - rpc_tcp_server_port { - desc { - en: """Listening port used by RPC local service.
-Note that this config only takes effect when rpc.port_discovery is set to manual.""" - zh: """RPC 本地服务使用的 TCP 端口。
-只有当 rpc.port_discovery 设置为 manual 时，此配置才会生效。""" - } - label { - en: "RPC TCP Server Port" - zh: "RPC TCP 服务监听端口" - } - } - - rpc_ssl_server_port { - desc { - en: """Listening port used by RPC local service.
-Note that this config only takes effect when rpc.port_discovery is set to manual -and driver is set to ssl.""" - zh: """RPC 本地服务使用的监听SSL端口。
-只有当 rpc.port_discovery 设置为 manual 且 dirver 设置为 ssl， -此配置才会生效。""" - } - label { - en: "RPC SSL Server Port" - zh: "RPC SSL 服务监听端口" - } - } - - rpc_tcp_client_num { - desc { - en: """Set the maximum number of RPC communication channels initiated by this node to each remote node.""" - zh: """设置本节点与远程节点之间的 RPC 通信通道的最大数量。""" - } - label { - en: "RPC TCP Client Num" - zh: "RPC TCP 客户端数量" - } - } - - rpc_connect_timeout { - desc { - en: """Timeout for establishing an RPC connection.""" - zh: """建立 RPC 连接的超时时间。""" - } - label { - en: "RPC Connect Timeout" - zh: "RPC 连接超时时间" - } - } - - rpc_certfile { - desc { - en: """Path to TLS certificate file used to validate identity of the cluster nodes. -Note that this config only takes effect when rpc.driver is set to ssl.""" - - zh: """TLS 证书文件的路径，用于验证集群节点的身份。 -只有当 rpc.driver 设置为 ssl 时，此配置才会生效。""" - } - label { - en: "RPC Certfile" - zh: "RPC 证书文件" - } - } - - rpc_keyfile { - desc { - en: """Path to the private key file for the rpc.certfile.
-Note: contents of this file are secret, so it's necessary to set permissions to 600.""" - zh: """rpc.certfile 的私钥文件的路径。
-注意：此文件内容是私钥，所以需要设置权限为 600。""" - } - label { - en: "RPC Keyfile" - zh: "RPC 私钥文件" - } - } - - rpc_cacertfile { - desc { - en: """Path to certification authority TLS certificate file used to validate rpc.certfile.
-Note: certificates of all nodes in the cluster must be signed by the same CA.""" - zh: """验证 rpc.certfile 的 CA 证书文件的路径。
-注意：集群中所有节点的证书必须使用同一个 CA 签发。""" - } - label { - en: "RPC Cacertfile" - zh: "RPC CA 证书文件" - } - } - - rpc_send_timeout { - desc { - en: """Timeout for sending the RPC request.""" - zh: """发送 RPC 请求的超时时间。""" - } - label { - en: "RPC Send Timeout" - zh: "RPC 发送超时时间" - } - } - - rpc_authentication_timeout { - desc { - en: """Timeout for the remote node authentication.""" - zh: """远程节点认证的超时时间。""" - } - label { - en: "RPC Authentication Timeout" - zh: "RPC 认证超时时间" - } - } - - rpc_call_receive_timeout { - desc { - en: """Timeout for the reply to a synchronous RPC.""" - zh: """同步 RPC 的回复超时时间。""" - } - label { - en: "RPC Call Receive Timeout" - zh: "RPC 调用接收超时时间" - } - } - - rpc_socket_keepalive_idle { - desc { - en: """How long the connections between the brokers should remain open after the last message is sent.""" - zh: """broker 之间的连接在最后一条消息发送后保持打开的时间。""" - } - label { - en: "RPC Socket Keepalive Idle" - zh: "RPC Socket Keepalive Idle" - } - } - - rpc_socket_keepalive_interval { - desc { - en: """The interval between keepalive messages.""" - zh: """keepalive 消息的间隔。""" - } - label { - en: "RPC Socket Keepalive Interval" - zh: "RPC Socket Keepalive 间隔" - } - } - - rpc_socket_keepalive_count { - desc { - en: """How many times the keepalive probe message can fail to receive a reply -until the RPC connection is considered lost.""" - zh: """keepalive 探测消息发送失败的次数，直到 RPC 连接被认为已经断开。""" - } - label { - en: "RPC Socket Keepalive Count" - zh: "RPC Socket Keepalive 次数" - } - } - - rpc_socket_sndbuf { - desc { - en: """TCP tuning parameters. TCP sending buffer size.""" - zh: """TCP 调节参数。TCP 发送缓冲区大小。""" - } - label { - en: "RPC Socket Sndbuf" - zh: "RPC 套接字发送缓冲区大小" - } - } - - rpc_socket_recbuf { - desc { - en: """TCP tuning parameters. TCP receiving buffer size.""" - zh: """TCP 调节参数。TCP 接收缓冲区大小。""" - } - label { - en: "RPC Socket Recbuf" - zh: "RPC 套接字接收缓冲区大小" - } - } - - rpc_socket_buffer { - desc { - en: """TCP tuning parameters. Socket buffer size in user mode.""" - zh: """TCP 调节参数。用户模式套接字缓冲区大小。""" - } - label { - en: "RPC Socket Buffer" - zh: "RPC 套接字缓冲区大小" - } - } - - rpc_insecure_fallback { - desc { - en: """Enable compatibility with old RPC authentication.""" - zh: """兼容旧的无鉴权模式""" - } - label { - en: "RPC insecure fallback" - zh: "向后兼容旧的无鉴权模式" - } - } - - log_file_handlers { - desc { - en: """File-based log handlers.""" - zh: """输出到文件的日志处理进程列表""" - } - label { - en: "File Handler" - zh: "File Handler" - } - } - - common_handler_enable { - desc { - en: """Enable this log handler.""" - zh: """启用此日志处理进程。""" - } - label { - en: "Enable Log Handler" - zh: "启用日志处理进程" - } - } - - common_handler_level { - desc { - en: """The log level for the current log handler. -Defaults to warning.""" - zh: """当前日志处理进程的日志级别。 -默认为 warning 级别。""" - } - label { - en: "Log Level" - zh: "日志级别" - } - } - - common_handler_time_offset { - desc { - en: """The time offset to be used when formatting the timestamp. -Can be one of: - - system: the time offset used by the local system - - utc: the UTC time offset - - +-[hh]:[mm]: user specified time offset, such as "-02:00" or "+00:00" -Defaults to: system.""" - zh: """日志中的时间戳使用的时间偏移量。 -可选值为： - - system: 本地系统使用的时区偏移量 - - utc: 0 时区的偏移量 - - +-[hh]:[mm]: 自定义偏移量，比如 "-02:00" 或者 "+00:00" -默认值为本地系统的时区偏移量：system。""" - } - label { - en: "Time Offset" - zh: "时间偏移量" - } - } - - common_handler_chars_limit { - desc { - en: """Set the maximum length of a single log message. If this length is exceeded, the log message will be truncated. -NOTE: Restrict char limiter if formatter is JSON , it will get a truncated incomplete JSON data, which is not recommended.""" - zh: """设置单个日志消息的最大长度。 如果超过此长度，则日志消息将被截断。最小可设置的长度为100。 -注意：如果日志格式为 JSON，限制字符长度可能会导致截断不完整的 JSON 数据。""" - } - label { - en: "Single Log Max Length" - zh: "单条日志长度限制" - } - } - - common_handler_formatter { - desc { - en: """Choose log formatter. text for free text, and json for structured logging.""" - zh: """选择日志格式类型。 text 用于纯文本，json 用于结构化日志记录。""" - } - label { - en: "Log Formatter" - zh: "日志格式类型" - } - } - - common_handler_single_line { - desc { - en: """Print logs in a single line if set to true. Otherwise, log messages may span multiple lines.""" - zh: """如果设置为 true，则单行打印日志。 否则，日志消息可能跨越多行。""" - } - label { - en: "Single Line Mode" - zh: "单行模式" - } - } - - common_handler_sync_mode_qlen { - desc { - en: """As long as the number of buffered log events is lower than this value, -all log events are handled asynchronously. This means that the client process sending the log event, -by calling a log function in the Logger API, does not wait for a response from the handler -but continues executing immediately after the event is sent. -It is not affected by the time it takes the handler to print the event to the log device. -If the message queue grows larger than this value, -the handler starts handling log events synchronously instead, -meaning that the client process sending the event must wait for a response. -When the handler reduces the message queue to a level below the sync_mode_qlen threshold, -asynchronous operation is resumed.""" - zh: """只要缓冲的日志事件的数量低于这个值，所有的日志事件都会被异步处理。 -这意味着，日志落地速度不会影响正常的业务进程，因为它们不需要等待日志处理进程的响应。 -如果消息队列的增长超过了这个值，处理程序开始同步处理日志事件。也就是说，发送事件的客户进程必须等待响应。 -当处理程序将消息队列减少到低于sync_mode_qlen阈值的水平时，异步操作就会恢复。 -默认为100条信息，当等待的日志事件大于100条时，就开始同步处理日志。""" - } - label { - en: "Queue Length before Entering Sync Mode" - zh: "进入异步模式的队列长度" - } - } - - common_handler_drop_mode_qlen { - desc { - en: """When the number of buffered log events is larger than this value, the new log events are dropped. -When drop mode is activated or deactivated, a message is printed in the logs.""" - zh: """当缓冲的日志事件数大于此值时，新的日志事件将被丢弃。起到过载保护的功能。 -为了使过载保护算法正常工作必须要： sync_mode_qlen =< drop_mode_qlen =< flush_qlen 且 drop_mode_qlen > 1 -要禁用某些模式，请执行以下操作。 -- 如果sync_mode_qlen被设置为0，所有的日志事件都被同步处理。也就是说，异步日志被禁用。 -- 如果sync_mode_qlen被设置为与drop_mode_qlen相同的值，同步模式被禁用。也就是说，处理程序总是以异步模式运行，除非调用drop或flushing。 -- 如果drop_mode_qlen被设置为与flush_qlen相同的值，则drop模式被禁用，永远不会发生。""" - } - label { - en: "Queue Length before Entering Drop Mode" - zh: "进入丢弃模式的队列长度" - } - } - - common_handler_flush_qlen { - desc { - en: """If the number of buffered log events grows larger than this threshold, a flush (delete) operation takes place. -To flush events, the handler discards the buffered log messages without logging.""" - zh: """如果缓冲日志事件的数量增长大于此阈值，则会发生冲刷（删除）操作。 日志处理进程会丢弃缓冲的日志消息。 -来缓解自身不会由于内存瀑涨而影响其它业务进程。日志内容会提醒有多少事件被删除。""" - } - label { - en: "Flush Threshold" - zh: "冲刷阈值" - } - } - - common_handler_supervisor_reports { - desc { - en: """Type of supervisor reports that are logged. Defaults to error
- - error: only log errors in the Erlang processes
. - - progress: log process startup.""" - - zh: """Supervisor 报告的类型。默认为 error 类型。
- - error：仅记录 Erlang 进程中的错误。 - - progress：除了 error 信息外，还需要记录进程启动的详细信息。""" - } - label { - en: "Report Type" - zh: "报告类型" - } - } - - common_handler_max_depth { - desc { - en: """Maximum depth for Erlang term log formatting and Erlang process message queue inspection.""" - zh: """Erlang 内部格式日志格式化和 Erlang 进程消息队列检查的最大深度。""" - } - label { - en: "Max Depth" - zh: "最大深度" - } - } - - log_file_handler_file { - desc { - en: """Name the log file.""" - zh: """日志文件路径及名字。""" - } - label { - en: "Log File Name" - zh: "日志文件名字" - } - } - - log_file_handler_max_size { - desc { - en: """This parameter controls log file rotation. The value `infinity` means the log file will grow indefinitely, otherwise the log file will be rotated once it reaches `max_size` in bytes.""" - zh: """此参数控制日志文件轮换。 `infinity` 意味着日志文件将无限增长，否则日志文件将在达到 `max_size`（以字节为单位）时进行轮换。 -与 rotation count配合使用。如果 counter 为 10，则是10个文件轮换。""" - } - label { - en: "Rotation Size" - zh: "日志文件轮换大小" - } - } - - log_rotation_enable { - desc { - en: """Enable log rotation feature.""" - zh: """启用日志轮换功能。启动后生成日志文件后缀会加上对应的索引数字，比如：log/emqx.log.1。 -系统会默认生成*.siz/*.idx用于记录日志位置，请不要手动修改这两个文件。""" - } - label { - en: "Rotation Enable" - zh: "日志轮换" - } - } - - log_rotation_count { - desc { - en: """Maximum number of log files.""" - zh: """轮换的最大日志文件数。""" - } - label { - en: "Max Log Files Number" - zh: "最大日志文件数" - } - } - - log_overload_kill_enable { - desc { - en: """Enable log handler overload kill feature.""" - zh: """日志处理进程过载时为保护自己节点其它的业务能正常，强制杀死日志处理进程。""" - } - label { - en: "Log Handler Overload Kill" - zh: "日志处理进程过载保护" - } - } - - log_overload_kill_mem_size { - desc { - en: """Maximum memory size that the log handler process is allowed to use.""" - zh: """日志处理进程允许使用的最大内存。""" - } - label { - en: "Log Handler Max Memory Size" - zh: "日志处理进程允许使用的最大内存" - } - } - - log_overload_kill_qlen { - desc { - en: """Maximum allowed queue length.""" - zh: """允许的最大队列长度。""" - } - label { - en: "Max Queue Length" - zh: "最大队列长度" - } - } - - log_overload_kill_restart_after { - desc { - en: """The handler restarts automatically after a delay in the event of termination, unless the value `infinity` is set, which blocks any subsequent restarts.""" - zh: """如果处理进程终止，它会在以指定的时间后后自动重新启动。 `infinity` 不自动重启。""" - } - label { - en: "Handler Restart Timer" - zh: "处理进程重启机制" - } - } - - log_burst_limit_enable { - desc { - en: """Enable log burst control feature.""" - zh: """启用日志限流保护机制。""" - } - label { - en: "Enable Burst" - zh: "日志限流保护" - } - } - - log_burst_limit_max_count { - desc { - en: """Maximum number of log events to handle within a `window_time` interval. After the limit is reached, successive events are dropped until the end of the `window_time`.""" - zh: """在 `window_time` 间隔内处理的最大日志事件数。 达到限制后，将丢弃连续事件，直到 `window_time` 结束。""" - } - label { - en: "Events Number" - zh: "日志事件数" - } - } - - log_burst_limit_window_time { - desc { - en: """See max_count.""" - zh: """参考 max_count。""" - } - label { - en: "Window Time" - zh: "Window Time" - } - } - - authorization { - desc { - en: """Authorization a.k.a. ACL.
+authorization.desc: +"""Authorization a.k.a. ACL.
In EMQX, MQTT client access control is extremely flexible.
An out-of-the-box set of authorization data sources are supported. For example,
@@ -1243,221 +551,238 @@ natively in the EMQX node;
'http' source to make EMQX call an external HTTP API to make the decision;
'PostgreSQL' etc. to look up clients or rules from external databases""" - zh: """授权（ACL）。EMQX 支持完整的客户端访问控制（ACL）。""" - } - label { - en: "Authorization" - zh: "授权" - } - } +authorization.label: +"""Authorization""" - desc_cluster { - desc { - en: """EMQX nodes can form a cluster to scale up the total capacity.
- Here holds the configs to instruct how individual nodes can discover each other.""" - zh: """EMQX 节点可以组成一个集群，以提高总容量。
这里指定了节点之间如何连接。""" - } - label { - en: "Cluster" - zh: "集群" - } - } +rpc_socket_keepalive_idle.desc: +"""How long the connections between the brokers should remain open after the last message is sent.""" - desc_cluster_static { - desc { - en: """Service discovery via static nodes. -The new node joins the cluster by connecting to one of the bootstrap nodes.""" - zh: """静态节点服务发现。新节点通过连接一个节点来加入集群。""" - } - label { - en: "Cluster Static" - zh: "静态节点服务发现" - } - } +rpc_socket_keepalive_idle.label: +"""RPC Socket Keepalive Idle""" - desc_cluster_mcast { - desc { - en: """Service discovery via UDP multicast.""" - zh: """UDP 组播服务发现。""" - } - label { - en: "Cluster Multicast" - zh: "UDP 组播服务发现" - } - } +desc_cluster_call.desc: +"""Options for the 'cluster call' feature that allows to execute a callback on all nodes in the cluster.""" - desc_cluster_dns { - desc { - en: """Service discovery via DNS SRV records.""" - zh: """DNS SRV 记录服务发现。""" - } - label { - en: "Cluster DNS" - zh: "DNS SRV 记录服务发现" - } - } +desc_cluster_call.label: +"""Cluster Call""" - desc_cluster_etcd { - desc { - en: """Service discovery using 'etcd' service.""" - zh: """使用 'etcd' 服务的服务发现。""" - } - label { - en: "Cluster Etcd" - zh: "'etcd' 服务的服务发现" - } - } +cluster_mcast_ports.desc: +"""List of UDP ports used for service discovery.
+Note: probe messages are broadcast to all the specified ports.""" - desc_cluster_k8s { - desc { - en: """Service discovery via Kubernetes API server.""" - zh: """Kubernetes 服务发现。""" - } - label { - en: "Cluster Kubernetes" - zh: "Kubernetes 服务发现" - } - } +cluster_mcast_ports.label: +"""Cluster Multicast Ports""" - desc_node { - desc { - en: """Node name, cookie, config & data directories and the Erlang virtual machine (BEAM) boot parameters.""" - zh: """节点名称、Cookie、配置文件、数据目录和 Erlang 虚拟机（BEAM）启动参数。""" - } - label { - en: "Node" - zh: "节点" - } - } +log_overload_kill_mem_size.desc: +"""Maximum memory size that the log handler process is allowed to use.""" - desc_db { - desc { - en: """Settings for the embedded database.""" - zh: """内置数据库的配置。""" - } - label { - en: "Database" - zh: "数据库" - } - } +log_overload_kill_mem_size.label: +"""Log Handler Max Memory Size""" - desc_cluster_call { - desc { - en: """Options for the 'cluster call' feature that allows to execute a callback on all nodes in the cluster.""" - zh: """集群调用功能的选项。""" - } - label { - en: "Cluster Call" - zh: "集群调用" - } - } +rpc_connect_timeout.desc: +"""Timeout for establishing an RPC connection.""" - desc_rpc { - desc { - en: """EMQX uses a library called gen_rpc for inter-broker communication.
+rpc_connect_timeout.label: +"""RPC Connect Timeout""" + +cluster_etcd_node_ttl.desc: +"""Expiration time of the etcd key associated with the node. +It is refreshed automatically, as long as the node is alive.""" + +cluster_etcd_node_ttl.label: +"""Cluster Etcd Node TTL""" + +rpc_call_receive_timeout.desc: +"""Timeout for the reply to a synchronous RPC.""" + +rpc_call_receive_timeout.label: +"""RPC Call Receive Timeout""" + +rpc_socket_recbuf.desc: +"""TCP tuning parameters. TCP receiving buffer size.""" + +rpc_socket_recbuf.label: +"""RPC Socket Recbuf""" + +db_tlog_push_mode.desc: +"""In sync mode the core node waits for an ack from the replicant nodes before sending the next +transaction log entry.""" + +db_tlog_push_mode.label: +"""Tlog Push Mode""" + +node_crash_dump_bytes.desc: +"""This variable sets the maximum size of a crash dump file in bytes. +The crash dump will be truncated if this limit is exceeded. +If setting it to 0, the runtime system does not even attempt to write a crash dump file.""" + +node_crash_dump_bytes.label: +"""Crash Dump Bytes""" + +rpc_certfile.desc: +"""Path to TLS certificate file used to validate identity of the cluster nodes. +Note that this config only takes effect when rpc.driver is set to ssl.""" + +rpc_certfile.label: +"""RPC Certfile""" + +node_crash_dump_seconds.desc: +"""This variable gives the number of seconds that the emulator is allowed to spend writing a crash dump. When the given number of seconds have elapsed, the emulator is terminated.
+- If setting to 0 seconds, the runtime system does not even attempt to write the crash dump file. It only terminates.
+- If setting to a positive value S, wait for S seconds to complete the crash dump file and then terminates the runtime system with a SIGALRM signal.
+- A negative value causes the termination of the runtime system to wait indefinitely until the crash dump file has been completely written.""" + +node_crash_dump_seconds.label: +"""Crash Dump Seconds""" + +log_file_handlers.desc: +"""File-based log handlers.""" + +log_file_handlers.label: +"""File Handler""" + +node_global_gc_interval.desc: +"""Periodic garbage collection interval. Set to disabled to have it disabled.""" + +node_global_gc_interval.label: +"""Global GC Interval""" + +common_handler_time_offset.desc: +"""The time offset to be used when formatting the timestamp. +Can be one of: + - system: the time offset used by the local system + - utc: the UTC time offset + - +-[hh]:[mm]: user specified time offset, such as "-02:00" or "+00:00" +Defaults to: system.""" + +common_handler_time_offset.label: +"""Time Offset""" + +rpc_mode.desc: +"""In sync mode the sending side waits for the ack from the receiving side.""" + +rpc_mode.label: +"""RPC Mode""" + +node_crash_dump_file.desc: +"""Location of the crash dump file.""" + +node_crash_dump_file.label: +"""Crash Dump File""" + +cluster_mcast_loop.desc: +"""If true, loop UDP datagrams back to the local socket.""" + +cluster_mcast_loop.label: +"""Cluster Multicast Loop""" + +rpc_socket_keepalive_interval.desc: +"""The interval between keepalive messages.""" + +rpc_socket_keepalive_interval.label: +"""RPC Socket Keepalive Interval""" + +common_handler_level.desc: +"""The log level for the current log handler. +Defaults to warning.""" + +common_handler_level.label: +"""Log Level""" + +desc_rpc.desc: +"""EMQX uses a library called gen_rpc for inter-broker communication.
Most of the time the default config should work, but in case you need to do performance fine-tuning or experiment a bit, this is where to look.""" - zh: """EMQX 使用 gen_rpc 库来实现跨节点通信。
-大多数情况下，默认的配置应该可以工作，但如果你需要做一些性能优化或者实验，可以尝试调整这些参数。""" - } - label { - en: "RPC" - zh: "RPC" - } - } - desc_log { - desc { - en: """EMQX logging supports multiple sinks for the log events. -Each sink is represented by a _log handler_, which can be configured independently.""" - zh: """EMQX 日志记录支持日志事件的多个接收器。 每个接收器由一个_log handler_表示，可以独立配置。""" - } - label { - en: "Log" - zh: "日志" - } - } +desc_rpc.label: +"""RPC""" - desc_console_handler { - desc { - en: """Log handler that prints log events to the EMQX console.""" - zh: """日志处理进程将日志事件打印到 EMQX 控制台。""" - } - label { - en: "Console Handler" - zh: "Console Handler" - } - } +rpc_ssl_server_port.desc: +"""Listening port used by RPC local service.
+Note that this config only takes effect when rpc.port_discovery is set to manual +and driver is set to ssl.""" - desc_log_file_handler { - desc { - en: """Log handler that prints log events to files.""" - zh: """日志处理进程将日志事件打印到文件。""" - } - label { - en: "Files Log Handler" - zh: "文件日志处理进程" - } - } +rpc_ssl_server_port.label: +"""RPC SSL Server Port""" - desc_log_rotation { - desc { - en: """By default, the logs are stored in `./log` directory (for installation from zip file) or in `/var/log/emqx` (for binary installation).
-This section of the configuration controls the number of files kept for each log handler.""" +desc_cluster.desc: +"""EMQX nodes can form a cluster to scale up the total capacity.
+ Here holds the configs to instruct how individual nodes can discover each other.""" - zh: """默认情况下，日志存储在 `./log` 目录（用于从 zip 文件安装）或 `/var/log/emqx`（用于二进制安装）。
-这部分配置，控制每个日志处理进程保留的文件数量。""" - } - label { - en: "Log Rotation" - zh: "日志轮换" - } - } +desc_cluster.label: +"""Cluster""" - desc_log_overload_kill { - desc { - en: """Log overload kill features an overload protection that activates when the log handlers use too much memory or have too many buffered log messages.
-When the overload is detected, the log handler is terminated and restarted after a cooldown period.""" +common_handler_sync_mode_qlen.desc: +"""As long as the number of buffered log events is lower than this value, +all log events are handled asynchronously. This means that the client process sending the log event, +by calling a log function in the Logger API, does not wait for a response from the handler +but continues executing immediately after the event is sent. +It is not affected by the time it takes the handler to print the event to the log device. +If the message queue grows larger than this value, +the handler starts handling log events synchronously instead, +meaning that the client process sending the event must wait for a response. +When the handler reduces the message queue to a level below the sync_mode_qlen threshold, +asynchronous operation is resumed.""" - zh: """日志过载终止，具有过载保护功能。当日志处理进程使用过多内存，或者缓存的日志消息过多时该功能被激活。
-检测到过载时，日志处理进程将终止，并在冷却期后重新启动。""" - } - label { - en: "Log Overload Kill" - zh: "日志过载保护" - } - } +common_handler_sync_mode_qlen.label: +"""Queue Length before Entering Sync Mode""" - desc_log_burst_limit { - desc { - en: """Large bursts of log events produced in a short time can potentially cause problems, such as: - - Log files grow very large - - Log files are rotated too quickly, and useful information gets overwritten - - Overall performance impact on the system +common_handler_formatter.desc: +"""Choose log formatter. text for free text, and json for structured logging.""" -Log burst limit feature can temporarily disable logging to avoid these issues.""" - zh: """短时间内产生的大量日志事件可能会导致问题，例如： - - 日志文件变得非常大 - - 日志文件轮换过快，有用信息被覆盖 - - 对系统的整体性能影响 +common_handler_formatter.label: +"""Log Formatter""" -日志突发限制功能可以暂时禁用日志记录以避免这些问题。""" - } - label { - en: "Log Burst Limit" - zh: "日志突发限制" - } - } +rpc_async_batch_size.desc: +"""The maximum number of batch messages sent in asynchronous mode. + Note that this configuration does not work in synchronous mode.""" + +rpc_async_batch_size.label: +"""Async Batch Size""" + +cluster_call_max_history.desc: +"""Retain the maximum number of completed transactions (for queries).""" + +cluster_call_max_history.label: +"""Cluster Call Max History""" + +cluster_discovery_strategy.desc: +"""Service discovery method for the cluster nodes. Possible values are: +- manual: Use emqx ctl cluster command to manage cluster.
+- static: Configure static nodes list by setting seeds in config file.
+- dns: Use DNS A record to discover peer nodes.
+- etcd: Use etcd to discover peer nodes.
+- k8s: Use Kubernetes API to discover peer pods.""" + +cluster_discovery_strategy.label: +"""Cluster Discovery Strategy""" + +rpc_send_timeout.desc: +"""Timeout for sending the RPC request.""" + +rpc_send_timeout.label: +"""RPC Send Timeout""" + +common_handler_single_line.desc: +"""Print logs in a single line if set to true. Otherwise, log messages may span multiple lines.""" + +common_handler_single_line.label: +"""Single Line Mode""" + +rpc_socket_buffer.desc: +"""TCP tuning parameters. Socket buffer size in user mode.""" + +rpc_socket_buffer.label: +"""RPC Socket Buffer""" + +db_shard_transports.desc: +"""Allows to tune the transport method used for transaction log replication, on a per-shard basis.
+gen_rpc uses the gen_rpc library, +distr uses the Erlang distribution.
If not specified, +the default is to use the value set in db.default_shard_transport.""" + +db_shard_transports.label: +"""Shard Transports""" - desc_authorization { - desc { - en: """Settings that control client authorization.""" - zh: """授权相关""" - } - label { - en: "Authorization" - zh: "授权" - } - } } diff --git a/rel/i18n/emqx_connector_api.hocon b/rel/i18n/emqx_connector_api.hocon index 8575f01aa..4942ce981 100644 --- a/rel/i18n/emqx_connector_api.hocon +++ b/rel/i18n/emqx_connector_api.hocon @@ -1,82 +1,46 @@ emqx_connector_api { - id { - desc { - en: "The connector ID. Must be of format {type}:{name}" - zh: "连接器 ID， 格式必须为 {type}:{name}" - } - label: { - en: "Connector ID" - zh: "连接器 ID" - } - } +conn_get.desc: +"""List all connectors""" - conn_test_post { - desc { - en: """Test creating a new connector by given ID
+conn_get.label: +"""List All Connectors""" + +conn_id_delete.desc: +"""Delete a connector by ID""" + +conn_id_delete.label: +"""Delete Connector""" + +conn_id_get.desc: +"""Get the connector by ID""" + +conn_id_get.label: +"""Get Connector""" + +conn_id_put.desc: +"""Update an existing connector by ID""" + +conn_id_put.label: +"""Update Connector""" + +conn_post.desc: +"""Create a new connector""" + +conn_post.label: +"""Create Connector""" + +conn_test_post.desc: +"""Test creating a new connector by given ID
The ID must be of format '{type}:{name}'""" - zh: """通过给定的 ID 测试创建一个新的连接器
-ID 的格式必须为“{type}:{name}”""" - } - label: { - en: "Create Test Connector" - zh: "创建测试连接器" - } - } - conn_get { - desc { - en: "List all connectors" - zh: "列出所有连接器" - } - label: { - en: "List All Connectors" - zh: "列出所有连接器" - } - } +conn_test_post.label: +"""Create Test Connector""" - conn_post { - desc { - en: "Create a new connector" - zh: "创建一个新的连接器" - } - label: { - en: "Create Connector" - zh: "创建连接器" - } - } +id.desc: +"""The connector ID. Must be of format {type}:{name}""" - conn_id_get { - desc { - en: "Get the connector by ID" - zh: "通过 ID 获取连接器" - } - label: { - en: "Get Connector" - zh: "获取连接器" - } - } - - conn_id_put { - desc { - en: "Update an existing connector by ID" - zh: "通过 ID 更新一个连接器" - } - label: { - en: "Update Connector" - zh: "更新连接器" - } - } - - conn_id_delete { - desc { - en: "Delete a connector by ID" - zh: "通过 ID 删除一个连接器" - } - label: { - en: "Delete Connector" - zh: "删除连接器" - } - } +id.label: +"""Connector ID""" } diff --git a/rel/i18n/emqx_connector_http.hocon b/rel/i18n/emqx_connector_http.hocon index c6efd03ca..70c644e33 100644 --- a/rel/i18n/emqx_connector_http.hocon +++ b/rel/i18n/emqx_connector_http.hocon @@ -1,139 +1,78 @@ emqx_connector_http { - base_url { - desc { - en: """The base URL is the URL includes only the scheme, host and port.
+ +base_url.desc: +"""The base URL is the URL includes only the scheme, host and port.
When send an HTTP request, the real URL to be used is the concatenation of the base URL and the path parameter
For example: `http://localhost:9901/`""" - zh: """base URL 只包含host和port。
-发送HTTP请求时，真实的URL是由base URL 和 path parameter连接而成。
-示例：`http://localhost:9901/`""" - } - label: { - en: "Base Url" - zh: "Base Url" - } - } - connect_timeout { - desc { - en: "The timeout when connecting to the HTTP server." - zh: "连接HTTP服务器的超时时间。" - } - label: { - en: "Connect Timeout" - zh: "连接超时" - } - } +base_url.label: +"""Base Url""" - max_retries { - desc { - en: "Max retry times if error on sending request." - zh: "请求出错时的最大重试次数。" - } - label: { - en: "Max Retries" - zh: "最大重试次数" - } - } +body.desc: +"""HTTP request body.""" - pool_type { - desc { - en: "The type of the pool. Can be one of `random`, `hash`." - zh: "连接池的类型，可用类型有`random`, `hash`。" - } - label: { - en: "Pool Type" - zh: "连接池类型" - } - } +body.label: +"""HTTP Body""" - pool_size { - desc { - en: "The pool size." - zh: "连接池大小。" - } - label: { - en: "Pool Size" - zh: "连接池大小" - } - } +connect_timeout.desc: +"""The timeout when connecting to the HTTP server.""" - enable_pipelining { - desc { - en: "A positive integer. Whether to send HTTP requests continuously, when set to 1, it means that after each HTTP request is sent, you need to wait for the server to return and then continue to send the next request." - zh: "正整数，设置最大可发送的异步 HTTP 请求数量。当设置为 1 时，表示每次发送完成 HTTP 请求后都需要等待服务器返回，再继续发送下一个请求。" - } - label: { - en: "HTTP Pipelineing" - zh: "HTTP 管道" - } - } +connect_timeout.label: +"""Connect Timeout""" - request { - desc { - en: """Configure HTTP request parameters.""" - zh: """设置 HTTP 请求的参数。""" - } - label: { - en: "Request" - zh: "HTTP 请求" - } - } +enable_pipelining.desc: +"""A positive integer. Whether to send HTTP requests continuously, when set to 1, it means that after each HTTP request is sent, you need to wait for the server to return and then continue to send the next request.""" - method { - desc { - en: "HTTP method." - zh: "HTTP 请求方法。" - } - label: { - en: "HTTP Method" - zh: "HTTP 请求方法" - } - } +enable_pipelining.label: +"""HTTP Pipelineing""" - path { - desc { - en: "URL path." - zh: "HTTP请求路径。" - } - label: { - en: "URL Path" - zh: "HTTP请求路径" - } - } +headers.desc: +"""List of HTTP headers.""" - body { - desc { - en: "HTTP request body." - zh: "HTTP请求报文主体。" - } - label: { - en: "HTTP Body" - zh: "HTTP请求报文主体" - } - } +headers.label: +"""HTTP Headers""" - headers { - desc { - en: "List of HTTP headers." - zh: "HTTP 头字段列表。" - } - label: { - en: "HTTP Headers" - zh: "HTTP 头字段列表" - } - } +max_retries.desc: +"""Max retry times if error on sending request.""" - request_timeout { - desc { - en: "HTTP request timeout." - zh: "HTTP 请求超时。" - } - label: { - en: "Request Timeout" - zh: "HTTP 请求超时" - } - } +max_retries.label: +"""Max Retries""" + +method.desc: +"""HTTP method.""" + +method.label: +"""HTTP Method""" + +path.desc: +"""URL path.""" + +path.label: +"""URL Path""" + +pool_size.desc: +"""The pool size.""" + +pool_size.label: +"""Pool Size""" + +pool_type.desc: +"""The type of the pool. Can be one of `random`, `hash`.""" + +pool_type.label: +"""Pool Type""" + +request.desc: +"""Configure HTTP request parameters.""" + +request.label: +"""Request""" + +request_timeout.desc: +"""HTTP request timeout.""" + +request_timeout.label: +"""Request Timeout""" } diff --git a/rel/i18n/emqx_connector_ldap.hocon b/rel/i18n/emqx_connector_ldap.hocon index 0bcb4869e..64a953816 100644 --- a/rel/i18n/emqx_connector_ldap.hocon +++ b/rel/i18n/emqx_connector_ldap.hocon @@ -1,37 +1,21 @@ emqx_connector_ldap { - bind_dn { - desc { - en: """LDAP's Binding Distinguished Name (DN)""" - zh: """LDAP 绑定的 DN 的值""" - } - label: { - en: "Bind DN" - zh: "Bind DN" - } - } +bind_dn.desc: +"""LDAP's Binding Distinguished Name (DN)""" - port { - desc { - en: """LDAP Port""" - zh: """LDAP 端口""" - } - label: { - en: "Port" - zh: "端口" - } - } +bind_dn.label: +"""Bind DN""" +port.desc: +"""LDAP Port""" - timeout { - desc { - en: """LDAP's query timeout""" - zh: """LDAP 查询超时时间""" - } - label: { - en: "timeout" - zh: "超时时间" - } - } +port.label: +"""Port""" + +timeout.desc: +"""LDAP's query timeout""" + +timeout.label: +"""timeout""" } diff --git a/rel/i18n/emqx_connector_mongo.hocon b/rel/i18n/emqx_connector_mongo.hocon index 6a2511ec8..bba26d736 100644 --- a/rel/i18n/emqx_connector_mongo.hocon +++ b/rel/i18n/emqx_connector_mongo.hocon @@ -1,277 +1,152 @@ emqx_connector_mongo { - single_mongo_type { - desc { - en: "Standalone instance. Must be set to 'single' when MongoDB server is running in standalone mode." - zh: "Standalone 模式。当 MongoDB 服务运行在 standalone 模式下，该配置必须设置为 'single'。" - } - label: { - en: "Standalone instance" - zh: "Standalone 模式" - } - } +auth_source.desc: +"""Database name associated with the user's credentials.""" - rs_mongo_type { - desc { - en: "Replica set. Must be set to 'rs' when MongoDB server is running in 'replica set' mode." - zh: "Replica set模式。当 MongoDB 服务运行在 replica-set 模式下，该配置必须设置为 'rs'。" - } - label: { - en: "Replica set" - zh: "Replica set 模式" - } - } +auth_source.label: +"""Auth Source""" - sharded_mongo_type { - desc { - en: "Sharded cluster. Must be set to 'sharded' when MongoDB server is running in 'sharded' mode." - zh: "Sharded cluster模式。当 MongoDB 服务运行在 sharded 模式下，该配置必须设置为 'sharded'。" - } - label: { - en: "Sharded cluster" - zh: "Sharded cluster 模式" - } - } +connect_timeout.desc: +"""The duration to attempt a connection before timing out.""" - auth_source { - desc { - en: "Database name associated with the user's credentials." - zh: "与用户证书关联的数据库名称。" - } - label: { - en: "Auth Source" - zh: "认证源" - } - } +connect_timeout.label: +"""Connect Timeout""" - server { - desc { - en: """The IPv4 or IPv6 address or the hostname to connect to.
+desc_rs.desc: +"""Settings for replica set.""" + +desc_rs.label: +"""Setting Replica Set""" + +desc_sharded.desc: +"""Settings for sharded cluster.""" + +desc_sharded.label: +"""Setting Sharded Cluster""" + +desc_single.desc: +"""Settings for a single MongoDB instance.""" + +desc_single.label: +"""Setting Single MongoDB""" + +desc_topology.desc: +"""Topology of MongoDB.""" + +desc_topology.label: +"""Setting Topology""" + +heartbeat_period.desc: +"""Controls when the driver checks the state of the MongoDB deployment. Specify the interval between checks, counted from the end of the previous check until the beginning of the next one. If the number of connections is increased (which will happen, for example, if you increase the pool size), you may need to increase this period as well to avoid creating too many log entries in the MongoDB log file.""" + +heartbeat_period.label: +"""Heartbeat period""" + +local_threshold.desc: +"""The size of the latency window for selecting among multiple suitable MongoDB instances.""" + +local_threshold.label: +"""Local Threshold""" + +max_overflow.desc: +"""Max Overflow.""" + +max_overflow.label: +"""Max Overflow""" + +min_heartbeat_period.desc: +"""Controls the minimum amount of time to wait between heartbeats.""" + +min_heartbeat_period.label: +"""Minimum Heartbeat Period""" + +overflow_check_period.desc: +"""Period for checking if there are more workers than configured ("overflow").""" + +overflow_check_period.label: +"""Overflow Check Period""" + +overflow_ttl.desc: +"""Period of time before workers that exceed the configured pool size ("overflow") to be terminated.""" + +overflow_ttl.label: +"""Overflow TTL""" + +r_mode.desc: +"""Read mode.""" + +r_mode.label: +"""Read Mode""" + +replica_set_name.desc: +"""Name of the replica set.""" + +replica_set_name.label: +"""Replica Set Name""" + +rs_mongo_type.desc: +"""Replica set. Must be set to 'rs' when MongoDB server is running in 'replica set' mode.""" + +rs_mongo_type.label: +"""Replica set""" + +server.desc: +"""The IPv4 or IPv6 address or the hostname to connect to.
A host entry has the following form: `Host[:Port]`.
The MongoDB default port 27017 is used if `[:Port]` is not specified.""" - zh: """将要连接的 IPv4 或 IPv6 地址，或者主机名。
-主机名具有以下形式：`Host[:Port]`。
-如果未指定 `[:Port]`，则使用 MongoDB 默认端口 27017。""" - } - label: { - en: "Server Host" - zh: "服务器地址" - } - } - servers { - desc { - en: """A Node list for Cluster to connect to. The nodes should be separated with commas, such as: `Node[,Node].` +server.label: +"""Server Host""" + +server_selection_timeout.desc: +"""Specifies how long to block for server selection before throwing an exception.""" + +server_selection_timeout.label: +"""Server Selection Timeout""" + +servers.desc: +"""A Node list for Cluster to connect to. The nodes should be separated with commas, such as: `Node[,Node].` For each Node should be: The IPv4 or IPv6 address or the hostname to connect to. A host entry has the following form: `Host[:Port]`. The MongoDB default port 27017 is used if `[:Port]` is not specified.""" - zh: """集群将要连接的节点列表。 节点之间用逗号分隔，如：`Node[,Node].` -每个节点的配置为：将要连接的 IPv4 或 IPv6 地址或主机名。 -主机名具有以下形式：`Host[:Port]`。 -如果未指定 `[:Port]`，则使用 MongoDB 默认端口 27017。""" - } - label: { - en: "Servers" - zh: "服务器列表" - } - } - w_mode { - desc { - en: "Write mode." - zh: "写模式。" - } - label: { - en: "Write Mode" - zh: "写模式" - } - } +servers.label: +"""Servers""" - r_mode { - desc { - en: "Read mode." - zh: "读模式。" - } - label: { - en: "Read Mode" - zh: "读模式" - } - } +sharded_mongo_type.desc: +"""Sharded cluster. Must be set to 'sharded' when MongoDB server is running in 'sharded' mode.""" - overflow_ttl { - desc { - en: "Period of time before workers that exceed the configured pool size (\"overflow\") to be terminated." - zh: "当池内工人太多时，等待多久清除多余工人。" - } - label { - en: "Overflow TTL" - zh: "溢出TTL" - } - } +sharded_mongo_type.label: +"""Sharded cluster""" - overflow_check_period { - desc { - en: "Period for checking if there are more workers than configured (\"overflow\")." - zh: "检查是否有超过配置的工人的周期（\"溢出\"）。" - } - label { - en: "Overflow Check Period" - zh: "溢出检查周期" - } - } +single_mongo_type.desc: +"""Standalone instance. Must be set to 'single' when MongoDB server is running in standalone mode.""" - local_threshold { - desc { - en: "The size of the latency window for selecting among multiple suitable MongoDB instances." - zh: "在多个合适的MongoDB实例中进行选择的延迟窗口的大小。" - } - label { - en: "Local Threshold" - zh: "本地阈值" - } - } +single_mongo_type.label: +"""Standalone instance""" - connect_timeout { - desc { - en: "The duration to attempt a connection before timing out." - zh: "超时重连的等待时间。" - } - label { - en: "Connect Timeout" - zh: "连接超时" - } - } +socket_timeout.desc: +"""The duration to attempt to send or to receive on a socket before the attempt times out.""" - socket_timeout { - desc { - en: "The duration to attempt to send or to receive on a socket before the attempt times out." - zh: "在尝试超时之前，在套接字上尝试发送或接收的持续时间。" - } - label { - en: "Socket Timeout" - zh: "套接字操作超时" - } - } +socket_timeout.label: +"""Socket Timeout""" - server_selection_timeout { - desc { - en: "Specifies how long to block for server selection before throwing an exception." - zh: "指定在抛出异常之前为服务器选择阻断多长时间。" - } - label { - en: "Server Selection Timeout" - zh: "服务器选择超时" - } - } +srv_record.desc: +"""Use DNS SRV record.""" - wait_queue_timeout { - desc { - en: "The maximum duration that a worker can wait for a connection to become available." - zh: "工作者等待连接可用的最长时间。" - } - label { - en: "Wait Queue Timeout" - zh: "等待队列超时" - } - } +srv_record.label: +"""Srv Record""" - heartbeat_period { - desc { - en: "Controls when the driver checks the state of the MongoDB deployment. Specify the interval between checks, counted from the end of the previous check until the beginning of the next one. If the number of connections is increased (which will happen, for example, if you increase the pool size), you may need to increase this period as well to avoid creating too many log entries in the MongoDB log file." - zh: "控制驱动程序何时检查MongoDB部署的状态。指定检查的间隔时间，从上一次检查结束到下一次检查开始计算。如果连接数增加（例如，如果你增加池子的大小，就会发生这种情况），你可能也需要增加这个周期，以避免在MongoDB日志文件中创建太多的日志条目。" - } - label { - en: "Heartbeat period" - zh: "心跳期" - } - } +w_mode.desc: +"""Write mode.""" - min_heartbeat_period { - desc { - en: "Controls the minimum amount of time to wait between heartbeats." - zh: "心跳间的最小间隙" - } - label { - en: "Minimum Heartbeat Period" - zh: "最小心跳周期" - } - } +w_mode.label: +"""Write Mode""" - max_overflow { - desc { - en: "Max Overflow." - zh: "最大溢出。" - } - label: { - en: "Max Overflow" - zh: "最大溢出" - } - } +wait_queue_timeout.desc: +"""The maximum duration that a worker can wait for a connection to become available.""" - replica_set_name { - desc { - en: "Name of the replica set." - zh: "副本集的名称。" - } - label: { - en: "Replica Set Name" - zh: "副本集名称" - } - } - - srv_record { - desc { - en: "Use DNS SRV record." - zh: "使用 DNS SRV 记录。" - } - label: { - en: "Srv Record" - zh: "SRV 记录" - } - } - - desc_single { - desc { - en: """Settings for a single MongoDB instance.""" - zh: """配置 Single 模式""" - } - label: { - en: "Setting Single MongoDB" - zh: "配置 Single 模式" - } - } - - desc_rs { - desc { - en: """Settings for replica set.""" - zh: """配置 Replica Set""" - } - label: { - en: "Setting Replica Set" - zh: "配置 Replica Set" - } - } - - desc_sharded { - desc { - en: """Settings for sharded cluster.""" - zh: """配置 Sharded Cluster""" - } - label: { - en: "Setting Sharded Cluster" - zh: "配置 Sharded Cluster" - } - } - - desc_topology { - desc { - en: """Topology of MongoDB.""" - zh: """配置 Topology""" - } - label: { - en: "Setting Topology" - zh: "配置 Topology" - } - } +wait_queue_timeout.label: +"""Wait Queue Timeout""" } diff --git a/rel/i18n/emqx_connector_mqtt.hocon b/rel/i18n/emqx_connector_mqtt.hocon index 5ade54670..80303b825 100644 --- a/rel/i18n/emqx_connector_mqtt.hocon +++ b/rel/i18n/emqx_connector_mqtt.hocon @@ -1,35 +1,21 @@ emqx_connector_mqtt { - num_of_bridges { - desc { - en: "The current number of bridges that are using this connector." - zh: "当前使用此连接器的网桥数量。" - } - label: { - en: "Num of Bridges" - zh: "网桥数量" - } - } - type { - desc { - en: "The Connector Type." - zh: "连接器类型。" - } - label: { - en: "Connector Type" - zh: "连接器类型" - } - } +name.desc: +"""Connector name, used as a human-readable description of the connector.""" - name { - desc { - en: "Connector name, used as a human-readable description of the connector." - zh: "连接器名称，人类可读的连接器描述。" - } - label: { - en: "Connector Name" - zh: "连接器名称" - } - } +name.label: +"""Connector Name""" + +num_of_bridges.desc: +"""The current number of bridges that are using this connector.""" + +num_of_bridges.label: +"""Num of Bridges""" + +type.desc: +"""The Connector Type.""" + +type.label: +"""Connector Type""" } diff --git a/rel/i18n/emqx_connector_mqtt_schema.hocon b/rel/i18n/emqx_connector_mqtt_schema.hocon index 0de97d84b..e37e87e49 100644 --- a/rel/i18n/emqx_connector_mqtt_schema.hocon +++ b/rel/i18n/emqx_connector_mqtt_schema.hocon @@ -1,319 +1,178 @@ emqx_connector_mqtt_schema { - ingress_desc { - desc { - en: """The ingress config defines how this bridge receive messages from the remote MQTT broker, and then + +bridge_mode.desc: +"""If enable bridge mode. +NOTE: This setting is only for MQTT protocol version older than 5.0, and the remote MQTT +broker MUST support this feature. +If bridge_mode is set to true, the bridge will indicate to the remote broker that it is a bridge not an ordinary client. +This means that loop detection will be more effective and that retained messages will be propagated correctly.""" + +bridge_mode.label: +"""Bridge Mode""" + +clean_start.desc: +"""Whether to start a clean session when reconnecting a remote broker for ingress bridge""" + +clean_start.label: +"""Clean Session""" + +clientid_prefix.desc: +"""Optional prefix to prepend to the clientid used by egress bridges.""" + +clientid_prefix.label: +"""Clientid Prefix""" + +egress_desc.desc: +"""The egress config defines how this bridge forwards messages from the local broker to the remote broker.
+Template with variables is allowed in 'remote.topic', 'local.qos', 'local.retain', 'local.payload'.
+NOTE: if this bridge is used as the action of a rule, and also 'local.topic' +is configured, then both the data got from the rule and the MQTT messages that matches +'local.topic' will be forwarded.""" + +egress_desc.label: +"""Egress Configs""" + +egress_local.desc: +"""The configs about receiving messages from local broker.""" + +egress_local.label: +"""Local Configs""" + +egress_local_topic.desc: +"""The local topic to be forwarded to the remote broker""" + +egress_local_topic.label: +"""Local Topic""" + +egress_remote.desc: +"""The configs about sending message to the remote broker.""" + +egress_remote.label: +"""Remote Configs""" + +egress_remote_qos.desc: +"""The QoS of the MQTT message to be sent.
+Template with variables is allowed.""" + +egress_remote_qos.label: +"""Remote QoS""" + +egress_remote_topic.desc: +"""Forward to which topic of the remote broker.
+Template with variables is allowed.""" + +egress_remote_topic.label: +"""Remote Topic""" + +ingress_desc.desc: +"""The ingress config defines how this bridge receive messages from the remote MQTT broker, and then send them to the local broker.
Template with variables is allowed in 'remote.qos', 'local.topic', 'local.qos', 'local.retain', 'local.payload'.
NOTE: if this bridge is used as the input of a rule, and also 'local.topic' is configured, then messages got from the remote broker will be sent to both the 'local.topic' and the rule.""" - zh: """入口配置定义了该桥接如何从远程 MQTT Broker 接收消息，然后将消息发送到本地 Broker。
- 以下字段中允许使用带有变量的模板：'remote.qos', 'local.topic', 'local.qos', 'local.retain', 'local.payload'。
- 注意：如果此桥接被用作规则的输入，并且配置了 'local.topic'，则从远程代理获取的消息将同时被发送到 'local.topic' 和规则。""" - } - label: { - en: "Ingress Configs" - zh: "入方向配置" - } - } - egress_desc { - desc { - en: """The egress config defines how this bridge forwards messages from the local broker to the remote broker.
-Template with variables is allowed in 'remote.topic', 'local.qos', 'local.retain', 'local.payload'.
-NOTE: if this bridge is used as the action of a rule, and also 'local.topic' -is configured, then both the data got from the rule and the MQTT messages that matches -'local.topic' will be forwarded.""" - zh: """出口配置定义了该桥接如何将消息从本地 Broker 转发到远程 Broker。 -以下字段中允许使用带有变量的模板：'remote.topic', 'local.qos', 'local.retain', 'local.payload'。
-注意：如果此桥接被用作规则的动作，并且配置了 'local.topic'，则从规则输出的数据以及匹配到 'local.topic' 的 MQTT 消息都会被转发。""" - } - label: { - en: "Egress Configs" - zh: "出方向配置" - } - } +ingress_desc.label: +"""Ingress Configs""" - ingress_remote { - desc { - en: """The configs about subscribing to the remote broker.""" - zh: """订阅远程 Broker 相关的配置。""" - } - label: { - en: "Remote Configs" - zh: "远程配置" - } - } +ingress_local.desc: +"""The configs about sending message to the local broker.""" - ingress_local { - desc { - en: """The configs about sending message to the local broker.""" - zh: """发送消息到本地 Broker 相关的配置。""" - } - label: { - en: "Local Configs" - zh: "本地配置" - } - } +ingress_local.label: +"""Local Configs""" - egress_remote { - desc { - en: """The configs about sending message to the remote broker.""" - zh: """发送消息到远程 Broker 相关的配置。""" - } - label: { - en: "Remote Configs" - zh: "远程配置" - } - } +ingress_local_qos.desc: +"""The QoS of the MQTT message to be sent.
+Template with variables is allowed.""" - egress_local { - desc { - en: """The configs about receiving messages from local broker.""" - zh: """如何从本地 Broker 接收消息相关的配置。""" - } - label: { - en: "Local Configs" - zh: "本地配置" - } - } +ingress_local_qos.label: +"""Local QoS""" - mode { - desc { - en: """The mode of the MQTT Bridge.
+ingress_local_topic.desc: +"""Send messages to which topic of the local broker.
+Template with variables is allowed.""" + +ingress_local_topic.label: +"""Local Topic""" + +ingress_remote.desc: +"""The configs about subscribing to the remote broker.""" + +ingress_remote.label: +"""Remote Configs""" + +ingress_remote_qos.desc: +"""The QoS level to be used when subscribing to the remote broker""" + +ingress_remote_qos.label: +"""Remote QoS""" + +ingress_remote_topic.desc: +"""Receive messages from which topic of the remote broker""" + +ingress_remote_topic.label: +"""Remote Topic""" + +max_inflight.desc: +"""Max inflight (sent, but un-acked) messages of the MQTT protocol""" + +max_inflight.label: +"""Max Inflight Message""" + +mode.desc: +"""The mode of the MQTT Bridge.
- cluster_shareload: create an MQTT connection on each node in the emqx cluster.
In 'cluster_shareload' mode, the incoming load from the remote broker is shared by using shared subscription.
Note that the 'clientid' is suffixed by the node name, this is to avoid clientid conflicts between different nodes. And we can only use shared subscription topic filters for remote.topic of ingress connections.""" - zh: """MQTT 桥的模式。
-- cluster_shareload：在 emqx 集群的每个节点上创建一个 MQTT 连接。
-在“cluster_shareload”模式下，来自远程代理的传入负载通过共享订阅的方式接收。
-请注意，clientid 以节点名称为后缀，这是为了避免不同节点之间的 clientid 冲突。 -而且对于入口连接的 remote.topic，我们只能使用共享订阅主题过滤器。""" - } - label: { - en: "MQTT Bridge Mode" - zh: "MQTT 桥接模式" - } - } - server { - desc { - en: "The host and port of the remote MQTT broker" - zh: "远程 MQTT Broker的主机和端口。" - } - label: { - en: "Broker IP And Port" - zh: "Broker主机和端口" - } - } +mode.label: +"""MQTT Bridge Mode""" - bridge_mode { - desc { - en: """If enable bridge mode. -NOTE: This setting is only for MQTT protocol version older than 5.0, and the remote MQTT -broker MUST support this feature. -If bridge_mode is set to true, the bridge will indicate to the remote broker that it is a bridge not an ordinary client. -This means that loop detection will be more effective and that retained messages will be propagated correctly.""" - zh: """是否启用 Bridge Mode。 -注意：此设置只针对 MQTT 协议版本 < 5.0 有效，并且需要远程 MQTT Broker 支持 Bridge Mode。 -如果设置为 true ，桥接会告诉远端服务器当前连接是一个桥接而不是一个普通的客户端。 -这意味着消息回环检测会更加高效，并且远端服务器收到的保留消息的标志位会透传给本地。""" - } - label { - en: "Bridge Mode" - zh: "Bridge 模式" - } - } +password.desc: +"""The password of the MQTT protocol""" - proto_ver { - desc { - en: "The MQTT protocol version" - zh: "MQTT 协议版本" - } - label: { - en: "Protocol Version" - zh: "协议版本" - } - } +password.label: +"""Password""" - username { - desc { - en: "The username of the MQTT protocol" - zh: "MQTT 协议的用户名" - } - label: { - en: "Username" - zh: "用户名" - } - } - - password { - desc { - en: "The password of the MQTT protocol" - zh: "MQTT 协议的密码" - } - label: { - en: "Password" - zh: "密码" - } - } - - clean_start { - desc { - en: "Whether to start a clean session when reconnecting a remote broker for ingress bridge" - zh: "与 ingress MQTT 桥的远程服务器重连时是否清除老的 MQTT 会话。" - } - label: { - en: "Clean Session" - zh: "清除会话" - } - } - - max_inflight { - desc { - en: "Max inflight (sent, but un-acked) messages of the MQTT protocol" - zh: "MQTT 协议的最大飞行（已发送但未确认）消息" - } - label: { - en: "Max Inflight Message" - zh: "最大飞行消息" - } - } - - ingress_remote_topic { - desc { - en: "Receive messages from which topic of the remote broker" - zh: "从远程broker的哪个topic接收消息" - } - label: { - en: "Remote Topic" - zh: "远程主题" - } - } - - ingress_remote_qos { - desc { - en: "The QoS level to be used when subscribing to the remote broker" - zh: "订阅远程borker时要使用的 QoS 级别" - } - label: { - en: "Remote QoS" - zh: "远程 QoS" - } - } - - ingress_local_topic { - desc { - en: """Send messages to which topic of the local broker.
+payload.desc: +"""The payload of the MQTT message to be sent.
Template with variables is allowed.""" - zh: """向本地broker的哪个topic发送消息。
-允许使用带有变量的模板。""" - } - label: { - en: "Local Topic" - zh: "本地主题" - } - } - ingress_local_qos { - desc { - en: """The QoS of the MQTT message to be sent.
+payload.label: +"""Payload""" + +proto_ver.desc: +"""The MQTT protocol version""" + +proto_ver.label: +"""Protocol Version""" + +retain.desc: +"""The 'retain' flag of the MQTT message to be sent.
Template with variables is allowed.""" - zh: """待发送 MQTT 消息的 QoS。
-允许使用带有变量的模板。""" - } - label: { - en: "Local QoS" - zh: "本地 QoS" - } - } - egress_local_topic { - desc { - en: "The local topic to be forwarded to the remote broker" - zh: "要转发到远程broker的本地主题" - } - label: { - en: "Local Topic" - zh: "本地主题" - } - } +retain.label: +"""Retain Flag""" - egress_remote_topic { - desc { - en: """Forward to which topic of the remote broker.
-Template with variables is allowed.""" - zh: """转发到远程broker的哪个topic。
-允许使用带有变量的模板。""" - } - label: { - en: "Remote Topic" - zh: "远程主题" - } - } +server.desc: +"""The host and port of the remote MQTT broker""" - egress_remote_qos { - desc { - en: """The QoS of the MQTT message to be sent.
-Template with variables is allowed.""" - zh: """待发送 MQTT 消息的 QoS。
-允许使用带有变量的模板。""" - } - label: { - en: "Remote QoS" - zh: "远程 QoS" - } - } +server.label: +"""Broker IP And Port""" - retain { - desc { - en: """The 'retain' flag of the MQTT message to be sent.
-Template with variables is allowed.""" - zh: """要发送的 MQTT 消息的“保留”标志。
-允许使用带有变量的模板。""" - } - label: { - en: "Retain Flag" - zh: "保留消息标志" - } - } +server_configs.desc: +"""Configs related to the server.""" - payload { - desc { - en: """The payload of the MQTT message to be sent.
-Template with variables is allowed.""" - zh: """要发送的 MQTT 消息的负载。
-允许使用带有变量的模板。""" - } - label: { - en: "Payload" - zh: "消息负载" - } - } +server_configs.label: +"""Server Configs""" - server_configs { - desc { - en: """Configs related to the server.""" - zh: """服务器相关的配置。""" - } - label: { - en: "Server Configs" - zh: "服务配置。" - } - } +username.desc: +"""The username of the MQTT protocol""" - clientid_prefix { - desc { - en: """Optional prefix to prepend to the clientid used by egress bridges.""" - zh: """可选的前缀，用于在出口网桥使用的clientid前加上前缀。""" - } - label: { - en: "Clientid Prefix" - zh: "客户ID前缀" - } - } +username.label: +"""Username""" } diff --git a/rel/i18n/emqx_connector_mysql.hocon b/rel/i18n/emqx_connector_mysql.hocon index 89e95534b..dd86b62c8 100644 --- a/rel/i18n/emqx_connector_mysql.hocon +++ b/rel/i18n/emqx_connector_mysql.hocon @@ -1,18 +1,11 @@ emqx_connector_mysql { - server { - desc { - en: """The IPv4 or IPv6 address or the hostname to connect to.
+server.desc: +"""The IPv4 or IPv6 address or the hostname to connect to.
A host entry has the following form: `Host[:Port]`.
The MySQL default port 3306 is used if `[:Port]` is not specified.""" - zh: """将要连接的 IPv4 或 IPv6 地址，或者主机名。
-主机名具有以下形式：`Host[:Port]`。
-如果未指定 `[:Port]`，则使用 MySQL 默认端口 3306。""" - } - label: { - en: "Server Host" - zh: "服务器地址" - } - } + +server.label: +"""Server Host""" } diff --git a/rel/i18n/emqx_connector_pgsql.hocon b/rel/i18n/emqx_connector_pgsql.hocon index 33246c844..485e666a0 100644 --- a/rel/i18n/emqx_connector_pgsql.hocon +++ b/rel/i18n/emqx_connector_pgsql.hocon @@ -1,18 +1,11 @@ emqx_connector_pgsql { - server { - desc { - en: """The IPv4 or IPv6 address or the hostname to connect to.
+server.desc: +"""The IPv4 or IPv6 address or the hostname to connect to.
A host entry has the following form: `Host[:Port]`.
The PostgreSQL default port 5432 is used if `[:Port]` is not specified.""" - zh: """将要连接的 IPv4 或 IPv6 地址，或者主机名。
-主机名具有以下形式：`Host[:Port]`。
-如果未指定 `[:Port]`，则使用 PostgreSQL 默认端口 5432。""" - } - label: { - en: "Server Host" - zh: "服务器地址" - } - } + +server.label: +"""Server Host""" } diff --git a/rel/i18n/emqx_connector_redis.hocon b/rel/i18n/emqx_connector_redis.hocon index 5915725a2..5dc887b72 100644 --- a/rel/i18n/emqx_connector_redis.hocon +++ b/rel/i18n/emqx_connector_redis.hocon @@ -1,90 +1,50 @@ emqx_connector_redis { - single { - desc { - en: "Single mode. Must be set to 'single' when Redis server is running in single mode." - zh: "单机模式。当 Redis 服务运行在单机模式下，该配置必须设置为 'single'。" - } - label: { - en: "Single Mode" - zh: "单机模式" - } - } +cluster.desc: +"""Cluster mode. Must be set to 'cluster' when Redis server is running in clustered mode.""" - cluster { - desc { - en: "Cluster mode. Must be set to 'cluster' when Redis server is running in clustered mode." - zh: "集群模式。当 Redis 服务运行在集群模式下，该配置必须设置为 'cluster'。" - } - label: { - en: "Cluster Mode" - zh: "集群模式" - } - } +cluster.label: +"""Cluster Mode""" - sentinel { - desc { - en: "Sentinel mode. Must be set to 'sentinel' when Redis server is running in sentinel mode." - zh: "哨兵模式。当 Redis 服务运行在哨兵模式下，该配置必须设置为 'sentinel'。" - } - label: { - en: "Sentinel Mode" - zh: "哨兵模式" - } - } +database.desc: +"""Redis database ID.""" - sentinel_desc { - desc { - en: "The cluster name in Redis sentinel mode." - zh: "Redis 哨兵模式下的集群名称。" - } - label: { - en: "Cluster Name" - zh: "集群名称" - } - } +database.label: +"""Database ID""" - server { - desc { - en: """The IPv4 or IPv6 address or the hostname to connect to.
+sentinel.desc: +"""Sentinel mode. Must be set to 'sentinel' when Redis server is running in sentinel mode.""" + +sentinel.label: +"""Sentinel Mode""" + +sentinel_desc.desc: +"""The cluster name in Redis sentinel mode.""" + +sentinel_desc.label: +"""Cluster Name""" + +server.desc: +"""The IPv4 or IPv6 address or the hostname to connect to.
A host entry has the following form: `Host[:Port]`.
The Redis default port 6379 is used if `[:Port]` is not specified.""" - zh: """将要连接的 IPv4 或 IPv6 地址，或者主机名。
-主机名具有以下形式：`Host[:Port]`。
-如果未指定 `[:Port]`，则使用 Redis 默认端口 6379。""" - } - label: { - en: "Server Host" - zh: "服务器地址" - } - } - servers { - desc { - en: """A Node list for Cluster to connect to. The nodes should be separated with commas, such as: `Node[,Node].` +server.label: +"""Server Host""" + +servers.desc: +"""A Node list for Cluster to connect to. The nodes should be separated with commas, such as: `Node[,Node].` For each Node should be: The IPv4 or IPv6 address or the hostname to connect to. A host entry has the following form: `Host[:Port]`. The Redis default port 6379 is used if `[:Port]` is not specified.""" - zh: """集群将要连接的节点列表。 节点之间用逗号分隔，如：`Node[,Node].` -每个节点的配置为：将要连接的 IPv4 或 IPv6 地址或主机名。 -主机名具有以下形式：`Host[:Port]`。 -如果未指定 `[:Port]`，则使用 Redis 默认端口 6379。""" - } - label: { - en: "Servers" - zh: "服务器列表" - } - } - database { - desc { - en: "Redis database ID." - zh: "Redis 数据库 ID。" - } - label: { - en: "Database ID" - zh: "数据库 ID" - } - } +servers.label: +"""Servers""" + +single.desc: +"""Single mode. Must be set to 'single' when Redis server is running in single mode.""" + +single.label: +"""Single Mode""" } diff --git a/rel/i18n/emqx_connector_schema_lib.hocon b/rel/i18n/emqx_connector_schema_lib.hocon index 1bc45c36d..0e8a2e9a3 100644 --- a/rel/i18n/emqx_connector_schema_lib.hocon +++ b/rel/i18n/emqx_connector_schema_lib.hocon @@ -1,80 +1,45 @@ emqx_connector_schema_lib { - ssl { - desc { - en: "SSL connection settings." - zh: "启用 SSL 连接。" - } - label: { - en: "Enable SSL" - zh: "启用SSL" - } - } +auto_reconnect.desc: +"""Deprecated. Enable automatic reconnect to the database.""" - prepare_statement { - desc { - en: "Key-value list of SQL prepared statements." - zh: "SQL 预处理语句列表。" - } - label: { - en: "SQL Prepared Statements List" - zh: "SQL 预处理语句列表" - } - } +auto_reconnect.label: +"""Deprecated. Auto Reconnect Database""" - database_desc { - desc { - en: "Database name." - zh: "数据库名字。" - } - label: { - en: "Database Name" - zh: "数据库名字" - } - } +database_desc.desc: +"""Database name.""" - pool_size { - desc { - en: "Size of the connection pool towards the bridge target service." - zh: "桥接远端服务时使用的连接池大小。" - } - label: { - en: "Connection Pool Size" - zh: "连接池大小" - } - } +database_desc.label: +"""Database Name""" - username { - desc { - en: "EMQX's username in the external database." - zh: "内部数据库的用户名。" - } - label: { - en: "Username" - zh: "用户名" - } - } +password.desc: +"""EMQX's password in the external database.""" - password { - desc { - en: "EMQX's password in the external database." - zh: "内部数据库密码。" - } - label: { - en: "Password" - zh: "密码" - } - } +password.label: +"""Password""" - auto_reconnect { - desc { - en: "Deprecated. Enable automatic reconnect to the database." - zh: "已弃用。自动重连数据库。" - } - label: { - en: "Deprecated. Auto Reconnect Database" - zh: "已弃用。自动重连数据库" - } - } +pool_size.desc: +"""Size of the connection pool towards the bridge target service.""" + +pool_size.label: +"""Connection Pool Size""" + +prepare_statement.desc: +"""Key-value list of SQL prepared statements.""" + +prepare_statement.label: +"""SQL Prepared Statements List""" + +ssl.desc: +"""SSL connection settings.""" + +ssl.label: +"""Enable SSL""" + +username.desc: +"""EMQX's username in the external database.""" + +username.label: +"""Username""" } diff --git a/rel/i18n/emqx_dashboard_api.hocon b/rel/i18n/emqx_dashboard_api.hocon index 9a6390a02..3e5bb6239 100644 --- a/rel/i18n/emqx_dashboard_api.hocon +++ b/rel/i18n/emqx_dashboard_api.hocon @@ -1,150 +1,66 @@ emqx_dashboard_api { - token { - desc { - en: """Dashboard Auth Token""" - zh: """Dashboard 认证 Token""" - } - } +change_pwd_api.desc: +"""Change dashboard user password""" - username { - desc { - en: """Dashboard Username""" - zh: """Dashboard 用户名""" - } - } +create_user_api.desc: +"""Create dashboard user""" - user_description { - desc { - en: """Dashboard User Description""" - zh: """Dashboard 用户描述""" - } - } +create_user_api_success.desc: +"""Create dashboard user success""" - password { - desc { - en: """Dashboard Password""" - zh: """Dashboard 密码""" - } - } +delete_user_api.desc: +"""Delete dashboard user""" - license { - desc { - en: """EMQX License. opensource or enterprise""" - zh: """EMQX 许可类型。可为 opensource 或 enterprise""" - } - } +license.desc: +"""EMQX License. opensource or enterprise""" - version { - desc { - en: """EMQX Version""" - zh: """EMQX 版本""" - } - } +list_users_api.desc: +"""Dashboard list users""" - login_api { - desc { - en: """Get Dashboard Auth Token.""" - zh: """获取 Dashboard 认证 Token。""" - } - } +login_api.desc: +"""Get Dashboard Auth Token.""" - login_success { - desc { - en: """Dashboard Auth Success""" - zh: """Dashboard 认证成功""" - } - } +login_failed401.desc: +"""Login failed. Bad username or password""" - login_failed401 { - desc { - en: """Login failed. Bad username or password""" - zh: """登录失败。用户名或密码错误""" - } - } +login_failed_response400.desc: +"""Login failed. Bad username or password""" - logout_api { - desc { - en: """Dashboard user logout""" - zh: """Dashboard 用户登出""" - } - } +login_success.desc: +"""Dashboard Auth Success""" - list_users_api { - desc { - en: """Dashboard list users""" - zh: """Dashboard 用户列表""" - } - } +logout_api.desc: +"""Dashboard user logout""" - create_user_api { - desc { - en: """Create dashboard user""" - zh: """创建 Dashboard 用户""" - } - } +new_pwd.desc: +"""New password""" - create_user_api_success { - desc { - en: """Create dashboard user success""" - zh: """创建 Dashboard 用户成功""" - } - } +old_pwd.desc: +"""Old password""" - update_user_api { - desc { - en: """Update dashboard user description""" - zh: """更新 Dashboard 用户描述""" - } - } +password.desc: +"""Dashboard Password""" - update_user_api200 { - desc { - en: """Update dashboard user success""" - zh: """更新 Dashboard 用户成功""" - } - } +token.desc: +"""Dashboard Auth Token""" - delete_user_api { - desc { - en: """Delete dashboard user""" - zh: """删除 Dashboard 用户""" - } - } +update_user_api.desc: +"""Update dashboard user description""" - users_api404 { - desc { - en: """Dashboard user not found""" - zh: """Dashboard 用户不存在""" - } - } +update_user_api200.desc: +"""Update dashboard user success""" - change_pwd_api { - desc { - en: """Change dashboard user password""" - zh: """更改 Dashboard 用户密码""" - } - } +user_description.desc: +"""Dashboard User Description""" - old_pwd { - desc { - en: """Old password""" - zh: """旧密码""" - } - } +username.desc: +"""Dashboard Username""" - new_pwd { - desc { - en: """New password""" - zh: """新密码""" - } - } +users_api404.desc: +"""Dashboard user not found""" - login_failed_response400 { - desc { - en: """Login failed. Bad username or password""" - zh: """登录失败。用户名或密码错误""" - } - } +version.desc: +"""EMQX Version""" } diff --git a/rel/i18n/emqx_dashboard_schema.hocon b/rel/i18n/emqx_dashboard_schema.hocon index f81816e63..6bd6ab016 100644 --- a/rel/i18n/emqx_dashboard_schema.hocon +++ b/rel/i18n/emqx_dashboard_schema.hocon @@ -1,225 +1,139 @@ emqx_dashboard_schema { - listeners { - desc { - en: """HTTP(s) listeners are identified by their protocol type and are + +backlog.desc: +"""Defines the maximum length that the queue of pending connections can grow to.""" + +backlog.label: +"""Backlog""" + +bind.desc: +"""Port without IP(18083) or port with specified IP(127.0.0.1:18083).""" + +bind.label: +"""Bind""" + +bootstrap_users_file.desc: +"""Deprecated, use api_key.bootstrap_file.""" + +bootstrap_users_file.label: +"""Deprecated""" + +cors.desc: +"""Support Cross-Origin Resource Sharing (CORS). +Allows a server to indicate any origins (domain, scheme, or port) other than +its own from which a browser should permit loading resources.""" + +cors.label: +"""CORS""" + +default_password.desc: +"""The initial default password for dashboard 'admin' user. +For safety, it should be changed as soon as possible. +This value is not valid when you log in to Dashboard for the first time via the web +and change to a complex password as prompted.""" + +default_password.label: +"""Default password""" + +default_username.desc: +"""The default username of the automatically created dashboard user.""" + +default_username.label: +"""Default username""" + +desc_dashboard.desc: +"""Configuration for EMQX dashboard.""" + +desc_dashboard.label: +"""Dashboard""" + +desc_http.desc: +"""Configuration for the dashboard listener (plaintext).""" + +desc_http.label: +"""HTTP""" + +desc_https.desc: +"""Configuration for the dashboard listener (TLS).""" + +desc_https.label: +"""HTTPS""" + +desc_listeners.desc: +"""Configuration for the dashboard listener.""" + +desc_listeners.label: +"""Listeners""" + +i18n_lang.desc: +"""Internationalization language support.""" + +i18n_lang.label: +"""I18n language""" + +inet6.desc: +"""Enable IPv6 support, default is false, which means IPv4 only.""" + +inet6.label: +"""IPv6""" + +ipv6_v6only.desc: +"""Disable IPv4-to-IPv6 mapping for the listener. +The configuration is only valid when the inet6 is true.""" + +ipv6_v6only.label: +"""IPv6 only""" + +listener_enable.desc: +"""Ignore or enable this listener""" + +listener_enable.label: +"""Enable""" + +listeners.desc: +"""HTTP(s) listeners are identified by their protocol type and are used to serve dashboard UI and restful HTTP API. Listeners must have a unique combination of port number and IP address. For example, an HTTP listener can listen on all configured IP addresses on a given port for a machine by specifying the IP address 0.0.0.0. Alternatively, the HTTP listener can specify a unique IP address for each listener, but use the same port.""" - zh: """Dashboard 监听器设置。监听器必须有唯一的端口号和IP地址的组合。 -例如，可以通过指定IP地址 0.0.0.0 来监听机器上给定端口上的所有配置的IP地址。 -或者，可以为每个监听器指定唯一的IP地址，但使用相同的端口。""" - } - label { - en: "Listeners" - zh: "监听器" - } - } - sample_interval { - desc { - en: """How often to update metrics displayed in the dashboard. + +listeners.label: +"""Listeners""" + +max_connections.desc: +"""Maximum number of simultaneous connections.""" + +max_connections.label: +"""Maximum connections""" + +num_acceptors.desc: +"""Socket acceptor pool size for TCP protocols. Default is the number of schedulers online""" + +num_acceptors.label: +"""Number of acceptors""" + +proxy_header.desc: +"""Enable support for `HAProxy` header. Be aware once enabled regular HTTP requests can't be handled anymore.""" + +proxy_header.label: +"""Enable support for HAProxy header""" + +sample_interval.desc: +"""How often to update metrics displayed in the dashboard. Note: `sample_interval` should be a divisor of 60, default is 10s.""" - zh: """Dashboard 中图表指标的时间间隔。必须小于60，且被60的整除，默认设置 10s。""" - } - } - token_expired_time { - desc { - en: "JWT token expiration time. Default is 60 minutes" - zh: "JWT token 过期时间。默认设置为 60 分钟。" - } - label { - en: "Token expired time" - zh: "JWT 过期时间" - } - } - num_acceptors { - desc { - en: "Socket acceptor pool size for TCP protocols. Default is the number of schedulers online" - zh: "TCP协议的Socket acceptor池大小, 默认设置在线的调度器数量（通常为 CPU 核数）" - } - label { - en: "Number of acceptors" - zh: "Acceptor 数量" - } - } - max_connections { - desc { - en: "Maximum number of simultaneous connections." - zh: "同时处理的最大连接数。" - } - label { - en: "Maximum connections" - zh: "最大连接数" - } - } - backlog { - desc { - en: "Defines the maximum length that the queue of pending connections can grow to." - zh: "排队等待连接的队列的最大长度。" - } - label { - en: "Backlog" - zh: "排队长度" - } - } - send_timeout { - desc { - en: "Send timeout for the socket." - zh: "Socket发送超时时间。" - } - label { - en: "Send timeout" - zh: "发送超时时间" - } - } - inet6 { - desc { - en: "Enable IPv6 support, default is false, which means IPv4 only." - zh: "启用IPv6， 如果机器不支持IPv6，请关闭此选项，否则会导致 Dashboard 无法使用。" - } - label { - en: "IPv6" - zh: "IPv6" - } - } - ipv6_v6only { - desc { - en: """Disable IPv4-to-IPv6 mapping for the listener. -The configuration is only valid when the inet6 is true.""" - zh: "当开启 inet6 功能的同时禁用 IPv4-to-IPv6 映射。该配置仅在 inet6 功能开启时有效。" - } - label { - en: "IPv6 only" - zh: "IPv6 only" - } - } - proxy_header { - desc { - en: "Enable support for `HAProxy` header. Be aware once enabled regular HTTP requests can't be handled anymore." - zh: "开启对 `HAProxy` 的支持，注意：一旦开启了这个功能，就无法再处理普通的 HTTP 请求了。" - } - label: { - en: "Enable support for HAProxy header" - zh: "开启对 `HAProxy` 的支持" - } - } - desc_dashboard { - desc { - en: "Configuration for EMQX dashboard." - zh: "EMQX Dashboard 配置。" - } - label { - en: "Dashboard" - zh: "Dashboard" - } - } - desc_listeners { - desc { - en: "Configuration for the dashboard listener." - zh: "Dashboard 监听器配置。" - } - label { - en: "Listeners" - zh: "监听器" - } - } - desc_http { - desc { - en: "Configuration for the dashboard listener (plaintext)." - zh: "Dashboard 监听器(HTTP)配置。" - } - label { - en: "HTTP" - zh: "HTTP" - } - } - desc_https { - desc { - en: "Configuration for the dashboard listener (TLS)." - zh: "Dashboard 监听器(HTTPS)配置。" - } - label { - en: "HTTPS" - zh: "HTTPS" - } - } - listener_enable { - desc { - en: "Ignore or enable this listener" - zh: "忽略或启用该监听器。" - } - label { - en: "Enable" - zh: "启用" - } - } - bind { - desc { - en: "Port without IP(18083) or port with specified IP(127.0.0.1:18083)." - zh: "监听地址和端口，热更新此配置时，会重启 Dashboard 服务。" - } - label { - en: "Bind" - zh: "绑定端口" - } - } - default_username { - desc { - en: "The default username of the automatically created dashboard user." - zh: "Dashboard 的默认用户名。" - } - label { - en: "Default username" - zh: "默认用户名" - } - } - default_password { - desc { - en: """The initial default password for dashboard 'admin' user. -For safety, it should be changed as soon as possible. -This value is not valid when you log in to Dashboard for the first time via the web -and change to a complex password as prompted.""" - zh: """Dashboard 的默认密码，为了安全，应该尽快修改密码。 -当通过网页首次登录 Dashboard 并按提示修改成复杂密码后，此值就会失效。""" - } - label { - en: "Default password" - zh: "默认密码" - } - } - cors { - desc { - en: """Support Cross-Origin Resource Sharing (CORS). -Allows a server to indicate any origins (domain, scheme, or port) other than -its own from which a browser should permit loading resources.""" - zh: """支持跨域资源共享(CORS)， -允许服务器指示任何来源(域名、协议或端口)，除了本服务器之外的任何浏览器应允许加载资源。""" - } - label { - en: "CORS" - zh: "跨域资源共享" - } - } - i18n_lang { - desc { - en: "Internationalization language support." - zh: "设置 Swagger 多语言的版本，可为 en 或 zh。" - } - label { - en: "I18n language" - zh: "多语言支持" - } - } - bootstrap_users_file { - desc { - en: "Deprecated, use api_key.bootstrap_file." - zh: "已废弃，请使用 api_key.bootstrap_file。" - } - label { - en: """Deprecated""" - zh: """已废弃""" - } - } + +send_timeout.desc: +"""Send timeout for the socket.""" + +send_timeout.label: +"""Send timeout""" + +token_expired_time.desc: +"""JWT token expiration time. Default is 60 minutes""" + +token_expired_time.label: +"""Token expired time""" + } diff --git a/rel/i18n/emqx_delayed_api.hocon b/rel/i18n/emqx_delayed_api.hocon index d16d61ccf..62e0fd775 100644 --- a/rel/i18n/emqx_delayed_api.hocon +++ b/rel/i18n/emqx_delayed_api.hocon @@ -1,164 +1,72 @@ emqx_delayed_api { - view_status_api { - desc { - en: "Get delayed status" - zh: "查看慢订阅状态" - } - } +bad_msgid_format.desc: +"""Bad Message ID format""" - update_api { - desc { - en: "Enable or disable delayed, set max delayed messages" - zh: "开启或者关闭功能，或者设置延迟消息数量上限" - } - } +count.desc: +"""Count of delayed messages""" - update_success { - desc { - en: "Enable or disable delayed successfully" - zh: "开启或者关闭功能操作成功" - } - } +delayed_interval.desc: +"""Delayed interval(second)""" - illegality_limit { - desc { - en: "Max limit illegality" - zh: "数量上限不合法" - } - } +delayed_remaining.desc: +"""Delayed remaining(second)""" - get_message_api { - desc { - en: "View delayed message" - zh: "查看延迟消息" - } - } +delete_api.desc: +"""Delete delayed message""" - node { - desc { - en: "The node where message from" - zh: "消息的来源节点" - } - } +expected_at.desc: +"""Expect publish time, in RFC 3339 format""" - msgid { - desc { - en: "Delayed Message ID" - zh: "延迟消息 ID" - } - } +from_clientid.desc: +"""From ClientID""" - bad_msgid_format { - desc { - en: "Bad Message ID format" - zh: "消息 ID 格式错误" - } - } +from_username.desc: +"""From Username""" - msgid_not_found { - desc { - en: "Message ID not found" - zh: "未找到对应消息" - } - } +get_message_api.desc: +"""View delayed message""" - delete_api { - desc { - en: "Delete delayed message" - zh: "删除延迟消息" - } - } +illegality_limit.desc: +"""Max limit illegality""" - list_api { - desc { - en: "List delayed messages" - zh: "查看延迟消息列表" - } - } +list_api.desc: +"""List delayed messages""" - view_page { - desc { - en: "View page" - zh: "查看的页数" - } - } +msgid.desc: +"""Delayed Message ID""" - view_limit { - desc { - en: "Page limit" - zh: "每页数量" - } - } +msgid_not_found.desc: +"""Message ID not found""" - count { - desc { - en: "Count of delayed messages" - zh: "延迟消息总数" - } - } +node.desc: +"""The node where message from""" - publish_at { - desc { - en: "Clinet publish message time, in RFC 3339 format" - zh: "客户端发送时间, RFC 3339 格式" - } - } +payload.desc: +"""Payload, base64 encoded. Payload will be set to 'PAYLOAD_TO_LARGE' if its length is larger than 2048 bytes""" - delayed_interval { - desc { - en: "Delayed interval(second)" - zh: "延迟时间(秒)" - } - } +publish_at.desc: +"""Clinet publish message time, in RFC 3339 format""" - delayed_remaining { - desc { - en: "Delayed remaining(second)" - zh: "剩余时间(秒)" - } - } +qos.desc: +"""QoS""" - expected_at { - desc { - en: "Expect publish time, in RFC 3339 format" - zh: "期望的发送时间, RFC 3339 格式" - } - } +topic.desc: +"""Topic""" - topic { - desc { - en: "Topic" - zh: "主题" - } - } +update_api.desc: +"""Enable or disable delayed, set max delayed messages""" - qos { - desc { - en: "QoS" - zh: "QoS" - } - } +update_success.desc: +"""Enable or disable delayed successfully""" - from_clientid { - desc { - en: "From ClientID" - zh: "消息的 ClientID" - } - } +view_limit.desc: +"""Page limit""" - from_username { - desc { - en: "From Username" - zh: "消息的 Username" - } - } +view_page.desc: +"""View page""" - payload { - desc { - en: "Payload, base64 encoded. Payload will be set to 'PAYLOAD_TO_LARGE' if its length is larger than 2048 bytes" - zh: "消息内容, base64 格式。如果消息的大小超过 2048 字节，则消息内容会被设置为 'PAYLOAD_TO_LARGE'" - } - } +view_status_api.desc: +"""Get delayed status""" } diff --git a/rel/i18n/emqx_ee_bridge_cassa.hocon b/rel/i18n/emqx_ee_bridge_cassa.hocon deleted file mode 100644 index 3bbac6658..000000000 --- a/rel/i18n/emqx_ee_bridge_cassa.hocon +++ /dev/null @@ -1,72 +0,0 @@ -emqx_ee_bridge_cassa { - - local_topic { - desc { - en: """The MQTT topic filter to be forwarded to Cassandra. All MQTT 'PUBLISH' messages with the topic -matching the local_topic will be forwarded.
-NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is -configured, then both the data got from the rule and the MQTT messages that match local_topic -will be forwarded.""" - zh: """发送到 'local_topic' 的消息都会转发到 Cassandra。
-注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。""" - } - label { - en: "Local Topic" - zh: "本地 Topic" - } - } - - cql_template { - desc { - en: """CQL Template""" - zh: """CQL 模板""" - } - label { - en: "CQL Template" - zh: "CQL 模板" - } - } - config_enable { - desc { - en: """Enable or disable this bridge""" - zh: """启用/禁用桥接""" - } - label { - en: "Enable Or Disable Bridge" - zh: "启用/禁用桥接" - } - } - - desc_config { - desc { - en: """Configuration for a Cassandra bridge.""" - zh: """Cassandra 桥接配置""" - } - label: { - en: "Cassandra Bridge Configuration" - zh: "Cassandra 桥接配置" - } - } - - desc_type { - desc { - en: """The Bridge Type""" - zh: """Bridge 类型""" - } - label { - en: "Bridge Type" - zh: "桥接类型" - } - } - - desc_name { - desc { - en: """Bridge name.""" - zh: """桥接名字""" - } - label { - en: "Bridge Name" - zh: "桥接名字" - } - } -} diff --git a/rel/i18n/emqx_ee_bridge_clickhouse.hocon b/rel/i18n/emqx_ee_bridge_clickhouse.hocon index b54f4dc70..6735aee22 100644 --- a/rel/i18n/emqx_ee_bridge_clickhouse.hocon +++ b/rel/i18n/emqx_ee_bridge_clickhouse.hocon @@ -1,81 +1,49 @@ emqx_ee_bridge_clickhouse { - local_topic { - desc { - en: """The MQTT topic filter to be forwarded to Clickhouse. All MQTT 'PUBLISH' messages with the topic +batch_value_separator.desc: +"""The default value ',' works for the VALUES format. You can also use other separator if other format is specified. See [INSERT INTO Statement](https://clickhouse.com/docs/en/sql-reference/statements/insert-into).""" + +batch_value_separator.label: +"""Batch Value Separator""" + +config_enable.desc: +"""Enable or disable this bridge""" + +config_enable.label: +"""Enable Or Disable Bridge""" + +desc_config.desc: +"""Configuration for a Clickhouse bridge.""" + +desc_config.label: +"""Clickhouse Bridge Configuration""" + +desc_name.desc: +"""Bridge name.""" + +desc_name.label: +"""Bridge Name""" + +desc_type.desc: +"""The Bridge Type""" + +desc_type.label: +"""Bridge Type""" + +local_topic.desc: +"""The MQTT topic filter to be forwarded to Clickhouse. All MQTT 'PUBLISH' messages with the topic matching the local_topic will be forwarded.
NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is configured, then both the data got from the rule and the MQTT messages that match local_topic will be forwarded.""" - zh: """发送到 'local_topic' 的消息都会转发到 Clickhouse。
-注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。""" - } - label { - en: "Local Topic" - zh: "本地 Topic" - } - } - sql_template { - desc { - en: """The template string can contain ${field} placeholders for message metadata and payload field. Make sure that the inserted values are formatted and escaped correctly. [Prepared Statement](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridges.html#Prepared-Statement) is not supported.""" - zh: """可以使用 ${field} 占位符来引用消息与客户端上下文中的变量，请确保对应字段存在且数据格式符合预期。此处不支持 [SQL 预处理](https://docs.emqx.com/zh/enterprise/v5.0/data-integration/data-bridges.html#sql-预处理)。""" - } - label { - en: "SQL Template" - zh: "SQL 模板" - } - } - batch_value_separator { - desc { - en: """The default value ',' works for the VALUES format. You can also use other separator if other format is specified. See [INSERT INTO Statement](https://clickhouse.com/docs/en/sql-reference/statements/insert-into).""" - zh: """默认为逗号 ','，适用于 VALUE 格式。您也可以使用其他分隔符， 请参考 [INSERT INTO 语句](https://clickhouse.com/docs/en/sql-reference/statements/insert-into)。""" - } - label { - en: "Batch Value Separator" - zh: "分隔符" - } - } - config_enable { - desc { - en: """Enable or disable this bridge""" - zh: """启用/禁用桥接""" - } - label { - en: "Enable Or Disable Bridge" - zh: "启用/禁用桥接" - } - } - desc_config { - desc { - en: """Configuration for a Clickhouse bridge.""" - zh: """Clickhouse 桥接配置""" - } - label: { - en: "Clickhouse Bridge Configuration" - zh: "Clickhouse 桥接配置" - } - } +local_topic.label: +"""Local Topic""" - desc_type { - desc { - en: """The Bridge Type""" - zh: """Bridge 类型""" - } - label { - en: "Bridge Type" - zh: "桥接类型" - } - } +sql_template.desc: +"""The template string can contain ${field} placeholders for message metadata and payload field. Make sure that the inserted values are formatted and escaped correctly. [Prepared Statement](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridges.html#Prepared-Statement) is not supported.""" + +sql_template.label: +"""SQL Template""" - desc_name { - desc { - en: """Bridge name.""" - zh: """桥接名字""" - } - label { - en: "Bridge Name" - zh: "桥接名字" - } - } } diff --git a/rel/i18n/emqx_ee_bridge_dynamo.hocon b/rel/i18n/emqx_ee_bridge_dynamo.hocon index 664b13174..7725130eb 100644 --- a/rel/i18n/emqx_ee_bridge_dynamo.hocon +++ b/rel/i18n/emqx_ee_bridge_dynamo.hocon @@ -1,72 +1,43 @@ emqx_ee_bridge_dynamo { - local_topic { - desc { - en: """The MQTT topic filter to be forwarded to DynamoDB. All MQTT `PUBLISH` messages with the topic +config_enable.desc: +"""Enable or disable this bridge""" + +config_enable.label: +"""Enable Or Disable Bridge""" + +desc_config.desc: +"""Configuration for a DynamoDB bridge.""" + +desc_config.label: +"""DynamoDB Bridge Configuration""" + +desc_name.desc: +"""Bridge name.""" + +desc_name.label: +"""Bridge Name""" + +desc_type.desc: +"""The Bridge Type""" + +desc_type.label: +"""Bridge Type""" + +local_topic.desc: +"""The MQTT topic filter to be forwarded to DynamoDB. All MQTT `PUBLISH` messages with the topic matching the `local_topic` will be forwarded.
NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also `local_topic` is configured, then both the data got from the rule and the MQTT messages that match `local_topic` will be forwarded.""" - zh: """发送到 'local_topic' 的消息都会转发到 DynamoDB。
-注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。""" - } - label { - en: "Local Topic" - zh: "本地 Topic" - } - } - template { - desc { - en: """Template, the default value is empty. When this value is empty the whole message will be stored in the database""" - zh: """模板, 默认为空，为空时将会将整个消息存入数据库""" - } - label { - en: "Template" - zh: "模板" - } - } - config_enable { - desc { - en: """Enable or disable this bridge""" - zh: """启用/禁用桥接""" - } - label { - en: "Enable Or Disable Bridge" - zh: "启用/禁用桥接" - } - } +local_topic.label: +"""Local Topic""" - desc_config { - desc { - en: """Configuration for an DynamoDB bridge.""" - zh: """DynamoDB 桥接配置""" - } - label: { - en: "DynamoDB Bridge Configuration" - zh: "DynamoDB 桥接配置" - } - } +template.desc: +"""Template, the default value is empty. When this value is empty the whole message will be stored in the database""" - desc_type { - desc { - en: """The Bridge Type""" - zh: """Bridge 类型""" - } - label { - en: "Bridge Type" - zh: "桥接类型" - } - } +template.label: +"""Template""" - desc_name { - desc { - en: """Bridge name.""" - zh: """桥接名字""" - } - label { - en: "Bridge Name" - zh: "桥接名字" - } - } } diff --git a/rel/i18n/emqx_ee_bridge_gcp_pubsub.hocon b/rel/i18n/emqx_ee_bridge_gcp_pubsub.hocon deleted file mode 100644 index b8fa3b43a..000000000 --- a/rel/i18n/emqx_ee_bridge_gcp_pubsub.hocon +++ /dev/null @@ -1,145 +0,0 @@ -emqx_ee_bridge_gcp_pubsub { - desc_config { - desc { - en: """Configuration for a GCP PubSub bridge.""" - zh: """GCP PubSub 桥接配置""" - } - label { - en: "GCP PubSub Bridge Configuration" - zh: "GCP PubSub 桥接配置" - } - } - - desc_type { - desc { - en: """The Bridge Type""" - zh: """桥接类型""" - } - label { - en: "Bridge Type" - zh: "桥接类型" - } - } - desc_name { - desc { - en: """Bridge name, used as a human-readable description of the bridge.""" - zh: """桥接名字，可读描述""" - } - label { - en: "Bridge Name" - zh: "桥接名字" - } - } - - connect_timeout { - desc { - en: "The timeout when connecting to the HTTP server." - zh: "连接 HTTP 服务器的超时时间。" - } - label: { - en: "Connect Timeout" - zh: "连接超时" - } - } - - max_retries { - desc { - en: "Max retry times if an error occurs when sending a request." - zh: "请求出错时的最大重试次数。" - } - label: { - en: "Max Retries" - zh: "最大重试次数" - } - } - - pool_size { - desc { - en: "The pool size." - zh: "连接池大小。" - } - label: { - en: "Pool Size" - zh: "连接池大小" - } - } - - pipelining { - desc { - en: "A positive integer. Whether to send HTTP requests continuously, when set to 1, it means that after each HTTP request is sent, you need to wait for the server to return and then continue to send the next request." - zh: "正整数，设置最大可发送的异步 HTTP 请求数量。当设置为 1 时，表示每次发送完成 HTTP 请求后都需要等待服务器返回，再继续发送下一个请求。" - } - label: { - en: "HTTP Pipelineing" - zh: "HTTP 流水线" - } - } - - request_timeout { - desc { - en: "Deprecated: Configure the request timeout in the buffer settings." - zh: "废弃的。在缓冲区设置中配置请求超时。" - } - label: { - en: "Request Timeout" - zh: "HTTP 请求超时" - } - } - - payload_template { - desc { - en: "The template for formatting the outgoing messages. If undefined, will send all the available context in JSON format." - zh: "用于格式化外发信息的模板。 如果未定义，将以JSON格式发送所有可用的上下文。" - } - label: { - en: "Payload template" - zh: "HTTP 请求消息体模板" - } - } - - local_topic { - desc { - en: """The MQTT topic filter to be forwarded to GCP PubSub. All MQTT 'PUBLISH' messages with the topic -matching `local_topic` will be forwarded.
-NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is -configured, then both the data got from the rule and the MQTT messages that match local_topic -will be forwarded.""" - zh: """发送到 'local_topic' 的消息都会转发到 GCP PubSub。
-注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发到 GCP PubSub。""" - } - label { - en: "Local Topic" - zh: "本地 Topic" - } - } - - pubsub_topic { - desc { - en: "The GCP PubSub topic to publish messages to." - zh: "要发布消息的GCP PubSub主题。" - } - label: { - en: "GCP PubSub Topic" - zh: "GCP PubSub 主题" - } - } - - service_account_json { - desc { - en: "JSON containing the GCP Service Account credentials to be used with PubSub.\n" - "When a GCP Service Account is created " - "(as described in https://developers.google.com/identity/protocols/oauth2/service-account#creatinganaccount), " - "you have the option of downloading the credentials in JSON form. That's the " - "file needed." - zh: "包含将与 PubSub 一起使用的 GCP 服务账户凭证的 JSON。\n" - "当创建GCP服务账户时" - "（如https://developers.google.com/identity/protocols/oauth2/service-account#creatinganaccount），" - "可以选择下载 JSON 形式的凭证，然后在该配置项中使用。" - } - label: { - en: "GCP Service Account Credentials" - zh: "GCP 服务账户凭证" - } - } - -} diff --git a/rel/i18n/emqx_ee_bridge_hstreamdb.hocon b/rel/i18n/emqx_ee_bridge_hstreamdb.hocon index dce40aa85..cb43d483a 100644 --- a/rel/i18n/emqx_ee_bridge_hstreamdb.hocon +++ b/rel/i18n/emqx_ee_bridge_hstreamdb.hocon @@ -1,90 +1,55 @@ emqx_ee_bridge_hstreamdb { - local_topic { - desc { - en: """The MQTT topic filter to be forwarded to the HStreamDB. All MQTT 'PUBLISH' messages with the topic + +config_direction.desc: +"""The direction of this bridge, MUST be 'egress'""" + +config_direction.label: +"""Bridge Direction""" + +config_enable.desc: +"""Enable or disable this bridge""" + +config_enable.label: +"""Enable Or Disable Bridge""" + +desc_config.desc: +"""Configuration for an HStreamDB bridge.""" + +desc_config.label: +"""HStreamDB Bridge Configuration""" + +desc_connector.desc: +"""Generic configuration for the connector.""" + +desc_connector.label: +"""Connector Generic Configuration""" + +desc_name.desc: +"""Bridge name, used as a human-readable description of the bridge.""" + +desc_name.label: +"""Bridge Name""" + +desc_type.desc: +"""The Bridge Type""" + +desc_type.label: +"""Bridge Type""" + +local_topic.desc: +"""The MQTT topic filter to be forwarded to the HStreamDB. All MQTT 'PUBLISH' messages with the topic matching the local_topic will be forwarded.
NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is configured, then both the data got from the rule and the MQTT messages that match local_topic will be forwarded.""" - zh: """发送到 'local_topic' 的消息都会转发到 HStreamDB。
-注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发到 HStreamDB。""" - } - label { - en: "Local Topic" - zh: "本地 Topic" - } - } - payload { - desc { - en: """The payload to be forwarded to the HStreamDB. Placeholders supported.""" - zh: """要转发到 HStreamDB 的数据内容，支持占位符""" - } - label { - en: "Payload" - zh: "消息内容" - } - } - config_enable { - desc { - en: """Enable or disable this bridge""" - zh: """启用/禁用桥接""" - } - label { - en: "Enable Or Disable Bridge" - zh: "启用/禁用桥接" - } - } - config_direction { - desc { - en: """The direction of this bridge, MUST be 'egress'""" - zh: """桥接的方向， 必须是 egress""" - } - label { - en: "Bridge Direction" - zh: "桥接方向" - } - } - desc_config { - desc { - en: """Configuration for an HStreamDB bridge.""" - zh: """HStreamDB 桥接配置""" - } - label: { - en: "HStreamDB Bridge Configuration" - zh: "HStreamDB 桥接配置" - } - } +local_topic.label: +"""Local Topic""" - desc_type { - desc { - en: """The Bridge Type""" - zh: """Bridge 类型""" - } - label { - en: "Bridge Type" - zh: "桥接类型" - } - } +payload.desc: +"""The payload to be forwarded to the HStreamDB. Placeholders supported.""" + +payload.label: +"""Payload""" - desc_name { - desc { - en: """Bridge name, used as a human-readable description of the bridge.""" - zh: """桥接名字，可读描述""" - } - label { - en: "Bridge Name" - zh: "桥接名字" - } - } - desc_connector { - desc { - en: """Generic configuration for the connector.""" - zh: """连接器的通用配置。""" - } - label: { - en: "Connector Generic Configuration" - zh: "连接器通用配置。" - } - } } diff --git a/rel/i18n/emqx_ee_bridge_influxdb.hocon b/rel/i18n/emqx_ee_bridge_influxdb.hocon index d73d62b14..c5cee2b66 100644 --- a/rel/i18n/emqx_ee_bridge_influxdb.hocon +++ b/rel/i18n/emqx_ee_bridge_influxdb.hocon @@ -1,22 +1,41 @@ emqx_ee_bridge_influxdb { - local_topic { - desc { - en: """The MQTT topic filter to be forwarded to the InfluxDB. All MQTT 'PUBLISH' messages with the topic + +config_enable.desc: +"""Enable or disable this bridge.""" + +config_enable.label: +"""Enable Or Disable Bridge""" + +desc_config.desc: +"""Configuration for an InfluxDB bridge.""" + +desc_config.label: +"""InfluxDB Bridge Configuration""" + +desc_name.desc: +"""Bridge name.""" + +desc_name.label: +"""Bridge Name""" + +desc_type.desc: +"""The Bridge Type.""" + +desc_type.label: +"""Bridge Type""" + +local_topic.desc: +"""The MQTT topic filter to be forwarded to the InfluxDB. All MQTT 'PUBLISH' messages with the topic matching the local_topic will be forwarded.
NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is configured, then both the data got from the rule and the MQTT messages that match local_topic will be forwarded.""" - zh: """发送到 'local_topic' 的消息都会转发到 InfluxDB。
-注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发到 InfluxDB。""" - } - label { - en: "Local Topic" - zh: "本地 Topic" - } - } - write_syntax { - desc { - en: """Conf of InfluxDB line protocol to write data points. It is a text-based format that provides the measurement, tag set, field set, and timestamp of a data point, and placeholder supported. + +local_topic.label: +"""Local Topic""" + +write_syntax.desc: +"""Conf of InfluxDB line protocol to write data points. It is a text-based format that provides the measurement, tag set, field set, and timestamp of a data point, and placeholder supported. See also [InfluxDB 2.3 Line Protocol](https://docs.influxdata.com/influxdb/v2.3/reference/syntax/line-protocol/) and [InfluxDB 1.8 Line Protocol](https://docs.influxdata.com/influxdb/v1.8/write_protocols/line_protocol_tutorial/)
TLDR:
@@ -24,62 +43,8 @@ TLDR:
[,=[,=]] =[,=] [] ``` Please note that a placeholder for an integer value must be annotated with a suffix `i`. For example `${payload.int_value}i`.""" - zh: """使用 InfluxDB API Line Protocol 写入 InfluxDB 的数据，支持占位符
-参考 [InfluxDB 2.3 Line Protocol](https://docs.influxdata.com/influxdb/v2.3/reference/syntax/line-protocol/) 及 -[InfluxDB 1.8 Line Protocol](https://docs.influxdata.com/influxdb/v1.8/write_protocols/line_protocol_tutorial/)
-TLDR:
-``` -[,=[,=]] =[,=] [] -``` -注意，整形数值占位符后需要添加一个字符 `i` 类型标识。例如 `${payload.int_value}i`""" - } - label { - en: "Write Syntax" - zh: "写语句" - } - } - config_enable { - desc { - en: """Enable or disable this bridge.""" - zh: """启用/禁用桥接。""" - } - label { - en: "Enable Or Disable Bridge" - zh: "启用/禁用桥接" - } - } - desc_config { - desc { - en: """Configuration for an InfluxDB bridge.""" - zh: """InfluxDB 桥接配置。""" - } - label: { - en: "InfluxDB Bridge Configuration" - zh: "InfluxDB 桥接配置" - } - } - - desc_type { - desc { - en: """The Bridge Type.""" - zh: """桥接类型。""" - } - label { - en: "Bridge Type" - zh: "桥接类型" - } - } - - desc_name { - desc { - en: """Bridge name.""" - zh: """桥接名称。""" - } - label { - en: "Bridge Name" - zh: "桥接名称" - } - } +write_syntax.label: +"""Write Syntax""" } diff --git a/rel/i18n/emqx_ee_bridge_kafka.hocon b/rel/i18n/emqx_ee_bridge_kafka.hocon deleted file mode 100644 index d1a017416..000000000 --- a/rel/i18n/emqx_ee_bridge_kafka.hocon +++ /dev/null @@ -1,667 +0,0 @@ -emqx_ee_bridge_kafka { - config_enable { - desc { - en: "Enable (true) or disable (false) this Kafka bridge." - zh: "启用（true）或停用该（false）Kafka 数据桥接。" - } - label { - en: "Enable or Disable" - zh: "启用或停用" - } - } - desc_config { - desc { - en: """Configuration for a Kafka bridge.""" - zh: """Kafka 桥接配置""" - } - label { - en: "Kafka Bridge Configuration" - zh: "Kafka 桥接配置" - } - } - desc_type { - desc { - en: """The Bridge Type""" - zh: """桥接类型""" - } - label { - en: "Bridge Type" - zh: "桥接类型" - } - } - desc_name { - desc { - en: """Bridge name, used as a human-readable description of the bridge.""" - zh: """桥接名字，可读描述""" - } - label { - en: "Bridge Name" - zh: "桥接名字" - } - } - kafka_producer { - desc { - en: "Kafka Producer configuration." - zh: "Kafka Producer 配置。" - } - label { - en: "Kafka Producer" - zh: "Kafka Producer" - } - } - producer_opts { - desc { - en: "Local MQTT data source and Kafka bridge configs." - zh: "本地 MQTT 数据源和 Kafka 桥接的配置。" - } - label { - en: "MQTT to Kafka" - zh: "MQTT 到 Kafka" - } - } - mqtt_topic { - desc { - en: "MQTT topic or topic as data source (bridge input). Should not configure this if the bridge is used as a rule action." - zh: "指定 MQTT 主题作为桥接的数据源。 若该桥接用于规则的动作，则必须将该配置项删除。" - } - label { - en: "Source MQTT Topic" - zh: "源 MQTT 主题" - } - } - producer_kafka_opts { - desc { - en: "Kafka producer configs." - zh: "Kafka 生产者参数。" - } - label { - en: "Kafka Producer" - zh: "生产者参数" - } - } - bootstrap_hosts { - desc { - en: "A comma separated list of Kafka host[:port] endpoints to bootstrap the client. Default port number is 9092." - zh: "用逗号分隔的 host[:port] 主机列表。默认端口号为 9092。" - } - label { - en: "Bootstrap Hosts" - zh: "主机列表" - } - } - connect_timeout { - desc { - en: "Maximum wait time for TCP connection establishment (including authentication time if enabled)." - zh: "建立 TCP 连接时的最大等待时长（若启用认证，这个等待时长也包含完成认证所需时间）。" - } - label { - en: "Connect Timeout" - zh: "连接超时" - } - } - min_metadata_refresh_interval { - desc { - en: "Minimum time interval the client has to wait before refreshing Kafka broker and topic metadata. " - "Setting too small value may add extra load on Kafka." - zh: "刷新 Kafka broker 和 Kafka 主题元数据段最短时间间隔。设置太小可能会增加 Kafka 压力。" - } - label { - en: "Min Metadata Refresh Interval" - zh: "元数据刷新最小间隔" - } - } - metadata_request_timeout { - desc { - en: "Maximum wait time when fetching metadata from Kafka." - zh: "刷新元数据时最大等待时长。" - } - label { - en: "Metadata Request Timeout" - zh: "元数据请求超时" - } - } - authentication { - desc { - en: "Authentication configs." - zh: "认证参数。" - } - label { - en: "Authentication" - zh: "认证" - } - } - socket_opts { - desc { - en: "Extra socket options." - zh: "更多 Socket 参数设置。" - } - label { - en: "Socket Options" - zh: "Socket 参数" - } - } - auth_sasl_mechanism { - desc { - en: "SASL authentication mechanism." - zh: "SASL 认证方法名称。" - } - label { - en: "Mechanism" - zh: "认证方法" - } - } - auth_sasl_username { - desc { - en: "SASL authentication username." - zh: "SASL 认证的用户名。" - } - label { - en: "Username" - zh: "用户名" - } - } - auth_sasl_password { - desc { - en: "SASL authentication password." - zh: "SASL 认证的密码。" - } - label { - en: "Password" - zh: "密码" - } - } - auth_kerberos_principal { - desc { - en: "SASL GSSAPI authentication Kerberos principal. " - "For example client_name@MY.KERBEROS.REALM.MYDOMAIN.COM, " - "NOTE: The realm in use has to be configured in /etc/krb5.conf in EMQX nodes." - zh: "SASL GSSAPI 认证方法的 Kerberos principal，" - "例如 client_name@MY.KERBEROS.REALM.MYDOMAIN.COM" - "注意：这里使用的 realm 需要配置在 EMQX 服务器的 /etc/krb5.conf 中" - } - label { - en: "Kerberos Principal" - zh: "Kerberos Principal" - } - } - auth_kerberos_keytab_file { - desc { - en: "SASL GSSAPI authentication Kerberos keytab file path. " - "NOTE: This file has to be placed in EMQX nodes, and the EMQX service runner user requires read permission." - zh: "SASL GSSAPI 认证方法的 Kerberos keytab 文件。" - "注意：该文件需要上传到 EMQX 服务器中，且运行 EMQX 服务的系统账户需要有读取权限。" - } - label { - en: "Kerberos keytab file" - zh: "Kerberos keytab 文件" - } - } - socket_send_buffer { - desc { - en: "Fine tune the socket send buffer. The default value is tuned for high throughput." - zh: "TCP socket 的发送缓存调优。默认值是针对高吞吐量的一个推荐值。" - } - label { - en: "Socket Send Buffer Size" - zh: "Socket 发送缓存大小" - } - } - socket_receive_buffer { - desc { - en: "Fine tune the socket receive buffer. The default value is tuned for high throughput." - zh: "TCP socket 的收包缓存调优。默认值是针对高吞吐量的一个推荐值。" - } - label { - en: "Socket Receive Buffer Size" - zh: "Socket 收包缓存大小" - } - } - # hidden - socket_nodelay { - desc { - en: "When set to 'true', TCP buffer is sent as soon as possible. " - "Otherwise, the OS kernel may buffer small TCP packets for a while (40 ms by default)." - zh: "设置‘true’让系统内核立即发送。否则当需要发送的内容很少时，可能会有一定延迟（默认 40 毫秒）。" - } - label { - en: "No Delay" - zh: "是否关闭延迟发送" - } - } - kafka_topic { - desc { - en: "Kafka topic name" - zh: "Kafka 主题名称" - } - label { - en: "Kafka Topic Name" - zh: "Kafka 主题名称" - } - } - kafka_message { - desc { - en: "Template to render a Kafka message." - zh: "用于生成 Kafka 消息的模版。" - } - label { - en: "Kafka Message Template" - zh: "Kafka 消息模版" - } - } - kafka_message_key { - desc { - en: "Template to render Kafka message key. " - "If the template is rendered into a NULL value (i.e. there is no such data field in Rule Engine context) " - "then Kafka's NULL (but not empty string) is used." - zh: "生成 Kafka 消息 Key 的模版。如果模版生成后为空值，" - "则会使用 Kafka 的 NULL ，而非空字符串。" - } - label { - en: "Message Key" - zh: "消息的 Key" - } - } - kafka_message_value { - desc { - en: "Template to render Kafka message value. " - "If the template is rendered " - "into a NULL value (i.e. there is no such data field in Rule Engine context) " - "then Kafka's NULL (but not empty string) is used." - zh: "生成 Kafka 消息 Value 的模版。如果模版生成后为空值，" - "则会使用 Kafka 的 NULL，而非空字符串。" - } - label { - en: "Message Value" - zh: "消息的 Value" - } - } - kafka_message_timestamp { - desc { - en: "Which timestamp to use. " - "The timestamp is expected to be a millisecond precision Unix epoch " - "which can be in string format, e.g. 1661326462115 or " - "'1661326462115'. " - "When the desired data field for this template is not found, " - "or if the found data is not a valid integer, " - "the current system timestamp will be used." - zh: "生成 Kafka 消息时间戳的模版。" - "该时间必需是一个整型数值（可以是字符串格式）例如 1661326462115 " - "或 '1661326462115'。" - "当所需的输入字段不存在，或不是一个整型时，" - "则会使用当前系统时间。" - } - label { - en: "Message Timestamp" - zh: "消息的时间戳" - } - } - max_batch_bytes { - desc { - en: "Maximum bytes to collect in a Kafka message batch. " - "Most of the Kafka brokers default to a limit of 1 MB batch size. " - "EMQX's default value is less than 1 MB in order to compensate " - "Kafka message encoding overheads (especially when each individual message is very small). " - "When a single message is over the limit, it is still sent (as a single element batch)." - zh: "最大消息批量字节数。" - "大多数 Kafka 环境的默认最低值是 1 MB，EMQX 的默认值比 1 MB 更小是因为需要" - "补偿 Kafka 消息编码所需要的额外字节（尤其是当每条消息都很小的情况下）。" - "当单个消息的大小超过该限制时，它仍然会被发送，（相当于该批量中只有单个消息）。" - } - label { - en: "Max Batch Bytes" - zh: "最大批量字节数" - } - } - compression { - desc { - en: "Compression method." - zh: "压缩方法。" - } - label { - en: "Compression" - zh: "压缩" - } - } - partition_strategy { - desc { - en: "Partition strategy is to tell the producer how to dispatch messages to Kafka partitions.\n\n" - "random: Randomly pick a partition for each message\n" - "key_dispatch: Hash Kafka message key to a partition number" - zh: "设置消息发布时应该如何选择 Kafka 分区。\n\n" - "random: 为每个消息随机选择一个分区。\n" - "key_dispatch: Hash Kafka message key to a partition number" - } - label { - en: "Partition Strategy" - zh: "分区选择策略" - } - } - required_acks { - desc { - en: "Required acknowledgements for Kafka partition leader to wait for its followers " - "before it sends back the acknowledgement to EMQX Kafka producer\n\n" - "all_isr: Require all in-sync replicas to acknowledge.\n" - "leader_only: Require only the partition-leader's acknowledgement.\n" - "none: No need for Kafka to acknowledge at all." - zh: "设置 Kafka leader 在返回给 EMQX 确认之前需要等待多少个 follower 的确认。\n\n" - "all_isr: 需要所有的在线复制者都确认。\n" - "leader_only: 仅需要分区 leader 确认。\n" - "none: 无需 Kafka 回复任何确认。" - } - label { - en: "Required Acks" - zh: "Kafka 确认数量" - } - } - partition_count_refresh_interval { - desc { - en: "The time interval for Kafka producer to discover increased number of partitions.\n" - "After the number of partitions is increased in Kafka, EMQX will start taking the \n" - "discovered partitions into account when dispatching messages per partition_strategy." - zh: "配置 Kafka 刷新分区数量的时间间隔。\n" - "EMQX 发现 Kafka 分区数量增加后，会开始按 partition_strategy 配置，把消息发送到新的分区中。" - } - label { - en: "Partition Count Refresh Interval" - zh: "分区数量刷新间隔" - } - } - max_inflight { - desc { - en: "Maximum number of batches allowed for Kafka producer (per-partition) to send before receiving acknowledgement from Kafka. " - "Greater value typically means better throughput. However, there can be a risk of message reordering when this " - "value is greater than 1." - zh: "设置 Kafka 生产者（每个分区一个）在收到 Kafka 的确认前最多发送多少个请求（批量）。" - "调大这个值通常可以增加吞吐量，但是，当该值设置大于 1 时存在消息乱序的风险。" - } - label { - en: "Max Inflight" - zh: "飞行窗口" - } - } - producer_buffer { - desc { - en: "Configure producer message buffer.\n\n" - "Tell Kafka producer how to buffer messages when EMQX has more messages to send than " - "Kafka can keep up, or when Kafka is down." - zh: "配置消息缓存的相关参数。\n\n" - "当 EMQX 需要发送的消息超过 Kafka 处理能力，或者当 Kafka 临时下线时，EMQX 内部会将消息缓存起来。" - } - label { - en: "Message Buffer" - zh: "消息缓存" - } - } - buffer_mode { - desc { - en: "Message buffer mode.\n\n" - "memory: Buffer all messages in memory. The messages will be lost in case of EMQX node restart\n" - "disk: Buffer all messages on disk. The messages on disk are able to survive EMQX node restart.\n" - "hybrid: Buffer message in memory first, when up to certain limit " - "(see segment_bytes config for more information), then start offloading " - "messages to disk, Like memory mode, the messages will be lost in case of " - "EMQX node restart." - zh: "消息缓存模式。\n" - "memory: 所有的消息都缓存在内存里。如果 EMQX 服务重启，缓存的消息会丢失。\n" - "disk: 缓存到磁盘上。EMQX 重启后会继续发送重启前未发送完成的消息。\n" - "hybrid: 先将消息缓存在内存中，当内存中的消息堆积超过一定限制" - "（配置项 segment_bytes 描述了该限制）后，后续的消息会缓存到磁盘上。" - "与 memory 模式一样，如果 EMQX 服务重启，缓存的消息会丢失。" - } - label { - en: "Buffer Mode" - zh: "缓存模式" - } - } - buffer_per_partition_limit { - desc { - en: "Number of bytes allowed to buffer for each Kafka partition. " - "When this limit is exceeded, old messages will be dropped in a trade for credits " - "for new messages to be buffered." - zh: "为每个 Kafka 分区设置的最大缓存字节数。当超过这个上限之后，老的消息会被丢弃，" - "为新的消息腾出空间。" - } - label { - en: "Per-partition Buffer Limit" - zh: "Kafka 分区缓存上限" - } - } - buffer_segment_bytes { - desc { - en: "Applicable when buffer mode is set to disk or hybrid.\n" - "This value is to specify the size of each on-disk buffer file." - zh: "当缓存模式是 disk 或 hybrid 时适用。" - "该配置用于指定缓存到磁盘上的文件的大小。" - } - label { - en: "Segment File Bytes" - zh: "缓存文件大小" - } - } - buffer_memory_overload_protection { - desc { - en: "Applicable when buffer mode is set to memory\n" - "EMQX will drop old buffered messages under high memory pressure. " - "The high memory threshold is defined in config sysmon.os.sysmem_high_watermark. " - "NOTE: This config only works on Linux." - zh: "缓存模式是 memory 或 hybrid 时适用。" - "当系统处于高内存压力时，从队列中丢弃旧的消息以减缓内存增长。" - "内存压力值由配置项 sysmon.os.sysmem_high_watermark 决定。" - "注意，该配置仅在 Linux 系统中有效。" - } - label { - en: "Memory Overload Protection" - zh: "内存过载保护" - } - } - auth_username_password { - desc { - en: "Username/password based authentication." - zh: "基于用户名密码的认证。" - } - label { - en: "Username/password Auth" - zh: "用户名密码认证" - } - } - auth_gssapi_kerberos { - desc { - en: "Use GSSAPI/Kerberos authentication." - zh: "使用 GSSAPI/Kerberos 认证。" - } - label { - en: "GSSAPI/Kerberos" - zh: "GSSAPI/Kerberos" - } - } - - kafka_consumer { - desc { - en: "Kafka Consumer configuration." - zh: "Kafka 消费者配置。" - } - label { - en: "Kafka Consumer" - zh: "Kafka 消费者" - } - } - consumer_opts { - desc { - en: "Local MQTT publish and Kafka consumer configs." - zh: "本地 MQTT 转发 和 Kafka 消费者配置。" - } - label { - en: "MQTT to Kafka" - zh: "MQTT 到 Kafka" - } - } - consumer_kafka_opts { - desc { - en: "Kafka consumer configs." - zh: "Kafka消费者配置。" - } - label { - en: "Kafka Consumer" - zh: "Kafka 消费者" - } - } - consumer_mqtt_opts { - desc { - en: "Local MQTT message publish." - zh: "本地 MQTT 消息转发。" - } - label { - en: "MQTT publish" - zh: "MQTT 转发" - } - } - consumer_mqtt_topic { - desc { - en: "Local topic to which consumed Kafka messages should be published to." - zh: "设置 Kafka 消息向哪个本地 MQTT 主题转发消息。" - } - label { - en: "MQTT Topic" - zh: "MQTT主题" - } - } - consumer_mqtt_qos { - desc { - en: "MQTT QoS used to publish messages consumed from Kafka." - zh: "转发 MQTT 消息时使用的 QoS。" - } - label { - en: "QoS" - zh: "QoS" - } - } - consumer_mqtt_payload { - desc { - en: "The template for transforming the incoming Kafka message." - " By default, it will use JSON format to serialize" - " inputs from the Kafka message. Such fields are:\n" - "headers: an object containing string key-value pairs.\n" - "key: Kafka message key (uses the chosen key encoding).\n" - "offset: offset for the message.\n" - "topic: Kafka topic.\n" - "ts: message timestamp.\n" - "ts_type: message timestamp type, which is one of" - " create, append or undefined.\n" - "value: Kafka message value (uses the chosen value encoding)." - zh: "用于转换收到的 Kafka 消息的模板。 " - "默认情况下，它将使用 JSON 格式来序列化来自 Kafka 的所有字段。 " - "这些字段包括：" - "headers：一个包含字符串键值对的 JSON 对象。\n" - "key：Kafka 消息的键（使用选择的编码方式编码）。\n" - "offset：消息的偏移量。\n" - "topic：Kafka 主题。\n" - "ts: 消息的时间戳。\n" - "ts_type：消息的时间戳类型，值可能是：" - " create， append 或 undefined。\n" - "value: Kafka 消息值（使用选择的编码方式编码）。" - - } - label { - en: "MQTT Payload Template" - zh: "MQTT Payload Template" - } - } - consumer_kafka_topic { - desc { - en: "Kafka topic to consume from." - zh: "指定从哪个 Kafka 主题消费消息。" - } - label { - en: "Kafka Topic" - zh: "Kafka 主题" - } - } - consumer_max_batch_bytes { - desc { - en: "Set how many bytes to pull from Kafka in each fetch request. " - "Please note that if the configured value is smaller than the message size in Kafka, it may negatively impact the fetch performance." - zh: "设置每次从 Kafka 拉取数据的字节数。" - "如该配置小于 Kafka 消息的大小，可能会影响消费性能。" - } - label { - en: "Fetch Bytes" - zh: "拉取字节数" - } - } - # hidden - consumer_max_rejoin_attempts { - desc { - en: "Maximum number of times allowed for a member to re-join the group. If the consumer group can not reach balance after this configured number of attempts, the consumer group member will restart after a delay." - zh: "消费组成员允许重新加入小组的最大次数。如超过该配置次数后仍未能成功加入消费组，则会在等待一段时间后重试。" - } - label { - en: "Max Rejoin Attempts" - zh: "最大的重新加入尝试" - } - } - consumer_offset_reset_policy { - desc { - en: "Defines from which offset a consumer should start fetching when there " - "is no commit history or when the commit history becomes invalid." - zh: "如不存在偏移量历史记录或历史记录失效，消费者应使用哪个偏移量开始消费。" - } - label { - en: "Offset Reset Policy" - zh: "偏移重置策略" - } - } - consumer_offset_commit_interval_seconds { - desc { - en: "Defines the time interval between two offset commit requests sent for each consumer group." - zh: "指定 Kafka 消费组偏移量提交的时间间隔。" - } - label { - en: "Offset Commit Interval" - zh: "偏移提交间隔" - } - } - consumer_topic_mapping { - desc { - en: "Defines the mapping between Kafka topics and MQTT topics. Must contain at least one item." - zh: "指定 Kafka 主题和 MQTT 主题之间的映射关系。 应至少包含一项。" - } - label { - en: "Topic Mapping" - zh: "主题映射关系" - } - } - consumer_key_encoding_mode { - desc { - en: "Defines how the key from the Kafka message is" - " encoded before being forwarded via MQTT.\n" - "none Uses the key from the Kafka message unchanged." - " Note: in this case, the key must be a valid UTF-8 string.\n" - "base64 Uses base-64 encoding on the received key." - zh: "通过 MQTT 转发之前，如何处理 Kafka 消息的 Key。" - "none 使用 Kafka 消息中的 Key 原始值，不进行编码。" - " 注意：在这种情况下，Key 必须是一个有效的 UTF-8 字符串。\n" - "base64 对收到的密钥或值使用 base-64 编码。" - } - label { - en: "Key Encoding Mode" - zh: "Key 编码模式" - } - } - consumer_value_encoding_mode { - desc { - en: "Defines how the value from the Kafka message is" - " encoded before being forwarded via MQTT.\n" - "none Uses the value from the Kafka message unchanged." - " Note: in this case, the value must be a valid UTF-8 string.\n" - "base64 Uses base-64 encoding on the received value." - zh: "通过 MQTT 转发之前，如何处理 Kafka 消息的 Value。" - "none 使用 Kafka 消息中的 Value 原始值，不进行编码。" - " 注意：在这种情况下，Value 必须是一个有效的 UTF-8 字符串。\n" - "base64 对收到的 Value 使用 base-64 编码。" - } - label { - en: "Value Encoding Mode" - zh: "Value 编码模式" - } - } -} diff --git a/rel/i18n/emqx_ee_bridge_mongodb.hocon b/rel/i18n/emqx_ee_bridge_mongodb.hocon index 053c932f7..fab371824 100644 --- a/rel/i18n/emqx_ee_bridge_mongodb.hocon +++ b/rel/i18n/emqx_ee_bridge_mongodb.hocon @@ -1,100 +1,57 @@ emqx_ee_bridge_mongodb { - desc_config { - desc { - en: "Configuration for MongoDB Bridge" - zh: "为MongoDB桥配置" - } - label { - en: "MongoDB Bridge Configuration" - zh: "MongoDB桥配置" - } - } - enable { - desc { - en: "Enable or disable this MongoDB Bridge" - zh: "启用或停用该MongoDB桥" - } - label { - en: "Enable or disable" - zh: "启用或禁用" - } - } +collection.desc: +"""The collection where data will be stored into""" - collection { - desc { - en: "The collection where data will be stored into" - zh: "数据将被存储到的集合" - } - label { - en: "Collection to be used" - zh: "将要使用的集合（Collection）" - } - } +collection.label: +"""Collection to be used""" - mongodb_rs_conf { - desc { - en: "MongoDB (Replica Set) configuration" - zh: "MongoDB（Replica Set）配置" - } - label { - en: "MongoDB (Replica Set) Configuration" - zh: "MongoDB（Replica Set）配置" - } - } +desc_config.desc: +"""Configuration for MongoDB Bridge""" - mongodb_sharded_conf { - desc { - en: "MongoDB (Sharded) configuration" - zh: "MongoDB (Sharded)配置" - } - label { - en: "MongoDB (Sharded) Configuration" - zh: "MongoDB (Sharded)配置" - } - } +desc_config.label: +"""MongoDB Bridge Configuration""" - mongodb_single_conf { - desc { - en: "MongoDB (Standalone) configuration" - zh: "MongoDB（独立）配置" - } - label { - en: "MongoDB (Standalone) Configuration" - zh: "MongoDB（独立）配置" - } - } +desc_name.desc: +"""Bridge name.""" - desc_type { - desc { - en: """The Bridge Type.""" - zh: """桥接类型。""" - } - label { - en: "Bridge Type" - zh: "桥接类型" - } - } +desc_name.label: +"""Bridge Name""" - desc_name { - desc { - en: """Bridge name.""" - zh: """桥接名称。""" - } - label { - en: "Bridge Name" - zh: "桥接名称" - } - } +desc_type.desc: +"""The Bridge Type.""" + +desc_type.label: +"""Bridge Type""" + +enable.desc: +"""Enable or disable this MongoDB Bridge""" + +enable.label: +"""Enable or disable""" + +mongodb_rs_conf.desc: +"""MongoDB (Replica Set) configuration""" + +mongodb_rs_conf.label: +"""MongoDB (Replica Set) Configuration""" + +mongodb_sharded_conf.desc: +"""MongoDB (Sharded) configuration""" + +mongodb_sharded_conf.label: +"""MongoDB (Sharded) Configuration""" + +mongodb_single_conf.desc: +"""MongoDB (Standalone) configuration""" + +mongodb_single_conf.label: +"""MongoDB (Standalone) Configuration""" + +payload_template.desc: +"""The template for formatting the outgoing messages. If undefined, rule engine will use JSON format to serialize all visible inputs, such as clientid, topic, payload etc.""" + +payload_template.label: +"""Payload template""" - payload_template { - desc { - en: "The template for formatting the outgoing messages. If undefined, rule engine will use JSON format to serialize all visible inputs, such as clientid, topic, payload etc." - zh: "用于格式化写入 MongoDB 的消息模板。 如果未定义，规则引擎会使用 JSON 格式序列化所有的可见输入，例如 clientid, topic, payload 等。" - } - label: { - en: "Payload template" - zh: "有效载荷模板" - } - } } diff --git a/rel/i18n/emqx_ee_bridge_mysql.hocon b/rel/i18n/emqx_ee_bridge_mysql.hocon index 345fd9cba..bd627f726 100644 --- a/rel/i18n/emqx_ee_bridge_mysql.hocon +++ b/rel/i18n/emqx_ee_bridge_mysql.hocon @@ -1,72 +1,43 @@ emqx_ee_bridge_mysql { - local_topic { - desc { - en: """The MQTT topic filter to be forwarded to MySQL. All MQTT 'PUBLISH' messages with the topic +config_enable.desc: +"""Enable or disable this bridge""" + +config_enable.label: +"""Enable Or Disable Bridge""" + +desc_config.desc: +"""Configuration for an HStreamDB bridge.""" + +desc_config.label: +"""HStreamDB Bridge Configuration""" + +desc_name.desc: +"""Bridge name, used as a human-readable description of the bridge.""" + +desc_name.label: +"""Bridge Name""" + +desc_type.desc: +"""The Bridge Type""" + +desc_type.label: +"""Bridge Type""" + +local_topic.desc: +"""The MQTT topic filter to be forwarded to MySQL. All MQTT 'PUBLISH' messages with the topic matching the local_topic will be forwarded.
NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is configured, then both the data got from the rule and the MQTT messages that match local_topic will be forwarded.""" - zh: """发送到 'local_topic' 的消息都会转发到 MySQL。
-注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。""" - } - label { - en: "Local Topic" - zh: "本地 Topic" - } - } - sql_template { - desc { - en: """SQL Template""" - zh: """SQL 模板""" - } - label { - en: "SQL Template" - zh: "SQL 模板" - } - } - config_enable { - desc { - en: """Enable or disable this bridge""" - zh: """启用/禁用桥接""" - } - label { - en: "Enable Or Disable Bridge" - zh: "启用/禁用桥接" - } - } +local_topic.label: +"""Local Topic""" - desc_config { - desc { - en: """Configuration for an HStreamDB bridge.""" - zh: """HStreamDB 桥接配置""" - } - label: { - en: "HStreamDB Bridge Configuration" - zh: "HStreamDB 桥接配置" - } - } +sql_template.desc: +"""SQL Template""" - desc_type { - desc { - en: """The Bridge Type""" - zh: """Bridge 类型""" - } - label { - en: "Bridge Type" - zh: "桥接类型" - } - } +sql_template.label: +"""SQL Template""" - desc_name { - desc { - en: """Bridge name, used as a human-readable description of the bridge.""" - zh: """桥接名字，可读描述""" - } - label { - en: "Bridge Name" - zh: "桥接名字" - } - } } diff --git a/rel/i18n/emqx_ee_bridge_pgsql.hocon b/rel/i18n/emqx_ee_bridge_pgsql.hocon index 81d2330c5..94c263a56 100644 --- a/rel/i18n/emqx_ee_bridge_pgsql.hocon +++ b/rel/i18n/emqx_ee_bridge_pgsql.hocon @@ -1,72 +1,43 @@ emqx_ee_bridge_pgsql { - local_topic { - desc { - en: """The MQTT topic filter to be forwarded to PostgreSQL. All MQTT 'PUBLISH' messages with the topic +config_enable.desc: +"""Enable or disable this bridge""" + +config_enable.label: +"""Enable Or Disable Bridge""" + +desc_config.desc: +"""Configuration for a PostgreSQL bridge.""" + +desc_config.label: +"""PostgreSQL Bridge Configuration""" + +desc_name.desc: +"""Bridge name.""" + +desc_name.label: +"""Bridge Name""" + +desc_type.desc: +"""The Bridge Type""" + +desc_type.label: +"""Bridge Type""" + +local_topic.desc: +"""The MQTT topic filter to be forwarded to PostgreSQL. All MQTT 'PUBLISH' messages with the topic matching the local_topic will be forwarded.
NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is configured, then both the data got from the rule and the MQTT messages that match local_topic will be forwarded.""" - zh: """发送到 'local_topic' 的消息都会转发到 PostgreSQL。
-注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。""" - } - label { - en: "Local Topic" - zh: "本地 Topic" - } - } - sql_template { - desc { - en: """SQL Template""" - zh: """SQL 模板""" - } - label { - en: "SQL Template" - zh: "SQL 模板" - } - } - config_enable { - desc { - en: """Enable or disable this bridge""" - zh: """启用/禁用桥接""" - } - label { - en: "Enable Or Disable Bridge" - zh: "启用/禁用桥接" - } - } +local_topic.label: +"""Local Topic""" - desc_config { - desc { - en: """Configuration for an PostgreSQL bridge.""" - zh: """PostgreSQL 桥接配置""" - } - label: { - en: "PostgreSQL Bridge Configuration" - zh: "PostgreSQL 桥接配置" - } - } +sql_template.desc: +"""SQL Template""" - desc_type { - desc { - en: """The Bridge Type""" - zh: """Bridge 类型""" - } - label { - en: "Bridge Type" - zh: "桥接类型" - } - } +sql_template.label: +"""SQL Template""" - desc_name { - desc { - en: """Bridge name.""" - zh: """桥接名字""" - } - label { - en: "Bridge Name" - zh: "桥接名字" - } - } } diff --git a/rel/i18n/emqx_ee_bridge_redis.hocon b/rel/i18n/emqx_ee_bridge_redis.hocon index 4de42b4e3..78db30196 100644 --- a/rel/i18n/emqx_ee_bridge_redis.hocon +++ b/rel/i18n/emqx_ee_bridge_redis.hocon @@ -1,74 +1,45 @@ emqx_ee_bridge_redis { - local_topic { - desc { - en: """The MQTT topic filter to be forwarded to Redis. All MQTT 'PUBLISH' messages with the topic + +command_template.desc: +"""Redis command template used to export messages. Each list element stands for a command name or its argument. +For example, to push payloads in a Redis list by key `msgs`, the elements should be the following: +`rpush`, `msgs`, `${payload}`.""" + +command_template.label: +"""Redis Command Template""" + +config_enable.desc: +"""Enable or disable this bridge""" + +config_enable.label: +"""Enable Or Disable Bridge""" + +desc_config.desc: +"""Configuration for a Redis bridge.""" + +desc_config.label: +"""Redis Bridge Configuration""" + +desc_name.desc: +"""Bridge name, used as a human-readable description of the bridge.""" + +desc_name.label: +"""Bridge Name""" + +desc_type.desc: +"""The Bridge Type""" + +desc_type.label: +"""Bridge Type""" + +local_topic.desc: +"""The MQTT topic filter to be forwarded to Redis. All MQTT 'PUBLISH' messages with the topic matching the local_topic will be forwarded.
NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is configured, then both the data got from the rule and the MQTT messages that match local_topic will be forwarded.""" - zh: """发送到 'local_topic' 的消息都会转发到 Redis。
-注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发到 Redis。""" - } - label { - en: "Local Topic" - zh: "本地 Topic" - } - } - command_template { - desc { - en: """Redis command template used to export messages. Each list element stands for a command name or its argument. -For example, to push payloads in a Redis list by key `msgs`, the elements should be the following: -`rpush`, `msgs`, `${payload}`.""" - zh: """用于推送数据的 Redis 命令模板。 每个列表元素代表一个命令名称或其参数。 -例如，要通过键值 `msgs` 将消息体推送到 Redis 列表中，数组元素应该是： `rpush`, `msgs`, `${payload}`。""" - } - label { - en: "Redis Command Template" - zh: "Redis Command 模板" - } - } - config_enable { - desc { - en: """Enable or disable this bridge""" - zh: """启用/禁用桥接""" - } - label { - en: "Enable Or Disable Bridge" - zh: "启用/禁用桥接" - } - } +local_topic.label: +"""Local Topic""" - desc_config { - desc { - en: """Configuration for a Redis bridge.""" - zh: """Resis 桥接配置""" - } - label: { - en: "Redis Bridge Configuration" - zh: "Redis 桥接配置" - } - } - - desc_type { - desc { - en: """The Bridge Type""" - zh: """Bridge 类型""" - } - label { - en: "Bridge Type" - zh: "桥接类型" - } - } - - desc_name { - desc { - en: """Bridge name, used as a human-readable description of the bridge.""" - zh: """桥接名字，可读描述""" - } - label { - en: "Bridge Name" - zh: "桥接名字" - } - } } diff --git a/rel/i18n/emqx_ee_bridge_rocketmq.hocon b/rel/i18n/emqx_ee_bridge_rocketmq.hocon index 2e33e6c07..a545a7fca 100644 --- a/rel/i18n/emqx_ee_bridge_rocketmq.hocon +++ b/rel/i18n/emqx_ee_bridge_rocketmq.hocon @@ -1,70 +1,41 @@ emqx_ee_bridge_rocketmq { - local_topic { - desc { - en: """The MQTT topic filter to be forwarded to RocketMQ. All MQTT `PUBLISH` messages with the topic +config_enable.desc: +"""Enable or disable this bridge""" + +config_enable.label: +"""Enable Or Disable Bridge""" + +desc_config.desc: +"""Configuration for a RocketMQ bridge.""" + +desc_config.label: +"""RocketMQ Bridge Configuration""" + +desc_name.desc: +"""Bridge name.""" + +desc_name.label: +"""Bridge Name""" + +desc_type.desc: +"""The Bridge Type""" + +desc_type.label: +"""Bridge Type""" + +local_topic.desc: +"""The MQTT topic filter to be forwarded to RocketMQ. All MQTT `PUBLISH` messages with the topic matching the `local_topic` will be forwarded.
NOTE: if the bridge is used as a rule action, `local_topic` should be left empty otherwise the messages will be duplicated.""" - zh: """发送到 'local_topic' 的消息都会转发到 RocketMQ。
-注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。""" - } - label { - en: "Local Topic" - zh: "本地 Topic" - } - } - template { - desc { - en: """Template, the default value is empty. When this value is empty the whole message will be stored in the RocketMQ""" - zh: """模板, 默认为空，为空时将会将整个消息转发给 RocketMQ""" - } - label { - en: "Template" - zh: "模板" - } - } - config_enable { - desc { - en: """Enable or disable this bridge""" - zh: """启用/禁用桥接""" - } - label { - en: "Enable Or Disable Bridge" - zh: "启用/禁用桥接" - } - } +local_topic.label: +"""Local Topic""" - desc_config { - desc { - en: """Configuration for a RocketMQ bridge.""" - zh: """RocketMQ 桥接配置""" - } - label: { - en: "RocketMQ Bridge Configuration" - zh: "RocketMQ 桥接配置" - } - } +template.desc: +"""Template, the default value is empty. When this value is empty the whole message will be stored in the RocketMQ""" - desc_type { - desc { - en: """The Bridge Type""" - zh: """Bridge 类型""" - } - label { - en: "Bridge Type" - zh: "桥接类型" - } - } +template.label: +"""Template""" - desc_name { - desc { - en: """Bridge name.""" - zh: """桥接名字""" - } - label { - en: "Bridge Name" - zh: "桥接名字" - } - } } diff --git a/rel/i18n/emqx_ee_bridge_sqlserver.hocon b/rel/i18n/emqx_ee_bridge_sqlserver.hocon new file mode 100644 index 000000000..396126622 --- /dev/null +++ b/rel/i18n/emqx_ee_bridge_sqlserver.hocon @@ -0,0 +1,49 @@ +emqx_ee_bridge_sqlserver { + +config_enable.desc: +"""Enable or disable this bridge""" + +config_enable.label: +"""Enable Or Disable Bridge""" + +desc_config.desc: +"""Configuration for a Microsoft SQL Server bridge.""" + +desc_config.label: +"""Microsoft SQL Server Bridge Configuration""" + +desc_name.desc: +"""Bridge name.""" + +desc_name.label: +"""Bridge Name""" + +desc_type.desc: +"""The Bridge Type""" + +desc_type.label: +"""Bridge Type""" + +driver.desc: +"""SQL Server Driver Name""" + +driver.label: +"""SQL Server Driver Name""" + +local_topic.desc: +"""The MQTT topic filter to be forwarded to Microsoft SQL Server. All MQTT 'PUBLISH' messages with the topic +matching the local_topic will be forwarded.
+NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is +configured, then both the data got from the rule and the MQTT messages that match local_topic +will be forwarded.""" + +local_topic.label: +"""Local Topic""" + +sql_template.desc: +"""SQL Template""" + +sql_template.label: +"""SQL Template""" + +} diff --git a/rel/i18n/emqx_ee_bridge_tdengine.hocon b/rel/i18n/emqx_ee_bridge_tdengine.hocon index 21fc013df..e6ece89c8 100644 --- a/rel/i18n/emqx_ee_bridge_tdengine.hocon +++ b/rel/i18n/emqx_ee_bridge_tdengine.hocon @@ -1,72 +1,43 @@ emqx_ee_bridge_tdengine { - local_topic { - desc { - en: """The MQTT topic filter to be forwarded to TDengine. All MQTT 'PUBLISH' messages with the topic +config_enable.desc: +"""Enable or disable this bridge""" + +config_enable.label: +"""Enable Or Disable Bridge""" + +desc_config.desc: +"""Configuration for a TDengine bridge.""" + +desc_config.label: +"""TDengine Bridge Configuration""" + +desc_name.desc: +"""Bridge name.""" + +desc_name.label: +"""Bridge Name""" + +desc_type.desc: +"""The Bridge Type""" + +desc_type.label: +"""Bridge Type""" + +local_topic.desc: +"""The MQTT topic filter to be forwarded to TDengine. All MQTT 'PUBLISH' messages with the topic matching the local_topic will be forwarded.
NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is configured, then both the data got from the rule and the MQTT messages that match local_topic will be forwarded.""" - zh: """发送到 'local_topic' 的消息都会转发到 TDengine。
-注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。""" - } - label { - en: "Local Topic" - zh: "本地 Topic" - } - } - sql_template { - desc { - en: """SQL Template""" - zh: """SQL 模板""" - } - label { - en: "SQL Template" - zh: "SQL 模板" - } - } - config_enable { - desc { - en: """Enable or disable this bridge""" - zh: """启用/禁用桥接""" - } - label { - en: "Enable Or Disable Bridge" - zh: "启用/禁用桥接" - } - } +local_topic.label: +"""Local Topic""" - desc_config { - desc { - en: """Configuration for an TDengine bridge.""" - zh: """TDengine 桥接配置""" - } - label: { - en: "TDengine Bridge Configuration" - zh: "TDengine 桥接配置" - } - } +sql_template.desc: +"""SQL Template""" - desc_type { - desc { - en: """The Bridge Type""" - zh: """Bridge 类型""" - } - label { - en: "Bridge Type" - zh: "桥接类型" - } - } +sql_template.label: +"""SQL Template""" - desc_name { - desc { - en: """Bridge name.""" - zh: """桥接名字""" - } - label { - en: "Bridge Name" - zh: "桥接名字" - } - } } diff --git a/rel/i18n/emqx_ee_connector_cassa.hocon b/rel/i18n/emqx_ee_connector_cassa.hocon deleted file mode 100644 index ecf004722..000000000 --- a/rel/i18n/emqx_ee_connector_cassa.hocon +++ /dev/null @@ -1,28 +0,0 @@ -emqx_ee_connector_cassa { - - servers { - desc { - en: """The IPv4 or IPv6 address or the hostname to connect to.
-A host entry has the following form: `Host[:Port][,Host2:Port]`.
-The Cassandra default port 9042 is used if `[:Port]` is not specified.""" - zh: """将要连接的 IPv4 或 IPv6 地址，或者主机名。
-主机名具有以下形式：`Host[:Port][,Host2:Port]`。
-如果未指定 `[:Port]`，则使用 Cassandra 默认端口 9042。""" - } - label: { - en: "Servers" - zh: "Servers" - } - } - - keyspace { - desc { - en: """Keyspace name to connect to.""" - zh: """要连接到的 Keyspace 名称。""" - } - label: { - en: "Keyspace" - zh: "Keyspace" - } - } -} diff --git a/rel/i18n/emqx_ee_connector_clickhouse.hocon b/rel/i18n/emqx_ee_connector_clickhouse.hocon index 4d30e1715..cebba5aef 100644 --- a/rel/i18n/emqx_ee_connector_clickhouse.hocon +++ b/rel/i18n/emqx_ee_connector_clickhouse.hocon @@ -1,25 +1,15 @@ emqx_ee_connector_clickhouse { - base_url { - desc { - en: """The HTTP URL to the Clickhouse server that you want to connect to (for example http://myhostname:8123)""" - zh: """你想连接到的Clickhouse服务器的HTTP URL（例如http://myhostname:8123）。""" - } - label: { - en: "Server URL" - zh: "服务器 URL" - } - } +base_url.desc: +"""The HTTP URL to the Clickhouse server that you want to connect to (for example http://myhostname:8123)""" - connect_timeout { - desc { - en: "The timeout when connecting to the Clickhouse server." - zh: "连接HTTP服务器的超时时间。" - } - label: { - en: "Clickhouse Timeout" - zh: "连接超时" - } - } +base_url.label: +"""Server URL""" + +connect_timeout.desc: +"""The timeout when connecting to the Clickhouse server.""" + +connect_timeout.label: +"""Clickhouse Timeout""" } diff --git a/rel/i18n/emqx_ee_connector_dynamo.hocon b/rel/i18n/emqx_ee_connector_dynamo.hocon index 295929a72..29b6bf99e 100644 --- a/rel/i18n/emqx_ee_connector_dynamo.hocon +++ b/rel/i18n/emqx_ee_connector_dynamo.hocon @@ -1,14 +1,27 @@ emqx_ee_connector_dynamo { - url { - desc { - en: """The url of DynamoDB endpoint.""" - zh: """DynamoDB 的地址。""" - } - label: { - en: "DynamoDB Endpoint" - zh: "DynamoDB 地址" - } - } +aws_access_key_id.desc: +"""Access Key ID for connecting to DynamoDB.""" + +aws_access_key_id.label: +"""AWS Access Key ID""" + +aws_secret_access_key.desc: +"""AWS Secret Access Key for connecting to DynamoDB.""" + +aws_secret_access_key.label: +"""AWS Secret Access Key""" + +table.desc: +"""DynamoDB Table.""" + +table.label: +"""Table """ + +url.desc: +"""The url of DynamoDB endpoint.""" + +url.label: +"""DynamoDB Endpoint""" } diff --git a/rel/i18n/emqx_ee_connector_hstreamdb.hocon b/rel/i18n/emqx_ee_connector_hstreamdb.hocon index 0826c8f0c..f6838297f 100644 --- a/rel/i18n/emqx_ee_connector_hstreamdb.hocon +++ b/rel/i18n/emqx_ee_connector_hstreamdb.hocon @@ -1,74 +1,45 @@ emqx_ee_connector_hstreamdb { - config { - desc { - en: "HStreamDB connection config" - zh: "HStreamDB 连接配置。" - } - label: { - en: "Connection config" - zh: "连接配置" - } - } - type { - desc { - en: "The Connector Type." - zh: "连接器类型。" - } - label: { - en: "Connector Type" - zh: "连接器类型" - } - } +config.desc: +"""HStreamDB connection config""" + +config.label: +"""Connection config""" + +name.desc: +"""Connector name, used as a human-readable description of the connector.""" + +name.label: +"""Connector Name""" + +ordering_key.desc: +"""HStreamDB Ordering Key""" + +ordering_key.label: +"""HStreamDB Ordering Key""" + +pool_size.desc: +"""HStreamDB Pool Size""" + +pool_size.label: +"""HStreamDB Pool Size""" + +stream_name.desc: +"""HStreamDB Stream Name""" + +stream_name.label: +"""HStreamDB Stream Name""" + +type.desc: +"""The Connector Type.""" + +type.label: +"""Connector Type""" + +url.desc: +"""HStreamDB Server URL""" + +url.label: +"""HStreamDB Server URL""" - name { - desc { - en: "Connector name, used as a human-readable description of the connector." - zh: "连接器名称，人类可读的连接器描述。" - } - label: { - en: "Connector Name" - zh: "连接器名称" - } - } - url { - desc { - en: """HStreamDB Server URL""" - zh: """HStreamDB 服务器 URL""" - } - label { - en: """HStreamDB Server URL""" - zh: """HStreamDB 服务器 URL""" - } - } - stream_name { - desc { - en: """HStreamDB Stream Name""" - zh: """HStreamDB 流名称""" - } - label { - en: """HStreamDB Stream Name""" - zh: """HStreamDB 流名称""" - } - } - ordering_key { - desc { - en: """HStreamDB Ordering Key""" - zh: """HStreamDB 分区键""" - } - label { - en: """HStreamDB Ordering Key""" - zh: """HStreamDB 分区键""" - } - } - pool_size { - desc { - en: """HStreamDB Pool Size""" - zh: """HStreamDB 连接池大小""" - } - label { - en: """HStreamDB Pool Size""" - zh: """HStreamDB 连接池大小""" - } - } } diff --git a/rel/i18n/emqx_ee_connector_influxdb.hocon b/rel/i18n/emqx_ee_connector_influxdb.hocon index 18ff48109..9c3b143a2 100644 --- a/rel/i18n/emqx_ee_connector_influxdb.hocon +++ b/rel/i18n/emqx_ee_connector_influxdb.hocon @@ -1,118 +1,71 @@ emqx_ee_connector_influxdb { - server { - desc { - en: """The IPv4 or IPv6 address or the hostname to connect to.
+bucket.desc: +"""InfluxDB bucket name.""" + +bucket.label: +"""Bucket""" + +database.desc: +"""InfluxDB database.""" + +database.label: +"""Database""" + +influxdb_api_v1.desc: +"""InfluxDB's protocol. Support InfluxDB v1.8 and before.""" + +influxdb_api_v1.label: +"""HTTP API Protocol""" + +influxdb_api_v2.desc: +"""InfluxDB's protocol. Support InfluxDB v2.0 and after.""" + +influxdb_api_v2.label: +"""HTTP API V2 Protocol""" + +org.desc: +"""Organization name of InfluxDB.""" + +org.label: +"""Organization""" + +password.desc: +"""InfluxDB password.""" + +password.label: +"""Password""" + +precision.desc: +"""InfluxDB time precision.""" + +precision.label: +"""Time Precision""" + +protocol.desc: +"""InfluxDB's protocol. HTTP API or HTTP API V2.""" + +protocol.label: +"""Protocol""" + +server.desc: +"""The IPv4 or IPv6 address or the hostname to connect to.
A host entry has the following form: `Host[:Port]`.
The InfluxDB default port 8086 is used if `[:Port]` is not specified.""" - zh: """将要连接的 IPv4 或 IPv6 地址，或者主机名。
-主机名具有以下形式：`Host[:Port]`。
-如果未指定 `[:Port]`，则使用 InfluxDB 默认端口 8086。""" - } - label { - en: "Server Host" - zh: "服务器地址" - } - } - precision { - desc { - en: """InfluxDB time precision.""" - zh: """InfluxDB 时间精度。""" - } - label { - en: """Time Precision""" - zh: """时间精度""" - } - } - protocol { - desc { - en: """InfluxDB's protocol. HTTP API or HTTP API V2.""" - zh: """InfluxDB 协议。HTTP API 或 HTTP API V2。""" - } - label { - en: """Protocol""" - zh: """协议""" - } - } - influxdb_api_v1 { - desc { - en: """InfluxDB's protocol. Support InfluxDB v1.8 and before.""" - zh: """InfluxDB HTTP API 协议。支持 Influxdb v1.8 以及之前的版本。""" - } - label { - en: """HTTP API Protocol""" - zh: """HTTP API 协议""" - } - } - influxdb_api_v2 { - desc { - en: """InfluxDB's protocol. Support InfluxDB v2.0 and after.""" - zh: """InfluxDB HTTP API V2 协议。支持 Influxdb v2.0 以及之后的版本。""" - } - label { - en: """HTTP API V2 Protocol""" - zh: """HTTP API V2 协议""" - } - } - database { - desc { - en: """InfluxDB database.""" - zh: """InfluxDB 数据库。""" - } - label { - en: "Database" - zh: "数据库" - } - } - username { - desc { - en: "InfluxDB username." - zh: "InfluxDB 用户名。" - } - label { - en: "Username" - zh: "用户名" - } - } - password { - desc { - en: "InfluxDB password." - zh: "InfluxDB 密码。" - } - label { - en: "Password" - zh: "密码" - } - } - bucket { - desc { - en: "InfluxDB bucket name." - zh: "InfluxDB bucket 名称。" - } - label { - en: "Bucket" - zh: "Bucket" - } - } - org { - desc { - en: """Organization name of InfluxDB.""" - zh: """InfluxDB 组织名称。""" - } - label { - en: """Organization""" - zh: """组织""" - } - } - token { - desc { - en: """InfluxDB token.""" - zh: """InfluxDB token。""" - } - label { - en: """Token""" - zh: """Token""" - } - } + +server.label: +"""Server Host""" + +token.desc: +"""InfluxDB token.""" + +token.label: +"""Token""" + +username.desc: +"""InfluxDB username.""" + +username.label: +"""Username""" } diff --git a/rel/i18n/emqx_ee_connector_rocketmq.hocon b/rel/i18n/emqx_ee_connector_rocketmq.hocon index 44dda7931..672dcafce 100644 --- a/rel/i18n/emqx_ee_connector_rocketmq.hocon +++ b/rel/i18n/emqx_ee_connector_rocketmq.hocon @@ -1,62 +1,35 @@ emqx_ee_connector_rocketmq { - server { - desc { - en: """The IPv4 or IPv6 address or the hostname to connect to.
+refresh_interval.desc: +"""RocketMQ Topic Route Refresh Interval.""" + +refresh_interval.label: +"""Topic Route Refresh Interval""" + +security_token.desc: +"""RocketMQ Server Security Token""" + +security_token.label: +"""Security Token""" + +send_buffer.desc: +"""The socket send buffer size of the RocketMQ driver client.""" + +send_buffer.label: +"""Send Buffer Size""" + +server.desc: +"""The IPv4 or IPv6 address or the hostname to connect to.
A host entry has the following form: `Host[:Port]`.
The RocketMQ default port 9876 is used if `[:Port]` is not specified.""" - zh: """将要连接的 IPv4 或 IPv6 地址，或者主机名。
-主机名具有以下形式：`Host[:Port]`。
-如果未指定 `[:Port]`，则使用 RocketMQ 默认端口 9876。""" - } - label: { - en: "Server Host" - zh: "服务器地址" - } - } - topic { - desc { - en: """RocketMQ Topic""" - zh: """RocketMQ 主题""" - } - label: { - en: "RocketMQ Topic" - zh: "RocketMQ 主题" - } - } +server.label: +"""Server Host""" - refresh_interval { - desc { - en: """RocketMQ Topic Route Refresh Interval.""" - zh: """RocketMQ 主题路由更新间隔。""" - } - label: { - en: "Topic Route Refresh Interval" - zh: "主题路由更新间隔" - } - } +topic.desc: +"""RocketMQ Topic""" - send_buffer { - desc { - en: """The socket send buffer size of the RocketMQ driver client.""" - zh: """RocketMQ 驱动的套字节发送消息的缓冲区大小""" - } - label: { - en: "Send Buffer Size" - zh: "发送消息的缓冲区大小" - } - } - - security_token { - desc { - en: """RocketMQ Server Security Token""" - zh: """RocketMQ 服务器安全令牌""" - } - label: { - en: "Security Token" - zh: "安全令牌" - } - } +topic.label: +"""RocketMQ Topic""" } diff --git a/rel/i18n/emqx_ee_connector_sqlserver.hocon b/rel/i18n/emqx_ee_connector_sqlserver.hocon new file mode 100644 index 000000000..ef68865fe --- /dev/null +++ b/rel/i18n/emqx_ee_connector_sqlserver.hocon @@ -0,0 +1,11 @@ +emqx_ee_connector_sqlserver { + +server.desc: +"""The IPv4 or IPv6 address or the hostname to connect to.
+A host entry has the following form: `Host[:Port]`.
+The SQL Server default port 1433 is used if `[:Port]` is not specified.""" + +server.label: +"""Server Host""" + +} diff --git a/rel/i18n/emqx_ee_connector_tdengine.hocon b/rel/i18n/emqx_ee_connector_tdengine.hocon index 02254124c..9a34b32ce 100644 --- a/rel/i18n/emqx_ee_connector_tdengine.hocon +++ b/rel/i18n/emqx_ee_connector_tdengine.hocon @@ -1,18 +1,11 @@ emqx_ee_connector_tdengine { - server { - desc { - en: """The IPv4 or IPv6 address or the hostname to connect to.
+server.desc: +"""The IPv4 or IPv6 address or the hostname to connect to.
A host entry has the following form: `Host[:Port]`.
The TDengine default port 6041 is used if `[:Port]` is not specified.""" - zh: """将要连接的 IPv4 或 IPv6 地址，或者主机名。
-主机名具有以下形式：`Host[:Port]`。
-如果未指定 `[:Port]`，则使用 TDengine 默认端口 6041。""" - } - label: { - en: "Server Host" - zh: "服务器地址" - } - } + +server.label: +"""Server Host""" } diff --git a/rel/i18n/emqx_ee_schema_registry_http_api.hocon b/rel/i18n/emqx_ee_schema_registry_http_api.hocon new file mode 100644 index 000000000..09f268459 --- /dev/null +++ b/rel/i18n/emqx_ee_schema_registry_http_api.hocon @@ -0,0 +1,39 @@ +emqx_ee_schema_registry_http_api { + +desc_param_path_schema_name.desc: +"""The schema name""" + +desc_param_path_schema_name.label: +"""Schema name""" + +desc_schema_registry_api_delete.desc: +"""Delete a schema""" + +desc_schema_registry_api_delete.label: +"""Delete schema""" + +desc_schema_registry_api_get.desc: +"""Get a schema by its name""" + +desc_schema_registry_api_get.label: +"""Get schema""" + +desc_schema_registry_api_list.desc: +"""List all registered schemas""" + +desc_schema_registry_api_list.label: +"""List schemas""" + +desc_schema_registry_api_post.desc: +"""Register a new schema""" + +desc_schema_registry_api_post.label: +"""Register schema""" + +desc_schema_registry_api_put.desc: +"""Update an existing schema""" + +desc_schema_registry_api_put.label: +"""Update schema""" + +} diff --git a/rel/i18n/emqx_ee_schema_registry_schema.hocon b/rel/i18n/emqx_ee_schema_registry_schema.hocon new file mode 100644 index 000000000..3d1ce4072 --- /dev/null +++ b/rel/i18n/emqx_ee_schema_registry_schema.hocon @@ -0,0 +1,51 @@ +emqx_ee_schema_registry_schema { + +avro_type.desc: +"""[Apache Avro](https://avro.apache.org/) serialization format.""" + +avro_type.label: +"""Apache Avro""" + +protobuf_type.desc: +"""[Protocol Buffers](https://protobuf.dev/) serialization format.""" + +protobuf_type.label: +"""Protocol Buffers""" + +schema_description.desc: +"""A description for this schema.""" + +schema_description.label: +"""Schema description""" + +schema_name.desc: +"""A name for the schema that will serve as its identifier.""" + +schema_name.label: +"""Schema name""" + +schema_registry_root.desc: +"""Schema registry configurations.""" + +schema_registry_root.label: +"""Schema registry""" + +schema_registry_schemas.desc: +"""Registered schemas.""" + +schema_registry_schemas.label: +"""Registered schemas""" + +schema_source.desc: +"""Source text for the schema.""" + +schema_source.label: +"""Schema source""" + +schema_type.desc: +"""Schema type.""" + +schema_type.label: +"""Schema type""" + +} diff --git a/rel/i18n/emqx_exhook_api.hocon b/rel/i18n/emqx_exhook_api.hocon index 3ec5367ed..9cb7177c1 100644 --- a/rel/i18n/emqx_exhook_api.hocon +++ b/rel/i18n/emqx_exhook_api.hocon @@ -1,180 +1,81 @@ emqx_exhook_api { - list_all_servers { - desc { - en: "List all servers" - zh: "查看ExHook 服务器列表" - } - } +add_server.desc: +"""Add a server""" - add_server { - desc { - en: "Add a server" - zh: "添加 ExHook 服务器" - } - } +delete_server.desc: +"""Delete the server""" - get_detail { - desc { - en: "Get the detail information of Exhook server" - zh: "查看 Exhook 服务器详细信息" - } - } +get_detail.desc: +"""Get the detail information of Exhook server""" - update_server { - desc { - en: "Update the server" - zh: "更新 Exhook 服务器" - } - } +get_hooks.desc: +"""Get the hooks information of server""" - delete_server { - desc { - en: "Delete the server" - zh: "删除 Exhook 服务器" - } - } +hook_metrics.desc: +"""Metrics information of this hook in the current node""" - get_hooks { - desc { - en: "Get the hooks information of server" - zh: "获取 Exhook 服务器的钩子信息" - } - } +hook_name.desc: +"""The hook's name""" - move_api { - desc { - en: """Move the server. -NOTE: The position should be \"front | rear | before:{name} | after:{name}""" - zh: """移动 Exhook 服务器顺序。 -注意: 移动的参数只能是：front | rear | before:{name} | after:{name}""" - } - label { - en: "Change order of execution for registered Exhook server" - zh: "改变已注册的Exhook服务器的执行顺序" - } - } +hook_params.desc: +"""The parameters used when the hook is registered""" - move_position { - desc { - en: "The target position to be moved" - zh: "移动的方向" - } - } +list_all_servers.desc: +"""List all servers""" - hook_name { - desc { - en: "The hook's name" - zh: "钩子的名称" - } - } +metric_failed.desc: +"""The number of times the hook execution failed""" - server_name { - desc { - en: "The Exhook server name" - zh: "Exhook 服务器的名称" - } - } +metric_max_rate.desc: +"""Maximum call rate of hooks""" - hook_params { - desc { - en: "The parameters used when the hook is registered" - zh: "钩子注册时使用的参数" - } - } +metric_rate.desc: +"""The call rate of hooks""" - server_metrics { - desc { - en: "Metrics information of this server in the current node" - zh: "当前节点中该服务器的指标信息" - } - } +metric_succeed.desc: +"""The number of times the hooks execution successful""" - node_metrics { - desc { - en: "Metrics information of this server in all nodes" - zh: "所有节点中该服务器的指标信息" - } - } +metrics.desc: +"""Metrics information""" - node_status { - desc { - en: "status of this server in all nodes" - zh: "所有节点中该服务器的状态信息" - } - } +move_api.desc: +"""Move the server. +NOTE: The position should be "front | rear | before:{name} | after:{name}""" - hook_metrics { - desc { - en: "Metrics information of this hook in the current node" - zh: "当前节点中该钩子的指标信息" - } - } +move_api.label: +"""Change order of execution for registered Exhook server""" - node_hook_metrics { - desc { - en: "Metrics information of this hook in all nodes" - zh: "所有节点中该钩子的指标信息" - } - } +move_position.desc: +"""The target position to be moved""" - node { - desc { - en: "Node name" - zh: "节点名称" - } - } +node.desc: +"""Node name""" - metrics { - desc { - en: "Metrics information" - zh: "指标信息" - } - } +node_hook_metrics.desc: +"""Metrics information of this hook in all nodes""" - status { - desc { - en: """The status of Exhook server.
+node_metrics.desc: +"""Metrics information of this server in all nodes""" + +node_status.desc: +"""status of this server in all nodes""" + +server_metrics.desc: +"""Metrics information of this server in the current node""" + +server_name.desc: +"""The Exhook server name""" + +status.desc: +"""The status of Exhook server.
connected: connection succeeded
connecting: connection failed, reconnecting
disconnected: failed to connect and didn't reconnect
disabled: this server is disabled
error: failed to view the status of this server""" - zh: """Exhook 服务器的状态。
-connected: 连接成功
-connecting: 连接失败，重连中
-disconnected: 连接失败，且未设置自动重连
-disabled: 该服务器未开启
-error: 查看该服务器状态时发生错误""" - } - } - metric_succeed { - desc { - en: "The number of times the hooks execution successful" - zh: "钩子执行成功的次数" - } - } - - metric_failed { - desc { - en: "The number of times the hook execution failed" - zh: "钩子执行失败的次数" - } - } - - metric_rate { - desc { - en: "The call rate of hooks" - zh: "钩子的调用速率" - } - } - - metric_max_rate { - desc { - en: "Maximum call rate of hooks" - zh: "钩子的最大调用速率" - } - } +update_server.desc: +"""Update the server""" } diff --git a/rel/i18n/emqx_exhook_schema.hocon b/rel/i18n/emqx_exhook_schema.hocon index 5b34a245a..6d262fae7 100644 --- a/rel/i18n/emqx_exhook_schema.hocon +++ b/rel/i18n/emqx_exhook_schema.hocon @@ -1,97 +1,45 @@ emqx_exhook_schema { - servers { - desc { - en: "List of exhook servers" - zh: "ExHook 服务器列表" - } - } - - name { - desc { - en: "Name of the exhook server" - zh: "ExHook 服务器名称" - } - } - - enable { - desc { - en: "Enable this Exhook server" - zh: "开启这个 Exhook 服务器" - } - } - - url { - desc { - en: "URL of the gRPC server" - zh: "gRPC 服务器地址" - } - } - - request_timeout { - desc { - en: "The timeout of request gRPC server" - zh: "gRPC 服务器请求超时时间" - } - } - - failed_action { - desc { - en: "The value that is returned when the request to the gRPC server fails for any reason" - zh: "当 gRPC 请求失败后的操作" - } - } - - auto_reconnect { - desc { - en: """Whether to automatically reconnect (initialize) the gRPC server. +auto_reconnect.desc: +"""Whether to automatically reconnect (initialize) the gRPC server. When gRPC is not available, Exhook tries to request the gRPC service at that interval and reinitialize the list of mounted hooks.""" - zh: """自动重连到 gRPC 服务器的设置。 -当 gRPC 服务器不可用时，Exhook 将会按照这里设置的间隔时间进行重连，并重新初始化注册的钩子""" - } - } - pool_size { - desc { - en: "The process pool size for gRPC client" - zh: "gRPC 客户端进程池大小" - } - } +enable.desc: +"""Enable this Exhook server""" - socket_options { - desc { - en: "Connection socket options" - zh: "连接套接字设置" - } - } +failed_action.desc: +"""The value that is returned when the request to the gRPC server fails for any reason""" - keepalive { - desc { - en: """Enables/disables periodic transmission on a connected socket when no other data is exchanged. +keepalive.desc: +"""Enables/disables periodic transmission on a connected socket when no other data is exchanged. If the other end does not respond, the connection is considered broken and an error message is sent to the controlling process.""" - zh: """当没有其他数据交换时，是否向连接的对端套接字定期的发送探测包。如果另一端没有响应，则认为连接断开，并向控制进程发送错误消息""" - } - } - nodelay { - desc { - en: """If true, option TCP_NODELAY is turned on for the socket, +name.desc: +"""Name of the exhook server""" + +nodelay.desc: +"""If true, option TCP_NODELAY is turned on for the socket, which means that also small amounts of data are sent immediately""" - zh: "如果为 true，则为套接字设置 TCP_NODELAY 选项，这意味着会立即发送数据包" - } - } - recbuf { - desc { - en: "The minimum size of receive buffer to use for the socket" - zh: "套接字的最小接收缓冲区大小" - } - } +pool_size.desc: +"""The process pool size for gRPC client""" + +recbuf.desc: +"""The minimum size of receive buffer to use for the socket""" + +request_timeout.desc: +"""The timeout of request gRPC server""" + +servers.desc: +"""List of exhook servers""" + +sndbuf.desc: +"""The minimum size of send buffer to use for the socket""" + +socket_options.desc: +"""Connection socket options""" + +url.desc: +"""URL of the gRPC server""" - sndbuf { - desc { - en: "The minimum size of send buffer to use for the socket" - zh: "套接字的最小发送缓冲区大小" - } - } } diff --git a/rel/i18n/emqx_exproto_schema.hocon b/rel/i18n/emqx_exproto_schema.hocon new file mode 100644 index 000000000..eed450208 --- /dev/null +++ b/rel/i18n/emqx_exproto_schema.hocon @@ -0,0 +1,24 @@ +emqx_exproto_schema { + +exproto.desc: +"""The Extension Protocol configuration""" + +exproto_grpc_handler_address.desc: +"""gRPC server address.""" + +exproto_grpc_handler_ssl.desc: +"""SSL configuration for the gRPC client.""" + +exproto_grpc_server_bind.desc: +"""Listening address and port for the gRPC server.""" + +exproto_grpc_server_ssl.desc: +"""SSL configuration for the gRPC server.""" + +exproto_handler.desc: +"""Configurations for request to ConnectionHandler service""" + +exproto_server.desc: +"""Configurations for starting the ConnectionAdapter service""" + +} diff --git a/rel/i18n/emqx_gateway_api.hocon b/rel/i18n/emqx_gateway_api.hocon index 18ab1f242..1e0e22456 100644 --- a/rel/i18n/emqx_gateway_api.hocon +++ b/rel/i18n/emqx_gateway_api.hocon @@ -1,166 +1,73 @@ emqx_gateway_api { - list_gateway { - desc { - en: """This API returns an overview info for the specified or all gateways. -including current running status, number of connections, listener status, etc.""" - zh: """该接口会返回指定或所有网关的概览状态， -包括当前状态、连接数、监听器状态等。""" - } - } +delete_gateway.desc: +"""Unload the specified gateway""" - enable_gateway { - desc { - en: """Enable a gateway by confs.""" - zh: """使用配置启动某一网关。""" - } - } +enable_gateway.desc: +"""Enable a gateway by confs.""" - get_gateway { - desc { - en: """Get the gateway configurations""" - zh: """获取网关配置详情""" - } - } +gateway_created_at.desc: +"""The Gateway created datetime""" - delete_gateway { - desc { - en: """Unload the specified gateway""" - zh: """停用指定网关""" - } - } +gateway_current_connections.desc: +"""The Gateway current connected connections/clients""" - update_gateway { - desc { - en: """Update the gateway basic configurations and running status.
-Note: The Authentication and Listener configurations should be updated by other special APIs.""" - zh: """更新指定网关的基础配置、和启用的状态。
-注：认证、和监听器的配置更新需参考对应的 API 接口。""" - } - } +gateway_enable_in_path.desc: +"""Whether to enable this gateway""" - gateway_name { - desc { - en: """Gateway Name""" - zh: """网关名称""" - } - } +gateway_listener_id.desc: +"""Listener ID""" - gateway_name_in_qs { - desc { - en: """Gateway Name.
+gateway_listener_name.desc: +"""Listener Name""" + +gateway_listener_running.desc: +"""Listener Running status""" + +gateway_listener_type.desc: +"""Listener Type""" + +gateway_listeners.desc: +"""The Gateway listeners overview""" + +gateway_max_connections.desc: +"""The Gateway allowed maximum connections/clients""" + +gateway_name.desc: +"""Gateway Name""" + +gateway_name_in_qs.desc: +"""Gateway Name.
It's enum with `stomp`, `mqttsn`, `coap`, `lwm2m`, `exproto`""" - zh: """网关名称.
-可取值为 `stomp`、`mqttsn`、`coap`、`lwm2m`、`exproto`""" - } - } - gateway_enable_in_path { - desc { - en: """Whether to enable this gateway""" +gateway_node_status.desc: +"""The status of the gateway on each node in the cluster""" - zh: """是否开启此网关""" - } - } +gateway_started_at.desc: +"""The Gateway started datetime""" - gateway_status { - desc { - en: """Gateway status""" - zh: """网关启用状态""" - } - } +gateway_status.desc: +"""Gateway status""" - gateway_status_in_qs { - desc { - en: """Filter gateways by status.
+gateway_status_in_qs.desc: +"""Filter gateways by status.
It is enum with `running`, `stopped`, `unloaded`""" - zh: """通过网关状态筛选
-可选值为 `running`、`stopped`、`unloaded`""" - } - } - gateway_created_at { - desc { - en: """The Gateway created datetime""" - zh: """网关创建时间""" - } - } +gateway_stopped_at.desc: +"""The Gateway stopped datetime""" - gateway_started_at { - desc { - en: """The Gateway started datetime""" - zh: """网关启用时间""" - } - } +get_gateway.desc: +"""Get the gateway configurations""" - gateway_stopped_at { - desc { - en: """The Gateway stopped datetime""" - zh: """网关停用时间""" - } - } +list_gateway.desc: +"""This API returns an overview info for the specified or all gateways. +including current running status, number of connections, listener status, etc.""" - gateway_max_connections { - desc { - en: """The Gateway allowed maximum connections/clients""" - zh: """最大连接数""" - } - } +node.desc: +"""Node Name""" - gateway_current_connections { - desc { - en: """The Gateway current connected connections/clients""" - zh: """当前连接数""" - } - } - - gateway_listeners { - desc { - en: """The Gateway listeners overview""" - zh: """网关监听器列表""" - } - } - - gateway_listener_id { - desc { - en: """Listener ID""" - zh: """监听器 ID""" - } - } - - gateway_listener_name { - desc { - en: """Listener Name""" - zh: """监听器名称""" - } - } - - gateway_listener_running { - desc { - en: """Listener Running status""" - zh: """监听器运行状态""" - } - } - - gateway_listener_type { - desc { - en: """Listener Type""" - zh: """监听器类型""" - } - } - - gateway_node_status { - desc { - en: """The status of the gateway on each node in the cluster""" - zh: """网关在集群中每个节点上的状态""" - } - } - - node { - desc { - en: """Node Name""" - zh: """节点名称""" - } - } +update_gateway.desc: +"""Update the gateway basic configurations and running status.
+Note: The Authentication and Listener configurations should be updated by other special APIs.""" } diff --git a/rel/i18n/emqx_gateway_api_authn.hocon b/rel/i18n/emqx_gateway_api_authn.hocon index a9ae33f0c..2d84eef54 100644 --- a/rel/i18n/emqx_gateway_api_authn.hocon +++ b/rel/i18n/emqx_gateway_api_authn.hocon @@ -1,99 +1,45 @@ emqx_gateway_api_authn { - get_authn { - desc { - en: """Gets the configuration of the specified gateway authenticator.
-Returns 404 when gateway or authentication is not enabled.""" - zh: """获取指定网关认证器的配置 -当网关或认证未启用时，返回 404。""" - } - } - - update_authn { - desc { - en: """Update the configuration of the specified gateway authenticator, or disable the authenticator.""" - zh: """更新指定网关认证器的配置，或停用认证器。""" - } - } - - add_authn { - desc { - en: """Enables the authenticator for client authentication for the specified gateway.
+add_authn.desc: +"""Enables the authenticator for client authentication for the specified gateway.
When the authenticator is not configured or turned off, all client connections are assumed to be allowed.
Note: Only one authenticator is allowed to be enabled at a time in the gateway, rather than allowing multiple authenticators to be configured to form an authentication chain as in MQTT.""" - zh: """为指定网关开启认证器实现客户端认证的功能。
-当未配置认证器或关闭认证器时，则认为允许所有客户端的连接。
-注：在网关中仅支持添加一个认证器，而不是像 MQTT 一样允许配置多个认证器构成认证链。""" - } - } - delete_authn { - desc { - en: """Delete the authenticator of the specified gateway.""" - zh: """删除指定网关的认证器。""" - } - } +add_user.desc: +"""Add user for the authenticator (only supports built_in_database).""" - list_users { - desc { - en: """Get the users for the authenticator (only supported by built_in_database).""" - zh: """获取用户列表（仅支持 built_in_database 类型的认证器）""" - } - } +delete_authn.desc: +"""Delete the authenticator of the specified gateway.""" - add_user { - desc { - en: """Add user for the authenticator (only supports built_in_database).""" - zh: """添加用户（仅支持 built_in_database 类型的认证器）""" - } - } +delete_user.desc: +"""Delete the user for the gateway authenticator (only supports built_in_database)""" - get_user { - desc { - en: """Get user info from the gateway authenticator (only supports built_in_database)""" - zh: """获取用户信息（仅支持 built_in_database 类型的认证器）""" - } - } +get_authn.desc: +"""Gets the configuration of the specified gateway authenticator.
+Returns 404 when gateway or authentication is not enabled.""" - update_user { - desc { - en: """Update the user info for the gateway authenticator (only supports built_in_database)""" - zh: """更新用户信息（仅支持 built_in_database 类型的认证器）""" - } - } +get_user.desc: +"""Get user info from the gateway authenticator (only supports built_in_database)""" - delete_user { - desc { - en: """Delete the user for the gateway authenticator (only supports built_in_database)""" - zh: """删除用户（仅支持 built_in_database 类型的认证器）""" - } - } +import_users.desc: +"""Import users into the gateway authenticator (only supports built_in_database)""" - import_users { - desc { - en: """Import users into the gateway authenticator (only supports built_in_database)""" - zh: """导入用户（仅支持 built_in_database 类型的认证器）""" - } - } +is_superuser.desc: +"""Is superuser""" - user_id { - desc { - en: """User ID""" - zh: """用户 ID""" - } - } +like_user_id.desc: +"""Fuzzy search using user ID (username or clientid), only supports search by substring.""" - like_user_id { - desc { - en: """Fuzzy search using user ID (username or clientid), only supports search by substring.""" - zh: """使用用户 ID （username 或 clientid）模糊搜索，仅支持按子串的方式进行搜索。""" - } - } +list_users.desc: +"""Get the users for the authenticator (only supported by built_in_database).""" + +update_authn.desc: +"""Update the configuration of the specified gateway authenticator, or disable the authenticator.""" + +update_user.desc: +"""Update the user info for the gateway authenticator (only supports built_in_database)""" + +user_id.desc: +"""User ID""" - is_superuser { - desc { - en: """Is superuser""" - zh: """是否是超级用户""" - } - } } diff --git a/rel/i18n/emqx_gateway_api_clients.hocon b/rel/i18n/emqx_gateway_api_clients.hocon index 1e6f575c3..4c95318ab 100644 --- a/rel/i18n/emqx_gateway_api_clients.hocon +++ b/rel/i18n/emqx_gateway_api_clients.hocon @@ -1,478 +1,207 @@ emqx_gateway_api_clients { - list_clients { - desc { - en: """Get the gateway client list""" - zh: """获取指定网关的客户端列表""" - } - } - - get_client { - desc { - en: """Get the gateway client information""" - zh: """获取客户端信息""" - } - } - - kick_client { - desc { - en: """Kick out the gateway client""" - zh: """踢出指定客户端""" - } - } - - list_subscriptions { - desc { - en: """Get the gateway client subscriptions""" - zh: """获取某客户端的主题订阅列表""" - } - } - - add_subscription { - desc { - en: """Create a subscription membership""" - zh: """为某客户端新增订阅关系""" - } - } - - delete_subscription { - desc { - en: """Delete a subscriptions membership""" - zh: """为某客户端删除某订阅关系""" - } - } - - param_node { - desc { - en: """Match the client's node name""" - zh: """匹配客户端的节点名称""" - } - } - - param_clientid { - desc { - en: """Match the client's ID""" - zh: """匹配客户端 ID""" - } - } - - param_username { - desc { - en: """Match the client's Username""" - zh: """匹配客户端 Username""" - } - } - - param_ip_address { - desc { - en: """Match the client's ip address""" - zh: """匹配客户端 IP 地址""" - } - } - - param_conn_state { - desc { - en: """Match the client's connection state""" - zh: """匹配客户端连接状态""" - } - } - - param_proto_ver { - desc { - en: """Match the client's protocol version""" - zh: """匹配客户端协议版本""" - } - } - - param_clean_start { - desc { - en: """Match the client's clean start flag""" - zh: """匹配客户端 `clean_start` 标记""" - } - } - - param_like_clientid { - desc { - en: """Use sub-string to match client's ID""" - zh: """子串匹配客户端 ID""" - } - } - - param_like_username { - desc { - en: """Use sub-string to match client's username""" - zh: """子串匹配 客户端 Username""" - } - } - - param_gte_created_at { - desc { - en: """Match the session created datetime greater than a certain value""" - zh: """匹配会话创建时间大于等于指定值的客户端""" - } - } - - param_lte_created_at { - desc { - en: """Match the session created datetime less than a certain value""" - zh: """匹配会话创建时间小于等于指定值的客户端""" - } - } - - param_gte_connected_at{ - desc { - en: """Match the client socket connected datetime greater than a certain value""" - zh: """匹配连接创建时间大于等于指定值的客户端""" - } - } - - param_lte_connected_at { - desc { - en: """Match the client socket connected datatime less than a certain value""" - zh: """匹配连接创建时间小于等于指定值的客户端""" - } - } - - param_endpoint_name { - desc { - en: """Match the lwm2m client's endpoint name""" - zh: """匹配 LwM2M 客户端 Endpoint Name""" - } - } - - param_like_endpoint_name { - desc { - en: """Use sub-string to match lwm2m client's endpoint name""" - zh: """子串匹配 LwM2M 客户端 Endpoint Name""" - } - } - - param_gte_lifetime { - desc { - en: """Match the lwm2m client registered lifetime greater than a certain value""" - zh: """匹配心跳时间大于等于指定值的 LwM2M 客户端""" - } - } - - param_lte_lifetime { - desc { - en: """Match the lwm2m client registered lifetime less than a certain value""" - zh: """匹配心跳时间小于等于指定值的 LwM2M 客户端""" - } - } - - clientid { - desc { - en: """Client ID""" - zh: """客户端 ID""" - } - } - - topic { - desc { - en: """Topic Filter/Name""" - zh: """主题过滤器或主题名称""" - } - } - - endpoint_name { - desc { - en: """The LwM2M client endpoint name""" - zh: """LwM2M 客户端 Endpoint Name""" - } - } - - lifetime { - desc { - en: """LwM2M Life time""" - zh: """LwM2M 客户端心跳周期""" - } - } - - qos { - desc { - en: """QoS level, enum: 0, 1, 2""" - zh: """QoS 等级，枚举：0，1，2""" - } - } - - nl { - desc { - en: """No Local option, enum: 0, 1""" - zh: """No Local 选项，枚举：0，1""" - } - } - - rap { - desc { - en: """Retain as Published option, enum: 0, 1""" - zh: """Retain as Published 选项，枚举：0，1""" - } - } - - rh { - desc { - en: """Retain Handling option, enum: 0, 1, 2""" - zh: """Retain Handling 选项，枚举：0，1，2""" - } - } - - sub_props { - desc { - en: """Subscription properties""" - zh: """订阅属性""" - } - } - - subid { - desc { - en: """Only stomp protocol, a unique identity for the subscription. range: 1-65535.""" - zh: """订阅ID，仅用于 Stomp 网关。用于创建订阅关系时指定订阅 ID。取值范围 1-65535。""" - } - } - - node { - desc { - en: """Name of the node to which the client is connected""" - zh: """客户端连接到的节点名称""" - } - } - - username { - desc { - en: """Username of client when connecting""" - zh: """客户端连接的用户名""" - } - } - - mountpoint { - desc { - en: """Topic mountpoint""" - zh: """主题固定前缀""" - } - } - - proto_name { - desc { - en: """Client protocol name""" - zh: """客户端使用的协议名称""" - } - } - - proto_ver { - desc { - en: """Protocol version used by the client""" - zh: """客户端使用的协议版本""" - } - } - - ip_address { - desc { - en: """Client's IP address""" - zh: """客户端 IP 地址""" - } - } - - port { - desc { - en: """Client's port""" - zh: """客户端端口""" - } - } - - is_bridge { - desc { - en: """Indicates whether the client is connected via bridge""" - zh: """标识客户端是否通过 is_bridge 标志连接""" - } - } - - connected_at { - desc { - en: """Client connection time""" - zh: """客户端连接时间""" - } - } - - disconnected_at { - desc { - en: """Client offline time, This field is only valid and returned when connected is false""" - zh: """客户端连接断开时间""" - } - } - - connected { - desc { - en: """Whether the client is connected""" - zh: """标识客户端是否已连接到网关""" - } - } - - keepalive { - desc { - en: """Keepalive time, with the unit of second""" - zh: """Keepalive 时间，单位：秒""" - } - } - - clean_start { - desc { - en: """Indicate whether the client is using a brand new session""" - zh: """标识客户端是否以 clean_start 的标志连接到网关""" - } - } - - expiry_interval { - desc { - en: """Session expiration interval, with the unit of second""" - zh: """会话超期时间，单位：秒""" - } - } - - created_at { - desc { - en: """Session creation time""" - zh: """会话创建时间""" - } - } - - subscriptions_cnt { - desc { - en: """Number of subscriptions established by this client""" - zh: """客户端已订阅主题数""" - } - } - - subscriptions_max { - desc { - en: """Maximum number of subscriptions allowed by this client""" - zh: """客户端允许订阅的最大主题数""" - } - } - - inflight_cnt { - desc { - en: """Current length of inflight""" - zh: """客户端当前飞行窗口大小""" - } - } - - inflight_max { - desc { - en: """Maximum length of inflight""" - zh: """客户端允许的飞行窗口最大值""" - } - } - - mqueue_len { - desc { - en: """Current length of message queue""" - zh: """客户端当前消息队列长度""" - } - } - - mqueue_max { - desc { - en: """Maximum length of message queue""" - zh: """客户端允许的最大消息队列长度""" - } - } - - mqueue_dropped { - desc { - en: """Number of messages dropped by the message queue due to exceeding the length""" - zh: """由于消息队列过程，客户端消息队列丢弃消息条数""" - } - } - - awaiting_rel_cnt { - desc { - en: """Number of awaiting acknowledge packet""" - zh: """客户端当前等待 PUBREL 确认的 PUBREC 消息的条数""" - } - } - - awaiting_rel_max { - desc { - en: """Maximum allowed number of awaiting PUBREC packet""" - zh: """客户端允许的最大 PUBREC 等待队列长度""" - } - } - - recv_oct { - desc { - en: """Number of bytes received""" - zh: """已接收的字节数""" - } - } - - recv_cnt { - desc { - en: """Number of socket packets received""" - zh: """已接收 Socket 报文次数""" - } - } - - recv_pkt { - desc { - en: """Number of protocol packets received""" - zh: """已接收应用层协议控制报文数""" - } - } - - recv_msg { - desc { - en: """Number of message packets received""" - zh: """已接收上行的消息条数""" - } - } - - send_oct { - desc { - en: """Number of bytes sent""" - zh: """已发送字节数""" - } - } - - send_cnt { - desc { - en: """Number of socket packets sent""" - zh: """已发送 Socket 报文次数""" - } - } - - send_pkt { - desc { - en: """Number of protocol packets sent""" - zh: """已发送应用层协议控制报文数""" - } - } - - send_msg { - desc { - en: """Number of message packets sent""" - zh: """已发送下行消息数条数""" - } - } - - mailbox_len { - desc { - en: """Process mailbox size""" - zh: """进程邮箱大小""" - } - } - - heap_size { - desc { - en: """Process heap size with the unit of byte""" - zh: """进程堆内存大小，单位：字节""" - } - } - - reductions { - desc { - en: """Erlang reduction""" - zh: """进程已消耗 Reduction 数""" - } - } +disconnected_at.desc: +"""Client offline time, This field is only valid and returned when connected is false""" + +heap_size.desc: +"""Process heap size with the unit of byte""" + +send_oct.desc: +"""Number of bytes sent""" + +get_client.desc: +"""Get the gateway client information""" + +param_gte_created_at.desc: +"""Match the session created datetime greater than a certain value""" + +param_conn_state.desc: +"""Match the client's connection state""" + +send_pkt.desc: +"""Number of protocol packets sent""" + +clean_start.desc: +"""Indicate whether the client is using a brand new session""" + +inflight_cnt.desc: +"""Current length of inflight""" + +delete_subscription.desc: +"""Delete a subscriptions membership""" + +param_lte_connected_at.desc: +"""Match the client socket connected datatime less than a certain value""" + +node.desc: +"""Name of the node to which the client is connected""" + +awaiting_rel_cnt.desc: +"""Number of awaiting acknowledge packet""" + +rap.desc: +"""Retain as Published option, enum: 0, 1""" + +inflight_max.desc: +"""Maximum length of inflight""" + +param_username.desc: +"""Match the client's Username""" + +param_like_endpoint_name.desc: +"""Use sub-string to match lwm2m client's endpoint name""" + +created_at.desc: +"""Session creation time""" + +sub_props.desc: +"""Subscription properties""" + +list_clients.desc: +"""Get the gateway client list""" + +subscriptions_cnt.desc: +"""Number of subscriptions established by this client""" + +mailbox_len.desc: +"""Process mailbox size""" + +send_cnt.desc: +"""Number of socket packets sent""" + +rh.desc: +"""Retain Handling option, enum: 0, 1, 2""" + +connected.desc: +"""Whether the client is connected""" + +qos.desc: +"""QoS level, enum: 0, 1, 2""" + +ip_address.desc: +"""Client's IP address""" + +param_gte_connected_at.desc: +"""Match the client socket connected datetime greater than a certain value""" + +awaiting_rel_max.desc: +"""Maximum allowed number of awaiting PUBREC packet""" + +param_like_username.desc: +"""Use sub-string to match client's username""" + +nl.desc: +"""No Local option, enum: 0, 1""" + +param_like_clientid.desc: +"""Use sub-string to match client's ID""" + +param_lte_created_at.desc: +"""Match the session created datetime less than a certain value""" + +topic.desc: +"""Topic Filter/Name""" + +proto_ver.desc: +"""Protocol version used by the client""" + +mountpoint.desc: +"""Topic mountpoint""" + +proto_name.desc: +"""Client protocol name""" + +param_lte_lifetime.desc: +"""Match the lwm2m client registered lifetime less than a certain value""" + +port.desc: +"""Client's port""" + +connected_at.desc: +"""Client connection time""" + +expiry_interval.desc: +"""Session expiration interval, with the unit of second""" + +username.desc: +"""Username of client when connecting""" + +param_clean_start.desc: +"""Match the client's clean start flag""" + +recv_msg.desc: +"""Number of message packets received""" + +list_subscriptions.desc: +"""Get the gateway client subscriptions""" + +recv_oct.desc: +"""Number of bytes received""" + +keepalive.desc: +"""Keepalive time, with the unit of second""" + +param_clientid.desc: +"""Match the client's ID""" + +subscriptions_max.desc: +"""Maximum number of subscriptions allowed by this client""" + +param_ip_address.desc: +"""Match the client's ip address""" + +mqueue_max.desc: +"""Maximum length of message queue""" + +mqueue_dropped.desc: +"""Number of messages dropped by the message queue due to exceeding the length""" + +subid.desc: +"""Only stomp protocol, a unique identity for the subscription. range: 1-65535.""" + +clientid.desc: +"""Client ID""" + +kick_client.desc: +"""Kick out the gateway client""" + +is_bridge.desc: +"""Indicates whether the client is connected via bridge""" + +lifetime.desc: +"""LwM2M Life time""" + +send_msg.desc: +"""Number of message packets sent""" + +add_subscription.desc: +"""Create a subscription membership""" + +param_endpoint_name.desc: +"""Match the lwm2m client's endpoint name""" + +param_node.desc: +"""Match the client's node name""" + +recv_cnt.desc: +"""Number of socket packets received""" + +recv_pkt.desc: +"""Number of protocol packets received""" + +endpoint_name.desc: +"""The LwM2M client endpoint name""" + +param_proto_ver.desc: +"""Match the client's protocol version""" + +reductions.desc: +"""Erlang reduction""" + +param_gte_lifetime.desc: +"""Match the lwm2m client registered lifetime greater than a certain value""" + +mqueue_len.desc: +"""Current length of message queue""" + } diff --git a/rel/i18n/emqx_gateway_api_listeners.hocon b/rel/i18n/emqx_gateway_api_listeners.hocon index 9319bb3e5..3b5bb65a1 100644 --- a/rel/i18n/emqx_gateway_api_listeners.hocon +++ b/rel/i18n/emqx_gateway_api_listeners.hocon @@ -1,146 +1,65 @@ emqx_gateway_api_listeners { - list_listeners { - desc { - en: """Gets a list of gateway listeners. This interface returns all the configs of the listener (including the authenticator on that listener), as well as the status of that listener running in the cluster.""" - zh: """获取网关监听器列表。该接口会返回监听器所有的配置（包括该监听器上的认证器），同时也会返回该监听器在集群中运行的状态。""" - } - } - - add_listener { - desc { - en: """Create the gateway listener.
+add_listener.desc: +"""Create the gateway listener.
Note: For listener types not supported by a gateway, this API returns `400: BAD_REQUEST`.""" - zh: """为指定网关添加监听器。
-注：对于某网关不支持的监听器类型，该接口会返回 `400: BAD_REQUEST`。""" - } - } - get_listener { - desc { - en: """Get the gateway listener configs""" - zh: """获取指定网关监听器的配置。""" - } - } - - delete_listener { - desc { - en: """Delete the gateway listener. All connected clients under the deleted listener will be disconnected.""" - zh: """删除指定监听器。被删除的监听器下所有已连接的客户端都会离线。""" - } - } - - update_listener { - desc { - en: """Update the gateway listener. The listener being updated performs a restart and all clients connected to that listener will be disconnected.""" - zh: """更新某网关监听器的配置。被更新的监听器会执行重启，所有已连接到该监听器上的客户端都会被断开。""" - } - } - - get_listener_authn { - desc { - en: """Get the listener's authenticator configs.""" - zh: """获取监听器的认证器配置。""" - } - } - - add_listener_authn { - desc { - en: """Enable authenticator for specified listener for client authentication.
+add_listener_authn.desc: +"""Enable authenticator for specified listener for client authentication.
When authenticator is enabled for a listener, all clients connecting to that listener will use that authenticator for authentication.""" - zh: """为指定监听器开启认证器以实现客户端认证的能力。
-当某一监听器开启认证后，所有连接到该监听器的客户端会使用该认证器进行认证。""" - } - } - update_listener_authn { - desc { - en: """Update authenticator configs for the listener, or disable/enable it.""" - zh: """更新指定监听器的认证器配置，或停用/启用该认证器。""" - } - } +add_user.desc: +"""Add user for the authenticator (only supports built_in_database)""" - delete_listener_authn { - desc { - en: """Remove authenticator for the listener.""" - zh: """移除指定监听器的认证器。""" - } - } +current_connections.desc: +"""Current Connections""" - list_users { - desc { - en: """Get the users for the authenticator (only supported by built_in_database)""" - zh: """获取用户列表（仅支持 built_in_database 类型的认证器）""" - } - } +delete_listener.desc: +"""Delete the gateway listener. All connected clients under the deleted listener will be disconnected.""" - add_user { - desc { - en: """Add user for the authenticator (only supports built_in_database)""" - zh: """添加用户（仅支持 built_in_database 类型的认证器）""" - } - } +delete_listener_authn.desc: +"""Remove authenticator for the listener.""" - get_user { - desc { - en: """Get user info from the gateway authenticator (only supports built_in_database)""" - zh: """获取用户信息（仅支持 built_in_database 类型的认证器）""" - } - } +delete_user.desc: +"""Delete the user for the gateway authenticator (only supports built_in_database)""" - update_user { - desc { - en: """Update the user info for the gateway authenticator (only supports built_in_database)""" - zh: """更新用户信息（仅支持 built_in_database 类型的认证器）""" - } - } +get_listener.desc: +"""Get the gateway listener configs""" - delete_user { - desc { - en: """Delete the user for the gateway authenticator (only supports built_in_database)""" - zh: """删除用户（仅支持 built_in_database 类型的认证器）""" - } - } +get_listener_authn.desc: +"""Get the listener's authenticator configs.""" - import_users { - desc { - en: """Import users into the gateway authenticator (only supports built_in_database)""" - zh: """导入用户（仅支持 built_in_database 类型的认证器）""" - } - } +get_user.desc: +"""Get user info from the gateway authenticator (only supports built_in_database)""" - listener_id { - desc { - en: """Listener ID""" - zh: """监听器 ID""" - } - } +import_users.desc: +"""Import users into the gateway authenticator (only supports built_in_database)""" - listener_status { - desc { - en: """listener status""" - zh: """监听器状态""" - } - } +list_listeners.desc: +"""Gets a list of gateway listeners. This interface returns all the configs of the listener (including the authenticator on that listener), as well as the status of that listener running in the cluster.""" - listener_node_status { - desc { - en: """listener status of each node in the cluster""" - zh: """监听器在集群中每个节点上的状态""" - } - } +list_users.desc: +"""Get the users for the authenticator (only supported by built_in_database)""" - node { - desc { - en: """Node Name""" - zh: """节点名称""" - } - } +listener_id.desc: +"""Listener ID""" + +listener_node_status.desc: +"""listener status of each node in the cluster""" + +listener_status.desc: +"""listener status""" + +node.desc: +"""Node Name""" + +update_listener.desc: +"""Update the gateway listener. The listener being updated performs a restart and all clients connected to that listener will be disconnected.""" + +update_listener_authn.desc: +"""Update authenticator configs for the listener, or disable/enable it.""" + +update_user.desc: +"""Update the user info for the gateway authenticator (only supports built_in_database)""" - current_connections { - desc { - en: """Current Connections""" - zh: """当前连接数""" - } - } } diff --git a/rel/i18n/emqx_gateway_schema.hocon b/rel/i18n/emqx_gateway_schema.hocon index ebc955557..5f7d71913 100644 --- a/rel/i18n/emqx_gateway_schema.hocon +++ b/rel/i18n/emqx_gateway_schema.hocon @@ -1,600 +1,117 @@ emqx_gateway_schema { - stomp { - desc { - en: """The Stomp Gateway configuration. -This gateway supports v1.2/1.1/1.0""" - zh: """Stomp 网关配置。当前实现支持 v1.2/1.1/1.0 协议版本""" - } - } +dtls_listener_acceptors.desc: +"""Size of the acceptor pool.""" - stom_frame_max_headers { - desc { - en: """The maximum number of Header""" - zh: """允许的 Header 最大数量""" - } - } +dtls_listener_dtls_opts.desc: +"""DTLS socket options""" - stomp_frame_max_headers_length { - desc { - en: """The maximum string length of the Header Value""" - zh: """允许的 Header 字符串的最大长度""" - } - } +gateway_common_authentication.desc: +"""Default authentication configs for all the gateway listeners. For per-listener overrides see authentication + in listener configs""" - stom_frame_max_body_length { - desc { - en: """Maximum number of bytes of Body allowed per Stomp packet""" - zh: """允许的 Stomp 报文 Body 的最大字节数""" - } - } +gateway_common_clientinfo_override.desc: +"""ClientInfo override.""" - mqttsn { - desc { - en: """The MQTT-SN Gateway configuration. -This gateway only supports the v1.2 protocol""" - zh: """MQTT-SN 网关配置。当前实现仅支持 v1.2 版本""" - } - } +gateway_common_clientinfo_override_clientid.desc: +"""Template for overriding clientid.""" - mqttsn_gateway_id { - desc { - en: """MQTT-SN Gateway ID. -When the broadcast option is enabled, the gateway will broadcast ADVERTISE message with this value""" - zh: """MQTT-SN 网关 ID。 -当 broadcast 打开时，MQTT-SN 网关会使用该 ID 来广播 ADVERTISE 消息""" - } - } +gateway_common_clientinfo_override_password.desc: +"""Template for overriding password.""" - mqttsn_broadcast { - desc { - en: """Whether to periodically broadcast ADVERTISE messages""" - zh: """是否周期性广播 ADVERTISE 消息""" - } - } +gateway_common_clientinfo_override_username.desc: +"""Template for overriding username.""" - mqttsn_enable_qos3 { - desc { - en: """Allows connectionless clients to publish messages with a Qos of -1. -This feature is defined for very simple client implementations which do not support any other features except this one. There is no connection setup nor tear down, no registration nor subscription. The client just sends its 'PUBLISH' messages to a GW""" - zh: """是否允许无连接的客户端发送 QoS 等于 -1 的消息。 -该功能主要用于支持轻量的 MQTT-SN 客户端实现，它不会向网关建立连接，注册主题，也不会发起订阅；它只使用 QoS 为 -1 来发布消息""" - } - } +gateway_common_enable.desc: +"""Whether to enable this gateway""" - mqttsn_subs_resume { - desc { - en: """Whether to initiate all subscribed topic name registration messages to the client after the Session has been taken over by a new channel""" - zh: """在会话被重用后，网关是否主动向客户端注册对已订阅主题名称""" - } - } +gateway_common_enable_stats.desc: +"""Whether to enable client process statistic""" - mqttsn_predefined { - desc { - en: """The pre-defined topic IDs and topic names. -A 'pre-defined' topic ID is a topic ID whose mapping to a topic name is known in advance by both the client's application and the gateway""" - zh: """预定义主题列表。 -预定义的主题列表，是一组 主题 ID 和 主题名称 的映射关系。使用预先定义的主题列表，可以减少 MQTT-SN 客户端和网关对于固定主题的注册请求""" - } - } - - mqttsn_predefined_id { - desc { - en: """Topic ID. Range: 1-65535""" - zh: """主题 ID。范围：1-65535""" - } - } - - mqttsn_predefined_topic { - desc { - en: """Topic Name""" - zh: """主题名称。注：不支持通配符""" - } - } - - coap { - desc { - en: """The CoAP Gateway configuration. -This gateway is implemented based on RFC-7252 and https://core-wg.github.io/coap-pubsub/draft-ietf-core-pubsub.html""" - zh: """CoAP 网关配置。 -该网关的实现基于 RFC-7252 和 https://core-wg.github.io/coap-pubsub/draft-ietf-core-pubsub.html""" - } - } - - coap_heartbeat { - desc { - en: """The gateway server required minimum heartbeat interval. -When connection mode is enabled, this parameter is used to set the minimum heartbeat interval for the connection to be alive""" - zh: """CoAP 网关要求客户端的最小心跳间隔时间。 -当 connection_required 开启后，该参数用于检查客户端连接是否存活""" - } - } - - coap_connection_required { - desc { - en: """Enable or disable connection mode. -Connection mode is a feature of non-standard protocols. When connection mode is enabled, it is necessary to maintain the creation, authentication and alive of connection resources""" - zh: """是否开启连接模式。 -连接模式是非标准协议的功能。它维护 CoAP 客户端上线、认证、和连接状态的保持""" - } - } - - coap_notify_type { - desc { - en: """The Notification Message will be delivered to the CoAP client if a new message received on an observed topic. -The type of delivered coap message can be set to:
- - non: Non-confirmable;
- - con: Confirmable;
- - qos: Mapping from QoS type of received message, QoS0 -> non, QoS1,2 -> con""" - zh: """投递给 CoAP 客户端的通知消息类型。当客户端 Observe 一个资源（或订阅某个主题）时，网关会向客户端推送新产生的消息。其消息类型可设置为：
- - non: 不需要客户端返回确认消息;
- - con: 需要客户端返回一个确认消息;
- - qos: 取决于消息的 QoS 等级; QoS 0 会以 `non` 类型下发，QoS 1/2 会以 `con` 类型下发""" - } - } - - coap_subscribe_qos { - desc { - en: """The Default QoS Level indicator for subscribe request. -This option specifies the QoS level for the CoAP Client when establishing a subscription membership, if the subscribe request is not carried `qos` option. The indicator can be set to:
- - qos0, qos1, qos2: Fixed default QoS level
- - coap: Dynamic QoS level by the message type of subscribe request
- * qos0: If the subscribe request is non-confirmable
- * qos1: If the subscribe request is confirmable""" - - zh: """客户端订阅请求的默认 QoS 等级。 -当 CoAP 客户端发起订阅请求时，如果未携带 `qos` 参数则会使用该默认值。默认值可设置为：
- - qos0、 qos1、qos2: 设置为固定的 QoS 等级
- - coap: 依据订阅操作的 CoAP 报文类型来动态决定
- * 当订阅请求为 `non-confirmable` 类型时，取值为 qos0
- * 当订阅请求为 `confirmable` 类型时，取值为 qos1""" - } - } - - coap_publish_qos { - desc { - en: """The Default QoS Level indicator for publish request. -This option specifies the QoS level for the CoAP Client when publishing a message to EMQX PUB/SUB system, if the publish request is not carried `qos` option. The indicator can be set to:
- - qos0, qos1, qos2: Fixed default QoS level
- - coap: Dynamic QoS level by the message type of publish request
- * qos0: If the publish request is non-confirmable
- * qos1: If the publish request is confirmable""" - - zh: """客户端发布请求的默认 QoS 等级。 -当 CoAP 客户端发起发布请求时，如果未携带 `qos` 参数则会使用该默认值。默认值可设置为：
- - qos0、qos1、qos2: 设置为固定的 QoS 等级
- - coap: 依据发布操作的 CoAP 报文类型来动态决定
- * 当发布请求为 `non-confirmable` 类型时，取值为 qos0
- * 当发布请求为 `confirmable` 类型时，取值为 qos1""" - } - } - - lwm2m { - desc { - en: """The LwM2M Gateway configuration. This gateway only supports the v1.0.1 protocol.""" - zh: """LwM2M 网关配置。仅支持 v1.0.1 协议。""" - } - } - - lwm2m_xml_dir { - desc { - en: """The Directory for LwM2M Resource definition.""" - zh: """LwM2M Resource 定义的 XML 文件目录路径。""" - } - } - - lwm2m_lifetime_min { - desc { - en: """Minimum value of lifetime allowed to be set by the LwM2M client.""" - zh: """允许 LwM2M 客户端允许设置的心跳最小值。""" - } - } - - lwm2m_lifetime_max { - desc { - en: """Maximum value of lifetime allowed to be set by the LwM2M client.""" - zh: """允许 LwM2M 客户端允许设置的心跳最大值。""" - } - } - - lwm2m_qmode_time_window { - desc { - en: """The value of the time window during which the network link is considered valid by the LwM2M Gateway in QMode mode. -For example, after receiving an update message from a client, any messages within this time window are sent directly to the LwM2M client, and all messages beyond this time window are temporarily stored in memory.""" - - zh: """在QMode模式下，LwM2M网关认为网络链接有效的时间窗口的值。 -例如，在收到客户端的更新信息后，在这个时间窗口内的任何信息都会直接发送到LwM2M客户端，而超过这个时间窗口的所有信息都会暂时储存在内存中。""" - } - } - - lwm2m_auto_observe { - desc { - en: """Automatically observe the object list of REGISTER packet.""" - zh: """自动 Observe REGISTER 数据包的 Object 列表。""" - } - } - - lwm2m_update_msg_publish_condition { - desc { - en: """Policy for publishing UPDATE event message.
- - always: send update events as long as the UPDATE request is received.
- - contains_object_list: send update events only if the UPDATE request carries any Object List""" - zh: """发布UPDATE事件消息的策略。
- - always: 只要收到 UPDATE 请求，就发送更新事件。
- - contains_object_list: 仅当 UPDATE 请求携带 Object 列表时才发送更新事件。""" - } - } - - lwm2m_translators { - desc { - en: """Topic configuration for LwM2M's gateway publishing and subscription.""" - zh: """LwM2M 网关订阅/发布消息的主题映射配置。""" - } - } - - lwm2m_translators_command { - desc { - en: """The topic for receiving downstream commands. -For each new LwM2M client that succeeds in going online, the gateway creates a subscription relationship to receive downstream commands and send it to the LwM2M client""" - - zh: """下行命令主题。 -对于每个成功上线的新 LwM2M 客户端，网关会创建一个订阅关系来接收下行消息并将其发送给客户端。""" - } - } - - lwm2m_translators_response { - desc { - en: """The topic for gateway to publish the acknowledge events from LwM2M client""" - zh: """用于网关发布来自 LwM2M 客户端的确认事件的主题。""" - } - } - - lwm2m_translators_notify { - desc { - en: """The topic for gateway to publish the notify events from LwM2M client. -After succeed observe a resource of LwM2M client, Gateway will send the notify events via this topic, if the client reports any resource changes""" - - zh: """用于发布来自 LwM2M 客户端的通知事件的主题。 -在成功 Observe 到 LwM2M 客户端的资源后，如果客户端报告任何资源状态的变化，网关将通过该主题发送通知事件。""" - } - } - - lwm2m_translators_register { - desc { - en: """The topic for gateway to publish the register events from LwM2M client.""" - zh: """用于发布来自 LwM2M 客户端的注册事件的主题。""" - } - } - - lwm2m_translators_update { - desc { - en: """The topic for gateway to publish the update events from LwM2M client""" - zh: """用于发布来自LwM2M客户端的更新事件的主题。""" - } - } - - translator { - desc { - en: """MQTT topic that corresponds to a particular type of event.""" - zh: """配置某网关客户端对于发布消息或订阅的主题和 QoS 等级。""" - } - } - - translator_topic { - desc { - en: """Topic Name""" - zh: """主题名称""" - } - } - - translator_qos { - desc { - en: """QoS Level""" - zh: """QoS 等级""" - } - } - - exproto { - desc { - en: """The Extension Protocol configuration""" - zh: """ExProto 网关""" - } - } - - exproto_server { - desc { - en: """Configurations for starting the ConnectionAdapter service""" - zh: """配置 ExProto 网关需要启动的 ConnectionAdapter 服务。 -该服务用于提供客户端的认证、发布、订阅和数据下行等功能。""" - } - } - - exproto_grpc_server_bind { - desc { - en: """Listening address and port for the gRPC server.""" - zh: """服务监听地址和端口。""" - } - } - - exproto_grpc_server_ssl { - desc { - en: """SSL configuration for the gRPC server.""" - zh: """服务 SSL 配置。""" - } - } - - exproto_handler { - desc { - en: """Configurations for request to ConnectionHandler service""" - zh: """配置 ExProto 网关需要请求的 ConnectionHandler 服务地址。 -该服务用于给 ExProto 提供客户端的 Socket 事件处理、字节解码、订阅消息接收等功能。""" - } - } - - exproto_grpc_handler_address { - desc { - en: """gRPC server address.""" - zh: """对端 gRPC 服务器地址。""" - } - } - - exproto_grpc_handler_ssl { - desc { - en: """SSL configuration for the gRPC client.""" - zh: """gRPC 客户端的 SSL 配置。""" - } - } - - gateway_common_enable { - desc { - en: """Whether to enable this gateway""" - zh: """是否启用该网关""" - } - } - - gateway_common_enable_stats { - desc { - en: """Whether to enable client process statistic""" - zh: """是否开启客户端统计""" - } - } - - gateway_common_idle_timeout { - desc { - en: """The idle time of the client connection process. It has two purposes: +gateway_common_idle_timeout.desc: +"""The idle time of the client connection process. It has two purposes: 1. A newly created client process that does not receive any client requests after that time will be closed directly. 2. A running client process that does not receive any client requests after this time will go into hibernation to save resources.""" - zh: """客户端连接过程的空闲时间。该配置用于： - 1. 一个新创建的客户端进程如果在该时间间隔内没有收到任何客户端请求，将被直接关闭。 - 2. 一个正在运行的客户进程如果在这段时间后没有收到任何客户请求，将进入休眠状态以节省资源。""" - } - } - gateway_common_clientinfo_override { - desc { - en: """ClientInfo override.""" - zh: """ClientInfo 重写。""" - } - } +gateway_common_listener_access_rules.desc: +"""The access control rules for this listener. +See: https://github.com/emqtt/esockd#allowdeny""" - gateway_common_clientinfo_override_username { - desc { - en: """Template for overriding username.""" - zh: """username 重写模板""" - } - } - gateway_common_clientinfo_override_password { - desc { - en: """Template for overriding password.""" - zh: """password 重写模板""" - } - } - gateway_common_clientinfo_override_clientid { - desc { - en: """Template for overriding clientid.""" - zh: """clientid 重写模板""" - } - } +gateway_common_listener_bind.desc: +"""The IP address and port that the listener will bind.""" - gateway_common_authentication { - desc { - en: """Default authentication configs for all the gateway listeners. For per-listener overrides see authentication\n in listener configs""" - zh: """网关的认证器配置，对该网关下所以的监听器生效。如果每个监听器需要配置不同的认证器，需要配置监听器下的 authentication 字段。""" - } - } +gateway_common_listener_enable.desc: +"""Enable the listener.""" - tcp_udp_listeners { - desc { - en: """Settings for the listeners.""" - zh: """监听器配置。""" - } - } - - tcp_listeners { - desc { - en: """Settings for the TCP listeners.""" - zh: """配置 TCP 类型的监听器。""" - } - } - - udp_listeners { - desc { - en: """Settings for the UDP listeners.""" - zh: """配置 UDP 类型的监听器。""" - } - } - - listener_name_to_settings_map{ - desc { - en: """A map from listener names to listener settings.""" - zh: """从监听器名称到配置参数的映射。""" - } - } - - tcp_listener_acceptors { - desc { - en: """Size of the acceptor pool.""" - zh: """Acceptor 进程池大小。""" - } - } - - tcp_listener_tcp_opts{ - desc { - en: """Setting the TCP socket options.""" - zh: """TCP Socket 配置。""" - } - } - - tcp_listener_proxy_protocol { - desc { - en: """Enable the Proxy Protocol V1/2 if the EMQX cluster is deployed behind HAProxy or Nginx. -See: https://www.haproxy.com/blog/haproxy/proxy-protocol/""" - zh: """是否开启 Proxy Protocol V1/2。当 EMQX 集群部署在 HAProxy 或 Nginx 后需要获取客户端真实 IP 时常用到该选项。参考：https://www.haproxy.com/blog/haproxy/proxy-protocol/""" - } - } - - tcp_listener_proxy_protocol_timeout { - desc { - en: """Timeout for proxy protocol. -EMQX will close the TCP connection if proxy protocol packet is not received within the timeout.""" - zh: """接收 Proxy Protocol 报文头的超时时间。如果在超时内没有收到 Proxy Protocol 包，EMQX 将关闭 TCP 连接。""" - } - } - - ssl_listener_options { - desc { - en: """SSL Socket options.""" - zh: """SSL Socket 配置。""" - } - } - - udp_listener_udp_opts { - desc { - en: """Settings for the UDP sockets.""" - zh: """UDP Socket 配置。""" - } - } - - udp_listener_active_n { - desc { - en: """Specify the {active, N} option for the socket. -See: https://erlang.org/doc/man/inet.html#setopts-2""" - zh: """为 Socket 指定 {active, N} 选项。 -参见：https://erlang.org/doc/man/inet.html#setopts-2""" - } - } - - udp_listener_recbuf { - desc { - en: """Size of the kernel-space receive buffer for the socket.""" - zh: """Socket 在内核空间接收缓冲区的大小。""" - } - } - - udp_listener_sndbuf { - desc { - en: """Size of the kernel-space send buffer for the socket.""" - zh: """Socket 在内核空间发送缓冲区的大小。""" - } - } - - udp_listener_buffer { - desc { - en: """Size of the user-space buffer for the socket.""" - zh: """Socket 在用户空间的缓冲区大小。""" - } - } - - udp_listener_reuseaddr { - desc { - en: """Allow local reuse of port numbers.""" - zh: """允许重用本地处于 TIME_WAIT 的端口号。""" - } - } - - dtls_listener_acceptors { - desc { - en: """Size of the acceptor pool.""" - zh: """Acceptor 进程池大小。""" - } - } - - dtls_listener_dtls_opts { - desc { - en: """DTLS socket options""" - zh: """DTLS Socket 配置""" - } - - } - - gateway_common_listener_enable { - desc { - en: """Enable the listener.""" - zh: """是否启用该监听器。""" - } - } - - gateway_common_listener_bind { - desc { - en: """The IP address and port that the listener will bind.""" - zh: """监听器绑定的 IP 地址或端口。""" - } - } - - gateway_common_listener_max_connections { - desc { - en: """Maximum number of concurrent connections.""" - zh: """监听器支持的最大连接数。""" - } - } - - gateway_common_listener_max_conn_rate { - desc { - en: """Maximum connections per second.""" - zh: """监听器支持的最大连接速率。""" - } - } - - gateway_common_listener_enable_authn { - desc { - en: """Set true (default) to enable client authentication on this listener. +gateway_common_listener_enable_authn.desc: +"""Set true (default) to enable client authentication on this listener. When set to false clients will be allowed to connect without authentication.""" - zh: """配置 true （默认值）启用客户端进行身份认证。 -配置 false 时，将不对客户端做任何认证。""" - } - } - gateway_mountpoint { - desc { - en: """When publishing or subscribing, prefix all topics with a mountpoint string. +gateway_common_listener_max_conn_rate.desc: +"""Maximum connections per second.""" + +gateway_common_listener_max_connections.desc: +"""Maximum number of concurrent connections.""" + +gateway_mountpoint.desc: +"""When publishing or subscribing, prefix all topics with a mountpoint string. The prefixed string will be removed from the topic name when the message is delivered to the subscriber. The mountpoint is a way that users can use to implement isolation of message routing between different listeners. For example if a client A subscribes to `t` with `listeners.tcp.\.mountpoint` set to `some_tenant`, then the client actually subscribes to the topic `some_tenant/t`. Similarly, if another client B (connected to the same listener as the client A) sends a message to topic `t`, the message is routed to all the clients subscribed `some_tenant/t`, -so client A will receive the message, with topic name `t`. Set to `\"\"` to disable the feature. +so client A will receive the message, with topic name `t`. Set to `""` to disable the feature. Variables in mountpoint string:
- ${clientid}: clientid
- ${username}: username""" - zh: """发布或订阅时，在所有主题前增加前缀字符串。 -当消息投递给订阅者时，前缀字符串将从主题名称中删除。挂载点是用户可以用来实现不同监听器之间的消息路由隔离的一种方式。 -例如，如果客户端 A 在 `listeners.tcp.\.mountpoint` 设置为 `some_tenant` 的情况下订阅 `t`， -则客户端实际上订阅了 `some_tenant/t` 主题。 -类似地，如果另一个客户端 B（连接到与客户端 A 相同的侦听器）向主题 `t` 发送消息， -则该消息被路由到所有订阅了 `some_tenant/t` 的客户端，因此客户端 A 将收到该消息，带有 主题名称`t`。 设置为 `\"\"` 以禁用该功能。 -挂载点字符串中可用的变量：
- - ${clientid}：clientid
- - ${username}：用户名""" - } - } - gateway_common_listener_access_rules { - desc { - en: """The access control rules for this listener. -See: https://github.com/emqtt/esockd#allowdeny""" - zh: """配置监听器的访问控制规则。 -见：https://github.com/emqtt/esockd#allowdeny""" - } - } +listener_name_to_settings_map.desc: +"""A map from listener names to listener settings.""" + +ssl_listener_options.desc: +"""SSL Socket options.""" + +tcp_listener_acceptors.desc: +"""Size of the acceptor pool.""" + +tcp_listener_proxy_protocol.desc: +"""Enable the Proxy Protocol V1/2 if the EMQX cluster is deployed behind HAProxy or Nginx. +See: https://www.haproxy.com/blog/haproxy/proxy-protocol/""" + +tcp_listener_proxy_protocol_timeout.desc: +"""Timeout for proxy protocol. +EMQX will close the TCP connection if proxy protocol packet is not received within the timeout.""" + +tcp_listener_tcp_opts.desc: +"""Setting the TCP socket options.""" + +tcp_listeners.desc: +"""Settings for the TCP listeners.""" + +tcp_udp_listeners.desc: +"""Settings for the listeners.""" + +udp_listener_active_n.desc: +"""Specify the {active, N} option for the socket. +See: https://erlang.org/doc/man/inet.html#setopts-2""" + +udp_listener_buffer.desc: +"""Size of the user-space buffer for the socket.""" + +udp_listener_recbuf.desc: +"""Size of the kernel-space receive buffer for the socket.""" + +udp_listener_reuseaddr.desc: +"""Allow local reuse of port numbers.""" + +udp_listener_sndbuf.desc: +"""Size of the kernel-space send buffer for the socket.""" + +udp_listener_udp_opts.desc: +"""Settings for the UDP sockets.""" + +udp_listeners.desc: +"""Settings for the UDP listeners.""" + } diff --git a/rel/i18n/emqx_license_http_api.hocon b/rel/i18n/emqx_license_http_api.hocon index 40a18bbf3..895041c18 100644 --- a/rel/i18n/emqx_license_http_api.hocon +++ b/rel/i18n/emqx_license_http_api.hocon @@ -1,23 +1,15 @@ emqx_license_http_api { - desc_license_info_api { - desc { - en: "Get license info" - zh: "获取许可证信息" - } - label: { - en: "License info" - zh: "许可证信息" - } - } - desc_license_key_api { - desc { - en: "Update a license key" - zh: "更新一个许可证密钥" - } - label: { - en: "Update license" - zh: "更新许可证" - } - } +desc_license_info_api.desc: +"""Get license info""" + +desc_license_info_api.label: +"""License info""" + +desc_license_key_api.desc: +"""Update a license key""" + +desc_license_key_api.label: +"""Update license""" + } diff --git a/rel/i18n/emqx_license_schema.hocon b/rel/i18n/emqx_license_schema.hocon index c330f1cb2..3e4e37bff 100644 --- a/rel/i18n/emqx_license_schema.hocon +++ b/rel/i18n/emqx_license_schema.hocon @@ -1,55 +1,33 @@ emqx_license_schema { - license_root { - desc { - en: "Defines the EMQX Enterprise license. \n\n" - "\n" - "The default license has 100 connections limit, it is " - "issued on 2023-01-09 and valid for 5 years (1825 days).\n" - "\n" - "EMQX comes with a default trial license. For production use, please \n" - "visit https://www.emqx.com/apply-licenses/emqx to apply." - zh: "EMQX企业许可证。\n" - "EMQX 自带一个默认的试用许可证，" - "默认试用许可允许最多接入 100 个连接，签发时间是 2023年1月9日，有效期是 5 年（1825 天）。" - "若需要在生产环境部署，\n" - "请访问 https://www.emqx.com/apply-licenses/emqx 来申请。" - } - label { - en: "License" - zh: "许可证" - } - } - key_field { - desc { - en: "License string" - zh: "许可证字符串" - } - label { - en: "License string" - zh: "许可证字符串" - } - } +connection_high_watermark_field.desc: +"""High watermark limit above which license connection quota usage alarms are activated""" - connection_low_watermark_field { - desc { - en: "Low watermark limit below which license connection quota usage alarms are deactivated" - zh: "低水位限制，低于此水位线时系统会清除连接配额使用告警" - } - label { - en: "Connection low watermark" - zh: "连接低水位线" - } - } +connection_high_watermark_field.label: +"""Connection high watermark""" + +connection_low_watermark_field.desc: +"""Low watermark limit below which license connection quota usage alarms are deactivated""" + +connection_low_watermark_field.label: +"""Connection low watermark""" + +key_field.desc: +"""License string""" + +key_field.label: +"""License string""" + +license_root.desc: +"""Defines the EMQX Enterprise license. + + +The default license has 100 connections limit, it is issued on 2023-01-09 and valid for 5 years (1825 days). + +EMQX comes with a default trial license. For production use, please +visit https://www.emqx.com/apply-licenses/emqx to apply.""" + +license_root.label: +"""License""" - connection_high_watermark_field { - desc { - en: "High watermark limit above which license connection quota usage alarms are activated" - zh: "高水位线，连接数超过这个水位线时，系统会触发许可证连接配额使用告警" - } - label { - en: "Connection high watermark" - zh: "连接高水位" - } - } } diff --git a/rel/i18n/emqx_limiter_schema.hocon b/rel/i18n/emqx_limiter_schema.hocon index 3657df694..c99840375 100644 --- a/rel/i18n/emqx_limiter_schema.hocon +++ b/rel/i18n/emqx_limiter_schema.hocon @@ -1,190 +1,94 @@ emqx_limiter_schema { - failure_strategy { - desc { - en: """The strategy when all the retries failed.""" - zh: """当所有的重试都失败后的处理策略""" - } - label: { - en: """Failure Strategy""" - zh: """失败策略""" - } - } +bucket_cfg.desc: +"""Bucket Configs""" - max_retry_time { - desc { - en: """The maximum retry time when acquire failed.""" - zh: """申请失败后，尝试重新申请的时长最大值""" - } - label: { - en: """Max Retry Time""" - zh: """最大重试时间""" - } - } +bucket_cfg.label: +"""Buckets""" - divisible { - desc { - en: """Is it possible to split the number of requested tokens?""" - zh: """申请的令牌数是否可以被分割""" - } - label: { - en: """Divisible""" - zh: """是否可分割""" - } - } - - client_bucket_capacity { - desc { - en: """The capacity of per user.""" - zh: """每个使用者的令牌容量上限""" - } - label: { - en: """Capacity""" - zh: """容量""" - } - } - - capacity { - desc { - en: """The capacity of this token bucket.""" - zh: """该令牌桶的容量""" - } - label: { - en: """Capacity""" - zh: """容量""" - } - } - - low_watermark { - desc { - en: """If the remaining tokens are lower than this value, -the check/consume will succeed, but it will be forced to wait for a short period of time.""" - zh: """当桶中剩余的令牌数低于这个值，即使令牌申请成功了，也会被强制暂停一会儿""" - } - label: { - en: """Low Watermark""" - zh: """低水位线""" - } - } - - initial { - desc { - en: """The initial number of tokens for this bucket.""" - zh: """桶中的初始令牌数""" - } - label: { - en: """Initial""" - zh: """初始令牌数""" - } - } - - rate { - desc { - en: """Rate for this bucket.""" - zh: """桶的令牌生成速率""" - } - label: { - en: """Rate""" - zh: """速率""" - } - } - - client { - desc { - en: """The rate limit for each user of the bucket""" - zh: """对桶的每个使用者的速率控制设置""" - } - label: { - en: """Per Client""" - zh: """每个使用者的限制""" - } - } - - bucket_cfg { - desc { - en: """Bucket Configs""" - zh: """桶的配置""" - } - label: { - en: """Buckets""" - zh: """桶的配置""" - } - } - - burst { - desc { - en: """The burst, This value is based on rate.
+burst.desc: +"""The burst, This value is based on rate.
This value + rate = the maximum limit that can be achieved when limiter burst.""" - zh: """突发速率。 -突发速率允许短时间内速率超过设置的速率值，突发速率 + 速率 = 当前桶能达到的最大速率值""" - } - label: { - en: """Burst""" - zh: """突发速率""" - } - } - message_routing { - desc { - en: """The message routing limiter. -This is used to limit the forwarding rate for this EMQX node. -Once the limit is reached, new publish will be refused""" - zh: """消息派发速率控制器。 -这个用来控制当前节点内的消息派发速率，当达到最大速率后，新的推送将会被拒绝""" - } - label: { - en: """Message Routing""" - zh: """消息派发""" - } - } +burst.label: +"""Burst""" - connection { - desc { - en: """The connection limiter. -This is used to limit the connection rate for this EMQX node. -Once the limit is reached, new connections will be refused""" - zh: """连接速率控制器。 -这个用来控制当前节点上的连接速率，当达到最大速率后，新的连接将会被拒绝""" - } - label: { - en: """Connection""" - zh: """连接速率""" - } - } - - message_in { - desc { - en: """The message in limiter. -This is used to limit the inbound message numbers for this EMQX node -Once the limit is reached, the restricted client will be slow down even be hung for a while.""" - zh: """流入速率控制器。 -这个用来控制当前节点上的消息流入速率，当达到最大速率后，会话将会被限速甚至被强制挂起一小段时间""" - } - label: { - en: """Message In""" - zh: """消息流入速率""" - } - } - - bytes_in { - desc { - en: """The bytes_in limiter. +bytes.desc: +"""The `bytes` limiter. This is used to limit the inbound bytes rate for this EMQX node. Once the limit is reached, the restricted client will be slow down even be hung for a while.""" - zh: """流入字节率控制器。 -这个是用来控制当前节点上的数据流入的字节率，每条消息将会消耗和其二进制大小等量的令牌，当达到最大速率后，会话将会被限速甚至被强制挂起一小段时间""" - } - label: { - en: """Bytes In""" - zh: """流入字节率""" - } - } - internal { - desc { - en: """Limiter for EMQX internal app.""" - zh: """EMQX 内部功能所用限制器。""" +bytes.label: +"""Bytes""" + +client.desc: +"""The rate limit for each user of the bucket""" + +client.label: +"""Per Client""" + +connection.desc: +"""The connection limiter. +This is used to limit the connection rate for this EMQX node. +Once the limit is reached, new connections will be refused""" + +connection.label: +"""Connection""" + +divisible.desc: +"""Is it possible to split the number of requested tokens?""" + +divisible.label: +"""Divisible""" + +failure_strategy.desc: +"""The strategy when all the retries failed.""" + +failure_strategy.label: +"""Failure Strategy""" + +initial.desc: +"""The initial number of tokens for this bucket.""" + +initial.label: +"""Initial""" + +internal.desc: +"""Limiter for EMQX internal app.""" + +low_watermark.desc: +"""If the remaining tokens are lower than this value, +the check/consume will succeed, but it will be forced to wait for a short period of time.""" + +low_watermark.label: +"""Low Watermark""" + +max_retry_time.desc: +"""The maximum retry time when acquire failed.""" + +max_retry_time.label: +"""Max Retry Time""" + +message_routing.desc: +"""The message routing limiter. +This is used to limit the forwarding rate for this EMQX node. +Once the limit is reached, new publish will be refused""" + +message_routing.label: +"""Message Routing""" + +messages.desc: +"""The `messages` limiter. +This is used to limit the inbound message numbers for this EMQX node +Once the limit is reached, the restricted client will be slow down even be hung for a while.""" + +messages.label: +"""Messages""" + +rate.desc: +"""Rate for this bucket.""" + +rate.label: +"""Rate""" - } - } } diff --git a/rel/i18n/emqx_lwm2m_api.hocon b/rel/i18n/emqx_lwm2m_api.hocon index 9cd7e27c0..7ff1fdcce 100644 --- a/rel/i18n/emqx_lwm2m_api.hocon +++ b/rel/i18n/emqx_lwm2m_api.hocon @@ -1,58 +1,27 @@ emqx_lwm2m_api { - lookup_resource { - desc { - en: """Look up a resource""" - zh: """查看指定资源状态""" - } - } +dataType.desc: +"""Data Type""" - observe_resource { - desc { - en: """Observe or Cancel observe a resource""" - zh: """Observe/Un-Observe 指定资源""" - } - } +lookup_resource.desc: +"""Look up a resource""" - read_resource { - desc { - en: """Send a read command to a resource""" - zh: """发送读指令到某资源""" - } - } +name.desc: +"""Resource Name""" - write_resource { - desc { - en: """Send a write command to a resource""" - zh: """发送写指令到某资源""" - } - } +observe_resource.desc: +"""Observe or Cancel observe a resource""" - operations { - desc { - en: """Resource Operations""" - zh: """资源可用操作列表""" - } - } +operations.desc: +"""Resource Operations""" - dataType { - desc { - en: """Data Type""" - zh: """数据类型""" - } - } +path.desc: +"""Resource Path""" - path { - desc { - en: """Resource Path""" - zh: """资源路径""" - } - } +read_resource.desc: +"""Send a read command to a resource""" + +write_resource.desc: +"""Send a write command to a resource""" - name { - desc { - en: """Resource Name""" - zh: """资源名称""" - } - } } diff --git a/rel/i18n/emqx_lwm2m_schema.hocon b/rel/i18n/emqx_lwm2m_schema.hocon new file mode 100644 index 000000000..0193ce88f --- /dev/null +++ b/rel/i18n/emqx_lwm2m_schema.hocon @@ -0,0 +1,56 @@ +emqx_lwm2m_schema { + +lwm2m.desc: +"""The LwM2M Gateway configuration. This gateway only supports the v1.0.1 protocol.""" + +lwm2m_auto_observe.desc: +"""Automatically observe the object list of REGISTER packet.""" + +lwm2m_lifetime_max.desc: +"""Maximum value of lifetime allowed to be set by the LwM2M client.""" + +lwm2m_lifetime_min.desc: +"""Minimum value of lifetime allowed to be set by the LwM2M client.""" + +lwm2m_qmode_time_window.desc: +"""The value of the time window during which the network link is considered valid by the LwM2M Gateway in QMode mode. +For example, after receiving an update message from a client, any messages within this time window are sent directly to the LwM2M client, and all messages beyond this time window are temporarily stored in memory.""" + +lwm2m_translators.desc: +"""Topic configuration for LwM2M's gateway publishing and subscription.""" + +lwm2m_translators_command.desc: +"""The topic for receiving downstream commands. +For each new LwM2M client that succeeds in going online, the gateway creates a subscription relationship to receive downstream commands and send it to the LwM2M client""" + +lwm2m_translators_notify.desc: +"""The topic for gateway to publish the notify events from LwM2M client. +After succeed observe a resource of LwM2M client, Gateway will send the notify events via this topic, if the client reports any resource changes""" + +lwm2m_translators_register.desc: +"""The topic for gateway to publish the register events from LwM2M client.""" + +lwm2m_translators_response.desc: +"""The topic for gateway to publish the acknowledge events from LwM2M client""" + +lwm2m_translators_update.desc: +"""The topic for gateway to publish the update events from LwM2M client""" + +lwm2m_update_msg_publish_condition.desc: +"""Policy for publishing UPDATE event message.
+ - always: send update events as long as the UPDATE request is received.
+ - contains_object_list: send update events only if the UPDATE request carries any Object List""" + +lwm2m_xml_dir.desc: +"""The Directory for LwM2M Resource definition.""" + +translator.desc: +"""MQTT topic that corresponds to a particular type of event.""" + +translator_qos.desc: +"""QoS Level""" + +translator_topic.desc: +"""Topic Name""" + +} diff --git a/rel/i18n/emqx_mgmt_api_alarms.hocon b/rel/i18n/emqx_mgmt_api_alarms.hocon index ca8bf0769..0327fffcd 100644 --- a/rel/i18n/emqx_mgmt_api_alarms.hocon +++ b/rel/i18n/emqx_mgmt_api_alarms.hocon @@ -1,84 +1,39 @@ emqx_mgmt_api_alarms { - list_alarms_api { - desc { - en: """List currently activated alarms or historical alarms, determined by query parameters.""" - zh: """列出当前激活的告警或历史告警，由查询参数决定。""" - } - } +activate_at.desc: +"""Alarm start time, using rfc3339 standard time format.""" - delete_alarms_api { - desc { - en: """Remove all historical alarms.""" - zh: """删除所有历史告警。""" - } - } +deactivate_at.desc: +"""Alarm end time, using rfc3339 standard time format.""" - delete_alarms_api_response204 { - desc { - en: """Historical alarms have been cleared successfully.""" - zh: """历史告警已成功清除。""" - } - } +delete_alarms_api.desc: +"""Remove all historical alarms.""" - get_alarms_qs_activated { - desc { - en: """It is used to specify the alarm type of the query. +delete_alarms_api_response204.desc: +"""Historical alarms have been cleared successfully.""" + +details.desc: +"""Alarm details, provides more alarm information, mainly for program processing.""" + +duration.desc: +"""Indicates how long the alarm has been active in milliseconds.""" + +get_alarms_qs_activated.desc: +"""It is used to specify the alarm type of the query. When true, it returns the currently activated alarm, and when it is false, it returns the historical alarm. The default is false.""" - zh: """用于指定查询的告警类型， -为 true 时返回当前激活的告警，为 false 时返回历史告警，默认为 false。""" - } - } - node { - desc { - en: """The name of the node that triggered this alarm.""" - zh: """触发此告警的节点名称。""" - } - } +list_alarms_api.desc: +"""List currently activated alarms or historical alarms, determined by query parameters.""" - name { - desc { - en: """Alarm name, used to distinguish different alarms.""" - zh: """告警名称，用于区分不同的告警。""" - } - } +message.desc: +"""Alarm message, which describes the alarm content in a human-readable format.""" - message { - desc { - en: """Alarm message, which describes the alarm content in a human-readable format.""" - zh: """告警消息，以人类可读的方式描述告警内容。""" - } - } +name.desc: +"""Alarm name, used to distinguish different alarms.""" - details { - desc { - en: """Alarm details, provides more alarm information, mainly for program processing.""" - zh: """告警详情，提供了更多的告警信息，主要提供给程序处理。""" - } - } - - duration { - desc { - en: """Indicates how long the alarm has been active in milliseconds.""" - zh: """表明告警已经持续了多久，单位：毫秒。""" - } - } - - activate_at { - desc { - en: """Alarm start time, using rfc3339 standard time format.""" - zh: """告警开始时间，使用 rfc3339 标准时间格式。""" - } - } - - deactivate_at { - desc { - en: """Alarm end time, using rfc3339 standard time format.""" - zh: """告警结束时间，使用 rfc3339 标准时间格式。""" - } - } +node.desc: +"""The name of the node that triggered this alarm.""" } diff --git a/rel/i18n/emqx_mgmt_api_banned.hocon b/rel/i18n/emqx_mgmt_api_banned.hocon index b45a40ba6..1a9700641 100644 --- a/rel/i18n/emqx_mgmt_api_banned.hocon +++ b/rel/i18n/emqx_mgmt_api_banned.hocon @@ -1,98 +1,54 @@ emqx_mgmt_api_banned { - list_banned_api { - desc { - en: """List all currently banned client IDs, usernames and IP addresses.""" - zh: """列出目前所有被封禁的客户端 ID、用户名和 IP 地址。""" - } - } +as.desc: +"""Ban method, which can be client ID, username or IP address.""" - create_banned_api { - desc { - en: """Add a client ID, username or IP address to the blacklist.""" - zh: """添加一个客户端 ID、用户名或者 IP 地址到黑名单。""" - } - } +as.label: +"""Ban Method""" - create_banned_api_response400 { - desc { - en: """Bad request, possibly due to wrong parameters or the existence of a banned object.""" - zh: """错误的请求，可能是参数错误或封禁对象已存在等原因。""" - } - } +at.desc: +"""The start time of the ban, the format is rfc3339, the default is the time when the operation was initiated.""" - delete_banned_api { - desc { - en: """Remove a client ID, username or IP address from the blacklist.""" - zh: """将一个客户端 ID、用户名或者 IP 地址从黑名单中删除。""" - } - } +at.label: +"""Ban Time""" - delete_banned_api_response404 { - desc { - en: """The banned object was not found in the blacklist.""" - zh: """未在黑名单中找到该封禁对象。""" - } - } +by.desc: +"""Initiator of the ban.""" + +by.label: +"""Ban Initiator""" + +create_banned_api.desc: +"""Add a client ID, username or IP address to the blacklist.""" + +create_banned_api_response400.desc: +"""Bad request, possibly due to wrong parameters or the existence of a banned object.""" + +delete_banned_api.desc: +"""Remove a client ID, username or IP address from the blacklist.""" + +delete_banned_api_response404.desc: +"""The banned object was not found in the blacklist.""" + +list_banned_api.desc: +"""List all currently banned client IDs, usernames and IP addresses.""" + +reason.desc: +"""Ban reason, record the reason why the current object was banned.""" + +reason.label: +"""Ban Reason""" + +until.desc: +"""The end time of the ban, the format is rfc3339, the default is the time when the operation was initiated + 1 year.""" + +until.label: +"""Ban End Time""" + +who.desc: +"""Ban object, specific client ID, username or IP address.""" + +who.label: +"""Ban Object""" - as { - desc { - en: """Ban method, which can be client ID, username or IP address.""" - zh: """封禁方式，可以通过客户端 ID、用户名或者 IP 地址等方式进行封禁。""" - } - label { - en: """Ban Method""" - zh: """封禁方式""" - } - } - who { - desc { - en: """Ban object, specific client ID, username or IP address.""" - zh: """封禁对象，具体的客户端 ID、用户名或者 IP 地址。""" - } - label { - en: """Ban Object""" - zh: """封禁对象""" - } - } - by { - desc { - en: """Initiator of the ban.""" - zh: """封禁的发起者。""" - } - label { - en: """Ban Initiator""" - zh: """封禁发起者""" - } - } - reason { - desc { - en: """Ban reason, record the reason why the current object was banned.""" - zh: """封禁原因，记录当前对象被封禁的原因。""" - } - label { - en: """Ban Reason""" - zh: """封禁原因""" - } - } - at { - desc { - en: """The start time of the ban, the format is rfc3339, the default is the time when the operation was initiated.""" - zh: """封禁的起始时间，格式为 rfc3339，默认为发起操作的时间。""" - } - label { - en: """Ban Time""" - zh: """封禁时间""" - } - } - until { - desc { - en: """The end time of the ban, the format is rfc3339, the default is the time when the operation was initiated + 1 year.""" - zh: """封禁的结束时间，格式为 rfc3339，默认值为发起操作的时间 + 1 年。""" - } - label { - en: """Ban End Time""" - zh: """封禁结束时间""" - } - } } diff --git a/rel/i18n/emqx_mgmt_api_key_schema.hocon b/rel/i18n/emqx_mgmt_api_key_schema.hocon index f96c5c11b..0dc11c7ac 100644 --- a/rel/i18n/emqx_mgmt_api_key_schema.hocon +++ b/rel/i18n/emqx_mgmt_api_key_schema.hocon @@ -1,32 +1,20 @@ emqx_mgmt_api_key_schema { - api_key { - desc { - en: """API Key, can be used to request API other than the management API key and the Dashboard user management API""" - zh: """API 密钥， 可用于请求除管理 API 密钥及 Dashboard 用户管理 API 的其它接口""" - } - label { - en: "API Key" - zh: "API 密钥" - } - } - bootstrap_file { - desc { - en: """Bootstrap file is used to add an api_key when emqx is launched, + +api_key.desc: +"""API Key, can be used to request API other than the management API key and the Dashboard user management API""" + +api_key.label: +"""API Key""" + +bootstrap_file.desc: +"""Bootstrap file is used to add an api_key when emqx is launched, the format is: ``` 7e729ae70d23144b:2QILI9AcQ9BYlVqLDHQNWN2saIjBV4egr1CZneTNKr9CpK ec3907f865805db0:Ee3taYltUKtoBVD9C3XjQl9C6NXheip8Z9B69BpUv5JxVHL ```""" - zh: """用于在启动 emqx 时，添加 API 密钥，其格式为： - ``` - 7e729ae70d23144b:2QILI9AcQ9BYlVqLDHQNWN2saIjBV4egr1CZneTNKr9CpK - ec3907f865805db0:Ee3taYltUKtoBVD9C3XjQl9C6NXheip8Z9B69BpUv5JxVHL - ```""" - } - label { - en: "Initialize api_key file." - zh: "API 密钥初始化文件" - } - } +bootstrap_file.label: +"""Initialize api_key file.""" + } diff --git a/rel/i18n/emqx_mgmt_api_publish.hocon b/rel/i18n/emqx_mgmt_api_publish.hocon index a09732cfc..50589f8d7 100644 --- a/rel/i18n/emqx_mgmt_api_publish.hocon +++ b/rel/i18n/emqx_mgmt_api_publish.hocon @@ -1,27 +1,51 @@ emqx_mgmt_api_publish { - publish_api { - desc { - en: """Possible HTTP status response codes are:
+ +error_message.desc: +"""Describes the failure reason in detail.""" + +message_id.desc: +"""A globally unique message ID for correlation/tracing.""" + +message_properties.desc: +"""The Properties of the PUBLISH message.""" + +msg_content_type.desc: +"""The Content Type MUST be a UTF-8 Encoded String.""" + +msg_correlation_data.desc: +"""Identifier of the Correlation Data. The Server MUST send the Correlation Data unaltered to all subscribers receiving the Application Message.""" + +msg_message_expiry_interval.desc: +"""Identifier of the Message Expiry Interval. If the Message Expiry Interval has passed and the Server has not managed to start onward delivery to a matching subscriber, then it MUST delete the copy of the message for that subscriber.""" + +msg_payload_format_indicator.desc: +"""0 (0x00) Byte Indicates that the Payload is unspecified bytes, which is equivalent to not sending a Payload Format Indicator. +1 (0x01) Byte Indicates that the Payload is UTF-8 Encoded Character Data. The UTF-8 data in the Payload MUST be well-formed UTF-8 as defined by the Unicode specification and restated in RFC 3629.""" + +msg_response_topic.desc: +"""Identifier of the Response Topic.The Response Topic MUST be a UTF-8 Encoded, It MUST NOT contain wildcard characters.""" + +msg_user_properties.desc: +"""The User-Property key-value pairs. Note: in case there are duplicated keys, only the last one will be used.""" + +payload.desc: +"""The MQTT message payload.""" + +payload_encoding.desc: +"""MQTT Payload Encoding, base64 or plain. When set to base64, the message is decoded before it is published.""" + +publish_api.desc: +"""Possible HTTP status response codes are:
200: The message is delivered to at least one subscriber;
202: No matched subscribers;
400: Message is invalid. for example bad topic name, or QoS is out of range;
503: Failed to deliver the message to subscriber(s)""" - zh: """发布一个消息。
-可能的 HTTP 状态码如下：
-200: 消息被成功发送到至少一个订阅。
-202: 没有匹配到任何订阅。
-400: 消息编码错误，如非法主题，或 QoS 超出范围等。
-503: 服务重启等过程中导致转发失败。""" - } - label { - en: "Publish a message" - zh: "发布一条信息" - } - } - publish_bulk_api { - desc { - en: """Possible HTTP response status code are:
+publish_api.label: +"""Publish a message""" + +publish_bulk_api.desc: +"""Possible HTTP response status code are:
200: All messages are delivered to at least one subscriber;
202: At least one message was not delivered to any subscriber;
400: At least one message is invalid. For example bad topic name, or QoS is out of range;
@@ -31,62 +55,15 @@ In case there is at lest one invalid message in the batch, the HTTP response bod is the same as for /publish API.
Otherwise the HTTP response body is an array of JSON objects indicating the publish result of each individual message in the batch.""" - zh: """批量发布一组消息。
-可能的 HTTP 状态码如下：
-200: 所有的消息都被成功发送到至少一个订阅。
-202: 至少有一个消息没有匹配到任何订阅。
-400: 至少有一个消息编码错误，如非法主题，或 QoS 超出范围等。
-503: 至少有一个小因为服务重启的原因导致转发失败。
-请求的 Body 或者 Body 中包含的某个消息无法通过 API 规范的类型检查时，HTTP 响应的消息与发布单个消息的 API - /publish 是一样的。 -如果所有的消息都是合法的，那么 HTTP 返回的内容是一个 JSON 数组，每个元素代表了该消息转发的状态。""" - } - label { - en: "Publish a batch of messages" - zh: "发布一批信息" - } - } +publish_bulk_api.label: +"""Publish a batch of messages""" - topic_name { - desc { - en: "Topic Name" - zh: "主题名称" - } - } - qos { - desc { - en: "MQTT message QoS" - zh: "MQTT 消息的 QoS" - } - } - payload { - desc { - en: "The MQTT message payload." - zh: "MQTT 消息体。" - } - } - retain { - desc { - en: "A boolean field to indicate if this message should be retained." - zh: "布尔型字段，用于表示该消息是否保留消息。" - } - } - payload_encoding { - desc { - en: "MQTT Payload Encoding, base64 or plain. When set to base64, the message is decoded before it is published." - zh: "MQTT 消息体的编码方式，可以是 base64 或 plain。当设置为 base64 时，消息在发布前会先被解码。" - } - } - message_id { - desc { - en: "A globally unique message ID for correlation/tracing." - zh: "全局唯一的一个消息 ID，方便用于关联和追踪。" - } - } - reason_code { - desc { - en: """The MQTT reason code, as the same ones used in PUBACK packet.
+qos.desc: +"""MQTT message QoS""" + +reason_code.desc: +"""The MQTT reason code, as the same ones used in PUBACK packet.
Currently supported codes are:
16(0x10): No matching subscribers;
@@ -95,63 +72,11 @@ Currently supported codes are:
151(0x97): Publish rate limited, or message size exceeded limit. The global size limit can be configured with mqtt.max_packet_size
NOTE: The message size is estimated with the received topic and payload size, meaning the actual size of serialized bytes (when sent to MQTT subscriber) might be slightly over the limit.""" - zh: """MQTT 消息发布的错误码，这些错误码也是 MQTT 规范中 PUBACK 消息可能携带的错误码。
-当前支持如下错误码：
-16(0x10)：没能匹配到任何订阅；
-131(0x81)：消息转发时发生错误，例如 EMQX 服务重启；
-144(0x90)：主题名称非法；
-151(0x97)：受到了速率限制，或者消息尺寸过大。全局消息大小限制可以通过配置项 mqtt.max_packet_size 来进行修改。
-注意：消息尺寸的是通过主题和消息体的字节数进行估算的。具体发布时所占用的字节数可能会稍大于这个估算的值。""" - } - } - error_message { - desc { - en: "Describes the failure reason in detail." - zh: "失败的详细原因。" - } - } - message_properties { - desc { - en: "The Properties of the PUBLISH message." - zh: "PUBLISH 消息里的 Property 字段。" - } - } - msg_payload_format_indicator { - desc { - en: """0 (0x00) Byte Indicates that the Payload is unspecified bytes, which is equivalent to not sending a Payload Format Indicator. -1 (0x01) Byte Indicates that the Payload is UTF-8 Encoded Character Data. The UTF-8 data in the Payload MUST be well-formed UTF-8 as defined by the Unicode specification and restated in RFC 3629.""" - zh: "载荷格式指示标识符，0 表示载荷是未指定格式的数据，相当于没有发送载荷格式指示；1 表示载荷是 UTF-8 编码的字符数据，载荷中的 UTF-8 数据必须是按照 Unicode 的规范和 RFC 3629 的标准要求进行编码的。" - } - } - msg_message_expiry_interval { - desc { - en: "Identifier of the Message Expiry Interval. If the Message Expiry Interval has passed and the Server has not managed to start onward delivery to a matching subscriber, then it MUST delete the copy of the message for that subscriber." - zh: "消息过期间隔标识符，以秒为单位。当消失已经过期时，如果服务端还没有开始向匹配的订阅者投递该消息，则服务端会删除该订阅者的消息副本。如果不设置，则消息永远不会过期" - } - } - msg_response_topic { - desc { - en: "Identifier of the Response Topic.The Response Topic MUST be a UTF-8 Encoded, It MUST NOT contain wildcard characters." - zh: "响应主题标识符， UTF-8 编码的字符串，用作响应消息的主题名。响应主题不能包含通配符，也不能包含多个主题，否则将造成协议错误。当存在响应主题时，消息将被视作请求报文。服务端在收到应用消息时必须将响应主题原封不动的发送给所有的订阅者。" - } - } - msg_correlation_data { - desc { - en: "Identifier of the Correlation Data. The Server MUST send the Correlation Data unaltered to all subscribers receiving the Application Message." - zh: "对比数据标识符，服务端在收到应用消息时必须原封不动的把对比数据发送给所有的订阅者。对比数据只对请求消息(Request Message)的发送端和响应消息(Response Message)的接收端有意义。" - } - } - msg_user_properties { - desc { - en: "The User-Property key-value pairs. Note: in case there are duplicated keys, only the last one will be used." - zh: "指定 MQTT 消息的 User Property 键值对。注意，如果出现重复的键，只有最后一个会保留。" - } - } - msg_content_type { - desc { - en: "The Content Type MUST be a UTF-8 Encoded String." - zh: "内容类型标识符，以 UTF-8 格式编码的字符串，用来描述应用消息的内容，服务端必须把收到的应用消息中的内容类型原封不动的发送给所有的订阅者。" - } - } +retain.desc: +"""A boolean field to indicate if this message should be retained.""" + +topic_name.desc: +"""Topic Name""" + } diff --git a/rel/i18n/emqx_mgmt_api_status.hocon b/rel/i18n/emqx_mgmt_api_status.hocon index d72fd0998..28278b747 100644 --- a/rel/i18n/emqx_mgmt_api_status.hocon +++ b/rel/i18n/emqx_mgmt_api_status.hocon @@ -1,48 +1,21 @@ emqx_mgmt_api_status { - get_status_api { - desc { - en: "Serves as a health check for the node. Returns a plain text response" - " describing the status of the node. This endpoint requires no" - " authentication.\n" - "\n" - "Returns status code 200 if the EMQX application is up and running, " - "503 otherwise." - "\n" - "This API was introduced in v5.0.10." - "\n" - "The GET `/status` endpoint (without the `/api/...` prefix) is also an alias" - " to this endpoint and works in the same way. This alias has been available since" - " v5.0.0." - zh: "作为节点的健康检查。 返回一个纯文本的响应，描述节点的状态。\n" - "\n" - "如果 EMQX 应用程序已经启动并运行，返回状态代码 200，否则返回 503。\n" - "\n" - "这个API是在v5.0.10中引入的。" - "\n" - "GET `/status`端点（没有`/api/...`前缀）也是这个端点的一个别名，工作方式相同。" - " 这个别名从v5.0.0开始就有了。" - } - label { - en: "Service health check" - zh: "服务健康检查" - } - } - get_status_response200 { - desc { - en: "Node emqx@127.0.0.1 is started\n" - "emqx is running" - zh: "Node emqx@127.0.0.1 is started\n" - "emqx is running" - } - } +get_status_api.desc: +"""Serves as a health check for the node. Returns a plain text response describing the status of the node. This endpoint requires no authentication. + +Returns status code 200 if the EMQX application is up and running, 503 otherwise. +This API was introduced in v5.0.10. +The GET `/status` endpoint (without the `/api/...` prefix) is also an alias to this endpoint and works in the same way. This alias has been available since v5.0.0.""" + +get_status_api.label: +"""Service health check""" + +get_status_response200.desc: +"""Node emqx@127.0.0.1 is started +emqx is running""" + +get_status_response503.desc: +"""Node emqx@127.0.0.1 is stopped +emqx is not_running""" - get_status_response503 { - desc { - en: "Node emqx@127.0.0.1 is stopped\n" - "emqx is not_running" - zh: "Node emqx@127.0.0.1 is stopped\n" - "emqx is not_running" - } - } } diff --git a/rel/i18n/emqx_modules_schema.hocon b/rel/i18n/emqx_modules_schema.hocon index 248f85341..25588b4e2 100644 --- a/rel/i18n/emqx_modules_schema.hocon +++ b/rel/i18n/emqx_modules_schema.hocon @@ -1,8 +1,13 @@ emqx_modules_schema { - rewrite { - desc { - en: """The topic rewriting function of EMQX supports rewriting topic A to topic B when the client subscribes to topics, publishes messages, and cancels subscriptions according to user-configured rules. +enable.desc: +"""Enable this feature""" + +max_delayed_messages.desc: +"""Maximum number of delayed messages (0 is no limit).""" + +rewrite.desc: +"""The topic rewriting function of EMQX supports rewriting topic A to topic B when the client subscribes to topics, publishes messages, and cancels subscriptions according to user-configured rules. Each rewrite rule consists of three parts: subject filter, regular expression, and target expression. Under the premise that the subject rewriting function is enabled, when EMQX receives a subject-based MQTT message such as a `PUBLISH` message, it will use the subject of the message to sequentially match the subject filter part of the rule in the configuration file. If the match is successful, @@ -13,78 +18,32 @@ It should be noted that EMQX uses reverse order to read the rewrite rules in the When a topic can match the topic filter of multiple topic rewrite rules at the same time, EMQX will only use the first rule it matches. Rewrite. If the regular expression in this rule does not match the subject of the MQTT message, the rewriting will fail, and no other rules will be attempted for rewriting. Therefore, users need to carefully design MQTT message topics and topic rewriting rules when using them.""" - zh: """EMQX 的主题重写功能支持根据用户配置的规则在客户端订阅主题、发布消息、取消订阅的时候将 A 主题重写为 B 主题。 -重写规则分为 Pub 规则和 Sub 规则，Pub 规则匹配 PUSHLISH 报文携带的主题，Sub 规则匹配 SUBSCRIBE、UNSUBSCRIBE 报文携带的主题。 -每条重写规则都由主题过滤器、正则表达式、目标表达式三部分组成。 -在主题重写功能开启的前提下，EMQX 在收到诸如 PUBLISH 报文等带有主题的 MQTT 报文时，将使用报文中的主题去依次匹配配置文件中规则的主题过滤器部分，一旦成功匹配，则使用正则表达式提取主题中的信息，然后替换至目标表达式以构成新的主题。 -目标表达式中可以使用 `$N` 这种格式的变量匹配正则表达中提取出来的元素，`$N` 的值为正则表达式中提取出来的第 N 个元素，比如 `$1` 即为正则表达式提取的第一个元素。 -需要注意的是，EMQX 使用倒序读取配置文件中的重写规则，当一条主题可以同时匹配多条主题重写规则的主题过滤器时，EMQX 仅会使用它匹配到的第一条规则进行重写，如果该条规则中的正则表达式与 MQTT 报文主题不匹配，则重写失败，不会再尝试使用其他的规则进行重写。 -因此用户在使用时需要谨慎的设计 MQTT 报文主题以及主题重写规则。""" - } - label { - en: """Topic Rewrite""" - zh: """主题重写""" - } - } - tr_source_topic { - desc { - en: """Source topic, specified by the client.""" - zh: """源主题，客户端业务指定的主题""" - } - label { - en: """Source Topic""" - zh: """源主题""" - } - } +rewrite.label: +"""Topic Rewrite""" - tr_action { - desc { - en: """Topic rewriting takes effect on the type of operation: +tr_action.desc: +"""Topic rewriting takes effect on the type of operation: - `subscribe`: Rewrite topic when client do subscribe. - `publish`: Rewrite topic when client do publish. - `all`: Both""" - zh: """主题重写在哪种操作上生效： - - `subscribe`：订阅时重写主题； - - `publish`：发布时重写主题； - -`all`：全部重写主题""" - } - label { - en: """Action""" - zh: """Action""" - } - } +tr_action.label: +"""Action""" - tr_re { - desc { - en: """Regular expressions""" - zh: """正则表达式""" - } - } +tr_dest_topic.desc: +"""Destination topic.""" - tr_dest_topic { - desc { - en: """Destination topic.""" - zh: """目标主题。""" - } - label { - en: """Destination Topic""" - zh: """目标主题""" - } - } +tr_dest_topic.label: +"""Destination Topic""" - enable { - desc { - en: "Enable this feature" - zh: "是否开启该功能" - } - } +tr_re.desc: +"""Regular expressions""" + +tr_source_topic.desc: +"""Source topic, specified by the client.""" + +tr_source_topic.label: +"""Source Topic""" - max_delayed_messages { - desc { - en: "Maximum number of delayed messages (0 is no limit)." - zh: "延迟消息的数量上限(0 代表无限)" - } - } } diff --git a/rel/i18n/emqx_mqttsn_schema.hocon b/rel/i18n/emqx_mqttsn_schema.hocon new file mode 100644 index 000000000..1541a8c78 --- /dev/null +++ b/rel/i18n/emqx_mqttsn_schema.hocon @@ -0,0 +1,31 @@ +emqx_mqttsn_schema { + +mqttsn.desc: +"""The MQTT-SN Gateway configuration. +This gateway only supports the v1.2 protocol""" + +mqttsn_broadcast.desc: +"""Whether to periodically broadcast ADVERTISE messages""" + +mqttsn_enable_qos3.desc: +"""Allows connectionless clients to publish messages with a Qos of -1. +This feature is defined for very simple client implementations which do not support any other features except this one. There is no connection setup nor tear down, no registration nor subscription. The client just sends its 'PUBLISH' messages to a GW""" + +mqttsn_gateway_id.desc: +"""MQTT-SN Gateway ID. +When the broadcast option is enabled, the gateway will broadcast ADVERTISE message with this value""" + +mqttsn_predefined.desc: +"""The pre-defined topic IDs and topic names. +A 'pre-defined' topic ID is a topic ID whose mapping to a topic name is known in advance by both the client's application and the gateway""" + +mqttsn_predefined_id.desc: +"""Topic ID. Range: 1-65535""" + +mqttsn_predefined_topic.desc: +"""Topic Name""" + +mqttsn_subs_resume.desc: +"""Whether to initiate all subscribed topic name registration messages to the client after the Session has been taken over by a new channel""" + +} diff --git a/rel/i18n/emqx_plugins_schema.hocon b/rel/i18n/emqx_plugins_schema.hocon index c3fed1884..b72c87054 100644 --- a/rel/i18n/emqx_plugins_schema.hocon +++ b/rel/i18n/emqx_plugins_schema.hocon @@ -1,85 +1,55 @@ emqx_plugins_schema { - plugins { - desc { - en: """Manage EMQX plugins.
-Plugins can be pre-built as a part of EMQX package, -or installed as a standalone package in a location specified by -install_dir config key
-The standalone-installed plugins are referred to as 'external' plugins.""" - zh: """管理EMQX插件。
-插件可以是EMQX安装包中的一部分，也可以是一个独立的安装包。
-独立安装的插件称为“外部插件”。""" - } - label { - en: "Plugins" - zh: "插件" - } - } - state { - desc { - en: "A per-plugin config to describe the desired state of the plugin." - zh: "描述插件的状态" - } - label { - en: "State" - zh: "插件状态" - } - } - name_vsn { - desc { - en: """The {name}-{version} of the plugin.
-It should match the plugin application name-version as the for the plugin release package name
-For example: my_plugin-0.1.0.""" - zh: """插件的名称{name}-{version}。
-它应该与插件的发布包名称一致，如my_plugin-0.1.0。""" - } - label { - en: "Name-Version" - zh: "名称-版本" - } - } - enable { - desc { - en: "Set to 'true' to enable this plugin" - zh: "设置为“true”以启用此插件" - } - label { - en: "Enable" - zh: "启用" - } - } - states { - desc { - en: """An array of plugins in the desired states.
-The plugins are started in the defined order""" - zh: """一组插件的状态。插件将按照定义的顺序启动""" - } - label { - en: "States" - zh: "插件启动顺序及状态" - } - } - install_dir { - desc { - en: """The installation directory for the external plugins. + +check_interval.desc: +"""Check interval: check if the status of the plugins in the cluster is consistent,
+if the results of 3 consecutive checks are not consistent, then alarm.""" + +enable.desc: +"""Set to 'true' to enable this plugin""" + +enable.label: +"""Enable""" + +install_dir.desc: +"""The installation directory for the external plugins. The plugin beam files and configuration files should reside in the subdirectory named as emqx_foo_bar-0.1.0.
NOTE: For security reasons, this directory should **NOT** be writable by anyone except emqx (or any user which runs EMQX).""" - zh: "插件安装包的目录，出于安全考虑，该目录应该值允许 emqx，或用于运行 EMQX 服务的用户拥有写入权限。" - } - label { - en: "Install Directory" - zh: "安装目录" - } - } - check_interval { - desc { - en: """Check interval: check if the status of the plugins in the cluster is consistent,
-if the results of 3 consecutive checks are not consistent, then alarm.""" - zh: """检查间隔：检查集群中插件的状态是否一致，
-如果连续3次检查结果不一致，则报警。""" - } - } + +install_dir.label: +"""Install Directory""" + +name_vsn.desc: +"""The {name}-{version} of the plugin.
+It should match the plugin application name-version as the for the plugin release package name
+For example: my_plugin-0.1.0.""" + +name_vsn.label: +"""Name-Version""" + +plugins.desc: +"""Manage EMQX plugins.
+Plugins can be pre-built as a part of EMQX package, +or installed as a standalone package in a location specified by +install_dir config key
+The standalone-installed plugins are referred to as 'external' plugins.""" + +plugins.label: +"""Plugins""" + +state.desc: +"""A per-plugin config to describe the desired state of the plugin.""" + +state.label: +"""State""" + +states.desc: +"""An array of plugins in the desired states.
+The plugins are started in the defined order""" + +states.label: +"""States""" + } diff --git a/rel/i18n/emqx_prometheus_schema.hocon b/rel/i18n/emqx_prometheus_schema.hocon index e9a0fc11e..d79685a4d 100644 --- a/rel/i18n/emqx_prometheus_schema.hocon +++ b/rel/i18n/emqx_prometheus_schema.hocon @@ -1,95 +1,47 @@ emqx_prometheus_schema { - prometheus { - desc { - en: """Settings for reporting metrics to Prometheus""" - zh: """Prometheus 监控数据推送""" - } - label { - en: """Prometheus""" - zh: """Prometheus""" - } - } +enable.desc: +"""Turn Prometheus data pushing on or off""" - push_gateway_server { - desc { - en: """URL of Prometheus server""" - zh: """Prometheus 服务器地址""" - } - } - - interval { - desc { - en: """Data reporting interval""" - zh: """数据推送间隔""" - } - } - - headers { - desc { - en: """A list of HTTP Headers when pushing to Push Gateway.
+headers.desc: +"""A list of HTTP Headers when pushing to Push Gateway.
For example, { Authorization = "some-authz-tokens"}""" - zh: """推送到 Push Gateway 的 HTTP Headers 列表。
-例如， { Authorization = "some-authz-tokens"}""" - } - } - job_name { - desc { - en: """Job Name that is pushed to the Push Gateway. Available variables:
+interval.desc: +"""Data reporting interval""" + +job_name.desc: +"""Job Name that is pushed to the Push Gateway. Available variables:
- ${name}: Name of EMQX node.
- ${host}: Host name of EMQX node.
For example, when the EMQX node name is emqx@127.0.0.1 then the name variable takes value emqx and the host variable takes value 127.0.0.1.
Default value is: ${name}/instance/${name}~${host}""" - zh: """推送到 Push Gateway 的 Job 名称。可用变量为：
-- ${name}: EMQX 节点的名称。 -- ${host}: EMQX 节点主机名。 -例如，当 EMQX 节点名为 emqx@127.0.0.1 则 name 变量的值为 emqx，host 变量的值为 127.0.0.1。
-默认值为: ${name}/instance/${name}~${host}""" - } - } - enable { - desc { - en: """Turn Prometheus data pushing on or off""" - zh: """开启或关闭 Prometheus 数据推送""" - } - } - vm_dist_collector { - desc { - en: """Enable or disable VM distribution collector, collects information about the sockets and processes involved in the Erlang distribution mechanism.""" - zh: """开启或关闭 VM 分布采集器，收集 Erlang 分布机制中涉及的套接字和进程的信息。""" - } - } - mnesia_collector { - desc { - en: """Enable or disable Mnesia collector, collects Mnesia metrics mainly using mnesia:system_info/1 .""" - zh: """开启或关闭 Mnesia 采集器, 使用 mnesia:system_info/1 收集 Mnesia 相关指标""" - } - } - vm_statistics_collector { - desc { - en: """Enable or disable VM statistics collector, collects Erlang VM metrics using erlang:statistics/1 .""" - zh: """开启或关闭 VM 统计采集器, 使用 erlang:statistics/1 收集 Erlang VM 相关指标""" - } - } +mnesia_collector.desc: +"""Enable or disable Mnesia collector, collects Mnesia metrics mainly using mnesia:system_info/1 .""" + +prometheus.desc: +"""Settings for reporting metrics to Prometheus""" + +prometheus.label: +"""Prometheus""" + +push_gateway_server.desc: +"""URL of Prometheus server""" + +vm_dist_collector.desc: +"""Enable or disable VM distribution collector, collects information about the sockets and processes involved in the Erlang distribution mechanism.""" + +vm_memory_collector.desc: +"""Enable or disable VM memory collector, collects information about memory dynamically allocated by the Erlang emulator using erlang:memory/0 , also provides basic (D)ETS statistics .""" + +vm_msacc_collector.desc: +"""Enable or disable VM msacc collector, collects microstate accounting metrics using erlang:statistics(microstate_accounting) .""" + +vm_statistics_collector.desc: +"""Enable or disable VM statistics collector, collects Erlang VM metrics using erlang:statistics/1 .""" + +vm_system_info_collector.desc: +"""Enable or disable VM system info collector, collects Erlang VM metrics using erlang:system_info/1 .""" - vm_system_info_collector { - desc { - en: """Enable or disable VM system info collector, collects Erlang VM metrics using erlang:system_info/1 .""" - zh: """开启或关闭 VM 系统信息采集器, 使用 erlang:system_info/1 收集 Erlang VM 相关指标""" - } - } - vm_memory_collector { - desc { - en: """Enable or disable VM memory collector, collects information about memory dynamically allocated by the Erlang emulator using erlang:memory/0 , also provides basic (D)ETS statistics .""" - zh: """开启或关闭 VM 内存采集器, 使用 erlang:memory/0 收集 Erlang 虚拟机动态分配的内存信息，同时提供基本的 (D)ETS 统计信息""" - } - } - vm_msacc_collector { - desc { - en: """Enable or disable VM msacc collector, collects microstate accounting metrics using erlang:statistics(microstate_accounting) .""" - zh: """开启或关闭 VM msacc 采集器, 使用 erlang:statistics(microstate_accounting) 收集微状态计数指标""" - } - } } diff --git a/rel/i18n/emqx_psk_schema.hocon b/rel/i18n/emqx_psk_schema.hocon index 60d45977a..1a99e1b19 100644 --- a/rel/i18n/emqx_psk_schema.hocon +++ b/rel/i18n/emqx_psk_schema.hocon @@ -1,8 +1,18 @@ emqx_psk_schema { - psk_authentication { - desc { - en: """PSK stands for 'Pre-Shared Keys'. +chunk_size.desc: +"""The size of each chunk used to import to the built-in database from PSK file""" + +enable.desc: +"""Whether to enable TLS PSK support""" + +init_file.desc: +"""If init_file is specified, EMQX will import PSKs from the file into the built-in database at startup for use by the runtime. +The file has to be structured line-by-line, each line must be in the format of PSKIdentity:SharedSecret. +For example: mydevice1:c2VjcmV0""" + +psk_authentication.desc: +"""PSK stands for 'Pre-Shared Keys'. This config to enable TLS-PSK authentication. Important! Make sure the SSL listener with only tlsv1.2 enabled, and also PSK cipher suites @@ -11,49 +21,8 @@ configured, such as RSA-PSK-AES256-GCM-SHA384. See listener SSL options config for more details. The IDs and secrets can be provided from a file which is configurable by the init_file field.""" - zh: """此配置用于启用 TLS-PSK 身份验证。 -PSK 是 “Pre-Shared-Keys” 的缩写。 - -注意: 确保 SSL 监听器仅启用了 'tlsv1.2'，并且配置了PSK 密码套件，例如 'RSA-PSK-AES256-GCM-SHA384'。 - -可以通过查看监听器中的 SSL 选项，了解更多详细信息。 - -可以通过配置 'init_file' 来设置初始化的 ID 和 密钥""" - } - } - - enable { - desc { - en: "Whether to enable TLS PSK support" - zh: "是否开启 TLS PSK 支持" - } - } - - init_file { - desc { - en: """If init_file is specified, EMQX will import PSKs from the file into the built-in database at startup for use by the runtime. -The file has to be structured line-by-line, each line must be in the format of PSKIdentity:SharedSecret. -For example: mydevice1:c2VjcmV0""" - zh: """如果设置了初始化文件，EMQX 将在启动时从初始化文件中导入 PSK 信息到内建数据库中。 -这个文件需要按行进行组织，每一行必须遵守如下格式: PSKIdentity:SharedSecret -例如: mydevice1:c2VjcmV0""" - } - } - - separator { - desc { - en: "The separator between PSKIdentity and SharedSecret in the PSK file" - - zh: "PSK 文件中 PSKIdentity 和 SharedSecret 之间的分隔符" - } - } - - chunk_size { - desc { - en: "The size of each chunk used to import to the built-in database from PSK file" - zh: "将 PSK 文件导入到内建数据时每个块的大小" - } - } +separator.desc: +"""The separator between PSKIdentity and SharedSecret in the PSK file""" } diff --git a/rel/i18n/emqx_resource_schema.hocon b/rel/i18n/emqx_resource_schema.hocon index c73f8b1aa..8fc781794 100644 --- a/rel/i18n/emqx_resource_schema.hocon +++ b/rel/i18n/emqx_resource_schema.hocon @@ -1,183 +1,116 @@ emqx_resource_schema { - resource_opts { - desc { - en: """Resource options.""" - zh: """资源相关的选项。""" - } - label { - en: """Resource Options""" - zh: """资源选项""" - } - } +auto_restart_interval.desc: +"""The auto restart interval after the resource is disconnected.""" - creation_opts { - desc { - en: """Creation options.""" - zh: """资源启动相关的选项。""" - } - label { - en: """Creation Options""" - zh: """资源启动选项""" - } - } +auto_restart_interval.label: +"""Auto Restart Interval""" - worker_pool_size { - desc { - en: """The number of buffer workers. Only applicable for egress type bridges. -For bridges only have ingress direction data flow, it can be set to 0 otherwise must be greater than 0.""" - zh: """缓存队列 worker 数量。仅对 egress 类型的桥接有意义。当桥接仅有 ingress 方向时，可设置为 0，否则必须大于 0。""" - } - label { - en: """Buffer Pool Size""" - zh: """缓存池大小""" - } - } +batch_size.desc: +"""Maximum batch count. If equal to 1, there's effectively no batching.""" - health_check_interval { - desc { - en: """Health check interval.""" - zh: """健康检查间隔。""" - } - label { - en: """Health Check Interval""" - zh: """健康检查间隔""" - } - } +batch_size.label: +"""Max batch size""" - resume_interval { - desc { - en: """The interval at which the buffer worker attempts to resend failed requests in the inflight window.""" - zh: """在发送失败后尝试重传飞行窗口中的请求的时间间隔。""" - } - label { - en: """Resume Interval""" - zh: """重试时间间隔""" - } - } +batch_time.desc: +"""Maximum waiting interval when accumulating a batch at a low message rates for more efficient resource usage.""" - start_after_created { - desc { - en: """Whether start the resource right after created.""" - zh: """是否在创建资源后立即启动资源。""" - } - label { - en: """Start After Created""" - zh: """创建后立即启动""" - } - } +batch_time.label: +"""Max batch wait time""" - start_timeout { - desc { - en: """Time interval to wait for an auto-started resource to become healthy before responding resource creation requests.""" - zh: """在回复资源创建请求前等待资源进入健康状态的时间。""" - } - label { - en: """Start Timeout""" - zh: """启动超时时间""" - } - } +buffer_mode.desc: +"""Buffer operation mode. +memory_only: Buffer all messages in memory.volatile_offload: Buffer message in memory first, when up to certain limit (see buffer_seg_bytes config for more information), then start offloading messages to disk""" - auto_restart_interval { - desc { - en: """The auto restart interval after the resource is disconnected.""" - zh: """资源断开以后，自动重连的时间间隔。""" - } - label { - en: """Auto Restart Interval""" - zh: """自动重连间隔""" - } - } +buffer_mode.label: +"""Buffer Mode""" - query_mode { - desc { - en: """Query mode. Optional 'sync/async', default 'async'.""" - zh: """请求模式。可选 '同步/异步'，默认为'异步'模式。""" - } - label { - en: """Query mode""" - zh: """请求模式""" - } - } +buffer_seg_bytes.desc: +"""Applicable when buffer mode is set to volatile_offload. +This value is to specify the size of each on-disk buffer file.""" - request_timeout { - desc { - en: """Starting from the moment when the request enters the buffer, if the request remains in the buffer for the specified time or is sent but does not receive a response or acknowledgement in time, the request is considered expired.""" - zh: """从请求进入缓冲区开始计时，如果请求在规定的时间内仍停留在缓冲区内或者已发送但未能及时收到响应或确认，该请求将被视为过期。""" - } - label { - en: """Request Expiry""" - zh: """请求超期""" - } - } +buffer_seg_bytes.label: +"""Segment File Bytes""" - enable_batch { - desc { - en: """Batch mode enabled.""" - zh: """启用批量模式。""" - } - label { - en: """Enable batch""" - zh: """启用批量模式""" - } - } +creation_opts.desc: +"""Creation options.""" - enable_queue { - desc { - en: """Enable disk buffer queue (only applicable for egress bridges). +creation_opts.label: +"""Creation Options""" + +enable_batch.desc: +"""Batch mode enabled.""" + +enable_batch.label: +"""Enable batch""" + +enable_queue.desc: +"""Enable disk buffer queue (only applicable for egress bridges). When Enabled, messages will be buffered on disk when the bridge connection is down. When disabled the messages are buffered in RAM only.""" - zh: """启用磁盘缓存队列（仅对 egress 方向桥接有用）。""" - } - label { - en: """Enable disk buffer queue""" - zh: """启用磁盘缓存队列""" - } - } - inflight_window { - desc { - en: """Query inflight window. When query_mode is set to async, this config has to be set to 1 if messages from the same MQTT client have to be strictly ordered.""" - zh: """请求飞行队列窗口大小。当请求模式为异步时，如果需要严格保证来自同一 MQTT 客户端的消息有序，则必须将此值设为 1。""""" - } - label { - en: """Inflight window""" - zh: """请求飞行队列窗口""" - } - } +enable_queue.label: +"""Enable disk buffer queue""" - batch_size { - desc { - en: """Maximum batch count. If equal to 1, there's effectively no batching.""" - zh: """最大批量请求大小。如果设为1，则无批处理。""" - } - label { - en: """Max batch size""" - zh: """最大批量请求大小""" - } - } +health_check_interval.desc: +"""Health check interval.""" - batch_time { - desc { - en: """Maximum waiting interval when accumulating a batch at a low message rates for more efficient resource usage.""" - zh: """在较低消息率情况下尝试累积批量输出时的最大等待间隔，以提高资源的利用率。""" - } - label { - en: """Max batch wait time""" - zh: """批量等待最大间隔""" - } - } +health_check_interval.label: +"""Health Check Interval""" - max_queue_bytes { - desc { - en: """Maximum number of bytes to buffer for each buffer worker.""" - zh: """每个缓存 worker 允许使用的最大字节数。""" - } - label { - en: """Max buffer queue size""" - zh: """缓存队列最大长度""" - } - } +inflight_window.desc: +"""Query inflight window. When query_mode is set to async, this config has to be set to 1 if messages from the same MQTT client have to be strictly ordered.""" +inflight_window.label: +"""Inflight window""" + +max_buffer_bytes.desc: +"""Maximum number of bytes to buffer for each buffer worker.""" + +max_buffer_bytes.label: +"""Max buffer queue size""" + +query_mode.desc: +"""Query mode. Optional 'sync/async', default 'async'.""" + +query_mode.label: +"""Query mode""" + +request_timeout.desc: +"""Starting from the moment when the request enters the buffer, if the request remains in the buffer for the specified time or is sent but does not receive a response or acknowledgement in time, the request is considered expired.""" + +request_timeout.label: +"""Request Expiry""" + +resource_opts.desc: +"""Resource options.""" + +resource_opts.label: +"""Resource Options""" + +resume_interval.desc: +"""The interval at which the buffer worker attempts to resend failed requests in the inflight window.""" + +resume_interval.label: +"""Resume Interval""" + +start_after_created.desc: +"""Whether start the resource right after created.""" + +start_after_created.label: +"""Start After Created""" + +start_timeout.desc: +"""Time interval to wait for an auto-started resource to become healthy before responding resource creation requests.""" + +start_timeout.label: +"""Start Timeout""" + +worker_pool_size.desc: +"""The number of buffer workers. Only applicable for egress type bridges. +For bridges only have ingress direction data flow, it can be set to 0 otherwise must be greater than 0.""" + +worker_pool_size.label: +"""Buffer Pool Size""" } diff --git a/rel/i18n/emqx_retainer_api.hocon b/rel/i18n/emqx_retainer_api.hocon index aced87076..5c7084778 100644 --- a/rel/i18n/emqx_retainer_api.hocon +++ b/rel/i18n/emqx_retainer_api.hocon @@ -1,143 +1,63 @@ emqx_retainer_api { - get_config_api { - desc { - en: "View config" - zh: "查看配置内容" - } - } +config_content.desc: +"""The config content""" - config_content { - desc { - en: "The config content" - zh: "配置内容" - } - } +config_not_found.desc: +"""Config not found.""" - config_not_found { - desc { - en: "Config not found." - zh: "配置不存在" - } - } +delete_matching_api.desc: +"""Delete matching messages.""" - update_retainer_api { - desc { - en: "Update retainer config." - zh: "更新配置" - } - } +from_clientid.desc: +"""The clientid of publisher.""" - update_config_success { - desc { - en: "Update configs successfully." - zh: "配置更新成功" - } - } +from_username.desc: +"""The username of publisher.""" - update_config_failed { - desc { - en: "Update config failed" - zh: "配置更新失败" - } - } +get_config_api.desc: +"""View config""" - list_retained_api { - desc { - en: "List retained messages." - zh: "查看保留消息列表" - } - } +list_retained_api.desc: +"""List retained messages.""" - retained_list { - desc { - en: "Retained messages list." - zh: "保留消息列表" - } - } +lookup_api.desc: +"""Lookup a message by a topic without wildcards.""" - unsupported_backend { - desc { - en: "Unsupported backend." - zh: "不支持的后端" - } - } +message_detail.desc: +"""Details of the message.""" - lookup_api { - desc { - en: "Lookup a message by a topic without wildcards." - zh: "通过不带通配符的主题查看对应的保留消息" - } - } +message_not_exist.desc: +"""Viewed message doesn't exist.""" - message_detail { - desc { - en: "Details of the message." - zh: "消息详情" - } - } +msgid.desc: +"""Message ID.""" - message_not_exist { - desc { - en: "Viewed message doesn't exist." - zh: "消息不存在" - } - } +payload.desc: +"""Payload.""" - delete_matching_api { - desc { - en: "Delete matching messages." - zh: "删除对应的消息" - } - } +publish_at.desc: +"""Message publish time, RFC 3339 format.""" - topic { - desc { - en: "Topic." - zh: "主题" - } - } +qos.desc: +"""QoS.""" - msgid { - desc { - en: "Message ID." - zh: "消息 ID" - } - } +retained_list.desc: +"""Retained messages list.""" - qos { - desc { - en: "QoS." - zh: "QoS" - } - } +topic.desc: +"""Topic.""" - publish_at { - desc { - en: "Message publish time, RFC 3339 format." - zh: "消息发送时间, RFC 3339 格式" - } - } +unsupported_backend.desc: +"""Unsupported backend.""" - from_clientid { - desc { - en: "The clientid of publisher." - zh: "发布者的 ClientID" - } - } +update_config_failed.desc: +"""Update config failed""" - from_username { - desc { - en: "The username of publisher." - zh: "发布者的用户名" - } - } +update_config_success.desc: +"""Update configs successfully.""" - payload { - desc { - en: "Payload." - zh: "消息内容" - } - } +update_retainer_api.desc: +"""Update retainer config.""" } diff --git a/rel/i18n/emqx_retainer_schema.hocon b/rel/i18n/emqx_retainer_schema.hocon index 274c260d4..9b2905da1 100644 --- a/rel/i18n/emqx_retainer_schema.hocon +++ b/rel/i18n/emqx_retainer_schema.hocon @@ -1,105 +1,49 @@ emqx_retainer_schema { - enable { - desc { - en: "Enable retainer feature" - zh: "是否开启消息保留功能" - } - } +backend.desc: +"""Settings for the database storing the retained messages.""" - msg_expiry_interval { - desc { - en: "Message retention time. 0 means message will never be expired." - zh: "消息保留时间。0 代表永久保留" - } - } - - flow_control { - desc { - en: "Flow control." - zh: "流控设置" - } - } - - msg_clear_interval { - desc { - en: """Periodic interval for cleaning up expired messages. -Never clear if the value is 0.""" - zh: "消息清理间隔。0 代表不进行清理" - } - } - - max_payload_size { - desc { - en: "Maximum retained message size." - zh: "消息大小最大值" - } - } - - stop_publish_clear_msg { - desc { - en: """When the retained flag of the `PUBLISH` message is set and Payload is empty, -whether to continue to publish the message. -See: -http://docs.oasis-open.org/mqtt/mqtt/v3.1.1/os/mqtt-v3.1.1-os.html#_Toc398718038""" - zh: """是否不发送保留消息的清理消息，在 MQTT 5.0 中如果一条保留消息的消息体为空，则会清除掉之前存储 -的对应的保留消息，通过这个值控制是否停止发送清理消息""" - } - } - - backend { - desc { - en: "Settings for the database storing the retained messages." - zh: "保留消息的存储后端" - } - } - - mnesia_config_type { - desc { - en: "Backend type." - zh: "后端类型" - } - } - - mnesia_config_storage_type { - desc { - en: "Specifies whether the messages are stored in RAM or persisted on disc." - zh: "选择消息是存放在磁盘还是内存中" - } - } - - max_retained_messages { - desc { - en: "Maximum number of retained messages. 0 means no limit." - zh: "消息保留的数量上限。0 表示无限" - } - } - - batch_read_number { - desc { - en: "Size of the batch when reading messages from storage. 0 means no limit." - zh: "从存储后端批量加载时的每批数量上限，0 代表一次性读取" - } - } - - batch_deliver_number { - desc { - en: "The number of retained messages can be delivered per batch." - zh: "批量派发时每批的数量。0 代表一次性全部派发" - } - } - - batch_deliver_limiter { - desc { - en: """The rate limiter name for retained messages' delivery. +batch_deliver_limiter.desc: +"""The rate limiter name for retained messages' delivery. Limiter helps to avoid delivering too many messages to the client at once, which may cause the client to block or crash, or drop messages due to exceeding the size of the message queue. The names of the available rate limiters are taken from the existing rate limiters under `limiter.batch`. If this field is empty, limiter is not used.""" - zh: """批量发送的限流器的名称。 -限流器可以用来防止短时间内向客户端发送太多的消息，从而避免过多的消息导致客户端队列堵塞甚至崩溃。 -这个名称需要是指向 `limiter.batch` 下的一个真实存在的限流器。 -如果这个字段为空，则不使用限流器。""" - } - } + +batch_deliver_number.desc: +"""The number of retained messages can be delivered per batch.""" + +batch_read_number.desc: +"""Size of the batch when reading messages from storage. 0 means no limit.""" + +enable.desc: +"""Enable retainer feature""" + +flow_control.desc: +"""Flow control.""" + +max_payload_size.desc: +"""Maximum retained message size.""" + +max_retained_messages.desc: +"""Maximum number of retained messages. 0 means no limit.""" + +mnesia_config_storage_type.desc: +"""Specifies whether the messages are stored in RAM or persisted on disc.""" + +mnesia_config_type.desc: +"""Backend type.""" + +msg_clear_interval.desc: +"""Periodic interval for cleaning up expired messages. +Never clear if the value is 0.""" + +msg_expiry_interval.desc: +"""Message retention time. 0 means message will never be expired.""" + +stop_publish_clear_msg.desc: +"""When the retained flag of the `PUBLISH` message is set and Payload is empty, +whether to continue to publish the message. +See: +http://docs.oasis-open.org/mqtt/mqtt/v3.1.1/os/mqtt-v3.1.1-os.html#_Toc398718038""" } diff --git a/rel/i18n/emqx_rewrite_api.hocon b/rel/i18n/emqx_rewrite_api.hocon index 91f15cb8c..7e090e590 100644 --- a/rel/i18n/emqx_rewrite_api.hocon +++ b/rel/i18n/emqx_rewrite_api.hocon @@ -1,25 +1,12 @@ emqx_rewrite_api { - list_topic_rewrite_api { - desc { - en: """List all rewrite rules""" - zh: """列出全部主题重写规则""" - } - } +list_topic_rewrite_api.desc: +"""List all rewrite rules""" - update_topic_rewrite_api { - desc { - en: """Update all rewrite rules""" - zh: """更新全部主题重写规则""" - } - } +update_topic_rewrite_api.desc: +"""Update all rewrite rules""" - - update_topic_rewrite_api_response413 { - desc { - en: """Rules count exceed max limit""" - zh: """超出主题重写规则数量上限""" - } - } +update_topic_rewrite_api_response413.desc: +"""Rules count exceed max limit""" } diff --git a/rel/i18n/emqx_rule_api_schema.hocon b/rel/i18n/emqx_rule_api_schema.hocon index f9b344666..29ecaa18e 100644 --- a/rel/i18n/emqx_rule_api_schema.hocon +++ b/rel/i18n/emqx_rule_api_schema.hocon @@ -1,685 +1,381 @@ emqx_rule_api_schema { - event_event_type { - desc { - en: "Event Type" - zh: "事件类型" - } - label: { - en: "Event Type" - zh: "事件类型" - } - } +event_action.desc: +"""Publish or Subscribe""" - event_id { - desc { - en: "Message ID" - zh: "消息 ID" - } - label: { - en: "Message ID" - zh: "消息 ID" - } - } +event_action.label: +"""Publish or Subscribe""" - event_clientid { - desc { - en: "The Client ID" - zh: "客户端 ID" - } - label: { - en: "Client ID" - zh: "客户端 ID" - } - } +event_payload.desc: +"""The Message Payload""" - event_username { - desc { - en: "Username" - zh: "用户名" - } - label: { - en: "Username" - zh: "用户名" - } - } +event_payload.label: +"""Message Payload""" - event_payload { - desc { - en: "The Message Payload" - zh: "消息负载" - } - label: { - en: "Message Payload" - zh: "消息负载" - } - } +metrics_actions_failed_out_of_service.desc: +"""How much times the rule failed to call actions due to the action is out of service. For example, a bridge is disabled or stopped.""" - event_peerhost { - desc { - en: "The IP Address of the Peer Client" - zh: "对等客户端的 IP 地址" - } - label: { - en: "Peer IP Address" - zh: "对等客户端的 IP" - } - } +metrics_actions_failed_out_of_service.label: +"""Fail Action""" - event_topic { - desc { - en: "Message Topic" - zh: "消息主题" - } - label: { - en: "Message Topic" - zh: "消息主题" - } - } +metrics_actions_failed_unknown.desc: +"""How much times the rule failed to call actions due to to an unknown error.""" - event_publish_received_at { - desc { - en: "The Time that this Message is Received" - zh: "消息被接受的时间" - } - label: { - en: "Message Received Time" - zh: "消息被接受的时间" - } - } +metrics_actions_failed_unknown.label: +"""Fail Action""" - event_qos { - desc { - en: "The Message QoS" - zh: "消息的 QoS" - } - label: { - en: "Message QoS" - zh: "消息 QoS" - } - } +event_server.desc: +"""The IP address (or hostname) and port of the MQTT broker, in IP:Port format""" - event_from_clientid { - desc { - en: "The Client ID" - zh: "事件来源客户端的 ID" - } - label: { - en: "Client ID" - zh: "客户端 ID" - } - } +event_server.label: +"""Server IP And Port""" - event_from_username { - desc { - en: "The User Name" - zh: "事件来源客户端的用户名" - } - label: { - en: "Username" - zh: "用户名" - } - } +metrics_actions_total.desc: +"""How much times the actions are called by the rule. This value may several times of 'matched', depending on the number of the actions of the rule.""" - event_mountpoint { - desc { - en: "The Mountpoint" - zh: "挂载点" - } - label: { - en: "Mountpoint" - zh: "挂载点" - } - } +metrics_actions_total.label: +"""Action Total""" - event_peername { - desc { - en: "The IP Address and Port of the Peer Client" - zh: "对等客户端的 IP 地址和端口" - } - label: { - en: "IP Address And Port" - zh: "IP 地址和端口" - } - } +event_ctx_disconnected_da.desc: +"""The Time that this Client is Disconnected""" - event_sockname { - desc { - en: "The IP Address and Port of the Local Listener" - zh: "本地监听的 IP 地址和端口" - } - label: { - en: "IP Address And Port" - zh: "IP 地址和端口" - } - } +event_ctx_disconnected_da.label: +"""Disconnected Time""" - event_proto_name { - desc { - en: "Protocol Name" - zh: "协议名称" - } - label: { - en: "Protocol Name" - zh: "协议名称" - } - } +event_topic.desc: +"""Message Topic""" - event_proto_ver { - desc { - en: "Protocol Version" - zh: "协议版本" - } - label: { - en: "Protocol Version" - zh: "协议版本" - } - } +event_topic.label: +"""Message Topic""" - event_keepalive { - desc { - en: "KeepAlive" - zh: "保持连接" - } - label: { - en: "KeepAlive" - zh: "保持连接" - } - } +event_peername.desc: +"""The IP Address and Port of the Peer Client""" - event_clean_start { - desc { - en: "Clean Start" - zh: "清除会话" - } - label: { - en: "Clean Start" - zh: "清除会话" - } - } +event_peername.label: +"""IP Address And Port""" - event_expiry_interval { - desc { - en: "Expiry Interval" - zh: "到期间隔" - } - label: { - en: "Expiry Interval" - zh: "到期间隔" - } - } +metrics_sql_passed.desc: +"""How much times the SQL is passed""" - event_is_bridge { - desc { - en: "Is Bridge" - zh: "是否桥接" - } - label: { - en: "Is Bridge" - zh: "是否桥接" - } - } +metrics_sql_passed.label: +"""SQL Passed""" - event_connected_at { - desc { - en: "The Time that this Client is Connected" - zh: "客户端连接完成时的时刻" - } - label: { - en: "Connected Time" - zh: "连接完成时的时刻" - } - } +test_context.desc: +"""The context of the event for testing""" - event_action { - desc { - en: "Publish or Subscribe" - zh: "订阅或发布" - } - label: { - en: "Publish or Subscribe" - zh: "订阅或发布" - } - } +test_context.label: +"""Event Conetxt""" - event_authz_source { - desc { - en: "Cache, Plugs or Default" - zh: "缓存，插件或者默认值" - } - label: { - en: "Auth Source" - zh: "认证源" - } - } +node_node.desc: +"""The node name""" - event_result { - desc { - en: "Allow or Deny" - zh: "允许或禁止" - } - label: { - en: "Auth Result" - zh: "认证结果" - } - } +node_node.label: +"""Node Name""" - event_server { - desc { - en: "The IP address (or hostname) and port of the MQTT broker, in IP:Port format" - zh: "MQTT broker的 IP 地址（或主机名）和端口，采用 IP:Port 格式" - } - label: { - en: "Server IP And Port" - zh: "服务器 IP 地址和端口" - } - } +event_from_clientid.desc: +"""The Client ID""" - event_dup { - desc { - en: "The DUP flag of the MQTT message" - zh: "MQTT 消息的 DUP 标志" - } - label: { - en: "DUP Flag" - zh: "DUP 标志" - } - } +event_from_clientid.label: +"""Client ID""" - event_retain { - desc { - en: "If is a retain message" - zh: "是否是保留消息" - } - label: { - en: "Retain Message" - zh: "保留消息" - } - } +event_keepalive.desc: +"""KeepAlive""" - event_ctx_dropped { - desc { - en: "The Reason for Dropping" - zh: "消息被丢弃的原因" - } - label: { - en: "Dropped Reason" - zh: "丢弃原因" - } - } +event_keepalive.label: +"""KeepAlive""" - event_ctx_disconnected_reason { - desc { - en: "The Reason for Disconnect" - zh: "断开连接的原因" - } - label: { - en: "Disconnect Reason" - zh: "断开连接原因" - } - } +event_connected_at.desc: +"""The Time that this Client is Connected""" - event_ctx_disconnected_da { - desc { - en: "The Time that this Client is Disconnected" - zh: "客户端断开连接的时刻" - } - label: { - en: "Disconnected Time" - zh: "客户端断开连接时刻" - } - } +event_connected_at.label: +"""Connected Time""" - event_ctx_connack_reason_code { - desc { - en: "The reason code" - zh: "错误码" - } - label: { - en: "Reason Code" - zh: "错误码" - } - } +metrics_sql_failed_exception.desc: +"""How much times the SQL is failed due to exceptions. This may because of a crash when calling a SQL function, or trying to do arithmetic operation on undefined variables""" - rule_id { - desc { - en: "The ID of the rule" - zh: "规则的 ID" - } - label: { - en: "Rule ID" - zh: "规则 ID" - } - } +metrics_sql_failed_exception.label: +"""SQL Exception""" - node_node { - desc { - en: "The node name" - zh: "节点名字" - } - label: { - en: "Node Name" - zh: "节点名字" - } - } +event_from_username.desc: +"""The User Name""" - metrics_sql_matched { - desc { - en: "How much times the FROM clause of the SQL is matched." - zh: "SQL 的 FROM 子句匹配的次数。" - } - label: { - en: "Matched" - zh: "命中数" - } - } +event_from_username.label: +"""Username""" - metrics_sql_matched_rate { - desc { - en: "The rate of matched, times/second" - zh: "命中速率，次/秒" - } - label: { - en: "命中速率" - zh: "Matched Rate" - } - } +event_ctx_connack_reason_code.desc: +"""The reason code""" - metrics_sql_matched_rate_max { - desc { - en: "The max rate of matched, times/second" - zh: "最大命中速率，次/秒" - } - label: { - en: "Max Matched Rate" - zh: "最大命中速率" - } - } +event_ctx_connack_reason_code.label: +"""Reason Code""" - metrics_sql_matched_rate_last5m { - desc { - en: "The average rate of matched in last 5 minutes, times/second" - zh: "5分钟平均命中速率，次/秒" - } - label: { - en: "Average Matched Rate" - zh: "平均命中速率" - } - } +rs_description.desc: +"""The description""" - metrics_sql_passed { - desc { - en: "How much times the SQL is passed" - zh: "SQL 通过的次数" - } - label: { - en: "SQL Passed" - zh: "SQL 通过" - } - } +rs_description.label: +"""Description""" - metrics_sql_failed { - desc { - en: "How much times the SQL is failed" - zh: "SQL 失败的次数" - } - label: { - en: "SQL Failed" - zh: "SQL 失败" - } - } +rule_id.desc: +"""The ID of the rule""" - metrics_sql_failed_exception { - desc { - en: "How much times the SQL is failed due to exceptions. This may because of a crash when calling a SQL function, or trying to do arithmetic operation on undefined variables" - zh: "SQL 由于执行异常而失败的次数。 这可能是因为调用 SQL 函数时崩溃，或者试图对未定义的变量进行算术运算" - } - label: { - en: "SQL Exception" - zh: "SQL 执行异常" - } - } +rule_id.label: +"""Rule ID""" - metrics_sql_failed_unknown { - desc { - en: "How much times the SQL is failed due to an unknown error." - zh: "由于未知错误导致 SQL 失败的次数。" - } - label: { - en: "SQL Unknown Error" - zh: "SQL 未知错误" - } - } +rs_event.desc: +"""The event topics""" - metrics_actions_total { - desc { - en: "How much times the actions are called by the rule. This value may several times of 'matched', depending on the number of the actions of the rule." - zh: "规则调用输出的次数。 该值可能是“sql.matched”的几倍，具体取决于规则输出的数量。" - } - label: { - en: "Action Total" - zh: "调用输出次数" - } - } +rs_event.label: +"""Event Topics""" - metrics_actions_success { - desc { - en: "How much times the rule success to call the actions." - zh: "规则成功调用输出的次数。" - } - label: { - en: "Success Action" - zh: "成功调用输出次数" - } - } +root_rule_engine.desc: +"""Rule engine configurations. This API can be used to change EMQX rule engine settings. But not for the rules. To list, create, or update rules, call the '/rules' API instead.""" - metrics_actions_failed { - desc { - en: "How much times the rule failed to call the actions." - zh: "规则调用输出失败的次数。" - } - label: { - en: "Failed Action" - zh: "调用输出失败次数" - } - } +root_rule_engine.label: +"""Rule engine configuration""" - metrics_actions_failed_out_of_service { - desc { - en: "How much times the rule failed to call actions due to the action is out of service. For example, a bridge is disabled or stopped." - zh: "由于输出停止服务而导致规则调用输出失败的次数。 例如，桥接被禁用或停止。" - } - label: { - en: "Fail Action" - zh: "调用输出失败次数" - } - } +event_sockname.desc: +"""The IP Address and Port of the Local Listener""" - metrics_actions_failed_unknown { - desc { - en: "How much times the rule failed to call actions due to to an unknown error." - zh: "由于未知错误，规则调用输出失败的次数。" - } - label: { - en: "Fail Action" - zh: "调用输出失败次数" - } - } +event_sockname.label: +"""IP Address And Port""" - test_context { - desc { - en: "The context of the event for testing" - zh: "测试事件的上下文" - } - label: { - en: "Event Conetxt" - zh: "事件上下文" - } - } +event_qos.desc: +"""The Message QoS""" - test_sql { - desc { - en: "The SQL of the rule for testing" - zh: "测试的 SQL" - } - label: { - en: "Test SQL" - zh: "测试 SQL" - } - } +event_qos.label: +"""Message QoS""" - rs_event { - desc { - en: "The event topics" - zh: "事件主题" - } - label: { - en: "Event Topics" - zh: "事件主题" - } - } +event_mountpoint.desc: +"""The Mountpoint""" - rs_title { - desc { - en: "The title" - zh: "标题" - } - label: { - en: "Title" - zh: "标题" - } - } +event_mountpoint.label: +"""Mountpoint""" - rs_description { - desc { - en: "The description" - zh: "描述" - } - label: { - en: "Description" - zh: "描述" - } - } +rs_title.desc: +"""The title""" - rs_columns { - desc { - en: "The columns" - zh: "列" - } - label: { - en: "Column" - zh: "列" - } - } +rs_title.label: +"""Title""" - rs_test_columns { - desc { - en: "The test columns" - zh: "测试列" - } - label: { - en: "Test Columns" - zh: "测试列" - } - } +ri_metrics.desc: +"""The metrics of the rule""" - rs_sql_example { - desc { - en: "The sql_example" - zh: "SQL 例子" - } - label: { - en: "SQL Example" - zh: "SQL 例子" - } - } +ri_metrics.label: +"""Rule Metrics""" - ri_metrics { - desc { - en: "The metrics of the rule" - zh: "规则的计数器" - } - label: { - en: "Rule Metrics" - zh: "规则计数器" - } - } +event_retain.desc: +"""If is a retain message""" - ri_node_metrics { - desc { - en: "The metrics of the rule for each node" - zh: "每个节点的规则计数器" - } - label: { - en: "Each Node Rule Metrics" - zh: "每个节点规则计数器" - } - } +event_retain.label: +"""Retain Message""" - ri_from { - desc { - en: "The topics of the rule" - zh: "规则指定的主题" - } - label: { - en: "Topics of Rule" - zh: "规则指定的主题" - } - } +event_event_type.desc: +"""Event Type""" - ri_created_at { - desc { - en: "The created time of the rule" - zh: "规则创建时间" - } - label: { - en: "Rule Create Time" - zh: "规则创建时间" - } - } +event_event_type.label: +"""Event Type""" - root_rule_creation { - desc { - en: "Schema for creating rules" - zh: "用于创建规则的 Schema" - } - label: { - en: "Create Schema" - zh: "用于创建规则的 Schema" - } - } +event_expiry_interval.desc: +"""Expiry Interval""" - root_rule_info { - desc { - en: "Schema for rule info" - zh: "用于规则信息的 Schema" - } - label: { - en: "Info Schema" - zh: "用于规则信息的 Schema" - } - } +event_expiry_interval.label: +"""Expiry Interval""" - root_rule_events { - desc { - en: "Schema for rule events" - zh: "用于事件的 Schema" - } - label: { - en: "Rule Events Schema" - zh: "用于规则事件的 Schema" - } - } +metrics_sql_matched.desc: +"""How much times the FROM clause of the SQL is matched.""" - root_rule_test { - desc { - en: "Schema for testing rules" - zh: "用于规则测试的 Schema" - } - label: { - en: "Rule Test Schema" - zh: "用于规则测试的 Schema" - } - } +metrics_sql_matched.label: +"""Matched""" + +event_clientid.desc: +"""The Client ID""" + +event_clientid.label: +"""Client ID""" + +metrics_actions_success.desc: +"""How much times the rule success to call the actions.""" + +metrics_actions_success.label: +"""Success Action""" + +metrics_actions_failed.desc: +"""How much times the rule failed to call the actions.""" + +metrics_actions_failed.label: +"""Failed Action""" + +metrics_sql_matched_rate.desc: +"""The rate of matched, times/second""" + +metrics_sql_matched_rate.label: +"""命中速率""" + +event_proto_ver.desc: +"""Protocol Version""" + +event_proto_ver.label: +"""Protocol Version""" + +event_publish_received_at.desc: +"""The Time that this Message is Received""" + +event_publish_received_at.label: +"""Message Received Time""" + +metrics_sql_matched_rate_last5m.desc: +"""The average rate of matched in last 5 minutes, times/second""" + +metrics_sql_matched_rate_last5m.label: +"""Average Matched Rate""" + +event_is_bridge.desc: +"""Is Bridge""" + +event_is_bridge.label: +"""Is Bridge""" + +event_authz_source.desc: +"""Cache, Plugs or Default""" + +event_authz_source.label: +"""Auth Source""" + +metrics_sql_failed_unknown.desc: +"""How much times the SQL is failed due to an unknown error.""" + +metrics_sql_failed_unknown.label: +"""SQL Unknown Error""" + +metrics_sql_failed.desc: +"""How much times the SQL is failed""" + +metrics_sql_failed.label: +"""SQL Failed""" + +event_ctx_dropped.desc: +"""The Reason for Dropping""" + +event_ctx_dropped.label: +"""Dropped Reason""" + +root_rule_test.desc: +"""Schema for testing rules""" + +root_rule_test.label: +"""Rule Test Schema""" + +rs_test_columns.desc: +"""The test columns""" + +rs_test_columns.label: +"""Test Columns""" + +event_peerhost.desc: +"""The IP Address of the Peer Client""" + +event_peerhost.label: +"""Peer IP Address""" + +event_proto_name.desc: +"""Protocol Name""" + +event_proto_name.label: +"""Protocol Name""" + +root_rule_events.desc: +"""Schema for rule events""" + +root_rule_events.label: +"""Rule Events Schema""" + +rs_sql_example.desc: +"""The sql_example""" + +rs_sql_example.label: +"""SQL Example""" + +metrics_sql_matched_rate_max.desc: +"""The max rate of matched, times/second""" + +metrics_sql_matched_rate_max.label: +"""Max Matched Rate""" + +event_clean_start.desc: +"""Clean Start""" + +event_clean_start.label: +"""Clean Start""" + +ri_created_at.desc: +"""The created time of the rule""" + +ri_created_at.label: +"""Rule Create Time""" + +event_dup.desc: +"""The DUP flag of the MQTT message""" + +event_dup.label: +"""DUP Flag""" + +ri_from.desc: +"""The topics of the rule""" + +ri_from.label: +"""Topics of Rule""" + +ri_node_metrics.desc: +"""The metrics of the rule for each node""" + +ri_node_metrics.label: +"""Each Node Rule Metrics""" + +root_rule_creation.desc: +"""Schema for creating rules""" + +root_rule_creation.label: +"""Create Schema""" + +event_result.desc: +"""Allow or Deny""" + +event_result.label: +"""Auth Result""" + +event_id.desc: +"""Message ID""" + +event_id.label: +"""Message ID""" + +event_username.desc: +"""Username""" + +event_username.label: +"""Username""" + +root_rule_info.desc: +"""Schema for rule info""" + +root_rule_info.label: +"""Info Schema""" + +rs_columns.desc: +"""The columns""" + +rs_columns.label: +"""Column""" + +test_sql.desc: +"""The SQL of the rule for testing""" + +test_sql.label: +"""Test SQL""" + +event_ctx_disconnected_reason.desc: +"""The Reason for Disconnect""" + +event_ctx_disconnected_reason.label: +"""Disconnect Reason""" } diff --git a/rel/i18n/emqx_rule_engine_api.hocon b/rel/i18n/emqx_rule_engine_api.hocon index 39fc3186c..385b71ddc 100644 --- a/rel/i18n/emqx_rule_engine_api.hocon +++ b/rel/i18n/emqx_rule_engine_api.hocon @@ -1,151 +1,93 @@ emqx_rule_engine_api { - api1 { - desc { - en: "List all rules" - zh: "列出所有规则" - } - label: { - en: "List All Rules" - zh: "列出所有规则" - } - } - api1_enable { - desc { - en: "Filter enable/disable rules" - zh: "根据规则是否开启条件过滤" - } - } +api1.desc: +"""List all rules""" - api1_from { - desc { - en: "Filter rules by from(topic), exact match" - zh: "根据规则来源 Topic 过滤, 需要完全匹配" - } - } +api1.label: +"""List All Rules""" - api1_like_id { - desc { - en: "Filter rules by id, Substring matching" - zh: "根据规则 id 过滤, 使用子串模糊匹配" - } - } +api10.desc: +"""Update rule engine configuration.""" - api1_like_from { - desc { - en: "Filter rules by from(topic), Substring matching" - zh: "根据规则来源 Topic 过滤, 使用子串模糊匹配" - } - } +api10.label: +"""Update configuration""" - api1_like_description { - desc { - en: "Filter rules by description, Substring matching" - zh: "根据规则描述过滤, 使用子串模糊匹配" - } - } - api1_match_from { - desc { - en: "Filter rules by from(topic), Mqtt topic matching" - zh: "根据规则来源 Topic 过滤, 使用 MQTT Topic 匹配" - } - } +api1_enable.desc: +"""Filter enable/disable rules""" - api2 { - desc { - en: "Create a new rule using given Id" - zh: "通过指定 ID 创建规则" - } - label: { - en: "Create Rule By ID" - zh: "通过指定 ID 创建规则" - } - } +api1_from.desc: +"""Filter rules by from(topic), exact match""" - api3 { - desc { - en: "List all events can be used in rules" - zh: "列出所有能被规则使用的事件" - } - label: { - en: "List All Events Can Be Used In Rule" - zh: "列出所有能被规则使用的事件" - } - } +api1_like_description.desc: +"""Filter rules by description, Substring matching""" - api4 { - desc { - en: "Get a rule by given Id" - zh: "通过 ID 查询规则" - } - label: { - en: "Get Rule" - zh: "查询规则" - } - } +api1_like_from.desc: +"""Filter rules by from(topic), Substring matching""" - api4_1 { - desc { - en: "Get a rule's metrics by given Id" - zh: "通过给定的 Id 获得规则的指标数据" - } - label: { - en: "Get Metric" - zh: "获得指标数据" - } - } +api1_like_id.desc: +"""Filter rules by id, Substring matching""" - api5 { - desc { - en: "Update a rule by given Id to all nodes in the cluster" - zh: "通过 ID 更新集群里所有节点上的规则" - } - label: { - en: "Update Cluster Rule" - zh: "更新集群规则" - } - } +api1_match_from.desc: +"""Filter rules by from(topic), Mqtt topic matching""" - api6 { - desc { - en: "Delete a rule by given Id from all nodes in the cluster" - zh: "通过 ID 删除集群里所有节点上的规则" - } - label: { - en: "Delete Cluster Rule" - zh: "删除集群规则" - } - } +api1_resp.desc: +"""List of rules""" - api7 { - desc { - en: "Reset a rule metrics" - zh: "重置规则计数" - } - label: { - en: "Reset Rule Metrics" - zh: "重置规则计数" - } - } +api1_resp.label: +"""List Rules""" + +api2.desc: +"""Create a new rule using given Id""" + +api2.label: +"""Create Rule By ID""" + +api3.desc: +"""List all events can be used in rules""" + +api3.label: +"""List All Events Can Be Used In Rule""" + +api4.desc: +"""Get a rule by given Id""" + +api4.label: +"""Get Rule""" + +api4_1.desc: +"""Get a rule's metrics by given Id""" + +api4_1.label: +"""Get Metric""" + +api5.desc: +"""Update a rule by given Id to all nodes in the cluster""" + +api5.label: +"""Update Cluster Rule""" + +api6.desc: +"""Delete a rule by given Id from all nodes in the cluster""" + +api6.label: +"""Delete Cluster Rule""" + +api7.desc: +"""Reset a rule metrics""" + +api7.label: +"""Reset Rule Metrics""" + +api8.desc: +"""Test a rule""" + +api8.label: +"""Test Rule""" + +api9.desc: +"""Get rule engine configuration.""" + +api9.label: +"""Get configuration""" - api8 { - desc { - en: "Test a rule" - zh: "测试一个规则" - } - label: { - en: "Test Rule" - zh: "测试规则" - } - } - desc9 { - desc { - en: "List of rules" - zh: "列出所有规则" - } - label: { - en: "List Rules" - zh: "列出所有规则" - } - } } diff --git a/rel/i18n/emqx_rule_engine_schema.hocon b/rel/i18n/emqx_rule_engine_schema.hocon index ca0a73f0f..9b1f1f802 100644 --- a/rel/i18n/emqx_rule_engine_schema.hocon +++ b/rel/i18n/emqx_rule_engine_schema.hocon @@ -1,326 +1,25 @@ emqx_rule_engine_schema { - rules_name { - desc { - en: "The name of the rule" - zh: "规则名字" - } - label: { - en: "Rule Name" - zh: "规则名字" - } - } +console_function.desc: +"""Print the actions to the console""" - rules_sql { - desc { - en: """SQL query to transform the messages. -Example: SELECT * FROM "test/topic" WHERE payload.x = 1""" - zh: """用于处理消息的 SQL 。 -示例：SELECT * FROM "test/topic" WHERE payload.x = 1""" - } - label: { - en: "Rule SQL" - zh: "规则 SQL" - } - } +console_function.label: +"""Console Function""" - rules_actions { - desc { - en: """A list of actions of the rule. -An action can be a string that refers to the channel ID of an EMQX bridge, or an object -that refers to a function. -There a some built-in functions like "republish" and "console", and we also support user -provided functions in the format: "{module}:{function}". -The actions in the list are executed sequentially. -This means that if one of the action is executing slowly, all the following actions will not -be executed until it returns. -If one of the action crashed, all other actions come after it will still be executed, in the -original order. -If there's any error when running an action, there will be an error message, and the 'failure' -counter of the function action or the bridge channel will increase.""" +desc_builtin_action_console.desc: +"""Configuration for a built-in action.""" - zh: """规则的动作列表。 -动作可以是指向 EMQX bridge 的引用，也可以是一个指向函数的对象。 -我们支持一些内置函数，如“republish”和“console”，我们还支持用户提供的函数，它的格式为：“{module}:{function}”。 -列表中的动作按顺序执行。这意味着如果其中一个动作执行缓慢，则以下所有动作都不会被执行直到它返回。 -如果其中一个动作崩溃，在它之后的所有动作仍然会被按照原始顺序执行。 -如果运行动作时出现任何错误，则会出现错误消息，并且相应的计数器会增加。""" - } - label: { - en: "Rule Action List" - zh: "动作列表" - } - } +desc_builtin_action_console.label: +"""Action Console Configuration""" - rules_enable { - desc { - en: "Enable or disable the rule" - zh: "启用或禁用规则引擎" - } - label: { - en: "Enable Or Disable Rule" - zh: "启用或禁用规则引擎" - } - } +desc_builtin_action_republish.desc: +"""Configuration for a built-in action.""" - rules_metadata { - desc { - en: "Rule metadata, do not change manually" - zh: "规则的元数据，不要手动修改" - } - label: { - en: "Rule metadata" - zh: "规则的元数据" - } - } +desc_builtin_action_republish.label: +"""Republish Configuration""" - rules_description { - desc { - en: "The description of the rule" - zh: "规则的描述" - } - label: { - en: "Rule Description" - zh: "规则描述" - } - } - - republish_function { - desc { - en: """Republish the message as a new MQTT message""" - zh: """将消息重新发布为新的 MQTT 消息""" - } - label: { - en: "Republish Function" - zh: "重新发布函数" - } - } - - console_function { - desc { - en: """Print the actions to the console""" - zh: "将输出打印到控制台" - } - label: { - en: "Console Function" - zh: "控制台函数" - } - } - - user_provided_function_function { - desc { - en: """The user provided function. Should be in the format: '{module}:{function}'. -Where {module} is the Erlang callback module and {function} is the Erlang function. - -To write your own function, checkout the function console and -republish in the source file: -apps/emqx_rule_engine/src/emqx_rule_actions.erl as an example.""" - - zh: """用户提供的函数。 格式应为：'{module}:{function}'。 -其中 {module} 是 Erlang 回调模块， {function} 是 Erlang 函数。 -要编写自己的函数，请检查源文件：apps/emqx_rule_engine/src/emqx_rule_actions.erl 中的示例函数 console 和republish 。""" - } - label: { - en: "User Provided Function" - zh: "用户提供的函数" - } - } - - user_provided_function_args { - desc { - en: """The args will be passed as the 3rd argument to module:function/3, -checkout the function console and republish in the source file: -apps/emqx_rule_engine/src/emqx_rule_actions.erl as an example.""" - zh: """用户提供的参数将作为函数 module:function/3 的第三个参数， -请检查源文件：apps/emqx_rule_engine/src/emqx_rule_actions.erl 中的示例函数 console 和republish 。""" - } - label: { - en: "User Provided Function Args" - zh: "用户提供函数的参数" - } - } - - republish_args_topic { - desc { - en: """The target topic of message to be re-published. -Template with variables is allowed, see description of the 'republish_args'.""" - zh: """重新发布消息的目标主题。 -允许使用带有变量的模板，请参阅“republish_args”的描述。""" - } - label: { - en: "Target Topic" - zh: "目标主题" - } - } - - republish_args_qos { - desc { - en: """The qos of the message to be re-published. -Template with variables is allowed, see description of the 'republish_args'. -Defaults to ${qos}. If variable ${qos} is not found from the selected result of the rule, -0 is used.""" - zh: """要重新发布的消息的 qos。允许使用带有变量的模板，请参阅“republish_args”的描述。 -默认为 ${qos}。 如果从规则的选择结果中没有找到变量 ${qos}，则使用 0。""" - } - label: { - en: "Message QoS" - zh: "消息 QoS 等级" - } - } - - republish_args_retain { - desc { - en: """The 'retain' flag of the message to be re-published. -Template with variables is allowed, see description of the 'republish_args'. -Defaults to ${retain}. If variable ${retain} is not found from the selected result -of the rule, false is used.""" - zh: """要重新发布的消息的“保留”标志。允许使用带有变量的模板，请参阅“republish_args”的描述。 -默认为 ${retain}。 如果从所选结果中未找到变量 ${retain}，则使用 false。""" - } - label: { - en: "Retain Flag" - zh: "保留消息标志" - } - } - - republish_args_payload { - desc { - en: """The payload of the message to be re-published. -Template with variables is allowed, see description of the 'republish_args'. -Defaults to ${payload}. If variable ${payload} is not found from the selected result -of the rule, then the string "undefined" is used.""" - zh: """要重新发布的消息的有效负载。允许使用带有变量的模板，请参阅“republish_args”的描述。 -默认为 ${payload}。 如果从所选结果中未找到变量 ${payload}，则使用字符串 "undefined"。""" - } - label: { - en: "Message Payload" - zh: "消息负载" - } - } - republish_args_user_properties { - desc { - en: """From which variable should the MQTT message's User-Property pairs be taken from. -The value must be a map. -You may configure it to ${pub_props.'User-Property'} or -use SELECT *,pub_props.'User-Property' as user_properties -to forward the original user properties to the republished message. -You may also call map_put function like -map_put('my-prop-name', 'my-prop-value', user_properties) as user_properties -to inject user properties. -NOTE: MQTT spec allows duplicated user property names, but EMQX Rule-Engine does not.""" - - zh: """指定使用哪个变量来填充 MQTT 消息的 User-Property 列表。这个变量的值必须是一个 map 类型。 -可以设置成 ${pub_props.'User-Property'} 或者 -使用 SELECT *,pub_props.'User-Property' as user_properties 来把源 MQTT 消息 -的 User-Property 列表用于填充。 -也可以使用 map_put 函数来添加新的 User-Property， -map_put('my-prop-name', 'my-prop-value', user_properties) as user_properties -注意：MQTT 协议允许一个消息中出现多次同一个 property 名，但是 EMQX 的规则引擎不允许。""" - } - } - - rule_engine_ignore_sys_message { - desc { - en: "When set to 'true' (default), rule-engine will ignore messages published to $SYS topics." - zh: "当设置为“true”（默认）时，规则引擎将忽略发布到 $SYS 主题的消息。" - } - label: { - en: "Ignore Sys Message" - zh: "忽略系统消息" - } - } - - rule_engine_rules { - desc { - en: """The rules""" - zh: "规则" - } - label: { - en: "Rules" - zh: "规则" - } - } - - rule_engine_jq_function_default_timeout { - desc { - en: "Default timeout for the `jq` rule engine function" - zh: "规则引擎内建函数 `jq` 默认时间限制" - } - label: { - en: "Rule engine jq function default timeout" - zh: "规则引擎 jq 函数时间限制" - } - } - - rule_engine_jq_implementation_module { - desc { - en: "The implementation module for the jq rule engine function. The two options are jq_nif and jq_port. With the jq_nif option an Erlang NIF library is used while with the jq_port option an implementation based on Erlang port programs is used. The jq_nif option (the default option) is the fastest implementation of the two but jq_port is safer as the jq programs will not execute in the same process as the Erlang VM." - zh: "jq 规则引擎功能的实现模块。可用的两个选项是 jq_nif 和 jq_port。jq_nif 使用 Erlang NIF 库访问 jq 库，而 jq_port 使用基于 Erlang Port 的实现。jq_nif 方式（默认选项）是这两个选项中最快的实现，但 jq_port 方式更安全，因为这种情况下 jq 程序不会在 Erlang VM 进程中执行。" - } - label: { - en: "JQ Implementation Module" - zh: "JQ 实现模块" - } - } - - desc_rule_engine { - desc { - en: """Configuration for the EMQX Rule Engine.""" - zh: """配置 EMQX 规则引擎。""" - } - label: { - en: "Rule Engine Configuration" - zh: "配置规则引擎" - } - } - - desc_rules { - desc { - en: """Configuration for a rule.""" - zh: """配置规则""" - } - label: { - en: "Rule Configuration" - zh: "配置规则" - } - } - - desc_builtin_action_republish { - desc { - en: """Configuration for a built-in action.""" - zh: """配置重新发布。""" - } - label: { - en: "Republish Configuration" - zh: "配置重新发布" - } - } - - desc_builtin_action_console { - desc { - en: """Configuration for a built-in action.""" - zh: """配置打印到控制台""" - } - label: { - en: "Action Console Configuration" - zh: "配置打印到控制台" - } - } - - desc_user_provided_function { - desc { - en: """Configuration for a built-in action.""" - zh: """配置用户函数""" - } - label: { - en: "User Provid Function Configuration" - zh: "配置用户函数" - } - } - - desc_republish_args { - desc { - en: """The arguments of the built-in 'republish' action.One can use variables in the args. +desc_republish_args.desc: +"""The arguments of the built-in 'republish' action.One can use variables in the args. The variables are selected by the rule. For example, if the rule SQL is defined as following: SELECT clientid, qos, payload FROM "t/1" @@ -337,27 +36,168 @@ Then there are 3 variables available: clientid, qos an When the rule is triggered by an MQTT message with payload = `hello`, qos = 1, clientid = `Steve`, the rule will republish a new MQTT message to topic `t/Steve`, payload = `msg: hello`, and `qos = 1`.""" - zh: """内置 'republish' 动作的参数。 -可以在参数中使用变量。 -变量是规则中选择的字段。 例如规则 SQL 定义如下： - - SELECT clientid, qos, payload FROM "t/1" - -然后有 3 个变量可用：clientid、qos 和 payload。 如果我们将参数设置为： - - { - topic = "t/${clientid}" - qos = "${qos}" - payload = "msg: ${payload}" - } - -当收到一条消息 payload = `hello`, qos = 1, clientid = `Steve` 时，将重新发布一条新的 MQTT 消息到主题 `t/Steve` -消息内容为 payload = `msg: hello`, and `qos = 1""" - } - label: { - en: "Republish Args" - zh: "重新发布参数" - } - } + +desc_republish_args.label: +"""Republish Args""" + +desc_rule_engine.desc: +"""Configuration for the EMQX Rule Engine.""" + +desc_rule_engine.label: +"""Rule Engine Configuration""" + +desc_rules.desc: +"""Configuration for a rule.""" + +desc_rules.label: +"""Rule Configuration""" + +desc_user_provided_function.desc: +"""Configuration for a built-in action.""" + +desc_user_provided_function.label: +"""User Provid Function Configuration""" + +republish_args_payload.desc: +"""The payload of the message to be re-published. +Template with variables is allowed, see description of the 'republish_args'. +Defaults to ${payload}. If variable ${payload} is not found from the selected result +of the rule, then the string "undefined" is used.""" + +republish_args_payload.label: +"""Message Payload""" + +republish_args_qos.desc: +"""The qos of the message to be re-published. +Template with variables is allowed, see description of the 'republish_args'. +Defaults to ${qos}. If variable ${qos} is not found from the selected result of the rule, +0 is used.""" + +republish_args_qos.label: +"""Message QoS""" + +republish_args_retain.desc: +"""The 'retain' flag of the message to be re-published. +Template with variables is allowed, see description of the 'republish_args'. +Defaults to ${retain}. If variable ${retain} is not found from the selected result +of the rule, false is used.""" + +republish_args_retain.label: +"""Retain Flag""" + +republish_args_topic.desc: +"""The target topic of message to be re-published. +Template with variables is allowed, see description of the 'republish_args'.""" + +republish_args_topic.label: +"""Target Topic""" + +republish_args_user_properties.desc: +"""From which variable should the MQTT message's User-Property pairs be taken from. +The value must be a map. +You may configure it to ${pub_props.'User-Property'} or +use SELECT *,pub_props.'User-Property' as user_properties +to forward the original user properties to the republished message. +You may also call map_put function like +map_put('my-prop-name', 'my-prop-value', user_properties) as user_properties +to inject user properties. +NOTE: MQTT spec allows duplicated user property names, but EMQX Rule-Engine does not.""" + +republish_function.desc: +"""Republish the message as a new MQTT message""" + +republish_function.label: +"""Republish Function""" + +rule_engine_ignore_sys_message.desc: +"""When set to 'true' (default), rule-engine will ignore messages published to $SYS topics.""" + +rule_engine_ignore_sys_message.label: +"""Ignore Sys Message""" + +rule_engine_jq_function_default_timeout.desc: +"""Default timeout for the `jq` rule engine function""" + +rule_engine_jq_function_default_timeout.label: +"""Rule engine jq function default timeout""" + +rule_engine_jq_implementation_module.desc: +"""The implementation module for the jq rule engine function. The two options are jq_nif and jq_port. With the jq_nif option an Erlang NIF library is used while with the jq_port option an implementation based on Erlang port programs is used. The jq_nif option (the default option) is the fastest implementation of the two but jq_port is safer as the jq programs will not execute in the same process as the Erlang VM.""" + +rule_engine_jq_implementation_module.label: +"""JQ Implementation Module""" + +rule_engine_rules.desc: +"""The rules""" + +rule_engine_rules.label: +"""Rules""" + +rules_actions.desc: +"""A list of actions of the rule. +An action can be a string that refers to the channel ID of an EMQX bridge, or an object +that refers to a function. +There a some built-in functions like "republish" and "console", and we also support user +provided functions in the format: "{module}:{function}". +The actions in the list are executed sequentially. +This means that if one of the action is executing slowly, all the following actions will not +be executed until it returns. +If one of the action crashed, all other actions come after it will still be executed, in the +original order. +If there's any error when running an action, there will be an error message, and the 'failure' +counter of the function action or the bridge channel will increase.""" + +rules_actions.label: +"""Rule Action List""" + +rules_description.desc: +"""The description of the rule""" + +rules_description.label: +"""Rule Description""" + +rules_enable.desc: +"""Enable or disable the rule""" + +rules_enable.label: +"""Enable Or Disable Rule""" + +rules_metadata.desc: +"""Rule metadata, do not change manually""" + +rules_metadata.label: +"""Rule metadata""" + +rules_name.desc: +"""The name of the rule""" + +rules_name.label: +"""Rule Name""" + +rules_sql.desc: +"""SQL query to transform the messages. +Example: SELECT * FROM "test/topic" WHERE payload.x = 1""" + +rules_sql.label: +"""Rule SQL""" + +user_provided_function_args.desc: +"""The args will be passed as the 3rd argument to module:function/3, +checkout the function console and republish in the source file: +apps/emqx_rule_engine/src/emqx_rule_actions.erl as an example.""" + +user_provided_function_args.label: +"""User Provided Function Args""" + +user_provided_function_function.desc: +"""The user provided function. Should be in the format: '{module}:{function}'. +Where {module} is the Erlang callback module and {function} is the Erlang function. + +To write your own function, checkout the function console and +republish in the source file: +apps/emqx_rule_engine/src/emqx_rule_actions.erl as an example.""" + +user_provided_function_function.label: +"""User Provided Function""" } diff --git a/rel/i18n/emqx_schema.hocon b/rel/i18n/emqx_schema.hocon index d36809c3b..9c2a1530d 100644 --- a/rel/i18n/emqx_schema.hocon +++ b/rel/i18n/emqx_schema.hocon @@ -1,1582 +1,441 @@ emqx_schema { - force_shutdown_enable { - desc { - en: "Enable `force_shutdown` feature." - zh: "启用 `force_shutdown` 功能。" - } - label { - en: "Enable `force_shutdown` feature" - zh: "启用 `force_shutdown` 功能" - } - } +fields_mqtt_quic_listener_peer_unidi_stream_count.desc: +"""Number of unidirectional streams to allow the peer to open.""" - force_shutdown_max_message_queue_len { - desc { - en: "Maximum message queue length." - zh: "消息队列的最大长度。" - } - label { - en: "Maximum mailbox queue length of process." - zh: "进程邮箱消息队列的最大长度" - } - } +fields_mqtt_quic_listener_peer_unidi_stream_count.label: +"""Peer unidi stream count""" - force_shutdown_max_heap_size { - desc { - en: "Total heap size" - zh: "Heap 的总大小。" - } - label { - en: "Total heap size" - zh: "Heap 的总大小" - } - } +fields_authorization_no_match.desc: +"""Default access control action if the user or client matches no ACL rules, +or if no such user or client is found by the configurable authorization +sources such as built_in_database, an HTTP API, or a query against PostgreSQL. +Find more details in 'authorization.sources' config.""" - overload_protection_enable { - desc { - en: "React on system overload or not." - zh: "是否对系统过载做出反应。" - } - label { - en: "React on system overload or not" - zh: "是否对系统过载做出反应" - } - } +fields_authorization_no_match.label: +"""Authorization no match""" - overload_protection_backoff_delay { - desc { - en: "The maximum duration of delay for background task execution during high load conditions." - zh: "高负载时，一些不重要的任务可能会延迟执行，在这里设置允许延迟的时间。" - } - label { - en: "Delay Time" - zh: "延迟时间" - } - } +sysmon_top_db_hostname.desc: +"""Hostname of the PostgreSQL database that collects the data points""" - overload_protection_backoff_gc { - desc { - en: "When at high load, skip forceful GC." - zh: "高负载时，跳过强制 GC。" - } - label { - en: "Skip GC" - zh: "跳过GC" - } - } +sysmon_top_db_hostname.label: +"""DB Hostname""" - overload_protection_backoff_hibernation { - desc { - en: "When at high load, skip process hibernation." - zh: "高负载时，跳过进程休眠。" - } - label { - en: "Skip hibernation" - zh: "跳过休眠" - } - } - - overload_protection_backoff_new_conn { - desc { - en: "When at high load, close new incoming connections." - zh: "高负载时，拒绝新进来的客户端连接。" - } - label { - en: "Close new connections" - zh: "关闭新连接" - } - } - - conn_congestion_enable_alarm { - desc { - en: "Enable or disable connection congestion alarm." - zh: "启用或者禁用连接阻塞告警功能。" - } - label { - en: "Enable/disable congestion alarm" - zh: "启用/禁用阻塞告警" - } - } - - conn_congestion_min_alarm_sustain_duration { - desc { - en: "Minimal time before clearing the alarm.
" - "The alarm is cleared only when there's no pending data in
" - "the queue, and at least min_alarm_sustain_duration" - "milliseconds passed since the last time we considered the connection 'congested'.
" - "This is to avoid clearing and raising the alarm again too often." - zh: "清除警报前的最短时间。
" - "只有当队列中没有挂起的数据，并且连接至少被堵塞了 min_alarm_sustain_duration 毫秒时，
" - "报警才会被清除。这是为了避免太频繁地清除和再次发出警报。" - } - label { - en: "Sustain duration" - zh: "告警维持时间" - } - } - - force_gc_enable { - desc { - en: "Enable forced garbage collection." - zh: "启用强制垃圾回收。" - } - label { - en: "Enable forced garbage collection" - zh: "启用强制垃圾回收" - } - } - - force_gc_count { - desc { - en: "GC the process after this many received messages." - zh: "在进程收到多少消息之后，对此进程执行垃圾回收。" - } - label { - en: "Process GC messages num" - zh: "垃圾回收消息数" - } - } - - force_gc_bytes { - desc { - en: "GC the process after specified number of bytes have passed through." - zh: "在进程处理过多少个字节之后，对此进程执行垃圾回收。" - } - label { - en: "Process GC bytes" - zh: "垃圾回收字节数" - } - } - - sysmon_vm_process_check_interval { - desc { - en: "The time interval for the periodic process limit check." - zh: "定期进程限制检查的时间间隔。" - } - label { - en: "Process limit check interval" - zh: "进程限制检查时间" - } - } - - sysmon_vm_process_high_watermark { - desc { - en: "The threshold, as percentage of processes, for how many\n" - " processes can simultaneously exist at the local node before the corresponding\n" - " alarm is raised." - zh: "在发出相应警报之前，本地节点上可以同时存在多少进程的阈值（以进程百分比表示）。" - } - label { - en: "Process high watermark" - zh: "进程数高水位线" - } - } - - sysmon_vm_process_low_watermark { - desc { - en: "The threshold, as percentage of processes, for how many\n" - " processes can simultaneously exist at the local node before the corresponding\n" - " alarm is cleared." - zh: "在清除相应警报之前，本地节点上可以同时存在多少进程的阈值（以进程百分比表示）。" - } - label { - en: "Process low watermark" - zh: "进程数低水位线" - } - } - - sysmon_vm_long_gc { - desc { - en: """When an Erlang process spends long time to perform garbage collection, a warning level long_gc log is emitted, -and an MQTT message is published to the system topic $SYS/sysmon/long_gc.""" - zh: """当系统检测到某个 Erlang 进程垃圾回收占用过长时间，会触发一条带有 long_gc 关键字的日志。 -同时还会发布一条主题为 $SYS/sysmon/long_gc 的 MQTT 系统消息。""" - } - label { - en: "Enable Long GC monitoring." - zh: "启用长垃圾回收监控" - } - } - - sysmon_vm_long_schedule { - desc { - en: """When the Erlang VM detect a task scheduled for too long, a warning level 'long_schedule' log is emitted, -and an MQTT message is published to the system topic $SYS/sysmon/long_schedule.""" - zh: """启用后，如果 Erlang VM 调度器出现某个任务占用时间过长时，会触发一条带有 'long_schedule' 关键字的日志。 -同时还会发布一条主题为 $SYS/sysmon/long_schedule 的 MQTT 系统消息。""" - } - label { - en: "Enable Long Schedule monitoring." - zh: "启用长调度监控" - } - } - - sysmon_vm_large_heap { - desc { - en: """When an Erlang process consumed a large amount of memory for its heap space, -the system will write a warning level large_heap log, and an MQTT message is published to -the system topic $SYS/sysmon/large_heap.""" - zh: """启用后，当一个 Erlang 进程申请了大量内存，系统会触发一条带有 large_heap 关键字的 -warning 级别日志。同时还会发布一条主题为 $SYS/sysmon/busy_dist_port 的 MQTT 系统消息。""" - } - label { - en: "Enable Large Heap monitoring." - zh: "启用大 heap 监控" - } - } - - sysmon_vm_busy_dist_port { - desc { - en: """When the RPC connection used to communicate with other nodes in the cluster is overloaded, -there will be a busy_dist_port warning log, -and an MQTT message is published to system topic $SYS/sysmon/busy_dist_port.""" - zh: """启用后，当用于集群接点之间 RPC 的连接过忙时，会触发一条带有 busy_dist_port 关键字的 warning 级别日志。 -同时还会发布一条主题为 $SYS/sysmon/busy_dist_port 的 MQTT 系统消息。""" - } - label { - en: "Enable Busy Distribution Port monitoring." - zh: "启用分布式端口过忙监控" - } - } - - sysmon_vm_busy_port { - desc { - en: """When a port (e.g. TCP socket) is overloaded, there will be a busy_port warning log, -and an MQTT message is published to the system topic $SYS/sysmon/busy_port.""" - zh: """当一个系统接口（例如 TCP socket）过忙，会触发一条带有 busy_port 关键字的 warning 级别的日志。 -同时还会发布一条主题为 $SYS/sysmon/busy_port 的 MQTT 系统消息。""" - } - label { - en: "Enable Busy Port monitoring." - zh: "启用端口过忙监控" - } - } - - sysmon_os_cpu_check_interval { - desc { - en: "The time interval for the periodic CPU check." - zh: "定期 CPU 检查的时间间隔。" - } - label { - en: "The time interval for the periodic CPU check." - zh: "定期 CPU 检查的时间间隔" - } - } - - sysmon_os_cpu_high_watermark { - desc { - en: "The threshold, as percentage of system CPU load,\n" - " for how much system cpu can be used before the corresponding alarm is raised." - zh: "在发出相应警报之前可以使用多少系统 CPU 的阈值，以系统CPU负载的百分比表示。" - } - label { - en: "CPU high watermark" - zh: "CPU 高水位线" - } - } - - sysmon_os_cpu_low_watermark { - desc { - en: "The threshold, as percentage of system CPU load,\n" - " for how much system cpu can be used before the corresponding alarm is cleared." - zh: "在解除相应警报之前可以使用多少系统 CPU 的阈值，以系统CPU负载的百分比表示。" - } - label { - en: "CPU low watermark" - zh: "CPU 低水位线" - } - } - - sysmon_os_mem_check_interval { - desc { - en: "The time interval for the periodic memory check." - zh: "定期内存检查的时间间隔。" - } - label { - en: "Mem check interval" - zh: "内存检查间隔" - } - } - - sysmon_os_sysmem_high_watermark { - desc { - en: "The threshold, as percentage of system memory,\n" - " for how much system memory can be allocated before the corresponding alarm is raised." - zh: "在发出相应报警之前可以分配多少系统内存的阈值，以系统内存的百分比表示。" - } - label { - en: "SysMem high wartermark" - zh: "系统内存高水位线" - } - } - - sysmon_os_procmem_high_watermark { - desc { - en: "The threshold, as percentage of system memory,\n" - " for how much system memory can be allocated by one Erlang process before\n" - " the corresponding alarm is raised." - zh: "在发出相应警报之前，一个Erlang进程可以分配多少系统内存的阈值，以系统内存的百分比表示。" - } - label { - en: "ProcMem high wartermark" - zh: "进程内存高水位线" - } - } - - sysmon_top_num_items { - desc { - en: "The number of top processes per monitoring group" - zh: "每个监视组的顶级进程数。" - } - label { - en: "Top num items" - zh: "顶级进程数" - } - } - - sysmon_top_sample_interval { - desc { - en: "Specifies how often process top should be collected" - zh: "指定应收集进程顶部的频率。" - } - label { - en: "Top sample interval" - zh: "取样时间" - } - } - - sysmon_top_max_procs { - desc { - en: "Stop collecting data when the number of processes\n" - "in the VM exceeds this value" - zh: "当 VM 中的进程数超过此值时，停止收集数据。" - } - label { - en: "Max procs" - zh: "最大进程数" - } - } - - sysmon_top_db_hostname { - desc { - en: "Hostname of the PostgreSQL database that collects the data points" - zh: "收集数据点的 PostgreSQL 数据库的主机名。" - } - label { - en: "DB Hostname" - zh: "数据库主机名" - } - } - - sysmon_top_db_port { - desc { - en: "Port of the PostgreSQL database that collects the data points." - zh: "收集数据点的 PostgreSQL 数据库的端口。" - } - label { - en: "DB Port" - zh: "数据库端口" - } - } - - sysmon_top_db_username { - desc { - en: "Username of the PostgreSQL database" - zh: "PostgreSQL 数据库的用户名" - } - label { - en: "DB Username" - zh: "数据库用户名" - } - } - - sysmon_top_db_password { - desc { - en: "EMQX user password in the PostgreSQL database" - zh: "PostgreSQL 数据库的密码" - } - label { - en: "DB Password" - zh: "数据库密码" - } - } - - sysmon_top_db_name { - desc { - en: "PostgreSQL database name" - zh: "PostgreSQL 数据库的数据库名" - } - label { - en: "DB Name" - zh: "数据库名" - } - } - - alarm_actions { - desc { - en: "The actions triggered when the alarm is activated.
" - "Currently, the following actions are supported: log and " - "publish.\n" - "log is to write the alarm to log (console or file).\n" - "publish is to publish the alarm as an MQTT message to " - "the system topics:\n" - "$SYS/brokers/emqx@xx.xx.xx.x/alarms/activate and\n" - "$SYS/brokers/emqx@xx.xx.xx.x/alarms/deactivate" - zh: "警报激活时触发的动作。
" - "目前，支持以下操作：log 和 " - "publish.\n" - "log 将告警写入日志 (控制台或者文件).\n" - "publish 将告警作为 MQTT 消息发布到系统主题:\n" - "$SYS/brokers/emqx@xx.xx.xx.x/alarms/activate and\n" - "$SYS/brokers/emqx@xx.xx.xx.x/alarms/deactivate" - } - label: { - en: "Alarm Actions" - zh: "告警动作" - } - } - - alarm_size_limit { - desc { - en: "The maximum total number of deactivated alarms to keep as history.
" - "When this limit is exceeded, the oldest deactivated alarms are " - "deleted to cap the total number." - zh: "要保留为历史记录的已停用报警的最大总数。当超过此限制时，将删除最旧的停用报警，以限制总数。" - } - label: { - en: "Alarm size limit" - zh: "告警总数限制" - } - } - - alarm_validity_period { - desc { - en: "Retention time of deactivated alarms. Alarms are not deleted immediately\n" - "when deactivated, but after the retention time." - zh: "停用报警的保留时间。报警在停用时不会立即删除，而是在保留时间之后删除。" - } - label: { - en: "Alarm validity period" - zh: "告警保留时间" - } - } - - flapping_detect_enable { - desc { - en: "Enable flapping connection detection feature." - zh: "启用抖动检测功能。" - } - label: { - en: "Enable flapping detection" - zh: "启用抖动检测" - } - } - - flapping_detect_max_count { - desc { - en: "The maximum number of disconnects allowed for a MQTT Client in `window_time`" - zh: "MQTT 客户端在“窗口”时间内允许的最大断开次数。" - } - label: { - en: "Max count" - zh: "最大断开次数" - } - } - - flapping_detect_window_time { - desc { - en: "The time window for flapping detection." - zh: "抖动检测的时间窗口。" - } - label: { - en: "Window time" - zh: "时间窗口" - } - } - - flapping_detect_ban_time { - desc { - en: "How long the flapping clientid will be banned." - zh: "抖动的客户端将会被禁止登录多长时间。" - } - label: { - en: "Ban time" - zh: "禁止登录时长" - } - } - - persistent_session_store_enabled { - desc { - en: "Use the database to store information about persistent sessions.\n" - "This makes it possible to migrate a client connection to another\n" - "cluster node if a node is stopped." - zh: "使用数据库存储有关持久会话的信息。\n" - "这使得在节点停止时，可以将客户端连接迁移到另一个群集节点。" - } - label: { - en: "Enable persistent session store" - zh: "启用持久会话保存" - } - } - - persistent_session_store_backend { - desc { - en: "Database management system used to store information about persistent sessions and messages.\n" - "- `builtin`: Use the embedded database (mria)" - zh: "用于存储持久性会话和信息的数据库管理后端\n" - "- `builtin`: 使用内置的数据库（mria）" - } - label: { - en: "Backend" - zh: "后端类型" - } - } - - persistent_store_on_disc { - desc { - en: "Save information about the persistent sessions on disc.\n" - "If this option is enabled, persistent sessions will survive full restart of the cluster.\n" - "Otherwise, all the data will be stored in RAM, and it will be lost when all the nodes in the cluster are stopped." - zh: "将持久会话数据保存在磁盘上。如果为 false 则存储在内存中。\n" - "如开启， 持久会话数据可在集群重启后恢复。\n" - "如关闭， 数据仅存储在内存中， 则在整个集群停止后丢失。" - } - label: { - en: "Persist on disc" - zh: "持久化在磁盘上" - } - } - - persistent_store_ram_cache { - desc { - en: "Maintain a copy of the data in RAM for faster access." - zh: "在内存中保持一份数据的副本，以便更快地访问。" - } - label: { - en: "RAM cache" - zh: "内存缓存" - } - } - - persistent_session_store_max_retain_undelivered { - desc { - en: "The time messages that was not delivered to a persistent session\n" - "is stored before being garbage collected if the node the previous\n" - "session was handled on restarts of is stopped." - zh: "如果重新启动时处理上一个会话的节点已停止，则未传递到持久会话的消息在垃圾收集之前会被存储。" - } - label: { - en: "Max retain undelivered" - zh: "未投递的消息保留条数" - } - } - - persistent_session_store_message_gc_interval { - desc { - en: "The starting interval for garbage collection of undelivered messages to\n" - "a persistent session. This affects how often the \"max_retain_undelivered\"\n" - "is checked for removal." - zh: "将未送达的消息垃圾收集到持久会话的开始间隔。\n" - "这会影响检查 \"max_retain_undelivered\"（最大保留未送达）的删除频率。" - } - label: { - en: "Message GC interval" - zh: "消息清理间隔" - } - } - - persistent_session_store_session_message_gc_interval { - desc { - en: "The starting interval for garbage collection of transient data for\n" - "persistent session messages. This does not affect the lifetime length\n" - "of persistent session messages." - zh: "持久会话消息的临时数据垃圾收集的开始间隔。\n" - "这不会影响持久会话消息的生命周期长度。" - } - label: { - en: "Session message GC interval" - zh: "会话消息清理间隔" - } - } - - persistent_session_builtin_session_table { - desc { - en: "Performance tuning options for built-in session table." - zh: "用于内建会话表的性能调优参数。" - } - label: { - en: "Persistent session" - zh: "持久会话" - } - } - - persistent_session_builtin_sess_msg_table { - desc { - en: "Performance tuning options for built-in session messages table." - zh: "优化内置的会话消息表的配置。" - } - label: { - en: "Persistent session messages" - zh: "用于内建会话管理表的性能调优参数" - } - } - - persistent_session_builtin_messages_table { - desc { - en: "Performance tuning options for built-in messages table." - zh: "用于内建消息表的性能调优参数。" - } - label: { - en: "Persistent messages" - zh: "持久化消息" - } - } - - stats_enable { - desc { - en: "Enable/disable statistic data collection." - zh: "启用/禁用统计数据收集功能。" - } - label: { - en: "Enable/disable statistic data collection." - zh: "启用/禁用统计数据收集功能" - } - } - - zones { - desc { - en: """A zone is a set of configs grouped by the zone name. +zones.desc: +"""A zone is a set of configs grouped by the zone name. For flexible configuration mapping, the name can be set to a listener's zone config. NOTE: A built-in zone named default is auto created and can not be deleted.""" - zh: """zone 是按name 分组的一组配置。 -对于灵活的配置映射，可以将 name 设置为侦听器的 zone 配置。 -注：名为 default 的内置区域是自动创建的，无法删除。""" - } - } - mqtt { - desc { - en: """Global MQTT configuration. -The configs here work as default values which can be overridden in zone configs""" - zh: """全局的 MQTT 配置项。 -mqtt 下所有的配置作为全局的默认值存在，它可以被 zone 中的配置覆盖。""" - } - } +fields_mqtt_quic_listener_certfile.desc: +"""Path to the certificate file. Will be deprecated in 5.1, use .ssl_options.certfile instead.""" - mqtt_idle_timeout { - desc { - en: """Configure the duration of time that a connection can remain idle (i.e., without any data transfer) before being: - - Automatically disconnected if no CONNECT package is received from the client yet. - - Put into hibernation mode to save resources if some CONNECT packages are already received. -Note: Please set the parameter with caution as long idle time will lead to resource waste.""" - zh: """设置连接被断开或进入休眠状态前的等待时间，空闲超时后， - - 如暂未收到客户端的 CONNECT 报文，连接将断开； - - 如已收到客户端的 CONNECT 报文，连接将进入休眠模式以节省系统资源。 +fields_mqtt_quic_listener_certfile.label: +"""Certificate file""" -注意：请合理设置该参数值，如等待时间设置过长，可能造成系统资源的浪费。""" - } - label: { - en: """Idle Timeout""" - zh: """空闲超时""" - } - } +fields_rate_limit_conn_bytes_in.desc: +"""Limit the rate of receiving packets for a MQTT connection. +The rate is counted by bytes of packets per second.""" - mqtt_max_packet_size { - desc { - en: """Maximum MQTT packet size allowed.""" - zh: """允许的最大 MQTT 报文大小。""" - } - label: { - en: """Max Packet Size""" - zh: """最大报文大小""" - } - } +fields_rate_limit_conn_bytes_in.label: +"""Connection bytes in""" - mqtt_max_clientid_len { - desc { - en: """Maximum allowed length of MQTT Client ID.""" - zh: """允许的最大 MQTT Client ID 长度。""" - } - label: { - en: """Max Client ID Length""" - zh: """最大 Client ID 长度""" - } - } +crl_cache_capacity.desc: +"""The maximum number of CRL URLs that can be held in cache. If the cache is at full capacity and a new URL must be fetched, then it'll evict the oldest inserted URL in the cache.""" - mqtt_max_topic_levels { - desc { - en: """Maximum topic levels allowed.""" - zh: """允许的最大主题层级。""" - } - label: { - en: """Max Topic Levels""" - zh: """最大主题层级""" - } - } +crl_cache_capacity.label: +"""CRL Cache Capacity""" - mqtt_max_qos_allowed { - desc { - en: """Maximum QoS allowed.""" - zh: """允许的最大 QoS 等级。""" - } - label: { - en: """Max QoS""" - zh: """最大 QoS""" - } - } +alarm_actions.desc: +"""The actions triggered when the alarm is activated.
Currently, the following actions are supported: log and publish. +log is to write the alarm to log (console or file). +publish is to publish the alarm as an MQTT message to the system topics: +$SYS/brokers/emqx@xx.xx.xx.x/alarms/activate and +$SYS/brokers/emqx@xx.xx.xx.x/alarms/deactivate""" - mqtt_max_topic_alias { - desc { - en: """Maximum topic alias, 0 means no topic alias supported.""" - zh: """允许的最大主题别名数，0 表示不支持主题别名。""" - } - label: { - en: """Max Topic Alias""" - zh: """最大主题别名""" - } - } +alarm_actions.label: +"""Alarm Actions""" - mqtt_retain_available { - desc { - en: """Whether to enable support for MQTT retained message.""" - zh: """是否启用对 MQTT 保留消息的支持。""" - } - label: { - en: """Retain Available""" - zh: """保留消息可用""" - } - } +base_listener_max_connections.desc: +"""The maximum number of concurrent connections allowed by the listener.""" - mqtt_wildcard_subscription { - desc { - en: """Whether to enable support for MQTT wildcard subscription.""" - zh: """是否启用对 MQTT 通配符订阅的支持。""" - } - label: { - en: """Wildcard Subscription Available""" - zh: """通配符订阅可用""" - } - } +base_listener_max_connections.label: +"""Max connections""" - mqtt_shared_subscription { - desc { - en: """Whether to enable support for MQTT shared subscription.""" - zh: """是否启用对 MQTT 共享订阅的支持。""" - } - label: { - en: """Shared Subscription Available""" - zh: """共享订阅可用""" - } - } - - mqtt_exclusive_subscription { - desc { - en: """Whether to enable support for MQTT exclusive subscription.""" - zh: """是否启用对 MQTT 排它订阅的支持。""" - } - label: { - en: """Exclusive Subscription""" - zh: """排它订阅""" - } - } - - mqtt_ignore_loop_deliver { - desc { - en: """Whether the messages sent by the MQTT v3.1.1/v3.1.0 client will be looped back to the publisher itself, similar to No Local in MQTT 5.0.""" - zh: """设置由 MQTT v3.1.1/v3.1.0 客户端发布的消息是否将转发给其本身；类似 MQTT 5.0 协议中的 No Local 选项。""" - } - label: { - en: """Ignore Loop Deliver""" - zh: """忽略循环投递""" - } - } - - mqtt_strict_mode { - desc { - en: """Whether to parse MQTT messages in strict mode. -In strict mode, invalid utf8 strings in for example client ID, topic name, etc. will cause the client to be disconnected.""" - zh: """是否以严格模式解析 MQTT 消息。 -严格模式下，如客户端 ID、主题名称等中包含无效 utf8 字符串，连接将被断开。""" - } - label: { - en: """Strict Mode""" - zh: """严格模式""" - } - } - - mqtt_response_information { - desc { - en: """UTF-8 string, for creating the response topic, for example, if set to reqrsp/, the publisher/subscriber will communicate using the topic prefix reqrsp/. -To disable this feature, input \"\" in the text box below. Only applicable to MQTT 5.0 clients.""" - zh: """UTF-8 字符串，用于指定返回给客户端的响应主题，如 reqrsp/，此时请求和应答客户端都需要使用 reqrsp/ 前缀的主题来完成通讯。 -如希望禁用此功能，请在下方的文字框中输入\"\"；仅适用于 MQTT 5.0 客户端。""" - } - label: { - en: """Response Information""" - zh: """响应信息""" - } - } - - mqtt_server_keepalive { - desc { - en: """The keep alive duration required by EMQX. To use the setting from the client side, choose disabled from the drop-down list. Only applicable to MQTT 5.0 clients.""" - zh: """EMQX 要求的保活时间，如设为 disabled，则将使用客户端指定的保持连接时间；仅适用于 MQTT 5.0 客户端。""" - } - label: { - en: """Server Keep Alive""" - zh: """服务端保活时间""" - } - } - - mqtt_keepalive_backoff { - desc { - en: """The coefficient EMQX uses to confirm whether the keep alive duration of the client expires. Formula: Keep Alive * Backoff * 2""" - zh: """EMQX 判定客户端保活超时使用的阈值系数。计算公式为：Keep Alive * Backoff * 2""" - } - label: { - en: """Keep Alive Backoff""" - zh: """保活超时阈值系数""" - } - } - - mqtt_max_subscriptions { - desc { - en: """Maximum number of subscriptions allowed per client.""" - zh: """允许每个客户端建立的最大订阅数量。""" - } - label: { - en: """Max Subscriptions""" - zh: """最大订阅数量""" - } - } - - mqtt_upgrade_qos { - desc { - en: """Force upgrade of QoS level according to subscription.""" - zh: """投递消息时，是否根据订阅主题时的 QoS 等级来强制提升派发的消息的 QoS 等级。""" - } - label: { - en: """Upgrade QoS""" - zh: """升级 QoS""" - } - } - - mqtt_max_inflight { - desc { - en: """Maximum number of QoS 1 and QoS 2 messages that are allowed to be delivered simultaneously before completing the acknowledgment.""" - zh: """允许在完成应答前同时投递的 QoS 1 和 QoS 2 消息的最大数量。""" - } - label: { - en: """Max Inflight""" - zh: """最大飞行窗口""" - } - - } - - mqtt_retry_interval { - desc { - en: """Retry interval for QoS 1/2 message delivering.""" - zh: """QoS 1/2 消息的重新投递间隔。""" - } - label: { - en: """Retry Interval""" - zh: """重试间隔""" - } - } - - mqtt_max_awaiting_rel { - desc { - en: """For each publisher session, the maximum number of outstanding QoS 2 messages pending on the client to send PUBREL. After reaching this limit, new QoS 2 PUBLISH requests will be rejected with `147(0x93)` until either PUBREL is received or timed out.""" - zh: """每个发布者的会话中，都存在一个队列来处理客户端发送的 QoS 2 消息。该队列会存储 QoS 2 消息的报文 ID 直到收到客户端的 PUBREL 或超时，达到队列长度的限制后，新的 QoS 2 消息发布会被拒绝，并返回 `147(0x93)` 错误。""" - } - label: { - en: """Max Awaiting PUBREL""" - zh: """PUBREL 等待队列长度""" - } - } - - mqtt_await_rel_timeout { - desc { - en: """For client to broker QoS 2 message, the time limit for the broker to wait before the `PUBREL` message is received. The wait is aborted after timed out, meaning the packet ID is freed for new `PUBLISH` requests. Receiving a stale `PUBREL` causes a warning level log. Note, the message is delivered to subscribers before entering the wait for PUBREL.""" - zh: """客户端发布 QoS 2 消息时，服务器等待 `PUBREL` 的最长时延。超过该时长后服务器会放弃等待，该PACKET ID 会被释放，从而允许后续新的 PUBLISH 消息使用。如果超时后收到 PUBREL，服务器将会产生一条告警日志。注意，向订阅客户端转发消息的动作发生在进入等待之前。""" - } - label: { - en: """Max Awaiting PUBREL TIMEOUT""" - zh: """PUBREL 最大等待时间""" - } - } - - mqtt_session_expiry_interval { - desc { - en: """Specifies how long the session will expire after the connection is disconnected, only for non-MQTT 5.0 connections.""" - zh: """指定会话将在连接断开后多久过期，仅适用于非 MQTT 5.0 的连接。""" - } - label: { - en: """Session Expiry Interval""" - zh: """会话过期间隔""" - } - } - - mqtt_max_mqueue_len { - desc { - en: """Maximum queue length. Enqueued messages when persistent client disconnected, or inflight window is full.""" - zh: """消息队列最大长度。持久客户端断开连接或飞行窗口已满时排队的消息长度。""" - } - label: { - en: """Max Message Queue Length""" - zh: """最大消息队列长度""" - } - } - - mqtt_mqueue_priorities { - desc { - en: """Topic priorities. Priority number [1-255] -There's no priority table by default, hence all messages are treated equal. - -**NOTE**: Comma and equal signs are not allowed for priority topic names. -**NOTE**: Messages for topics not in the priority table are treated as either highest or lowest priority depending on the configured value for mqtt.mqueue_default_priority. - -**Examples**: -To configure \"topic/1\" > \"topic/2\": -mqueue_priorities: {\"topic/1\": 10, \"topic/2\": 8}""" - zh: """主题优先级。取值范围 [1-255] -默认优先级表为空，即所有的主题优先级相同。 - -注：优先主题名称中不支持使用逗号和等号。 -注：不在此列表中的主题，被视为最高/最低优先级，这取决于mqtt.mqueue_default_priority 的配置 - -示例： -配置 \"topic/1\" > \"topic/2\": -mqueue_priorities: {\"topic/1\": 10, \"topic/2\": 8}""" - } - label: { - en: """Topic Priorities""" - zh: """主题优先级""" - } - } - - mqtt_mqueue_default_priority { - desc { - en: """Default topic priority, which will be used by topics not in Topic Priorities (mqueue_priorities).""" - zh: """默认的主题优先级，不在 主题优先级（mqueue_priorities） 中的主题将会使用该优先级。""" - } - label: { - en: """Default Topic Priorities""" - zh: """默认主题优先级""" - } - } - - mqtt_mqueue_store_qos0 { - desc { - en: """Specifies whether to store QoS 0 messages in the message queue while the connection is down but the session remains.""" - zh: """指定在连接断开但会话保持期间，是否需要在消息队列中存储 QoS 0 消息。""" - } - label: { - en: """Store QoS 0 Message""" - zh: """存储 QoS 0 消息""" - } - } - - mqtt_use_username_as_clientid { - desc { - en: """Whether to use Username as Client ID. -This setting takes effect later than Use Peer Certificate as Username and Use peer certificate as Client ID.""" - zh: """是否使用用户名作为客户端 ID。 -此设置的作用时间晚于 对端证书作为用户名 和 对端证书作为客户端 ID。""" - } - label: { - en: """Use Username as Client ID""" - zh: """用户名作为客户端 ID""" - } - } - - mqtt_peer_cert_as_username { - desc { - en: """Use the CN, DN field in the peer certificate or the entire certificate content as Username. Only works for the TLS connection. +mqtt_peer_cert_as_username.desc: +"""Use the CN, DN field in the peer certificate or the entire certificate content as Username. Only works for the TLS connection. Supported configurations are the following: - cn: CN field of the certificate - dn: DN field of the certificate - crt: Content of the DER or PEM certificate - pem: Convert DER certificate content to PEM format and use as Username - md5: MD5 value of the DER or PEM certificate""" - zh: """使用对端证书中的 CN、DN 字段或整个证书内容来作为用户名；仅适用于 TLS 连接。 -目前支持： -- cn: 取证书的 CN 字段 -- dn: 取证书的 DN 字段 -- crt: 取 DER 或 PEM 的证书内容 -- pem: 将 DER 证书转换为 PEM 格式作为用户名 -- md5: 取 DER 或 PEM 证书内容的 MD5 值""" - } - label: { - en: """Use Peer Certificate as Username""" - zh: """对端证书作为用户名""" - } - } - mqtt_peer_cert_as_clientid { - desc { - en: """Use the CN, DN field in the peer certificate or the entire certificate content as Client ID. Only works for the TLS connection. -Supported configurations are the following: -- cn: CN field of the certificate -- dn: DN field of the certificate -- crt: DER or PEM certificate -- pem: Convert DER certificate content to PEM format and use as Client ID -- md5: MD5 value of the DER or PEM certificate""" - zh: """使用对端证书中的 CN、DN 字段或整个证书内容来作为客户端 ID。仅适用于 TLS 连接； -目前支持： -- cn: 取证书的 CN 字段 -- dn: 取证书的 DN 字段 -- crt: 取 DER 或 PEM 证书的内容 -- pem: 将 DER 证书内容转换为 PEM 格式作为客户端 ID -- md5: 取 DER 或 PEM 证书内容的 MD5 值""" - } - label: { - en: """Use Peer Certificate as Client ID""" - zh: """对端证书作为客户端 ID""" - } - } +mqtt_peer_cert_as_username.label: +"""Use Peer Certificate as Username""" - broker { - desc { - en: """Message broker options.""" - zh: """Broker 相关配置项。""" - } - } +fields_cache_enable.desc: +"""Enable or disable the authorization cache.""" - broker_enable_session_registry { - desc { - en: """Enable session registry""" - zh: """是否启用 Session Registry""" - } - } +fields_cache_enable.label: +"""Enable or disable the authorization cache.""" - broker_session_locking_strategy { - desc { - en: """Session locking strategy in a cluster. - - `local`: only lock the session on the current node - - `one`: select only one remote node to lock the session - - `quorum`: select some nodes to lock the session - - `all`: lock the session on all the nodes in the cluster""" +fields_mqtt_quic_listener_disconnect_timeout_ms.desc: +"""How long to wait for an ACK before declaring a path dead and disconnecting. Default: 16000""" - zh: """Session 在集群中的锁策略。 - - `loca`：仅锁本节点的 Session； - - `one`：任选一个其它节点加锁； - - `quorum`：选择集群中半数以上的节点加锁； - - `all`：选择所有节点加锁。""" - } - } +fields_mqtt_quic_listener_disconnect_timeout_ms.label: +"""Disconnect timeout ms""" - broker_shared_subscription_strategy { - desc { - en: """Dispatch strategy for shared subscription. - - `random`: dispatch the message to a random selected subscriber - - `round_robin`: select the subscribers in a round-robin manner - - `round_robin_per_group`: select the subscribers in round-robin fashion within each shared subscriber group - - `local`: select random local subscriber otherwise select random cluster-wide - - `sticky`: always use the last selected subscriber to dispatch, until the subscriber disconnects. - - `hash_clientid`: select the subscribers by hashing the `clientIds` - - `hash_topic`: select the subscribers by hashing the source topic""" +mqtt_max_topic_alias.desc: +"""Maximum topic alias, 0 means no topic alias supported.""" - zh: """共享订阅消息派发策略。 - - `random`：随机挑选一个共享订阅者派发； - - `round_robin`：使用 round-robin 策略派发； - - `round_robin_per_group`：在共享组内循环选择下一个成员； - - `local`：选择随机的本地成员，否则选择随机的集群范围内成员; - - `sticky`：总是使用上次选中的订阅者派发，直到它断开连接； - - `hash_clientid`：通过对发送者的客户端 ID 进行 Hash 处理来选择订阅者; - - `hash_topic`：通过对源主题进行 Hash 处理来选择订阅者。""" - } - } +mqtt_max_topic_alias.label: +"""Max Topic Alias""" - broker_shared_dispatch_ack_enabled { - desc { - en: """Deprecated, will be removed in 5.1. -Enable/disable shared dispatch acknowledgement for QoS 1 and QoS 2 messages. -This should allow messages to be dispatched to a different subscriber in the group in case the picked (based on `shared_subscription_strategy`) subscriber is offline.""" +common_ssl_opts_schema_user_lookup_fun.desc: +"""EMQX-internal callback that is used to lookup pre-shared key (PSK) identity.""" - zh: """该配置项已废弃，会在 5.1 中移除。 -启用/禁用 QoS 1 和 QoS 2 消息的共享派发确认。 -开启后，允许将消息从未及时回复 ACK 的订阅者 （例如，客户端离线） 重新派发给另外一个订阅者。""" - } - } +common_ssl_opts_schema_user_lookup_fun.label: +"""SSL PSK user lookup fun""" - broker_route_batch_clean { - desc { - en: """Enable batch clean for deleted routes.""" - zh: """是否开启批量清除路由。""" - } - } +fields_listeners_wss.desc: +"""HTTPS websocket listeners.""" - shared_subscription_group_strategy { - desc { - en: """Per group dispatch strategy for shared subscription. -This config is a map from shared subscription group name to the strategy -name. The group name should be of format `[A-Za-z0-9]`. i.e. no -special characters are allowed.""" - zh: """设置共享订阅组为单位的分发策略。该配置是一个从组名到 -策略名的一个map，组名不得包含 `[A-Za-z0-9]` 之外的特殊字符。""" - } +fields_listeners_wss.label: +"""HTTPS websocket listeners""" - } +sysmon_top_max_procs.desc: +"""Stop collecting data when the number of processes +in the VM exceeds this value""" - shared_subscription_strategy_enum { - desc { - en: """Dispatch strategy for shared subscription. -- `random`: dispatch the message to a random selected subscriber -- `round_robin`: select the subscribers in a round-robin manner -- `round_robin_per_group`: select the subscribers in round-robin fashion within each shared subscriber group -- `sticky`: always use the last selected subscriber to dispatch, -until the subscriber disconnects. -- `hash`: select the subscribers by the hash of `clientIds` -- `local`: send to a random local subscriber. If local -subscriber was not found, send to a random subscriber cluster-wide""" - zh: """共享订阅的分发策略名称。 -- `random`：随机选择一个组内成员； -- `round_robin`：循环选择下一个成员； -- `round_robin_per_group`：在共享组内循环选择下一个成员； -- `sticky`：使用上一次选中的成员； -- `hash`：根据 ClientID 哈希映射到一个成员； -- `local`：随机分发到节点本地成成员，如果本地成员不存在，则随机分发到任意一个成员。""" +sysmon_top_max_procs.label: +"""Max procs""" - } - } +mqtt_use_username_as_clientid.desc: +"""Whether to use Username as Client ID. +This setting takes effect later than Use Peer Certificate as Username and Use peer certificate as Client ID.""" - broker_perf_route_lock_type { - desc { - en: """Performance tuning for subscribing/unsubscribing a wildcard topic. -Change this parameter only when there are many wildcard topics. +mqtt_use_username_as_clientid.label: +"""Use Username as Client ID""" -NOTE: when changing from/to `global` lock, it requires all nodes in the cluster to be stopped before the change. - - `key`: mnesia transactional updates with per-key locks. Recommended for a single-node setup. - - `tab`: mnesia transactional updates with table lock. Recommended for a cluster setup. - - `global`: updates are protected with a global lock. Recommended for large clusters.""" - zh: """通配主题订阅/取消订阅性能调优。 -建议仅当通配符主题较多时才更改此参数。 +mqtt_max_qos_allowed.desc: +"""Maximum QoS allowed.""" -注：当从/更改为 `global` 锁时，它要求集群中的所有节点在更改之前停止。 - - `key`：为 Mnesia 事务涉及到的每个 key 上锁，建议单节点时使用。 - - `tab`：为 Mnesia 事务涉及到的表上锁，建议在集群中使用。 - - `global`：所以更新操作都被全局的锁保护，仅建议在超大规模集群中使用。""" - } - } +mqtt_max_qos_allowed.label: +"""Max QoS""" - broker_perf_trie_compaction { - desc { - en: """Enable trie path compaction. -Enabling it significantly improves wildcard topic subscribe rate, if wildcard topics have unique prefixes like: 'sensor/{{id}}/+/', where ID is unique per subscriber. -Topic match performance (when publishing) may degrade if messages are mostly published to topics with large number of levels. +fields_mqtt_quic_listener_max_binding_stateless_operations.desc: +"""The maximum number of stateless operations that may be queued on a binding at any one time. Default: 100""" -NOTE: This is a cluster-wide configuration. It requires all nodes to be stopped before changing it.""" - zh: """是否开启主题表压缩存储。 -启用它会显着提高通配符主题订阅率，如果通配符主题具有唯一前缀，例如：'sensor/{{id}}/+/'，其中每个订阅者的 ID 是唯一的。 -如果消息主要发布到具有大量级别的主题，则主题匹配性能（发布时）可能会降低。 +fields_mqtt_quic_listener_max_binding_stateless_operations.label: +"""Max binding stateless operations""" -注意：这是一个集群范围的配置。 它要求在更改之前停止所有节点。""" - } - } +fields_mqtt_quic_listener_stream_recv_buffer_default.desc: +"""Stream initial buffer size. Default: 4096""" - sys_topics { - desc { - en: """System topics configuration.""" - zh: """系统主题配置。""" - } - } +fields_mqtt_quic_listener_stream_recv_buffer_default.label: +"""Stream recv buffer default""" - sys_msg_interval { - desc { - en: """Time interval of publishing `$SYS` messages.""" - zh: """发送 `$SYS` 主题的间隔时间。""" - } - } +fields_mqtt_quic_listener_pacing_enabled.desc: +"""Pace sending to avoid overfilling buffers on the path. Default: 1 (Enabled)""" - sys_heartbeat_interval { - desc { - en: """Time interval for publishing following heartbeat messages: - - `$SYS/brokers//uptime` - - `$SYS/brokers//datetime`""" - zh: """发送心跳系统消息的间隔时间，它包括： - - `$SYS/brokers//uptime` - - `$SYS/brokers//datetime`""" - } - } +fields_mqtt_quic_listener_pacing_enabled.label: +"""Pacing enabled""" - sys_event_messages { - desc { - en: """Client events messages.""" - zh: """客户端事件消息。""" - } - } +mqtt_max_subscriptions.desc: +"""Maximum number of subscriptions allowed per client.""" - sys_event_client_connected { - desc { - en: """Enable to publish client connected event messages""" - zh: """是否开启客户端已连接事件消息。""" - } - } +mqtt_max_subscriptions.label: +"""Max Subscriptions""" - sys_event_client_disconnected { - desc { - en: """Enable to publish client disconnected event messages.""" - zh: """是否开启客户端已断开连接事件消息。""" - } - } +persistent_session_builtin_messages_table.desc: +"""Performance tuning options for built-in messages table.""" - sys_event_client_subscribed { - desc { - en: """Enable to publish event message that client subscribed a topic successfully.""" - zh: """是否开启客户端已成功订阅主题事件消息。""" - } - } +persistent_session_builtin_messages_table.label: +"""Persistent messages""" - sys_event_client_unsubscribed { - desc { - en: """Enable to publish event message that client unsubscribed a topic successfully.""" - zh: """是否开启客户端已成功取消订阅主题事件消息。""" - } - } +sysmon_os_cpu_low_watermark.desc: +"""The threshold, as percentage of system CPU load, + for how much system cpu can be used before the corresponding alarm is cleared.""" +sysmon_os_cpu_low_watermark.label: +"""CPU low watermark""" -fields_authorization_no_match { - desc { - en: """Default access control action if the user or client matches no ACL rules, -or if no such user or client is found by the configurable authorization -sources such as built_in_database, an HTTP API, or a query against PostgreSQL. -Find more details in 'authorization.sources' config.""" - zh: """如果用户或客户端不匹配ACL规则，或者从可配置授权源(比如内置数据库、HTTP API 或 PostgreSQL 等。)内未找 -到此类用户或客户端时，模式的认访问控制操作。 -在“授权”中查找更多详细信息。""" - } - label: { - en: "Authorization no match" - zh: "未匹时的默认授权动作" - } -} +fields_mqtt_quic_listener_tls_server_max_send_buffer.desc: +"""How much Server TLS data to buffer. Default: 8192""" -fields_authorization_deny_action { - desc { - en: """The action when the authorization check rejects an operation.""" - zh: """授权检查拒绝操作时的操作。""" - } - label: { - en: "Authorization deny action" - zh: "授权检查拒绝操作时的操作" - } -} +fields_mqtt_quic_listener_tls_server_max_send_buffer.label: +"""TLS server max send buffer""" -fields_cache_enable { - desc { - en: """Enable or disable the authorization cache.""" - zh: """启用或禁用授权缓存。""" - } - label: { - en: "Enable or disable the authorization cache." - zh: "启用或禁用授权缓存" - } -} +base_listener_bind.desc: +"""IP address and port for the listening socket.""" -fields_cache_max_size { - desc { - en: """Maximum number of cached items.""" - zh: """缓存项的最大数量。""" - } - label: { - en: "Maximum number of cached items." - zh: "缓存项的最大数量" - } -} +base_listener_bind.label: +"""IP address and port""" -fields_cache_ttl { - desc { - en: """Time to live for the cached data.""" - zh: """缓存数据的生存时间。""" - } - label: { - en: "Time to live for the cached data." - zh: "缓存数据的生存时间。" - } -} +server_ssl_opts_schema_handshake_timeout.desc: +"""Maximum time duration allowed for the handshake to complete""" -fields_deflate_opts_level { - desc { - en: """Compression level.""" - zh: """压缩级别""" - } - label: { - en: "Compression level" - zh: "压缩级别" - } -} +server_ssl_opts_schema_handshake_timeout.label: +"""Handshake timeout""" -fields_deflate_opts_mem_level { - desc { - en: """Specifies the size of the compression state.
-Lower values decrease memory usage per connection.""" - zh: """指定压缩状态的大小
-较低的值会减少每个连接的内存使用。""" - } - label: { - en: "Size of the compression state" - zh: "压缩状态大小" - } -} +fields_deflate_opts_server_context_takeover.desc: +"""Takeover means the compression state is retained between server messages.""" -fields_deflate_opts_strategy { - desc { - en: """Specifies the compression strategy.""" - zh: """指定压缩策略。""" - } - label: { - en: "compression strategy" - zh: "指定压缩策略" - } -} +fields_deflate_opts_server_context_takeover.label: +"""Server context takeover""" -fields_deflate_opts_server_context_takeover { - desc { - en: """Takeover means the compression state is retained between server messages.""" - zh: """接管意味着在服务器消息之间保留压缩状态。""" - } - label: { - en: "Server context takeover" - zh: "服务上下文接管" - } -} +mqtt_session_expiry_interval.desc: +"""Specifies how long the session will expire after the connection is disconnected, only for non-MQTT 5.0 connections.""" -fields_deflate_opts_client_context_takeover { - desc { - en: """Takeover means the compression state is retained between client messages.""" - zh: """接管意味着在客户端消息之间保留压缩状态。""" - } - label: { - en: "Client context takeover" - zh: "客户端上下文接管" - } -} +mqtt_session_expiry_interval.label: +"""Session Expiry Interval""" -fields_deflate_opts_server_max_window_bits { - desc { - en: """Specifies the size of the compression context for the server.""" - zh: """指定服务器压缩上下文的大小。""" - } - label: { - en: "Server compression max window size" - zh: "服务器压缩窗口大小" - } -} +fields_listener_enabled.desc: +"""Enable listener.""" -fields_deflate_opts_client_max_window_bits { - desc { - en: """Specifies the size of the compression context for the client.""" - zh: """指定客户端压缩上下文的大小。""" - } - label: { - en: "Client compression max window size" - zh: "压缩窗口大小" - } -} +fields_listener_enabled.label: +"""Enable listener""" -client_ssl_opts_schema_enable { - desc { - en: """Enable TLS.""" - zh: """启用 TLS。""" - } - label: { - en: "Enable TLS." - zh: "启用 TLS" - } -} +mqtt.desc: +"""Global MQTT configuration. +The configs here work as default values which can be overridden in zone configs""" -common_ssl_opts_schema_cacertfile { - desc { - en: """Trusted PEM format CA certificates bundle file.
+crl_cache_refresh_http_timeout.desc: +"""The timeout for the HTTP request when fetching CRLs. This is a global setting for all listeners.""" + +crl_cache_refresh_http_timeout.label: +"""CRL Cache Refresh HTTP Timeout""" + +fields_tcp_opts_backlog.desc: +"""TCP backlog defines the maximum length that the queue of +pending connections can grow to.""" + +fields_tcp_opts_backlog.label: +"""TCP backlog length""" + +broker_route_batch_clean.desc: +"""Enable batch clean for deleted routes.""" + +fields_mqtt_quic_listener_initial_window_packets.desc: +"""The size (in packets) of the initial congestion window for a connection. Default: 10""" + +fields_mqtt_quic_listener_initial_window_packets.label: +"""Initial window packets""" + +flapping_detect_enable.desc: +"""Enable flapping connection detection feature.""" + +flapping_detect_enable.label: +"""Enable flapping detection""" + +sysmon_top_db_password.desc: +"""EMQX user password in the PostgreSQL database""" + +sysmon_top_db_password.label: +"""DB Password""" + +fields_ws_opts_check_origins.desc: +"""List of allowed origins.
See check_origin_enable.""" + +fields_ws_opts_check_origins.label: +"""Allowed origins""" + +fields_deflate_opts_client_context_takeover.desc: +"""Takeover means the compression state is retained between client messages.""" + +fields_deflate_opts_client_context_takeover.label: +"""Client context takeover""" + +base_listener_acceptors.desc: +"""The size of the listener's receiving pool.""" + +base_listener_acceptors.label: +"""Acceptors Num""" + +common_ssl_opts_schema_cacertfile.desc: +"""Trusted PEM format CA certificates bundle file.
The certificates in this file are used to verify the TLS peer's certificates. Append new certificates to the file if new CAs are to be trusted. There is no need to restart EMQX to have the updated file loaded, because the system regularly checks if file has been updated (and reload).
NOTE: invalidating (deleting) a certificate from the file will not affect already established connections.""" - zh: """受信任的PEM格式 CA 证书捆绑文件
-此文件中的证书用于验证TLS对等方的证书。 -如果要信任新 CA，请将新证书附加到文件中。 -无需重启EMQX即可加载更新的文件，因为系统会定期检查文件是否已更新（并重新加载）
-注意：从文件中失效（删除）证书不会影响已建立的连接。""" - } - label: { - en: "CACertfile" - zh: "CA 证书文件" - } -} -common_ssl_opts_schema_certfile { - desc { - en: """PEM format certificates chain file.
-The certificates in this file should be in reversed order of the certificate -issue chain. That is, the host's certificate should be placed in the beginning -of the file, followed by the immediate issuer certificate and so on. -Although the root CA certificate is optional, it should be placed at the end of -the file if it is to be added.""" - zh: """PEM格式证书链文件
-此文件中的证书应与证书颁发链的顺序相反。也就是说，主机的证书应该放在文件的开头， -然后是直接颁发者 CA 证书，依此类推，一直到根 CA 证书。 -根 CA 证书是可选的，如果想要添加，应加到文件到最末端。""" - } - label: { - en: "Certfile" - zh: "证书文件" - } -} +common_ssl_opts_schema_cacertfile.label: +"""CACertfile""" -common_ssl_opts_schema_keyfile { - desc { - en: """PEM format private key file.""" - zh: """PEM格式的私钥文件。""" - } - label: { - en: "Keyfile" - zh: "私钥文件" - } -} +fields_ws_opts_mqtt_path.desc: +"""WebSocket's MQTT protocol path. So the address of EMQX Broker's WebSocket is: +ws://{ip}:{port}/mqtt""" -common_ssl_opts_schema_verify { - desc { - en: """Enable or disable peer verification.""" - zh: """启用或禁用对等验证。""" - } - label: { - en: "Verify peer" - zh: "对等验证" - } -} +fields_ws_opts_mqtt_path.label: +"""WS MQTT Path""" -common_ssl_opts_schema_reuse_sessions { - desc { - en: """Enable TLS session reuse.""" - zh: """启用 TLS 会话重用。""" - } - label: { - en: "TLS session reuse" - zh: "TLS 会话重用" - } -} +sysmon_os_procmem_high_watermark.desc: +"""The threshold, as percentage of system memory, + for how much system memory can be allocated by one Erlang process before + the corresponding alarm is raised.""" -common_ssl_opts_schema_depth { - desc { - en: """Maximum number of non-self-issued intermediate certificates that can follow the peer certificate in a valid certification path. -So, if depth is 0 the PEER must be signed by the trusted ROOT-CA directly;
-if 1 the path can be PEER, Intermediate-CA, ROOT-CA;
-if 2 the path can be PEER, Intermediate-CA1, Intermediate-CA2, ROOT-CA.""" - zh: """在有效的证书路径中，可以跟随对等证书的非自颁发中间证书的最大数量。 -因此，如果深度为0，则对等方必须由受信任的根 CA 直接签名；
-如果是1，路径可以是 PEER、中间 CA、ROOT-CA；
-如果是2，则路径可以是PEER、中间 CA1、中间 CA2、ROOT-CA。""" - } - label: { - en: "CACert Depth" - zh: "CA 证书深度" - } -} +sysmon_os_procmem_high_watermark.label: +"""ProcMem high wartermark""" -common_ssl_opts_schema_password { - desc { - en: """String containing the user's password. Only used if the private key file is password-protected.""" - zh: """包含用户密码的字符串。仅在私钥文件受密码保护时使用。""" - } - label: { - en: "Keyfile passphrase" - zh: "秘钥文件密码" - } -} +fields_listeners_quic.desc: +"""QUIC listeners.""" -common_ssl_opts_schema_versions { - desc { - en: """All TLS/DTLS versions to be supported.
-NOTE: PSK ciphers are suppressed by 'tlsv1.3' version config.
-In case PSK cipher suites are intended, make sure to configure -['tlsv1.2', 'tlsv1.1'] here.""" - zh: """支持所有TLS/DTLS版本
-注：PSK 的 Ciphers 无法在 tlsv1.3 中使用，如果打算使用 PSK 密码套件，请确保这里配置为 ["tlsv1.2","tlsv1.1"]。""" - } - label: { - en: "SSL versions" - zh: "SSL 版本" - } -} +fields_listeners_quic.label: +"""QUIC listeners""" -common_ssl_opts_schema_hibernate_after { - desc { - en: """Hibernate the SSL process after idling for amount of time reducing its memory footprint.""" - zh: """在闲置一定时间后休眠 SSL 进程，减少其内存占用。""" - } - label: { - en: "hibernate after" - zh: "闲置多久后休眠" - } -} +fields_listeners_ws.desc: +"""HTTP websocket listeners.""" -ciphers_schema_common { - desc { - en: """This config holds TLS cipher suite names separated by comma, -or as an array of strings. e.g. -"TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256" or -["TLS_AES_256_GCM_SHA384","TLS_AES_128_GCM_SHA256"]. -
-Ciphers (and their ordering) define the way in which the -client and server encrypts information over the network connection. -Selecting a good cipher suite is critical for the -application's data security, confidentiality and performance. +fields_listeners_ws.label: +"""HTTP websocket listeners""" -The names should be in OpenSSL string format (not RFC format). -All default values and examples provided by EMQX config -documentation are all in OpenSSL format.
+mqtt_retry_interval.desc: +"""Retry interval for QoS 1/2 message delivering.""" -NOTE: Certain cipher suites are only compatible with -specific TLS versions ('tlsv1.1', 'tlsv1.2' or 'tlsv1.3') -incompatible cipher suites will be silently dropped. -For instance, if only 'tlsv1.3' is given in the versions, -configuring cipher suites for other versions will have no effect. -
+mqtt_retry_interval.label: +"""Retry Interval""" -NOTE: PSK ciphers are suppressed by 'tlsv1.3' version config
-If PSK cipher suites are intended, 'tlsv1.3' should be disabled from versions.
-PSK cipher suites: "RSA-PSK-AES256-GCM-SHA384,RSA-PSK-AES256-CBC-SHA384, -RSA-PSK-AES128-GCM-SHA256,RSA-PSK-AES128-CBC-SHA256, -RSA-PSK-AES256-CBC-SHA,RSA-PSK-AES128-CBC-SHA, -RSA-PSK-DES-CBC3-SHA,RSA-PSK-RC4-SHA"""" - zh: """此配置保存由逗号分隔的 TLS 密码套件名称，或作为字符串数组。例如 -"TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256"或 -["TLS_AES_256_GCM_SHA384","TLS_AES_128_GCM_SHA256"]。 -
-密码（及其顺序）定义了客户端和服务器通过网络连接加密信息的方式。 -选择一个好的密码套件对于应用程序的数据安全性、机密性和性能至关重要。 +stats_enable.desc: +"""Enable/disable statistic data collection.""" -名称应为 OpenSSL 字符串格式（而不是 RFC 格式）。 -EMQX 配置文档提供的所有默认值和示例都是 OpenSSL 格式
-注意：某些密码套件仅与特定的 TLS 版本兼容（'tlsv1.1'、'tlsv1.2'或'tlsv1.3'）。 -不兼容的密码套件将被自动删除。 +stats_enable.label: +"""Enable/disable statistic data collection.""" -例如，如果只有 versions 仅配置为 tlsv1.3。为其他版本配置密码套件将无效。 +fields_authorization_deny_action.desc: +"""The action when the authorization check rejects an operation.""" -
-注：PSK 的 Ciphers 不支持 tlsv1.3
-如果打算使用PSK密码套件 tlsv1.3。应在ssl.versions中禁用。 +fields_authorization_deny_action.label: +"""Authorization deny action""" -
-PSK 密码套件： -"RSA-PSK-AES256-GCM-SHA384,RSA-PSK-AES256-CBC-SHA384, -RSA-PSK-AES128-GCM-SHA256,RSA-PSK-AES128-CBC-SHA256, -RSA-PSK-AES256-CBC-SHA,RSA-PSK-AES128-CBC-SHA, -RSA-PSK-DES-CBC3-SHA,RSA-PSK-RC4-SHA"""" - } - label: { - en: "" - zh: "" - } -} +fields_deflate_opts_server_max_window_bits.desc: +"""Specifies the size of the compression context for the server.""" -ciphers_schema_quic { - desc { - en: """This config holds TLS cipher suite names separated by comma, +fields_deflate_opts_server_max_window_bits.label: +"""Server compression max window size""" + +client_ssl_opts_schema_server_name_indication.desc: +"""Specify the host name to be used in TLS Server Name Indication extension.
+For instance, when connecting to "server.example.net", the genuine server +which accepts the connection and performs TLS handshake may differ from the +host the TLS client initially connects to, e.g. when connecting to an IP address +or when the host has multiple resolvable DNS records
+If not specified, it will default to the host name string which is used +to establish the connection, unless it is IP addressed used.
+The host name is then also used in the host name verification of the peer +certificate.
The special value 'disable' prevents the Server Name +Indication extension from being sent and disables the hostname +verification check.""" + +client_ssl_opts_schema_server_name_indication.label: +"""Server Name Indication""" + +fields_mqtt_quic_listener_retry_memory_limit.desc: +"""The percentage of available memory usable for handshake connections before stateless retry is used. Calculated as `N/65535`. Default: 65""" + +fields_mqtt_quic_listener_retry_memory_limit.label: +"""Retry memory limit""" + +force_shutdown_max_message_queue_len.desc: +"""Maximum message queue length.""" + +force_shutdown_max_message_queue_len.label: +"""Maximum mailbox queue length of process.""" + +sys_heartbeat_interval.desc: +"""Time interval for publishing following heartbeat messages: + - `$SYS/brokers//uptime` + - `$SYS/brokers//datetime`""" + +flapping_detect_ban_time.desc: +"""How long the flapping clientid will be banned.""" + +flapping_detect_ban_time.label: +"""Ban time""" + +sysmon_top_num_items.desc: +"""The number of top processes per monitoring group""" + +sysmon_top_num_items.label: +"""Top num items""" + +persistent_session_builtin_session_table.desc: +"""Performance tuning options for built-in session table.""" + +persistent_session_builtin_session_table.label: +"""Persistent session""" + +mqtt_upgrade_qos.desc: +"""Force upgrade of QoS level according to subscription.""" + +mqtt_upgrade_qos.label: +"""Upgrade QoS""" + +mqtt_shared_subscription.desc: +"""Whether to enable support for MQTT shared subscription.""" + +mqtt_shared_subscription.label: +"""Shared Subscription Available""" + +fields_tcp_opts_sndbuf.desc: +"""The TCP send buffer (OS kernel) for the connections.""" + +fields_tcp_opts_sndbuf.label: +"""TCP send buffer""" + +sysmon_os_mem_check_interval.desc: +"""The time interval for the periodic memory check.""" + +sysmon_os_mem_check_interval.label: +"""Mem check interval""" + +server_ssl_opts_schema_gc_after_handshake.desc: +"""Memory usage tuning. If enabled, will immediately perform a garbage collection after the TLS/SSL handshake.""" + +server_ssl_opts_schema_gc_after_handshake.label: +"""Perform GC after handshake""" + +fields_mqtt_quic_listener_ssl_options.desc: +"""TLS options for QUIC transport""" + +fields_mqtt_quic_listener_ssl_options.label: +"""TLS Options""" + +fields_ws_opts_mqtt_piggyback.desc: +"""Whether a WebSocket message is allowed to contain multiple MQTT packets.""" + +fields_ws_opts_mqtt_piggyback.label: +"""MQTT Piggyback""" + +base_listener_mountpoint.desc: +"""When publishing or subscribing, prefix all topics with a mountpoint string. +The prefixed string will be removed from the topic name when the message +is delivered to the subscriber. The mountpoint is a way that users can use +to implement isolation of message routing between different listeners. +For example if a client A subscribes to `t` with `listeners.tcp.\.mountpoint` +set to `some_tenant`, then the client actually subscribes to the topic +`some_tenant/t`. Similarly, if another client B (connected to the same listener +as the client A) sends a message to topic `t`, the message is routed +to all the clients subscribed `some_tenant/t`, so client A will receive the +message, with topic name `t`.
+Set to `""` to disable the feature.
+ +Variables in mountpoint string: + - ${clientid}: clientid + - ${username}: username""" + +base_listener_mountpoint.label: +"""mountpoint""" + +mqtt_max_awaiting_rel.desc: +"""For each publisher session, the maximum number of outstanding QoS 2 messages pending on the client to send PUBREL. After reaching this limit, new QoS 2 PUBLISH requests will be rejected with `147(0x93)` until either PUBREL is received or timed out.""" + +mqtt_max_awaiting_rel.label: +"""Max Awaiting PUBREL""" + +ciphers_schema_quic.desc: +"""This config holds TLS cipher suite names separated by comma, or as an array of strings. e.g. "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256" or ["TLS_AES_256_GCM_SHA384","TLS_AES_128_GCM_SHA256"]. @@ -1606,115 +465,744 @@ RSA-PSK-DES-CBC3-SHA,RSA-PSK-RC4-SHA"
NOTE: QUIC listener supports only 'tlsv1.3' ciphers""" - zh: """此配置保存由逗号分隔的 TLS 密码套件名称，或作为字符串数组。例如 -"TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256"或 -["TLS_AES_256_GCM_SHA384","TLS_AES_128_GCM_SHA256"]。 -
-密码（及其顺序）定义了客户端和服务器通过网络连接加密信息的方式。 -选择一个好的密码套件对于应用程序的数据安全性、机密性和性能至关重要。 +ciphers_schema_quic.label: +"""""" -名称应为 OpenSSL 字符串格式（而不是 RFC 格式）。 -EMQX 配置文档提供的所有默认值和示例都是 OpenSSL 格式
-注意：某些密码套件仅与特定的 TLS 版本兼容（'tlsv1.1'、'tlsv1.2'或'tlsv1.3'）。 -不兼容的密码套件将被自动删除。 +fields_mqtt_quic_listener_max_bytes_per_key.desc: +"""Maximum number of bytes to encrypt with a single 1-RTT encryption key before initiating key update. Default: 274877906944""" -例如，如果只有 versions 仅配置为 tlsv1.3。为其他版本配置密码套件将无效。 +fields_mqtt_quic_listener_max_bytes_per_key.label: +"""Max bytes per key""" -
-注：PSK 的 Ciphers 不支持 tlsv1.3
-如果打算使用PSK密码套件，tlsv1.3。应在ssl.versions中禁用。 +fields_mqtt_quic_listener_mtu_discovery_search_complete_timeout_us.desc: +"""The time in microseconds to wait before reattempting MTU probing if max was not reached. Default: 600000000""" -
-PSK 密码套件： -"RSA-PSK-AES256-GCM-SHA384,RSA-PSK-AES256-CBC-SHA384, -RSA-PSK-AES128-GCM-SHA256,RSA-PSK-AES128-CBC-SHA256, -RSA-PSK-AES256-CBC-SHA,RSA-PSK-AES128-CBC-SHA, -RSA-PSK-DES-CBC3-SHA,RSA-PSK-RC4-SHA"
+fields_mqtt_quic_listener_mtu_discovery_search_complete_timeout_us.label: +"""MTU discovery search complete timeout us""" -注：QUIC 监听器不支持 tlsv1.3 的 ciphers""" - } - label: { - en: "" - zh: "" - } -} +fields_ws_opts_check_origin_enable.desc: +"""If true, origin HTTP header will be + validated against the list of allowed origins configured in check_origins + parameter.""" -common_ssl_opts_schema_user_lookup_fun { - desc { - en: """EMQX-internal callback that is used to lookup pre-shared key (PSK) identity.""" - zh: """用于查找预共享密钥（PSK）标识的 EMQX 内部回调。""" - } - label: { - en: "SSL PSK user lookup fun" - zh: "SSL PSK 用户回调" - } -} +fields_ws_opts_check_origin_enable.label: +"""Check origin""" -common_ssl_opts_schema_secure_renegotiate { - desc { - en: """SSL parameter renegotiation is a feature that allows a client and a server -to renegotiate the parameters of the SSL connection on the fly. -RFC 5746 defines a more secure way of doing this. By enabling secure renegotiation, -you drop support for the insecure renegotiation, prone to MitM attacks.""" - zh: """SSL 参数重新协商是一种允许客户端和服务器动态重新协商 SSL 连接参数的功能。 -RFC 5746 定义了一种更安全的方法。通过启用安全的重新协商，您就失去了对不安全的重新协商的支持，从而容易受到 MitM 攻击。""" - } - label: { - en: "SSL renegotiate" - zh: "SSL 重新协商" - } -} +sysmon_vm_busy_dist_port.desc: +"""When the RPC connection used to communicate with other nodes in the cluster is overloaded, +there will be a busy_dist_port warning log, +and an MQTT message is published to system topic $SYS/sysmon/busy_dist_port.""" -server_ssl_opts_schema_dhfile { - desc { - en: """Path to a file containing PEM-encoded Diffie-Hellman parameters +sysmon_vm_busy_dist_port.label: +"""Enable Busy Distribution Port monitoring.""" + +mqtt_max_mqueue_len.desc: +"""Maximum queue length. Enqueued messages when persistent client disconnected, or inflight window is full.""" + +mqtt_max_mqueue_len.label: +"""Max Message Queue Length""" + +mqtt_max_inflight.desc: +"""Maximum number of QoS 1 and QoS 2 messages that are allowed to be delivered simultaneously before completing the acknowledgment.""" + +mqtt_max_inflight.label: +"""Max Inflight""" + +persistent_session_store_enabled.desc: +"""Use the database to store information about persistent sessions. +This makes it possible to migrate a client connection to another +cluster node if a node is stopped.""" + +persistent_session_store_enabled.label: +"""Enable persistent session store""" + +fields_deflate_opts_level.desc: +"""Compression level.""" + +fields_deflate_opts_level.label: +"""Compression level""" + +mqtt_server_keepalive.desc: +"""The keep alive duration required by EMQX. To use the setting from the client side, choose disabled from the drop-down list. Only applicable to MQTT 5.0 clients.""" + +mqtt_server_keepalive.label: +"""Server Keep Alive""" + +global_authentication.desc: +"""Default authentication configs for all MQTT listeners. + +For per-listener overrides see authentication in listener configs + +This option can be configured with: +
    +
  • []: The default value, it allows *ALL* logins
    • +
  • one: For example {enable:true,backend:"built_in_database",mechanism="password_based"}
    • +
  • chain: An array of structs.
    • +
+ +When a chain is configured, the login credentials are checked against the backends per the configured order, until an 'allow' or 'deny' decision can be made. + +If there is no decision after a full chain exhaustion, the login is rejected.""" + +fields_mqtt_quic_listener_load_balancing_mode.desc: +"""0: Disabled, 1: SERVER_ID_IP, 2: SERVER_ID_FIXED. default: 0""" + +fields_mqtt_quic_listener_load_balancing_mode.label: +"""Load balancing mode""" + +persistent_session_store_session_message_gc_interval.desc: +"""The starting interval for garbage collection of transient data for +persistent session messages. This does not affect the lifetime length +of persistent session messages.""" + +persistent_session_store_session_message_gc_interval.label: +"""Session message GC interval""" + +server_ssl_opts_schema_ocsp_refresh_http_timeout.desc: +"""The timeout for the HTTP request when checking OCSP responses.""" + +server_ssl_opts_schema_ocsp_refresh_http_timeout.label: +"""OCSP Refresh HTTP Timeout""" + +fields_tcp_opts_send_timeout.desc: +"""The TCP send timeout for the connections.""" + +fields_tcp_opts_send_timeout.label: +"""TCP send timeout""" + +sysmon_vm_process_high_watermark.desc: +"""The threshold, as percentage of processes, for how many + processes can simultaneously exist at the local node before the corresponding + alarm is raised.""" + +sysmon_vm_process_high_watermark.label: +"""Process high watermark""" + +fields_tcp_opts_buffer.desc: +"""The size of the user-space buffer used by the driver.""" + +fields_tcp_opts_buffer.label: +"""TCP user-space buffer""" + +server_ssl_opts_schema_honor_cipher_order.desc: +"""An important security setting, it forces the cipher to be set based + on the server-specified order instead of the client-specified order, + hence enforcing the (usually more properly configured) security + ordering of the server administrator.""" + +server_ssl_opts_schema_honor_cipher_order.label: +"""SSL honor cipher order""" + +conn_congestion_min_alarm_sustain_duration.desc: +"""Minimal time before clearing the alarm.
The alarm is cleared only when there's no pending data in
the queue, and at least min_alarm_sustain_durationmilliseconds passed since the last time we considered the connection 'congested'.
This is to avoid clearing and raising the alarm again too often.""" + +conn_congestion_min_alarm_sustain_duration.label: +"""Sustain duration""" + +fields_mqtt_quic_listener_keep_alive_interval_ms.desc: +"""How often to send PING frames to keep a connection alive.""" + +fields_mqtt_quic_listener_keep_alive_interval_ms.label: +"""Keep alive interval ms""" + +fields_mqtt_quic_listener_handshake_idle_timeout_ms.desc: +"""How long a handshake can idle before it is discarded""" + +fields_mqtt_quic_listener_handshake_idle_timeout_ms.label: +"""Handshake idle timeout ms""" + +broker_session_locking_strategy.desc: +"""Session locking strategy in a cluster. + - `local`: only lock the session on the current node + - `one`: select only one remote node to lock the session + - `quorum`: select some nodes to lock the session + - `all`: lock the session on all the nodes in the cluster""" + +persistent_store_ram_cache.desc: +"""Maintain a copy of the data in RAM for faster access.""" + +persistent_store_ram_cache.label: +"""RAM cache""" + +fields_mqtt_quic_listener_stream_recv_window_default.desc: +"""Initial stream receive window size. Default: 32678""" + +fields_mqtt_quic_listener_stream_recv_window_default.label: +"""Stream recv window default""" + +mqtt_mqueue_priorities.desc: +"""Topic priorities. Priority number [1-255] +There's no priority table by default, hence all messages are treated equal. + +**NOTE**: Comma and equal signs are not allowed for priority topic names. +**NOTE**: Messages for topics not in the priority table are treated as either highest or lowest priority depending on the configured value for mqtt.mqueue_default_priority. + +**Examples**: +To configure "topic/1" > "topic/2": +mqueue_priorities: {"topic/1": 10, "topic/2": 8}""" + +mqtt_mqueue_priorities.label: +"""Topic Priorities""" + +fields_rate_limit_conn_messages_in.desc: +"""Message limit for the external MQTT connections.""" + +fields_rate_limit_conn_messages_in.label: +"""connecting messages in""" + +fields_rate_limit_max_conn_rate.desc: +"""Maximum connections per second.""" + +fields_rate_limit_max_conn_rate.label: +"""Max connection rate""" + +alarm_size_limit.desc: +"""The maximum total number of deactivated alarms to keep as history.
When this limit is exceeded, the oldest deactivated alarms are deleted to cap the total number.""" + +alarm_size_limit.label: +"""Alarm size limit""" + +fields_cache_max_size.desc: +"""Maximum number of cached items.""" + +fields_cache_max_size.label: +"""Maximum number of cached items.""" + +fields_listeners_tcp.desc: +"""TCP listeners.""" + +fields_listeners_tcp.label: +"""TCP listeners""" + +conn_congestion_enable_alarm.desc: +"""Enable or disable connection congestion alarm.""" + +conn_congestion_enable_alarm.label: +"""Enable/disable congestion alarm""" + +fields_ws_opts_proxy_port_header.desc: +"""HTTP header used to pass information about the client port. Relevant when the EMQX cluster is deployed behind a load-balancer.""" + +fields_ws_opts_proxy_port_header.label: +"""Proxy port header""" + +overload_protection_enable.desc: +"""React on system overload or not.""" + +overload_protection_enable.label: +"""React on system overload or not""" + +fields_mqtt_quic_listener_minimum_mtu.desc: +"""The minimum MTU supported by a connection. This will be used as the starting MTU. Default: 1248""" + +fields_mqtt_quic_listener_minimum_mtu.label: +"""Minimum MTU""" + +sys_msg_interval.desc: +"""Time interval of publishing `$SYS` messages.""" + +mqtt_await_rel_timeout.desc: +"""For client to broker QoS 2 message, the time limit for the broker to wait before the `PUBREL` message is received. The wait is aborted after timed out, meaning the packet ID is freed for new `PUBLISH` requests. Receiving a stale `PUBREL` causes a warning level log. Note, the message is delivered to subscribers before entering the wait for PUBREL.""" + +mqtt_await_rel_timeout.label: +"""Max Awaiting PUBREL TIMEOUT""" + +common_ssl_opts_schema_verify.desc: +"""Enable or disable peer verification.""" + +common_ssl_opts_schema_verify.label: +"""Verify peer""" + +fields_listeners_ssl.desc: +"""SSL listeners.""" + +fields_listeners_ssl.label: +"""SSL listeners""" + +fields_deflate_opts_client_max_window_bits.desc: +"""Specifies the size of the compression context for the client.""" + +fields_deflate_opts_client_max_window_bits.label: +"""Client compression max window size""" + +common_ssl_opts_schema_keyfile.desc: +"""PEM format private key file.""" + +common_ssl_opts_schema_keyfile.label: +"""Keyfile""" + +sysmon_os_cpu_high_watermark.desc: +"""The threshold, as percentage of system CPU load, + for how much system cpu can be used before the corresponding alarm is raised.""" + +sysmon_os_cpu_high_watermark.label: +"""CPU high watermark""" + +flapping_detect_window_time.desc: +"""The time window for flapping detection.""" + +flapping_detect_window_time.label: +"""Window time""" + +mqtt_mqueue_default_priority.desc: +"""Default topic priority, which will be used by topics not in Topic Priorities (mqueue_priorities).""" + +mqtt_mqueue_default_priority.label: +"""Default Topic Priorities""" + +client_ssl_opts_schema_enable.desc: +"""Enable TLS.""" + +client_ssl_opts_schema_enable.label: +"""Enable TLS.""" + +fields_mqtt_quic_listener_mtu_discovery_missing_probe_count.desc: +"""The maximum number of stateless operations that may be queued on a binding at any one time. Default: 3""" + +fields_mqtt_quic_listener_mtu_discovery_missing_probe_count.label: +"""MTU discovery missing probe count""" + +fields_tcp_opts_recbuf.desc: +"""The TCP receive buffer (OS kernel) for the connections.""" + +fields_tcp_opts_recbuf.label: +"""TCP receive buffer""" + +sysmon_vm_process_check_interval.desc: +"""The time interval for the periodic process limit check.""" + +sysmon_vm_process_check_interval.label: +"""Process limit check interval""" + +fields_mqtt_quic_listener_server_resumption_level.desc: +"""Controls resumption tickets and/or 0-RTT server support. Default: 0 (No resumption)""" + +fields_mqtt_quic_listener_server_resumption_level.label: +"""Server resumption level""" + +fields_ws_opts_proxy_address_header.desc: +"""HTTP header used to pass information about the client IP address. +Relevant when the EMQX cluster is deployed behind a load-balancer.""" + +fields_ws_opts_proxy_address_header.label: +"""Proxy address header""" + +sysmon_os_sysmem_high_watermark.desc: +"""The threshold, as percentage of system memory, + for how much system memory can be allocated before the corresponding alarm is raised.""" + +sysmon_os_sysmem_high_watermark.label: +"""SysMem high wartermark""" + +fields_tcp_opts_high_watermark.desc: +"""The socket is set to a busy state when the amount of data queued internally +by the VM socket implementation reaches this limit.""" + +fields_tcp_opts_high_watermark.label: +"""TCP 高水位线""" + +fields_mqtt_quic_listener_stateless_operation_expiration_ms.desc: +"""The time limit between operations for the same endpoint, in milliseconds. Default: 100""" + +fields_mqtt_quic_listener_stateless_operation_expiration_ms.label: +"""Stateless operation expiration ms""" + +server_ssl_opts_schema_dhfile.desc: +"""Path to a file containing PEM-encoded Diffie-Hellman parameters to be used by the server if a cipher suite using Diffie-Hellman key exchange is negotiated. If not specified, default parameters are used.
NOTE: The dhfile option is not supported by TLS 1.3.""" - zh: """如果协商使用Diffie-Hellman密钥交换的密码套件，则服务器将使用包含PEM编码的Diffie-Hellman参数的文件的路径。如果未指定，则使用默认参数。
-注意：TLS 1.3不支持dhfile选项。""" - } - label: { - en: "SSL dhfile" - zh: "SSL dhfile" - } -} -server_ssl_opts_schema_fail_if_no_peer_cert { - desc { - en: """Used together with {verify, verify_peer} by an TLS/DTLS server. +server_ssl_opts_schema_dhfile.label: +"""SSL dhfile""" + +flapping_detect_max_count.desc: +"""The maximum number of disconnects allowed for a MQTT Client in `window_time`""" + +flapping_detect_max_count.label: +"""Max count""" + +mqtt_max_topic_levels.desc: +"""Maximum topic levels allowed.""" + +mqtt_max_topic_levels.label: +"""Max Topic Levels""" + +force_shutdown_max_heap_size.desc: +"""Total heap size""" + +force_shutdown_max_heap_size.label: +"""Total heap size""" + +persistent_store_on_disc.desc: +"""Save information about the persistent sessions on disc. +If this option is enabled, persistent sessions will survive full restart of the cluster. +Otherwise, all the data will be stored in RAM, and it will be lost when all the nodes in the cluster are stopped.""" + +persistent_store_on_disc.label: +"""Persist on disc""" + +mqtt_ignore_loop_deliver.desc: +"""Whether the messages sent by the MQTT v3.1.1/v3.1.0 client will be looped back to the publisher itself, similar to No Local in MQTT 5.0.""" + +mqtt_ignore_loop_deliver.label: +"""Ignore Loop Deliver""" + +common_ssl_opts_schema_certfile.desc: +"""PEM format certificates chain file.
+The certificates in this file should be in reversed order of the certificate +issue chain. That is, the host's certificate should be placed in the beginning +of the file, followed by the immediate issuer certificate and so on. +Although the root CA certificate is optional, it should be placed at the end of +the file if it is to be added.""" + +common_ssl_opts_schema_certfile.label: +"""Certfile""" + +mqtt_exclusive_subscription.desc: +"""Whether to enable support for MQTT exclusive subscription.""" + +mqtt_exclusive_subscription.label: +"""Exclusive Subscription""" + +mqtt_retain_available.desc: +"""Whether to enable support for MQTT retained message.""" + +mqtt_retain_available.label: +"""Retain Available""" + +fields_tcp_opts_reuseaddr.desc: +"""The SO_REUSEADDR flag for the connections.""" + +fields_tcp_opts_reuseaddr.label: +"""SO_REUSEADDR""" + +sysmon_vm_long_schedule.desc: +"""When the Erlang VM detect a task scheduled for too long, a warning level 'long_schedule' log is emitted, +and an MQTT message is published to the system topic $SYS/sysmon/long_schedule.""" + +sysmon_vm_long_schedule.label: +"""Enable Long Schedule monitoring.""" + +mqtt_keepalive_backoff.desc: +"""The coefficient EMQX uses to confirm whether the keep alive duration of the client expires. Formula: Keep Alive * Backoff * 2""" + +mqtt_keepalive_backoff.label: +"""Keep Alive Backoff""" + +force_gc_bytes.desc: +"""GC the process after specified number of bytes have passed through.""" + +force_gc_bytes.label: +"""Process GC bytes""" + +server_ssl_opts_schema_fail_if_no_peer_cert.desc: +"""Used together with {verify, verify_peer} by an TLS/DTLS server. If set to true, the server fails if the client does not have a certificate to send, that is, sends an empty certificate. If set to false, it fails only if the client sends an invalid certificate (an empty certificate is considered valid).""" - zh: """TLS/DTLS 服务器与 {verify，verify_peer} 一起使用。 -如果设置为true，则如果客户端没有要发送的证书，即发送空证书，服务器将失败。 -如果设置为false，则仅当客户端发送无效证书（空证书被视为有效证书）时才会失败。""" - } - label: { - en: "SSL fail if no peer cert" - zh: "没有证书则 SSL 失败" - } -} -server_ssl_opts_schema_honor_cipher_order { - desc { - en: """An important security setting, it forces the cipher to be set based - on the server-specified order instead of the client-specified order, - hence enforcing the (usually more properly configured) security - ordering of the server administrator.""" - zh: """一个重要的安全设置，它强制根据服务器指定的顺序而不是客户机指定的顺序设置密码，从而强制服务器管理员执行（通常配置得更正确）安全顺序。""" - } - label: { - en: "SSL honor cipher order" - zh: "SSL honor cipher order" - } -} +server_ssl_opts_schema_fail_if_no_peer_cert.label: +"""SSL fail if no peer cert""" -server_ssl_opts_schema_client_renegotiation { - desc { - en: """In protocols that support client-initiated renegotiation, +fields_ws_opts_compress.desc: +"""If true, compress WebSocket messages using zlib.
+The configuration items under deflate_opts belong to the compression-related parameter configuration.""" + +fields_ws_opts_compress.label: +"""Ws compress""" + +fields_mqtt_quic_listener_keep_alive_interval.desc: +"""How often to send PING frames to keep a connection alive. 0 means disabled.""" + +fields_mqtt_quic_listener_keep_alive_interval.label: +"""Keep Alive Interval""" + +fields_cache_ttl.desc: +"""Time to live for the cached data.""" + +fields_cache_ttl.label: +"""Time to live for the cached data.""" + +sys_topics.desc: +"""System topics configuration.""" + +sys_event_client_subscribed.desc: +"""Enable to publish event message that client subscribed a topic successfully.""" + +sysmon_top_db_port.desc: +"""Port of the PostgreSQL database that collects the data points.""" + +sysmon_top_db_port.label: +"""DB Port""" + +fields_mqtt_quic_listener_max_operations_per_drain.desc: +"""The maximum number of operations to drain per connection quantum. Default: 16""" + +fields_mqtt_quic_listener_max_operations_per_drain.label: +"""Max operations per drain""" + +fields_mqtt_quic_listener_datagram_receive_enabled.desc: +"""Advertise support for QUIC datagram extension. Reserve for the future. Default 0 (FALSE)""" + +fields_mqtt_quic_listener_datagram_receive_enabled.label: +"""Datagram receive enabled""" + +fields_mqtt_quic_listener_initial_rtt_ms.desc: +"""Initial RTT estimate.""" + +fields_mqtt_quic_listener_initial_rtt_ms.label: +"""Initial RTT ms""" + +overload_protection_backoff_gc.desc: +"""When at high load, skip forceful GC.""" + +overload_protection_backoff_gc.label: +"""Skip GC""" + +broker_perf_route_lock_type.desc: +"""Performance tuning for subscribing/unsubscribing a wildcard topic. +Change this parameter only when there are many wildcard topics. + +NOTE: when changing from/to `global` lock, it requires all nodes in the cluster to be stopped before the change. + - `key`: mnesia transactional updates with per-key locks. Recommended for a single-node setup. + - `tab`: mnesia transactional updates with table lock. Recommended for a cluster setup. + - `global`: updates are protected with a global lock. Recommended for large clusters.""" + +fields_tcp_opts_nodelay.desc: +"""The TCP_NODELAY flag for the connections.""" + +fields_tcp_opts_nodelay.label: +"""TCP_NODELAY""" + +sysmon_top_db_username.desc: +"""Username of the PostgreSQL database""" + +sysmon_top_db_username.label: +"""DB Username""" + +broker.desc: +"""Message broker options.""" + +force_gc_count.desc: +"""GC the process after this many received messages.""" + +force_gc_count.label: +"""Process GC messages num""" + +mqtt_max_clientid_len.desc: +"""Maximum allowed length of MQTT Client ID.""" + +mqtt_max_clientid_len.label: +"""Max Client ID Length""" + +fields_ws_opts_supported_subprotocols.desc: +"""Comma-separated list of supported subprotocols.""" + +fields_ws_opts_supported_subprotocols.label: +"""Supported subprotocols""" + +broker_shared_subscription_strategy.desc: +"""Dispatch strategy for shared subscription. + - `random`: dispatch the message to a random selected subscriber + - `round_robin`: select the subscribers in a round-robin manner + - `round_robin_per_group`: select the subscribers in round-robin fashion within each shared subscriber group + - `local`: select random local subscriber otherwise select random cluster-wide + - `sticky`: always use the last selected subscriber to dispatch, until the subscriber disconnects. + - `hash_clientid`: select the subscribers by hashing the `clientIds` + - `hash_topic`: select the subscribers by hashing the source topic""" + +fields_deflate_opts_mem_level.desc: +"""Specifies the size of the compression state.
+Lower values decrease memory usage per connection.""" + +fields_deflate_opts_mem_level.label: +"""Size of the compression state""" + +fields_mqtt_quic_listener_send_idle_timeout_ms.desc: +"""Reset congestion control after being idle for amount of time. Default: 1000""" + +fields_mqtt_quic_listener_send_idle_timeout_ms.label: +"""Send idle timeout ms""" + +base_listener_limiter.desc: +"""Type of the rate limit.""" + +base_listener_limiter.label: +"""Type of the rate limit.""" + +persistent_session_store_backend.desc: +"""Database management system used to store information about persistent sessions and messages. +- `builtin`: Use the embedded database (mria)""" + +persistent_session_store_backend.label: +"""Backend""" + +alarm_validity_period.desc: +"""Retention time of deactivated alarms. Alarms are not deleted immediately +when deactivated, but after the retention time.""" + +alarm_validity_period.label: +"""Alarm validity period""" + +server_ssl_opts_schema_ocsp_issuer_pem.desc: +"""PEM-encoded certificate of the OCSP issuer for the server certificate.""" + +server_ssl_opts_schema_ocsp_issuer_pem.label: +"""OCSP Issuer Certificate""" + +fields_tcp_opts_active_n.desc: +"""Specify the {active, N} option for this Socket.
+See: https://erlang.org/doc/man/inet.html#setopts-2""" + +fields_tcp_opts_active_n.label: +"""active_n""" + +listener_authentication.desc: +"""Per-listener authentication override. +Authentication can be one single authenticator instance or a chain of authenticators as an array. +When authenticating a login (username, client ID, etc.) the authenticators are checked in the configured order.""" + +listener_authentication.label: +"""Per-listener authentication override""" + +fields_trace_payload_encode.desc: +"""Determine the format of the payload format in the trace file.
+`text`: Text-based protocol or plain text protocol. + It is recommended when payload is JSON encoded.
+`hex`: Binary hexadecimal encode. It is recommended when payload is a custom binary protocol.
+`hidden`: payload is obfuscated as `******`""" + +fields_trace_payload_encode.label: +"""Payload encode""" + +mqtt_response_information.desc: +"""UTF-8 string, for creating the response topic, for example, if set to reqrsp/, the publisher/subscriber will communicate using the topic prefix reqrsp/. +To disable this feature, input "" in the text box below. Only applicable to MQTT 5.0 clients.""" + +mqtt_response_information.label: +"""Response Information""" + +persistent_session_store_max_retain_undelivered.desc: +"""The time messages that was not delivered to a persistent session +is stored before being garbage collected if the node the previous +session was handled on restarts of is stopped.""" + +persistent_session_store_max_retain_undelivered.label: +"""Max retain undelivered""" + +fields_mqtt_quic_listener_migration_enabled.desc: +"""Enable clients to migrate IP addresses and tuples. Requires a cooperative load-balancer, or no load-balancer. Default: 1 (Enabled)""" + +fields_mqtt_quic_listener_migration_enabled.label: +"""Migration enabled""" + +common_ssl_opts_schema_password.desc: +"""String containing the user's password. Only used if the private key file is password-protected.""" + +common_ssl_opts_schema_password.label: +"""Keyfile passphrase""" + +common_ssl_opts_schema_hibernate_after.desc: +"""Hibernate the SSL process after idling for amount of time reducing its memory footprint.""" + +common_ssl_opts_schema_hibernate_after.label: +"""hibernate after""" + +fields_mqtt_quic_listener_send_buffering_enabled.desc: +"""Buffer send data instead of holding application buffers until sent data is acknowledged. Default: 1 (Enabled)""" + +fields_mqtt_quic_listener_send_buffering_enabled.label: +"""Send buffering enabled""" + +sys_event_client_unsubscribed.desc: +"""Enable to publish event message that client unsubscribed a topic successfully.""" + +overload_protection_backoff_new_conn.desc: +"""When at high load, close new incoming connections.""" + +overload_protection_backoff_new_conn.label: +"""Close new connections""" + +server_ssl_opts_schema_ocsp_responder_url.desc: +"""URL for the OCSP responder to check the server certificate against.""" + +server_ssl_opts_schema_ocsp_responder_url.label: +"""OCSP Responder URL""" + +mqtt_idle_timeout.desc: +"""Configure the duration of time that a connection can remain idle (i.e., without any data transfer) before being: + - Automatically disconnected if no CONNECT package is received from the client yet. + - Put into hibernation mode to save resources if some CONNECT packages are already received. +Note: Please set the parameter with caution as long idle time will lead to resource waste.""" + +mqtt_idle_timeout.label: +"""Idle Timeout""" + +fields_mqtt_quic_listener_conn_flow_control_window.desc: +"""Connection-wide flow control window. Default: 16777216""" + +fields_mqtt_quic_listener_conn_flow_control_window.label: +"""Conn flow control window""" + +fields_mqtt_quic_listener_maximum_mtu.desc: +"""The maximum MTU supported by a connection. This will be the maximum probed value. Default: 1500""" + +fields_mqtt_quic_listener_maximum_mtu.label: +"""Maximum MTU""" + +sysmon_top_db_name.desc: +"""PostgreSQL database name""" + +sysmon_top_db_name.label: +"""DB Name""" + +mqtt_strict_mode.desc: +"""Whether to parse MQTT messages in strict mode. +In strict mode, invalid utf8 strings in for example client ID, topic name, etc. will cause the client to be disconnected.""" + +mqtt_strict_mode.label: +"""Strict Mode""" + +shared_subscription_group_strategy.desc: +"""Per group dispatch strategy for shared subscription. +This config is a map from shared subscription group name to the strategy +name. The group name should be of format `[A-Za-z0-9]`. i.e. no +special characters are allowed.""" + +fields_deflate_opts_strategy.desc: +"""Specifies the compression strategy.""" + +fields_deflate_opts_strategy.label: +"""compression strategy""" + +shared_subscription_strategy_enum.desc: +"""Dispatch strategy for shared subscription. +- `random`: dispatch the message to a random selected subscriber +- `round_robin`: select the subscribers in a round-robin manner +- `round_robin_per_group`: select the subscribers in round-robin fashion within each shared subscriber group +- `sticky`: always use the last selected subscriber to dispatch, +until the subscriber disconnects. +- `hash`: select the subscribers by the hash of `clientIds` +- `local`: send to a random local subscriber. If local +subscriber was not found, send to a random subscriber cluster-wide""" + +persistent_session_builtin_sess_msg_table.desc: +"""Performance tuning options for built-in session messages table.""" + +persistent_session_builtin_sess_msg_table.label: +"""Persistent session messages""" + +mqtt_mqueue_store_qos0.desc: +"""Specifies whether to store QoS 0 messages in the message queue while the connection is down but the session remains.""" + +mqtt_mqueue_store_qos0.label: +"""Store QoS 0 Message""" + +server_ssl_opts_schema_client_renegotiation.desc: +"""In protocols that support client-initiated renegotiation, the cost of resources of such an operation is higher for the server than the client. This can act as a vector for denial of service attacks. The SSL application already takes measures to counter-act such attempts, @@ -1722,1147 +1210,331 @@ but client-initiated renegotiation can be strictly disabled by setting this opti The default value is true. Note that disabling renegotiation can result in long-lived connections becoming unusable due to limits on the number of messages the underlying cipher suite can encipher.""" - zh: """在支持客户机发起的重新协商的协议中，这种操作的资源成本对于服务器来说高于客户机。 -这可能会成为拒绝服务攻击的载体。 -SSL 应用程序已经采取措施来反击此类尝试，但通过将此选项设置为 false，可以严格禁用客户端发起的重新协商。 -默认值为 true。请注意，由于基础密码套件可以加密的消息数量有限，禁用重新协商可能会导致长期连接变得不可用。""" - } - label: { - en: "SSL client renegotiation" - zh: "SSL 客户端冲协商" - } -} - -server_ssl_opts_schema_handshake_timeout { - desc { - en: """Maximum time duration allowed for the handshake to complete""" - zh: """握手完成所允许的最长时间""" - } - label: { - en: "Handshake timeout" - zh: "握手超时时间" - } -} - -server_ssl_opts_schema_gc_after_handshake { - desc { - en: """Memory usage tuning. If enabled, will immediately perform a garbage collection after the TLS/SSL handshake.""" - zh: """内存使用调优。如果启用，将在TLS/SSL握手完成后立即执行垃圾回收。TLS/SSL握手建立后立即进行GC。""" - } - label: { - en: "Perform GC after handshake" - zh: "握手后执行GC" - } -} - -server_ssl_opts_schema_enable_ocsp_stapling { - desc { - en: "Whether to enable Online Certificate Status Protocol (OCSP) stapling for the listener." - " If set to true, requires defining the OCSP responder URL and issuer PEM path." - zh: "是否为监听器启用 OCSP Stapling 功能。 如果设置为 true，" - "需要定义 OCSP Responder 的 URL 和证书签发者的 PEM 文件路径。" - } - label: { - en: "Enable OCSP Stapling" - zh: "启用 OCSP Stapling" - } -} - -server_ssl_opts_schema_ocsp_responder_url { - desc { - en: "URL for the OCSP responder to check the server certificate against." - zh: "用于检查服务器证书的 OCSP Responder 的 URL。" - } - label: { - en: "OCSP Responder URL" - zh: "OCSP Responder 的 URL" - } -} - -server_ssl_opts_schema_ocsp_issuer_pem { - desc { - en: "PEM-encoded certificate of the OCSP issuer for the server certificate." - zh: "服务器证书的 OCSP 签发者的 PEM 编码证书。" - } - label: { - en: "OCSP Issuer Certificate" - zh: "OCSP 签发者证书" - } -} - -server_ssl_opts_schema_ocsp_refresh_interval { - desc { - en: "The period to refresh the OCSP response for the server." - zh: "为服务器刷新OCSP响应的周期。" - } - label: { - en: "OCSP Refresh Interval" - zh: "OCSP 刷新间隔" - } -} - -server_ssl_opts_schema_ocsp_refresh_http_timeout { - desc { - en: "The timeout for the HTTP request when checking OCSP responses." - zh: "检查 OCSP 响应时，HTTP 请求的超时。" - } - label: { - en: "OCSP Refresh HTTP Timeout" - zh: "OCSP 刷新 HTTP 超时" - } -} - -server_ssl_opts_schema_enable_crl_check { - desc { - en: "Whether to enable CRL verification for this listener." - zh: "是否为该监听器启用 CRL 检查。" - } - label: { - en: "Enable CRL Check" - zh: "启用 CRL 检查" - } -} - -crl_cache_refresh_http_timeout { - desc { - en: "The timeout for the HTTP request when fetching CRLs. This is" - " a global setting for all listeners." - zh: "获取 CRLs 时 HTTP 请求的超时。 该配置对所有启用 CRL 检查的监听器监听器有效。" - } - label: { - en: "CRL Cache Refresh HTTP Timeout" - zh: "CRL 缓存刷新 HTTP 超时" - } -} - -crl_cache_refresh_interval { - desc { - en: "The period to refresh the CRLs from the servers. This is a global setting" - " for all URLs and listeners." - zh: "从服务器刷新CRL的周期。 该配置对所有 URL 和监听器有效。" - } - label: { - en: "CRL Cache Refresh Interval" - zh: "CRL 缓存刷新间隔" - } -} - -crl_cache_capacity { - desc { - en: "The maximum number of CRL URLs that can be held in cache. If the cache is at" - " full capacity and a new URL must be fetched, then it'll evict the oldest" - " inserted URL in the cache." - zh: "缓存中可容纳的 CRL URL 的最大数量。" - " 如果缓存的容量已满，并且必须获取一个新的 URL，" - "那么它将驱逐缓存中插入的最老的 URL。" - } - label: { - en: "CRL Cache Capacity" - zh: "CRL 缓存容量" - } -} - -fields_listeners_tcp { - desc { - en: """TCP listeners.""" - zh: """TCP 监听器。""" - } - label: { - en: "TCP listeners" - zh: "TCP 监听器" - } -} - -fields_listeners_ssl { - desc { - en: """SSL listeners.""" - zh: """SSL 监听器。""" - } - label: { - en: "SSL listeners" - zh: "SSL 监听器" - } -} - -fields_listeners_ws { - desc { - en: """HTTP websocket listeners.""" - zh: """HTTP websocket 监听器。""" - } - label: { - en: "HTTP websocket listeners" - zh: "HTTP websocket 监听器" - } -} - -fields_listeners_wss { - desc { - en: """HTTPS websocket listeners.""" - zh: """HTTPS websocket 监听器。""" - } - label: { - en: "HTTPS websocket listeners" - zh: "HTTPS websocket 监听器" - } -} - -fields_listeners_quic { - desc { - en: """QUIC listeners.""" - zh: """QUIC 监听器。""" - } - label: { - en: "QUIC listeners" - zh: "QUIC 监听器" - } -} - -fields_listener_enabled { - desc { - en: """Enable listener.""" - zh: """启停监听器。""" - } - label: { - en: "Enable listener" - zh: "启停监听器" - } -} - -fields_mqtt_quic_listener_certfile { - desc { - en: """Path to the certificate file. Will be deprecated in 5.1, use .ssl_options.certfile instead.""" - zh: """证书文件。在 5.1 中会被废弃，使用 .ssl_options.certfile 代替。""" - } - label: { - en: "Certificate file" - zh: "证书文件" - } -} - -fields_mqtt_quic_listener_keyfile { - desc { - en: """Path to the secret key file. Will be deprecated in 5.1, use .ssl_options.keyfile instead.""" - zh: """私钥文件。在 5.1 中会被废弃，使用 .ssl_options.keyfile 代替。""" - } - label: { - en: "Key file" - zh: "私钥文件" - } -} - -fields_mqtt_quic_listener_idle_timeout { - desc { - en: """How long a connection can go idle before it is gracefully shut down. 0 to disable""" - zh: """一个连接在被关闭之前可以空闲多长时间。0表示禁用。""" - } - label: { - en: "Idle Timeout" - zh: "空闲超时时间" - } -} - -fields_mqtt_quic_listener_handshake_idle_timeout { - desc { - en: """How long a handshake can idle before it is discarded.""" - zh: """一个握手在被丢弃之前可以空闲多长时间。""" - } - label: { - en: "Handshake Idle Timeout" - zh: "握手空闲超时时间" - } -} - -fields_mqtt_quic_listener_keep_alive_interval { - desc { - en: """How often to send PING frames to keep a connection alive. 0 means disabled.""" - zh: """发送 PING 帧的频率，以保活连接. 设为 0 表示禁用。""" - } - label: { - en: "Keep Alive Interval" - zh: "PING 保活频率" - } -} - -fields_mqtt_quic_listener_ssl_options { - desc { - en: """TLS options for QUIC transport""" - zh: """QUIC 传输层的 TLS 选项""" - } - label: { - en: "TLS Options" - zh: "TLS 选项" - } -} - -base_listener_bind { - desc { - en: """IP address and port for the listening socket.""" - zh: """监听套接字的 IP 地址和端口。""" - } - label: { - en: "IP address and port" - zh: "IP 地址和端口" - } -} - -base_listener_acceptors { - desc { - en: """The size of the listener's receiving pool.""" - zh: """监听器接收池的大小。""" - } - label: { - en: "Acceptors Num" - zh: "接收器数量" - } -} - -fields_mqtt_quic_listener_max_bytes_per_key { - desc { - en: "Maximum number of bytes to encrypt with a single 1-RTT encryption key before initiating key update. Default: 274877906944" - zh: "在启动密钥更新之前，用单个 1-RTT 加密密钥加密的最大字节数。默认值：274877906944" - } - label { - en: "Max bytes per key" - zh: "每个密钥的最大字节数" - } -} - -fields_mqtt_quic_listener_handshake_idle_timeout_ms { - desc { - en: "How long a handshake can idle before it is discarded. Default: 10 000" - zh: "一个握手在被丢弃之前可以空闲多长时间。 默认值：10 000" - } - label { - en: "Handshake idle timeout ms" - zh: "握手空闲超时毫秒" - } -} - -fields_mqtt_quic_listener_tls_server_max_send_buffer { - desc { - en: "How much Server TLS data to buffer. Default: 8192" - zh: "缓冲多少TLS数据。 默认值：8192" - } - label { - en: "TLS server max send buffer" - zh: "TLS 服务器最大发送缓冲区" - } -} - -fields_mqtt_quic_listener_stream_recv_window_default { - desc { - en: "Initial stream receive window size. Default: 32678" - zh: "初始流接收窗口大小。 默认值：32678" - } - label { - en: "Stream recv window default" - zh: "流接收窗口默认" - } -} - -fields_mqtt_quic_listener_stream_recv_buffer_default { - desc { - en: "Stream initial buffer size. Default: 4096" - zh: "流的初始缓冲区大小。默认：4096" - } - label { - en: "Stream recv buffer default" - zh: "流媒体接收缓冲区默认值" - } -} - -fields_mqtt_quic_listener_conn_flow_control_window { - desc { - en: "Connection-wide flow control window. Default: 16777216" - zh: "连接的流控窗口。默认：16777216" - } - label { - en: "Conn flow control window" - zh: "流控窗口" - } -} - -fields_mqtt_quic_listener_max_stateless_operations { - desc { - en: "The maximum number of stateless operations that may be queued on a worker at any one time. Default: 16" - zh: "无状态操作的最大数量，在任何时候都可以在一个工作者上排队。默认值：16" - } - label { - en: "Max stateless operations" - zh: "最大无状态操作数" - } -} - -fields_mqtt_quic_listener_initial_window_packets { - desc { - en: "The size (in packets) of the initial congestion window for a connection. Default: 10" - zh: "一个连接的初始拥堵窗口的大小（以包为单位）。默认值：10" - } - label { - en: "Initial window packets" - zh: "初始窗口数据包" - } -} - -fields_mqtt_quic_listener_send_idle_timeout_ms { - desc { - en: "Reset congestion control after being idle for amount of time. Default: 1000" - zh: "在闲置一定时间后重置拥堵控制。默认值：1000" - } - label { - en: "Send idle timeout ms" - zh: "发送空闲超时毫秒" - } -} - -fields_mqtt_quic_listener_initial_rtt_ms { - desc { - en: "Initial RTT estimate." - zh: "初始RTT估计" - } - label { - en: "Initial RTT ms" - zh: "Initial RTT 毫秒" - } -} - -fields_mqtt_quic_listener_max_ack_delay_ms { - desc { - en: "How long to wait after receiving data before sending an ACK. Default: 25" - zh: "在收到数据后要等待多长时间才能发送一个ACK。默认值：25" - } - label { - en: "Max ack delay ms" - zh: "最大应答延迟 毫秒" - } -} - -fields_mqtt_quic_listener_disconnect_timeout_ms { - desc { - en: "How long to wait for an ACK before declaring a path dead and disconnecting. Default: 16000" - zh: "在判定路径无效和断开连接之前，要等待多长时间的ACK。默认：16000" - } - label { - en: "Disconnect timeout ms" - zh: "断开连接超时 毫秒" - } -} - -fields_mqtt_quic_listener_idle_timeout_ms { - desc { - en: "How long a connection can go idle before it is gracefully shut down. 0 to disable timeout" - zh: "一个连接在被优雅地关闭之前可以空闲多长时间。0 表示禁用超时" - } - label { - en: "Idle timeout ms" - zh: "空闲超时 毫秒" - } -} - -fields_mqtt_quic_listener_handshake_idle_timeout_ms { - desc { - en: "How long a handshake can idle before it is discarded" - zh: "一个握手在被丢弃之前可以空闲多长时间" - } - label { - en: "Handshake idle timeout ms" - zh: "握手空闲超时 毫秒" - } -} - -fields_mqtt_quic_listener_keep_alive_interval_ms { - desc { - en: "How often to send PING frames to keep a connection alive." - zh: "多长时间发送一次PING帧以保活连接。" - } - label { - en: "Keep alive interval ms" - zh: "保持活着的时间间隔 毫秒" - } -} - -fields_mqtt_quic_listener_peer_bidi_stream_count { - desc { - en: "Number of bidirectional streams to allow the peer to open." - zh: "允许对端打开的双向流的数量" - } - label { - en: "Peer bidi stream count" - zh: "对端双向流的数量" - } -} - -fields_mqtt_quic_listener_peer_unidi_stream_count { - desc { - en: "Number of unidirectional streams to allow the peer to open." - zh: "允许对端打开的单向流的数量" - } - label { - en: "Peer unidi stream count" - zh: "对端单向流的数量" - } -} - -fields_mqtt_quic_listener_retry_memory_limit { - desc { - en: "The percentage of available memory usable for handshake connections before stateless retry is used. Calculated as `N/65535`. Default: 65" - zh: "在使用无状态重试之前，可用于握手连接的可用内存的百分比。计算为`N/65535`。默认值：65" - } - label { - en: "Retry memory limit" - zh: "重试内存限制" - } -} - -fields_mqtt_quic_listener_load_balancing_mode { - desc { - en: "0: Disabled, 1: SERVER_ID_IP, 2: SERVER_ID_FIXED. default: 0" - zh: "0: 禁用, 1: SERVER_ID_IP, 2: SERVER_ID_FIXED. 默认: 0" - } - label { - en: "Load balancing mode" - zh: "负载平衡模式" - } -} - -fields_mqtt_quic_listener_max_operations_per_drain { - desc { - en: "The maximum number of operations to drain per connection quantum. Default: 16" - zh: "每个连接操作的最大耗费操作数。默认：16" - } - label { - en: "Max operations per drain" - zh: "每次操作最大操作数" - } -} - -fields_mqtt_quic_listener_send_buffering_enabled { - desc { - en: "Buffer send data instead of holding application buffers until sent data is acknowledged. Default: 1 (Enabled)" - zh: "缓冲发送数据，而不是保留应用缓冲区，直到发送数据被确认。默认值：1（启用）" - } - label { - en: "Send buffering enabled" - zh: "启用发送缓冲功能" - } -} - -fields_mqtt_quic_listener_pacing_enabled { - desc { - en: "Pace sending to avoid overfilling buffers on the path. Default: 1 (Enabled)" - zh: "有节奏的发送，以避免路径上的缓冲区过度填充。默认值：1（已启用）" - } - label { - en: "Pacing enabled" - zh: "启用节奏发送" - } -} - -fields_mqtt_quic_listener_migration_enabled { - desc { - en: "Enable clients to migrate IP addresses and tuples. Requires a cooperative load-balancer, or no load-balancer. Default: 1 (Enabled)" - zh: "开启客户端地址迁移功能。需要一个支持的负载平衡器，或者没有负载平衡器。默认值：1（已启用）" - } - label { - en: "Migration enabled" - zh: "启用地址迁移" - } -} - -fields_mqtt_quic_listener_datagram_receive_enabled { - desc { - en: "Advertise support for QUIC datagram extension. Reserve for the future. Default 0 (FALSE)" - zh: "宣传对QUIC Datagram 扩展的支持。为将来保留。默认为0（FALSE）" - } - label { - en: "Datagram receive enabled" - zh: "启用 Datagram 接收" - } -} - -fields_mqtt_quic_listener_server_resumption_level { - desc { - en: "Controls resumption tickets and/or 0-RTT server support. Default: 0 (No resumption)" - zh: "连接恢复 和/或 0-RTT 服务器支持。默认值：0（无恢复功能）" - } - label { - en: "Server resumption level" - zh: "服务端连接恢复支持" - } -} - -fields_mqtt_quic_listener_minimum_mtu { - desc { - en: "The minimum MTU supported by a connection. This will be used as the starting MTU. Default: 1248" - zh: "一个连接所支持的最小MTU。这将被作为起始MTU使用。默认值：1248" - } - label { - en: "Minimum MTU" - zh: "最小 MTU" - } -} - -fields_mqtt_quic_listener_maximum_mtu { - desc { - en: "The maximum MTU supported by a connection. This will be the maximum probed value. Default: 1500" - zh: "一个连接所支持的最大MTU。这将是最大的探测值。默认值：1500" - } - label { - en: "Maximum MTU" - zh: "最大 MTU" - } -} - -fields_mqtt_quic_listener_mtu_discovery_search_complete_timeout_us { - desc { - en: "The time in microseconds to wait before reattempting MTU probing if max was not reached. Default: 600000000" - zh: "如果没有达到 max ，在重新尝试 MTU 探测之前要等待的时间，单位是微秒。默认值：600000000" - } - label { - en: "MTU discovery search complete timeout us" - zh: "" - } -} - -fields_mqtt_quic_listener_mtu_discovery_missing_probe_count { - desc { - en: "The maximum number of stateless operations that may be queued on a binding at any one time. Default: 3" - zh: "在任何时候都可以在一个绑定上排队的无状态操作的最大数量。默认值：3" - } - label { - en: "MTU discovery missing probe count" - zh: "MTU发现丢失的探针数量" - } -} - -fields_mqtt_quic_listener_max_binding_stateless_operations { - desc { - en: "The maximum number of stateless operations that may be queued on a binding at any one time. Default: 100" - zh: "在任何时候可以在一个绑定上排队的无状态操作的最大数量。默认值：100" - } - label { - en: "Max binding stateless operations" - zh: "最大绑定无状态操作" - } -} - -fields_mqtt_quic_listener_stateless_operation_expiration_ms { - desc { - en: "The time limit between operations for the same endpoint, in milliseconds. Default: 100" - zh: "同一个对端的操作之间的时间限制，单位是毫秒。 默认：100" - } - label { - en: "Stateless operation expiration ms" - zh: "无状态操作过期 毫秒" - } -} - -base_listener_max_connections { - desc { - en: """The maximum number of concurrent connections allowed by the listener.""" - zh: """监听器允许的最大并发连接数。""" - } - label: { - en: "Max connections" - zh: "最大并发连接数" - } -} - -base_listener_mountpoint { - desc { - en: """When publishing or subscribing, prefix all topics with a mountpoint string. -The prefixed string will be removed from the topic name when the message -is delivered to the subscriber. The mountpoint is a way that users can use -to implement isolation of message routing between different listeners. -For example if a client A subscribes to `t` with `listeners.tcp.\.mountpoint` -set to `some_tenant`, then the client actually subscribes to the topic -`some_tenant/t`. Similarly, if another client B (connected to the same listener -as the client A) sends a message to topic `t`, the message is routed -to all the clients subscribed `some_tenant/t`, so client A will receive the -message, with topic name `t`.
-Set to `""` to disable the feature.
- -Variables in mountpoint string: - - ${clientid}: clientid - - ${username}: username""" - zh: """发布或订阅时，请在所有主题前面加上 mountpoint 字符串。 - -将消息传递给订阅者时，将从主题名称中删除带前缀的字符串。挂载点是一种用户可以用来实现不同侦听器之间消息路由隔离的方法。 - -例如，如果客户机 A 使用 listeners.tcp.\.mountpoint 设置为'some_tenant'，那么客户端实际上订阅了主题'some_tenant/t'。
-类似地，如果另一个客户端B（与客户端A连接到同一个侦听器）向主题 't' 发送消息，该消息将路由到所有订阅了'some_租户/t'的客户端，因此客户端 A 将接收主题名为't'的消息
- -设置为"" 以禁用该功能
- -mountpoint 字符串中的变量： -- ${clientid}: clientid -- ${username}: username""" - } - label: { - en: "mountpoint" - zh: "mountpoint" - } -} - -base_listener_zone { - desc { - en: """The configuration zone to which the listener belongs.""" - zh: """监听器所属的配置组。""" - } - label: { - en: "Zone" - zh: "配置组" - } -} - -base_listener_limiter { - desc { - en: """Type of the rate limit.""" - zh: """速率限制类型""" - } - label: { - en: "Type of the rate limit." - zh: "速率限制类型" - } -} - -base_listener_enable_authn { - desc { - en: """Set true (default) to enable client authentication on this listener, the authentication + +server_ssl_opts_schema_client_renegotiation.label: +"""SSL client renegotiation""" + +server_ssl_opts_schema_enable_crl_check.desc: +"""Whether to enable CRL verification for this listener.""" + +server_ssl_opts_schema_enable_crl_check.label: +"""Enable CRL Check""" + +fields_mqtt_quic_listener_peer_bidi_stream_count.desc: +"""Number of bidirectional streams to allow the peer to open.""" + +fields_mqtt_quic_listener_peer_bidi_stream_count.label: +"""Peer bidi stream count""" + +fields_mqtt_quic_listener_max_stateless_operations.desc: +"""The maximum number of stateless operations that may be queued on a worker at any one time. Default: 16""" + +fields_mqtt_quic_listener_max_stateless_operations.label: +"""Max stateless operations""" + +fields_ws_opts_idle_timeout.desc: +"""Close transport-layer connections from the clients that have not sent MQTT CONNECT message within this interval.""" + +fields_ws_opts_idle_timeout.label: +"""WS idle timeout""" + +fields_mqtt_quic_listener_max_ack_delay_ms.desc: +"""How long to wait after receiving data before sending an ACK. Default: 25""" + +fields_mqtt_quic_listener_max_ack_delay_ms.label: +"""Max ack delay ms""" + +base_listener_zone.desc: +"""The configuration zone to which the listener belongs.""" + +base_listener_zone.label: +"""Zone""" + +fields_mqtt_quic_listener_handshake_idle_timeout.desc: +"""How long a handshake can idle before it is discarded.""" + +fields_mqtt_quic_listener_handshake_idle_timeout.label: +"""Handshake Idle Timeout""" + +force_gc_enable.desc: +"""Enable forced garbage collection.""" + +force_gc_enable.label: +"""Enable forced garbage collection""" + +fields_ws_opts_allow_origin_absence.desc: +"""If false and check_origin_enable is + true, the server will reject requests that don't have origin + HTTP header.""" + +fields_ws_opts_allow_origin_absence.label: +"""Allow origin absence""" + +common_ssl_opts_schema_versions.desc: +"""All TLS/DTLS versions to be supported.
+NOTE: PSK ciphers are suppressed by 'tlsv1.3' version config.
+In case PSK cipher suites are intended, make sure to configure +['tlsv1.2', 'tlsv1.1'] here.""" + +common_ssl_opts_schema_versions.label: +"""SSL versions""" + +mqtt_listener_proxy_protocol_timeout.desc: +"""Timeout for proxy protocol. EMQX will close the TCP connection if proxy protocol packet is not received within the timeout.""" + +mqtt_listener_proxy_protocol_timeout.label: +"""Proxy protocol timeout""" + +fields_mqtt_quic_listener_idle_timeout.desc: +"""How long a connection can go idle before it is gracefully shut down. 0 to disable""" + +fields_mqtt_quic_listener_idle_timeout.label: +"""Idle Timeout""" + +common_ssl_opts_schema_secure_renegotiate.desc: +"""SSL parameter renegotiation is a feature that allows a client and a server +to renegotiate the parameters of the SSL connection on the fly. +RFC 5746 defines a more secure way of doing this. By enabling secure renegotiation, +you drop support for the insecure renegotiation, prone to MitM attacks.""" + +common_ssl_opts_schema_secure_renegotiate.label: +"""SSL renegotiate""" + +sysmon_vm_busy_port.desc: +"""When a port (e.g. TCP socket) is overloaded, there will be a busy_port warning log, +and an MQTT message is published to the system topic $SYS/sysmon/busy_port.""" + +sysmon_vm_busy_port.label: +"""Enable Busy Port monitoring.""" + +sys_event_client_connected.desc: +"""Enable to publish client connected event messages""" + +sysmon_vm_process_low_watermark.desc: +"""The threshold, as percentage of processes, for how many + processes can simultaneously exist at the local node before the corresponding + alarm is cleared.""" + +sysmon_vm_process_low_watermark.label: +"""Process low watermark""" + +mqtt_max_packet_size.desc: +"""Maximum MQTT packet size allowed.""" + +mqtt_max_packet_size.label: +"""Max Packet Size""" + +common_ssl_opts_schema_reuse_sessions.desc: +"""Enable TLS session reuse.""" + +common_ssl_opts_schema_reuse_sessions.label: +"""TLS session reuse""" + +common_ssl_opts_schema_depth.desc: +"""Maximum number of non-self-issued intermediate certificates that can follow the peer certificate in a valid certification path. +So, if depth is 0 the PEER must be signed by the trusted ROOT-CA directly;
+if 1 the path can be PEER, Intermediate-CA, ROOT-CA;
+if 2 the path can be PEER, Intermediate-CA1, Intermediate-CA2, ROOT-CA.""" + +common_ssl_opts_schema_depth.label: +"""CACert Depth""" + +sysmon_vm_long_gc.desc: +"""When an Erlang process spends long time to perform garbage collection, a warning level long_gc log is emitted, +and an MQTT message is published to the system topic $SYS/sysmon/long_gc.""" + +sysmon_vm_long_gc.label: +"""Enable Long GC monitoring.""" + +fields_mqtt_quic_listener_keyfile.desc: +"""Path to the secret key file. Will be deprecated in 5.1, use .ssl_options.keyfile instead.""" + +fields_mqtt_quic_listener_keyfile.label: +"""Key file""" + +mqtt_peer_cert_as_clientid.desc: +"""Use the CN, DN field in the peer certificate or the entire certificate content as Client ID. Only works for the TLS connection. +Supported configurations are the following: +- cn: CN field of the certificate +- dn: DN field of the certificate +- crt: DER or PEM certificate +- pem: Convert DER certificate content to PEM format and use as Client ID +- md5: MD5 value of the DER or PEM certificate""" + +mqtt_peer_cert_as_clientid.label: +"""Use Peer Certificate as Client ID""" + +persistent_session_store_message_gc_interval.desc: +"""The starting interval for garbage collection of undelivered messages to +a persistent session. This affects how often the "max_retain_undelivered" +is checked for removal.""" + +persistent_session_store_message_gc_interval.label: +"""Message GC interval""" + +broker_shared_dispatch_ack_enabled.desc: +"""Deprecated, will be removed in 5.1. +Enable/disable shared dispatch acknowledgement for QoS 1 and QoS 2 messages. +This should allow messages to be dispatched to a different subscriber in the group in case the picked (based on `shared_subscription_strategy`) subscriber is offline.""" + +base_listener_enable_authn.desc: +"""Set true (default) to enable client authentication on this listener, the authentication process goes through the configured authentication chain. When set to false to allow any clients with or without authentication information such as username or password to log in. When set to quick_deny_anonymous, it behaves like when set to true, but clients will be denied immediately without going through any authenticators if username is not provided. This is useful to fence off anonymous clients early.""" - zh: """配置 true （默认值）启用客户端进行身份认证，通过检查认配置的认认证器链来决定是否允许接入。 -配置 false 时，将不对客户端做任何认证，任何客户端，不论是不是携带用户名等认证信息，都可以接入。 -配置 quick_deny_anonymous 时，行为跟 true 类似，但是会对匿名 -客户直接拒绝，不做使用任何认证器对客户端进行身份检查。""" - } - label: { - en: "Enable authentication" - zh: "启用身份认证" - } -} -mqtt_listener_access_rules { - desc { - en: """The access control rules for this listener.
See: https://github.com/emqtt/esockd#allowdeny""" - zh: """此监听器的访问控制规则。""" - } - label: { - en: "Access rules" - zh: "访问控制规则" - } -} +base_listener_enable_authn.label: +"""Enable authentication""" -mqtt_listener_proxy_protocol { - desc { - en: """Enable the Proxy Protocol V1/2 if the EMQX cluster is deployed behind HAProxy or Nginx.
+force_shutdown_enable.desc: +"""Enable `force_shutdown` feature.""" + +force_shutdown_enable.label: +"""Enable `force_shutdown` feature""" + +broker_enable_session_registry.desc: +"""Enable session registry""" + +overload_protection_backoff_delay.desc: +"""The maximum duration of delay for background task execution during high load conditions.""" + +overload_protection_backoff_delay.label: +"""Delay Time""" + +ciphers_schema_common.desc: +"""This config holds TLS cipher suite names separated by comma, +or as an array of strings. e.g. +"TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256" or +["TLS_AES_256_GCM_SHA384","TLS_AES_128_GCM_SHA256"]. +
+Ciphers (and their ordering) define the way in which the +client and server encrypts information over the network connection. +Selecting a good cipher suite is critical for the +application's data security, confidentiality and performance. + +The names should be in OpenSSL string format (not RFC format). +All default values and examples provided by EMQX config +documentation are all in OpenSSL format.
+ +NOTE: Certain cipher suites are only compatible with +specific TLS versions ('tlsv1.1', 'tlsv1.2' or 'tlsv1.3') +incompatible cipher suites will be silently dropped. +For instance, if only 'tlsv1.3' is given in the versions, +configuring cipher suites for other versions will have no effect. +
+ +NOTE: PSK ciphers are suppressed by 'tlsv1.3' version config
+If PSK cipher suites are intended, 'tlsv1.3' should be disabled from versions.
+PSK cipher suites: "RSA-PSK-AES256-GCM-SHA384,RSA-PSK-AES256-CBC-SHA384, +RSA-PSK-AES128-GCM-SHA256,RSA-PSK-AES128-CBC-SHA256, +RSA-PSK-AES256-CBC-SHA,RSA-PSK-AES128-CBC-SHA, +RSA-PSK-DES-CBC3-SHA,RSA-PSK-RC4-SHA"""" + +ciphers_schema_common.label: +"""""" + +sys_event_client_disconnected.desc: +"""Enable to publish client disconnected event messages.""" + +crl_cache_refresh_interval.desc: +"""The period to refresh the CRLs from the servers. This is a global setting for all URLs and listeners.""" + +crl_cache_refresh_interval.label: +"""CRL Cache Refresh Interval""" + +mqtt_listener_proxy_protocol.desc: +"""Enable the Proxy Protocol V1/2 if the EMQX cluster is deployed behind HAProxy or Nginx.
See: https://www.haproxy.com/blog/haproxy/proxy-protocol/""" - zh: """如果EMQX集群部署在 HAProxy 或 Nginx 之后，请启用代理协议 V1/2
-详情见: https://www.haproxy.com/blog/haproxy/proxy-protocol/""" - } - label: { - en: "Proxy protocol" - zh: "Proxy protocol" - } -} -mqtt_listener_proxy_protocol_timeout { - desc { - en: """Timeout for proxy protocol. EMQX will close the TCP connection if proxy protocol packet is not received within the timeout.""" - zh: """代理协议超时。如果在超时时间内未收到代理协议数据包，EMQX将关闭TCP连接。""" - } - label: { - en: "Proxy protocol timeout" - zh: "Proxy protocol 超时时间" - } -} +mqtt_listener_proxy_protocol.label: +"""Proxy protocol""" -global_authentication { - desc { - en: """Default authentication configs for all MQTT listeners. +mqtt_listener_access_rules.desc: +"""The access control rules for this listener.
See: https://github.com/emqtt/esockd#allowdeny""" -For per-listener overrides see authentication in listener configs +mqtt_listener_access_rules.label: +"""Access rules""" -This option can be configured with: -
    -
  • []: The default value, it allows *ALL* logins
    • -
  • one: For example {enable:true,backend:\"built_in_database\",mechanism=\"password_based\"}
    • -
  • chain: An array of structs.
    • -
+server_ssl_opts_schema_enable_ocsp_stapling.desc: +"""Whether to enable Online Certificate Status Protocol (OCSP) stapling for the listener. If set to true, requires defining the OCSP responder URL and issuer PEM path.""" -When a chain is configured, the login credentials are checked against the backends per the configured order, until an 'allow' or 'deny' decision can be made. +server_ssl_opts_schema_enable_ocsp_stapling.label: +"""Enable OCSP Stapling""" -If there is no decision after a full chain exhaustion, the login is rejected.""" - zh: """全局 MQTT 监听器的默认认证配置。 为每个监听器配置认证参考监听器器配置中的authentication 配置。 +fields_tcp_opts_send_timeout_close.desc: +"""Close the connection if send timeout.""" -该配置可以被配置为： -
    -
  • []: 默认值，允许所有的登录请求 -
  • 配置为单认证器，例如 {enable:true,backend:\"built_in_database\",mechanism=\"password_based\"}
    • -
  • 配置为认证器数组
    • -
+fields_tcp_opts_send_timeout_close.label: +"""TCP send timeout close""" -当配置为认证链后，登录凭证会按照配置的顺序进行检查，直到做出allow 或 deny的结果。 +sysmon_os_cpu_check_interval.desc: +"""The time interval for the periodic CPU check.""" -如果在所有的认证器都执行完后，还是没有结果，登录将被拒绝。""" - } -} +sysmon_os_cpu_check_interval.label: +"""The time interval for the periodic CPU check.""" -listener_authentication { - desc { - en: """Per-listener authentication override. -Authentication can be one single authenticator instance or a chain of authenticators as an array. -When authenticating a login (username, client ID, etc.) the authenticators are checked in the configured order.""" - zh: """监听器认证重载。 -认证配置可以是单个认证器实例，也可以是一个认证器数组组成的认证链。 -执行登录验证时（用户名、客户端 ID 等），将按配置的顺序执行。""" - } - label: { - en: "Per-listener authentication override" - zh: "每个监听器的认证覆盖" - } -} +sysmon_top_sample_interval.desc: +"""Specifies how often process top should be collected""" -fields_rate_limit_max_conn_rate { - desc { - en: """Maximum connections per second.""" - zh: """每秒最大连接数。""" - } - label: { - en: "Max connection rate" - zh: "每秒最大连接数" - } -} +sysmon_top_sample_interval.label: +"""Top sample interval""" -fields_rate_limit_conn_messages_in { - desc { - en: """Message limit for the external MQTT connections.""" - zh: """外部 MQTT 连接的消息限制。""" - } - label: { - en: "connecting messages in" - zh: "外部 MQTT 连接的消息限制" - } -} +fields_mqtt_quic_listener_idle_timeout_ms.desc: +"""How long a connection can go idle before it is gracefully shut down. 0 to disable timeout""" -fields_rate_limit_conn_bytes_in { - desc { - en: """Limit the rate of receiving packets for a MQTT connection. -The rate is counted by bytes of packets per second.""" - zh: """限制 MQTT 连接接收数据包的速率。 速率以每秒的数据包字节数计算。""" - } - label: { - en: "Connection bytes in" - zh: "数据包速率" - } -} +fields_mqtt_quic_listener_idle_timeout_ms.label: +"""Idle timeout ms""" -client_ssl_opts_schema_server_name_indication { - desc { - en: """Specify the host name to be used in TLS Server Name Indication extension.
-For instance, when connecting to "server.example.net", the genuine server -which accepts the connection and performs TLS handshake may differ from the -host the TLS client initially connects to, e.g. when connecting to an IP address -or when the host has multiple resolvable DNS records
-If not specified, it will default to the host name string which is used -to establish the connection, unless it is IP addressed used.
-The host name is then also used in the host name verification of the peer -certificate.
The special value 'disable' prevents the Server Name -Indication extension from being sent and disables the hostname -verification check.""" - zh: """指定要在 TLS 服务器名称指示扩展中使用的主机名。
-例如，当连接到 "server.example.net" 时，接受连接并执行 TLS 握手的真正服务器可能与 TLS 客户端最初连接到的主机不同， -例如，当连接到 IP 地址时，或者当主机具有多个可解析的 DNS 记录时
-如果未指定，它将默认为使用的主机名字符串 -建立连接，除非使用 IP 地址
-然后，主机名也用于对等机的主机名验证证书
-特殊值 disable 阻止发送服务器名称指示扩展，并禁用主机名验证检查。""" - } - label: { - en: "Server Name Indication" - zh: "服务器名称指示" - } -} - -fields_tcp_opts_active_n { - desc { - en: """Specify the {active, N} option for this Socket.
-See: https://erlang.org/doc/man/inet.html#setopts-2""" - zh: """为此套接字指定{active，N}选项
-See: https://erlang.org/doc/man/inet.html#setopts-2""" - } - label: { - en: "active_n" - zh: "active_n" - } -} - -fields_tcp_opts_backlog { - desc { - en: """TCP backlog defines the maximum length that the queue of -pending connections can grow to.""" - zh: """TCP backlog 定义了挂起连接队列可以增长到的最大长度。""" - } - label: { - en: "TCP backlog length" - zh: "TCP 连接队列长度" - } -} - -fields_tcp_opts_send_timeout { - desc { - en: """The TCP send timeout for the connections.""" - zh: """连接的 TCP 发送超时。""" - } - label: { - en: "TCP send timeout" - zh: "TCP 发送超时" - } -} - -fields_tcp_opts_send_timeout_close { - desc { - en: """Close the connection if send timeout.""" - zh: """如果发送超时，则关闭连接。""" - } - label: { - en: "TCP send timeout close" - zh: "TCP 发送超时关闭连接" - } -} - -fields_tcp_opts_recbuf { - desc { - en: """The TCP receive buffer (OS kernel) for the connections.""" - zh: """连接的 TCP 接收缓冲区（OS 内核）。""" - } - label: { - en: "TCP receive buffer" - zh: "TCP 接收缓冲区" - } -} - -fields_tcp_opts_sndbuf { - desc { - en: """The TCP send buffer (OS kernel) for the connections.""" - zh: """连接的 TCP 发送缓冲区（OS 内核）。""" - } - label: { - en: "TCP send buffer" - zh: "TCP 发送缓冲区" - } -} - -fields_tcp_opts_buffer { - desc { - en: """The size of the user-space buffer used by the driver.""" - zh: """驱动程序使用的用户空间缓冲区的大小。""" - } - label: { - en: "TCP user-space buffer" - zh: "TCP 用户态缓冲区" - } -} - -fields_tcp_opts_high_watermark { - desc { - en: """The socket is set to a busy state when the amount of data queued internally -by the VM socket implementation reaches this limit.""" - zh: """当 VM 套接字实现内部排队的数据量达到此限制时，套接字将设置为忙碌状态。""" - } - label: { - en: "TCP 高水位线" - zh: "" - } -} - -fields_tcp_opts_nodelay { - desc { - en: """The TCP_NODELAY flag for the connections.""" - zh: """连接的 TCP_NODELAY 标识""" - } - label: { - en: "TCP_NODELAY" - zh: "TCP_NODELAY" - } -} - -fields_tcp_opts_reuseaddr { - desc { - en: """The SO_REUSEADDR flag for the connections.""" - zh: """连接的 SO_REUSEADDR 标识。""" - } - label: { - en: "SO_REUSEADDR" - zh: "SO_REUSEADDR" - } -} - -fields_trace_payload_encode { - desc { - en: """Determine the format of the payload format in the trace file.
-`text`: Text-based protocol or plain text protocol. - It is recommended when payload is JSON encoded.
-`hex`: Binary hexadecimal encode. It is recommended when payload is a custom binary protocol.
-`hidden`: payload is obfuscated as `******`""" - zh: """确定跟踪文件中有效负载格式的格式。
-`text`：基于文本的协议或纯文本协议。 -建议在有效负载为JSON编码时使用
-`hex`：二进制十六进制编码。当有效负载是自定义二进制协议时，建议使用此选项
-`hidden`：有效负载被模糊化为 `******`""" - } - label: { - en: "Payload encode" - zh: "有效负载编码" - } -} - -fields_ws_opts_mqtt_path { - desc { - en: """WebSocket's MQTT protocol path. So the address of EMQX Broker's WebSocket is: -ws://{ip}:{port}/mqtt""" - zh: """WebSocket 的 MQTT 协议路径。因此，EMQX Broker的WebSocket地址为： -ws://{ip}:{port}/mqtt""" - } - label: { - en: "WS MQTT Path" - zh: "WS MQTT 路径" - } -} - -fields_ws_opts_mqtt_piggyback { - desc { - en: """Whether a WebSocket message is allowed to contain multiple MQTT packets.""" - zh: """WebSocket消息是否允许包含多个 MQTT 数据包。""" - } - label: { - en: "MQTT Piggyback" - zh: "MQTT Piggyback" - } -} - -fields_ws_opts_compress { - desc { - en: """If true, compress WebSocket messages using zlib.
-The configuration items under deflate_opts belong to the compression-related parameter configuration.""" - zh: """如果 true，则使用zlib 压缩 WebSocket 消息
-deflate_opts 下的配置项属于压缩相关参数配置。""" - } - label: { - en: "Ws compress" - zh: "Ws 压缩" - } -} - -fields_ws_opts_idle_timeout { - desc { - en: """Close transport-layer connections from the clients that have not sent MQTT CONNECT message within this interval.""" - zh: """关闭在此间隔内未发送 MQTT CONNECT 消息的客户端的传输层连接。""" - } - label: { - en: "WS idle timeout" - zh: "WS 空闲时间" - } -} - -fields_ws_opts_max_frame_size { - desc { - en: """The maximum length of a single MQTT packet.""" - zh: """单个 MQTT 数据包的最大长度。""" - } - label: { - en: "Max frame size" - zh: "最大数据包长度" - } -} - -fields_ws_opts_fail_if_no_subprotocol { - desc { - en: """If true, the server will return an error when +fields_ws_opts_fail_if_no_subprotocol.desc: +"""If true, the server will return an error when the client does not carry the Sec-WebSocket-Protocol field.
Note: WeChat applet needs to disable this verification.""" - zh: """如果true，当客户端未携带Sec WebSocket Protocol字段时，服务器将返回一个错误。 -
注意：微信小程序需要禁用此验证。""" - } - label: { - en: "Fail if no subprotocol" - zh: "无 subprotocol 则失败" - } -} -fields_ws_opts_supported_subprotocols { - desc { - en: """Comma-separated list of supported subprotocols.""" - zh: """逗号分隔的 subprotocols 支持列表。""" - } - label: { - en: "Supported subprotocols" - zh: "Subprotocols 支持列表" - } -} +fields_ws_opts_fail_if_no_subprotocol.label: +"""Fail if no subprotocol""" -fields_ws_opts_check_origin_enable { - desc { - en: """If true, origin HTTP header will be - validated against the list of allowed origins configured in check_origins - parameter.""" - zh: """如果true，originHTTP 头将根据check_origins参数中配置的允许来源列表进行验证。""" - } - label: { - en: "Check origin" - zh: "检查 origin" - } -} +mqtt_wildcard_subscription.desc: +"""Whether to enable support for MQTT wildcard subscription.""" -fields_ws_opts_allow_origin_absence { - desc { - en: """If false and check_origin_enable is - true, the server will reject requests that don't have origin - HTTP header.""" - zh: """If false and check_origin_enable is true, the server will reject requests that don't have origin HTTP header.""" - } - label: { - en: "Allow origin absence" - zh: "允许 origin 缺失" - } -} +mqtt_wildcard_subscription.label: +"""Wildcard Subscription Available""" -fields_ws_opts_check_origins { - desc { - en: """List of allowed origins.
See check_origin_enable.""" - zh: """允许的 origins 列表""" - } - label: { - en: "Allowed origins" - zh: "允许的 origins" - } -} +server_ssl_opts_schema_ocsp_refresh_interval.desc: +"""The period to refresh the OCSP response for the server.""" -fields_ws_opts_proxy_address_header { - desc { - en: """HTTP header used to pass information about the client IP address. -Relevant when the EMQX cluster is deployed behind a load-balancer.""" - zh: """HTTP 头，用于传递有关客户端 IP 地址的信息。 -当 EMQX 集群部署在负载平衡器后面时，这一点非常重要。""" - } - label: { - en: "Proxy address header" - zh: "客户端地址头" - } -} +server_ssl_opts_schema_ocsp_refresh_interval.label: +"""OCSP Refresh Interval""" -fields_ws_opts_proxy_port_header { - desc { - en: """HTTP header used to pass information about the client port. Relevant when the EMQX cluster is deployed behind a load-balancer.""" - zh: """HTTP 头，用于传递有关客户端端口的信息。当 EMQX 集群部署在负载平衡器后面时，这一点非常重要。""" - } - label: { - en: "Proxy port header" - zh: "客户端端口头" - } -} +overload_protection_backoff_hibernation.desc: +"""When at high load, skip process hibernation.""" + +overload_protection_backoff_hibernation.label: +"""Skip hibernation""" + +fields_ws_opts_max_frame_size.desc: +"""The maximum length of a single MQTT packet.""" + +fields_ws_opts_max_frame_size.label: +"""Max frame size""" + +sys_event_messages.desc: +"""Client events messages.""" + +broker_perf_trie_compaction.desc: +"""Enable trie path compaction. +Enabling it significantly improves wildcard topic subscribe rate, if wildcard topics have unique prefixes like: 'sensor/{{id}}/+/', where ID is unique per subscriber. +Topic match performance (when publishing) may degrade if messages are mostly published to topics with large number of levels. + +NOTE: This is a cluster-wide configuration. It requires all nodes to be stopped before changing it.""" + +sysmon_vm_large_heap.desc: +"""When an Erlang process consumed a large amount of memory for its heap space, +the system will write a warning level large_heap log, and an MQTT message is published to +the system topic $SYS/sysmon/large_heap.""" + +sysmon_vm_large_heap.label: +"""Enable Large Heap monitoring.""" } diff --git a/rel/i18n/emqx_slow_subs_api.hocon b/rel/i18n/emqx_slow_subs_api.hocon index 92862bc98..edf473487 100644 --- a/rel/i18n/emqx_slow_subs_api.hocon +++ b/rel/i18n/emqx_slow_subs_api.hocon @@ -1,66 +1,30 @@ emqx_slow_subs_api { - clear_records_api { - desc { - en: "Clear current data and re count slow topic" - zh: "清除当前记录，然后重新开始统计" - } - } +clear_records_api.desc: +"""Clear current data and re count slow topic""" - get_records_api { - desc { - en: "View slow topics statistics record data" - zh: "查看慢订阅的统计数据" - } - } +clientid.desc: +"""Message clientid""" - get_setting_api { - desc { - en: "View slow subs settings" - zh: "查看配置" - } - } +get_records_api.desc: +"""View slow topics statistics record data""" - update_setting_api { - desc { - en: "Update slow subs settings" - zh: "更新配置" - } - } +get_setting_api.desc: +"""View slow subs settings""" - clientid { - desc { - en: "Message clientid" - zh: "消息的客户端 ID" - } - } +last_update_time.desc: +"""The timestamp of last update""" - node { - desc { - en: "Message node name" - zh: "消息的节点名称" - } - } +node.desc: +"""Message node name""" - topic { - desc { - en: "Message topic" - zh: "消息的主题" - } - } +timespan.desc: +"""Timespan for message transmission""" - timespan { - desc { - en: "Timespan for message transmission" - zh: "消息的传输耗时" - } - } +topic.desc: +"""Message topic""" - last_update_time { - desc { - en: "The timestamp of last update" - zh: "记录的更新时间戳" - } - } +update_setting_api.desc: +"""Update slow subs settings""" } diff --git a/rel/i18n/emqx_slow_subs_schema.hocon b/rel/i18n/emqx_slow_subs_schema.hocon index e65e802c2..4164db75a 100644 --- a/rel/i18n/emqx_slow_subs_schema.hocon +++ b/rel/i18n/emqx_slow_subs_schema.hocon @@ -1,38 +1,18 @@ emqx_slow_subs_schema { - enable { - desc { - en: "Enable this feature" - zh: "开启慢订阅" - } - } +enable.desc: +"""Enable this feature""" - threshold { - desc { - en: "The latency threshold for statistics" - zh: "慢订阅统计的阈值" - } - } +expire_interval.desc: +"""The eviction time of the record, which in the statistics record table""" - expire_interval { - desc { - en: "The eviction time of the record, which in the statistics record table" - zh: "慢订阅记录的有效时间" - } - } +stats_type.desc: +"""The method to calculate the latency""" - top_k_num { - desc { - en: "The maximum number of records in the slow subscription statistics record table" - zh: "慢订阅统计表的记录数量上限" - } - } +threshold.desc: +"""The latency threshold for statistics""" - stats_type { - desc { - en: "The method to calculate the latency" - zh: "慢订阅的统计类型" - } - } +top_k_num.desc: +"""The maximum number of records in the slow subscription statistics record table""" } diff --git a/rel/i18n/emqx_statsd_api.hocon b/rel/i18n/emqx_statsd_api.hocon index 2721188bd..d8bab13a7 100644 --- a/rel/i18n/emqx_statsd_api.hocon +++ b/rel/i18n/emqx_statsd_api.hocon @@ -1,16 +1,9 @@ emqx_statsd_api { - get_statsd_config_api { - desc { - en: """List the configuration of StatsD metrics collection and push service.""" - zh: """列出 StatsD 指标采集和推送服务的的配置。""" - } - } +get_statsd_config_api.desc: +"""List the configuration of StatsD metrics collection and push service.""" + +update_statsd_config_api.desc: +"""Update the configuration of StatsD metrics collection and push service.""" - update_statsd_config_api { - desc { - en: """Update the configuration of StatsD metrics collection and push service.""" - zh: """更新 StatsD 指标采集和推送服务的配置。""" - } - } } diff --git a/rel/i18n/emqx_statsd_schema.hocon b/rel/i18n/emqx_statsd_schema.hocon index 46d654a46..fc21710c4 100644 --- a/rel/i18n/emqx_statsd_schema.hocon +++ b/rel/i18n/emqx_statsd_schema.hocon @@ -1,61 +1,30 @@ emqx_statsd_schema { - get_statsd_config_api { - desc { - en: """List the configuration of StatsD metrics collection and push service.""" - zh: """列出 StatsD 指标采集和推送服务的的配置。""" - } - } +enable.desc: +"""Enable or disable StatsD metrics collection and push service.""" - update_statsd_config_api { - desc { - en: """Update the configuration of StatsD metrics collection and push service.""" - zh: """更新 StatsD 指标采集和推送服务的配置。""" - } - } +flush_interval.desc: +"""The push interval for metrics.""" - statsd { - desc { - en: """StatsD metrics collection and push configuration.""" - zh: """StatsD 指标采集与推送配置。""" - } - label { - en: """StatsD""" - zh: """StatsD""" - } - } +get_statsd_config_api.desc: +"""List the configuration of StatsD metrics collection and push service.""" - server { - desc { - en: """StatsD server address.""" - zh: """StatsD 服务器地址。""" - } - } +sample_interval.desc: +"""The sampling interval for metrics.""" - sample_interval { - desc { - en: """The sampling interval for metrics.""" - zh: """指标的采样间隔。""" - } - } +server.desc: +"""StatsD server address.""" - flush_interval { - desc { - en: """The push interval for metrics.""" - zh: """指标的推送间隔。""" - } - } - tags { - desc { - en: """The tags for metrics.""" - zh: """指标的标签。""" - } - } +statsd.desc: +"""StatsD metrics collection and push configuration.""" + +statsd.label: +"""StatsD""" + +tags.desc: +"""The tags for metrics.""" + +update_statsd_config_api.desc: +"""Update the configuration of StatsD metrics collection and push service.""" - enable { - desc { - en: """Enable or disable StatsD metrics collection and push service.""" - zh: """启用或禁用 StatsD 指标采集和推送服务。""" - } - } } diff --git a/rel/i18n/emqx_stomp_schema.hocon b/rel/i18n/emqx_stomp_schema.hocon new file mode 100644 index 000000000..05d5b9d18 --- /dev/null +++ b/rel/i18n/emqx_stomp_schema.hocon @@ -0,0 +1,16 @@ +emqx_stomp_schema { + +stom_frame_max_body_length.desc: +"""Maximum number of bytes of Body allowed per Stomp packet""" + +stom_frame_max_headers.desc: +"""The maximum number of Header""" + +stomp.desc: +"""The Stomp Gateway configuration. +This gateway supports v1.2/1.1/1.0""" + +stomp_frame_max_headers_length.desc: +"""The maximum string length of the Header Value""" + +} diff --git a/rel/i18n/emqx_telemetry_api.hocon b/rel/i18n/emqx_telemetry_api.hocon index a8f562065..5c61b8d3c 100644 --- a/rel/i18n/emqx_telemetry_api.hocon +++ b/rel/i18n/emqx_telemetry_api.hocon @@ -1,121 +1,54 @@ emqx_telemetry_api { - get_telemetry_status_api { - desc { - en: """Get telemetry status""" - zh: """获取遥测状态""" - } - } +active_modules.desc: +"""Get active modules""" - update_telemetry_status_api { - desc { - en: """Enable or disable telemetry""" - zh: """更新遥测状态""" - } - } +active_plugins.desc: +"""Get active plugins""" - get_telemetry_data_api { - desc { - en: """Get telemetry data""" - zh: """获取遥测数据""" - } - } +emqx_version.desc: +"""Get emqx version""" - enable { - desc { - en: """Enable telemetry""" - zh: """启用遥测""" - } - } +enable.desc: +"""Enable telemetry""" - emqx_version { - desc { - en: """Get emqx version""" - zh: """获取 emqx 版本""" - } - } +get_telemetry_data_api.desc: +"""Get telemetry data""" - license { - desc { - en: """Get license information""" - zh: """获取 license 信息""" - } - } +get_telemetry_status_api.desc: +"""Get telemetry status""" - os_name { - desc { - en: """Get OS name""" - zh: """获取操作系统名称""" - } - } +license.desc: +"""Get license information""" - os_version { - desc { - en: """Get OS version""" - zh: """获取操作系统版本""" - } - } +messages_received.desc: +"""Get number of messages received""" - otp_version { - desc { - en: """Get Erlang OTP version""" - zh: """获取 OTP 版本""" - } - } +messages_sent.desc: +"""Get number of messages sent""" - up_time { - desc { - en: """Get uptime""" - zh: """获取运行时间""" - } - } +nodes_uuid.desc: +"""Get nodes UUID""" - uuid { - desc { - en: """Get UUID""" - zh: """获取 UUID""" - } - } +num_clients.desc: +"""Get number of clients""" - nodes_uuid { - desc { - en: """Get nodes UUID""" - zh: """获取节点 UUID""" - } - } +os_name.desc: +"""Get OS name""" - active_plugins { - desc { - en: """Get active plugins""" - zh: """获取活跃插件""" - } - } +os_version.desc: +"""Get OS version""" - active_modules { - desc { - en: """Get active modules""" - zh: """获取活跃模块""" - } - } +otp_version.desc: +"""Get Erlang OTP version""" - num_clients { - desc { - en: """Get number of clients""" - zh: """获取客户端数量""" - } - } +up_time.desc: +"""Get uptime""" - messages_received { - desc { - en: """Get number of messages received""" - zh: """获取接收到的消息数量""" - } - } +update_telemetry_status_api.desc: +"""Enable or disable telemetry""" + +uuid.desc: +"""Get UUID""" - messages_sent { - desc { - en: """Get number of messages sent""" - zh: """获取发送的消息数量""" - } - } } diff --git a/rel/i18n/emqx_topic_metrics_api.hocon b/rel/i18n/emqx_topic_metrics_api.hocon index 22f038d4e..94c58f0cd 100644 --- a/rel/i18n/emqx_topic_metrics_api.hocon +++ b/rel/i18n/emqx_topic_metrics_api.hocon @@ -1,240 +1,105 @@ emqx_topic_metrics_api { - get_topic_metrics_api { - desc { - en: """List topic metrics""" - zh: """获取主题监控数据""" - } - } - reset_topic_metrics_api{ - desc { - en: """Reset telemetry status""" - zh: """重置主题监控状态""" - } - } +message_qos1_in_rate.desc: +"""QoS1 in messages rate""" - post_topic_metrics_api { - desc { - en: """Create topic metrics""" - zh: """创建主题监控数据""" - } - } +message_out_count.desc: +"""Out messages count""" - gat_topic_metrics_data_api { - desc { - en: """Get topic metrics""" - zh: """获取主题监控数据""" - } - } +message_qos2_out_rate.desc: +"""QoS2 out messages rate""" - delete_topic_metrics_data_api { - desc { - en: """Delete topic metrics""" - zh: """删除主题监控数据""" - } - } +message_qos0_in_rate.desc: +"""QoS0 in messages rate""" - topic_metrics_api_response409 { - desc { - en: """Conflict. Topic metrics exceeded max limit 512""" - zh: """冲突。主题监控数据超过最大限制512""" - } - } +get_topic_metrics_api.desc: +"""List topic metrics""" - topic_metrics_api_response400 { - desc { - en: """Bad request. Already exists or bad topic name""" - zh: """错误请求。已存在或错误的主题名称""" - } - } +reset_time.desc: +"""Reset time. In rfc3339. Nullable if never reset""" - topic_metrics_api_response404 { - desc { - en: """Not Found. Topic metrics not found""" - zh: """未找到。主题监控数据未找到""" - } - } +topic_metrics_api_response400.desc: +"""Bad request. Already exists or bad topic name""" - reset_topic_desc { - desc { - en: """Topic Name. If this parameter is not present,all created topic metrics will be reset.""" - zh: """主题名称。如果此参数不存在，则所有创建的主题监控数据都将重置。""" - } - } +reset_topic_desc.desc: +"""Topic Name. If this parameter is not present,all created topic metrics will be reset.""" - topic { - desc { - en: """Topic""" - zh: """主题""" - } - } +topic_metrics_api_response409.desc: +"""Conflict. Topic metrics exceeded max limit 512""" - topic_in_body { - desc { - en: """Raw topic string""" - zh: """主题字符串""" - } - } +post_topic_metrics_api.desc: +"""Create topic metrics""" - topic_in_path { - desc { - en: """Topic string. Notice: Topic string in url path must be encoded""" - zh: """主题字符串。注意：主题字符串在url路径中必须编码""" - } - } +message_dropped_rate.desc: +"""Dropped messages rate""" - action { - desc { - en: """Action. Only support reset""" - zh: """操作，仅支持 reset""" - } - } +message_qos2_in_rate.desc: +"""QoS2 in messages rate""" - create_time { - desc { - en: """Create time""" - zh: """创建时间。标准 rfc3339 时间格式，例如：2018-01-01T12:00:00Z""" - } - } +message_in_rate.desc: +"""In messages rate""" - reset_time { - desc { - en: """Reset time. In rfc3339. Nullable if never reset""" - zh: """重置时间。标准 rfc3339 时间格式，例如：2018-01-01T12:00:00Z。如果从未重置则为空""" - } - } +message_qos0_out_rate.desc: +"""QoS0 out messages rate""" - metrics { - desc { - en: """Metrics""" - zh: """监控数据""" - } - } +message_qos2_in_count.desc: +"""QoS2 in messages count""" - message_dropped_count { - desc { - en: """Dropped messages count""" - zh: """丢弃消息数量""" - } - } +message_dropped_count.desc: +"""Dropped messages count""" - message_dropped_rate { - desc { - en: """Dropped messages rate""" - zh: """丢弃消息速率""" - } - } +topic_metrics_api_response404.desc: +"""Not Found. Topic metrics not found""" - message_in_count { - desc { - en: """In messages count""" - zh: """接收消息数量""" - } - } +topic_in_path.desc: +"""Topic string. Notice: Topic string in url path must be encoded""" - message_in_rate { - desc { - en: """In messages rate""" - zh: """接收消息速率""" - } - } +action.desc: +"""Action. Only support reset""" - message_out_count { - desc { - en: """Out messages count""" - zh: """发送消息数量""" - } - } +message_qos0_in_count.desc: +"""QoS0 in messages count""" - message_out_rate { - desc { - en: """Out messages rate""" - zh: """发送消息速率""" - } - } +message_qos1_out_rate.desc: +"""QoS1 out messages rate""" - message_qos0_in_count { - desc { - en: """QoS0 in messages count""" - zh: """QoS0 接收消息数量""" - } - } +topic.desc: +"""Topic""" - message_qos0_in_rate { - desc { - en: """QoS0 in messages rate""" - zh: """QoS0 接收消息速率""" - } - } +reset_topic_metrics_api.desc: +"""Reset telemetry status""" - message_qos0_out_count { - desc { - en: """QoS0 out messages count""" - zh: """QoS0 发送消息数量""" - } - } +create_time.desc: +"""Create time""" - message_qos0_out_rate { - desc { - en: """QoS0 out messages rate""" - zh: """QoS0 发送消息速率""" - } - } +metrics.desc: +"""Metrics""" - message_qos1_in_count { - desc { - en: """QoS1 in messages count""" - zh: """QoS1 接收消息数量""" - } - } +message_qos1_out_count.desc: +"""QoS1 out messages count""" - message_qos1_in_rate { - desc { - en: """QoS1 in messages rate""" - zh: """QoS1 接收消息速率""" - } - } +gat_topic_metrics_data_api.desc: +"""Get topic metrics""" - message_qos1_out_count { - desc { - en: """QoS1 out messages count""" - zh: """QoS1 发送消息数量""" - } - } +message_qos1_in_count.desc: +"""QoS1 in messages count""" - message_qos1_out_rate { - desc { - en: """QoS1 out messages rate""" - zh: """QoS1 发送消息速率""" - } - } +delete_topic_metrics_data_api.desc: +"""Delete topic metrics""" - message_qos2_in_count { - desc { - en: """QoS2 in messages count""" - zh: """QoS2 接收消息数量""" - } - } +message_qos0_out_count.desc: +"""QoS0 out messages count""" - message_qos2_in_rate { - desc { - en: """QoS2 in messages rate""" - zh: """QoS2 接收消息速率""" - } - } +topic_in_body.desc: +"""Raw topic string""" - message_qos2_out_count { - desc { - en: """QoS2 out messages count""" - zh: """QoS2 发送消息数量""" - } - } +message_in_count.desc: +"""In messages count""" - message_qos2_out_rate { - desc { - en: """QoS2 out messages rate""" - zh: """QoS2 发送消息速率""" - } - } +message_qos2_out_count.desc: +"""QoS2 out messages count""" + +message_out_rate.desc: +"""Out messages rate""" } diff --git a/rel/i18n/zh/emqx_authn_api.hocon b/rel/i18n/zh/emqx_authn_api.hocon new file mode 100644 index 000000000..bd8332ac7 --- /dev/null +++ b/rel/i18n/zh/emqx_authn_api.hocon @@ -0,0 +1,96 @@ +emqx_authn_api { + +authentication_get.desc: +"""列出全局认证链上的认证器。""" + +authentication_id_delete.desc: +"""删除全局认证链上的指定认证器。""" + +authentication_id_get.desc: +"""获取全局认证链上的指定认证器。""" + +authentication_id_position_put.desc: +"""更改全局认证链上指定认证器的顺序。""" + +authentication_id_put.desc: +"""更新全局认证链上的指定认证器。""" + +authentication_id_status_get.desc: +"""获取全局认证链上指定认证器的状态。""" + +authentication_id_users_get.desc: +"""获取全局认证链上指定认证器中的用户数据。""" + +authentication_id_users_post.desc: +"""为全局认证链上的指定认证器创建用户数据。""" + +authentication_id_users_user_id_delete.desc: +"""删除全局认证链上指定认证器中的指定用户数据。""" + +authentication_id_users_user_id_get.desc: +"""获取全局认证链上指定认证器中的指定用户数据。""" + +authentication_id_users_user_id_put.desc: +"""更新全局认证链上指定认证器中的指定用户数据。""" + +authentication_post.desc: +"""为全局认证链创建认证器。""" + +is_superuser.desc: +"""是否是超级用户""" + +like_user_id.desc: +"""使用用户 ID （username 或 clientid）模糊查询。""" + +like_user_id.label: +"""like_user_id""" + +listeners_listener_id_authentication_get.desc: +"""列出监听器认证链上的认证器。""" + +listeners_listener_id_authentication_id_delete.desc: +"""删除监听器认证链上的指定认证器。""" + +listeners_listener_id_authentication_id_get.desc: +"""获取监听器认证链上的指定认证器。""" + +listeners_listener_id_authentication_id_position_put.desc: +"""更改监听器认证链上指定认证器的顺序。""" + +listeners_listener_id_authentication_id_put.desc: +"""更新监听器认证链上的指定认证器。""" + +listeners_listener_id_authentication_id_status_get.desc: +"""获取监听器认证链上指定认证器的状态。""" + +listeners_listener_id_authentication_id_users_get.desc: +"""列出监听器认证链上指定认证器中的用户数据。""" + +listeners_listener_id_authentication_id_users_post.desc: +"""为监听器认证链上的指定认证器创建用户数据。""" + +listeners_listener_id_authentication_id_users_user_id_delete.desc: +"""删除监听器认证链上指定认证器中的指定用户数据。""" + +listeners_listener_id_authentication_id_users_user_id_get.desc: +"""获取监听器认证链上指定认证器中的指定用户数据。""" + +listeners_listener_id_authentication_id_users_user_id_put.desc: +"""更新监听器认证链上指定认证器中的指定用户数据。""" + +listeners_listener_id_authentication_post.desc: +"""在监听器认证链上创建认证器。""" + +param_auth_id.desc: +"""认证器 ID。""" + +param_listener_id.desc: +"""监听器 ID。""" + +param_position.desc: +"""认证者在链中的位置。可能的值是 'front', 'rear', 'before:{other_authenticator}', 'after:{other_authenticator}'""" + +param_user_id.desc: +"""用户 ID。""" + +} diff --git a/rel/i18n/zh/emqx_authn_http.hocon b/rel/i18n/zh/emqx_authn_http.hocon new file mode 100644 index 000000000..17c922b33 --- /dev/null +++ b/rel/i18n/zh/emqx_authn_http.hocon @@ -0,0 +1,45 @@ +emqx_authn_http { + +body.desc: +"""HTTP request body。""" + +body.label: +"""Request Body""" + +get.desc: +"""使用 HTTP Server 作为认证服务的认证器的配置项 (使用 GET 请求)。""" + +headers.desc: +"""HTTP Headers 列表""" + +headers.label: +"""请求头""" + +headers_no_content_type.desc: +"""HTTP Headers 列表 (无 content-type) 。""" + +headers_no_content_type.label: +"""请求头(无 content-type)""" + +method.desc: +"""HTTP 请求方法。""" + +method.label: +"""请求方法""" + +post.desc: +"""使用 HTTP Server 作为认证服务的认证器的配置项 (使用 POST 请求)。""" + +request_timeout.desc: +"""HTTP 请求超时时长。""" + +request_timeout.label: +"""请求超时时间""" + +url.desc: +"""认证 HTTP 服务器地址。""" + +url.label: +"""URL""" + +} diff --git a/rel/i18n/zh/emqx_authn_jwt.hocon b/rel/i18n/zh/emqx_authn_jwt.hocon new file mode 100644 index 000000000..76b999101 --- /dev/null +++ b/rel/i18n/zh/emqx_authn_jwt.hocon @@ -0,0 +1,118 @@ +emqx_authn_jwt { + +acl_claim_name.desc: +"""JWT claim name to use for getting ACL rules.""" + +acl_claim_name.label: +"""ACL claim name""" + +algorithm.desc: +"""JWT 签名算法，支持 HMAC (配置为 hmac-based）和 RSA、ECDSA (配置为 public-key)。""" + +algorithm.label: +"""JWT 签名算法""" + +cacertfile.desc: +"""包含 PEM 编码的 CA 证书的文件的路径。""" + +cacertfile.label: +"""CA 证书文件""" + +certfile.desc: +"""包含用户证书的文件的路径。""" + +certfile.label: +"""证书文件""" + +enable.desc: +"""启用/禁用 SSL。""" + +enable.label: +"""启用/禁用 SSL""" + +endpoint.desc: +"""JWKS 端点， 它是一个以 JWKS 格式返回服务端的公钥集的只读端点。""" + +endpoint.label: +"""JWKS Endpoint""" + +from.desc: +"""要从中获取 JWT 的字段。""" + +from.label: +"""源字段""" + +jwt_hmac.desc: +"""用于认证的 JWT 使用 HMAC 算法签发时的配置。""" + +jwt_jwks.desc: +"""用于认证的 JWTs 需要从 JWKS 端点获取时的配置。""" + +keyfile.desc: +"""包含 PEM 编码的用户私钥的文件的路径。""" + +keyfile.label: +"""私钥文件""" + +jwt_public_key.desc: +"""用于认证的 JWT 使用 RSA 或 ECDSA 算法签发时的配置。""" + +public_key.desc: +"""用于验证 JWT 的公钥。""" + +public_key.label: +"""公钥""" + +refresh_interval.desc: +"""JWKS 刷新间隔。""" + +refresh_interval.label: +"""JWKS 刷新间隔""" + +secret.desc: +"""使用 HMAC 算法时用于验证 JWT 的密钥""" + +secret.label: +"""Secret""" + +secret_base64_encoded.desc: +"""密钥是否为 Base64 编码。""" + +secret_base64_encoded.label: +"""密钥是否为 Base64 编码""" + +server_name_indication.desc: +"""服务器名称指示（SNI）。""" + +server_name_indication.label: +"""服务器名称指示""" + +ssl.desc: +"""SSL 选项。""" + +ssl.label: +"""SSL 选项""" + +use_jwks.desc: +"""是否使用 JWKS。""" + +use_jwks.label: +"""是否使用 JWKS""" + +verify.desc: +"""指定握手过程中是否校验对端证书。""" + +verify.label: +"""Verify""" + +verify_claims.desc: +"""需要验证的自定义声明列表，它是一个名称/值对列表。 +值可以使用以下占位符： +- ${username}: 将在运行时被替换为客户端连接时使用的用户名 +- ${clientid}: 将在运行时被替换为客户端连接时使用的客户端标识符 +认证时将验证 JWT（取自 Password 字段）中 claims 的值是否与 verify_claims 中要求的相匹配。""" + +verify_claims.label: +"""Verify Claims""" + +} diff --git a/rel/i18n/zh/emqx_authn_mnesia.hocon b/rel/i18n/zh/emqx_authn_mnesia.hocon new file mode 100644 index 000000000..78fc96c75 --- /dev/null +++ b/rel/i18n/zh/emqx_authn_mnesia.hocon @@ -0,0 +1,12 @@ +emqx_authn_mnesia { + +builtin_db.desc: +"""使用内置数据库作为认证数据源的认证器的配置项。""" + +user_id_type.desc: +"""指定使用客户端ID `clientid` 还是用户名 `username` 进行认证。""" + +user_id_type.label: +"""认证 ID 类型""" + +} diff --git a/rel/i18n/zh/emqx_authn_mongodb.hocon b/rel/i18n/zh/emqx_authn_mongodb.hocon new file mode 100644 index 000000000..949322a76 --- /dev/null +++ b/rel/i18n/zh/emqx_authn_mongodb.hocon @@ -0,0 +1,45 @@ +emqx_authn_mongodb { + +collection.desc: +"""存储认证数据的集合。""" + +collection.label: +"""集合""" + +filter.desc: +"""在查询中定义过滤条件的条件表达式。 +过滤器支持如下占位符： +- ${username}: 将在运行时被替换为客户端连接时使用的用户名 +- ${clientid}: 将在运行时被替换为客户端连接时使用的客户端标识符""" + +filter.label: +"""过滤器""" + +is_superuser_field.desc: +"""文档中用于定义用户是否具有超级用户权限的字段。""" + +is_superuser_field.label: +"""超级用户字段""" + +password_hash_field.desc: +"""文档中用于存放密码散列的字段。""" + +password_hash_field.label: +"""密码散列字段""" + +replica-set.desc: +"""使用 MongoDB (Replica Set) 作为认证数据源的认证器的配置项。""" + +salt_field.desc: +"""文档中用于存放盐值的字段。""" + +salt_field.label: +"""盐值字段""" + +sharded-cluster.desc: +"""使用 MongoDB (Sharded Cluster) 作为认证数据源的认证器的配置项。""" + +single.desc: +"""使用 MongoDB (Standalone) 作为认证数据源的认证器的配置项。""" + +} diff --git a/rel/i18n/zh/emqx_authn_mysql.hocon b/rel/i18n/zh/emqx_authn_mysql.hocon new file mode 100644 index 000000000..c8b72d720 --- /dev/null +++ b/rel/i18n/zh/emqx_authn_mysql.hocon @@ -0,0 +1,18 @@ +emqx_authn_mysql { + +mysql.desc: +"""使用 MySQL 作为认证数据源的认证器的配置项。""" + +query.desc: +"""用于查询密码散列等用于认证的数据的 SQL 语句。""" + +query.label: +"""查询语句""" + +query_timeout.desc: +"""SQL 查询的超时时间。""" + +query_timeout.label: +"""查询超时""" + +} diff --git a/rel/i18n/zh/emqx_authn_pgsql.hocon b/rel/i18n/zh/emqx_authn_pgsql.hocon new file mode 100644 index 000000000..b9d47828b --- /dev/null +++ b/rel/i18n/zh/emqx_authn_pgsql.hocon @@ -0,0 +1,12 @@ +emqx_authn_pgsql { + +postgresql.desc: +"""使用 PostgreSQL 作为认证数据源的认证器的配置项。""" + +query.desc: +"""用于查询密码散列等用于认证的数据的 SQL 语句。""" + +query.label: +"""查询语句""" + +} diff --git a/rel/i18n/zh/emqx_authn_redis.hocon b/rel/i18n/zh/emqx_authn_redis.hocon new file mode 100644 index 000000000..58137f195 --- /dev/null +++ b/rel/i18n/zh/emqx_authn_redis.hocon @@ -0,0 +1,18 @@ +emqx_authn_redis { + +cluster.desc: +"""使用 Redis (Cluster) 作为认证数据源的认证器的配置项。""" + +cmd.desc: +"""用于查询密码散列等用于认证的数据的 Redis Command，目前仅支持 HGET 与 HMGET。""" + +cmd.label: +"""Command""" + +sentinel.desc: +"""使用 Redis (Sentinel) 作为认证数据源的认证器的配置项。""" + +single.desc: +"""使用 Redis (Standalone) 作为认证数据源的认证器的配置项。""" + +} diff --git a/rel/i18n/zh/emqx_authn_schema.hocon b/rel/i18n/zh/emqx_authn_schema.hocon new file mode 100644 index 000000000..e6e76e8ab --- /dev/null +++ b/rel/i18n/zh/emqx_authn_schema.hocon @@ -0,0 +1,135 @@ +emqx_authn_schema { + +backend.desc: +"""后端类型。""" + +backend.label: +"""后端类型""" + +enable.desc: +"""设为 true 或 false 以启用或禁用此认证数据源。""" + +enable.label: +"""启用""" + +failed.desc: +"""请求失败次数。""" + +failed.label: +"""失败""" + +matched.desc: +"""请求命中次数。""" + +matched.label: +"""已命中""" + +mechanism.desc: +"""认证机制。""" + +mechanism.label: +"""认证机制""" + +metrics.desc: +"""资源统计指标。""" + +metrics.label: +"""指标""" + +metrics_failed.desc: +"""在当前实例中找到需要的认证信息，并且实例返回认证失败的次数。""" + +metrics_failed.label: +"""实例认证失败的次数""" + +metrics_nomatch.desc: +"""在当前实例中没有找到需要的认证信息，实例被忽略的次数。""" + +metrics_nomatch.label: +"""实例被忽略的次数""" + +metrics_rate.desc: +"""实例被触发的速率。触发速率等于匹配速率 + 忽略速率，单位：次/秒。""" + +metrics_rate.label: +"""实例被触发的速率""" + +metrics_rate_last5m.desc: +"""实例5分钟内平均触发速率，单位：次/秒。""" + +metrics_rate_last5m.label: +"""实例5分钟内平均触发速率""" + +metrics_rate_max.desc: +"""实例曾经达到的最高触发速率，单位：次/秒。""" + +metrics_rate_max.label: +"""实例曾经达到的最高触发速率""" + +metrics_success.desc: +"""在当前实例中找到需要的认证信息，并且实例返回认证成功的次数。""" + +metrics_success.label: +"""实例认证成功的次数""" + +metrics_total.desc: +"""当前实例被触发的总次数。""" + +metrics_total.label: +"""当前实例被触发的总次数""" + +node.desc: +"""节点名称。""" + +node.label: +"""节点名称。""" + +node_error.desc: +"""节点上产生的错误。""" + +node_error.label: +"""节点产生的错误""" + +node_metrics.desc: +"""每个节点上资源的统计指标。""" + +node_metrics.label: +"""节点资源指标""" + +node_status.desc: +"""每个节点上资源的状态。""" + +node_status.label: +"""节点资源状态""" + +rate.desc: +"""命中速率，单位：次/秒。""" + +rate.label: +"""速率""" + +rate_last5m.desc: +"""5分钟内平均命中速率，单位：次/秒。""" + +rate_last5m.label: +"""5分钟内速率""" + +rate_max.desc: +"""最大命中速率，单位：次/秒。""" + +rate_max.label: +"""最大速率""" + +status.desc: +"""资源状态。""" + +status.label: +"""状态""" + +success.desc: +"""请求成功次数。""" + +success.label: +"""成功""" + +} diff --git a/rel/i18n/zh/emqx_authn_user_import_api.hocon b/rel/i18n/zh/emqx_authn_user_import_api.hocon new file mode 100644 index 000000000..a546066bb --- /dev/null +++ b/rel/i18n/zh/emqx_authn_user_import_api.hocon @@ -0,0 +1,9 @@ +emqx_authn_user_import_api { + +authentication_id_import_users_post.desc: +"""为全局认证链上的指定认证器导入用户数据。""" + +listeners_listener_id_authentication_id_import_users_post.desc: +"""为监听器认证链上的指定认证器导入用户数据。""" + +} diff --git a/rel/i18n/zh/emqx_authz_api_cache.hocon b/rel/i18n/zh/emqx_authz_api_cache.hocon new file mode 100644 index 000000000..94486e3e0 --- /dev/null +++ b/rel/i18n/zh/emqx_authz_api_cache.hocon @@ -0,0 +1,6 @@ +emqx_authz_api_cache { + +authorization_cache_delete.desc: +"""清除集群中所有授权数据缓存。""" + +} diff --git a/rel/i18n/zh/emqx_authz_api_mnesia.hocon b/rel/i18n/zh/emqx_authz_api_mnesia.hocon new file mode 100644 index 000000000..bc5121cc3 --- /dev/null +++ b/rel/i18n/zh/emqx_authz_api_mnesia.hocon @@ -0,0 +1,87 @@ +emqx_authz_api_mnesia { + +action.desc: +"""被授权的行为 (发布/订阅/所有)""" + +action.label: +"""行为""" + +clientid.desc: +"""客户端标识符""" + +clientid.label: +"""客户端标识符""" + +fuzzy_clientid.desc: +"""使用字串匹配模糊搜索客户端标识符""" + +fuzzy_clientid.label: +"""客户端标识符子串""" + +fuzzy_username.desc: +"""使用字串匹配模糊搜索用户名""" + +fuzzy_username.label: +"""用户名子串""" + +permission.desc: +"""权限""" + +permission.label: +"""权限""" + +rules_all_delete.desc: +"""删除 `all` 规则""" + +rules_all_get.desc: +"""列出为所有客户端启用的规则列表""" + +rules_all_post.desc: +"""创建/更新 为所有客户端启用的规则列表。""" + +rules_delete.desc: +"""清除内置数据库中的所有类型（'users' 、'clients' 、'all'）的所有规则""" + +topic.desc: +"""在指定主题上的规则""" + +topic.label: +"""主题""" + +user_clientid_delete.desc: +"""删除内置数据库中指定客户端标识符类型的规则记录""" + +user_clientid_get.desc: +"""获取内置数据库中指定客户端标识符类型的规则记录""" + +user_clientid_put.desc: +"""更新内置数据库中指定客户端标识符类型的规则记录""" + +user_username_delete.desc: +"""删除内置数据库中指定用户名类型的规则记录""" + +user_username_get.desc: +"""获取内置数据库中指定用户名类型的规则记录""" + +user_username_put.desc: +"""更新内置数据库中指定用户名类型的规则记录""" + +username.desc: +"""用户名""" + +username.label: +"""用户名""" + +users_clientid_get.desc: +"""获取内置数据库中所有客户端标识符类型的规则记录""" + +users_clientid_post.desc: +"""添加内置数据库中客户端标识符类型的规则记录""" + +users_username_get.desc: +"""获取内置数据库中所有用户名类型的规则记录""" + +users_username_post.desc: +"""添加内置数据库中用户名类型的规则记录""" + +} diff --git a/rel/i18n/zh/emqx_authz_api_schema.hocon b/rel/i18n/zh/emqx_authz_api_schema.hocon new file mode 100644 index 000000000..41e3819cd --- /dev/null +++ b/rel/i18n/zh/emqx_authz_api_schema.hocon @@ -0,0 +1,90 @@ +emqx_authz_api_schema { + +body.desc: +"""HTTP 请求体。""" + +body.label: +"""请求体""" + +cmd.desc: +"""访问控制数据查询命令。""" + +cmd.label: +"""查询命令""" + +collection.desc: +"""`MongoDB` 授权数据集。""" + +collection.label: +"""数据集""" + +enable.desc: +"""设为 true 或 false 以启用或禁用此访问控制数据源。""" + +enable.label: +"""enable""" + +filter.desc: +"""在查询中定义过滤条件的条件表达式。 +过滤器支持如下占位符： +- ${username}: 将在运行时被替换为客户端连接时使用的用户名 +- ${clientid}: 将在运行时被替换为客户端连接时使用的客户端标识符""" + +filter.label: +"""过滤器""" + +headers.desc: +"""HTTP Headers 列表""" + +headers.label: +"""请求头""" + +headers_no_content_type.desc: +"""HTTP Headers 列表（无 content-type）。""" + +headers_no_content_type.label: +"""请求头(无 content-type)""" + +method.desc: +"""HTTP 请求方法。""" + +method.label: +"""method""" + +position.desc: +"""认证数据源位置。""" + +position.label: +"""位置""" + +query.desc: +"""访问控制数据查询语句。""" + +query.label: +"""查询语句""" + +request_timeout.desc: +"""请求超时时间。""" + +request_timeout.label: +"""请求超时""" + +rules.desc: +"""静态授权文件规则。""" + +rules.label: +"""规则""" + +type.desc: +"""数据后端类型。""" + +type.label: +"""type""" + +url.desc: +"""认证服务器 URL。""" + +url.label: +"""url""" + +} diff --git a/rel/i18n/zh/emqx_authz_api_settings.hocon b/rel/i18n/zh/emqx_authz_api_settings.hocon new file mode 100644 index 000000000..c78fb7a0b --- /dev/null +++ b/rel/i18n/zh/emqx_authz_api_settings.hocon @@ -0,0 +1,9 @@ +emqx_authz_api_settings { + +authorization_settings_get.desc: +"""获取授权配置""" + +authorization_settings_put.desc: +"""更新授权配置""" + +} diff --git a/rel/i18n/zh/emqx_authz_api_sources.hocon b/rel/i18n/zh/emqx_authz_api_sources.hocon new file mode 100644 index 000000000..8e9bfef9c --- /dev/null +++ b/rel/i18n/zh/emqx_authz_api_sources.hocon @@ -0,0 +1,48 @@ +emqx_authz_api_sources { + +authorization_sources_get.desc: +"""列出所有授权数据源""" + +authorization_sources_post.desc: +"""添加授权数据源""" + +authorization_sources_type_delete.desc: +"""删除指定类型的授权数据源""" + +authorization_sources_type_get.desc: +"""获取指定类型的授权数据源""" + +authorization_sources_type_move_post.desc: +"""更新授权数据源的优先执行顺序""" + +authorization_sources_type_put.desc: +"""更新指定类型的授权数据源""" + +authorization_sources_type_status_get.desc: +"""获取指定授权数据源的状态""" + +source.desc: +"""授权数据源""" + +source.label: +"""数据源""" + +source_config.desc: +"""数据源配置""" + +source_config.label: +"""数据源配置""" + +source_type.desc: +"""数据源类型""" + +source_type.label: +"""数据源类型""" + +sources.desc: +"""授权数据源列表""" + +sources.label: +"""数据源列表""" + +} diff --git a/rel/i18n/zh/emqx_authz_schema.hocon b/rel/i18n/zh/emqx_authz_schema.hocon new file mode 100644 index 000000000..f0f973a55 --- /dev/null +++ b/rel/i18n/zh/emqx_authz_schema.hocon @@ -0,0 +1,285 @@ +emqx_authz_schema { + +deny.desc: +"""授权失败的次数。""" + +deny.label: +"""授权失败次数""" + +redis_sentinel.desc: +"""使用 Redis 授权（哨兵模式）。""" + +redis_sentinel.label: +"""redis_sentinel""" + +rate.desc: +"""命中速率，单位：次/秒。""" + +rate.label: +"""速率""" + +status.desc: +"""资源状态。""" + +status.label: +"""状态""" + +method.desc: +"""HTTP 请求方法""" + +method.label: +"""method""" + +query.desc: +"""访问控制数据查询语句/查询命令。""" + +query.label: +"""查询语句""" + +metrics_total.desc: +"""授权实例被触发的总次数。""" + +metrics_total.label: +"""授权实例被触发的总次数""" + +redis_cluster.desc: +"""使用 Redis 授权（集群模式）。""" + +redis_cluster.label: +"""redis_cluster""" + +mysql.desc: +"""使用 MySOL 数据库授权""" + +mysql.label: +"""mysql""" + +postgresql.desc: +"""使用 PostgreSQL 数据库授权""" + +postgresql.label: +"""postgresql""" + +mongo_rs.desc: +"""使用 MongoDB 授权（副本集模式）""" + +mongo_rs.label: +"""mongo_rs""" + +type.desc: +"""数据后端类型""" + +type.label: +"""type""" + +mongo_sharded.desc: +"""使用 MongoDB 授权（分片集群模式）。""" + +mongo_sharded.label: +"""mongo_sharded""" + +body.desc: +"""HTTP request body。""" + +body.label: +"""Request Body""" + +url.desc: +"""授权 HTTP 服务器地址。""" + +url.label: +"""URL""" + +node.desc: +"""节点名称。""" + +node.label: +"""节点名称。""" + +headers.desc: +"""HTTP Headers 列表""" + +headers.label: +"""请求头""" + +rate_last5m.desc: +"""5分钟内平均命中速率，单位：次/秒。""" + +rate_last5m.label: +"""5分钟内速率""" + +headers_no_content_type.desc: +"""HTTP Headers 列表 (无 content-type) 。""" + +headers_no_content_type.label: +"""请求头(无 content-type)""" + +node_error.desc: +"""节点上产生的错误。""" + +node_error.label: +"""节点产生的错误""" + +builtin_db.desc: +"""使用内部数据库授权（mnesia）。""" + +builtin_db.label: +"""Buitin Database""" + +enable.desc: +"""设为 true 或 false 以启用或禁用此访问控制数据源""" + +enable.label: +"""enable""" + +matched.desc: +"""请求命中次数。""" + +matched.label: +"""已命中""" + +node_status.desc: +"""每个节点上资源的状态。""" + +node_status.label: +"""节点资源状态""" + +rate_max.desc: +"""最大命中速率，单位：次/秒。""" + +rate_max.label: +"""最大速率""" + +filter.desc: +"""在查询中定义过滤条件的条件表达式。 +过滤器支持如下占位符：
+- ${username}：将在运行时被替换为客户端连接时使用的用户名
+- ${clientid}：将在运行时被替换为客户端连接时使用的客户端标识符""" + +filter.label: +"""过滤器""" + +path.desc: +"""包含 ACL 规则的文件路径。 +如果在启动 EMQX 节点前预先配置该路径， +那么可以将该文件置于任何 EMQX 可以访问到的位置。 + +如果从 EMQX Dashboard 或 HTTP API 创建或修改了规则集， +那么EMQX将会生成一个新的文件并将它存放在 `data_dir` 下的 `authz` 子目录中， +并从此弃用旧的文件。""" + +path.label: +"""path""" + +redis_single.desc: +"""使用 Redis 授权（单实例）。""" + +redis_single.label: +"""redis_single""" + +failed.desc: +"""请求失败次数。""" + +failed.label: +"""失败""" + +metrics.desc: +"""资源统计指标。""" + +metrics.label: +"""指标""" + +authorization.desc: +"""客户端授权相关配置""" + +authorization.label: +"""授权""" + +collection.desc: +"""`MongoDB` 授权数据集。""" + +collection.label: +"""数据集""" + +mongo_single.desc: +"""使用 MongoDB 授权（单实例）。""" + +mongo_single.label: +"""mongo_single""" + +file.desc: +"""使用静态文件授权""" + +file.label: +"""文件""" + +http_post.desc: +"""使用外部 HTTP 服务器授权(POST 请求)。""" + +http_post.label: +"""http_post""" + +request_timeout.desc: +"""HTTP 请求超时时长。""" + +request_timeout.label: +"""请求超时时间""" + +allow.desc: +"""授权成功的次数。""" + +allow.label: +"""授权成功次数""" + +cmd.desc: +"""访问控制数据查查询命令""" + +cmd.label: +"""查询命令""" + +nomatch.desc: +"""没有匹配到任何授权规则的次数。""" + +nomatch.label: +"""没有匹配到任何授权规则的次数""" + +sources.desc: +"""授权数据源。
+授权（ACL）数据源的列表。 +它被设计为一个数组，而不是一个散列映射， +所以可以作为链式访问控制。
+ +当授权一个 'publish' 或 'subscribe' 行为时， +该配置列表中的所有数据源将按顺序进行检查。 +如果在某个客户端未找到时(使用 ClientID 或 Username)。 +将会移动到下一个数据源。直至得到 'allow' 或 'deny' 的结果。
+ +如果在任何数据源中都未找到对应的客户端信息。 +配置的默认行为 ('authorization.no_match') 将生效。
+ +注意： +数据源使用 'type' 进行标识。 +使用同一类型的数据源多于一次不被允许。""" + +sources.label: +"""数据源""" + +node_metrics.desc: +"""每个节点上资源的统计指标。""" + +node_metrics.label: +"""节点资源指标""" + +success.desc: +"""请求成功次数。""" + +success.label: +"""成功""" + +http_get.desc: +"""使用外部 HTTP 服务器授权(GET 请求)。""" + +http_get.label: +"""http_get""" + +} diff --git a/rel/i18n/zh/emqx_auto_subscribe_api.hocon b/rel/i18n/zh/emqx_auto_subscribe_api.hocon new file mode 100644 index 000000000..7a42ece2c --- /dev/null +++ b/rel/i18n/zh/emqx_auto_subscribe_api.hocon @@ -0,0 +1,12 @@ +emqx_auto_subscribe_api { + +list_auto_subscribe_api.desc: +"""获取自动订阅主题列表""" + +update_auto_subscribe_api.desc: +"""更新自动订阅主题列表""" + +update_auto_subscribe_api_response409.desc: +"""超出自定订阅主题列表长度限制""" + +} diff --git a/rel/i18n/zh/emqx_auto_subscribe_schema.hocon b/rel/i18n/zh/emqx_auto_subscribe_schema.hocon new file mode 100644 index 000000000..f4156fe34 --- /dev/null +++ b/rel/i18n/zh/emqx_auto_subscribe_schema.hocon @@ -0,0 +1,48 @@ +emqx_auto_subscribe_schema { + +auto_subscribe.desc: +"""设备登录成功之后，通过预设的订阅表示符，为设备自动完成订阅。支持使用占位符。""" + +auto_subscribe.label: +"""自动订阅""" + +nl.desc: +"""缺省值为0， +MQTT v3.1.1：如果设备订阅了自己发布消息的主题，那么将收到自己发布的所有消息。 +MQTT v5：如果设备在订阅时将此选项设置为 1，那么服务端将不会向设备转发自己发布的消息""" + +nl.label: +"""No Local""" + +qos.desc: +"""缺省值为 0，服务质量， +QoS 0：消息最多传递一次，如果当时客户端不可用，则会丢失该消息。 +QoS 1：消息传递至少 1 次。 +QoS 2：消息仅传送一次。""" + +qos.label: +"""服务质量""" + +rap.desc: +"""缺省值为 0，这一选项用来指定服务端向客户端转发消息时是否要保留其中的 RETAIN 标识，注意这一选项不会影响保留消息中的 RETAIN 标识。因此当 Retain As Publish 选项被设置为 0 时，客户端直接依靠消息中的 RETAIN 标识来区分这是一个正常的转发消息还是一个保留消息，而不是去判断消息是否是自己订阅后收到的第一个消息（转发消息甚至可能会先于保留消息被发送，视不同 Broker 的具体实现而定）。""" + +rap.label: +"""Retain As Publish""" + +rh.desc: +"""指定订阅建立时服务端是否向客户端发送保留消息， +可选值 0：只要客户端订阅成功，服务端就发送保留消息。 +可选值 1：客户端订阅成功且该订阅此前不存在，服务端才发送保留消息。毕竟有些时候客户端重新发起订阅可能只是为了改变一下 QoS，并不意味着它想再次接收保留消息。 +可选值 2：即便客户订阅成功，服务端也不会发送保留消息。""" + +rh.label: +"""Retain Handling""" + +topic.desc: +"""订阅标识符，支持使用占位符，例如 client/${clientid}/username/${username}/host/${host}/port/${port} +必填，且不可为空字符串""" + +topic.label: +"""订阅标识符""" + +} diff --git a/rel/i18n/zh/emqx_bridge_api.hocon b/rel/i18n/zh/emqx_bridge_api.hocon new file mode 100644 index 000000000..06887a711 --- /dev/null +++ b/rel/i18n/zh/emqx_bridge_api.hocon @@ -0,0 +1,100 @@ +emqx_bridge_api { + +desc_api1.desc: +"""列出所有 Bridge""" + +desc_api1.label: +"""列出所有 Bridge""" + +desc_api2.desc: +"""通过类型和名字创建 Bridge""" + +desc_api2.label: +"""创建 Bridge""" + +desc_api3.desc: +"""通过 ID 获取 Bridge""" + +desc_api3.label: +"""获取 Bridge""" + +desc_api4.desc: +"""通过 ID 更新 Bridge""" + +desc_api4.label: +"""更新 Bridge""" + +desc_api5.desc: +"""通过 ID 删除 Bridge""" + +desc_api5.label: +"""删除 Bridge""" + +desc_api6.desc: +"""通过 ID 重置 Bridge 的计数""" + +desc_api6.label: +"""重置 Bridge 计数""" + +desc_api7.desc: +"""停止或启用所有节点上的桥接""" + +desc_api7.label: +"""集群 Bridge 操作""" + +desc_api8.desc: +"""在某个节点上停止/重新启动 Bridge。""" + +desc_api8.label: +"""单节点 Bridge 操作""" + +desc_api9.desc: +"""通过给定的 ID 测试创建一个新的桥接。
+ID 的格式必须为 ’{type}:{name}”""" + +desc_api9.label: +"""测试桥接创建""" + +desc_bridge_metrics.desc: +"""通过 Id 来获取桥接的指标信息""" + +desc_bridge_metrics.label: +"""获取桥接的指标""" + +desc_enable_bridge.desc: +"""启用或禁用所有节点上的桥接""" + +desc_enable_bridge.label: +"""是否启用集群内的桥接""" + +desc_param_path_enable.desc: +"""是否启用桥接""" + +desc_param_path_enable.label: +"""启用桥接""" + +desc_param_path_id.desc: +"""Bridge ID ， 格式为 {type}:{name}""" + +desc_param_path_id.label: +"""Bridge ID""" + +desc_param_path_node.desc: +"""节点名，比如 emqx@127.0.0.1""" + +desc_param_path_node.label: +"""节点名""" + +desc_param_path_operation_cluster.desc: +"""集群可用操作：停止、重新启动""" + +desc_param_path_operation_cluster.label: +"""集群可用操作""" + +desc_param_path_operation_on_node.desc: +"""节点可用操作：停止、重新启动""" + +desc_param_path_operation_on_node.label: +"""节点可用操作""" + +} diff --git a/rel/i18n/zh/emqx_bridge_gcp_pubsub.hocon b/rel/i18n/zh/emqx_bridge_gcp_pubsub.hocon new file mode 100644 index 000000000..19bd7058e --- /dev/null +++ b/rel/i18n/zh/emqx_bridge_gcp_pubsub.hocon @@ -0,0 +1,77 @@ +emqx_bridge_gcp_pubsub { + +connect_timeout.desc: +"""连接 HTTP 服务器的超时时间。""" + +connect_timeout.label: +"""连接超时""" + +desc_config.desc: +"""GCP PubSub 桥接配置""" + +desc_config.label: +"""GCP PubSub 桥接配置""" + +desc_name.desc: +"""桥接名字，可读描述""" + +desc_name.label: +"""桥接名字""" + +desc_type.desc: +"""桥接类型""" + +desc_type.label: +"""桥接类型""" + +local_topic.desc: +"""发送到 'local_topic' 的消息都会转发到 GCP PubSub。
+注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发到 GCP PubSub。""" + +local_topic.label: +"""本地 Topic""" + +max_retries.desc: +"""请求出错时的最大重试次数。""" + +max_retries.label: +"""最大重试次数""" + +payload_template.desc: +"""用于格式化外发信息的模板。 如果未定义，将以JSON格式发送所有可用的上下文。""" + +payload_template.label: +"""HTTP 请求消息体模板""" + +pipelining.desc: +"""正整数，设置最大可发送的异步 HTTP 请求数量。当设置为 1 时，表示每次发送完成 HTTP 请求后都需要等待服务器返回，再继续发送下一个请求。""" + +pipelining.label: +"""HTTP 流水线""" + +pool_size.desc: +"""连接池大小。""" + +pool_size.label: +"""连接池大小""" + +pubsub_topic.desc: +"""要发布消息的GCP PubSub主题。""" + +pubsub_topic.label: +"""GCP PubSub 主题""" + +request_timeout.desc: +"""废弃的。在缓冲区设置中配置请求超时。""" + +request_timeout.label: +"""HTTP 请求超时""" + +service_account_json.desc: +"""包含将与 PubSub 一起使用的 GCP 服务账户凭证的 JSON。 +当创建GCP服务账户时（如https://developers.google.com/identity/protocols/oauth2/service-account#creatinganaccount），可以选择下载 JSON 形式的凭证，然后在该配置项中使用。""" + +service_account_json.label: +"""GCP 服务账户凭证""" + +} diff --git a/rel/i18n/zh/emqx_bridge_kafka.hocon b/rel/i18n/zh/emqx_bridge_kafka.hocon new file mode 100644 index 000000000..31bae51d3 --- /dev/null +++ b/rel/i18n/zh/emqx_bridge_kafka.hocon @@ -0,0 +1,354 @@ +emqx_bridge_kafka { + +connect_timeout.desc: +"""建立 TCP 连接时的最大等待时长（若启用认证，这个等待时长也包含完成认证所需时间）。""" + +connect_timeout.label: +"""连接超时""" + +producer_opts.desc: +"""本地 MQTT 数据源和 Kafka 桥接的配置。""" + +producer_opts.label: +"""MQTT 到 Kafka""" + +min_metadata_refresh_interval.desc: +"""刷新 Kafka broker 和 Kafka 主题元数据段最短时间间隔。设置太小可能会增加 Kafka 压力。""" + +min_metadata_refresh_interval.label: +"""元数据刷新最小间隔""" + +kafka_producer.desc: +"""Kafka Producer 配置。""" + +kafka_producer.label: +"""Kafka Producer""" + +producer_buffer.desc: +"""配置消息缓存的相关参数。 + +当 EMQX 需要发送的消息超过 Kafka 处理能力，或者当 Kafka 临时下线时，EMQX 内部会将消息缓存起来。""" + +producer_buffer.label: +"""消息缓存""" + +socket_send_buffer.desc: +"""TCP socket 的发送缓存调优。默认值是针对高吞吐量的一个推荐值。""" + +socket_send_buffer.label: +"""Socket 发送缓存大小""" + +desc_name.desc: +"""桥接名字，可读描述""" + +desc_name.label: +"""桥接名字""" + +consumer_offset_commit_interval_seconds.desc: +"""指定 Kafka 消费组偏移量提交的时间间隔。""" + +consumer_offset_commit_interval_seconds.label: +"""偏移提交间隔""" + +consumer_max_batch_bytes.desc: +"""设置每次从 Kafka 拉取数据的字节数。如该配置小于 Kafka 消息的大小，可能会影响消费性能。""" + +consumer_max_batch_bytes.label: +"""拉取字节数""" + +socket_receive_buffer.desc: +"""TCP socket 的收包缓存调优。默认值是针对高吞吐量的一个推荐值。""" + +socket_receive_buffer.label: +"""Socket 收包缓存大小""" + +consumer_topic_mapping.desc: +"""指定 Kafka 主题和 MQTT 主题之间的映射关系。 应至少包含一项。""" + +consumer_topic_mapping.label: +"""主题映射关系""" + +producer_kafka_opts.desc: +"""Kafka 生产者参数。""" + +producer_kafka_opts.label: +"""生产者参数""" + +kafka_topic.desc: +"""Kafka 主题名称""" + +kafka_topic.label: +"""Kafka 主题名称""" + +consumer_kafka_topic.desc: +"""指定从哪个 Kafka 主题消费消息。""" + +consumer_kafka_topic.label: +"""Kafka 主题""" + +auth_username_password.desc: +"""基于用户名密码的认证。""" + +auth_username_password.label: +"""用户名密码认证""" + +auth_sasl_password.desc: +"""SASL 认证的密码。""" + +auth_sasl_password.label: +"""密码""" + +kafka_message_timestamp.desc: +"""生成 Kafka 消息时间戳的模版。该时间必需是一个整型数值（可以是字符串格式）例如 1661326462115 或 '1661326462115'。当所需的输入字段不存在，或不是一个整型时，则会使用当前系统时间。""" + +kafka_message_timestamp.label: +"""消息的时间戳""" + +buffer_mode.desc: +"""消息缓存模式。 +memory: 所有的消息都缓存在内存里。如果 EMQX 服务重启，缓存的消息会丢失。 +disk: 缓存到磁盘上。EMQX 重启后会继续发送重启前未发送完成的消息。 +hybrid: 先将消息缓存在内存中，当内存中的消息堆积超过一定限制（配置项 segment_bytes 描述了该限制）后，后续的消息会缓存到磁盘上。与 memory 模式一样，如果 EMQX 服务重启，缓存的消息会丢失。""" + +buffer_mode.label: +"""缓存模式""" + +consumer_mqtt_qos.desc: +"""转发 MQTT 消息时使用的 QoS。""" + +consumer_mqtt_qos.label: +"""QoS""" + +consumer_key_encoding_mode.desc: +"""通过 MQTT 转发之前，如何处理 Kafka 消息的 Key。none 使用 Kafka 消息中的 Key 原始值，不进行编码。 注意：在这种情况下，Key 必须是一个有效的 UTF-8 字符串。 +base64 对收到的密钥或值使用 base-64 编码。""" + +consumer_key_encoding_mode.label: +"""Key 编码模式""" + +auth_gssapi_kerberos.desc: +"""使用 GSSAPI/Kerberos 认证。""" + +auth_gssapi_kerberos.label: +"""GSSAPI/Kerberos""" + +consumer_mqtt_opts.desc: +"""本地 MQTT 消息转发。""" + +consumer_mqtt_opts.label: +"""MQTT 转发""" + +auth_kerberos_principal.desc: +"""SASL GSSAPI 认证方法的 Kerberos principal，例如 client_name@MY.KERBEROS.REALM.MYDOMAIN.COM注意：这里使用的 realm 需要配置在 EMQX 服务器的 /etc/krb5.conf 中""" + +auth_kerberos_principal.label: +"""Kerberos Principal""" + +socket_opts.desc: +"""更多 Socket 参数设置。""" + +socket_opts.label: +"""Socket 参数""" + +consumer_mqtt_topic.desc: +"""设置 Kafka 消息向哪个本地 MQTT 主题转发消息。""" + +consumer_mqtt_topic.label: +"""MQTT主题""" + +consumer_offset_reset_policy.desc: +"""如不存在偏移量历史记录或历史记录失效，消费者应使用哪个偏移量开始消费。""" + +consumer_offset_reset_policy.label: +"""偏移重置策略""" + +partition_count_refresh_interval.desc: +"""配置 Kafka 刷新分区数量的时间间隔。 +EMQX 发现 Kafka 分区数量增加后，会开始按 partition_strategy 配置，把消息发送到新的分区中。""" + +partition_count_refresh_interval.label: +"""分区数量刷新间隔""" + +max_batch_bytes.desc: +"""最大消息批量字节数。大多数 Kafka 环境的默认最低值是 1 MB，EMQX 的默认值比 1 MB 更小是因为需要补偿 Kafka 消息编码所需要的额外字节（尤其是当每条消息都很小的情况下）。当单个消息的大小超过该限制时，它仍然会被发送，（相当于该批量中只有单个消息）。""" + +max_batch_bytes.label: +"""最大批量字节数""" + +required_acks.desc: +"""设置 Kafka leader 在返回给 EMQX 确认之前需要等待多少个 follower 的确认。 + +all_isr: 需要所有的在线复制者都确认。 +leader_only: 仅需要分区 leader 确认。 +none: 无需 Kafka 回复任何确认。""" + +required_acks.label: +"""Kafka 确认数量""" + +metadata_request_timeout.desc: +"""刷新元数据时最大等待时长。""" + +metadata_request_timeout.label: +"""元数据请求超时""" + +desc_type.desc: +"""桥接类型""" + +desc_type.label: +"""桥接类型""" + +socket_nodelay.desc: +"""设置‘true’让系统内核立即发送。否则当需要发送的内容很少时，可能会有一定延迟（默认 40 毫秒）。""" + +socket_nodelay.label: +"""是否关闭延迟发送""" + +authentication.desc: +"""认证参数。""" + +authentication.label: +"""认证""" + +buffer_memory_overload_protection.desc: +"""缓存模式是 memory 或 hybrid 时适用。当系统处于高内存压力时，从队列中丢弃旧的消息以减缓内存增长。内存压力值由配置项 sysmon.os.sysmem_high_watermark 决定。注意，该配置仅在 Linux 系统中有效。""" + +buffer_memory_overload_protection.label: +"""内存过载保护""" + +auth_sasl_mechanism.desc: +"""SASL 认证方法名称。""" + +auth_sasl_mechanism.label: +"""认证方法""" + +config_enable.desc: +"""启用（true）或停用该（false）Kafka 数据桥接。""" + +config_enable.label: +"""启用或停用""" + +consumer_mqtt_payload.desc: +"""用于转换收到的 Kafka 消息的模板。 默认情况下，它将使用 JSON 格式来序列化来自 Kafka 的所有字段。 这些字段包括：headers：一个包含字符串键值对的 JSON 对象。 +key：Kafka 消息的键（使用选择的编码方式编码）。 +offset：消息的偏移量。 +topic：Kafka 主题。 +ts: 消息的时间戳。 +ts_type：消息的时间戳类型，值可能是： create， append 或 undefined。 +value: Kafka 消息值（使用选择的编码方式编码）。""" + +consumer_mqtt_payload.label: +"""MQTT Payload Template""" + +consumer_opts.desc: +"""本地 MQTT 转发 和 Kafka 消费者配置。""" + +consumer_opts.label: +"""MQTT 到 Kafka""" + +kafka_consumer.desc: +"""Kafka 消费者配置。""" + +kafka_consumer.label: +"""Kafka 消费者""" + +desc_config.desc: +"""Kafka 桥接配置""" + +desc_config.label: +"""Kafka 桥接配置""" + +consumer_value_encoding_mode.desc: +"""通过 MQTT 转发之前，如何处理 Kafka 消息的 Value。none 使用 Kafka 消息中的 Value 原始值，不进行编码。 注意：在这种情况下，Value 必须是一个有效的 UTF-8 字符串。 +base64 对收到的 Value 使用 base-64 编码。""" + +consumer_value_encoding_mode.label: +"""Value 编码模式""" + +buffer_per_partition_limit.desc: +"""为每个 Kafka 分区设置的最大缓存字节数。当超过这个上限之后，老的消息会被丢弃，为新的消息腾出空间。""" + +buffer_per_partition_limit.label: +"""Kafka 分区缓存上限""" + +bootstrap_hosts.desc: +"""用逗号分隔的 host[:port] 主机列表。默认端口号为 9092。""" + +bootstrap_hosts.label: +"""主机列表""" + +consumer_max_rejoin_attempts.desc: +"""消费组成员允许重新加入小组的最大次数。如超过该配置次数后仍未能成功加入消费组，则会在等待一段时间后重试。""" + +consumer_max_rejoin_attempts.label: +"""最大的重新加入尝试""" + +kafka_message_key.desc: +"""生成 Kafka 消息 Key 的模版。如果模版生成后为空值，则会使用 Kafka 的 NULL ，而非空字符串。""" + +kafka_message_key.label: +"""消息的 Key""" + +kafka_message.desc: +"""用于生成 Kafka 消息的模版。""" + +kafka_message.label: +"""Kafka 消息模版""" + +mqtt_topic.desc: +"""MQTT 主题数据源由桥接指定，或留空由规则动作指定。""" + +mqtt_topic.label: +"""源 MQTT 主题""" + +kafka_message_value.desc: +"""生成 Kafka 消息 Value 的模版。如果模版生成后为空值，则会使用 Kafka 的 NULL，而非空字符串。""" + +kafka_message_value.label: +"""消息的 Value""" + +partition_strategy.desc: +"""设置消息发布时应该如何选择 Kafka 分区。 + +random: 为每个消息随机选择一个分区。 +key_dispatch: Hash Kafka message key to a partition number""" + +partition_strategy.label: +"""分区选择策略""" + +buffer_segment_bytes.desc: +"""当缓存模式是 disk 或 hybrid 时适用。该配置用于指定缓存到磁盘上的文件的大小。""" + +buffer_segment_bytes.label: +"""缓存文件大小""" + +consumer_kafka_opts.desc: +"""Kafka消费者配置。""" + +consumer_kafka_opts.label: +"""Kafka 消费者""" + +max_inflight.desc: +"""设置 Kafka 生产者（每个分区一个）在收到 Kafka 的确认前最多发送多少个请求（批量）。调大这个值通常可以增加吞吐量，但是，当该值设置大于 1 时存在消息乱序的风险。""" + +max_inflight.label: +"""飞行窗口""" + +auth_sasl_username.desc: +"""SASL 认证的用户名。""" + +auth_sasl_username.label: +"""用户名""" + +auth_kerberos_keytab_file.desc: +"""SASL GSSAPI 认证方法的 Kerberos keytab 文件。注意：该文件需要上传到 EMQX 服务器中，且运行 EMQX 服务的系统账户需要有读取权限。""" + +auth_kerberos_keytab_file.label: +"""Kerberos keytab 文件""" + +compression.desc: +"""压缩方法。""" + +compression.label: +"""压缩""" + +} diff --git a/rel/i18n/zh/emqx_bridge_mqtt_schema.hocon b/rel/i18n/zh/emqx_bridge_mqtt_schema.hocon new file mode 100644 index 000000000..669d50398 --- /dev/null +++ b/rel/i18n/zh/emqx_bridge_mqtt_schema.hocon @@ -0,0 +1,21 @@ +emqx_bridge_mqtt_schema { + +config.desc: +"""MQTT Bridge 的配置。""" + +config.label: +"""配置""" + +desc_name.desc: +"""Bridge 名字，Bridge 的可读描述""" + +desc_name.label: +"""Bridge 名字""" + +desc_type.desc: +"""Bridge 的类型""" + +desc_type.label: +"""Bridge 类型""" + +} diff --git a/rel/i18n/zh/emqx_bridge_schema.hocon b/rel/i18n/zh/emqx_bridge_schema.hocon new file mode 100644 index 000000000..7512efa67 --- /dev/null +++ b/rel/i18n/zh/emqx_bridge_schema.hocon @@ -0,0 +1,158 @@ +emqx_bridge_schema { + +bridges_mqtt.desc: +"""桥接到另一个 MQTT Broker 的 MQTT Bridge""" + +bridges_mqtt.label: +"""MQTT Bridge""" + +bridges_webhook.desc: +"""转发消息到 HTTP 服务器的 WebHook""" + +bridges_webhook.label: +"""WebHook""" + +desc_bridges.desc: +"""MQTT Bridge 配置""" + +desc_bridges.label: +"""MQTT Bridge 配置""" + +desc_enable.desc: +"""启用/禁用 Bridge""" + +desc_enable.label: +"""启用/禁用 Bridge""" + +desc_metrics.desc: +"""Bridge 计数""" + +desc_metrics.label: +"""Bridge 计数""" + +desc_node_metrics.desc: +"""节点的计数器""" + +desc_node_metrics.label: +"""节点的计数器""" + +desc_node_name.desc: +"""节点的名字""" + +desc_node_name.label: +"""节点名字""" + +desc_node_status.desc: +"""节点的状态""" + +desc_node_status.label: +"""节点的状态""" + +desc_status.desc: +"""Bridge 的连接状态
+- connecting: 启动时的初始状态。
+- connected: 桥接驱动健康检查正常。
+- disconnected: 当桥接无法通过健康检查。
+- stopped: 桥接处于停用状态。
+- inconsistent: 集群中有各节点汇报的状态不一致。""" + +desc_status.label: +"""Bridge 状态""" + +desc_status_reason.desc: +"""桥接连接失败的原因。""" + +desc_status_reason.label: +"""失败原因""" + +metric_dropped.desc: +"""被丢弃的消息个数。""" + +metric_dropped.label: +"""丢弃""" + +metric_dropped_other.desc: +"""因为其他原因被丢弃的消息个数。""" + +metric_dropped_other.label: +"""其他丢弃""" + +metric_dropped_queue_full.desc: +"""因为队列已满被丢弃的消息个数。""" + +metric_dropped_queue_full.label: +"""队列已满被丢弃""" + +metric_dropped_resource_not_found.desc: +"""因为资源不存在被丢弃的消息个数。""" + +metric_dropped_resource_not_found.label: +"""资源不存在被丢弃""" + +metric_dropped_resource_stopped.desc: +"""因为资源已停用被丢弃的消息个数。""" + +metric_dropped_resource_stopped.label: +"""资源停用被丢弃""" + +metric_inflight.desc: +"""已异步地发送但没有收到 ACK 的消息个数。""" + +metric_inflight.label: +"""已发送未确认""" + +metric_matched.desc: +"""Bridge 被匹配到（被请求）的次数。""" + +metric_matched.label: +"""匹配次数""" + +metric_queuing.desc: +"""当前被缓存到磁盘队列的消息个数。""" + +metric_queuing.label: +"""被缓存""" + +metric_rate.desc: +"""执行操作的速率，次/秒""" + +metric_rate.label: +"""速率""" + +metric_rate_last5m.desc: +"""5 分钟平均速率，次/秒""" + +metric_rate_last5m.label: +"""5 分钟平均速率""" + +metric_rate_max.desc: +"""执行操作的最大速率，次/秒""" + +metric_rate_max.label: +"""执行操作的最大速率""" + +metric_received.desc: +"""从远程系统收到的消息个数。""" + +metric_received.label: +"""已接收""" + +metric_retried.desc: +"""重试的次数。""" + +metric_retried.label: +"""已重试""" + +metric_sent_failed.desc: +"""发送失败的消息个数。""" + +metric_sent_failed.label: +"""发送失败""" + +metric_sent_success.desc: +"""已经发送成功的消息个数。""" + +metric_sent_success.label: +"""发送成功""" + +} diff --git a/rel/i18n/zh/emqx_bridge_webhook_schema.hocon b/rel/i18n/zh/emqx_bridge_webhook_schema.hocon new file mode 100644 index 000000000..d7dd9dae0 --- /dev/null +++ b/rel/i18n/zh/emqx_bridge_webhook_schema.hocon @@ -0,0 +1,87 @@ +emqx_bridge_webhook_schema { + +config_body.desc: +"""HTTP 请求的正文。
+如果没有设置该字段，请求正文将是包含所有可用字段的 JSON object。
+如果该 webhook 是由于收到 MQTT 消息触发的，'所有可用字段' 将是 MQTT 消息的 +上下文信息；如果该 webhook 是由于规则触发的，'所有可用字段' 则为触发事件的上下文信息。
+允许使用带有变量的模板。""" + +config_body.label: +"""HTTP 请求正文""" + +config_direction.desc: +"""已废弃，Bridge 的方向，必须是 egress""" + +config_direction.label: +"""Bridge 方向""" + +config_enable.desc: +"""启用/禁用 Bridge""" + +config_enable.label: +"""启用/禁用 Bridge""" + +config_headers.desc: +"""HTTP 请求的标头。
+允许使用带有变量的模板。""" + +config_headers.label: +"""HTTP 请求标头""" + +config_local_topic.desc: +"""发送到 'local_topic' 的消息都会转发到 HTTP 服务器。
+注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发到 HTTP 服务器。""" + +config_local_topic.label: +"""本地 Topic""" + +config_max_retries.desc: +"""HTTP 请求失败最大重试次数""" + +config_max_retries.label: +"""HTTP 请求重试次数""" + +config_method.desc: +"""HTTP 请求的方法。 所有可用的方法包括：post、put、get、delete。
+允许使用带有变量的模板。""" + +config_method.label: +"""HTTP 请求方法""" + +config_request_timeout.desc: +"""HTTP 请求超时""" + +config_request_timeout.label: +"""HTTP 请求超时""" + +config_url.desc: +"""HTTP Bridge 的 URL。
+路径中允许使用带变量的模板，但是 host， port 不允许使用变量模板。
+例如， http://localhost:9901/${topic} 是允许的， +但是 http://${host}:9901/message +或 http://localhost:${port}/message +不允许。""" + +config_url.label: +"""HTTP Bridge""" + +desc_config.desc: +"""HTTP Bridge 配置""" + +desc_config.label: +"""HTTP Bridge 配置""" + +desc_name.desc: +"""Bridge 名字，Bridge 的可读描述""" + +desc_name.label: +"""Bridge 名字""" + +desc_type.desc: +"""Bridge 类型""" + +desc_type.label: +"""Bridge 类型""" + +} diff --git a/rel/i18n/zh/emqx_coap_api.hocon b/rel/i18n/zh/emqx_coap_api.hocon new file mode 100644 index 000000000..f3bee543e --- /dev/null +++ b/rel/i18n/zh/emqx_coap_api.hocon @@ -0,0 +1,27 @@ +emqx_coap_api { + +content_type.desc: +"""Payload 类型""" + +message_id.desc: +"""消息 ID""" + +method.desc: +"""请求 Method 类型""" + +payload.desc: +"""Payload 内容""" + +response_code.desc: +"""应答码""" + +send_coap_request.desc: +"""发送 CoAP 消息到指定客户端""" + +timeout.desc: +"""请求超时时间""" + +token.desc: +"""消息 Token, 可以为空""" + +} diff --git a/rel/i18n/zh/emqx_coap_schema.hocon b/rel/i18n/zh/emqx_coap_schema.hocon new file mode 100644 index 000000000..132c0f03a --- /dev/null +++ b/rel/i18n/zh/emqx_coap_schema.hocon @@ -0,0 +1,37 @@ +emqx_coap_schema { + +coap.desc: +"""CoAP 网关配置。 +该网关的实现基于 RFC-7252 和 https://core-wg.github.io/coap-pubsub/draft-ietf-core-pubsub.html""" + +coap_connection_required.desc: +"""是否开启连接模式。 +连接模式是非标准协议的功能。它维护 CoAP 客户端上线、认证、和连接状态的保持""" + +coap_heartbeat.desc: +"""CoAP 网关要求客户端的最小心跳间隔时间。 +当 connection_required 开启后，该参数用于检查客户端连接是否存活""" + +coap_notify_type.desc: +"""投递给 CoAP 客户端的通知消息类型。当客户端 Observe 一个资源（或订阅某个主题）时，网关会向客户端推送新产生的消息。其消息类型可设置为：
+ - non: 不需要客户端返回确认消息;
+ - con: 需要客户端返回一个确认消息;
+ - qos: 取决于消息的 QoS 等级; QoS 0 会以 `non` 类型下发，QoS 1/2 会以 `con` 类型下发""" + +coap_publish_qos.desc: +"""客户端发布请求的默认 QoS 等级。 +当 CoAP 客户端发起发布请求时，如果未携带 `qos` 参数则会使用该默认值。默认值可设置为：
+ - qos0、qos1、qos2: 设置为固定的 QoS 等级
+ - coap: 依据发布操作的 CoAP 报文类型来动态决定
+ * 当发布请求为 `non-confirmable` 类型时，取值为 qos0
+ * 当发布请求为 `confirmable` 类型时，取值为 qos1""" + +coap_subscribe_qos.desc: +"""客户端订阅请求的默认 QoS 等级。 +当 CoAP 客户端发起订阅请求时，如果未携带 `qos` 参数则会使用该默认值。默认值可设置为：
+ - qos0、 qos1、qos2: 设置为固定的 QoS 等级
+ - coap: 依据订阅操作的 CoAP 报文类型来动态决定
+ * 当订阅请求为 `non-confirmable` 类型时，取值为 qos0
+ * 当订阅请求为 `confirmable` 类型时，取值为 qos1""" + +} diff --git a/rel/i18n/zh/emqx_conf_schema.hocon b/rel/i18n/zh/emqx_conf_schema.hocon new file mode 100644 index 000000000..4c1edbdee --- /dev/null +++ b/rel/i18n/zh/emqx_conf_schema.hocon @@ -0,0 +1,774 @@ +emqx_conf_schema { + +common_handler_drop_mode_qlen.desc: +"""当缓冲的日志事件数大于此值时，新的日志事件将被丢弃。起到过载保护的功能。 +为了使过载保护算法正常工作必须要： sync_mode_qlen =< drop_mode_qlen =< flush_qlen 且 drop_mode_qlen > 1 +要禁用某些模式，请执行以下操作。 +- 如果sync_mode_qlen被设置为0，所有的日志事件都被同步处理。也就是说，异步日志被禁用。 +- 如果sync_mode_qlen被设置为与drop_mode_qlen相同的值，同步模式被禁用。也就是说，处理程序总是以异步模式运行，除非调用drop或flushing。 +- 如果drop_mode_qlen被设置为与flush_qlen相同的值，则drop模式被禁用，永远不会发生。""" + +common_handler_drop_mode_qlen.label: +"""进入丢弃模式的队列长度""" + +cluster_mcast_addr.desc: +"""指定多播 IPv4 地址。 +当 cluster.discovery_strategy 为 mcast 时，此配置项才有效。""" + +cluster_mcast_addr.label: +"""多播地址""" + +desc_cluster_dns.desc: +"""DNS SRV 记录服务发现。""" + +desc_cluster_dns.label: +"""DNS SRV 记录服务发现""" + +cluster_dns_name.desc: +"""指定 DNS A 记录的名字。emqx 会通过访问这个 DNS A 记录来获取 IP 地址列表。 +当cluster.discovery_strategy 为 dns 时有效。""" + +cluster_dns_name.label: +"""DNS名称""" + +rpc_keyfile.desc: +"""rpc.certfile 的私钥文件的路径。
+注意：此文件内容是私钥，所以需要设置权限为 600。""" + +rpc_keyfile.label: +"""RPC 私钥文件""" + +cluster_mcast_recbuf.desc: +"""接收数据报的内核级缓冲区的大小。 +当 cluster.discovery_strategy 为 mcast 时，此配置项才有效。""" + +cluster_mcast_recbuf.label: +"""多播接收数据缓冲区""" + +cluster_autoheal.desc: +"""集群脑裂自动恢复机制开关。""" + +cluster_autoheal.label: +"""节点脑裂自动修复机制""" + +log_overload_kill_enable.desc: +"""日志处理进程过载时为保护自己节点其它的业务能正常，强制杀死日志处理进程。""" + +log_overload_kill_enable.label: +"""日志处理进程过载保护""" + +node_etc_dir.desc: +"""etc 存放目录""" + +node_etc_dir.label: +"""Etc 目录""" + +cluster_proto_dist.desc: +"""分布式 Erlang 集群协议类型。可选值为:
+- inet_tcp: 使用 IPv4
+- inet_tls: 使用 TLS，需要配合 etc/ssl_dist.conf 一起使用。""" + +cluster_proto_dist.label: +"""集群内部通信协议""" + +log_burst_limit_enable.desc: +"""启用日志限流保护机制。""" + +log_burst_limit_enable.label: +"""日志限流保护""" + +dist_buffer_size.desc: +"""Erlang分布式缓冲区的繁忙阈值，单位是KB。""" + +dist_buffer_size.label: +"""Erlang分布式缓冲区的繁忙阈值(KB)""" + +common_handler_max_depth.desc: +"""Erlang 内部格式日志格式化和 Erlang 进程消息队列检查的最大深度。""" + +common_handler_max_depth.label: +"""最大深度""" + +desc_log.desc: +"""EMQX 日志记录支持日志事件的多个接收器。 每个接收器由一个_log handler_表示，可以独立配置。""" + +desc_log.label: +"""日志""" + +common_handler_flush_qlen.desc: +"""如果缓冲日志事件的数量增长大于此阈值，则会发生冲刷（删除）操作。 日志处理进程会丢弃缓冲的日志消息。 +来缓解自身不会由于内存瀑涨而影响其它业务进程。日志内容会提醒有多少事件被删除。""" + +common_handler_flush_qlen.label: +"""冲刷阈值""" + +common_handler_chars_limit.desc: +"""设置单个日志消息的最大长度。 如果超过此长度，则日志消息将被截断。最小可设置的长度为100。 +注意：如果日志格式为 JSON，限制字符长度可能会导致截断不完整的 JSON 数据。""" + +common_handler_chars_limit.label: +"""单条日志长度限制""" + +cluster_k8s_namespace.desc: +"""当使用 k8s 方式并且 cluster.k8s.address_type 指定为 dns 类型时， +可设置 emqx 节点名的命名空间。与 cluster.k8s.suffix 一起使用用以拼接得到节点名列表。""" + +cluster_k8s_namespace.label: +"""K8s 命名空间""" + +node_name.desc: +"""节点名。格式为 \@\。其中 可以是 IP 地址，也可以是 FQDN。 +详见 http://erlang.org/doc/reference_manual/distributed.html。""" + +node_name.label: +"""节点名""" + +rpc_port_discovery.desc: +"""manual: 通过 tcp_server_port 来发现端口。 +
stateless: 使用无状态的方式来发现端口，使用如下算法。如果节点名称是 +emqxN@127.0.0.1, N 是一个数字，那么监听端口就是 5370 + N。""" + +rpc_port_discovery.label: +"""RPC 端口发现策略""" + +log_overload_kill_restart_after.desc: +"""处理进程停止后，会在该延迟时间后自动重新启动。除非该值设置为 infinity，这会阻止任何后续的重启。""" + +log_overload_kill_restart_after.label: +"""处理进程重启延迟""" + +log_file_handler_max_size.desc: +"""此参数控制日志文件轮换。 `infinity` 意味着日志文件将无限增长，否则日志文件将在达到 `max_size`（以字节为单位）时进行轮换。 +与 rotation count配合使用。如果 counter 为 10，则是10个文件轮换。""" + +log_file_handler_max_size.label: +"""日志文件轮换大小""" + +desc_log_file_handler.desc: +"""日志处理进程将日志事件打印到文件。""" + +desc_log_file_handler.label: +"""文件日志处理进程""" + +rpc_socket_keepalive_count.desc: +"""keepalive 探测消息发送失败的次数，直到 RPC 连接被认为已经断开。""" + +rpc_socket_keepalive_count.label: +"""RPC Socket Keepalive 次数""" + +cluster_etcd_server.desc: +"""指定 etcd 服务的地址。如有多个服务使用逗号 , 分隔。 +当 cluster.discovery_strategy 为 etcd 时，此配置项才有效。""" + +cluster_etcd_server.label: +"""Etcd 服务器地址""" + +db_backend.desc: +"""配置后端数据库驱动，默认值为 rlog 它适用于大规模的集群。 +mnesia 是备选数据库，适合中小集群。""" + +db_backend.label: +"""内置数据库""" + +desc_authorization.desc: +"""授权相关""" + +desc_authorization.label: +"""授权""" + +cluster_etcd_ssl.desc: +"""当使用 TLS 连接 etcd 时的配置选项。 +当 cluster.discovery_strategy 为 etcd 时，此配置项才有效。""" + +cluster_etcd_ssl.label: +"""Etcd SSL 选项""" + +rpc_insecure_fallback.desc: +"""兼容旧的无鉴权模式""" + +rpc_insecure_fallback.label: +"""向后兼容旧的无鉴权模式""" + +cluster_mcast_buffer.desc: +"""用户级缓冲区的大小。 +当 cluster.discovery_strategy 为 mcast 时，此配置项才有效。""" + +cluster_mcast_buffer.label: +"""多播用户级缓冲区""" + +rpc_authentication_timeout.desc: +"""远程节点认证的超时时间。""" + +rpc_authentication_timeout.label: +"""RPC 认证超时时间""" + +cluster_call_retry_interval.desc: +"""当集群间调用出错时，多长时间重试一次。""" + +cluster_call_retry_interval.label: +"""重试时间间隔""" + +cluster_mcast_sndbuf.desc: +"""外发数据报的内核级缓冲区的大小。 +当 cluster.discovery_strategy 为 mcast 时，此配置项才有效。""" + +cluster_mcast_sndbuf.label: +"""多播发送缓存区""" + +rpc_driver.desc: +"""集群间通信使用的传输协议。""" + +rpc_driver.label: +"""RPC 驱动""" + +max_ets_tables.desc: +"""Erlang ETS 表的最大数量""" + +max_ets_tables.label: +"""Erlang 表的最大数量""" + +desc_db.desc: +"""内置数据库的配置。""" + +desc_db.label: +"""数据库""" + +desc_cluster_etcd.desc: +"""使用 'etcd' 服务的服务发现。""" + +desc_cluster_etcd.label: +"""'etcd' 服务的服务发现""" + +cluster_name.desc: +"""EMQX集群名称。每个集群都有一个唯一的名称。服务发现时会用于做路径的一部分。""" + +cluster_name.label: +"""集群名称""" + +log_rotation_enable.desc: +"""启用日志轮换功能。启动后生成日志文件后缀会加上对应的索引数字，比如：log/emqx.log.1。 +系统会默认生成*.siz/*.idx用于记录日志位置，请不要手动修改这两个文件。""" + +log_rotation_enable.label: +"""日志轮换""" + +cluster_call_cleanup_interval.desc: +"""清理过期事务的时间间隔""" + +cluster_call_cleanup_interval.label: +"""清理间隔""" + +desc_cluster_static.desc: +"""静态节点服务发现。新节点通过连接一个节点来加入集群。""" + +desc_cluster_static.label: +"""静态节点服务发现""" + +db_default_shard_transport.desc: +"""定义用于推送事务日志的默认传输。
+这可以在 db.shard_transports 中基于每个分片被覆盖。 +gen_rpc 使用 gen_rpc 库， +distr 使用 Erlang 发行版。""" + +db_default_shard_transport.label: +"""事务日志传输默认协议""" + +cluster_static_seeds.desc: +"""集群中的EMQX节点名称列表， +指定固定的节点列表，多个节点间使用逗号 , 分隔。 +当 cluster.discovery_strategy 为 static 时，此配置项才有效。 +适合于节点数量较少且固定的集群。""" + +cluster_static_seeds.label: +"""集群静态节点""" + +log_overload_kill_qlen.desc: +"""允许的最大队列长度。""" + +log_overload_kill_qlen.label: +"""最大队列长度""" + +node_backtrace_depth.desc: +"""错误信息中打印的最大堆栈层数""" + +node_backtrace_depth.label: +"""最大堆栈导数""" + +desc_log_burst_limit.desc: +"""短时间内产生的大量日志事件可能会导致问题，例如： + - 日志文件变得非常大 + - 日志文件轮换过快，有用信息被覆盖 + - 对系统的整体性能影响 + +日志突发限制功能可以暂时禁用日志记录以避免这些问题。""" + +desc_log_burst_limit.label: +"""日志突发限制""" + +common_handler_enable.desc: +"""启用此日志处理进程。""" + +common_handler_enable.label: +"""启用日志处理进程""" + +cluster_k8s_service_name.desc: +"""指定 Kubernetes 中 EMQX 的服务名。 +当 cluster.discovery_strategy 为 k8s 时，此配置项才有效。""" + +cluster_k8s_service_name.label: +"""K8s 服务别名""" + +log_rotation_count.desc: +"""轮换的最大日志文件数。""" + +log_rotation_count.label: +"""最大日志文件数""" + +node_cookie.desc: +"""分布式 Erlang 集群使用的 cookie 值。集群间保持一致""" + +node_cookie.label: +"""节点 Cookie""" + +db_role.desc: +"""选择节点的角色。
+core 节点提供数据的持久性，并负责写入。建议将核心节点放置在不同的机架或不同的可用区。
+repliant 节点是临时工作节点。 从集群中删除它们，不影响数据库冗余
+建议复制节点多于核心节点。
+注意：该参数仅在设置backend时生效到 rlog。""" + +db_role.label: +"""数据库角色""" + +rpc_tcp_server_port.desc: +"""RPC 本地服务使用的 TCP 端口。
+只有当 rpc.port_discovery 设置为 manual 时，此配置才会生效。""" + +rpc_tcp_server_port.label: +"""RPC TCP 服务监听端口""" + +desc_console_handler.desc: +"""日志处理进程将日志事件打印到 EMQX 控制台。""" + +desc_console_handler.label: +"""Console Handler""" + +node_applications.desc: +"""当新EMQX 加入集群时，应重启的Erlang应用程序的列表。""" + +node_applications.label: +"""应用""" + +log_burst_limit_max_count.desc: +"""在 `window_time` 间隔内处理的最大日志事件数。 达到限制后，将丢弃连续事件，直到 `window_time` 结束。""" + +log_burst_limit_max_count.label: +"""日志事件数""" + +rpc_tcp_client_num.desc: +"""设置本节点与远程节点之间的 RPC 通信通道的最大数量。""" + +rpc_tcp_client_num.label: +"""RPC TCP 客户端数量""" + +cluster_k8s_address_type.desc: +"""当使用 k8s 方式集群时，address_type 用来从 Kubernetes 接口的应答里获取什么形式的 Host 列表。 +指定 cluster.k8s.address_type 为 ip，则将从 Kubernetes 接口中获取集群中其他节点 +的IP地址。""" + +cluster_k8s_address_type.label: +"""K8s 地址类型""" + +rpc_socket_sndbuf.desc: +"""TCP 调节参数。TCP 发送缓冲区大小。""" + +rpc_socket_sndbuf.label: +"""RPC 套接字发送缓冲区大小""" + +cluster_mcast_ttl.desc: +"""指定多播的 Time-To-Live 值。 +当 cluster.discovery_strategy 为 mcast 时，此配置项才有效。""" + +cluster_mcast_ttl.label: +"""多播TTL""" + +db_core_nodes.desc: +"""当前节点连接的核心节点列表。
+注意：该参数仅在设置backend时生效到 rlog +并且设置role为replicant时生效。
+该值需要在手动或静态集群发现机制下设置。
+如果使用了自动集群发现机制（如etcd），则不需要设置该值。""" + +db_core_nodes.label: +"""数据库核心节点""" + +log_file_handler_file.desc: +"""日志文件路径及名字。""" + +log_file_handler_file.label: +"""日志文件名字""" + +node_dist_net_ticktime.desc: +"""系统调优参数，此配置将覆盖 vm.args 文件里的 -kernel net_ticktime 参数。当一个节点持续无响应多久之后，认为其已经宕机并断开连接。""" + +node_dist_net_ticktime.label: +"""节点间心跳间隔""" + +desc_cluster_k8s.desc: +"""Kubernetes 服务发现。""" + +desc_cluster_k8s.label: +"""Kubernetes 服务发现""" + +desc_cluster_mcast.desc: +"""UDP 组播服务发现。""" + +desc_cluster_mcast.label: +"""UDP 组播服务发现""" + +rpc_cacertfile.desc: +"""验证 rpc.certfile 的 CA 证书文件的路径。
+注意：集群中所有节点的证书必须使用同一个 CA 签发。""" + +rpc_cacertfile.label: +"""RPC CA 证书文件""" + +desc_node.desc: +"""节点名称、Cookie、配置文件、数据目录和 Erlang 虚拟机（BEAM）启动参数。""" + +desc_node.label: +"""节点""" + +cluster_k8s_apiserver.desc: +"""指定 Kubernetes API Server。如有多个 Server 使用逗号 , 分隔。 +当 cluster.discovery_strategy 为 k8s 时，此配置项才有效。""" + +cluster_k8s_apiserver.label: +"""K8s 服务地址""" + +common_handler_supervisor_reports.desc: +"""Supervisor 报告的类型。默认为 error 类型。
+ - error：仅记录 Erlang 进程中的错误。 + - progress：除了 error 信息外，还需要记录进程启动的详细信息。""" + +common_handler_supervisor_reports.label: +"""报告类型""" + +node_data_dir.desc: +"""节点数据存放目录，可能会自动创建的子目录如下：
+- `mnesia/`。EMQX的内置数据库目录。例如，`mnesia/emqx@127.0.0.1`。
+如果节点要被重新命名（例如，`emqx@10.0.1.1`）。旧目录应该首先被删除。
+- `configs`。在启动时生成的配置，以及集群/本地覆盖的配置。
+- `patches`: 热补丁文件将被放在这里。
+- `trace`: 日志跟踪文件。
+ +**注意**: 一个数据dir不能被两个或更多的EMQX节点同时使用。""" + +node_data_dir.label: +"""节点数据目录""" + +cluster_k8s_suffix.desc: +"""当使用 k8s 方式并且 cluster.k8s.address_type 指定为 dns 类型时，可设置 emqx 节点名的后缀。 +与 cluster.k8s.namespace 一起使用用以拼接得到节点名列表。""" + +cluster_k8s_suffix.label: +"""K8s 前缀""" + +db_rpc_module.desc: +"""集群间推送事务日志到复制节点使用的协议。""" + +db_rpc_module.label: +"""RPC协议""" + +cluster_etcd_prefix.desc: +"""指定 etcd 路径的前缀。每个节点在 etcd 中都会创建一个路径: +v2/keys///
+当 cluster.discovery_strategy 为 etcd 时，此配置项才有效。""" + +cluster_etcd_prefix.label: +"""Etcd 路径前缀""" + +cluster_mcast_iface.desc: +"""指定节点发现服务需要绑定到本地 IP 地址。 +当 cluster.discovery_strategy 为 mcast 时，此配置项才有效。""" + +cluster_mcast_iface.label: +"""多播绑定地址""" + +log_burst_limit_window_time.desc: +"""参考 max_count。""" + +log_burst_limit_window_time.label: +"""Window Time""" + +cluster_dns_record_type.desc: +"""DNS 记录类型。""" + +cluster_dns_record_type.label: +"""DNS记录类型""" + +cluster_autoclean.desc: +"""指定多久之后从集群中删除离线节点。""" + +cluster_autoclean.label: +"""自动删除离线节点时间""" + +process_limit.desc: +"""Erlang系统同时存在的最大进程数。 +实际选择的最大值可能比设置的数字大得多。 +参考: https://www.erlang.org/doc/man/erl.html""" + +process_limit.label: +"""Erlang 最大进程数""" + +max_ports.desc: +"""Erlang系统同时存在的最大端口数。 +实际选择的最大值可能比设置的数字大得多。 +参考: https://www.erlang.org/doc/man/erl.html""" + +max_ports.label: +"""Erlang 最大端口数""" + +desc_log_rotation.desc: +"""默认情况下，日志存储在 `./log` 目录（用于从 zip 文件安装）或 `/var/log/emqx`（用于二进制安装）。
+这部分配置，控制每个日志处理进程保留的文件数量。""" + +desc_log_rotation.label: +"""日志轮换""" + +desc_log_overload_kill.desc: +"""日志过载终止，具有过载保护功能。当日志处理进程使用过多内存，或者缓存的日志消息过多时该功能被激活。
+检测到过载时，日志处理进程将终止，并在冷却期后重新启动。""" + +desc_log_overload_kill.label: +"""日志过载保护""" + +authorization.desc: +"""授权（ACL）。EMQX 支持完整的客户端访问控制（ACL）。""" + +authorization.label: +"""授权""" + +rpc_socket_keepalive_idle.desc: +"""broker 之间的连接在最后一条消息发送后保持打开的时间。""" + +rpc_socket_keepalive_idle.label: +"""RPC Socket Keepalive Idle""" + +desc_cluster_call.desc: +"""集群调用功能的选项。""" + +desc_cluster_call.label: +"""集群调用""" + +cluster_mcast_ports.desc: +"""指定多播端口。如有多个端口使用逗号 , 分隔。 +当 cluster.discovery_strategy 为 mcast 时，此配置项才有效。""" + +cluster_mcast_ports.label: +"""多播端口""" + +log_overload_kill_mem_size.desc: +"""日志处理进程允许使用的最大内存。""" + +log_overload_kill_mem_size.label: +"""日志处理进程允许使用的最大内存""" + +rpc_connect_timeout.desc: +"""建立 RPC 连接的超时时间。""" + +rpc_connect_timeout.label: +"""RPC 连接超时时间""" + +cluster_etcd_node_ttl.desc: +"""指定 etcd 中节点信息的过期时间。 +当 cluster.discovery_strategy 为 etcd 时，此配置项才有效。""" + +cluster_etcd_node_ttl.label: +"""Etcd 节点过期时间""" + +rpc_call_receive_timeout.desc: +"""同步 RPC 的回复超时时间。""" + +rpc_call_receive_timeout.label: +"""RPC 调用接收超时时间""" + +rpc_socket_recbuf.desc: +"""TCP 调节参数。TCP 接收缓冲区大小。""" + +rpc_socket_recbuf.label: +"""RPC 套接字接收缓冲区大小""" + +db_tlog_push_mode.desc: +"""同步模式下，核心节点等待复制节点的确认信息，然后再发送下一条事务日志。""" + +db_tlog_push_mode.label: +"""Tlog推送模式""" + +node_crash_dump_bytes.desc: +"""限制崩溃文件的大小，当崩溃时节点内存太大， +如果为了保存现场，需要全部存到崩溃文件中，此处限制最多能保存多大的文件。 +如果超过此限制，崩溃转储将被截断。如果设置为0，系统不会尝试写入崩溃转储文件。""" + +node_crash_dump_bytes.label: +"""崩溃文件最大容量""" + +rpc_certfile.desc: +"""TLS 证书文件的路径，用于验证集群节点的身份。 +只有当 rpc.driver 设置为 ssl 时，此配置才会生效。""" + +rpc_certfile.label: +"""RPC 证书文件""" + +node_crash_dump_seconds.desc: +"""该配置给出了运行时系统允许花费的写入崩溃转储的秒数。当给定的秒数已经过去，运行时系统将被终止。
+- 如果设置为0秒，运行时会立即终止，不会尝试写入崩溃转储文件。
+- 如果设置为一个正数 S，节点会等待 S 秒来完成崩溃转储文件，然后用SIGALRM信号终止运行时系统。
+- 如果设置为一个负值导致运行时系统的终止等待无限期地直到崩溃转储文件已经完全写入。""" + +node_crash_dump_seconds.label: +"""保存崩溃文件最长时间""" + +log_file_handlers.desc: +"""输出到文件的日志处理进程列表""" + +log_file_handlers.label: +"""File Handler""" + +node_global_gc_interval.desc: +"""系统调优参数，设置节点运行多久强制进行一次全局垃圾回收。禁用设置为 disabled。""" + +node_global_gc_interval.label: +"""全局垃圾回收""" + +common_handler_time_offset.desc: +"""日志中的时间戳使用的时间偏移量。 +可选值为： + - system: 本地系统使用的时区偏移量 + - utc: 0 时区的偏移量 + - +-[hh]:[mm]: 自定义偏移量，比如 "-02:00" 或者 "+00:00" +默认值为本地系统的时区偏移量：system。""" + +common_handler_time_offset.label: +"""时间偏移量""" + +rpc_mode.desc: +"""在 sync 模式下，发送端等待接收端的 ack信号。""" + +rpc_mode.label: +"""RPC 模式""" + +node_crash_dump_file.desc: +"""设置 Erlang crash_dump 文件的存储路径和文件名。""" + +node_crash_dump_file.label: +"""节点崩溃时的Dump文件""" + +cluster_mcast_loop.desc: +"""设置多播的报文是否投递到本地回环地址。 +当 cluster.discovery_strategy 为 mcast 时，此配置项才有效。""" + +cluster_mcast_loop.label: +"""多播回环开关""" + +rpc_socket_keepalive_interval.desc: +"""keepalive 消息的间隔。""" + +rpc_socket_keepalive_interval.label: +"""RPC Socket Keepalive 间隔""" + +common_handler_level.desc: +"""当前日志处理进程的日志级别。 +默认为 warning 级别。""" + +common_handler_level.label: +"""日志级别""" + +desc_rpc.desc: +"""EMQX 使用 gen_rpc 库来实现跨节点通信。
+大多数情况下，默认的配置应该可以工作，但如果你需要做一些性能优化或者实验，可以尝试调整这些参数。""" + +desc_rpc.label: +"""RPC""" + +rpc_ssl_server_port.desc: +"""RPC 本地服务使用的监听SSL端口。
+只有当 rpc.port_discovery 设置为 manual 且 dirver 设置为 ssl， +此配置才会生效。""" + +rpc_ssl_server_port.label: +"""RPC SSL 服务监听端口""" + +desc_cluster.desc: +"""EMQX 节点可以组成一个集群，以提高总容量。
这里指定了节点之间如何连接。""" + +desc_cluster.label: +"""集群""" + +common_handler_sync_mode_qlen.desc: +"""只要缓冲的日志事件的数量低于这个值，所有的日志事件都会被异步处理。 +这意味着，日志落地速度不会影响正常的业务进程，因为它们不需要等待日志处理进程的响应。 +如果消息队列的增长超过了这个值，处理程序开始同步处理日志事件。也就是说，发送事件的客户进程必须等待响应。 +当处理程序将消息队列减少到低于sync_mode_qlen阈值的水平时，异步操作就会恢复。 +默认为100条信息，当等待的日志事件大于100条时，就开始同步处理日志。""" + +common_handler_sync_mode_qlen.label: +"""进入异步模式的队列长度""" + +common_handler_formatter.desc: +"""选择日志格式类型。 text 用于纯文本，json 用于结构化日志记录。""" + +common_handler_formatter.label: +"""日志格式类型""" + +rpc_async_batch_size.desc: +"""异步模式下，发送的批量消息的最大数量。""" + +rpc_async_batch_size.label: +"""异步模式下的批量消息数量""" + +cluster_call_max_history.desc: +"""集群间调用最多保留的历史记录数。只用于排错时查看。""" + +cluster_call_max_history.label: +"""最大历史记录""" + +cluster_discovery_strategy.desc: +"""集群节点发现方式。可选值为: +- manual: 使用 emqx ctl cluster 命令管理集群。
+- static: 配置静态节点。配置几个固定的节点，新节点通过连接固定节点中的某一个来加入集群。
+- dns: 使用 DNS A 记录的方式发现节点。
+- etcd: 使用 etcd 发现节点。
+- k8s: 使用 Kubernetes API 发现节点。""" + +cluster_discovery_strategy.label: +"""集群服务发现策略""" + +rpc_send_timeout.desc: +"""发送 RPC 请求的超时时间。""" + +rpc_send_timeout.label: +"""RPC 发送超时时间""" + +common_handler_single_line.desc: +"""如果设置为 true，则单行打印日志。 否则，日志消息可能跨越多行。""" + +common_handler_single_line.label: +"""单行模式""" + +rpc_socket_buffer.desc: +"""TCP 调节参数。用户模式套接字缓冲区大小。""" + +rpc_socket_buffer.label: +"""RPC 套接字缓冲区大小""" + +db_shard_transports.desc: +"""允许为每个 shard 下的事务日志复制操作的传输方法进行调优。
+gen_rpc 使用 gen_rpc 库， +distr 使用 Erlang 自带的 rpc 库。
如果未指定， +默认是使用 db.default_shard_transport 中设置的值。""" + +db_shard_transports.label: +"""事务日志传输协议""" + +} diff --git a/rel/i18n/zh/emqx_connector_api.hocon b/rel/i18n/zh/emqx_connector_api.hocon new file mode 100644 index 000000000..ab651c102 --- /dev/null +++ b/rel/i18n/zh/emqx_connector_api.hocon @@ -0,0 +1,46 @@ +emqx_connector_api { + +conn_get.desc: +"""列出所有连接器""" + +conn_get.label: +"""列出所有连接器""" + +conn_id_delete.desc: +"""通过 ID 删除一个连接器""" + +conn_id_delete.label: +"""删除连接器""" + +conn_id_get.desc: +"""通过 ID 获取连接器""" + +conn_id_get.label: +"""获取连接器""" + +conn_id_put.desc: +"""通过 ID 更新一个连接器""" + +conn_id_put.label: +"""更新连接器""" + +conn_post.desc: +"""创建一个新的连接器""" + +conn_post.label: +"""创建连接器""" + +conn_test_post.desc: +"""通过给定的 ID 测试创建一个新的连接器
+ID 的格式必须为“{type}:{name}”""" + +conn_test_post.label: +"""创建测试连接器""" + +id.desc: +"""连接器 ID， 格式必须为 {type}:{name}""" + +id.label: +"""连接器 ID""" + +} diff --git a/rel/i18n/zh/emqx_connector_http.hocon b/rel/i18n/zh/emqx_connector_http.hocon new file mode 100644 index 000000000..5d6398b2e --- /dev/null +++ b/rel/i18n/zh/emqx_connector_http.hocon @@ -0,0 +1,77 @@ +emqx_connector_http { + +base_url.desc: +"""base URL 只包含host和port。
+发送HTTP请求时，真实的URL是由base URL 和 path parameter连接而成。
+示例：`http://localhost:9901/`""" + +base_url.label: +"""Base Url""" + +body.desc: +"""HTTP请求报文主体。""" + +body.label: +"""HTTP请求报文主体""" + +connect_timeout.desc: +"""连接HTTP服务器的超时时间。""" + +connect_timeout.label: +"""连接超时""" + +enable_pipelining.desc: +"""正整数，设置最大可发送的异步 HTTP 请求数量。当设置为 1 时，表示每次发送完成 HTTP 请求后都需要等待服务器返回，再继续发送下一个请求。""" + +enable_pipelining.label: +"""HTTP 管道""" + +headers.desc: +"""HTTP 头字段列表。""" + +headers.label: +"""HTTP 头字段列表""" + +max_retries.desc: +"""请求出错时的最大重试次数。""" + +max_retries.label: +"""最大重试次数""" + +method.desc: +"""HTTP 请求方法。""" + +method.label: +"""HTTP 请求方法""" + +path.desc: +"""HTTP请求路径。""" + +path.label: +"""HTTP请求路径""" + +pool_size.desc: +"""连接池大小。""" + +pool_size.label: +"""连接池大小""" + +pool_type.desc: +"""连接池的类型，可用类型有`random`, `hash`。""" + +pool_type.label: +"""连接池类型""" + +request.desc: +"""设置 HTTP 请求的参数。""" + +request.label: +"""HTTP 请求""" + +request_timeout.desc: +"""HTTP 请求超时。""" + +request_timeout.label: +"""HTTP 请求超时""" + +} diff --git a/rel/i18n/zh/emqx_connector_ldap.hocon b/rel/i18n/zh/emqx_connector_ldap.hocon new file mode 100644 index 000000000..6826af6e6 --- /dev/null +++ b/rel/i18n/zh/emqx_connector_ldap.hocon @@ -0,0 +1,21 @@ +emqx_connector_ldap { + +bind_dn.desc: +"""LDAP 绑定的 DN 的值""" + +bind_dn.label: +"""Bind DN""" + +port.desc: +"""LDAP 端口""" + +port.label: +"""端口""" + +timeout.desc: +"""LDAP 查询超时时间""" + +timeout.label: +"""超时时间""" + +} diff --git a/rel/i18n/zh/emqx_connector_mongo.hocon b/rel/i18n/zh/emqx_connector_mongo.hocon new file mode 100644 index 000000000..cd84a242e --- /dev/null +++ b/rel/i18n/zh/emqx_connector_mongo.hocon @@ -0,0 +1,152 @@ +emqx_connector_mongo { + +auth_source.desc: +"""与用户证书关联的数据库名称。""" + +auth_source.label: +"""认证源""" + +connect_timeout.desc: +"""超时重连的等待时间。""" + +connect_timeout.label: +"""连接超时""" + +desc_rs.desc: +"""配置 Replica Set""" + +desc_rs.label: +"""配置 Replica Set""" + +desc_sharded.desc: +"""配置 Sharded Cluster""" + +desc_sharded.label: +"""配置 Sharded Cluster""" + +desc_single.desc: +"""配置 Single 模式""" + +desc_single.label: +"""配置 Single 模式""" + +desc_topology.desc: +"""配置 Topology""" + +desc_topology.label: +"""配置 Topology""" + +heartbeat_period.desc: +"""控制驱动程序何时检查MongoDB部署的状态。指定检查的间隔时间，从上一次检查结束到下一次检查开始计算。如果连接数增加（例如，如果你增加池子的大小，就会发生这种情况），你可能也需要增加这个周期，以避免在MongoDB日志文件中创建太多的日志条目。""" + +heartbeat_period.label: +"""心跳期""" + +local_threshold.desc: +"""在多个合适的MongoDB实例中进行选择的延迟窗口的大小。""" + +local_threshold.label: +"""本地阈值""" + +max_overflow.desc: +"""最大溢出。""" + +max_overflow.label: +"""最大溢出""" + +min_heartbeat_period.desc: +"""心跳间的最小间隙""" + +min_heartbeat_period.label: +"""最小心跳周期""" + +overflow_check_period.desc: +"""检查是否有超过配置的工人的周期（"溢出"）。""" + +overflow_check_period.label: +"""溢出检查周期""" + +overflow_ttl.desc: +"""当池内工人太多时，等待多久清除多余工人。""" + +overflow_ttl.label: +"""溢出TTL""" + +r_mode.desc: +"""读模式。""" + +r_mode.label: +"""读模式""" + +replica_set_name.desc: +"""副本集的名称。""" + +replica_set_name.label: +"""副本集名称""" + +rs_mongo_type.desc: +"""Replica set模式。当 MongoDB 服务运行在 replica-set 模式下，该配置必须设置为 'rs'。""" + +rs_mongo_type.label: +"""Replica set 模式""" + +server.desc: +"""将要连接的 IPv4 或 IPv6 地址，或者主机名。
+主机名具有以下形式：`Host[:Port]`。
+如果未指定 `[:Port]`，则使用 MongoDB 默认端口 27017。""" + +server.label: +"""服务器地址""" + +server_selection_timeout.desc: +"""指定在抛出异常之前为服务器选择阻断多长时间。""" + +server_selection_timeout.label: +"""服务器选择超时""" + +servers.desc: +"""集群将要连接的节点列表。 节点之间用逗号分隔，如：`Node[,Node].` +每个节点的配置为：将要连接的 IPv4 或 IPv6 地址或主机名。 +主机名具有以下形式：`Host[:Port]`。 +如果未指定 `[:Port]`，则使用 MongoDB 默认端口 27017。""" + +servers.label: +"""服务器列表""" + +sharded_mongo_type.desc: +"""Sharded cluster模式。当 MongoDB 服务运行在 sharded 模式下，该配置必须设置为 'sharded'。""" + +sharded_mongo_type.label: +"""Sharded cluster 模式""" + +single_mongo_type.desc: +"""Standalone 模式。当 MongoDB 服务运行在 standalone 模式下，该配置必须设置为 'single'。""" + +single_mongo_type.label: +"""Standalone 模式""" + +socket_timeout.desc: +"""在尝试超时之前，在套接字上尝试发送或接收的持续时间。""" + +socket_timeout.label: +"""套接字操作超时""" + +srv_record.desc: +"""使用 DNS SRV 记录。""" + +srv_record.label: +"""SRV 记录""" + +w_mode.desc: +"""写模式。""" + +w_mode.label: +"""写模式""" + +wait_queue_timeout.desc: +"""工作者等待连接可用的最长时间。""" + +wait_queue_timeout.label: +"""等待队列超时""" + +} diff --git a/rel/i18n/zh/emqx_connector_mqtt.hocon b/rel/i18n/zh/emqx_connector_mqtt.hocon new file mode 100644 index 000000000..1df6a89b6 --- /dev/null +++ b/rel/i18n/zh/emqx_connector_mqtt.hocon @@ -0,0 +1,21 @@ +emqx_connector_mqtt { + +name.desc: +"""连接器名称，人类可读的连接器描述。""" + +name.label: +"""连接器名称""" + +num_of_bridges.desc: +"""当前使用此连接器的网桥数量。""" + +num_of_bridges.label: +"""网桥数量""" + +type.desc: +"""连接器类型。""" + +type.label: +"""连接器类型""" + +} diff --git a/rel/i18n/zh/emqx_connector_mqtt_schema.hocon b/rel/i18n/zh/emqx_connector_mqtt_schema.hocon new file mode 100644 index 000000000..66ba2accf --- /dev/null +++ b/rel/i18n/zh/emqx_connector_mqtt_schema.hocon @@ -0,0 +1,170 @@ +emqx_connector_mqtt_schema { + +bridge_mode.desc: +"""是否启用 Bridge Mode。 +注意：此设置只针对 MQTT 协议版本 < 5.0 有效，并且需要远程 MQTT Broker 支持 Bridge Mode。 +如果设置为 true ，桥接会告诉远端服务器当前连接是一个桥接而不是一个普通的客户端。 +这意味着消息回环检测会更加高效，并且远端服务器收到的保留消息的标志位会透传给本地。""" + +bridge_mode.label: +"""Bridge 模式""" + +clean_start.desc: +"""与 ingress MQTT 桥的远程服务器重连时是否清除老的 MQTT 会话。""" + +clean_start.label: +"""清除会话""" + +clientid_prefix.desc: +"""可选的前缀，用于在出口网桥使用的clientid前加上前缀。""" + +clientid_prefix.label: +"""客户ID前缀""" + +egress_desc.desc: +"""出口配置定义了该桥接如何将消息从本地 Broker 转发到远程 Broker。 +以下字段中允许使用带有变量的模板：'remote.topic', 'local.qos', 'local.retain', 'local.payload'。
+注意：如果此桥接被用作规则的动作，并且配置了 'local.topic'，则从规则输出的数据以及匹配到 'local.topic' 的 MQTT 消息都会被转发。""" + +egress_desc.label: +"""出方向配置""" + +egress_local.desc: +"""如何从本地 Broker 接收消息相关的配置。""" + +egress_local.label: +"""本地配置""" + +egress_local_topic.desc: +"""要转发到远程broker的本地主题""" + +egress_local_topic.label: +"""本地主题""" + +egress_remote.desc: +"""发送消息到远程 Broker 相关的配置。""" + +egress_remote.label: +"""远程配置""" + +egress_remote_qos.desc: +"""待发送 MQTT 消息的 QoS。
+允许使用带有变量的模板。""" + +egress_remote_qos.label: +"""远程 QoS""" + +egress_remote_topic.desc: +"""转发到远程broker的哪个topic。
+允许使用带有变量的模板。""" + +egress_remote_topic.label: +"""远程主题""" + +ingress_desc.desc: +"""入口配置定义了该桥接如何从远程 MQTT Broker 接收消息，然后将消息发送到本地 Broker。
+ 以下字段中允许使用带有变量的模板：'remote.qos', 'local.topic', 'local.qos', 'local.retain', 'local.payload'。
+ 注意：如果此桥接被用作规则的输入，并且配置了 'local.topic'，则从远程代理获取的消息将同时被发送到 'local.topic' 和规则。""" + +ingress_desc.label: +"""入方向配置""" + +ingress_local.desc: +"""发送消息到本地 Broker 相关的配置。""" + +ingress_local.label: +"""本地配置""" + +ingress_local_qos.desc: +"""待发送 MQTT 消息的 QoS。
+允许使用带有变量的模板。""" + +ingress_local_qos.label: +"""本地 QoS""" + +ingress_local_topic.desc: +"""向本地broker的哪个topic发送消息。
+允许使用带有变量的模板。""" + +ingress_local_topic.label: +"""本地主题""" + +ingress_remote.desc: +"""订阅远程 Broker 相关的配置。""" + +ingress_remote.label: +"""远程配置""" + +ingress_remote_qos.desc: +"""订阅远程borker时要使用的 QoS 级别""" + +ingress_remote_qos.label: +"""远程 QoS""" + +ingress_remote_topic.desc: +"""从远程broker的哪个topic接收消息""" + +ingress_remote_topic.label: +"""远程主题""" + +max_inflight.desc: +"""MQTT 协议的最大飞行（已发送但未确认）消息""" + +max_inflight.label: +"""最大飞行消息""" + +mode.desc: +"""MQTT 桥的模式。
+- cluster_shareload：在 emqx 集群的每个节点上创建一个 MQTT 连接。
+在“cluster_shareload”模式下，来自远程代理的传入负载通过共享订阅的方式接收。
+请注意，clientid 以节点名称为后缀，这是为了避免不同节点之间的 clientid 冲突。 +而且对于入口连接的 remote.topic，我们只能使用共享订阅主题过滤器。""" + +mode.label: +"""MQTT 桥接模式""" + +password.desc: +"""MQTT 协议的密码""" + +password.label: +"""密码""" + +payload.desc: +"""要发送的 MQTT 消息的负载。
+允许使用带有变量的模板。""" + +payload.label: +"""消息负载""" + +proto_ver.desc: +"""MQTT 协议版本""" + +proto_ver.label: +"""协议版本""" + +retain.desc: +"""要发送的 MQTT 消息的“保留”标志。
+允许使用带有变量的模板。""" + +retain.label: +"""保留消息标志""" + +server.desc: +"""远程 MQTT Broker的主机和端口。""" + +server.label: +"""Broker主机和端口""" + +server_configs.desc: +"""服务器相关的配置。""" + +server_configs.label: +"""服务配置。""" + +username.desc: +"""MQTT 协议的用户名""" + +username.label: +"""用户名""" + +} diff --git a/rel/i18n/zh/emqx_connector_mysql.hocon b/rel/i18n/zh/emqx_connector_mysql.hocon new file mode 100644 index 000000000..a6900056f --- /dev/null +++ b/rel/i18n/zh/emqx_connector_mysql.hocon @@ -0,0 +1,11 @@ +emqx_connector_mysql { + +server.desc: +"""将要连接的 IPv4 或 IPv6 地址，或者主机名。
+主机名具有以下形式：`Host[:Port]`。
+如果未指定 `[:Port]`，则使用 MySQL 默认端口 3306。""" + +server.label: +"""服务器地址""" + +} diff --git a/rel/i18n/zh/emqx_connector_pgsql.hocon b/rel/i18n/zh/emqx_connector_pgsql.hocon new file mode 100644 index 000000000..c391034fd --- /dev/null +++ b/rel/i18n/zh/emqx_connector_pgsql.hocon @@ -0,0 +1,11 @@ +emqx_connector_pgsql { + +server.desc: +"""将要连接的 IPv4 或 IPv6 地址，或者主机名。
+主机名具有以下形式：`Host[:Port]`。
+如果未指定 `[:Port]`，则使用 PostgreSQL 默认端口 5432。""" + +server.label: +"""服务器地址""" + +} diff --git a/rel/i18n/zh/emqx_connector_redis.hocon b/rel/i18n/zh/emqx_connector_redis.hocon new file mode 100644 index 000000000..a0819876e --- /dev/null +++ b/rel/i18n/zh/emqx_connector_redis.hocon @@ -0,0 +1,50 @@ +emqx_connector_redis { + +cluster.desc: +"""集群模式。当 Redis 服务运行在集群模式下，该配置必须设置为 'cluster'。""" + +cluster.label: +"""集群模式""" + +database.desc: +"""Redis 数据库 ID。""" + +database.label: +"""数据库 ID""" + +sentinel.desc: +"""哨兵模式。当 Redis 服务运行在哨兵模式下，该配置必须设置为 'sentinel'。""" + +sentinel.label: +"""哨兵模式""" + +sentinel_desc.desc: +"""Redis 哨兵模式下的集群名称。""" + +sentinel_desc.label: +"""集群名称""" + +server.desc: +"""将要连接的 IPv4 或 IPv6 地址，或者主机名。
+主机名具有以下形式：`Host[:Port]`。
+如果未指定 `[:Port]`，则使用 Redis 默认端口 6379。""" + +server.label: +"""服务器地址""" + +servers.desc: +"""集群将要连接的节点列表。 节点之间用逗号分隔，如：`Node[,Node].` +每个节点的配置为：将要连接的 IPv4 或 IPv6 地址或主机名。 +主机名具有以下形式：`Host[:Port]`。 +如果未指定 `[:Port]`，则使用 Redis 默认端口 6379。""" + +servers.label: +"""服务器列表""" + +single.desc: +"""单机模式。当 Redis 服务运行在单机模式下，该配置必须设置为 'single'。""" + +single.label: +"""单机模式""" + +} diff --git a/rel/i18n/zh/emqx_connector_schema_lib.hocon b/rel/i18n/zh/emqx_connector_schema_lib.hocon new file mode 100644 index 000000000..318b5c31c --- /dev/null +++ b/rel/i18n/zh/emqx_connector_schema_lib.hocon @@ -0,0 +1,45 @@ +emqx_connector_schema_lib { + +auto_reconnect.desc: +"""已弃用。自动重连数据库。""" + +auto_reconnect.label: +"""已弃用。自动重连数据库""" + +database_desc.desc: +"""数据库名字。""" + +database_desc.label: +"""数据库名字""" + +password.desc: +"""内部数据库密码。""" + +password.label: +"""密码""" + +pool_size.desc: +"""桥接远端服务时使用的连接池大小。""" + +pool_size.label: +"""连接池大小""" + +prepare_statement.desc: +"""SQL 预处理语句列表。""" + +prepare_statement.label: +"""SQL 预处理语句列表""" + +ssl.desc: +"""启用 SSL 连接。""" + +ssl.label: +"""启用SSL""" + +username.desc: +"""内部数据库的用户名。""" + +username.label: +"""用户名""" + +} diff --git a/rel/i18n/zh/emqx_dashboard_api.hocon b/rel/i18n/zh/emqx_dashboard_api.hocon new file mode 100644 index 000000000..8292d8bba --- /dev/null +++ b/rel/i18n/zh/emqx_dashboard_api.hocon @@ -0,0 +1,66 @@ +emqx_dashboard_api { + +change_pwd_api.desc: +"""更改 Dashboard 用户密码""" + +create_user_api.desc: +"""创建 Dashboard 用户""" + +create_user_api_success.desc: +"""创建 Dashboard 用户成功""" + +delete_user_api.desc: +"""删除 Dashboard 用户""" + +license.desc: +"""EMQX 许可类型。可为 opensource 或 enterprise""" + +list_users_api.desc: +"""Dashboard 用户列表""" + +login_api.desc: +"""获取 Dashboard 认证 Token。""" + +login_failed401.desc: +"""登录失败。用户名或密码错误""" + +login_failed_response400.desc: +"""登录失败。用户名或密码错误""" + +login_success.desc: +"""Dashboard 认证成功""" + +logout_api.desc: +"""Dashboard 用户登出""" + +new_pwd.desc: +"""新密码""" + +old_pwd.desc: +"""旧密码""" + +password.desc: +"""Dashboard 密码""" + +token.desc: +"""Dashboard 认证 Token""" + +update_user_api.desc: +"""更新 Dashboard 用户描述""" + +update_user_api200.desc: +"""更新 Dashboard 用户成功""" + +user_description.desc: +"""Dashboard 用户描述""" + +username.desc: +"""Dashboard 用户名""" + +users_api404.desc: +"""Dashboard 用户不存在""" + +version.desc: +"""EMQX 版本""" + +} diff --git a/rel/i18n/zh/emqx_dashboard_schema.hocon b/rel/i18n/zh/emqx_dashboard_schema.hocon new file mode 100644 index 000000000..a07f3e6d8 --- /dev/null +++ b/rel/i18n/zh/emqx_dashboard_schema.hocon @@ -0,0 +1,130 @@ +emqx_dashboard_schema { + +backlog.desc: +"""排队等待连接的队列的最大长度。""" + +backlog.label: +"""排队长度""" + +bind.desc: +"""监听地址和端口，热更新此配置时，会重启 Dashboard 服务。""" + +bind.label: +"""绑定端口""" + +bootstrap_users_file.desc: +"""已废弃，请使用 api_key.bootstrap_file。""" + +bootstrap_users_file.label: +"""已废弃""" + +cors.desc: +"""支持跨域资源共享(CORS)， +允许服务器指示任何来源(域名、协议或端口)，除了本服务器之外的任何浏览器应允许加载资源。""" + +cors.label: +"""跨域资源共享""" + +default_password.desc: +"""Dashboard 的默认密码，为了安全，应该尽快修改密码。 +当通过网页首次登录 Dashboard 并按提示修改成复杂密码后，此值就会失效。""" + +default_password.label: +"""默认密码""" + +default_username.desc: +"""Dashboard 的默认用户名。""" + +default_username.label: +"""默认用户名""" + +desc_dashboard.desc: +"""EMQX Dashboard 配置。""" + +desc_dashboard.label: +"""Dashboard""" + +desc_http.desc: +"""Dashboard 监听器(HTTP)配置。""" + +desc_http.label: +"""HTTP""" + +desc_https.desc: +"""Dashboard 监听器(HTTPS)配置。""" + +desc_https.label: +"""HTTPS""" + +desc_listeners.desc: +"""Dashboard 监听器配置。""" + +desc_listeners.label: +"""监听器""" + +i18n_lang.desc: +"""设置 Swagger 多语言的版本，可为 en 或 zh。""" + +i18n_lang.label: +"""多语言支持""" + +inet6.desc: +"""启用IPv6， 如果机器不支持IPv6，请关闭此选项，否则会导致 Dashboard 无法使用。""" + +inet6.label: +"""IPv6""" + +ipv6_v6only.desc: +"""当开启 inet6 功能的同时禁用 IPv4-to-IPv6 映射。该配置仅在 inet6 功能开启时有效。""" + +ipv6_v6only.label: +"""IPv6 only""" + +listener_enable.desc: +"""忽略或启用该监听器。""" + +listener_enable.label: +"""启用""" + +listeners.desc: +"""Dashboard 监听器设置。监听器必须有唯一的端口号和IP地址的组合。 +例如，可以通过指定IP地址 0.0.0.0 来监听机器上给定端口上的所有配置的IP地址。 +或者，可以为每个监听器指定唯一的IP地址，但使用相同的端口。""" + +listeners.label: +"""监听器""" + +max_connections.desc: +"""同时处理的最大连接数。""" + +max_connections.label: +"""最大连接数""" + +num_acceptors.desc: +"""TCP协议的Socket acceptor池大小, 默认设置在线的调度器数量（通常为 CPU 核数）""" + +num_acceptors.label: +"""Acceptor 数量""" + +proxy_header.desc: +"""开启对 `HAProxy` 的支持，注意：一旦开启了这个功能，就无法再处理普通的 HTTP 请求了。""" + +proxy_header.label: +"""开启对 `HAProxy` 的支持""" + +sample_interval.desc: +"""Dashboard 中图表指标的时间间隔。必须小于60，且被60的整除，默认设置 10s。""" + +send_timeout.desc: +"""Socket发送超时时间。""" + +send_timeout.label: +"""发送超时时间""" + +token_expired_time.desc: +"""JWT token 过期时间。默认设置为 60 分钟。""" + +token_expired_time.label: +"""JWT 过期时间""" + +} diff --git a/rel/i18n/zh/emqx_delayed_api.hocon b/rel/i18n/zh/emqx_delayed_api.hocon new file mode 100644 index 000000000..d783a17c3 --- /dev/null +++ b/rel/i18n/zh/emqx_delayed_api.hocon @@ -0,0 +1,72 @@ +emqx_delayed_api { + +bad_msgid_format.desc: +"""消息 ID 格式错误""" + +count.desc: +"""延迟消息总数""" + +delayed_interval.desc: +"""延迟时间(秒)""" + +delayed_remaining.desc: +"""剩余时间(秒)""" + +delete_api.desc: +"""删除延迟消息""" + +expected_at.desc: +"""期望的发送时间, RFC 3339 格式""" + +from_clientid.desc: +"""消息的 ClientID""" + +from_username.desc: +"""消息的 Username""" + +get_message_api.desc: +"""查看延迟消息""" + +illegality_limit.desc: +"""数量上限不合法""" + +list_api.desc: +"""查看延迟消息列表""" + +msgid.desc: +"""延迟消息 ID""" + +msgid_not_found.desc: +"""未找到对应消息""" + +node.desc: +"""消息的来源节点""" + +payload.desc: +"""消息内容, base64 格式。如果消息的大小超过 2048 字节，则消息内容会被设置为 'PAYLOAD_TO_LARGE'""" + +publish_at.desc: +"""客户端发送时间, RFC 3339 格式""" + +qos.desc: +"""QoS""" + +topic.desc: +"""主题""" + +update_api.desc: +"""开启或者关闭功能，或者设置延迟消息数量上限""" + +update_success.desc: +"""开启或者关闭功能操作成功""" + +view_limit.desc: +"""每页数量""" + +view_page.desc: +"""查看的页数""" + +view_status_api.desc: +"""查看慢订阅状态""" + +} diff --git a/rel/i18n/zh/emqx_ee_bridge_cassa.hocon b/rel/i18n/zh/emqx_ee_bridge_cassa.hocon new file mode 100644 index 000000000..2d1125a6b --- /dev/null +++ b/rel/i18n/zh/emqx_ee_bridge_cassa.hocon @@ -0,0 +1,40 @@ +emqx_ee_bridge_cassa { + +config_enable.desc: +"""启用/禁用桥接""" + +config_enable.label: +"""启用/禁用桥接""" + +cql_template.desc: +"""CQL 模板""" + +cql_template.label: +"""CQL 模板""" + +desc_config.desc: +"""Cassandra 桥接配置""" + +desc_config.label: +"""Cassandra 桥接配置""" + +desc_name.desc: +"""桥接名字""" + +desc_name.label: +"""桥接名字""" + +desc_type.desc: +"""Bridge 类型""" + +desc_type.label: +"""桥接类型""" + +local_topic.desc: +"""发送到 'local_topic' 的消息都会转发到 Cassandra。
+注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。""" + +local_topic.label: +"""本地 Topic""" + +} diff --git a/rel/i18n/zh/emqx_ee_bridge_clickhouse.hocon b/rel/i18n/zh/emqx_ee_bridge_clickhouse.hocon new file mode 100644 index 000000000..a3ede08ba --- /dev/null +++ b/rel/i18n/zh/emqx_ee_bridge_clickhouse.hocon @@ -0,0 +1,46 @@ +emqx_ee_bridge_clickhouse { + +batch_value_separator.desc: +"""默认为逗号 ','，适用于 VALUE 格式。您也可以使用其他分隔符， 请参考 [INSERT INTO 语句](https://clickhouse.com/docs/en/sql-reference/statements/insert-into)。""" + +batch_value_separator.label: +"""分隔符""" + +config_enable.desc: +"""启用/禁用桥接""" + +config_enable.label: +"""启用/禁用桥接""" + +desc_config.desc: +"""Clickhouse 桥接配置""" + +desc_config.label: +"""Clickhouse 桥接配置""" + +desc_name.desc: +"""桥接名字""" + +desc_name.label: +"""桥接名字""" + +desc_type.desc: +"""Bridge 类型""" + +desc_type.label: +"""桥接类型""" + +local_topic.desc: +"""发送到 'local_topic' 的消息都会转发到 Clickhouse。
+注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。""" + +local_topic.label: +"""本地 Topic""" + +sql_template.desc: +"""可以使用 ${field} 占位符来引用消息与客户端上下文中的变量，请确保对应字段存在且数据格式符合预期。此处不支持 [SQL 预处理](https://docs.emqx.com/zh/enterprise/v5.0/data-integration/data-bridges.html#sql-预处理)。""" + +sql_template.label: +"""SQL 模板""" + +} diff --git a/rel/i18n/zh/emqx_ee_bridge_dynamo.hocon b/rel/i18n/zh/emqx_ee_bridge_dynamo.hocon new file mode 100644 index 000000000..adf33b9e8 --- /dev/null +++ b/rel/i18n/zh/emqx_ee_bridge_dynamo.hocon @@ -0,0 +1,40 @@ +emqx_ee_bridge_dynamo { + +config_enable.desc: +"""启用/禁用桥接""" + +config_enable.label: +"""启用/禁用桥接""" + +desc_config.desc: +"""DynamoDB 桥接配置""" + +desc_config.label: +"""DynamoDB 桥接配置""" + +desc_name.desc: +"""桥接名字""" + +desc_name.label: +"""桥接名字""" + +desc_type.desc: +"""Bridge 类型""" + +desc_type.label: +"""桥接类型""" + +local_topic.desc: +"""发送到 'local_topic' 的消息都会转发到 DynamoDB。
+注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。""" + +local_topic.label: +"""本地 Topic""" + +template.desc: +"""模板, 默认为空，为空时将会将整个消息存入数据库""" + +template.label: +"""模板""" + +} diff --git a/rel/i18n/zh/emqx_ee_bridge_hstreamdb.hocon b/rel/i18n/zh/emqx_ee_bridge_hstreamdb.hocon new file mode 100644 index 000000000..55cdebe3c --- /dev/null +++ b/rel/i18n/zh/emqx_ee_bridge_hstreamdb.hocon @@ -0,0 +1,52 @@ +emqx_ee_bridge_hstreamdb { + +config_direction.desc: +"""桥接的方向， 必须是 egress""" + +config_direction.label: +"""桥接方向""" + +config_enable.desc: +"""启用/禁用桥接""" + +config_enable.label: +"""启用/禁用桥接""" + +desc_config.desc: +"""HStreamDB 桥接配置""" + +desc_config.label: +"""HStreamDB 桥接配置""" + +desc_connector.desc: +"""连接器的通用配置。""" + +desc_connector.label: +"""连接器通用配置。""" + +desc_name.desc: +"""桥接名字，可读描述""" + +desc_name.label: +"""桥接名字""" + +desc_type.desc: +"""Bridge 类型""" + +desc_type.label: +"""桥接类型""" + +local_topic.desc: +"""发送到 'local_topic' 的消息都会转发到 HStreamDB。
+注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发到 HStreamDB。""" + +local_topic.label: +"""本地 Topic""" + +payload.desc: +"""要转发到 HStreamDB 的数据内容，支持占位符""" + +payload.label: +"""消息内容""" + +} diff --git a/rel/i18n/zh/emqx_ee_bridge_influxdb.hocon b/rel/i18n/zh/emqx_ee_bridge_influxdb.hocon new file mode 100644 index 000000000..c9c7c6a54 --- /dev/null +++ b/rel/i18n/zh/emqx_ee_bridge_influxdb.hocon @@ -0,0 +1,47 @@ +emqx_ee_bridge_influxdb { + +config_enable.desc: +"""启用/禁用桥接。""" + +config_enable.label: +"""启用/禁用桥接""" + +desc_config.desc: +"""InfluxDB 桥接配置。""" + +desc_config.label: +"""InfluxDB 桥接配置""" + +desc_name.desc: +"""桥接名称。""" + +desc_name.label: +"""桥接名称""" + +desc_type.desc: +"""桥接类型。""" + +desc_type.label: +"""桥接类型""" + +local_topic.desc: +"""发送到 'local_topic' 的消息都会转发到 InfluxDB。
+注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发到 InfluxDB。""" + +local_topic.label: +"""本地 Topic""" + +write_syntax.desc: +"""使用 InfluxDB API Line Protocol 写入 InfluxDB 的数据，支持占位符
+参考 [InfluxDB 2.3 Line Protocol](https://docs.influxdata.com/influxdb/v2.3/reference/syntax/line-protocol/) 及 +[InfluxDB 1.8 Line Protocol](https://docs.influxdata.com/influxdb/v1.8/write_protocols/line_protocol_tutorial/)
+TLDR:
+``` +[,=[,=]] =[,=] [] +``` +注意，整形数值占位符后需要添加一个字符 `i` 类型标识。例如 `${payload.int_value}i`""" + +write_syntax.label: +"""写语句""" + +} diff --git a/rel/i18n/zh/emqx_ee_bridge_mongodb.hocon b/rel/i18n/zh/emqx_ee_bridge_mongodb.hocon new file mode 100644 index 000000000..71703c662 --- /dev/null +++ b/rel/i18n/zh/emqx_ee_bridge_mongodb.hocon @@ -0,0 +1,57 @@ +emqx_ee_bridge_mongodb { + +collection.desc: +"""数据将被存储到的集合""" + +collection.label: +"""将要使用的集合（Collection）""" + +desc_config.desc: +"""为MongoDB桥配置""" + +desc_config.label: +"""MongoDB桥配置""" + +desc_name.desc: +"""桥接名称。""" + +desc_name.label: +"""桥接名称""" + +desc_type.desc: +"""桥接类型。""" + +desc_type.label: +"""桥接类型""" + +enable.desc: +"""启用或停用该MongoDB桥""" + +enable.label: +"""启用或禁用""" + +mongodb_rs_conf.desc: +"""MongoDB（Replica Set）配置""" + +mongodb_rs_conf.label: +"""MongoDB（Replica Set）配置""" + +mongodb_sharded_conf.desc: +"""MongoDB (Sharded)配置""" + +mongodb_sharded_conf.label: +"""MongoDB (Sharded)配置""" + +mongodb_single_conf.desc: +"""MongoDB（独立）配置""" + +mongodb_single_conf.label: +"""MongoDB（独立）配置""" + +payload_template.desc: +"""用于格式化写入 MongoDB 的消息模板。 如果未定义，规则引擎会使用 JSON 格式序列化所有的可见输入，例如 clientid, topic, payload 等。""" + +payload_template.label: +"""有效载荷模板""" + +} diff --git a/rel/i18n/zh/emqx_ee_bridge_mysql.hocon b/rel/i18n/zh/emqx_ee_bridge_mysql.hocon new file mode 100644 index 000000000..e03b147b0 --- /dev/null +++ b/rel/i18n/zh/emqx_ee_bridge_mysql.hocon @@ -0,0 +1,40 @@ +emqx_ee_bridge_mysql { + +config_enable.desc: +"""启用/禁用桥接""" + +config_enable.label: +"""启用/禁用桥接""" + +desc_config.desc: +"""HStreamDB 桥接配置""" + +desc_config.label: +"""HStreamDB 桥接配置""" + +desc_name.desc: +"""桥接名字，可读描述""" + +desc_name.label: +"""桥接名字""" + +desc_type.desc: +"""Bridge 类型""" + +desc_type.label: +"""桥接类型""" + +local_topic.desc: +"""发送到 'local_topic' 的消息都会转发到 MySQL。
+注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。""" + +local_topic.label: +"""本地 Topic""" + +sql_template.desc: +"""SQL 模板""" + +sql_template.label: +"""SQL 模板""" + +} diff --git a/rel/i18n/zh/emqx_ee_bridge_pgsql.hocon b/rel/i18n/zh/emqx_ee_bridge_pgsql.hocon new file mode 100644 index 000000000..ebf7f331a --- /dev/null +++ b/rel/i18n/zh/emqx_ee_bridge_pgsql.hocon @@ -0,0 +1,40 @@ +emqx_ee_bridge_pgsql { + +config_enable.desc: +"""启用/禁用桥接""" + +config_enable.label: +"""启用/禁用桥接""" + +desc_config.desc: +"""PostgreSQL 桥接配置""" + +desc_config.label: +"""PostgreSQL 桥接配置""" + +desc_name.desc: +"""桥接名字""" + +desc_name.label: +"""桥接名字""" + +desc_type.desc: +"""Bridge 类型""" + +desc_type.label: +"""桥接类型""" + +local_topic.desc: +"""发送到 'local_topic' 的消息都会转发到 PostgreSQL。
+注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。""" + +local_topic.label: +"""本地 Topic""" + +sql_template.desc: +"""SQL 模板""" + +sql_template.label: +"""SQL 模板""" + +} diff --git a/rel/i18n/zh/emqx_ee_bridge_redis.hocon b/rel/i18n/zh/emqx_ee_bridge_redis.hocon new file mode 100644 index 000000000..378df508f --- /dev/null +++ b/rel/i18n/zh/emqx_ee_bridge_redis.hocon @@ -0,0 +1,41 @@ +emqx_ee_bridge_redis { + +command_template.desc: +"""用于推送数据的 Redis 命令模板。 每个列表元素代表一个命令名称或其参数。 +例如，要通过键值 `msgs` 将消息体推送到 Redis 列表中，数组元素应该是： `rpush`, `msgs`, `${payload}`。""" + +command_template.label: +"""Redis Command 模板""" + +config_enable.desc: +"""启用/禁用桥接""" + +config_enable.label: +"""启用/禁用桥接""" + +desc_config.desc: +"""Resis 桥接配置""" + +desc_config.label: +"""Redis 桥接配置""" + +desc_name.desc: +"""桥接名字，可读描述""" + +desc_name.label: +"""桥接名字""" + +desc_type.desc: +"""Bridge 类型""" + +desc_type.label: +"""桥接类型""" + +local_topic.desc: +"""发送到 'local_topic' 的消息都会转发到 Redis。
+注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发到 Redis。""" + +local_topic.label: +"""本地 Topic""" + +} diff --git a/rel/i18n/zh/emqx_ee_bridge_rocketmq.hocon b/rel/i18n/zh/emqx_ee_bridge_rocketmq.hocon new file mode 100644 index 000000000..924004361 --- /dev/null +++ b/rel/i18n/zh/emqx_ee_bridge_rocketmq.hocon @@ -0,0 +1,40 @@ +emqx_ee_bridge_rocketmq { + +config_enable.desc: +"""启用/禁用桥接""" + +config_enable.label: +"""启用/禁用桥接""" + +desc_config.desc: +"""RocketMQ 桥接配置""" + +desc_config.label: +"""RocketMQ 桥接配置""" + +desc_name.desc: +"""桥接名字""" + +desc_name.label: +"""桥接名字""" + +desc_type.desc: +"""Bridge 类型""" + +desc_type.label: +"""桥接类型""" + +local_topic.desc: +"""发送到 'local_topic' 的消息都会转发到 RocketMQ。
+注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。""" + +local_topic.label: +"""本地 Topic""" + +template.desc: +"""模板, 默认为空，为空时将会将整个消息转发给 RocketMQ""" + +template.label: +"""模板""" + +} diff --git a/rel/i18n/zh/emqx_ee_bridge_sqlserver.hocon b/rel/i18n/zh/emqx_ee_bridge_sqlserver.hocon new file mode 100644 index 000000000..0958d4b7a --- /dev/null +++ b/rel/i18n/zh/emqx_ee_bridge_sqlserver.hocon @@ -0,0 +1,46 @@ +emqx_ee_bridge_sqlserver { + +config_enable.desc: +"""启用/禁用桥接""" + +config_enable.label: +"""启用/禁用桥接""" + +desc_config.desc: +"""Microsoft SQL Server 桥接配置""" + +desc_config.label: +"""Microsoft SQL Server 桥接配置""" + +desc_name.desc: +"""桥接名字""" + +desc_name.label: +"""桥接名字""" + +desc_type.desc: +"""Bridge 类型""" + +desc_type.label: +"""桥接类型""" + +driver.desc: +"""SQL Server Driver 名称""" + +driver.label: +"""SQL Server Driver 名称""" + +local_topic.desc: +"""发送到 'local_topic' 的消息都会转发到 Microsoft SQL Server。
+注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。""" + +local_topic.label: +"""本地 Topic""" + +sql_template.desc: +"""SQL 模板""" + +sql_template.label: +"""SQL 模板""" + +} diff --git a/rel/i18n/zh/emqx_ee_bridge_tdengine.hocon b/rel/i18n/zh/emqx_ee_bridge_tdengine.hocon new file mode 100644 index 000000000..5e417a1c7 --- /dev/null +++ b/rel/i18n/zh/emqx_ee_bridge_tdengine.hocon @@ -0,0 +1,40 @@ +emqx_ee_bridge_tdengine { + +config_enable.desc: +"""启用/禁用桥接""" + +config_enable.label: +"""启用/禁用桥接""" + +desc_config.desc: +"""TDengine 桥接配置""" + +desc_config.label: +"""TDengine 桥接配置""" + +desc_name.desc: +"""桥接名字""" + +desc_name.label: +"""桥接名字""" + +desc_type.desc: +"""Bridge 类型""" + +desc_type.label: +"""桥接类型""" + +local_topic.desc: +"""发送到 'local_topic' 的消息都会转发到 TDengine。
+注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。""" + +local_topic.label: +"""本地 Topic""" + +sql_template.desc: +"""SQL 模板""" + +sql_template.label: +"""SQL 模板""" + +} diff --git a/rel/i18n/zh/emqx_ee_connector_cassa.hocon b/rel/i18n/zh/emqx_ee_connector_cassa.hocon new file mode 100644 index 000000000..ffbadc7de --- /dev/null +++ b/rel/i18n/zh/emqx_ee_connector_cassa.hocon @@ -0,0 +1,17 @@ +emqx_ee_connector_cassa { + +keyspace.desc: +"""要连接到的 Keyspace 名称。""" + +keyspace.label: +"""Keyspace""" + +servers.desc: +"""将要连接的 IPv4 或 IPv6 地址，或者主机名。
+主机名具有以下形式：`Host[:Port][,Host2:Port]`。
+如果未指定 `[:Port]`，则使用 Cassandra 默认端口 9042。""" + +servers.label: +"""Servers""" + +} diff --git a/rel/i18n/zh/emqx_ee_connector_clickhouse.hocon b/rel/i18n/zh/emqx_ee_connector_clickhouse.hocon new file mode 100644 index 000000000..f1457a1f6 --- /dev/null +++ b/rel/i18n/zh/emqx_ee_connector_clickhouse.hocon @@ -0,0 +1,15 @@ +emqx_ee_connector_clickhouse { + +base_url.desc: +"""你想连接到的Clickhouse服务器的HTTP URL（例如http://myhostname:8123）。""" + +base_url.label: +"""服务器 URL""" + +connect_timeout.desc: +"""连接HTTP服务器的超时时间。""" + +connect_timeout.label: +"""连接超时""" + +} diff --git a/rel/i18n/zh/emqx_ee_connector_dynamo.hocon b/rel/i18n/zh/emqx_ee_connector_dynamo.hocon new file mode 100644 index 000000000..e7b911c1e --- /dev/null +++ b/rel/i18n/zh/emqx_ee_connector_dynamo.hocon @@ -0,0 +1,27 @@ +emqx_ee_connector_dynamo { + +aws_access_key_id.desc: +"""DynamoDB 的访问 ID。""" + +aws_access_key_id.label: +"""连接访问 ID""" + +aws_secret_access_key.desc: +"""DynamoDB 的访问密钥。""" + +aws_secret_access_key.label: +"""连接访问密钥""" + +table.desc: +"""DynamoDB 的表。""" + +table.label: +"""表""" + +url.desc: +"""DynamoDB 的地址。""" + +url.label: +"""DynamoDB 地址""" + +} diff --git a/rel/i18n/zh/emqx_ee_connector_hstreamdb.hocon b/rel/i18n/zh/emqx_ee_connector_hstreamdb.hocon new file mode 100644 index 000000000..d9d8fb3ed --- /dev/null +++ b/rel/i18n/zh/emqx_ee_connector_hstreamdb.hocon @@ -0,0 +1,45 @@ +emqx_ee_connector_hstreamdb { + +config.desc: +"""HStreamDB 连接配置。""" + +config.label: +"""连接配置""" + +name.desc: +"""连接器名称，人类可读的连接器描述。""" + +name.label: +"""连接器名称""" + +ordering_key.desc: +"""HStreamDB 分区键""" + +ordering_key.label: +"""HStreamDB 分区键""" + +pool_size.desc: +"""HStreamDB 连接池大小""" + +pool_size.label: +"""HStreamDB 连接池大小""" + +stream_name.desc: +"""HStreamDB 流名称""" + +stream_name.label: +"""HStreamDB 流名称""" + +type.desc: +"""连接器类型。""" + +type.label: +"""连接器类型""" + +url.desc: +"""HStreamDB 服务器 URL""" + +url.label: +"""HStreamDB 服务器 URL""" + +} diff --git a/rel/i18n/zh/emqx_ee_connector_influxdb.hocon b/rel/i18n/zh/emqx_ee_connector_influxdb.hocon new file mode 100644 index 000000000..6148b400a --- /dev/null +++ b/rel/i18n/zh/emqx_ee_connector_influxdb.hocon @@ -0,0 +1,71 @@ +emqx_ee_connector_influxdb { + +bucket.desc: +"""InfluxDB bucket 名称。""" + +bucket.label: +"""Bucket""" + +database.desc: +"""InfluxDB 数据库。""" + +database.label: +"""数据库""" + +influxdb_api_v1.desc: +"""InfluxDB HTTP API 协议。支持 Influxdb v1.8 以及之前的版本。""" + +influxdb_api_v1.label: +"""HTTP API 协议""" + +influxdb_api_v2.desc: +"""InfluxDB HTTP API V2 协议。支持 Influxdb v2.0 以及之后的版本。""" + +influxdb_api_v2.label: +"""HTTP API V2 协议""" + +org.desc: +"""InfluxDB 组织名称。""" + +org.label: +"""组织""" + +password.desc: +"""InfluxDB 密码。""" + +password.label: +"""密码""" + +precision.desc: +"""InfluxDB 时间精度。""" + +precision.label: +"""时间精度""" + +protocol.desc: +"""InfluxDB 协议。HTTP API 或 HTTP API V2。""" + +protocol.label: +"""协议""" + +server.desc: +"""将要连接的 IPv4 或 IPv6 地址，或者主机名。
+主机名具有以下形式：`Host[:Port]`。
+如果未指定 `[:Port]`，则使用 InfluxDB 默认端口 8086。""" + +server.label: +"""服务器地址""" + +token.desc: +"""InfluxDB token。""" + +token.label: +"""Token""" + +username.desc: +"""InfluxDB 用户名。""" + +username.label: +"""用户名""" + +} diff --git a/rel/i18n/zh/emqx_ee_connector_rocketmq.hocon b/rel/i18n/zh/emqx_ee_connector_rocketmq.hocon new file mode 100644 index 000000000..d32e6ea01 --- /dev/null +++ b/rel/i18n/zh/emqx_ee_connector_rocketmq.hocon @@ -0,0 +1,35 @@ +emqx_ee_connector_rocketmq { + +refresh_interval.desc: +"""RocketMQ 主题路由更新间隔。""" + +refresh_interval.label: +"""主题路由更新间隔""" + +security_token.desc: +"""RocketMQ 服务器安全令牌""" + +security_token.label: +"""安全令牌""" + +send_buffer.desc: +"""RocketMQ 驱动的套字节发送消息的缓冲区大小""" + +send_buffer.label: +"""发送消息的缓冲区大小""" + +server.desc: +"""将要连接的 IPv4 或 IPv6 地址，或者主机名。
+主机名具有以下形式：`Host[:Port]`。
+如果未指定 `[:Port]`，则使用 RocketMQ 默认端口 9876。""" + +server.label: +"""服务器地址""" + +topic.desc: +"""RocketMQ 主题""" + +topic.label: +"""RocketMQ 主题""" + +} diff --git a/rel/i18n/zh/emqx_ee_connector_sqlserver.hocon b/rel/i18n/zh/emqx_ee_connector_sqlserver.hocon new file mode 100644 index 000000000..44377c86d --- /dev/null +++ b/rel/i18n/zh/emqx_ee_connector_sqlserver.hocon @@ -0,0 +1,11 @@ +emqx_ee_connector_sqlserver { + +server.desc: +"""将要连接的 IPv4 或 IPv6 地址，或者主机名。
+主机名具有以下形式：`Host[:Port]`。
+如果未指定 `[:Port]`，则使用 SQL Server 默认端口 1433。""" + +server.label: +"""服务器地址""" + +} diff --git a/rel/i18n/zh/emqx_ee_connector_tdengine.hocon b/rel/i18n/zh/emqx_ee_connector_tdengine.hocon new file mode 100644 index 000000000..f3064aeb5 --- /dev/null +++ b/rel/i18n/zh/emqx_ee_connector_tdengine.hocon @@ -0,0 +1,11 @@ +emqx_ee_connector_tdengine { + +server.desc: +"""将要连接的 IPv4 或 IPv6 地址，或者主机名。
+主机名具有以下形式：`Host[:Port]`。
+如果未指定 `[:Port]`，则使用 TDengine 默认端口 6041。""" + +server.label: +"""服务器地址""" + +} diff --git a/rel/i18n/zh/emqx_ee_schema_registry_http_api.hocon b/rel/i18n/zh/emqx_ee_schema_registry_http_api.hocon new file mode 100644 index 000000000..1dab50a3f --- /dev/null +++ b/rel/i18n/zh/emqx_ee_schema_registry_http_api.hocon @@ -0,0 +1,39 @@ +emqx_ee_schema_registry_http_api { + +desc_param_path_schema_name.desc: +"""模式名称""" + +desc_param_path_schema_name.label: +"""模式名称""" + +desc_schema_registry_api_delete.desc: +"""删除一个模式""" + +desc_schema_registry_api_delete.label: +"""删除模式""" + +desc_schema_registry_api_get.desc: +"""通过名称获取模式""" + +desc_schema_registry_api_get.label: +"""获取模式""" + +desc_schema_registry_api_list.desc: +"""列出所有注册的模式""" + +desc_schema_registry_api_list.label: +"""列表模式""" + +desc_schema_registry_api_post.desc: +"""注册一个新的模式""" + +desc_schema_registry_api_post.label: +"""注册模式""" + +desc_schema_registry_api_put.desc: +"""更新一个现有的模式""" + +desc_schema_registry_api_put.label: +"""更新模式""" + +} diff --git a/rel/i18n/zh/emqx_ee_schema_registry_schema.hocon b/rel/i18n/zh/emqx_ee_schema_registry_schema.hocon new file mode 100644 index 000000000..2f4c972ec --- /dev/null +++ b/rel/i18n/zh/emqx_ee_schema_registry_schema.hocon @@ -0,0 +1,51 @@ +emqx_ee_schema_registry_schema { + +avro_type.desc: +"""[阿帕奇-阿夫罗](https://avro.apache.org/) 序列化格式。""" + +avro_type.label: +"""阿帕奇-阿夫罗""" + +protobuf_type.desc: +"""[协议缓冲器](https://protobuf.dev/) 序列化格式。""" + +protobuf_type.label: +"""协议缓冲器""" + +schema_description.desc: +"""对该模式的描述。""" + +schema_description.label: +"""模式描述""" + +schema_name.desc: +"""模式的一个名称，将作为其标识符。""" + +schema_name.label: +"""模式名称""" + +schema_registry_root.desc: +"""模式注册表的配置。""" + +schema_registry_root.label: +"""模式注册表""" + +schema_registry_schemas.desc: +"""注册的模式。""" + +schema_registry_schemas.label: +"""注册的模式""" + +schema_source.desc: +"""模式的源文本。""" + +schema_source.label: +"""模式来源""" + +schema_type.desc: +"""模式类型。""" + +schema_type.label: +"""模式类型""" + +} diff --git a/rel/i18n/zh/emqx_exhook_api.hocon b/rel/i18n/zh/emqx_exhook_api.hocon new file mode 100644 index 000000000..aba0e3e58 --- /dev/null +++ b/rel/i18n/zh/emqx_exhook_api.hocon @@ -0,0 +1,81 @@ +emqx_exhook_api { + +add_server.desc: +"""添加 ExHook 服务器""" + +delete_server.desc: +"""删除 Exhook 服务器""" + +get_detail.desc: +"""查看 Exhook 服务器详细信息""" + +get_hooks.desc: +"""获取 Exhook 服务器的钩子信息""" + +hook_metrics.desc: +"""当前节点中该钩子的指标信息""" + +hook_name.desc: +"""钩子的名称""" + +hook_params.desc: +"""钩子注册时使用的参数""" + +list_all_servers.desc: +"""查看ExHook 服务器列表""" + +metric_failed.desc: +"""钩子执行失败的次数""" + +metric_max_rate.desc: +"""钩子的最大调用速率""" + +metric_rate.desc: +"""钩子的调用速率""" + +metric_succeed.desc: +"""钩子执行成功的次数""" + +metrics.desc: +"""指标信息""" + +move_api.desc: +"""移动 Exhook 服务器顺序。 +注意: 移动的参数只能是：front | rear | before:{name} | after:{name}""" + +move_api.label: +"""改变已注册的Exhook服务器的执行顺序""" + +move_position.desc: +"""移动的方向""" + +node.desc: +"""节点名称""" + +node_hook_metrics.desc: +"""所有节点中该钩子的指标信息""" + +node_metrics.desc: +"""所有节点中该服务器的指标信息""" + +node_status.desc: +"""所有节点中该服务器的状态信息""" + +server_metrics.desc: +"""当前节点中该服务器的指标信息""" + +server_name.desc: +"""Exhook 服务器的名称""" + +status.desc: +"""Exhook 服务器的状态。
+connected: 连接成功
+connecting: 连接失败，重连中
+disconnected: 连接失败，且未设置自动重连
+disabled: 该服务器未开启
+error: 查看该服务器状态时发生错误""" + +update_server.desc: +"""更新 Exhook 服务器""" + +} diff --git a/rel/i18n/zh/emqx_exhook_schema.hocon b/rel/i18n/zh/emqx_exhook_schema.hocon new file mode 100644 index 000000000..54d86169b --- /dev/null +++ b/rel/i18n/zh/emqx_exhook_schema.hocon @@ -0,0 +1,43 @@ +emqx_exhook_schema { + +auto_reconnect.desc: +"""自动重连到 gRPC 服务器的设置。 +当 gRPC 服务器不可用时，Exhook 将会按照这里设置的间隔时间进行重连，并重新初始化注册的钩子""" + +enable.desc: +"""开启这个 Exhook 服务器""" + +failed_action.desc: +"""当 gRPC 请求失败后的操作""" + +keepalive.desc: +"""当没有其他数据交换时，是否向连接的对端套接字定期的发送探测包。如果另一端没有响应，则认为连接断开，并向控制进程发送错误消息""" + +name.desc: +"""ExHook 服务器名称""" + +nodelay.desc: +"""如果为 true，则为套接字设置 TCP_NODELAY 选项，这意味着会立即发送数据包""" + +pool_size.desc: +"""gRPC 客户端进程池大小""" + +recbuf.desc: +"""套接字的最小接收缓冲区大小""" + +request_timeout.desc: +"""gRPC 服务器请求超时时间""" + +servers.desc: +"""ExHook 服务器列表""" + +sndbuf.desc: +"""套接字的最小发送缓冲区大小""" + +socket_options.desc: +"""连接套接字设置""" + +url.desc: +"""gRPC 服务器地址""" + +} diff --git a/rel/i18n/zh/emqx_exproto_schema.hocon b/rel/i18n/zh/emqx_exproto_schema.hocon new file mode 100644 index 000000000..794a563f3 --- /dev/null +++ b/rel/i18n/zh/emqx_exproto_schema.hocon @@ -0,0 +1,26 @@ +emqx_exproto_schema { + +exproto.desc: +"""ExProto 网关""" + +exproto_grpc_handler_address.desc: +"""对端 gRPC 服务器地址。""" + +exproto_grpc_handler_ssl.desc: +"""gRPC 客户端的 SSL 配置。""" + +exproto_grpc_server_bind.desc: +"""服务监听地址和端口。""" + +exproto_grpc_server_ssl.desc: +"""服务 SSL 配置。""" + +exproto_handler.desc: +"""配置 ExProto 网关需要请求的 ConnectionHandler 服务地址。 +该服务用于给 ExProto 提供客户端的 Socket 事件处理、字节解码、订阅消息接收等功能。""" + +exproto_server.desc: +"""配置 ExProto 网关需要启动的 ConnectionAdapter 服务。 +该服务用于提供客户端的认证、发布、订阅和数据下行等功能。""" + +} diff --git a/rel/i18n/zh/emqx_gateway_api.hocon b/rel/i18n/zh/emqx_gateway_api.hocon new file mode 100644 index 000000000..c9ea582a3 --- /dev/null +++ b/rel/i18n/zh/emqx_gateway_api.hocon @@ -0,0 +1,73 @@ +emqx_gateway_api { + +delete_gateway.desc: +"""停用指定网关""" + +enable_gateway.desc: +"""使用配置启动某一网关。""" + +gateway_created_at.desc: +"""网关创建时间""" + +gateway_current_connections.desc: +"""当前连接数""" + +gateway_enable_in_path.desc: +"""是否开启此网关""" + +gateway_listener_id.desc: +"""监听器 ID""" + +gateway_listener_name.desc: +"""监听器名称""" + +gateway_listener_running.desc: +"""监听器运行状态""" + +gateway_listener_type.desc: +"""监听器类型""" + +gateway_listeners.desc: +"""网关监听器列表""" + +gateway_max_connections.desc: +"""最大连接数""" + +gateway_name.desc: +"""网关名称""" + +gateway_name_in_qs.desc: +"""网关名称.
+可取值为 `stomp`、`mqttsn`、`coap`、`lwm2m`、`exproto`""" + +gateway_node_status.desc: +"""网关在集群中每个节点上的状态""" + +gateway_started_at.desc: +"""网关启用时间""" + +gateway_status.desc: +"""网关启用状态""" + +gateway_status_in_qs.desc: +"""通过网关状态筛选
+可选值为 `running`、`stopped`、`unloaded`""" + +gateway_stopped_at.desc: +"""网关停用时间""" + +get_gateway.desc: +"""获取网关配置详情""" + +list_gateway.desc: +"""该接口会返回指定或所有网关的概览状态， +包括当前状态、连接数、监听器状态等。""" + +node.desc: +"""节点名称""" + +update_gateway.desc: +"""更新指定网关的基础配置、和启用的状态。
+注：认证、和监听器的配置更新需参考对应的 API 接口。""" + +} diff --git a/rel/i18n/zh/emqx_gateway_api_authn.hocon b/rel/i18n/zh/emqx_gateway_api_authn.hocon new file mode 100644 index 000000000..662789551 --- /dev/null +++ b/rel/i18n/zh/emqx_gateway_api_authn.hocon @@ -0,0 +1,45 @@ +emqx_gateway_api_authn { + +add_authn.desc: +"""为指定网关开启认证器实现客户端认证的功能。
+当未配置认证器或关闭认证器时，则认为允许所有客户端的连接。
+注：在网关中仅支持添加一个认证器，而不是像 MQTT 一样允许配置多个认证器构成认证链。""" + +add_user.desc: +"""添加用户（仅支持 built_in_database 类型的认证器）""" + +delete_authn.desc: +"""删除指定网关的认证器。""" + +delete_user.desc: +"""删除用户（仅支持 built_in_database 类型的认证器）""" + +get_authn.desc: +"""获取指定网关认证器的配置 +当网关或认证未启用时，返回 404。""" + +get_user.desc: +"""获取用户信息（仅支持 built_in_database 类型的认证器）""" + +import_users.desc: +"""导入用户（仅支持 built_in_database 类型的认证器）""" + +is_superuser.desc: +"""是否是超级用户""" + +like_user_id.desc: +"""使用用户 ID （username 或 clientid）模糊搜索，仅支持按子串的方式进行搜索。""" + +list_users.desc: +"""获取用户列表（仅支持 built_in_database 类型的认证器）""" + +update_authn.desc: +"""更新指定网关认证器的配置，或停用认证器。""" + +update_user.desc: +"""更新用户信息（仅支持 built_in_database 类型的认证器）""" + +user_id.desc: +"""用户 ID""" + +} diff --git a/rel/i18n/zh/emqx_gateway_api_clients.hocon b/rel/i18n/zh/emqx_gateway_api_clients.hocon new file mode 100644 index 000000000..5dc7ff22a --- /dev/null +++ b/rel/i18n/zh/emqx_gateway_api_clients.hocon @@ -0,0 +1,207 @@ +emqx_gateway_api_clients { + +disconnected_at.desc: +"""客户端连接断开时间""" + +heap_size.desc: +"""进程堆内存大小，单位：字节""" + +send_oct.desc: +"""已发送字节数""" + +get_client.desc: +"""获取客户端信息""" + +param_gte_created_at.desc: +"""匹配会话创建时间大于等于指定值的客户端""" + +param_conn_state.desc: +"""匹配客户端连接状态""" + +send_pkt.desc: +"""已发送应用层协议控制报文数""" + +clean_start.desc: +"""标识客户端是否以 clean_start 的标志连接到网关""" + +inflight_cnt.desc: +"""客户端当前飞行窗口大小""" + +delete_subscription.desc: +"""为某客户端删除某订阅关系""" + +param_lte_connected_at.desc: +"""匹配连接创建时间小于等于指定值的客户端""" + +node.desc: +"""客户端连接到的节点名称""" + +awaiting_rel_cnt.desc: +"""客户端当前等待 PUBREL 确认的 PUBREC 消息的条数""" + +rap.desc: +"""Retain as Published 选项，枚举：0，1""" + +inflight_max.desc: +"""客户端允许的飞行窗口最大值""" + +param_username.desc: +"""匹配客户端 Username""" + +param_like_endpoint_name.desc: +"""子串匹配 LwM2M 客户端 Endpoint Name""" + +created_at.desc: +"""会话创建时间""" + +sub_props.desc: +"""订阅属性""" + +list_clients.desc: +"""获取指定网关的客户端列表""" + +subscriptions_cnt.desc: +"""客户端已订阅主题数""" + +mailbox_len.desc: +"""进程邮箱大小""" + +send_cnt.desc: +"""已发送 Socket 报文次数""" + +rh.desc: +"""Retain Handling 选项，枚举：0，1，2""" + +connected.desc: +"""标识客户端是否已连接到网关""" + +qos.desc: +"""QoS 等级，枚举：0，1，2""" + +ip_address.desc: +"""客户端 IP 地址""" + +param_gte_connected_at.desc: +"""匹配连接创建时间大于等于指定值的客户端""" + +awaiting_rel_max.desc: +"""客户端允许的最大 PUBREC 等待队列长度""" + +param_like_username.desc: +"""子串匹配 客户端 Username""" + +nl.desc: +"""No Local 选项，枚举：0，1""" + +param_like_clientid.desc: +"""子串匹配客户端 ID""" + +param_lte_created_at.desc: +"""匹配会话创建时间小于等于指定值的客户端""" + +topic.desc: +"""主题过滤器或主题名称""" + +proto_ver.desc: +"""客户端使用的协议版本""" + +mountpoint.desc: +"""主题固定前缀""" + +proto_name.desc: +"""客户端使用的协议名称""" + +param_lte_lifetime.desc: +"""匹配心跳时间小于等于指定值的 LwM2M 客户端""" + +port.desc: +"""客户端端口""" + +connected_at.desc: +"""客户端连接时间""" + +expiry_interval.desc: +"""会话超期时间，单位：秒""" + +username.desc: +"""客户端连接的用户名""" + +param_clean_start.desc: +"""匹配客户端 `clean_start` 标记""" + +recv_msg.desc: +"""已接收上行的消息条数""" + +list_subscriptions.desc: +"""获取某客户端的主题订阅列表""" + +recv_oct.desc: +"""已接收的字节数""" + +keepalive.desc: +"""Keepalive 时间，单位：秒""" + +param_clientid.desc: +"""匹配客户端 ID""" + +subscriptions_max.desc: +"""客户端允许订阅的最大主题数""" + +param_ip_address.desc: +"""匹配客户端 IP 地址""" + +mqueue_max.desc: +"""客户端允许的最大消息队列长度""" + +mqueue_dropped.desc: +"""由于消息队列过程，客户端消息队列丢弃消息条数""" + +subid.desc: +"""订阅ID，仅用于 Stomp 网关。用于创建订阅关系时指定订阅 ID。取值范围 1-65535。""" + +clientid.desc: +"""客户端 ID""" + +kick_client.desc: +"""踢出指定客户端""" + +is_bridge.desc: +"""标识客户端是否通过 is_bridge 标志连接""" + +lifetime.desc: +"""LwM2M 客户端心跳周期""" + +send_msg.desc: +"""已发送下行消息数条数""" + +add_subscription.desc: +"""为某客户端新增订阅关系""" + +param_endpoint_name.desc: +"""匹配 LwM2M 客户端 Endpoint Name""" + +param_node.desc: +"""匹配客户端的节点名称""" + +recv_cnt.desc: +"""已接收 Socket 报文次数""" + +recv_pkt.desc: +"""已接收应用层协议控制报文数""" + +endpoint_name.desc: +"""LwM2M 客户端 Endpoint Name""" + +param_proto_ver.desc: +"""匹配客户端协议版本""" + +reductions.desc: +"""进程已消耗 Reduction 数""" + +param_gte_lifetime.desc: +"""匹配心跳时间大于等于指定值的 LwM2M 客户端""" + +mqueue_len.desc: +"""客户端当前消息队列长度""" + +} diff --git a/rel/i18n/zh/emqx_gateway_api_listeners.hocon b/rel/i18n/zh/emqx_gateway_api_listeners.hocon new file mode 100644 index 000000000..731b14b74 --- /dev/null +++ b/rel/i18n/zh/emqx_gateway_api_listeners.hocon @@ -0,0 +1,65 @@ +emqx_gateway_api_listeners { + +add_listener.desc: +"""为指定网关添加监听器。
+注：对于某网关不支持的监听器类型，该接口会返回 `400: BAD_REQUEST`。""" + +add_listener_authn.desc: +"""为指定监听器开启认证器以实现客户端认证的能力。
+当某一监听器开启认证后，所有连接到该监听器的客户端会使用该认证器进行认证。""" + +add_user.desc: +"""添加用户（仅支持 built_in_database 类型的认证器）""" + +current_connections.desc: +"""当前连接数""" + +delete_listener.desc: +"""删除指定监听器。被删除的监听器下所有已连接的客户端都会离线。""" + +delete_listener_authn.desc: +"""移除指定监听器的认证器。""" + +delete_user.desc: +"""删除用户（仅支持 built_in_database 类型的认证器）""" + +get_listener.desc: +"""获取指定网关监听器的配置。""" + +get_listener_authn.desc: +"""获取监听器的认证器配置。""" + +get_user.desc: +"""获取用户信息（仅支持 built_in_database 类型的认证器）""" + +import_users.desc: +"""导入用户（仅支持 built_in_database 类型的认证器）""" + +list_listeners.desc: +"""获取网关监听器列表。该接口会返回监听器所有的配置（包括该监听器上的认证器），同时也会返回该监听器在集群中运行的状态。""" + +list_users.desc: +"""获取用户列表（仅支持 built_in_database 类型的认证器）""" + +listener_id.desc: +"""监听器 ID""" + +listener_node_status.desc: +"""监听器在集群中每个节点上的状态""" + +listener_status.desc: +"""监听器状态""" + +node.desc: +"""节点名称""" + +update_listener.desc: +"""更新某网关监听器的配置。被更新的监听器会执行重启，所有已连接到该监听器上的客户端都会被断开。""" + +update_listener_authn.desc: +"""更新指定监听器的认证器配置，或停用/启用该认证器。""" + +update_user.desc: +"""更新用户信息（仅支持 built_in_database 类型的认证器）""" + +} diff --git a/rel/i18n/zh/emqx_gateway_schema.hocon b/rel/i18n/zh/emqx_gateway_schema.hocon new file mode 100644 index 000000000..40cee4efb --- /dev/null +++ b/rel/i18n/zh/emqx_gateway_schema.hocon @@ -0,0 +1,112 @@ +emqx_gateway_schema { + +dtls_listener_acceptors.desc: +"""Acceptor 进程池大小。""" + +dtls_listener_dtls_opts.desc: +"""DTLS Socket 配置""" + +gateway_common_authentication.desc: +"""网关的认证器配置，对该网关下所以的监听器生效。如果每个监听器需要配置不同的认证器，需要配置监听器下的 authentication 字段。""" + +gateway_common_clientinfo_override.desc: +"""ClientInfo 重写。""" + +gateway_common_clientinfo_override_clientid.desc: +"""clientid 重写模板""" + +gateway_common_clientinfo_override_password.desc: +"""password 重写模板""" + +gateway_common_clientinfo_override_username.desc: +"""username 重写模板""" + +gateway_common_enable.desc: +"""是否启用该网关""" + +gateway_common_enable_stats.desc: +"""是否开启客户端统计""" + +gateway_common_idle_timeout.desc: +"""客户端连接过程的空闲时间。该配置用于： + 1. 一个新创建的客户端进程如果在该时间间隔内没有收到任何客户端请求，将被直接关闭。 + 2. 一个正在运行的客户进程如果在这段时间后没有收到任何客户请求，将进入休眠状态以节省资源。""" + +gateway_common_listener_access_rules.desc: +"""配置监听器的访问控制规则。 +见：https://github.com/emqtt/esockd#allowdeny""" + +gateway_common_listener_bind.desc: +"""监听器绑定的 IP 地址或端口。""" + +gateway_common_listener_enable.desc: +"""是否启用该监听器。""" + +gateway_common_listener_enable_authn.desc: +"""配置 true （默认值）启用客户端进行身份认证。 +配置 false 时，将不对客户端做任何认证。""" + +gateway_common_listener_max_conn_rate.desc: +"""监听器支持的最大连接速率。""" + +gateway_common_listener_max_connections.desc: +"""监听器支持的最大连接数。""" + +gateway_mountpoint.desc: +"""发布或订阅时，在所有主题前增加前缀字符串。 +当消息投递给订阅者时，前缀字符串将从主题名称中删除。挂载点是用户可以用来实现不同监听器之间的消息路由隔离的一种方式。 +例如，如果客户端 A 在 `listeners.tcp.\.mountpoint` 设置为 `some_tenant` 的情况下订阅 `t`， +则客户端实际上订阅了 `some_tenant/t` 主题。 +类似地，如果另一个客户端 B（连接到与客户端 A 相同的侦听器）向主题 `t` 发送消息， +则该消息被路由到所有订阅了 `some_tenant/t` 的客户端，因此客户端 A 将收到该消息，带有 主题名称`t`。 设置为 `""` 以禁用该功能。 +挂载点字符串中可用的变量：
+ - ${clientid}：clientid
+ - ${username}：用户名""" + +listener_name_to_settings_map.desc: +"""从监听器名称到配置参数的映射。""" + +ssl_listener_options.desc: +"""SSL Socket 配置。""" + +tcp_listener_acceptors.desc: +"""Acceptor 进程池大小。""" + +tcp_listener_proxy_protocol.desc: +"""是否开启 Proxy Protocol V1/2。当 EMQX 集群部署在 HAProxy 或 Nginx 后需要获取客户端真实 IP 时常用到该选项。参考：https://www.haproxy.com/blog/haproxy/proxy-protocol/""" + +tcp_listener_proxy_protocol_timeout.desc: +"""接收 Proxy Protocol 报文头的超时时间。如果在超时内没有收到 Proxy Protocol 包，EMQX 将关闭 TCP 连接。""" + +tcp_listener_tcp_opts.desc: +"""TCP Socket 配置。""" + +tcp_listeners.desc: +"""配置 TCP 类型的监听器。""" + +tcp_udp_listeners.desc: +"""监听器配置。""" + +udp_listener_active_n.desc: +"""为 Socket 指定 {active, N} 选项。 +参见：https://erlang.org/doc/man/inet.html#setopts-2""" + +udp_listener_buffer.desc: +"""Socket 在用户空间的缓冲区大小。""" + +udp_listener_recbuf.desc: +"""Socket 在内核空间接收缓冲区的大小。""" + +udp_listener_reuseaddr.desc: +"""允许重用本地处于 TIME_WAIT 的端口号。""" + +udp_listener_sndbuf.desc: +"""Socket 在内核空间发送缓冲区的大小。""" + +udp_listener_udp_opts.desc: +"""UDP Socket 配置。""" + +udp_listeners.desc: +"""配置 UDP 类型的监听器。""" + +} diff --git a/rel/i18n/zh/emqx_license_http_api.hocon b/rel/i18n/zh/emqx_license_http_api.hocon new file mode 100644 index 000000000..4ad471684 --- /dev/null +++ b/rel/i18n/zh/emqx_license_http_api.hocon @@ -0,0 +1,15 @@ +emqx_license_http_api { + +desc_license_info_api.desc: +"""获取许可证信息""" + +desc_license_info_api.label: +"""许可证信息""" + +desc_license_key_api.desc: +"""更新一个许可证密钥""" + +desc_license_key_api.label: +"""更新许可证""" + +} diff --git a/rel/i18n/zh/emqx_license_schema.hocon b/rel/i18n/zh/emqx_license_schema.hocon new file mode 100644 index 000000000..0bf5256e8 --- /dev/null +++ b/rel/i18n/zh/emqx_license_schema.hocon @@ -0,0 +1,29 @@ +emqx_license_schema { + +connection_high_watermark_field.desc: +"""高水位线，连接数超过这个水位线时，系统会触发许可证连接配额使用告警""" + +connection_high_watermark_field.label: +"""连接高水位""" + +connection_low_watermark_field.desc: +"""低水位限制，低于此水位线时系统会清除连接配额使用告警""" + +connection_low_watermark_field.label: +"""连接低水位线""" + +key_field.desc: +"""许可证字符串""" + +key_field.label: +"""许可证字符串""" + +license_root.desc: +"""EMQX企业许可证。 +EMQX 自带一个默认的试用许可证，默认试用许可允许最多接入 100 个连接，签发时间是 2023年1月9日，有效期是 5 年（1825 天）。若需要在生产环境部署， +请访问 https://www.emqx.com/apply-licenses/emqx 来申请。""" + +license_root.label: +"""许可证""" + +} diff --git a/rel/i18n/zh/emqx_limiter_schema.hocon b/rel/i18n/zh/emqx_limiter_schema.hocon new file mode 100644 index 000000000..4f5a0ce2f --- /dev/null +++ b/rel/i18n/zh/emqx_limiter_schema.hocon @@ -0,0 +1,89 @@ +emqx_limiter_schema { + +bucket_cfg.desc: +"""桶的配置""" + +bucket_cfg.label: +"""桶的配置""" + +burst.desc: +"""突发速率。 +突发速率允许短时间内速率超过设置的速率值，突发速率 + 速率 = 当前桶能达到的最大速率值""" + +burst.label: +"""突发速率""" + +bytes.desc: +"""流入字节率控制器。 +这个是用来控制当前节点上的数据流入的字节率，每条消息将会消耗和其二进制大小等量的令牌，当达到最大速率后，会话将会被限速甚至被强制挂起一小段时间""" + +bytes.label: +"""流入字节率""" + +client.desc: +"""对桶的每个使用者的速率控制设置""" + +client.label: +"""每个使用者的限制""" + +connection.desc: +"""连接速率控制器。 +这个用来控制当前节点上的连接速率，当达到最大速率后，新的连接将会被拒绝""" + +connection.label: +"""连接速率""" + +divisible.desc: +"""申请的令牌数是否可以被分割""" + +divisible.label: +"""是否可分割""" + +failure_strategy.desc: +"""当所有的重试都失败后的处理策略""" + +failure_strategy.label: +"""失败策略""" + +initial.desc: +"""桶中的初始令牌数""" + +initial.label: +"""初始令牌数""" + +internal.desc: +"""EMQX 内部功能所用限制器。""" + +low_watermark.desc: +"""当桶中剩余的令牌数低于这个值，即使令牌申请成功了，也会被强制暂停一会儿""" + +low_watermark.label: +"""低水位线""" + +max_retry_time.desc: +"""申请失败后，尝试重新申请的时长最大值""" + +max_retry_time.label: +"""最大重试时间""" + +message_routing.desc: +"""消息派发速率控制器。 +这个用来控制当前节点内的消息派发速率，当达到最大速率后，新的推送将会被拒绝""" + +message_routing.label: +"""消息派发""" + +messages.desc: +"""流入速率控制器。 +这个用来控制当前节点上的消息流入速率，当达到最大速率后，会话将会被限速甚至被强制挂起一小段时间""" + +messages.label: +"""消息流入速率""" + +rate.desc: +"""桶的令牌生成速率""" + +rate.label: +"""速率""" + +} diff --git a/rel/i18n/zh/emqx_lwm2m_api.hocon b/rel/i18n/zh/emqx_lwm2m_api.hocon new file mode 100644 index 000000000..7cf53060a --- /dev/null +++ b/rel/i18n/zh/emqx_lwm2m_api.hocon @@ -0,0 +1,27 @@ +emqx_lwm2m_api { + +dataType.desc: +"""数据类型""" + +lookup_resource.desc: +"""查看指定资源状态""" + +name.desc: +"""资源名称""" + +observe_resource.desc: +"""Observe/Un-Observe 指定资源""" + +operations.desc: +"""资源可用操作列表""" + +path.desc: +"""资源路径""" + +read_resource.desc: +"""发送读指令到某资源""" + +write_resource.desc: +"""发送写指令到某资源""" + +} diff --git a/rel/i18n/zh/emqx_lwm2m_schema.hocon b/rel/i18n/zh/emqx_lwm2m_schema.hocon new file mode 100644 index 000000000..3dea6a0c6 --- /dev/null +++ b/rel/i18n/zh/emqx_lwm2m_schema.hocon @@ -0,0 +1,56 @@ +emqx_lwm2m_schema { + +lwm2m.desc: +"""LwM2M 网关配置。仅支持 v1.0.1 协议。""" + +lwm2m_auto_observe.desc: +"""自动 Observe REGISTER 数据包的 Object 列表。""" + +lwm2m_lifetime_max.desc: +"""允许 LwM2M 客户端允许设置的心跳最大值。""" + +lwm2m_lifetime_min.desc: +"""允许 LwM2M 客户端允许设置的心跳最小值。""" + +lwm2m_qmode_time_window.desc: +"""在QMode模式下，LwM2M网关认为网络链接有效的时间窗口的值。 +例如，在收到客户端的更新信息后，在这个时间窗口内的任何信息都会直接发送到LwM2M客户端，而超过这个时间窗口的所有信息都会暂时储存在内存中。""" + +lwm2m_translators.desc: +"""LwM2M 网关订阅/发布消息的主题映射配置。""" + +lwm2m_translators_command.desc: +"""下行命令主题。 +对于每个成功上线的新 LwM2M 客户端，网关会创建一个订阅关系来接收下行消息并将其发送给客户端。""" + +lwm2m_translators_notify.desc: +"""用于发布来自 LwM2M 客户端的通知事件的主题。 +在成功 Observe 到 LwM2M 客户端的资源后，如果客户端报告任何资源状态的变化，网关将通过该主题发送通知事件。""" + +lwm2m_translators_register.desc: +"""用于发布来自 LwM2M 客户端的注册事件的主题。""" + +lwm2m_translators_response.desc: +"""用于网关发布来自 LwM2M 客户端的确认事件的主题。""" + +lwm2m_translators_update.desc: +"""用于发布来自LwM2M客户端的更新事件的主题。""" + +lwm2m_update_msg_publish_condition.desc: +"""发布UPDATE事件消息的策略。
+ - always: 只要收到 UPDATE 请求，就发送更新事件。
+ - contains_object_list: 仅当 UPDATE 请求携带 Object 列表时才发送更新事件。""" + +lwm2m_xml_dir.desc: +"""LwM2M Resource 定义的 XML 文件目录路径。""" + +translator.desc: +"""配置某网关客户端对于发布消息或订阅的主题和 QoS 等级。""" + +translator_qos.desc: +"""QoS 等级""" + +translator_topic.desc: +"""主题名称""" + +} diff --git a/rel/i18n/zh/emqx_mgmt_api_alarms.hocon b/rel/i18n/zh/emqx_mgmt_api_alarms.hocon new file mode 100644 index 000000000..d9dafd867 --- /dev/null +++ b/rel/i18n/zh/emqx_mgmt_api_alarms.hocon @@ -0,0 +1,37 @@ +emqx_mgmt_api_alarms { + +activate_at.desc: +"""告警开始时间，使用 rfc3339 标准时间格式。""" + +deactivate_at.desc: +"""告警结束时间，使用 rfc3339 标准时间格式。""" + +delete_alarms_api.desc: +"""删除所有历史告警。""" + +delete_alarms_api_response204.desc: +"""历史告警已成功清除。""" + +details.desc: +"""告警详情，提供了更多的告警信息，主要提供给程序处理。""" + +duration.desc: +"""表明告警已经持续了多久，单位：毫秒。""" + +get_alarms_qs_activated.desc: +"""用于指定查询的告警类型， +为 true 时返回当前激活的告警，为 false 时返回历史告警，默认为 false。""" + +list_alarms_api.desc: +"""列出当前激活的告警或历史告警，由查询参数决定。""" + +message.desc: +"""告警消息，以人类可读的方式描述告警内容。""" + +name.desc: +"""告警名称，用于区分不同的告警。""" + +node.desc: +"""触发此告警的节点名称。""" + +} diff --git a/rel/i18n/zh/emqx_mgmt_api_banned.hocon b/rel/i18n/zh/emqx_mgmt_api_banned.hocon new file mode 100644 index 000000000..cee3ba288 --- /dev/null +++ b/rel/i18n/zh/emqx_mgmt_api_banned.hocon @@ -0,0 +1,54 @@ +emqx_mgmt_api_banned { + +as.desc: +"""封禁方式，可以通过客户端 ID、用户名或者 IP 地址等方式进行封禁。""" + +as.label: +"""封禁方式""" + +at.desc: +"""封禁的起始时间，格式为 rfc3339，默认为发起操作的时间。""" + +at.label: +"""封禁时间""" + +by.desc: +"""封禁的发起者。""" + +by.label: +"""封禁发起者""" + +create_banned_api.desc: +"""添加一个客户端 ID、用户名或者 IP 地址到黑名单。""" + +create_banned_api_response400.desc: +"""错误的请求，可能是参数错误或封禁对象已存在等原因。""" + +delete_banned_api.desc: +"""将一个客户端 ID、用户名或者 IP 地址从黑名单中删除。""" + +delete_banned_api_response404.desc: +"""未在黑名单中找到该封禁对象。""" + +list_banned_api.desc: +"""列出目前所有被封禁的客户端 ID、用户名和 IP 地址。""" + +reason.desc: +"""封禁原因，记录当前对象被封禁的原因。""" + +reason.label: +"""封禁原因""" + +until.desc: +"""封禁的结束时间，格式为 rfc3339，默认值为发起操作的时间 + 1 年。""" + +until.label: +"""封禁结束时间""" + +who.desc: +"""封禁对象，具体的客户端 ID、用户名或者 IP 地址。""" + +who.label: +"""封禁对象""" + +} diff --git a/rel/i18n/zh/emqx_mgmt_api_key_schema.hocon b/rel/i18n/zh/emqx_mgmt_api_key_schema.hocon new file mode 100644 index 000000000..9a0536fe6 --- /dev/null +++ b/rel/i18n/zh/emqx_mgmt_api_key_schema.hocon @@ -0,0 +1,19 @@ +emqx_mgmt_api_key_schema { + +api_key.desc: +"""API 密钥， 可用于请求除管理 API 密钥及 Dashboard 用户管理 API 的其它接口""" + +api_key.label: +"""API 密钥""" + +bootstrap_file.desc: +"""用于在启动 emqx 时，添加 API 密钥，其格式为： + ``` + 7e729ae70d23144b:2QILI9AcQ9BYlVqLDHQNWN2saIjBV4egr1CZneTNKr9CpK + ec3907f865805db0:Ee3taYltUKtoBVD9C3XjQl9C6NXheip8Z9B69BpUv5JxVHL + ```""" + +bootstrap_file.label: +"""API 密钥初始化文件""" + +} diff --git a/rel/i18n/zh/emqx_mgmt_api_publish.hocon b/rel/i18n/zh/emqx_mgmt_api_publish.hocon new file mode 100644 index 000000000..2a532fbdd --- /dev/null +++ b/rel/i18n/zh/emqx_mgmt_api_publish.hocon @@ -0,0 +1,81 @@ +emqx_mgmt_api_publish { + +error_message.desc: +"""失败的详细原因。""" + +message_id.desc: +"""全局唯一的一个消息 ID，方便用于关联和追踪。""" + +message_properties.desc: +"""PUBLISH 消息里的 Property 字段。""" + +msg_content_type.desc: +"""内容类型标识符，以 UTF-8 格式编码的字符串，用来描述应用消息的内容，服务端必须把收到的应用消息中的内容类型原封不动的发送给所有的订阅者。""" + +msg_correlation_data.desc: +"""对比数据标识符，服务端在收到应用消息时必须原封不动的把对比数据发送给所有的订阅者。对比数据只对请求消息(Request Message)的发送端和响应消息(Response Message)的接收端有意义。""" + +msg_message_expiry_interval.desc: +"""消息过期间隔标识符，以秒为单位。当消失已经过期时，如果服务端还没有开始向匹配的订阅者投递该消息，则服务端会删除该订阅者的消息副本。如果不设置，则消息永远不会过期""" + +msg_payload_format_indicator.desc: +"""载荷格式指示标识符，0 表示载荷是未指定格式的数据，相当于没有发送载荷格式指示；1 表示载荷是 UTF-8 编码的字符数据，载荷中的 UTF-8 数据必须是按照 Unicode 的规范和 RFC 3629 的标准要求进行编码的。""" + +msg_response_topic.desc: +"""响应主题标识符， UTF-8 编码的字符串，用作响应消息的主题名。响应主题不能包含通配符，也不能包含多个主题，否则将造成协议错误。当存在响应主题时，消息将被视作请求报文。服务端在收到应用消息时必须将响应主题原封不动的发送给所有的订阅者。""" + +msg_user_properties.desc: +"""指定 MQTT 消息的 User Property 键值对。注意，如果出现重复的键，只有最后一个会保留。""" + +payload.desc: +"""MQTT 消息体。""" + +payload_encoding.desc: +"""MQTT 消息体的编码方式，可以是 base64 或 plain。当设置为 base64 时，消息在发布前会先被解码。""" + +publish_api.desc: +"""发布一个消息。
+可能的 HTTP 状态码如下：
+200: 消息被成功发送到至少一个订阅。
+202: 没有匹配到任何订阅。
+400: 消息编码错误，如非法主题，或 QoS 超出范围等。
+503: 服务重启等过程中导致转发失败。""" + +publish_api.label: +"""发布一条信息""" + +publish_bulk_api.desc: +"""批量发布一组消息。
+可能的 HTTP 状态码如下：
+200: 所有的消息都被成功发送到至少一个订阅。
+202: 至少有一个消息没有匹配到任何订阅。
+400: 至少有一个消息编码错误，如非法主题，或 QoS 超出范围等。
+503: 至少有一个小因为服务重启的原因导致转发失败。
+ +请求的 Body 或者 Body 中包含的某个消息无法通过 API 规范的类型检查时，HTTP 响应的消息与发布单个消息的 API + /publish 是一样的。 +如果所有的消息都是合法的，那么 HTTP 返回的内容是一个 JSON 数组，每个元素代表了该消息转发的状态。""" + +publish_bulk_api.label: +"""发布一批信息""" + +qos.desc: +"""MQTT 消息的 QoS""" + +reason_code.desc: +"""MQTT 消息发布的错误码，这些错误码也是 MQTT 规范中 PUBACK 消息可能携带的错误码。
+当前支持如下错误码：
+ +16(0x10)：没能匹配到任何订阅；
+131(0x81)：消息转发时发生错误，例如 EMQX 服务重启；
+144(0x90)：主题名称非法；
+151(0x97)：受到了速率限制，或者消息尺寸过大。全局消息大小限制可以通过配置项 mqtt.max_packet_size 来进行修改。
+注意：消息尺寸的是通过主题和消息体的字节数进行估算的。具体发布时所占用的字节数可能会稍大于这个估算的值。""" + +retain.desc: +"""布尔型字段，用于表示该消息是否保留消息。""" + +topic_name.desc: +"""主题名称""" + +} diff --git a/rel/i18n/zh/emqx_mgmt_api_status.hocon b/rel/i18n/zh/emqx_mgmt_api_status.hocon new file mode 100644 index 000000000..3625db967 --- /dev/null +++ b/rel/i18n/zh/emqx_mgmt_api_status.hocon @@ -0,0 +1,22 @@ +emqx_mgmt_api_status { + +get_status_api.desc: +"""作为节点的健康检查。 返回一个纯文本的响应，描述节点的状态。 + +如果 EMQX 应用程序已经启动并运行，返回状态代码 200，否则返回 503。 + +这个API是在v5.0.10中引入的。 +GET `/status`端点（没有`/api/...`前缀）也是这个端点的一个别名，工作方式相同。 这个别名从v5.0.0开始就有了。""" + +get_status_api.label: +"""服务健康检查""" + +get_status_response200.desc: +"""Node emqx@127.0.0.1 is started +emqx is running""" + +get_status_response503.desc: +"""Node emqx@127.0.0.1 is stopped +emqx is not_running""" + +} diff --git a/rel/i18n/zh/emqx_modules_schema.hocon b/rel/i18n/zh/emqx_modules_schema.hocon new file mode 100644 index 000000000..e1c2ca913 --- /dev/null +++ b/rel/i18n/zh/emqx_modules_schema.hocon @@ -0,0 +1,45 @@ +emqx_modules_schema { + +enable.desc: +"""是否开启该功能""" + +max_delayed_messages.desc: +"""延迟消息的数量上限(0 代表无限)""" + +rewrite.desc: +"""EMQX 的主题重写功能支持根据用户配置的规则在客户端订阅主题、发布消息、取消订阅的时候将 A 主题重写为 B 主题。 +重写规则分为 Pub 规则和 Sub 规则，Pub 规则匹配 PUSHLISH 报文携带的主题，Sub 规则匹配 SUBSCRIBE、UNSUBSCRIBE 报文携带的主题。 +每条重写规则都由主题过滤器、正则表达式、目标表达式三部分组成。 +在主题重写功能开启的前提下，EMQX 在收到诸如 PUBLISH 报文等带有主题的 MQTT 报文时，将使用报文中的主题去依次匹配配置文件中规则的主题过滤器部分，一旦成功匹配，则使用正则表达式提取主题中的信息，然后替换至目标表达式以构成新的主题。 +目标表达式中可以使用 `$N` 这种格式的变量匹配正则表达中提取出来的元素，`$N` 的值为正则表达式中提取出来的第 N 个元素，比如 `$1` 即为正则表达式提取的第一个元素。 +需要注意的是，EMQX 使用倒序读取配置文件中的重写规则，当一条主题可以同时匹配多条主题重写规则的主题过滤器时，EMQX 仅会使用它匹配到的第一条规则进行重写，如果该条规则中的正则表达式与 MQTT 报文主题不匹配，则重写失败，不会再尝试使用其他的规则进行重写。 +因此用户在使用时需要谨慎的设计 MQTT 报文主题以及主题重写规则。""" + +rewrite.label: +"""主题重写""" + +tr_action.desc: +"""主题重写在哪种操作上生效： + - `subscribe`：订阅时重写主题； + - `publish`：发布时重写主题； + -`all`：全部重写主题""" + +tr_action.label: +"""Action""" + +tr_dest_topic.desc: +"""目标主题。""" + +tr_dest_topic.label: +"""目标主题""" + +tr_re.desc: +"""正则表达式""" + +tr_source_topic.desc: +"""源主题，客户端业务指定的主题""" + +tr_source_topic.label: +"""源主题""" + +} diff --git a/rel/i18n/zh/emqx_mqttsn_schema.hocon b/rel/i18n/zh/emqx_mqttsn_schema.hocon new file mode 100644 index 000000000..c6d3a98a6 --- /dev/null +++ b/rel/i18n/zh/emqx_mqttsn_schema.hocon @@ -0,0 +1,30 @@ +emqx_mqttsn_schema { + +mqttsn.desc: +"""MQTT-SN 网关配置。当前实现仅支持 v1.2 版本""" + +mqttsn_broadcast.desc: +"""是否周期性广播 ADVERTISE 消息""" + +mqttsn_enable_qos3.desc: +"""是否允许无连接的客户端发送 QoS 等于 -1 的消息。 +该功能主要用于支持轻量的 MQTT-SN 客户端实现，它不会向网关建立连接，注册主题，也不会发起订阅；它只使用 QoS 为 -1 来发布消息""" + +mqttsn_gateway_id.desc: +"""MQTT-SN 网关 ID。 +当 broadcast 打开时，MQTT-SN 网关会使用该 ID 来广播 ADVERTISE 消息""" + +mqttsn_predefined.desc: +"""预定义主题列表。 +预定义的主题列表，是一组 主题 ID 和 主题名称 的映射关系。使用预先定义的主题列表，可以减少 MQTT-SN 客户端和网关对于固定主题的注册请求""" + +mqttsn_predefined_id.desc: +"""主题 ID。范围：1-65535""" + +mqttsn_predefined_topic.desc: +"""主题名称。注：不支持通配符""" + +mqttsn_subs_resume.desc: +"""在会话被重用后，网关是否主动向客户端注册对已订阅主题名称""" + +} diff --git a/rel/i18n/zh/emqx_plugins_schema.hocon b/rel/i18n/zh/emqx_plugins_schema.hocon new file mode 100644 index 000000000..4da5c5a7b --- /dev/null +++ b/rel/i18n/zh/emqx_plugins_schema.hocon @@ -0,0 +1,46 @@ +emqx_plugins_schema { + +check_interval.desc: +"""检查间隔：检查集群中插件的状态是否一致，
+如果连续3次检查结果不一致，则报警。""" + +enable.desc: +"""设置为“true”以启用此插件""" + +enable.label: +"""启用""" + +install_dir.desc: +"""插件安装包的目录，出于安全考虑，该目录应该值允许 emqx，或用于运行 EMQX 服务的用户拥有写入权限。""" + +install_dir.label: +"""安装目录""" + +name_vsn.desc: +"""插件的名称{name}-{version}。
+它应该与插件的发布包名称一致，如my_plugin-0.1.0。""" + +name_vsn.label: +"""名称-版本""" + +plugins.desc: +"""管理EMQX插件。
+插件可以是EMQX安装包中的一部分，也可以是一个独立的安装包。
+独立安装的插件称为“外部插件”。""" + +plugins.label: +"""插件""" + +state.desc: +"""描述插件的状态""" + +state.label: +"""插件状态""" + +states.desc: +"""一组插件的状态。插件将按照定义的顺序启动""" + +states.label: +"""插件启动顺序及状态""" + +} diff --git a/rel/i18n/zh/emqx_prometheus_schema.hocon b/rel/i18n/zh/emqx_prometheus_schema.hocon new file mode 100644 index 000000000..a1e59e517 --- /dev/null +++ b/rel/i18n/zh/emqx_prometheus_schema.hocon @@ -0,0 +1,47 @@ +emqx_prometheus_schema { + +enable.desc: +"""开启或关闭 Prometheus 数据推送""" + +headers.desc: +"""推送到 Push Gateway 的 HTTP Headers 列表。
+例如， { Authorization = "some-authz-tokens"}""" + +interval.desc: +"""数据推送间隔""" + +job_name.desc: +"""推送到 Push Gateway 的 Job 名称。可用变量为：
+- ${name}: EMQX 节点的名称。 +- ${host}: EMQX 节点主机名。 +例如，当 EMQX 节点名为 emqx@127.0.0.1 则 name 变量的值为 emqx，host 变量的值为 127.0.0.1。
+默认值为: ${name}/instance/${name}~${host}""" + +mnesia_collector.desc: +"""开启或关闭 Mnesia 采集器, 使用 mnesia:system_info/1 收集 Mnesia 相关指标""" + +prometheus.desc: +"""Prometheus 监控数据推送""" + +prometheus.label: +"""Prometheus""" + +push_gateway_server.desc: +"""Prometheus 服务器地址""" + +vm_dist_collector.desc: +"""开启或关闭 VM 分布采集器，收集 Erlang 分布机制中涉及的套接字和进程的信息。""" + +vm_memory_collector.desc: +"""开启或关闭 VM 内存采集器, 使用 erlang:memory/0 收集 Erlang 虚拟机动态分配的内存信息，同时提供基本的 (D)ETS 统计信息""" + +vm_msacc_collector.desc: +"""开启或关闭 VM msacc 采集器, 使用 erlang:statistics(microstate_accounting) 收集微状态计数指标""" + +vm_statistics_collector.desc: +"""开启或关闭 VM 统计采集器, 使用 erlang:statistics/1 收集 Erlang VM 相关指标""" + +vm_system_info_collector.desc: +"""开启或关闭 VM 系统信息采集器, 使用 erlang:system_info/1 收集 Erlang VM 相关指标""" + +} diff --git a/rel/i18n/zh/emqx_psk_schema.hocon b/rel/i18n/zh/emqx_psk_schema.hocon new file mode 100644 index 000000000..0fc14e730 --- /dev/null +++ b/rel/i18n/zh/emqx_psk_schema.hocon @@ -0,0 +1,28 @@ +emqx_psk_schema { + +chunk_size.desc: +"""将 PSK 文件导入到内建数据时每个块的大小""" + +enable.desc: +"""是否开启 TLS PSK 支持""" + +init_file.desc: +"""如果设置了初始化文件，EMQX 将在启动时从初始化文件中导入 PSK 信息到内建数据库中。 +这个文件需要按行进行组织，每一行必须遵守如下格式: PSKIdentity:SharedSecret +例如: mydevice1:c2VjcmV0""" + +psk_authentication.desc: +"""此配置用于启用 TLS-PSK 身份验证。 + +PSK 是 “Pre-Shared-Keys” 的缩写。 + +注意: 确保 SSL 监听器仅启用了 'tlsv1.2'，并且配置了PSK 密码套件，例如 'RSA-PSK-AES256-GCM-SHA384'。 + +可以通过查看监听器中的 SSL 选项，了解更多详细信息。 + +可以通过配置 'init_file' 来设置初始化的 ID 和 密钥""" + +separator.desc: +"""PSK 文件中 PSKIdentity 和 SharedSecret 之间的分隔符""" + +} diff --git a/rel/i18n/zh/emqx_resource_schema.hocon b/rel/i18n/zh/emqx_resource_schema.hocon new file mode 100644 index 000000000..9365b1026 --- /dev/null +++ b/rel/i18n/zh/emqx_resource_schema.hocon @@ -0,0 +1,112 @@ +emqx_resource_schema { + +auto_restart_interval.desc: +"""资源断开以后，自动重连的时间间隔。""" + +auto_restart_interval.label: +"""自动重连间隔""" + +batch_size.desc: +"""最大批量请求大小。如果设为1，则无批处理。""" + +batch_size.label: +"""最大批量请求大小""" + +batch_time.desc: +"""在较低消息率情况下尝试累积批量输出时的最大等待间隔，以提高资源的利用率。""" + +batch_time.label: +"""批量等待最大间隔""" + +buffer_mode.desc: +"""队列操作模式。 +memory_only: 所有的消息都缓存在内存里。volatile_offload: 先将消息缓存在内存中，当内存中的消息堆积超过一定限制（配置项 buffer_seg_bytes 指定该限制）后， 消息会开始缓存到磁盘上。""" + +buffer_mode.label: +"""缓存模式""" + +buffer_seg_bytes.desc: +"""当缓存模式是 volatile_offload 时适用。该配置用于指定缓存到磁盘上的文件的大小。""" + +buffer_seg_bytes.label: +"""缓存文件大小""" + +creation_opts.desc: +"""资源启动相关的选项。""" + +creation_opts.label: +"""资源启动选项""" + +enable_batch.desc: +"""启用批量模式。""" + +enable_batch.label: +"""启用批量模式""" + +enable_queue.desc: +"""启用磁盘缓存队列（仅对 egress 方向桥接有用）。""" + +enable_queue.label: +"""启用磁盘缓存队列""" + +health_check_interval.desc: +"""健康检查间隔。""" + +health_check_interval.label: +"""健康检查间隔""" + +inflight_window.desc: +"""请求飞行队列窗口大小。当请求模式为异步时，如果需要严格保证来自同一 MQTT 客户端的消息有序，则必须将此值设为 1。""" + +inflight_window.label: +"""请求飞行队列窗口""" + +max_buffer_bytes.desc: +"""每个缓存 worker 允许使用的最大字节数。""" + +max_buffer_bytes.label: +"""缓存队列最大长度""" + +query_mode.desc: +"""请求模式。可选 '同步/异步'，默认为'异步'模式。""" + +query_mode.label: +"""请求模式""" + +request_timeout.desc: +"""从请求进入缓冲区开始计时，如果请求在规定的时间内仍停留在缓冲区内或者已发送但未能及时收到响应或确认，该请求将被视为过期。""" + +request_timeout.label: +"""请求超期""" + +resource_opts.desc: +"""资源相关的选项。""" + +resource_opts.label: +"""资源选项""" + +resume_interval.desc: +"""在发送失败后尝试重传飞行窗口中的请求的时间间隔。""" + +resume_interval.label: +"""重试时间间隔""" + +start_after_created.desc: +"""是否在创建资源后立即启动资源。""" + +start_after_created.label: +"""创建后立即启动""" + +start_timeout.desc: +"""在回复资源创建请求前等待资源进入健康状态的时间。""" + +start_timeout.label: +"""启动超时时间""" + +worker_pool_size.desc: +"""缓存队列 worker 数量。仅对 egress 类型的桥接有意义。当桥接仅有 ingress 方向时，可设置为 0，否则必须大于 0。""" + +worker_pool_size.label: +"""缓存池大小""" + +} diff --git a/rel/i18n/zh/emqx_retainer_api.hocon b/rel/i18n/zh/emqx_retainer_api.hocon new file mode 100644 index 000000000..f8107f8ce --- /dev/null +++ b/rel/i18n/zh/emqx_retainer_api.hocon @@ -0,0 +1,63 @@ +emqx_retainer_api { + +config_content.desc: +"""配置内容""" + +config_not_found.desc: +"""配置不存在""" + +delete_matching_api.desc: +"""删除对应的消息""" + +from_clientid.desc: +"""发布者的 ClientID""" + +from_username.desc: +"""发布者的用户名""" + +get_config_api.desc: +"""查看配置内容""" + +list_retained_api.desc: +"""查看保留消息列表""" + +lookup_api.desc: +"""通过不带通配符的主题查看对应的保留消息""" + +message_detail.desc: +"""消息详情""" + +message_not_exist.desc: +"""消息不存在""" + +msgid.desc: +"""消息 ID""" + +payload.desc: +"""消息内容""" + +publish_at.desc: +"""消息发送时间, RFC 3339 格式""" + +qos.desc: +"""QoS""" + +retained_list.desc: +"""保留消息列表""" + +topic.desc: +"""主题""" + +unsupported_backend.desc: +"""不支持的后端""" + +update_config_failed.desc: +"""配置更新失败""" + +update_config_success.desc: +"""配置更新成功""" + +update_retainer_api.desc: +"""更新配置""" + +} diff --git a/rel/i18n/zh/emqx_retainer_schema.hocon b/rel/i18n/zh/emqx_retainer_schema.hocon new file mode 100644 index 000000000..1e8630007 --- /dev/null +++ b/rel/i18n/zh/emqx_retainer_schema.hocon @@ -0,0 +1,46 @@ +emqx_retainer_schema { + +backend.desc: +"""保留消息的存储后端""" + +batch_deliver_limiter.desc: +"""批量发送的限流器的名称。 +限流器可以用来防止短时间内向客户端发送太多的消息，从而避免过多的消息导致客户端队列堵塞甚至崩溃。 +这个名称需要是指向 `limiter.batch` 下的一个真实存在的限流器。 +如果这个字段为空，则不使用限流器。""" + +batch_deliver_number.desc: +"""批量派发时每批的数量。0 代表一次性全部派发""" + +batch_read_number.desc: +"""从存储后端批量加载时的每批数量上限，0 代表一次性读取""" + +enable.desc: +"""是否开启消息保留功能""" + +flow_control.desc: +"""流控设置""" + +max_payload_size.desc: +"""消息大小最大值""" + +max_retained_messages.desc: +"""消息保留的数量上限。0 表示无限""" + +mnesia_config_storage_type.desc: +"""选择消息是存放在磁盘还是内存中""" + +mnesia_config_type.desc: +"""后端类型""" + +msg_clear_interval.desc: +"""消息清理间隔。0 代表不进行清理""" + +msg_expiry_interval.desc: +"""消息保留时间。0 代表永久保留""" + +stop_publish_clear_msg.desc: +"""是否不发送保留消息的清理消息，在 MQTT 5.0 中如果一条保留消息的消息体为空，则会清除掉之前存储 +的对应的保留消息，通过这个值控制是否停止发送清理消息""" + +} diff --git a/rel/i18n/zh/emqx_rewrite_api.hocon b/rel/i18n/zh/emqx_rewrite_api.hocon new file mode 100644 index 000000000..2be95d38b --- /dev/null +++ b/rel/i18n/zh/emqx_rewrite_api.hocon @@ -0,0 +1,12 @@ +emqx_rewrite_api { + +list_topic_rewrite_api.desc: +"""列出全部主题重写规则""" + +update_topic_rewrite_api.desc: +"""更新全部主题重写规则""" + +update_topic_rewrite_api_response413.desc: +"""超出主题重写规则数量上限""" + +} diff --git a/rel/i18n/zh/emqx_rule_api_schema.hocon b/rel/i18n/zh/emqx_rule_api_schema.hocon new file mode 100644 index 000000000..854f7707f --- /dev/null +++ b/rel/i18n/zh/emqx_rule_api_schema.hocon @@ -0,0 +1,381 @@ +emqx_rule_api_schema { + +event_action.desc: +"""订阅或发布""" + +event_action.label: +"""订阅或发布""" + +event_payload.desc: +"""消息负载""" + +event_payload.label: +"""消息负载""" + +metrics_actions_failed_out_of_service.desc: +"""由于输出停止服务而导致规则调用输出失败的次数。 例如，桥接被禁用或停止。""" + +metrics_actions_failed_out_of_service.label: +"""调用输出失败次数""" + +metrics_actions_failed_unknown.desc: +"""由于未知错误，规则调用输出失败的次数。""" + +metrics_actions_failed_unknown.label: +"""调用输出失败次数""" + +event_server.desc: +"""MQTT broker的 IP 地址（或主机名）和端口，采用 IP:Port 格式""" + +event_server.label: +"""服务器 IP 地址和端口""" + +metrics_actions_total.desc: +"""规则调用输出的次数。 该值可能是“sql.matched”的几倍，具体取决于规则输出的数量。""" + +metrics_actions_total.label: +"""调用输出次数""" + +event_ctx_disconnected_da.desc: +"""客户端断开连接的时刻""" + +event_ctx_disconnected_da.label: +"""客户端断开连接时刻""" + +event_topic.desc: +"""消息主题""" + +event_topic.label: +"""消息主题""" + +event_peername.desc: +"""对等客户端的 IP 地址和端口""" + +event_peername.label: +"""IP 地址和端口""" + +metrics_sql_passed.desc: +"""SQL 通过的次数""" + +metrics_sql_passed.label: +"""SQL 通过""" + +test_context.desc: +"""测试事件的上下文""" + +test_context.label: +"""事件上下文""" + +node_node.desc: +"""节点名字""" + +node_node.label: +"""节点名字""" + +event_from_clientid.desc: +"""事件来源客户端的 ID""" + +event_from_clientid.label: +"""客户端 ID""" + +event_keepalive.desc: +"""保持连接""" + +event_keepalive.label: +"""保持连接""" + +event_connected_at.desc: +"""客户端连接完成时的时刻""" + +event_connected_at.label: +"""连接完成时的时刻""" + +metrics_sql_failed_exception.desc: +"""SQL 由于执行异常而失败的次数。 这可能是因为调用 SQL 函数时崩溃，或者试图对未定义的变量进行算术运算""" + +metrics_sql_failed_exception.label: +"""SQL 执行异常""" + +event_from_username.desc: +"""事件来源客户端的用户名""" + +event_from_username.label: +"""用户名""" + +event_ctx_connack_reason_code.desc: +"""错误码""" + +event_ctx_connack_reason_code.label: +"""错误码""" + +rs_description.desc: +"""描述""" + +rs_description.label: +"""描述""" + +rule_id.desc: +"""规则的 ID""" + +rule_id.label: +"""规则 ID""" + +rs_event.desc: +"""事件主题""" + +rs_event.label: +"""事件主题""" + +root_rule_engine.desc: +"""规则引擎配置。该 API 可用于查看和修改规则引擎相关的一些设置。但不可用于规则，如需查看或修改规则，请调用 '/rules' API 进行操作。""" + +root_rule_engine.label: +"""规则引擎配置""" + +event_sockname.desc: +"""本地监听的 IP 地址和端口""" + +event_sockname.label: +"""IP 地址和端口""" + +event_qos.desc: +"""消息的 QoS""" + +event_qos.label: +"""消息 QoS""" + +event_mountpoint.desc: +"""挂载点""" + +event_mountpoint.label: +"""挂载点""" + +rs_title.desc: +"""标题""" + +rs_title.label: +"""标题""" + +ri_metrics.desc: +"""规则的计数器""" + +ri_metrics.label: +"""规则计数器""" + +event_retain.desc: +"""是否是保留消息""" + +event_retain.label: +"""保留消息""" + +event_event_type.desc: +"""事件类型""" + +event_event_type.label: +"""事件类型""" + +event_expiry_interval.desc: +"""到期间隔""" + +event_expiry_interval.label: +"""到期间隔""" + +metrics_sql_matched.desc: +"""SQL 的 FROM 子句匹配的次数。""" + +metrics_sql_matched.label: +"""命中数""" + +event_clientid.desc: +"""客户端 ID""" + +event_clientid.label: +"""客户端 ID""" + +metrics_actions_success.desc: +"""规则成功调用输出的次数。""" + +metrics_actions_success.label: +"""成功调用输出次数""" + +metrics_actions_failed.desc: +"""规则调用输出失败的次数。""" + +metrics_actions_failed.label: +"""调用输出失败次数""" + +metrics_sql_matched_rate.desc: +"""命中速率，次/秒""" + +metrics_sql_matched_rate.label: +"""Matched Rate""" + +event_proto_ver.desc: +"""协议版本""" + +event_proto_ver.label: +"""协议版本""" + +event_publish_received_at.desc: +"""消息被接受的时间""" + +event_publish_received_at.label: +"""消息被接受的时间""" + +metrics_sql_matched_rate_last5m.desc: +"""5分钟平均命中速率，次/秒""" + +metrics_sql_matched_rate_last5m.label: +"""平均命中速率""" + +event_is_bridge.desc: +"""是否桥接""" + +event_is_bridge.label: +"""是否桥接""" + +event_authz_source.desc: +"""缓存，插件或者默认值""" + +event_authz_source.label: +"""认证源""" + +metrics_sql_failed_unknown.desc: +"""由于未知错误导致 SQL 失败的次数。""" + +metrics_sql_failed_unknown.label: +"""SQL 未知错误""" + +metrics_sql_failed.desc: +"""SQL 失败的次数""" + +metrics_sql_failed.label: +"""SQL 失败""" + +event_ctx_dropped.desc: +"""消息被丢弃的原因""" + +event_ctx_dropped.label: +"""丢弃原因""" + +root_rule_test.desc: +"""用于规则测试的 Schema""" + +root_rule_test.label: +"""用于规则测试的 Schema""" + +rs_test_columns.desc: +"""测试列""" + +rs_test_columns.label: +"""测试列""" + +event_peerhost.desc: +"""对等客户端的 IP 地址""" + +event_peerhost.label: +"""对等客户端的 IP""" + +event_proto_name.desc: +"""协议名称""" + +event_proto_name.label: +"""协议名称""" + +root_rule_events.desc: +"""用于事件的 Schema""" + +root_rule_events.label: +"""用于规则事件的 Schema""" + +rs_sql_example.desc: +"""SQL 例子""" + +rs_sql_example.label: +"""SQL 例子""" + +metrics_sql_matched_rate_max.desc: +"""最大命中速率，次/秒""" + +metrics_sql_matched_rate_max.label: +"""最大命中速率""" + +event_clean_start.desc: +"""清除会话""" + +event_clean_start.label: +"""清除会话""" + +ri_created_at.desc: +"""规则创建时间""" + +ri_created_at.label: +"""规则创建时间""" + +event_dup.desc: +"""MQTT 消息的 DUP 标志""" + +event_dup.label: +"""DUP 标志""" + +ri_from.desc: +"""规则指定的主题""" + +ri_from.label: +"""规则指定的主题""" + +ri_node_metrics.desc: +"""每个节点的规则计数器""" + +ri_node_metrics.label: +"""每个节点规则计数器""" + +root_rule_creation.desc: +"""用于创建规则的 Schema""" + +root_rule_creation.label: +"""用于创建规则的 Schema""" + +event_result.desc: +"""允许或禁止""" + +event_result.label: +"""认证结果""" + +event_id.desc: +"""消息 ID""" + +event_id.label: +"""消息 ID""" + +event_username.desc: +"""用户名""" + +event_username.label: +"""用户名""" + +root_rule_info.desc: +"""用于规则信息的 Schema""" + +root_rule_info.label: +"""用于规则信息的 Schema""" + +rs_columns.desc: +"""列""" + +rs_columns.label: +"""列""" + +test_sql.desc: +"""测试的 SQL""" + +test_sql.label: +"""测试 SQL""" + +event_ctx_disconnected_reason.desc: +"""断开连接的原因""" + +event_ctx_disconnected_reason.label: +"""断开连接原因""" + +} diff --git a/rel/i18n/zh/emqx_rule_engine_api.hocon b/rel/i18n/zh/emqx_rule_engine_api.hocon new file mode 100644 index 000000000..eb1be4e73 --- /dev/null +++ b/rel/i18n/zh/emqx_rule_engine_api.hocon @@ -0,0 +1,93 @@ +emqx_rule_engine_api { + +api1.desc: +"""列出所有规则""" + +api1.label: +"""列出所有规则""" + +api10.desc: +"""更新规则引擎配置。""" + +api10.label: +"""更新配置""" + +api1_enable.desc: +"""根据规则是否开启条件过滤""" + +api1_from.desc: +"""根据规则来源 Topic 过滤, 需要完全匹配""" + +api1_like_description.desc: +"""根据规则描述过滤, 使用子串模糊匹配""" + +api1_like_from.desc: +"""根据规则来源 Topic 过滤, 使用子串模糊匹配""" + +api1_like_id.desc: +"""根据规则 id 过滤, 使用子串模糊匹配""" + +api1_match_from.desc: +"""根据规则来源 Topic 过滤, 使用 MQTT Topic 匹配""" + +api1_resp.desc: +"""规则列表""" + +api1_resp.label: +"""列出所有规则""" + +api2.desc: +"""通过指定 ID 创建规则""" + +api2.label: +"""通过指定 ID 创建规则""" + +api3.desc: +"""列出所有能被规则使用的事件""" + +api3.label: +"""列出所有能被规则使用的事件""" + +api4.desc: +"""通过 ID 查询规则""" + +api4.label: +"""查询规则""" + +api4_1.desc: +"""通过给定的 Id 获得规则的指标数据""" + +api4_1.label: +"""获得指标数据""" + +api5.desc: +"""通过 ID 更新集群里所有节点上的规则""" + +api5.label: +"""更新集群规则""" + +api6.desc: +"""通过 ID 删除集群里所有节点上的规则""" + +api6.label: +"""基于给定 ID 新建一条规则""" + +api7.desc: +"""重置规则计数""" + +api7.label: +"""重置规则计数""" + +api8.desc: +"""测试一个规则""" + +api8.label: +"""测试规则""" + +api9.desc: +"""获取规则引擎配置。""" + +api9.label: +"""获取配置""" + +} diff --git a/rel/i18n/zh/emqx_rule_engine_schema.hocon b/rel/i18n/zh/emqx_rule_engine_schema.hocon new file mode 100644 index 000000000..26858e10f --- /dev/null +++ b/rel/i18n/zh/emqx_rule_engine_schema.hocon @@ -0,0 +1,184 @@ +emqx_rule_engine_schema { + +console_function.desc: +"""将输出打印到控制台""" + +console_function.label: +"""控制台函数""" + +desc_builtin_action_console.desc: +"""配置打印到控制台""" + +desc_builtin_action_console.label: +"""配置打印到控制台""" + +desc_builtin_action_republish.desc: +"""配置重新发布。""" + +desc_builtin_action_republish.label: +"""配置重新发布""" + +desc_republish_args.desc: +"""内置 'republish' 动作的参数。 +可以在参数中使用变量。 +变量是规则中选择的字段。 例如规则 SQL 定义如下： + + SELECT clientid, qos, payload FROM "t/1" + +然后有 3 个变量可用：clientid、qos 和 payload。 如果我们将参数设置为： + + { + topic = "t/${clientid}" + qos = "${qos}" + payload = "msg: ${payload}" + } + +当收到一条消息 payload = `hello`, qos = 1, clientid = `Steve` 时，将重新发布一条新的 MQTT 消息到主题 `t/Steve` +消息内容为 payload = `msg: hello`, and `qos = 1""" + +desc_republish_args.label: +"""重新发布参数""" + +desc_rule_engine.desc: +"""配置 EMQX 规则引擎。""" + +desc_rule_engine.label: +"""配置规则引擎""" + +desc_rules.desc: +"""配置规则""" + +desc_rules.label: +"""配置规则""" + +desc_user_provided_function.desc: +"""配置用户函数""" + +desc_user_provided_function.label: +"""配置用户函数""" + +republish_args_payload.desc: +"""要重新发布的消息的有效负载。允许使用带有变量的模板，请参阅“republish_args”的描述。 +默认为 ${payload}。 如果从所选结果中未找到变量 ${payload}，则使用字符串 "undefined"。""" + +republish_args_payload.label: +"""消息负载""" + +republish_args_qos.desc: +"""要重新发布的消息的 qos。允许使用带有变量的模板，请参阅“republish_args”的描述。 +默认为 ${qos}。 如果从规则的选择结果中没有找到变量 ${qos}，则使用 0。""" + +republish_args_qos.label: +"""消息 QoS 等级""" + +republish_args_retain.desc: +"""要重新发布的消息的“保留”标志。允许使用带有变量的模板，请参阅“republish_args”的描述。 +默认为 ${retain}。 如果从所选结果中未找到变量 ${retain}，则使用 false。""" + +republish_args_retain.label: +"""保留消息标志""" + +republish_args_topic.desc: +"""重新发布消息的目标主题。 +允许使用带有变量的模板，请参阅“republish_args”的描述。""" + +republish_args_topic.label: +"""目标主题""" + +republish_args_user_properties.desc: +"""指定使用哪个变量来填充 MQTT 消息的 User-Property 列表。这个变量的值必须是一个 map 类型。 +可以设置成 ${pub_props.'User-Property'} 或者 +使用 SELECT *,pub_props.'User-Property' as user_properties 来把源 MQTT 消息 +的 User-Property 列表用于填充。 +也可以使用 map_put 函数来添加新的 User-Property， +map_put('my-prop-name', 'my-prop-value', user_properties) as user_properties +注意：MQTT 协议允许一个消息中出现多次同一个 property 名，但是 EMQX 的规则引擎不允许。""" + +republish_function.desc: +"""将消息重新发布为新的 MQTT 消息""" + +republish_function.label: +"""重新发布函数""" + +rule_engine_ignore_sys_message.desc: +"""当设置为“true”（默认）时，规则引擎将忽略发布到 $SYS 主题的消息。""" + +rule_engine_ignore_sys_message.label: +"""忽略系统消息""" + +rule_engine_jq_function_default_timeout.desc: +"""规则引擎内建函数 `jq` 默认时间限制""" + +rule_engine_jq_function_default_timeout.label: +"""规则引擎 jq 函数时间限制""" + +rule_engine_jq_implementation_module.desc: +"""jq 规则引擎功能的实现模块。可用的两个选项是 jq_nif 和 jq_port。jq_nif 使用 Erlang NIF 库访问 jq 库，而 jq_port 使用基于 Erlang Port 的实现。jq_nif 方式（默认选项）是这两个选项中最快的实现，但 jq_port 方式更安全，因为这种情况下 jq 程序不会在 Erlang VM 进程中执行。""" + +rule_engine_jq_implementation_module.label: +"""JQ 实现模块""" + +rule_engine_rules.desc: +"""规则""" + +rule_engine_rules.label: +"""规则""" + +rules_actions.desc: +"""规则的动作列表。 +动作可以是指向 EMQX bridge 的引用，也可以是一个指向函数的对象。 +我们支持一些内置函数，如“republish”和“console”，我们还支持用户提供的函数，它的格式为：“{module}:{function}”。 +列表中的动作按顺序执行。这意味着如果其中一个动作执行缓慢，则以下所有动作都不会被执行直到它返回。 +如果其中一个动作崩溃，在它之后的所有动作仍然会被按照原始顺序执行。 +如果运行动作时出现任何错误，则会出现错误消息，并且相应的计数器会增加。""" + +rules_actions.label: +"""动作列表""" + +rules_description.desc: +"""规则的描述""" + +rules_description.label: +"""规则描述""" + +rules_enable.desc: +"""启用或禁用规则引擎""" + +rules_enable.label: +"""启用或禁用规则引擎""" + +rules_metadata.desc: +"""规则的元数据，不要手动修改""" + +rules_metadata.label: +"""规则的元数据""" + +rules_name.desc: +"""规则名字""" + +rules_name.label: +"""规则名字""" + +rules_sql.desc: +"""用于处理消息的 SQL 。 +示例：SELECT * FROM "test/topic" WHERE payload.x = 1""" + +rules_sql.label: +"""规则 SQL""" + +user_provided_function_args.desc: +"""用户提供的参数将作为函数 module:function/3 的第三个参数， +请检查源文件：apps/emqx_rule_engine/src/emqx_rule_actions.erl 中的示例函数 console 和republish 。""" + +user_provided_function_args.label: +"""用户提供函数的参数""" + +user_provided_function_function.desc: +"""用户提供的函数。 格式应为：'{module}:{function}'。 +其中 {module} 是 Erlang 回调模块， {function} 是 Erlang 函数。 +要编写自己的函数，请检查源文件：apps/emqx_rule_engine/src/emqx_rule_actions.erl 中的示例函数 console 和republish 。""" + +user_provided_function_function.label: +"""用户提供的函数""" + +} diff --git a/rel/i18n/zh/emqx_schema.hocon b/rel/i18n/zh/emqx_schema.hocon new file mode 100644 index 000000000..3616abe91 --- /dev/null +++ b/rel/i18n/zh/emqx_schema.hocon @@ -0,0 +1,1473 @@ +emqx_schema { + +fields_mqtt_quic_listener_peer_unidi_stream_count.desc: +"""允许对端打开的单向流的数量""" + +fields_mqtt_quic_listener_peer_unidi_stream_count.label: +"""对端单向流的数量""" + +fields_authorization_no_match.desc: +"""如果用户或客户端不匹配ACL规则，或者从可配置授权源(比如内置数据库、HTTP API 或 PostgreSQL 等。)内未找 +到此类用户或客户端时，模式的认访问控制操作。 +在“授权”中查找更多详细信息。""" + +fields_authorization_no_match.label: +"""未匹时的默认授权动作""" + +sysmon_top_db_hostname.desc: +"""收集数据点的 PostgreSQL 数据库的主机名。""" + +sysmon_top_db_hostname.label: +"""数据库主机名""" + +zones.desc: +"""zone 是按name 分组的一组配置。 +对于灵活的配置映射，可以将 name 设置为侦听器的 zone 配置。 +注：名为 default 的内置区域是自动创建的，无法删除。""" + +fields_mqtt_quic_listener_certfile.desc: +"""证书文件。在 5.1 中会被废弃，使用 .ssl_options.certfile 代替。""" + +fields_mqtt_quic_listener_certfile.label: +"""证书文件""" + +fields_rate_limit_conn_bytes_in.desc: +"""限制 MQTT 连接接收数据包的速率。 速率以每秒的数据包字节数计算。""" + +fields_rate_limit_conn_bytes_in.label: +"""数据包速率""" + +crl_cache_capacity.desc: +"""缓存中可容纳的 CRL URL 的最大数量。 如果缓存的容量已满，并且必须获取一个新的 URL，那么它将驱逐缓存中插入的最老的 URL。""" + +crl_cache_capacity.label: +"""CRL 缓存容量""" + +alarm_actions.desc: +"""警报激活时触发的动作。
目前，支持以下操作：log 和 publish. +log 将告警写入日志 (控制台或者文件). +publish 将告警作为 MQTT 消息发布到系统主题: +$SYS/brokers/emqx@xx.xx.xx.x/alarms/activate and +$SYS/brokers/emqx@xx.xx.xx.x/alarms/deactivate""" + +alarm_actions.label: +"""告警动作""" + +base_listener_max_connections.desc: +"""监听器允许的最大并发连接数。""" + +base_listener_max_connections.label: +"""最大并发连接数""" + +mqtt_peer_cert_as_username.desc: +"""使用对端证书中的 CN、DN 字段或整个证书内容来作为用户名；仅适用于 TLS 连接。 +目前支持： +- cn: 取证书的 CN 字段 +- dn: 取证书的 DN 字段 +- crt: 取 DER 或 PEM 的证书内容 +- pem: 将 DER 证书转换为 PEM 格式作为用户名 +- md5: 取 DER 或 PEM 证书内容的 MD5 值""" + +mqtt_peer_cert_as_username.label: +"""对端证书作为用户名""" + +fields_cache_enable.desc: +"""启用或禁用授权缓存。""" + +fields_cache_enable.label: +"""启用或禁用授权缓存""" + +fields_mqtt_quic_listener_disconnect_timeout_ms.desc: +"""在判定路径无效和断开连接之前，要等待多长时间的ACK。默认：16000""" + +fields_mqtt_quic_listener_disconnect_timeout_ms.label: +"""断开连接超时 毫秒""" + +mqtt_max_topic_alias.desc: +"""允许的最大主题别名数，0 表示不支持主题别名。""" + +mqtt_max_topic_alias.label: +"""最大主题别名""" + +common_ssl_opts_schema_user_lookup_fun.desc: +"""用于查找预共享密钥（PSK）标识的 EMQX 内部回调。""" + +common_ssl_opts_schema_user_lookup_fun.label: +"""SSL PSK 用户回调""" + +fields_listeners_wss.desc: +"""HTTPS websocket 监听器。""" + +fields_listeners_wss.label: +"""HTTPS websocket 监听器""" + +sysmon_top_max_procs.desc: +"""当 VM 中的进程数超过此值时，停止收集数据。""" + +sysmon_top_max_procs.label: +"""最大进程数""" + +mqtt_use_username_as_clientid.desc: +"""是否使用用户名作为客户端 ID。 +此设置的作用时间晚于 对端证书作为用户名 和 对端证书作为客户端 ID。""" + +mqtt_use_username_as_clientid.label: +"""用户名作为客户端 ID""" + +mqtt_max_qos_allowed.desc: +"""允许的最大 QoS 等级。""" + +mqtt_max_qos_allowed.label: +"""最大 QoS""" + +fields_mqtt_quic_listener_max_binding_stateless_operations.desc: +"""在任何时候可以在一个绑定上排队的无状态操作的最大数量。默认值：100""" + +fields_mqtt_quic_listener_max_binding_stateless_operations.label: +"""最大绑定无状态操作""" + +fields_mqtt_quic_listener_stream_recv_buffer_default.desc: +"""流的初始缓冲区大小。默认：4096""" + +fields_mqtt_quic_listener_stream_recv_buffer_default.label: +"""流媒体接收缓冲区默认值""" + +fields_mqtt_quic_listener_pacing_enabled.desc: +"""有节奏的发送，以避免路径上的缓冲区过度填充。默认值：1（已启用）""" + +fields_mqtt_quic_listener_pacing_enabled.label: +"""启用节奏发送""" + +mqtt_max_subscriptions.desc: +"""允许每个客户端建立的最大订阅数量。""" + +mqtt_max_subscriptions.label: +"""最大订阅数量""" + +persistent_session_builtin_messages_table.desc: +"""用于内建消息表的性能调优参数。""" + +persistent_session_builtin_messages_table.label: +"""持久化消息""" + +sysmon_os_cpu_low_watermark.desc: +"""在解除相应警报之前可以使用多少系统 CPU 的阈值，以系统CPU负载的百分比表示。""" + +sysmon_os_cpu_low_watermark.label: +"""CPU 低水位线""" + +fields_mqtt_quic_listener_tls_server_max_send_buffer.desc: +"""缓冲多少TLS数据。 默认值：8192""" + +fields_mqtt_quic_listener_tls_server_max_send_buffer.label: +"""TLS 服务器最大发送缓冲区""" + +base_listener_bind.desc: +"""监听套接字的 IP 地址和端口。""" + +base_listener_bind.label: +"""IP 地址和端口""" + +server_ssl_opts_schema_handshake_timeout.desc: +"""握手完成所允许的最长时间""" + +server_ssl_opts_schema_handshake_timeout.label: +"""握手超时时间""" + +fields_deflate_opts_server_context_takeover.desc: +"""接管意味着在服务器消息之间保留压缩状态。""" + +fields_deflate_opts_server_context_takeover.label: +"""服务上下文接管""" + +mqtt_session_expiry_interval.desc: +"""指定会话将在连接断开后多久过期，仅适用于非 MQTT 5.0 的连接。""" + +mqtt_session_expiry_interval.label: +"""会话过期间隔""" + +fields_listener_enabled.desc: +"""启停监听器。""" + +fields_listener_enabled.label: +"""启停监听器""" + +mqtt.desc: +"""全局的 MQTT 配置项。 +mqtt 下所有的配置作为全局的默认值存在，它可以被 zone 中的配置覆盖。""" + +crl_cache_refresh_http_timeout.desc: +"""获取 CRLs 时 HTTP 请求的超时。 该配置对所有启用 CRL 检查的监听器监听器有效。""" + +crl_cache_refresh_http_timeout.label: +"""CRL 缓存刷新 HTTP 超时""" + +fields_tcp_opts_backlog.desc: +"""TCP backlog 定义了挂起连接队列可以增长到的最大长度。""" + +fields_tcp_opts_backlog.label: +"""TCP 连接队列长度""" + +broker_route_batch_clean.desc: +"""是否开启批量清除路由。""" + +fields_mqtt_quic_listener_initial_window_packets.desc: +"""一个连接的初始拥堵窗口的大小（以包为单位）。默认值：10""" + +fields_mqtt_quic_listener_initial_window_packets.label: +"""初始窗口数据包""" + +flapping_detect_enable.desc: +"""启用抖动检测功能。""" + +flapping_detect_enable.label: +"""启用抖动检测""" + +sysmon_top_db_password.desc: +"""PostgreSQL 数据库的密码""" + +sysmon_top_db_password.label: +"""数据库密码""" + +fields_ws_opts_check_origins.desc: +"""允许的 origins 列表""" + +fields_ws_opts_check_origins.label: +"""允许的 origins""" + +fields_deflate_opts_client_context_takeover.desc: +"""接管意味着在客户端消息之间保留压缩状态。""" + +fields_deflate_opts_client_context_takeover.label: +"""客户端上下文接管""" + +base_listener_acceptors.desc: +"""监听器接收池的大小。""" + +base_listener_acceptors.label: +"""接收器数量""" + +common_ssl_opts_schema_cacertfile.desc: +"""受信任的PEM格式 CA 证书捆绑文件
+此文件中的证书用于验证TLS对等方的证书。 +如果要信任新 CA，请将新证书附加到文件中。 +无需重启EMQX即可加载更新的文件，因为系统会定期检查文件是否已更新（并重新加载）
+注意：从文件中失效（删除）证书不会影响已建立的连接。""" + +common_ssl_opts_schema_cacertfile.label: +"""CA 证书文件""" + +fields_ws_opts_mqtt_path.desc: +"""WebSocket 的 MQTT 协议路径。因此，EMQX Broker的WebSocket地址为： +ws://{ip}:{port}/mqtt""" + +fields_ws_opts_mqtt_path.label: +"""WS MQTT 路径""" + +sysmon_os_procmem_high_watermark.desc: +"""在发出相应警报之前，一个Erlang进程可以分配多少系统内存的阈值，以系统内存的百分比表示。""" + +sysmon_os_procmem_high_watermark.label: +"""进程内存高水位线""" + +fields_listeners_quic.desc: +"""QUIC 监听器。""" + +fields_listeners_quic.label: +"""QUIC 监听器""" + +fields_listeners_ws.desc: +"""HTTP websocket 监听器。""" + +fields_listeners_ws.label: +"""HTTP websocket 监听器""" + +mqtt_retry_interval.desc: +"""QoS 1/2 消息的重新投递间隔。""" + +mqtt_retry_interval.label: +"""重试间隔""" + +stats_enable.desc: +"""启用/禁用统计数据收集功能。""" + +stats_enable.label: +"""启用/禁用统计数据收集功能""" + +fields_authorization_deny_action.desc: +"""授权检查拒绝操作时的操作。""" + +fields_authorization_deny_action.label: +"""授权检查拒绝操作时的操作""" + +fields_deflate_opts_server_max_window_bits.desc: +"""指定服务器压缩上下文的大小。""" + +fields_deflate_opts_server_max_window_bits.label: +"""服务器压缩窗口大小""" + +client_ssl_opts_schema_server_name_indication.desc: +"""指定要在 TLS 服务器名称指示扩展中使用的主机名。
+例如，当连接到 "server.example.net" 时，接受连接并执行 TLS 握手的真正服务器可能与 TLS 客户端最初连接到的主机不同， +例如，当连接到 IP 地址时，或者当主机具有多个可解析的 DNS 记录时
+如果未指定，它将默认为使用的主机名字符串 +建立连接，除非使用 IP 地址
+然后，主机名也用于对等机的主机名验证证书
+特殊值 disable 阻止发送服务器名称指示扩展，并禁用主机名验证检查。""" + +client_ssl_opts_schema_server_name_indication.label: +"""服务器名称指示""" + +fields_mqtt_quic_listener_retry_memory_limit.desc: +"""在使用无状态重试之前，可用于握手连接的可用内存的百分比。计算为`N/65535`。默认值：65""" + +fields_mqtt_quic_listener_retry_memory_limit.label: +"""重试内存限制""" + +force_shutdown_max_message_queue_len.desc: +"""消息队列的最大长度。""" + +force_shutdown_max_message_queue_len.label: +"""进程邮箱消息队列的最大长度""" + +sys_heartbeat_interval.desc: +"""发送心跳系统消息的间隔时间，它包括： + - `$SYS/brokers//uptime` + - `$SYS/brokers//datetime`""" + +flapping_detect_ban_time.desc: +"""抖动的客户端将会被禁止登录多长时间。""" + +flapping_detect_ban_time.label: +"""禁止登录时长""" + +sysmon_top_num_items.desc: +"""每个监视组的顶级进程数。""" + +sysmon_top_num_items.label: +"""顶级进程数""" + +persistent_session_builtin_session_table.desc: +"""用于内建会话表的性能调优参数。""" + +persistent_session_builtin_session_table.label: +"""持久会话""" + +mqtt_upgrade_qos.desc: +"""投递消息时，是否根据订阅主题时的 QoS 等级来强制提升派发的消息的 QoS 等级。""" + +mqtt_upgrade_qos.label: +"""升级 QoS""" + +mqtt_shared_subscription.desc: +"""是否启用对 MQTT 共享订阅的支持。""" + +mqtt_shared_subscription.label: +"""共享订阅可用""" + +fields_tcp_opts_sndbuf.desc: +"""连接的 TCP 发送缓冲区（OS 内核）。""" + +fields_tcp_opts_sndbuf.label: +"""TCP 发送缓冲区""" + +sysmon_os_mem_check_interval.desc: +"""定期内存检查的时间间隔。""" + +sysmon_os_mem_check_interval.label: +"""内存检查间隔""" + +server_ssl_opts_schema_gc_after_handshake.desc: +"""内存使用调优。如果启用，将在TLS/SSL握手完成后立即执行垃圾回收。TLS/SSL握手建立后立即进行GC。""" + +server_ssl_opts_schema_gc_after_handshake.label: +"""握手后执行GC""" + +fields_mqtt_quic_listener_ssl_options.desc: +"""QUIC 传输层的 TLS 选项""" + +fields_mqtt_quic_listener_ssl_options.label: +"""TLS 选项""" + +fields_ws_opts_mqtt_piggyback.desc: +"""WebSocket消息是否允许包含多个 MQTT 数据包。""" + +fields_ws_opts_mqtt_piggyback.label: +"""MQTT Piggyback""" + +base_listener_mountpoint.desc: +"""发布或订阅时，请在所有主题前面加上 mountpoint 字符串。 + +将消息传递给订阅者时，将从主题名称中删除带前缀的字符串。挂载点是一种用户可以用来实现不同侦听器之间消息路由隔离的方法。 + +例如，如果客户机 A 使用 listeners.tcp.\.mountpoint 设置为'some_tenant'，那么客户端实际上订阅了主题'some_tenant/t'。
+类似地，如果另一个客户端B（与客户端A连接到同一个侦听器）向主题 't' 发送消息，该消息将路由到所有订阅了'some_租户/t'的客户端，因此客户端 A 将接收主题名为't'的消息
+ +设置为"" 以禁用该功能
+ +mountpoint 字符串中的变量： +- ${clientid}: clientid +- ${username}: username""" + +base_listener_mountpoint.label: +"""mountpoint""" + +mqtt_max_awaiting_rel.desc: +"""每个发布者的会话中，都存在一个队列来处理客户端发送的 QoS 2 消息。该队列会存储 QoS 2 消息的报文 ID 直到收到客户端的 PUBREL 或超时，达到队列长度的限制后，新的 QoS 2 消息发布会被拒绝，并返回 `147(0x93)` 错误。""" + +mqtt_max_awaiting_rel.label: +"""PUBREL 等待队列长度""" + +ciphers_schema_quic.desc: +"""此配置保存由逗号分隔的 TLS 密码套件名称，或作为字符串数组。例如 +"TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256"或 +["TLS_AES_256_GCM_SHA384","TLS_AES_128_GCM_SHA256"]。 +
+密码（及其顺序）定义了客户端和服务器通过网络连接加密信息的方式。 +选择一个好的密码套件对于应用程序的数据安全性、机密性和性能至关重要。 + +名称应为 OpenSSL 字符串格式（而不是 RFC 格式）。 +EMQX 配置文档提供的所有默认值和示例都是 OpenSSL 格式
+注意：某些密码套件仅与特定的 TLS 版本兼容（'tlsv1.1'、'tlsv1.2'或'tlsv1.3'）。 +不兼容的密码套件将被自动删除。 + +例如，如果只有 versions 仅配置为 tlsv1.3。为其他版本配置密码套件将无效。 + +
+注：PSK 的 Ciphers 不支持 tlsv1.3
+如果打算使用PSK密码套件，tlsv1.3。应在ssl.versions中禁用。 + +
+PSK 密码套件： +"RSA-PSK-AES256-GCM-SHA384,RSA-PSK-AES256-CBC-SHA384, +RSA-PSK-AES128-GCM-SHA256,RSA-PSK-AES128-CBC-SHA256, +RSA-PSK-AES256-CBC-SHA,RSA-PSK-AES128-CBC-SHA, +RSA-PSK-DES-CBC3-SHA,RSA-PSK-RC4-SHA"
+ +注：QUIC 监听器不支持 tlsv1.3 的 ciphers""" + +ciphers_schema_quic.label: +"""""" + +fields_mqtt_quic_listener_max_bytes_per_key.desc: +"""在启动密钥更新之前，用单个 1-RTT 加密密钥加密的最大字节数。默认值：274877906944""" + +fields_mqtt_quic_listener_max_bytes_per_key.label: +"""每个密钥的最大字节数""" + +fields_mqtt_quic_listener_mtu_discovery_search_complete_timeout_us.desc: +"""如果没有达到 max ，在重新尝试 MTU 探测之前要等待的时间，单位是微秒。默认值：600000000""" + +fields_mqtt_quic_listener_mtu_discovery_search_complete_timeout_us.label: +"""""" + +fields_ws_opts_check_origin_enable.desc: +"""如果true，originHTTP 头将根据check_origins参数中配置的允许来源列表进行验证。""" + +fields_ws_opts_check_origin_enable.label: +"""检查 origin""" + +sysmon_vm_busy_dist_port.desc: +"""启用后，当用于集群接点之间 RPC 的连接过忙时，会触发一条带有 busy_dist_port 关键字的 warning 级别日志。 +同时还会发布一条主题为 $SYS/sysmon/busy_dist_port 的 MQTT 系统消息。""" + +sysmon_vm_busy_dist_port.label: +"""启用分布式端口过忙监控""" + +mqtt_max_mqueue_len.desc: +"""消息队列最大长度。持久客户端断开连接或飞行窗口已满时排队的消息长度。""" + +mqtt_max_mqueue_len.label: +"""最大消息队列长度""" + +mqtt_max_inflight.desc: +"""允许在完成应答前同时投递的 QoS 1 和 QoS 2 消息的最大数量。""" + +mqtt_max_inflight.label: +"""最大飞行窗口""" + +persistent_session_store_enabled.desc: +"""使用数据库存储有关持久会话的信息。 +这使得在节点停止时，可以将客户端连接迁移到另一个群集节点。""" + +persistent_session_store_enabled.label: +"""启用持久会话保存""" + +fields_deflate_opts_level.desc: +"""压缩级别""" + +fields_deflate_opts_level.label: +"""压缩级别""" + +mqtt_server_keepalive.desc: +"""EMQX 要求的保活时间，如设为 disabled，则将使用客户端指定的保持连接时间；仅适用于 MQTT 5.0 客户端。""" + +mqtt_server_keepalive.label: +"""服务端保活时间""" + +global_authentication.desc: +"""全局 MQTT 监听器的默认认证配置。 为每个监听器配置认证参考监听器器配置中的authentication 配置。 + +该配置可以被配置为： +
    +
  • []: 默认值，允许所有的登录请求 +
  • 配置为单认证器，例如 {enable:true,backend:"built_in_database",mechanism="password_based"}
    • +
  • 配置为认证器数组
    • +
+ +当配置为认证链后，登录凭证会按照配置的顺序进行检查，直到做出allow 或 deny的结果。 + +如果在所有的认证器都执行完后，还是没有结果，登录将被拒绝。""" + +fields_mqtt_quic_listener_load_balancing_mode.desc: +"""0: 禁用, 1: SERVER_ID_IP, 2: SERVER_ID_FIXED. 默认: 0""" + +fields_mqtt_quic_listener_load_balancing_mode.label: +"""负载平衡模式""" + +persistent_session_store_session_message_gc_interval.desc: +"""持久会话消息的临时数据垃圾收集的开始间隔。 +这不会影响持久会话消息的生命周期长度。""" + +persistent_session_store_session_message_gc_interval.label: +"""会话消息清理间隔""" + +server_ssl_opts_schema_ocsp_refresh_http_timeout.desc: +"""检查 OCSP 响应时，HTTP 请求的超时。""" + +server_ssl_opts_schema_ocsp_refresh_http_timeout.label: +"""OCSP 刷新 HTTP 超时""" + +fields_tcp_opts_send_timeout.desc: +"""连接的 TCP 发送超时。""" + +fields_tcp_opts_send_timeout.label: +"""TCP 发送超时""" + +sysmon_vm_process_high_watermark.desc: +"""在发出相应警报之前，本地节点上可以同时存在多少进程的阈值（以进程百分比表示）。""" + +sysmon_vm_process_high_watermark.label: +"""进程数高水位线""" + +fields_tcp_opts_buffer.desc: +"""驱动程序使用的用户空间缓冲区的大小。""" + +fields_tcp_opts_buffer.label: +"""TCP 用户态缓冲区""" + +server_ssl_opts_schema_honor_cipher_order.desc: +"""一个重要的安全设置，它强制根据服务器指定的顺序而不是客户机指定的顺序设置密码，从而强制服务器管理员执行（通常配置得更正确）安全顺序。""" + +server_ssl_opts_schema_honor_cipher_order.label: +"""SSL honor cipher order""" + +conn_congestion_min_alarm_sustain_duration.desc: +"""清除警报前的最短时间。
只有当队列中没有挂起的数据，并且连接至少被堵塞了 min_alarm_sustain_duration 毫秒时，
报警才会被清除。这是为了避免太频繁地清除和再次发出警报。""" + +conn_congestion_min_alarm_sustain_duration.label: +"""告警维持时间""" + +fields_mqtt_quic_listener_keep_alive_interval_ms.desc: +"""多长时间发送一次PING帧以保活连接。""" + +fields_mqtt_quic_listener_keep_alive_interval_ms.label: +"""保持活着的时间间隔 毫秒""" + +fields_mqtt_quic_listener_handshake_idle_timeout_ms.desc: +"""一个握手在被丢弃之前可以空闲多长时间""" + +fields_mqtt_quic_listener_handshake_idle_timeout_ms.label: +"""握手空闲超时 毫秒""" + +broker_session_locking_strategy.desc: +"""Session 在集群中的锁策略。 + - `loca`：仅锁本节点的 Session； + - `one`：任选一个其它节点加锁； + - `quorum`：选择集群中半数以上的节点加锁； + - `all`：选择所有节点加锁。""" + +persistent_store_ram_cache.desc: +"""在内存中保持一份数据的副本，以便更快地访问。""" + +persistent_store_ram_cache.label: +"""内存缓存""" + +fields_mqtt_quic_listener_stream_recv_window_default.desc: +"""初始流接收窗口大小。 默认值：32678""" + +fields_mqtt_quic_listener_stream_recv_window_default.label: +"""流接收窗口默认""" + +mqtt_mqueue_priorities.desc: +"""主题优先级。取值范围 [1-255] +默认优先级表为空，即所有的主题优先级相同。 + +注：优先主题名称中不支持使用逗号和等号。 +注：不在此列表中的主题，被视为最高/最低优先级，这取决于mqtt.mqueue_default_priority 的配置 + +示例： +配置 "topic/1" > "topic/2": +mqueue_priorities: {"topic/1": 10, "topic/2": 8}""" + +mqtt_mqueue_priorities.label: +"""主题优先级""" + +fields_rate_limit_conn_messages_in.desc: +"""外部 MQTT 连接的消息限制。""" + +fields_rate_limit_conn_messages_in.label: +"""外部 MQTT 连接的消息限制""" + +fields_rate_limit_max_conn_rate.desc: +"""每秒最大连接数。""" + +fields_rate_limit_max_conn_rate.label: +"""每秒最大连接数""" + +alarm_size_limit.desc: +"""要保留为历史记录的已停用报警的最大总数。当超过此限制时，将删除最旧的停用报警，以限制总数。""" + +alarm_size_limit.label: +"""告警总数限制""" + +fields_cache_max_size.desc: +"""缓存项的最大数量。""" + +fields_cache_max_size.label: +"""缓存项的最大数量""" + +fields_listeners_tcp.desc: +"""TCP 监听器。""" + +fields_listeners_tcp.label: +"""TCP 监听器""" + +conn_congestion_enable_alarm.desc: +"""启用或者禁用连接阻塞告警功能。""" + +conn_congestion_enable_alarm.label: +"""启用/禁用阻塞告警""" + +fields_ws_opts_proxy_port_header.desc: +"""HTTP 头，用于传递有关客户端端口的信息。当 EMQX 集群部署在负载平衡器后面时，这一点非常重要。""" + +fields_ws_opts_proxy_port_header.label: +"""客户端端口头""" + +overload_protection_enable.desc: +"""是否对系统过载做出反应。""" + +overload_protection_enable.label: +"""是否对系统过载做出反应""" + +fields_mqtt_quic_listener_minimum_mtu.desc: +"""一个连接所支持的最小MTU。这将被作为起始MTU使用。默认值：1248""" + +fields_mqtt_quic_listener_minimum_mtu.label: +"""最小 MTU""" + +sys_msg_interval.desc: +"""发送 `$SYS` 主题的间隔时间。""" + +mqtt_await_rel_timeout.desc: +"""客户端发布 QoS 2 消息时，服务器等待 `PUBREL` 的最长时延。超过该时长后服务器会放弃等待，该PACKET ID 会被释放，从而允许后续新的 PUBLISH 消息使用。如果超时后收到 PUBREL，服务器将会产生一条告警日志。注意，向订阅客户端转发消息的动作发生在进入等待之前。""" + +mqtt_await_rel_timeout.label: +"""PUBREL 最大等待时间""" + +common_ssl_opts_schema_verify.desc: +"""启用或禁用对等验证。""" + +common_ssl_opts_schema_verify.label: +"""对等验证""" + +fields_listeners_ssl.desc: +"""SSL 监听器。""" + +fields_listeners_ssl.label: +"""SSL 监听器""" + +fields_deflate_opts_client_max_window_bits.desc: +"""指定客户端压缩上下文的大小。""" + +fields_deflate_opts_client_max_window_bits.label: +"""压缩窗口大小""" + +common_ssl_opts_schema_keyfile.desc: +"""PEM格式的私钥文件。""" + +common_ssl_opts_schema_keyfile.label: +"""私钥文件""" + +sysmon_os_cpu_high_watermark.desc: +"""在发出相应警报之前可以使用多少系统 CPU 的阈值，以系统CPU负载的百分比表示。""" + +sysmon_os_cpu_high_watermark.label: +"""CPU 高水位线""" + +flapping_detect_window_time.desc: +"""抖动检测的时间窗口。""" + +flapping_detect_window_time.label: +"""时间窗口""" + +mqtt_mqueue_default_priority.desc: +"""默认的主题优先级，不在 主题优先级（mqueue_priorities） 中的主题将会使用该优先级。""" + +mqtt_mqueue_default_priority.label: +"""默认主题优先级""" + +client_ssl_opts_schema_enable.desc: +"""启用 TLS。""" + +client_ssl_opts_schema_enable.label: +"""启用 TLS""" + +fields_mqtt_quic_listener_mtu_discovery_missing_probe_count.desc: +"""在任何时候都可以在一个绑定上排队的无状态操作的最大数量。默认值：3""" + +fields_mqtt_quic_listener_mtu_discovery_missing_probe_count.label: +"""MTU发现丢失的探针数量""" + +fields_tcp_opts_recbuf.desc: +"""连接的 TCP 接收缓冲区（OS 内核）。""" + +fields_tcp_opts_recbuf.label: +"""TCP 接收缓冲区""" + +sysmon_vm_process_check_interval.desc: +"""定期进程限制检查的时间间隔。""" + +sysmon_vm_process_check_interval.label: +"""进程限制检查时间""" + +fields_mqtt_quic_listener_server_resumption_level.desc: +"""连接恢复 和/或 0-RTT 服务器支持。默认值：0（无恢复功能）""" + +fields_mqtt_quic_listener_server_resumption_level.label: +"""服务端连接恢复支持""" + +fields_ws_opts_proxy_address_header.desc: +"""HTTP 头，用于传递有关客户端 IP 地址的信息。 +当 EMQX 集群部署在负载平衡器后面时，这一点非常重要。""" + +fields_ws_opts_proxy_address_header.label: +"""客户端地址头""" + +sysmon_os_sysmem_high_watermark.desc: +"""在发出相应报警之前可以分配多少系统内存的阈值，以系统内存的百分比表示。""" + +sysmon_os_sysmem_high_watermark.label: +"""系统内存高水位线""" + +fields_tcp_opts_high_watermark.desc: +"""当 VM 套接字实现内部排队的数据量达到此限制时，套接字将设置为忙碌状态。""" + +fields_tcp_opts_high_watermark.label: +"""""" + +fields_mqtt_quic_listener_stateless_operation_expiration_ms.desc: +"""同一个对端的操作之间的时间限制，单位是毫秒。 默认：100""" + +fields_mqtt_quic_listener_stateless_operation_expiration_ms.label: +"""无状态操作过期 毫秒""" + +server_ssl_opts_schema_dhfile.desc: +"""如果协商使用Diffie-Hellman密钥交换的密码套件，则服务器将使用包含PEM编码的Diffie-Hellman参数的文件的路径。如果未指定，则使用默认参数。
+注意：TLS 1.3不支持dhfile选项。""" + +server_ssl_opts_schema_dhfile.label: +"""SSL dhfile""" + +flapping_detect_max_count.desc: +"""MQTT 客户端在“窗口”时间内允许的最大断开次数。""" + +flapping_detect_max_count.label: +"""最大断开次数""" + +mqtt_max_topic_levels.desc: +"""允许的最大主题层级。""" + +mqtt_max_topic_levels.label: +"""最大主题层级""" + +force_shutdown_max_heap_size.desc: +"""Heap 的总大小。""" + +force_shutdown_max_heap_size.label: +"""Heap 的总大小""" + +persistent_store_on_disc.desc: +"""将持久会话数据保存在磁盘上。如果为 false 则存储在内存中。 +如开启， 持久会话数据可在集群重启后恢复。 +如关闭， 数据仅存储在内存中， 则在整个集群停止后丢失。""" + +persistent_store_on_disc.label: +"""持久化在磁盘上""" + +mqtt_ignore_loop_deliver.desc: +"""设置由 MQTT v3.1.1/v3.1.0 客户端发布的消息是否将转发给其本身；类似 MQTT 5.0 协议中的 No Local 选项。""" + +mqtt_ignore_loop_deliver.label: +"""忽略循环投递""" + +common_ssl_opts_schema_certfile.desc: +"""PEM格式证书链文件
+此文件中的证书应与证书颁发链的顺序相反。也就是说，主机的证书应该放在文件的开头， +然后是直接颁发者 CA 证书，依此类推，一直到根 CA 证书。 +根 CA 证书是可选的，如果想要添加，应加到文件到最末端。""" + +common_ssl_opts_schema_certfile.label: +"""证书文件""" + +mqtt_exclusive_subscription.desc: +"""是否启用对 MQTT 排它订阅的支持。""" + +mqtt_exclusive_subscription.label: +"""排它订阅""" + +mqtt_retain_available.desc: +"""是否启用对 MQTT 保留消息的支持。""" + +mqtt_retain_available.label: +"""保留消息可用""" + +fields_tcp_opts_reuseaddr.desc: +"""连接的 SO_REUSEADDR 标识。""" + +fields_tcp_opts_reuseaddr.label: +"""SO_REUSEADDR""" + +sysmon_vm_long_schedule.desc: +"""启用后，如果 Erlang VM 调度器出现某个任务占用时间过长时，会触发一条带有 'long_schedule' 关键字的日志。 +同时还会发布一条主题为 $SYS/sysmon/long_schedule 的 MQTT 系统消息。""" + +sysmon_vm_long_schedule.label: +"""启用长调度监控""" + +mqtt_keepalive_backoff.desc: +"""EMQX 判定客户端保活超时使用的阈值系数。计算公式为：Keep Alive * Backoff * 2""" + +mqtt_keepalive_backoff.label: +"""保活超时阈值系数""" + +force_gc_bytes.desc: +"""在进程处理过多少个字节之后，对此进程执行垃圾回收。""" + +force_gc_bytes.label: +"""垃圾回收字节数""" + +server_ssl_opts_schema_fail_if_no_peer_cert.desc: +"""TLS/DTLS 服务器与 {verify，verify_peer} 一起使用。 +如果设置为true，则如果客户端没有要发送的证书，即发送空证书，服务器将失败。 +如果设置为false，则仅当客户端发送无效证书（空证书被视为有效证书）时才会失败。""" + +server_ssl_opts_schema_fail_if_no_peer_cert.label: +"""没有证书则 SSL 失败""" + +fields_ws_opts_compress.desc: +"""如果 true，则使用zlib 压缩 WebSocket 消息
+deflate_opts 下的配置项属于压缩相关参数配置。""" + +fields_ws_opts_compress.label: +"""Ws 压缩""" + +fields_mqtt_quic_listener_keep_alive_interval.desc: +"""发送 PING 帧的频率，以保活连接. 设为 0 表示禁用。""" + +fields_mqtt_quic_listener_keep_alive_interval.label: +"""PING 保活频率""" + +fields_cache_ttl.desc: +"""缓存数据的生存时间。""" + +fields_cache_ttl.label: +"""缓存数据的生存时间。""" + +sys_topics.desc: +"""系统主题配置。""" + +sys_event_client_subscribed.desc: +"""是否开启客户端已成功订阅主题事件消息。""" + +sysmon_top_db_port.desc: +"""收集数据点的 PostgreSQL 数据库的端口。""" + +sysmon_top_db_port.label: +"""数据库端口""" + +fields_mqtt_quic_listener_max_operations_per_drain.desc: +"""每个连接操作的最大耗费操作数。默认：16""" + +fields_mqtt_quic_listener_max_operations_per_drain.label: +"""每次操作最大操作数""" + +fields_mqtt_quic_listener_datagram_receive_enabled.desc: +"""宣传对QUIC Datagram 扩展的支持。为将来保留。默认为0（FALSE）""" + +fields_mqtt_quic_listener_datagram_receive_enabled.label: +"""启用 Datagram 接收""" + +fields_mqtt_quic_listener_initial_rtt_ms.desc: +"""初始RTT估计""" + +fields_mqtt_quic_listener_initial_rtt_ms.label: +"""Initial RTT 毫秒""" + +overload_protection_backoff_gc.desc: +"""高负载时，跳过强制 GC。""" + +overload_protection_backoff_gc.label: +"""跳过GC""" + +broker_perf_route_lock_type.desc: +"""通配主题订阅/取消订阅性能调优。 +建议仅当通配符主题较多时才更改此参数。 + +注：当从/更改为 `global` 锁时，它要求集群中的所有节点在更改之前停止。 + - `key`：为 Mnesia 事务涉及到的每个 key 上锁，建议单节点时使用。 + - `tab`：为 Mnesia 事务涉及到的表上锁，建议在集群中使用。 + - `global`：所以更新操作都被全局的锁保护，仅建议在超大规模集群中使用。""" + +fields_tcp_opts_nodelay.desc: +"""连接的 TCP_NODELAY 标识""" + +fields_tcp_opts_nodelay.label: +"""TCP_NODELAY""" + +sysmon_top_db_username.desc: +"""PostgreSQL 数据库的用户名""" + +sysmon_top_db_username.label: +"""数据库用户名""" + +broker.desc: +"""Broker 相关配置项。""" + +force_gc_count.desc: +"""在进程收到多少消息之后，对此进程执行垃圾回收。""" + +force_gc_count.label: +"""垃圾回收消息数""" + +mqtt_max_clientid_len.desc: +"""允许的最大 MQTT Client ID 长度。""" + +mqtt_max_clientid_len.label: +"""最大 Client ID 长度""" + +fields_ws_opts_supported_subprotocols.desc: +"""逗号分隔的 subprotocols 支持列表。""" + +fields_ws_opts_supported_subprotocols.label: +"""Subprotocols 支持列表""" + +broker_shared_subscription_strategy.desc: +"""共享订阅消息派发策略。 + - `random`：随机挑选一个共享订阅者派发； + - `round_robin`：使用 round-robin 策略派发； + - `round_robin_per_group`：在共享组内循环选择下一个成员； + - `local`：选择随机的本地成员，否则选择随机的集群范围内成员; + - `sticky`：总是使用上次选中的订阅者派发，直到它断开连接； + - `hash_clientid`：通过对发送者的客户端 ID 进行 Hash 处理来选择订阅者; + - `hash_topic`：通过对源主题进行 Hash 处理来选择订阅者。""" + +fields_deflate_opts_mem_level.desc: +"""指定压缩状态的大小
+较低的值会减少每个连接的内存使用。""" + +fields_deflate_opts_mem_level.label: +"""压缩状态大小""" + +fields_mqtt_quic_listener_send_idle_timeout_ms.desc: +"""在闲置一定时间后重置拥堵控制。默认值：1000""" + +fields_mqtt_quic_listener_send_idle_timeout_ms.label: +"""发送空闲超时毫秒""" + +base_listener_limiter.desc: +"""速率限制类型""" + +base_listener_limiter.label: +"""速率限制类型""" + +persistent_session_store_backend.desc: +"""用于存储持久性会话和信息的数据库管理后端 +- `builtin`: 使用内置的数据库（mria）""" + +persistent_session_store_backend.label: +"""后端类型""" + +alarm_validity_period.desc: +"""停用报警的保留时间。报警在停用时不会立即删除，而是在保留时间之后删除。""" + +alarm_validity_period.label: +"""告警保留时间""" + +server_ssl_opts_schema_ocsp_issuer_pem.desc: +"""服务器证书的 OCSP 签发者的 PEM 编码证书。""" + +server_ssl_opts_schema_ocsp_issuer_pem.label: +"""OCSP 签发者证书""" + +fields_tcp_opts_active_n.desc: +"""为此套接字指定{active，N}选项
+See: https://erlang.org/doc/man/inet.html#setopts-2""" + +fields_tcp_opts_active_n.label: +"""active_n""" + +listener_authentication.desc: +"""监听器认证重载。 +认证配置可以是单个认证器实例，也可以是一个认证器数组组成的认证链。 +执行登录验证时（用户名、客户端 ID 等），将按配置的顺序执行。""" + +listener_authentication.label: +"""每个监听器的认证覆盖""" + +fields_trace_payload_encode.desc: +"""确定跟踪文件中有效负载格式的格式。
+`text`：基于文本的协议或纯文本协议。 +建议在有效负载为JSON编码时使用
+`hex`：二进制十六进制编码。当有效负载是自定义二进制协议时，建议使用此选项
+`hidden`：有效负载被模糊化为 `******`""" + +fields_trace_payload_encode.label: +"""有效负载编码""" + +mqtt_response_information.desc: +"""UTF-8 字符串，用于指定返回给客户端的响应主题，如 reqrsp/，此时请求和应答客户端都需要使用 reqrsp/ 前缀的主题来完成通讯。 +如希望禁用此功能，请在下方的文字框中输入""；仅适用于 MQTT 5.0 客户端。""" + +mqtt_response_information.label: +"""响应信息""" + +persistent_session_store_max_retain_undelivered.desc: +"""如果重新启动时处理上一个会话的节点已停止，则未传递到持久会话的消息在垃圾收集之前会被存储。""" + +persistent_session_store_max_retain_undelivered.label: +"""未投递的消息保留条数""" + +fields_mqtt_quic_listener_migration_enabled.desc: +"""开启客户端地址迁移功能。需要一个支持的负载平衡器，或者没有负载平衡器。默认值：1（已启用）""" + +fields_mqtt_quic_listener_migration_enabled.label: +"""启用地址迁移""" + +common_ssl_opts_schema_password.desc: +"""包含用户密码的字符串。仅在私钥文件受密码保护时使用。""" + +common_ssl_opts_schema_password.label: +"""秘钥文件密码""" + +common_ssl_opts_schema_hibernate_after.desc: +"""在闲置一定时间后休眠 SSL 进程，减少其内存占用。""" + +common_ssl_opts_schema_hibernate_after.label: +"""闲置多久后休眠""" + +fields_mqtt_quic_listener_send_buffering_enabled.desc: +"""缓冲发送数据，而不是保留应用缓冲区，直到发送数据被确认。默认值：1（启用）""" + +fields_mqtt_quic_listener_send_buffering_enabled.label: +"""启用发送缓冲功能""" + +sys_event_client_unsubscribed.desc: +"""是否开启客户端已成功取消订阅主题事件消息。""" + +overload_protection_backoff_new_conn.desc: +"""高负载时，拒绝新进来的客户端连接。""" + +overload_protection_backoff_new_conn.label: +"""关闭新连接""" + +server_ssl_opts_schema_ocsp_responder_url.desc: +"""用于检查服务器证书的 OCSP Responder 的 URL。""" + +server_ssl_opts_schema_ocsp_responder_url.label: +"""OCSP Responder 的 URL""" + +mqtt_idle_timeout.desc: +"""设置连接被断开或进入休眠状态前的等待时间，空闲超时后， + - 如暂未收到客户端的 CONNECT 报文，连接将断开； + - 如已收到客户端的 CONNECT 报文，连接将进入休眠模式以节省系统资源。 + +注意：请合理设置该参数值，如等待时间设置过长，可能造成系统资源的浪费。""" + +mqtt_idle_timeout.label: +"""空闲超时""" + +fields_mqtt_quic_listener_conn_flow_control_window.desc: +"""连接的流控窗口。默认：16777216""" + +fields_mqtt_quic_listener_conn_flow_control_window.label: +"""流控窗口""" + +fields_mqtt_quic_listener_maximum_mtu.desc: +"""一个连接所支持的最大MTU。这将是最大的探测值。默认值：1500""" + +fields_mqtt_quic_listener_maximum_mtu.label: +"""最大 MTU""" + +sysmon_top_db_name.desc: +"""PostgreSQL 数据库的数据库名""" + +sysmon_top_db_name.label: +"""数据库名""" + +mqtt_strict_mode.desc: +"""是否以严格模式解析 MQTT 消息。 +严格模式下，如客户端 ID、主题名称等中包含无效 utf8 字符串，连接将被断开。""" + +mqtt_strict_mode.label: +"""严格模式""" + +shared_subscription_group_strategy.desc: +"""设置共享订阅组为单位的分发策略。该配置是一个从组名到 +策略名的一个map，组名不得包含 `[A-Za-z0-9]` 之外的特殊字符。""" + +fields_deflate_opts_strategy.desc: +"""指定压缩策略。""" + +fields_deflate_opts_strategy.label: +"""指定压缩策略""" + +shared_subscription_strategy_enum.desc: +"""共享订阅的分发策略名称。 +- `random`：随机选择一个组内成员； +- `round_robin`：循环选择下一个成员； +- `round_robin_per_group`：在共享组内循环选择下一个成员； +- `sticky`：使用上一次选中的成员； +- `hash`：根据 ClientID 哈希映射到一个成员； +- `local`：随机分发到节点本地成成员，如果本地成员不存在，则随机分发到任意一个成员。""" + +persistent_session_builtin_sess_msg_table.desc: +"""优化内置的会话消息表的配置。""" + +persistent_session_builtin_sess_msg_table.label: +"""用于内建会话管理表的性能调优参数""" + +mqtt_mqueue_store_qos0.desc: +"""指定在连接断开但会话保持期间，是否需要在消息队列中存储 QoS 0 消息。""" + +mqtt_mqueue_store_qos0.label: +"""存储 QoS 0 消息""" + +server_ssl_opts_schema_client_renegotiation.desc: +"""在支持客户机发起的重新协商的协议中，这种操作的资源成本对于服务器来说高于客户机。 +这可能会成为拒绝服务攻击的载体。 +SSL 应用程序已经采取措施来反击此类尝试，但通过将此选项设置为 false，可以严格禁用客户端发起的重新协商。 +默认值为 true。请注意，由于基础密码套件可以加密的消息数量有限，禁用重新协商可能会导致长期连接变得不可用。""" + +server_ssl_opts_schema_client_renegotiation.label: +"""SSL 客户端冲协商""" + +server_ssl_opts_schema_enable_crl_check.desc: +"""是否为该监听器启用 CRL 检查。""" + +server_ssl_opts_schema_enable_crl_check.label: +"""启用 CRL 检查""" + +fields_mqtt_quic_listener_peer_bidi_stream_count.desc: +"""允许对端打开的双向流的数量""" + +fields_mqtt_quic_listener_peer_bidi_stream_count.label: +"""对端双向流的数量""" + +fields_mqtt_quic_listener_max_stateless_operations.desc: +"""无状态操作的最大数量，在任何时候都可以在一个工作者上排队。默认值：16""" + +fields_mqtt_quic_listener_max_stateless_operations.label: +"""最大无状态操作数""" + +fields_ws_opts_idle_timeout.desc: +"""关闭在此间隔内未发送 MQTT CONNECT 消息的客户端的传输层连接。""" + +fields_ws_opts_idle_timeout.label: +"""WS 空闲时间""" + +fields_mqtt_quic_listener_max_ack_delay_ms.desc: +"""在收到数据后要等待多长时间才能发送一个ACK。默认值：25""" + +fields_mqtt_quic_listener_max_ack_delay_ms.label: +"""最大应答延迟 毫秒""" + +base_listener_zone.desc: +"""监听器所属的配置组。""" + +base_listener_zone.label: +"""配置组""" + +fields_mqtt_quic_listener_handshake_idle_timeout.desc: +"""一个握手在被丢弃之前可以空闲多长时间。""" + +fields_mqtt_quic_listener_handshake_idle_timeout.label: +"""握手空闲超时时间""" + +force_gc_enable.desc: +"""启用强制垃圾回收。""" + +force_gc_enable.label: +"""启用强制垃圾回收""" + +fields_ws_opts_allow_origin_absence.desc: +"""If false and check_origin_enable is true, the server will reject requests that don't have origin HTTP header.""" + +fields_ws_opts_allow_origin_absence.label: +"""允许 origin 缺失""" + +common_ssl_opts_schema_versions.desc: +"""支持所有TLS/DTLS版本
+注：PSK 的 Ciphers 无法在 tlsv1.3 中使用，如果打算使用 PSK 密码套件，请确保这里配置为 ["tlsv1.2","tlsv1.1"]。""" + +common_ssl_opts_schema_versions.label: +"""SSL 版本""" + +mqtt_listener_proxy_protocol_timeout.desc: +"""代理协议超时。如果在超时时间内未收到代理协议数据包，EMQX将关闭TCP连接。""" + +mqtt_listener_proxy_protocol_timeout.label: +"""Proxy protocol 超时时间""" + +fields_mqtt_quic_listener_idle_timeout.desc: +"""一个连接在被关闭之前可以空闲多长时间。0表示禁用。""" + +fields_mqtt_quic_listener_idle_timeout.label: +"""空闲超时时间""" + +common_ssl_opts_schema_secure_renegotiate.desc: +"""SSL 参数重新协商是一种允许客户端和服务器动态重新协商 SSL 连接参数的功能。 +RFC 5746 定义了一种更安全的方法。通过启用安全的重新协商，您就失去了对不安全的重新协商的支持，从而容易受到 MitM 攻击。""" + +common_ssl_opts_schema_secure_renegotiate.label: +"""SSL 重新协商""" + +sysmon_vm_busy_port.desc: +"""当一个系统接口（例如 TCP socket）过忙，会触发一条带有 busy_port 关键字的 warning 级别的日志。 +同时还会发布一条主题为 $SYS/sysmon/busy_port 的 MQTT 系统消息。""" + +sysmon_vm_busy_port.label: +"""启用端口过忙监控""" + +sys_event_client_connected.desc: +"""是否开启客户端已连接事件消息。""" + +sysmon_vm_process_low_watermark.desc: +"""在清除相应警报之前，本地节点上可以同时存在多少进程的阈值（以进程百分比表示）。""" + +sysmon_vm_process_low_watermark.label: +"""进程数低水位线""" + +mqtt_max_packet_size.desc: +"""允许的最大 MQTT 报文大小。""" + +mqtt_max_packet_size.label: +"""最大报文大小""" + +common_ssl_opts_schema_reuse_sessions.desc: +"""启用 TLS 会话重用。""" + +common_ssl_opts_schema_reuse_sessions.label: +"""TLS 会话重用""" + +common_ssl_opts_schema_depth.desc: +"""在有效的证书路径中，可以跟随对等证书的非自颁发中间证书的最大数量。 +因此，如果深度为0，则对等方必须由受信任的根 CA 直接签名；
+如果是1，路径可以是 PEER、中间 CA、ROOT-CA；
+如果是2，则路径可以是PEER、中间 CA1、中间 CA2、ROOT-CA。""" + +common_ssl_opts_schema_depth.label: +"""CA 证书深度""" + +sysmon_vm_long_gc.desc: +"""当系统检测到某个 Erlang 进程垃圾回收占用过长时间，会触发一条带有 long_gc 关键字的日志。 +同时还会发布一条主题为 $SYS/sysmon/long_gc 的 MQTT 系统消息。""" + +sysmon_vm_long_gc.label: +"""启用长垃圾回收监控""" + +fields_mqtt_quic_listener_keyfile.desc: +"""私钥文件。在 5.1 中会被废弃，使用 .ssl_options.keyfile 代替。""" + +fields_mqtt_quic_listener_keyfile.label: +"""私钥文件""" + +mqtt_peer_cert_as_clientid.desc: +"""使用对端证书中的 CN、DN 字段或整个证书内容来作为客户端 ID。仅适用于 TLS 连接； +目前支持： +- cn: 取证书的 CN 字段 +- dn: 取证书的 DN 字段 +- crt: 取 DER 或 PEM 证书的内容 +- pem: 将 DER 证书内容转换为 PEM 格式作为客户端 ID +- md5: 取 DER 或 PEM 证书内容的 MD5 值""" + +mqtt_peer_cert_as_clientid.label: +"""对端证书作为客户端 ID""" + +persistent_session_store_message_gc_interval.desc: +"""将未送达的消息垃圾收集到持久会话的开始间隔。 +这会影响检查 "max_retain_undelivered"（最大保留未送达）的删除频率。""" + +persistent_session_store_message_gc_interval.label: +"""消息清理间隔""" + +broker_shared_dispatch_ack_enabled.desc: +"""该配置项已废弃，会在 5.1 中移除。 +启用/禁用 QoS 1 和 QoS 2 消息的共享派发确认。 +开启后，允许将消息从未及时回复 ACK 的订阅者 （例如，客户端离线） 重新派发给另外一个订阅者。""" + +base_listener_enable_authn.desc: +"""配置 true （默认值）启用客户端进行身份认证，通过检查认配置的认认证器链来决定是否允许接入。 +配置 false 时，将不对客户端做任何认证，任何客户端，不论是不是携带用户名等认证信息，都可以接入。 +配置 quick_deny_anonymous 时，行为跟 true 类似，但是会对匿名 +客户直接拒绝，不做使用任何认证器对客户端进行身份检查。""" + +base_listener_enable_authn.label: +"""启用身份认证""" + +force_shutdown_enable.desc: +"""启用 `force_shutdown` 功能。""" + +force_shutdown_enable.label: +"""启用 `force_shutdown` 功能""" + +broker_enable_session_registry.desc: +"""是否启用 Session Registry""" + +overload_protection_backoff_delay.desc: +"""高负载时，一些不重要的任务可能会延迟执行，在这里设置允许延迟的时间。""" + +overload_protection_backoff_delay.label: +"""延迟时间""" + +ciphers_schema_common.desc: +"""此配置保存由逗号分隔的 TLS 密码套件名称，或作为字符串数组。例如 +"TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256"或 +["TLS_AES_256_GCM_SHA384","TLS_AES_128_GCM_SHA256"]。 +
+密码（及其顺序）定义了客户端和服务器通过网络连接加密信息的方式。 +选择一个好的密码套件对于应用程序的数据安全性、机密性和性能至关重要。 + +名称应为 OpenSSL 字符串格式（而不是 RFC 格式）。 +EMQX 配置文档提供的所有默认值和示例都是 OpenSSL 格式
+注意：某些密码套件仅与特定的 TLS 版本兼容（'tlsv1.1'、'tlsv1.2'或'tlsv1.3'）。 +不兼容的密码套件将被自动删除。 + +例如，如果只有 versions 仅配置为 tlsv1.3。为其他版本配置密码套件将无效。 + +
+注：PSK 的 Ciphers 不支持 tlsv1.3
+如果打算使用PSK密码套件 tlsv1.3。应在ssl.versions中禁用。 + +
+PSK 密码套件： +"RSA-PSK-AES256-GCM-SHA384,RSA-PSK-AES256-CBC-SHA384, +RSA-PSK-AES128-GCM-SHA256,RSA-PSK-AES128-CBC-SHA256, +RSA-PSK-AES256-CBC-SHA,RSA-PSK-AES128-CBC-SHA, +RSA-PSK-DES-CBC3-SHA,RSA-PSK-RC4-SHA"""" + +ciphers_schema_common.label: +"""""" + +sys_event_client_disconnected.desc: +"""是否开启客户端已断开连接事件消息。""" + +crl_cache_refresh_interval.desc: +"""从服务器刷新CRL的周期。 该配置对所有 URL 和监听器有效。""" + +crl_cache_refresh_interval.label: +"""CRL 缓存刷新间隔""" + +mqtt_listener_proxy_protocol.desc: +"""如果EMQX集群部署在 HAProxy 或 Nginx 之后，请启用代理协议 V1/2
+详情见: https://www.haproxy.com/blog/haproxy/proxy-protocol/""" + +mqtt_listener_proxy_protocol.label: +"""Proxy protocol""" + +mqtt_listener_access_rules.desc: +"""此监听器的访问控制规则。""" + +mqtt_listener_access_rules.label: +"""访问控制规则""" + +server_ssl_opts_schema_enable_ocsp_stapling.desc: +"""是否为监听器启用 OCSP Stapling 功能。 如果设置为 true，需要定义 OCSP Responder 的 URL 和证书签发者的 PEM 文件路径。""" + +server_ssl_opts_schema_enable_ocsp_stapling.label: +"""启用 OCSP Stapling""" + +fields_tcp_opts_send_timeout_close.desc: +"""如果发送超时，则关闭连接。""" + +fields_tcp_opts_send_timeout_close.label: +"""TCP 发送超时关闭连接""" + +sysmon_os_cpu_check_interval.desc: +"""定期 CPU 检查的时间间隔。""" + +sysmon_os_cpu_check_interval.label: +"""定期 CPU 检查的时间间隔""" + +sysmon_top_sample_interval.desc: +"""指定应收集进程顶部的频率。""" + +sysmon_top_sample_interval.label: +"""取样时间""" + +fields_mqtt_quic_listener_idle_timeout_ms.desc: +"""一个连接在被优雅地关闭之前可以空闲多长时间。0 表示禁用超时""" + +fields_mqtt_quic_listener_idle_timeout_ms.label: +"""空闲超时 毫秒""" + +fields_ws_opts_fail_if_no_subprotocol.desc: +"""如果true，当客户端未携带Sec WebSocket Protocol字段时，服务器将返回一个错误。 +
注意：微信小程序需要禁用此验证。""" + +fields_ws_opts_fail_if_no_subprotocol.label: +"""无 subprotocol 则失败""" + +mqtt_wildcard_subscription.desc: +"""是否启用对 MQTT 通配符订阅的支持。""" + +mqtt_wildcard_subscription.label: +"""通配符订阅可用""" + +server_ssl_opts_schema_ocsp_refresh_interval.desc: +"""为服务器刷新OCSP响应的周期。""" + +server_ssl_opts_schema_ocsp_refresh_interval.label: +"""OCSP 刷新间隔""" + +overload_protection_backoff_hibernation.desc: +"""高负载时，跳过进程休眠。""" + +overload_protection_backoff_hibernation.label: +"""跳过休眠""" + +fields_ws_opts_max_frame_size.desc: +"""单个 MQTT 数据包的最大长度。""" + +fields_ws_opts_max_frame_size.label: +"""最大数据包长度""" + +sys_event_messages.desc: +"""客户端事件消息。""" + +broker_perf_trie_compaction.desc: +"""是否开启主题表压缩存储。 +启用它会显着提高通配符主题订阅率，如果通配符主题具有唯一前缀，例如：'sensor/{{id}}/+/'，其中每个订阅者的 ID 是唯一的。 +如果消息主要发布到具有大量级别的主题，则主题匹配性能（发布时）可能会降低。 + +注意：这是一个集群范围的配置。 它要求在更改之前停止所有节点。""" + +sysmon_vm_large_heap.desc: +"""启用后，当一个 Erlang 进程申请了大量内存，系统会触发一条带有 large_heap 关键字的 +warning 级别日志。同时还会发布一条主题为 $SYS/sysmon/busy_dist_port 的 MQTT 系统消息。""" + +sysmon_vm_large_heap.label: +"""启用大 heap 监控""" + +} diff --git a/rel/i18n/zh/emqx_slow_subs_api.hocon b/rel/i18n/zh/emqx_slow_subs_api.hocon new file mode 100644 index 000000000..b0e801ca3 --- /dev/null +++ b/rel/i18n/zh/emqx_slow_subs_api.hocon @@ -0,0 +1,30 @@ +emqx_slow_subs_api { + +clear_records_api.desc: +"""清除当前记录，然后重新开始统计""" + +clientid.desc: +"""消息的客户端 ID""" + +get_records_api.desc: +"""查看慢订阅的统计数据""" + +get_setting_api.desc: +"""查看配置""" + +last_update_time.desc: +"""记录的更新时间戳""" + +node.desc: +"""消息的节点名称""" + +timespan.desc: +"""消息的传输耗时""" + +topic.desc: +"""消息的主题""" + +update_setting_api.desc: +"""更新配置""" + +} diff --git a/rel/i18n/zh/emqx_slow_subs_schema.hocon b/rel/i18n/zh/emqx_slow_subs_schema.hocon new file mode 100644 index 000000000..beadac1ea --- /dev/null +++ b/rel/i18n/zh/emqx_slow_subs_schema.hocon @@ -0,0 +1,18 @@ +emqx_slow_subs_schema { + +enable.desc: +"""开启慢订阅""" + +expire_interval.desc: +"""慢订阅记录的有效时间""" + +stats_type.desc: +"""慢订阅的统计类型""" + +threshold.desc: +"""慢订阅统计的阈值""" + +top_k_num.desc: +"""慢订阅统计表的记录数量上限""" + +} diff --git a/rel/i18n/zh/emqx_statsd_api.hocon b/rel/i18n/zh/emqx_statsd_api.hocon new file mode 100644 index 000000000..5761e2431 --- /dev/null +++ b/rel/i18n/zh/emqx_statsd_api.hocon @@ -0,0 +1,9 @@ +emqx_statsd_api { + +get_statsd_config_api.desc: +"""列出 StatsD 指标采集和推送服务的的配置。""" + +update_statsd_config_api.desc: +"""更新 StatsD 指标采集和推送服务的配置。""" + +} diff --git a/rel/i18n/zh/emqx_statsd_schema.hocon b/rel/i18n/zh/emqx_statsd_schema.hocon new file mode 100644 index 000000000..548ad33bc --- /dev/null +++ b/rel/i18n/zh/emqx_statsd_schema.hocon @@ -0,0 +1,30 @@ +emqx_statsd_schema { + +enable.desc: +"""启用或禁用 StatsD 指标采集和推送服务。""" + +flush_interval.desc: +"""指标的推送间隔。""" + +get_statsd_config_api.desc: +"""列出 StatsD 指标采集和推送服务的的配置。""" + +sample_interval.desc: +"""指标的采样间隔。""" + +server.desc: +"""StatsD 服务器地址。""" + +statsd.desc: +"""StatsD 指标采集与推送配置。""" + +statsd.label: +"""StatsD""" + +tags.desc: +"""指标的标签。""" + +update_statsd_config_api.desc: +"""更新 StatsD 指标采集和推送服务的配置。""" + +} diff --git a/rel/i18n/zh/emqx_stomp_schema.hocon b/rel/i18n/zh/emqx_stomp_schema.hocon new file mode 100644 index 000000000..13cfc0397 --- /dev/null +++ b/rel/i18n/zh/emqx_stomp_schema.hocon @@ -0,0 +1,15 @@ +emqx_stomp_schema { + +stom_frame_max_body_length.desc: +"""允许的 Stomp 报文 Body 的最大字节数""" + +stom_frame_max_headers.desc: +"""允许的 Header 最大数量""" + +stomp.desc: +"""Stomp 网关配置。当前实现支持 v1.2/1.1/1.0 协议版本""" + +stomp_frame_max_headers_length.desc: +"""允许的 Header 字符串的最大长度""" + +} diff --git a/rel/i18n/zh/emqx_telemetry_api.hocon b/rel/i18n/zh/emqx_telemetry_api.hocon new file mode 100644 index 000000000..4e445a56f --- /dev/null +++ b/rel/i18n/zh/emqx_telemetry_api.hocon @@ -0,0 +1,54 @@ +emqx_telemetry_api { + +active_modules.desc: +"""获取活跃模块""" + +active_plugins.desc: +"""获取活跃插件""" + +emqx_version.desc: +"""获取 emqx 版本""" + +enable.desc: +"""启用遥测""" + +get_telemetry_data_api.desc: +"""获取遥测数据""" + +get_telemetry_status_api.desc: +"""获取遥测状态""" + +license.desc: +"""获取 license 信息""" + +messages_received.desc: +"""获取接收到的消息数量""" + +messages_sent.desc: +"""获取发送的消息数量""" + +nodes_uuid.desc: +"""获取节点 UUID""" + +num_clients.desc: +"""获取客户端数量""" + +os_name.desc: +"""获取操作系统名称""" + +os_version.desc: +"""获取操作系统版本""" + +otp_version.desc: +"""获取 OTP 版本""" + +up_time.desc: +"""获取运行时间""" + +update_telemetry_status_api.desc: +"""更新遥测状态""" + +uuid.desc: +"""获取 UUID""" + +} diff --git a/rel/i18n/zh/emqx_topic_metrics_api.hocon b/rel/i18n/zh/emqx_topic_metrics_api.hocon new file mode 100644 index 000000000..23a5791b3 --- /dev/null +++ b/rel/i18n/zh/emqx_topic_metrics_api.hocon @@ -0,0 +1,105 @@ +emqx_topic_metrics_api { + +message_qos1_in_rate.desc: +"""QoS1 接收消息速率""" + +message_out_count.desc: +"""发送消息数量""" + +message_qos2_out_rate.desc: +"""QoS2 发送消息速率""" + +message_qos0_in_rate.desc: +"""QoS0 接收消息速率""" + +get_topic_metrics_api.desc: +"""获取主题监控数据""" + +reset_time.desc: +"""重置时间。标准 rfc3339 时间格式，例如：2018-01-01T12:00:00Z。如果从未重置则为空""" + +topic_metrics_api_response400.desc: +"""错误请求。已存在或错误的主题名称""" + +reset_topic_desc.desc: +"""主题名称。如果此参数不存在，则所有创建的主题监控数据都将重置。""" + +topic_metrics_api_response409.desc: +"""冲突。主题监控数据超过最大限制512""" + +post_topic_metrics_api.desc: +"""创建主题监控数据""" + +message_dropped_rate.desc: +"""丢弃消息速率""" + +message_qos2_in_rate.desc: +"""QoS2 接收消息速率""" + +message_in_rate.desc: +"""接收消息速率""" + +message_qos0_out_rate.desc: +"""QoS0 发送消息速率""" + +message_qos2_in_count.desc: +"""QoS2 接收消息数量""" + +message_dropped_count.desc: +"""丢弃消息数量""" + +topic_metrics_api_response404.desc: +"""未找到。主题监控数据未找到""" + +topic_in_path.desc: +"""主题字符串。注意：主题字符串在url路径中必须编码""" + +action.desc: +"""操作，仅支持 reset""" + +message_qos0_in_count.desc: +"""QoS0 接收消息数量""" + +message_qos1_out_rate.desc: +"""QoS1 发送消息速率""" + +topic.desc: +"""主题""" + +reset_topic_metrics_api.desc: +"""重置主题监控状态""" + +create_time.desc: +"""创建时间。标准 rfc3339 时间格式，例如：2018-01-01T12:00:00Z""" + +metrics.desc: +"""监控数据""" + +message_qos1_out_count.desc: +"""QoS1 发送消息数量""" + +gat_topic_metrics_data_api.desc: +"""获取主题监控数据""" + +message_qos1_in_count.desc: +"""QoS1 接收消息数量""" + +delete_topic_metrics_data_api.desc: +"""删除主题监控数据""" + +message_qos0_out_count.desc: +"""QoS0 发送消息数量""" + +topic_in_body.desc: +"""主题字符串""" + +message_in_count.desc: +"""接收消息数量""" + +message_qos2_out_count.desc: +"""QoS2 发送消息数量""" + +message_out_rate.desc: +"""发送消息速率""" + +} diff --git a/scripts/check-i18n-style.escript b/scripts/check-i18n-style.escript index cbe79c82e..b8b6bdac7 100755 --- a/scripts/check-i18n-style.escript +++ b/scripts/check-i18n-style.escript @@ -9,7 +9,7 @@ -define(RESET, "\e[39m"). main([Files0]) -> - io:format(user, "checking i18n file styles", []), + io:format(user, "checking i18n file styles~n", []), _ = put(errors, 0), Files = string:tokens(Files0, "\n"), ok = load_hocon(), @@ -20,7 +20,7 @@ main([Files0]) -> N when is_integer(N) andalso N > 1 -> logerr("~p errors found~n", [N]); _ -> - io:format(user, "OK~n", []) + io:format(user, "~nOK~n", []) end. load_hocon() -> @@ -42,7 +42,7 @@ die(Msg, Args) -> halt(1). logerr(Fmt, Args) -> - io:format(standard_error, ?RED ++ "ERROR: " ++ Fmt ++ ?RESET, Args), + io:format(standard_error, "~n" ++ ?RED ++ "ERROR: " ++ Fmt ++ ?RESET, Args), N = get(errors), _ = put(errors, N + 1), ok. @@ -76,22 +76,16 @@ check_label(_Name, _) -> ok. check_desc(Name, #{<<"desc">> := Desc}) -> - do_check_desc(Name, Desc); + check_desc_string(Name, Desc); check_desc(Name, _) -> die("~s: no 'desc'~n", [Name]). -do_check_desc(Name, #{<<"zh">> := Zh, <<"en">> := En}) -> - ok = check_desc_string(Name, "zh", Zh), - ok = check_desc_string(Name, "en", En); -do_check_desc(Name, _) -> - die("~s: missing 'zh' or 'en'~n", [Name]). - -check_desc_string(Name, Tr, <<>>) -> - logerr("~s.~s: empty string~n", [Name, Tr]); -check_desc_string(Name, Tr, BinStr) -> +check_desc_string(Name, <<>>) -> + logerr("~s: empty string~n", [Name]); +check_desc_string(Name, BinStr) -> Str = unicode:characters_to_list(BinStr, utf8), Err = fun(Reason) -> - logerr("~s.~s: ~s~n", [Name, Tr, Reason]) + logerr("~s: ~s~n", [Name, Reason]) end, case Str of [$\s | _] -> diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index 9644ec8b9..66478a474 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -39,6 +39,7 @@ ONLY_UP='no' ATTACH='no' STOP='no' IS_CI='no' +ODBC_REQUEST='no' while [ "$#" -gt 0 ]; do case $1 in -h|--help) @@ -113,6 +114,13 @@ case "${WHICH_APP}" in ## ensure enterprise profile when testing lib-ee applications export PROFILE='emqx-enterprise' ;; + apps/*) + if [[ -f "${WHICH_APP}/BSL.txt" ]]; then + export PROFILE='emqx-enterprise' + else + export PROFILE='emqx' + fi + ;; *) export PROFILE="${PROFILE:-emqx}" ;; @@ -182,6 +190,10 @@ for dep in ${CT_DEPS}; do cassandra) FILES+=( '.ci/docker-compose-file/docker-compose-cassandra.yaml' ) ;; + sqlserver) + ODBC_REQUEST='yes' + FILES+=( '.ci/docker-compose-file/docker-compose-sqlserver.yaml' ) + ;; minio) FILES+=( '.ci/docker-compose-file/docker-compose-minio-tcp.yaml' '.ci/docker-compose-file/docker-compose-minio-tls.yaml' ) @@ -193,6 +205,12 @@ for dep in ${CT_DEPS}; do esac done +if [ "$ODBC_REQUEST" = 'yes' ]; then + INSTALL_ODBC="./scripts/install-odbc-driver.sh" +else + INSTALL_ODBC="echo 'Driver msodbcsql driver not requested'" +fi + F_OPTIONS="" for file in "${FILES[@]}"; do @@ -235,6 +253,7 @@ docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "openssl rand -base # the user must exist inside the container for `whoami` to work docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "useradd --uid $DOCKER_USER -M -d / emqx" || true docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "chown -R $DOCKER_USER /var/lib/secret" || true +docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "$INSTALL_ODBC" || true if [ "$ONLY_UP" = 'yes' ]; then exit 0 diff --git a/scripts/find-apps.sh b/scripts/find-apps.sh index f07cd2f7d..bfb6ba2cc 100755 --- a/scripts/find-apps.sh +++ b/scripts/find-apps.sh @@ -73,7 +73,11 @@ describe_app() { fi case "${app}" in apps/*) - profile='emqx' + if [[ -f "${app}/BSL.txt" ]]; then + profile='emqx-enterprise' + else + profile='emqx' + fi ;; lib-ee/*) profile='emqx-enterprise' diff --git a/scripts/git-hook-pre-commit.sh b/scripts/git-hook-pre-commit.sh index b0e0699b9..cebd92b22 100755 --- a/scripts/git-hook-pre-commit.sh +++ b/scripts/git-hook-pre-commit.sh @@ -4,6 +4,9 @@ set -euo pipefail OPT="${1:--c}" +# mix format check is quite fast +mix format --check-formatted + files_dirty="$(git diff --name-only | grep -E '.*\.erl' || true)" files_cached="$(git diff --cached --name-only | grep -E '.*\.erl' || true)" if [[ "${files_dirty}" == '' ]] && [[ "${files_cached}" == '' ]]; then diff --git a/scripts/install-odbc-driver.sh b/scripts/install-odbc-driver.sh new file mode 100755 index 000000000..7ceab95a0 --- /dev/null +++ b/scripts/install-odbc-driver.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +## this script install msodbcsql17 and unixodbc-dev on ci environment +## specific to ubuntu 16.04, 18.04, 20.04, 22.04 +set -euo pipefail + +# install msodbcsql17 +VERSION=$(lsb_release -rs) +if ! [[ "16.04 18.04 20.04 22.04" == *"$VERSION"* ]]; +then + echo "Ubuntu $VERSION is not currently supported."; + exit 1; +fi + +curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - && \ +curl https://packages.microsoft.com/config/ubuntu/"$VERSION"/prod.list > /etc/apt/sources.list.d/mssql-release.list && \ +## TODO: upgrade builder image +apt-get update && \ +ACCEPT_EULA=Y apt-get install -y msodbcsql17 unixodbc-dev mssql-tools && \ +## and not needed to modify /etc/odbcinst.ini +## docker-compose will mount one in .ci/docker-compose-file/odbc +sed -i 's/ODBC Driver 17 for SQL Server/ms-sql/g' /etc/odbcinst.ini diff --git a/scripts/merge-config.escript b/scripts/merge-config.escript index d30a0ca68..b3c214dd7 100755 --- a/scripts/merge-config.escript +++ b/scripts/merge-config.escript @@ -34,7 +34,10 @@ main(_) -> ok = file:write_file("apps/emqx_conf/etc/emqx-enterprise.conf.all", EnterpriseConf); false -> ok - end. + end, + merge_desc_files_per_lang("en"), + %% TODO: remove this when we have zh translation moved to dashboard package + merge_desc_files_per_lang("zh"). is_enterprise() -> Profile = os:getenv("PROFILE", "emqx"), @@ -96,3 +99,48 @@ try_enter_child(Dir, Files, Cfgs) -> true -> get_all_cfgs(filename:join([Dir, "src"]), Cfgs) end. + +%% Desc files merge is for now done locally in emqx.git repo for all languages. +%% When zh and other languages are moved to a separate repo, +%% we will only merge the en files. +%% The file for other languages will be merged in the other repo, +%% the built as a part of the dashboard package, +%% finally got pulled at build time as a part of the dashboard package. +merge_desc_files_per_lang(Lang) -> + BaseConf = <<"">>, + Cfgs0 = get_all_desc_files(Lang), + Conf = do_merge_desc_files_per_lang(BaseConf, Cfgs0), + OutputFile = case Lang of + "en" -> + %% en desc will always be in the priv dir of emqx_dashboard + "apps/emqx_dashboard/priv/desc.en.hocon"; + "zh" -> + %% so far we inject zh desc as if it's extracted from dashboard package + %% TODO: remove this when we have zh translation moved to dashboard package + "apps/emqx_dashboard/priv/www/static/desc.zh.hocon" + end, + ok = filelib:ensure_dir(OutputFile), + ok = file:write_file(OutputFile, Conf). + +do_merge_desc_files_per_lang(BaseConf, Cfgs) -> + lists:foldl( + fun(CfgFile, Acc) -> + case filelib:is_regular(CfgFile) of + true -> + {ok, Bin1} = file:read_file(CfgFile), + [Acc, io_lib:nl(), Bin1]; + false -> Acc + end + end, BaseConf, Cfgs). + +get_all_desc_files(Lang) -> + Dir = + case Lang of + "en" -> + filename:join(["rel", "i18n"]); + "zh" -> + %% TODO: remove this when we have zh translation moved to dashboard package + filename:join(["rel", "i18n", "zh"]) + end, + Files = filelib:wildcard("*.hocon", Dir), + lists:map(fun(Name) -> filename:join([Dir, Name]) end, Files). diff --git a/scripts/merge-i18n.escript b/scripts/merge-i18n.escript deleted file mode 100755 index b2501d10a..000000000 --- a/scripts/merge-i18n.escript +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env escript - --mode(compile). - -main(_) -> - BaseConf = <<"">>, - Cfgs0 = get_all_files(), - Conf = merge(BaseConf, Cfgs0), - OutputFile = "apps/emqx_dashboard/priv/i18n.conf", - ok = filelib:ensure_dir(OutputFile), - ok = file:write_file(OutputFile, Conf). - -merge(BaseConf, Cfgs) -> - lists:foldl( - fun(CfgFile, Acc) -> - case filelib:is_regular(CfgFile) of - true -> - {ok, Bin1} = file:read_file(CfgFile), - [Acc, io_lib:nl(), Bin1]; - false -> Acc - end - end, BaseConf, Cfgs). - -get_all_files() -> - Dir = filename:join(["rel","i18n"]), - Files = filelib:wildcard("*.hocon", Dir), - lists:map(fun(Name) -> filename:join([Dir, Name]) end, Files). diff --git a/scripts/pre-compile.sh b/scripts/pre-compile.sh index 56b7d47b4..71251a03e 100755 --- a/scripts/pre-compile.sh +++ b/scripts/pre-compile.sh @@ -20,5 +20,4 @@ cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." ./scripts/get-dashboard.sh "$dashboard_version" ./scripts/merge-config.escript -./scripts/merge-i18n.escript ./scripts/update-bom.sh "$PROFILE_STR" ./rel diff --git a/scripts/rel/cut.sh b/scripts/rel/cut.sh index 71033035a..4a0a7ab6c 100755 --- a/scripts/rel/cut.sh +++ b/scripts/rel/cut.sh @@ -233,19 +233,19 @@ if [ -d "${CHECKS_DIR}" ]; then fi generate_changelog () { - local from_tag num_en num_zh + local from_tag from_tag="${PREV_TAG:-}" if [[ -z $from_tag ]]; then from_tag="$(./scripts/find-prev-rel-tag.sh "$PROFILE")" fi - num_en=$(git diff --name-only -a "${from_tag}...HEAD" "changes" | grep -c '.en.md') - num_zh=$(git diff --name-only -a "${from_tag}...HEAD" "changes" | grep -c '.zh.md') - if [ "$num_en" -ne "$num_zh" ]; then - echo "Number of English and Chinese changelog files added since ${from_tag} do not match." - exit 1 - fi + # num_en=$(git diff --name-only -a "${from_tag}...HEAD" "changes" | grep -c '.en.md') + # num_zh=$(git diff --name-only -a "${from_tag}...HEAD" "changes" | grep -c '.zh.md') + # if [ "$num_en" -ne "$num_zh" ]; then + # echo "Number of English and Chinese changelog files added since ${from_tag} do not match." + # exit 1 + # fi ./scripts/rel/format-changelog.sh -b "${from_tag}" -l 'en' -v "$TAG" > "changes/${TAG}.en.md" - ./scripts/rel/format-changelog.sh -b "${from_tag}" -l 'zh' -v "$TAG" > "changes/${TAG}.zh.md" + # ./scripts/rel/format-changelog.sh -b "${from_tag}" -l 'zh' -v "$TAG" > "changes/${TAG}.zh.md" git add changes/"${TAG}".*.md if [ -n "$(git diff --staged --stat)" ]; then git commit -m "docs: Generate changelog for ${TAG}" diff --git a/scripts/spellcheck/dicts/emqx.txt b/scripts/spellcheck/dicts/emqx.txt index 79c8b7e3a..168275e1e 100644 --- a/scripts/spellcheck/dicts/emqx.txt +++ b/scripts/spellcheck/dicts/emqx.txt @@ -1,6 +1,7 @@ ACL AES APIs +Avro BPAPI BSON Backplane diff --git a/scripts/split-i18n-files.escript b/scripts/split-i18n-files.escript new file mode 100755 index 000000000..b9f558925 --- /dev/null +++ b/scripts/split-i18n-files.escript @@ -0,0 +1,84 @@ +#!/usr/bin/env escript + +%% This script is for one-time use. +%% will be deleted after the migration is done. + +-mode(compile). + +main([]) -> + %% we need to parse hocon + %% so we'll just add all compiled libs to path + code:add_pathsz(find_ebin_paths("_build/default/lib/*")), + Files = filelib:wildcard("rel/i18n/*.hocon"), + ok = lists:foreach(fun split_file/1, Files), + ok. + +find_ebin_paths(DirPattern) -> + LibDirs = filelib:wildcard(DirPattern), + lists:filtermap(fun add_ebin/1, LibDirs). + +add_ebin(Dir) -> + EbinDir = filename:join(Dir, "ebin"), + case filelib:is_dir(EbinDir) of + true -> {true, EbinDir}; + false -> false + end. + +split_file(Path) -> + {ok, DescMap} = hocon:load(Path), + [{Module, Descs}] = maps:to_list(DescMap), + try + ok = split(Path, Module, <<"en">>, Descs), + ok = split(Path, Module, <<"zh">>, Descs) + catch + throw : already_done -> + ok + end. + +split(Path, Module, Lang, Fields) when is_map(Fields) -> + split(Path, Module, Lang, maps:to_list(Fields)); +split(Path, Module, Lang, Fields) when is_list(Fields) -> + Split = lists:map(fun({Name, Desc})-> do_split(Path, Name, Lang, Desc) end, Fields), + IoData = [Module, " {\n\n", Split, "}\n"], + %% assert it's a valid HOCON object + {ok, _} = hocon:binary(IoData), + %io:format(user, "~s", [IoData]). + WritePath = case Lang of + <<"en">> -> + Path; + <<"zh">> -> + rename(Path, "zh") + end, + ok = filelib:ensure_dir(WritePath), + ok = file:write_file(WritePath, IoData), + ok. + +rename(FilePath, Lang) -> + Dir = filename:dirname(FilePath), + BaseName = filename:basename(FilePath), + filename:join([Dir, Lang, BaseName]). + +do_split(_Path, _Name, _Lang, #{<<"desc">> := Desc}) when is_binary(Desc) -> + throw(already_done); +do_split(Path, Name, Lang, #{<<"desc">> := Desc} = D) -> + try + Label = maps:get(<<"label">>, D, #{}), + DescL = maps:get(Lang, Desc), + LabelL = maps:get(Lang, Label, undefined), + [fmt([Name, ".desc:\n"], DescL), + fmt([Name, ".label:\n"], LabelL) + ] + catch + C : E : S-> + erlang:raise(C, {Path, Name, E}, S) + end. + + +tq() -> + "\"\"\"". + +fmt(_Key, undefined) -> + []; +fmt(Key, Content) -> + [Key, tq(), Content, tq(), "\n\n"]. + diff --git a/scripts/test/influx/influx-bridge.conf b/scripts/test/influx/influx-bridge.conf index 0416e42b6..0574ac38a 100644 --- a/scripts/test/influx/influx-bridge.conf +++ b/scripts/test/influx/influx-bridge.conf @@ -11,7 +11,7 @@ bridges { batch_size = 100 batch_time = "10ms" health_check_interval = "15s" - max_queue_bytes = "1GB" + max_buffer_bytes = "1GB" query_mode = "sync" request_timeout = "15s" start_after_created = "true"