diff --git a/.ci/build_packages/tests.sh b/.ci/build_packages/tests.sh index 53ab9ac57..da0ec7362 100755 --- a/.ci/build_packages/tests.sh +++ b/.ci/build_packages/tests.sh @@ -36,9 +36,14 @@ emqx_test(){ "zip") packagename=$(basename "${PACKAGE_PATH}/${EMQX_NAME}"-*.zip) unzip -q "${PACKAGE_PATH}/${packagename}" - export EMQX_ZONES__DEFAULT__MQTT__SERVER_KEEPALIVE=60 \ - EMQX_MQTT__MAX_TOPIC_ALIAS=10 - [[ $(arch) == *arm* || $(arch) == aarch64 ]] && export EMQX_LISTENERS__QUIC__DEFAULT__ENABLED=false + export EMQX_ZONES__DEFAULT__MQTT__SERVER_KEEPALIVE=60 + export EMQX_MQTT__MAX_TOPIC_ALIAS=10 + export EMQX_LOG__CONSOLE_HANDLER__LEVEL=debug + export EMQX_LOG__FILE_HANDLERS__DEFAULT__LEVEL=debug + if [[ $(arch) == *arm* || $(arch) == aarch64 ]]; then + export EMQX_LISTENERS__QUIC__DEFAULT__ENABLED=false + export WAIT_FOR_ERLANG_STOP=120 + fi # sed -i '/emqx_telemetry/d' "${PACKAGE_PATH}"/emqx/data/loaded_plugins echo "running ${packagename} start" @@ -58,7 +63,11 @@ emqx_test(){ IDLE_TIME=$((IDLE_TIME+1)) done pytest -v /paho-mqtt-testing/interoperability/test_client/V5/test_connect.py::test_basic - "${PACKAGE_PATH}"/emqx/bin/emqx stop + if ! "${PACKAGE_PATH}"/emqx/bin/emqx stop; then + cat "${PACKAGE_PATH}"/emqx/log/erlang.log.1 || true + cat "${PACKAGE_PATH}"/emqx/log/emqx.log.1 || true + exit 1 + fi echo "running ${packagename} stop" rm -rf "${PACKAGE_PATH}"/emqx ;; @@ -133,6 +142,7 @@ EOF ## for ARM, due to CI env issue, skip start of quic listener for the moment [[ $(arch) == *arm* || $(arch) == aarch64 ]] && tee -a "$emqx_env_vars" < ./_packages/${{ matrix.profile }}/$pkg_name.sha256 + openssl dgst -sha256 $pkg_name | awk '{print $2}' > $pkg_name.sha256 - uses: actions/upload-artifact@v1 if: startsWith(github.ref, 'refs/tags/') with: @@ -386,7 +386,7 @@ jobs: username: ${{ secrets.DOCKER_HUB_USER }} password: ${{ secrets.DOCKER_HUB_TOKEN }} - uses: docker/build-push-action@v2 - if: github.event_name == 'release' + if: github.event_name == 'release' && github.event.release.prerelease with: push: true pull: true @@ -400,6 +400,23 @@ jobs: EMQX_NAME=${{ matrix.profile }} file: source/deploy/docker/Dockerfile context: source + - uses: docker/build-push-action@v2 + if: github.event_name == 'release' && !github.event.release.prerelease + with: + push: true + pull: true + no-cache: true + platforms: linux/amd64,linux/arm64 + tags: | + emqx/${{ matrix.profile }}:latest + emqx/${{ matrix.profile }}:${{ steps.version.outputs.version }} + build-args: | + PKG_VSN=${{ steps.version.outputs.version }} + BUILD_FROM=ghcr.io/emqx/emqx-builder-helper/5.0:${{ matrix.otp }}-alpine3.14 + RUN_FROM=alpine:3.14 + EMQX_NAME=${{ matrix.profile }} + file: source/deploy/docker/Dockerfile + context: source delete-artifact: @@ -487,15 +504,6 @@ jobs: -X POST \ -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ env.version }}\" }" \ ${{ secrets.EMQX_IO_RELEASE_API }} - - name: push docker image to docker hub - if: github.event_name == 'release' - run: | - set -e -x -u - sudo make docker-prepare - cd _packages/${{ matrix.profile }} && for var in $(ls |grep docker |grep -v sha256); do unzip $var; sudo docker load < ${var%.*}; rm -f ${var%.*}; done && cd - - echo ${{ secrets.DOCKER_HUB_TOKEN }} |sudo docker login -u ${{ secrets.DOCKER_HUB_USER }} --password-stdin - sudo TARGET=emqx/${{ matrix.profile }} make docker-push - sudo TARGET=emqx/${{ matrix.profile }} make docker-manifest-list - name: update repo.emqx.io if: github.event_name == 'release' && endsWith(github.repository, 'enterprise') && matrix.profile == 'emqx-ee' run: | diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 2fb447d26..293bcb82b 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -13,6 +13,7 @@ jobs: runs-on: ubuntu-20.04 strategy: + fail-fast: false matrix: otp: - 24.0.5-emqx-1 @@ -53,13 +54,18 @@ jobs: path: _packages/**/*.zip mac: - runs-on: macos-10.15 strategy: + fail-fast: false matrix: + macos: + - macos-11 + - macos-10.15 otp: - 24.0.5-emqx-1 + runs-on: ${{ matrix.macos }} + steps: - uses: actions/checkout@v1 - name: prepare @@ -82,16 +88,12 @@ jobs: id: cache with: path: ~/.kerl - key: erl${{ matrix.otp }}-macos10.15 + key: otp-${{ matrix.otp }}-${{ matrix.macos }} - name: build erlang if: steps.cache.outputs.cache-hit != 'true' timeout-minutes: 60 - env: - KERL_BUILD_BACKEND: git - OTP_GITHUB_URL: https://github.com/emqx/otp run: | - kerl update releases - kerl build ${{ matrix.otp }} + kerl build git https://github.com/emqx/otp.git OTP-${{ matrix.otp }} ${{ matrix.otp }} kerl install ${{ matrix.otp }} $HOME/.kerl/${{ matrix.otp }} - name: build run: | @@ -106,8 +108,7 @@ jobs: path: ./rebar3.crashdump - name: test run: | - pkg_name=$(basename _packages/${EMQX_NAME}/emqx-*.zip) - unzip -q _packages/${EMQX_NAME}/$pkg_name + unzip -q $(find _packages/${EMQX_NAME} -mindepth 1 -maxdepth 1 -iname \*.zip | head) # gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins ./emqx/bin/emqx start || cat emqx/log/erlang.log.1 ready='no' diff --git a/.github/workflows/run_api_tests.yaml b/.github/workflows/run_api_tests.yaml index af9be07e0..1f2bc1eec 100644 --- a/.github/workflows/run_api_tests.yaml +++ b/.github/workflows/run_api_tests.yaml @@ -45,10 +45,19 @@ jobs: - api_login - api_banned - api_alarms + - api_nodes + - api_topic_metrics + - api_retainer + - api_auto_subscribe + - api_delayed_publish + - api_topic_rewrite + - api_event_message + - api_stats steps: - uses: actions/checkout@v2 with: repository: emqx/emqx-fvt + ref: v1.2.0 path: . - uses: actions/setup-java@v1 with: @@ -74,7 +83,7 @@ jobs: cd /tmp && tar -xvf apache-jmeter.tgz echo "jmeter.save.saveservice.output_format=xml" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties echo "jmeter.save.saveservice.response_data.on_error=true" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties - wget --no-verbose -O /tmp/apache-jmeter-$JMETER_VERSION/lib/ext/mqtt-xmeter-2.0.2-jar-with-dependencies.jar https://raw.githubusercontent.com/xmeter-net/mqtt-jmeter/master/Download/v2.0.2/mqtt-xmeter-2.0.2-jar-with-dependencies.jar + wget --no-verbose -O /tmp/apache-jmeter-$JMETER_VERSION/lib/ext/mqtt-xmeter-fuse-2.0.2-jar-with-dependencies.jar https://raw.githubusercontent.com/xmeter-net/mqtt-jmeter/master/Download/v2.0.2/mqtt-xmeter-fuse-2.0.2-jar-with-dependencies.jar ln -s /tmp/apache-jmeter-$JMETER_VERSION /opt/jmeter - name: run ${{ matrix.script_name }} run: | diff --git a/.gitignore b/.gitignore index 57be83882..4188126da 100644 --- a/.gitignore +++ b/.gitignore @@ -50,3 +50,5 @@ _upgrade_base/ TAGS erlang_ls.config .els_cache/ +.vs/ +.vscode/ diff --git a/Makefile b/Makefile index 2ef0e1806..6d159d144 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ BUILD = $(CURDIR)/build SCRIPTS = $(CURDIR)/scripts export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh) export EMQX_DESC ?= EMQ X -export EMQX_DASHBOARD_VERSION ?= v5.0.0-beta.13 +export EMQX_DASHBOARD_VERSION ?= v5.0.0-beta.16 ifeq ($(OS),Windows_NT) export REBAR_COLOR=none endif diff --git a/README.md b/README.md index dac48c2f2..bf3d93b64 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ English | [简体中文](./README-CN.md) | [日本語](./README-JP.md) | [рус *EMQ X* broker is a fully open source, highly scalable, highly available distributed MQTT messaging broker for IoT, M2M and Mobile applications that can handle tens of millions of concurrent clients. -Starting from 3.0 release, *EMQ X* broker fully supports MQTT V5.0 protocol specifications and backward compatible with MQTT V3.1 and V3.1.1, as well as other communication protocols such as MQTT-SN, CoAP, LwM2M, WebSocket and STOMP. The 3.0 release of the *EMQ X* broker can scaled to 10+ million concurrent MQTT connections on one cluster. +Starting from 3.0 release, *EMQ X* broker fully supports MQTT V5.0 protocol specifications and backward compatible with MQTT V3.1 and V3.1.1, as well as other communication protocols such as MQTT-SN, CoAP, LwM2M, WebSocket and STOMP. The 3.0 release of the *EMQ X* broker can scale to 10+ million concurrent MQTT connections on one cluster. - For full list of new features, please read [EMQ X Release Notes](https://github.com/emqx/emqx/releases). - For more information, please visit [EMQ X homepage](https://www.emqx.io/). diff --git a/apps/emqx/etc/emqx.conf b/apps/emqx/etc/emqx.conf index 42d9305c8..df5ae9034 100644 --- a/apps/emqx/etc/emqx.conf +++ b/apps/emqx/etc/emqx.conf @@ -194,12 +194,17 @@ listeners.ssl.default { mountpoint = "" ## SSL options - ## See ${example_common_ssl_options} for more information - ssl.versions = ["tlsv1.3", "tlsv1.2", "tlsv1.1", "tlsv1"] ssl.keyfile = "{{ platform_etc_dir }}/certs/key.pem" ssl.certfile = "{{ platform_etc_dir }}/certs/cert.pem" ssl.cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" + # ssl.versions = ["tlsv1.3", "tlsv1.2", "tlsv1.1", "tlsv1"] + # TLS 1.3: "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_CCM_SHA256,TLS_AES_128_CCM_8_SHA256" + # TLS 1-1.2 "ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA" + # PSK: "PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA" + # NOTE: If PSK cipher-suites are intended, tlsv1.3 should not be enabled in 'versions' config + # ssl.ciphers = "" + ## TCP options ## See ${example_common_tcp_options} for more information tcp.backlog = 1024 @@ -1345,12 +1350,13 @@ example_common_ssl_options { ## Default: true ssl.honor_cipher_order = true - ## TLS versions only to protect from POODLE attack. - ## - ## @doc listeners..ssl.versions - ## ValueType: Array - ## Default: ["tlsv1.3", "tlsv1.2", "tlsv1.1", "tlsv1"] - ssl.versions = ["tlsv1.3", "tlsv1.2", "tlsv1.1", "tlsv1"] + # ssl.versions = ["tlsv1.3", "tlsv1.2", "tlsv1.1", "tlsv1"] + # TLS 1.3: "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_CCM_SHA256,TLS_AES_128_CCM_8_SHA256" + # TLS 1-1.2 "ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA" + # PSK: "PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA" + # NOTE: If PSK cipher-suites are intended, tlsv1.3 should not be enabled in 'versions' config + # NOTE: by default, ALL ciphers are enabled + # ssl.ciphers = "" ## TLS Handshake timeout. ## @@ -1446,27 +1452,6 @@ example_common_ssl_options { ## Default: true ssl.fail_if_no_peer_cert = false - ## This is the single most important configuration option of an Erlang SSL - ## application. Ciphers (and their ordering) define the way the client and - ## server encrypt information over the wire, from the initial Diffie-Helman - ## key exchange, the session key encryption ## algorithm and the message - ## digest algorithm. Selecting a good cipher suite is critical for the - ## application’s data security, confidentiality and performance. - ## - ## The cipher list above offers: - ## - ## A good balance between compatibility with older browsers. - ## It can get stricter for Machine-To-Machine scenarios. - ## Perfect Forward Secrecy. - ## No old/insecure encryption and HMAC algorithms - ## - ## Most of it was copied from Mozilla’s Server Side TLS article - ## - ## @doc listeners..ssl.ciphers - ## ValueType: Array - ## Default: [ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA,PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA] - ssl.ciphers = [ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA,PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA] - } ## Socket options for websocket connections diff --git a/apps/emqx/include/emqx.hrl b/apps/emqx/include/emqx.hrl index 550e650a2..486143912 100644 --- a/apps/emqx/include/emqx.hrl +++ b/apps/emqx/include/emqx.hrl @@ -17,18 +17,18 @@ -ifndef(EMQ_X_HRL). -define(EMQ_X_HRL, true). +%% Shard %%-------------------------------------------------------------------- -%% Common -%%-------------------------------------------------------------------- - --define(Otherwise, true). - -define(COMMON_SHARD, emqx_common_shard). -define(SHARED_SUB_SHARD, emqx_shared_sub_shard). --define(MOD_DELAYED_SHARD, emqx_delayed_shard). -define(CM_SHARD, emqx_cm_shard). +-define(ROUTE_SHARD, route_shard). + +-define(BOOT_SHARDS, [ ?ROUTE_SHARD + , ?COMMON_SHARD + , ?SHARED_SUB_SHARD + ]). -%%-------------------------------------------------------------------- %% Banner %%-------------------------------------------------------------------- @@ -71,7 +71,9 @@ %% Message Payload payload :: emqx_types:payload(), %% Timestamp (Unit: millisecond) - timestamp :: integer() + timestamp :: integer(), + %% not used so far, for future extension + extra = [] :: term() }). -record(delivery, { @@ -83,11 +85,6 @@ %% Route %%-------------------------------------------------------------------- --define(ROUTE_SHARD, route_shard). - - --define(RULE_ENGINE_SHARD, emqx_rule_engine_shard). - -record(route, { topic :: binary(), dest :: node() | {binary(), node()} @@ -133,8 +130,6 @@ until :: integer() }). --endif. - %%-------------------------------------------------------------------- %% Authentication %%-------------------------------------------------------------------- @@ -149,4 +144,6 @@ -record(chain, { name :: atom() , authenticators :: [#authenticator{}] - }). \ No newline at end of file + }). + +-endif. diff --git a/apps/emqx/include/emqx_mqtt.hrl b/apps/emqx/include/emqx_mqtt.hrl index 5dd9a317c..ae13fcf14 100644 --- a/apps/emqx/include/emqx_mqtt.hrl +++ b/apps/emqx/include/emqx_mqtt.hrl @@ -542,4 +542,9 @@ -define(SHARE(Group, Topic), emqx_topic:join([<>, Group, Topic])). -define(IS_SHARE(Topic), case Topic of <> -> true; _ -> false end). +-define(FRAME_PARSE_ERROR(Reason), {frame_parse_error, Reason}). +-define(FRAME_SERIALIZE_ERROR(Reason), {frame_serialize_error, Reason}). +-define(THROW_FRAME_ERROR(Reason), erlang:throw(?FRAME_PARSE_ERROR(Reason))). +-define(THROW_SERIALIZE_ERROR(Reason), erlang:throw(?FRAME_SERIALIZE_ERROR(Reason))). + -endif. diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index 7391b765a..5b4bc7d00 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "5.0-alpha.6"}). +-define(EMQX_RELEASE, {opensource, "5.0-beta.1"}). -else. diff --git a/apps/emqx/include/logger.hrl b/apps/emqx/include/logger.hrl index fac01696d..c2ee5ab95 100644 --- a/apps/emqx/include/logger.hrl +++ b/apps/emqx/include/logger.hrl @@ -41,6 +41,7 @@ -define(LOG(Level, Format), ?LOG(Level, Format, [])). +%% deprecated -define(LOG(Level, Format, Args, Meta), %% check 'allow' here so we do not have to pass an anonymous function %% down to logger which may cause `badfun` exception during upgrade @@ -58,8 +59,15 @@ %% structured logging -define(SLOG(Level, Data), - logger:log(Level, Data, #{ mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY} - , line => ?LINE})). + %% check 'allow' here, only evaluate Data when necessary + case logger:allow(Level, ?MODULE) of + true -> + logger:log(Level, (Data), #{ mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY} + , line => ?LINE + }); + false -> + ok + end). %% print to 'user' group leader -define(ULOG(Fmt, Args), io:format(user, Fmt, Args)). diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 54735360b..4ec7c7dc5 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -10,13 +10,13 @@ %% `git_subdir` dependency in other projects. {deps, [ {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} - , {typerefl, {git, "https://github.com/k32/typerefl", {tag, "0.8.4"}}} + , {typerefl, {git, "https://github.com/k32/typerefl", {tag, "0.8.5"}}} , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.8.3"}}} - , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.2"}}} + , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.3"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.10.8"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.19.0"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.19.5"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} , {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.14.1"}}} diff --git a/apps/emqx/src/emqx_alarm.erl b/apps/emqx/src/emqx_alarm.erl index 6bd2d5d49..14f05dc5c 100644 --- a/apps/emqx/src/emqx_alarm.erl +++ b/apps/emqx/src/emqx_alarm.erl @@ -239,11 +239,11 @@ handle_call({get_alarms, deactivated}, _From, State) -> {reply, Alarms, State}; handle_call(Req, _From, State) -> - ?LOG(error, "Unexpected call: ~p", [Req]), + ?SLOG(error, #{msg => "unexpected_call", call => Req}), {reply, ignored, State}. handle_cast(Msg, State) -> - ?LOG(error, "Unexpected msg: ~p", [Msg]), + ?SLOG(error, #{msg => "unexpected_cast", cast => Msg}), {noreply, State}. handle_info({timeout, _TRef, delete_expired_deactivated_alarm}, @@ -253,11 +253,11 @@ handle_info({timeout, _TRef, delete_expired_deactivated_alarm}, {noreply, State#state{timer = ensure_timer(TRef, Period)}}; handle_info({update_timer, Period}, #state{timer = TRef} = State) -> - ?LOG(warning, "update the 'validity_period' timer to ~p", [Period]), + ?SLOG(warning, #{msg => "validity_timer_updated", period => Period}), {noreply, State#state{timer = ensure_timer(TRef, Period)}}; handle_info(Info, State) -> - ?LOG(error, "Unexpected info: ~p", [Info]), + ?SLOG(error, #{msg => "unexpected_info", info => Info}), {noreply, State}. terminate(_Reason, _State) -> @@ -323,8 +323,11 @@ deactivate_all_alarms() -> clear_table(TableName) -> case ekka_mnesia:clear_table(TableName) of {aborted, Reason} -> - ?LOG(warning, "Faile to clear table ~p reason: ~p", - [TableName, Reason]); + ?SLOG(warning, #{ + msg => "fail_to_clear_table", + table_name => TableName, + reason => Reason + }); {atomic, ok} -> ok end. @@ -354,10 +357,17 @@ delete_expired_deactivated_alarms(ActivatedAt, Checkpoint) -> do_actions(_, _, []) -> ok; do_actions(activate, Alarm = #activated_alarm{name = Name, message = Message}, [log | More]) -> - ?LOG(warning, "Alarm ~s is activated, ~s", [Name, Message]), + ?SLOG(warning, #{ + msg => "alarm_is_activated", + name => Name, + message => Message + }), do_actions(activate, Alarm, More); do_actions(deactivate, Alarm = #deactivated_alarm{name = Name}, [log | More]) -> - ?LOG(warning, "Alarm ~s is deactivated", [Name]), + ?SLOG(warning, #{ + msg => "alarm_is_deactivated", + name => Name + }), do_actions(deactivate, Alarm, More); do_actions(Operation, Alarm, [publish | More]) -> Topic = topic(Operation), diff --git a/apps/emqx/src/emqx_app.erl b/apps/emqx/src/emqx_app.erl index 1585946d1..2d667b805 100644 --- a/apps/emqx/src/emqx_app.erl +++ b/apps/emqx/src/emqx_app.erl @@ -24,6 +24,7 @@ , get_description/0 , get_release/0 , set_init_config_load_done/0 + , get_init_config_load_done/0 , set_override_conf_file/1 ]). @@ -33,25 +34,18 @@ -define(APP, emqx). --define(EMQX_SHARDS, [ ?ROUTE_SHARD - , ?COMMON_SHARD - , ?SHARED_SUB_SHARD - , ?RULE_ENGINE_SHARD - , ?MOD_DELAYED_SHARD - ]). - - %%-------------------------------------------------------------------- %% Application callbacks %%-------------------------------------------------------------------- start(_Type, _Args) -> ok = maybe_load_config(), + %% Load application first for ekka_mnesia scanner - mnesia:change_table_copy_type(schema, node(), disc_copies), - ekka:start(), - ok = ekka_rlog:wait_for_shards(?EMQX_SHARDS, infinity), + %% mnesia:change_table_copy_type(schema, node(), disc_copies), + ok = maybe_start_quicer(), + ensure_ekka_started(), {ok, Sup} = emqx_sup:start_link(), ok = maybe_start_listeners(), ok = emqx_alarm_handler:load(), @@ -65,21 +59,28 @@ prep_stop(_State) -> stop(_State) -> ok. +ensure_ekka_started() -> + ekka:start(), + ok = ekka_rlog:wait_for_shards(?BOOT_SHARDS, infinity). + %% @doc Call this function to make emqx boot without loading config, %% in case we want to delegate the config load to a higher level app %% which manages emqx app. set_init_config_load_done() -> application:set_env(emqx, init_config_load_done, true). +get_init_config_load_done() -> + application:get_env(emqx, init_config_load_done, false). + %% @doc This API is mostly for testing. %% The override config file is typically located in the 'data' dir when -%% it is a emqx release, but emqx app should not have to konw where the +%% it is a emqx release, but emqx app should not have to know where the %% 'data' dir is located. set_override_conf_file(File) -> application:set_env(emqx, override_conf_file, File). maybe_load_config() -> - case application:get_env(emqx, init_config_load_done, false) of + case get_init_config_load_done() of true -> ok; false -> diff --git a/apps/emqx/src/emqx_authentication.erl b/apps/emqx/src/emqx_authentication.erl index 4200190ac..ea077e171 100644 --- a/apps/emqx/src/emqx_authentication.erl +++ b/apps/emqx/src/emqx_authentication.erl @@ -289,22 +289,16 @@ check_config(Config) -> %%------------------------------------------------------------------------------ authenticate(#{listener := Listener, protocol := Protocol} = Credential, _AuthResult) -> - case ets:lookup(?CHAINS_TAB, Listener) of - [#chain{authenticators = Authenticators}] when Authenticators =/= [] -> - do_authenticate(Authenticators, Credential); - _ -> - case ets:lookup(?CHAINS_TAB, global_chain(Protocol)) of - [#chain{authenticators = Authenticators}] when Authenticators =/= [] -> - do_authenticate(Authenticators, Credential); - _ -> - ignore - end + Authenticators = get_authenticators(Listener, global_chain(Protocol)), + case get_enabled(Authenticators) of + [] -> ignore; + NAuthenticators -> do_authenticate(NAuthenticators, Credential) end. do_authenticate([], _) -> {stop, {error, not_authorized}}; -do_authenticate([#authenticator{provider = Provider, state = State} | More], Credential) -> - case Provider:authenticate(Credential, State) of +do_authenticate([#authenticator{id = ID, provider = Provider, state = State} | More], Credential) -> + try Provider:authenticate(Credential, State) of ignore -> do_authenticate(More, Credential); Result -> @@ -314,8 +308,32 @@ do_authenticate([#authenticator{provider = Provider, state = State} | More], Cre %% {continue, AuthData, AuthCache} %% {error, Reason} {stop, Result} + catch + Class:Reason:Stacktrace -> + ?SLOG(warning, #{msg => "unexpected_error_in_authentication", + exception => Class, + reason => Reason, + stacktrace => Stacktrace, + authenticator => ID}), + do_authenticate(More, Credential) end. +get_authenticators(Listener, Global) -> + case ets:lookup(?CHAINS_TAB, Listener) of + [#chain{authenticators = Authenticators}] -> + Authenticators; + _ -> + case ets:lookup(?CHAINS_TAB, Global) of + [#chain{authenticators = Authenticators}] -> + Authenticators; + _ -> + [] + end + end. + +get_enabled(Authenticators) -> + [Authenticator || Authenticator <- Authenticators, Authenticator#authenticator.enable =:= true]. + %%------------------------------------------------------------------------------ %% APIs %%------------------------------------------------------------------------------ @@ -331,7 +349,11 @@ initialize_authentication(ChainName, AuthenticatorsConfig) -> {ok, _} -> ok; {error, Reason} -> - ?LOG(error, "Failed to create authenticator '~s': ~p", [generate_id(AuthenticatorConfig), Reason]) + ?SLOG(error, #{ + msg => "failed_to_create_authenticator", + authenticator => generate_id(AuthenticatorConfig), + reason => Reason + }) end end, CheckedConfig). @@ -536,7 +558,7 @@ handle_call({create_authenticator, ChainName, Config}, _From, #{providers := Pro false -> case do_create_authenticator(ChainName, AuthenticatorID, Config, Providers) of {ok, Authenticator} -> - NAuthenticators = Authenticators ++ [Authenticator], + NAuthenticators = Authenticators ++ [Authenticator#authenticator{enable = maps:get(enable, Config)}], true = ets:insert(?CHAINS_TAB, Chain#chain{authenticators = NAuthenticators}), {ok, serialize_authenticator(Authenticator)}; {error, Reason} -> @@ -575,7 +597,8 @@ handle_call({update_authenticator, ChainName, AuthenticatorID, Config}, _From, S Unique = unique(ChainName, AuthenticatorID, Version), case Provider:update(Config#{'_unique' => Unique}, ST) of {ok, NewST} -> - NewAuthenticator = Authenticator#authenticator{state = switch_version(NewST)}, + NewAuthenticator = Authenticator#authenticator{state = switch_version(NewST), + enable = maps:get(enable, Config)}, NewAuthenticators = replace_authenticator(AuthenticatorID, NewAuthenticator, Authenticators), true = ets:insert(?CHAINS_TAB, Chain#chain{authenticators = NewAuthenticators}), {ok, serialize_authenticator(NewAuthenticator)}; @@ -629,15 +652,15 @@ handle_call({list_users, ChainName, AuthenticatorID}, _From, State) -> reply(Reply, State); handle_call(Req, _From, State) -> - ?LOG(error, "Unexpected call: ~p", [Req]), + ?SLOG(error, #{msg => "unexpected_call", call => Req}), {reply, ignored, State}. handle_cast(Req, State) -> - ?LOG(error, "Unexpected case: ~p", [Req]), + ?SLOG(error, #{msg => "unexpected_cast", cast => Req}), {noreply, State}. handle_info(Info, State) -> - ?LOG(error, "Unexpected info: ~p", [Info]), + ?SLOG(error, #{msg => "unexpected_info", info => Info}), {noreply, State}. terminate(_Reason, _State) -> diff --git a/apps/emqx/src/emqx_banned.erl b/apps/emqx/src/emqx_banned.erl index 608734363..759c9f955 100644 --- a/apps/emqx/src/emqx_banned.erl +++ b/apps/emqx/src/emqx_banned.erl @@ -187,11 +187,11 @@ init([]) -> {ok, ensure_expiry_timer(#{expiry_timer => undefined})}. handle_call(Req, _From, State) -> - ?LOG(error, "unexpected call: ~p", [Req]), + ?SLOG(error, #{msg => "unexpected_call", call => Req}), {reply, ignored, State}. handle_cast(Msg, State) -> - ?LOG(error, "unexpected msg: ~p", [Msg]), + ?SLOG(error, #{msg => "unexpected_msg", cast => Msg}), {noreply, State}. handle_info({timeout, TRef, expire}, State = #{expiry_timer := TRef}) -> @@ -199,7 +199,7 @@ handle_info({timeout, TRef, expire}, State = #{expiry_timer := TRef}) -> {noreply, ensure_expiry_timer(State), hibernate}; handle_info(Info, State) -> - ?LOG(error, "unexpected info: ~p", [Info]), + ?SLOG(error, #{msg => "unexpected_info", info => Info}), {noreply, State}. terminate(_Reason, #{expiry_timer := TRef}) -> diff --git a/apps/emqx/src/emqx_broker.erl b/apps/emqx/src/emqx_broker.erl index 56ac348da..e556361c7 100644 --- a/apps/emqx/src/emqx_broker.erl +++ b/apps/emqx/src/emqx_broker.erl @@ -202,7 +202,8 @@ publish(Msg) when is_record(Msg, message) -> emqx_message:is_sys(Msg) orelse emqx_metrics:inc('messages.publish'), case emqx_hooks:run_fold('message.publish', [], emqx_message:clean_dup(Msg)) of #message{headers = #{allow_publish := false}} -> - ?LOG(notice, "Stop publishing: ~s", [emqx_message:format(Msg)]), + ?SLOG(debug, #{msg => "message_not_published", + payload => emqx_message:to_log_map(Msg)}), []; Msg1 = #message{topic = Topic} -> route(aggre(emqx_router:match_routes(Topic)), delivery(Msg1)) @@ -214,9 +215,14 @@ safe_publish(Msg) when is_record(Msg, message) -> try publish(Msg) catch - _:Error:Stk-> - ?LOG(error, "Publish error: ~0p~n~s~n~0p", - [Error, emqx_message:format(Msg), Stk]), + Error : Reason : Stk-> + ?SLOG(error,#{ + msg => "publishing_error", + exception => Error, + reason => Reason, + payload => emqx_message:to_log_map(Msg), + stacktrace => Stk + }), [] end. @@ -266,14 +272,22 @@ forward(Node, To, Delivery, async) -> case emqx_rpc:cast(To, Node, ?BROKER, dispatch, [To, Delivery]) of true -> emqx_metrics:inc('messages.forward'); {badrpc, Reason} -> - ?LOG(error, "Ansync forward msg to ~s failed due to ~p", [Node, Reason]), + ?SLOG(error, #{ + msg => "async_forward_msg_to_node_failed", + node => Node, + reason => Reason + }), {error, badrpc} end; forward(Node, To, Delivery, sync) -> case emqx_rpc:call(To, Node, ?BROKER, dispatch, [To, Delivery]) of {badrpc, Reason} -> - ?LOG(error, "Sync forward msg to ~s failed due to ~p", [Node, Reason]), + ?SLOG(error, #{ + msg => "sync_forward_msg_to_node_failed", + node => Node, + reason => Reason + }), {error, badrpc}; Result -> emqx_metrics:inc('messages.forward'), Result @@ -450,14 +464,14 @@ handle_call({subscribe, Topic, I}, _From, State) -> {reply, Ok, State}; handle_call(Req, _From, State) -> - ?LOG(error, "Unexpected call: ~p", [Req]), + ?SLOG(error, #{msg => "unexpected_call", call => Req}), {reply, ignored, State}. handle_cast({subscribe, Topic}, State) -> case emqx_router:do_add_route(Topic) of ok -> ok; {error, Reason} -> - ?LOG(error, "Failed to add route: ~p", [Reason]) + ?SLOG(error, #{msg => "failed_to_add_route", reason => Reason}) end, {noreply, State}; @@ -481,11 +495,11 @@ handle_cast({unsubscribed, Topic, I}, State) -> {noreply, State}; handle_cast(Msg, State) -> - ?LOG(error, "Unexpected cast: ~p", [Msg]), + ?SLOG(error, #{msg => "unexpected_cast", cast => Msg}), {noreply, State}. handle_info(Info, State) -> - ?LOG(error, "Unexpected info: ~p", [Info]), + ?SLOG(error, #{msg => "unexpected_info", info => Info}), {noreply, State}. terminate(_Reason, #{pool := Pool, id := Id}) -> diff --git a/apps/emqx/src/emqx_broker_helper.erl b/apps/emqx/src/emqx_broker_helper.erl index fdd1a55d9..f31f5b164 100644 --- a/apps/emqx/src/emqx_broker_helper.erl +++ b/apps/emqx/src/emqx_broker_helper.erl @@ -118,7 +118,7 @@ init([]) -> {ok, #{pmon => emqx_pmon:new()}}. handle_call(Req, _From, State) -> - ?LOG(error, "Unexpected call: ~p", [Req]), + ?SLOG(error, #{msg => "unexpected_call", call => Req}), {reply, ignored, State}. handle_cast({register_sub, SubPid, SubId}, State = #{pmon := PMon}) -> @@ -127,7 +127,7 @@ handle_cast({register_sub, SubPid, SubId}, State = #{pmon := PMon}) -> {noreply, State#{pmon := emqx_pmon:monitor(SubPid, PMon)}}; handle_cast(Msg, State) -> - ?LOG(error, "Unexpected cast: ~p", [Msg]), + ?SLOG(error, #{msg => "unexpected_cast", cast => Msg}), {noreply, State}. handle_info({'DOWN', _MRef, process, SubPid, _Reason}, State = #{pmon := PMon}) -> @@ -138,7 +138,7 @@ handle_info({'DOWN', _MRef, process, SubPid, _Reason}, State = #{pmon := PMon}) {noreply, State#{pmon := PMon1}}; handle_info(Info, State) -> - ?LOG(error, "Unexpected info: ~p", [Info]), + ?SLOG(error, #{msg => "unexpected_info", info => Info}), {noreply, State}. terminate(_Reason, _State) -> diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index 0b1ff7e25..61ccdae16 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -373,11 +373,11 @@ handle_in(?PUBACK_PACKET(PacketId, _ReasonCode, Properties), Channel ok = after_message_acked(ClientInfo, Msg, Properties), handle_out(publish, Publishes, Channel#channel{session = NSession}); {error, ?RC_PACKET_IDENTIFIER_IN_USE} -> - ?LOG(warning, "The PUBACK PacketId ~w is inuse.", [PacketId]), + ?SLOG(warning, #{msg => "puback_packetId_inuse", packetId => PacketId}), ok = emqx_metrics:inc('packets.puback.inuse'), {ok, Channel}; {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND} -> - ?LOG(warning, "The PUBACK PacketId ~w is not found.", [PacketId]), + ?SLOG(warning, #{msg => "puback_packetId_not_found", packetId => PacketId}), ok = emqx_metrics:inc('packets.puback.missed'), {ok, Channel} end; @@ -390,11 +390,11 @@ handle_in(?PUBREC_PACKET(PacketId, _ReasonCode, Properties), Channel NChannel = Channel#channel{session = NSession}, handle_out(pubrel, {PacketId, ?RC_SUCCESS}, NChannel); {error, RC = ?RC_PACKET_IDENTIFIER_IN_USE} -> - ?LOG(warning, "The PUBREC PacketId ~w is inuse.", [PacketId]), + ?SLOG(warning, #{msg => "pubrec_packetId_inuse", packetId => PacketId}), ok = emqx_metrics:inc('packets.pubrec.inuse'), handle_out(pubrel, {PacketId, RC}, Channel); {error, RC = ?RC_PACKET_IDENTIFIER_NOT_FOUND} -> - ?LOG(warning, "The PUBREC ~w is not found.", [PacketId]), + ?SLOG(warning, #{msg => "pubrec_packetId_not_found", packetId => PacketId}), ok = emqx_metrics:inc('packets.pubrec.missed'), handle_out(pubrel, {PacketId, RC}, Channel) end; @@ -405,7 +405,7 @@ handle_in(?PUBREL_PACKET(PacketId, _ReasonCode), Channel = #channel{session = Se NChannel = Channel#channel{session = NSession}, handle_out(pubcomp, {PacketId, ?RC_SUCCESS}, NChannel); {error, RC = ?RC_PACKET_IDENTIFIER_NOT_FOUND} -> - ?LOG(warning, "The PUBREL PacketId ~w is not found.", [PacketId]), + ?SLOG(warning, #{msg => "pubrec_packetId_not_found", packetId => PacketId}), ok = emqx_metrics:inc('packets.pubrel.missed'), handle_out(pubcomp, {PacketId, RC}, Channel) end; @@ -420,7 +420,7 @@ handle_in(?PUBCOMP_PACKET(PacketId, _ReasonCode), Channel = #channel{session = S ok = emqx_metrics:inc('packets.pubcomp.inuse'), {ok, Channel}; {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND} -> - ?LOG(warning, "The PUBCOMP PacketId ~w is not found", [PacketId]), + ?SLOG(warning, #{msg => "pubcomp_packetId_not_found", packetId => PacketId}), ok = emqx_metrics:inc('packets.pubcomp.missed'), {ok, Channel} end; @@ -501,11 +501,11 @@ handle_in({frame_error, Reason}, Channel = #channel{conn_state = ConnState}) handle_out(disconnect, {?RC_MALFORMED_PACKET, Reason}, Channel); handle_in({frame_error, Reason}, Channel = #channel{conn_state = disconnected}) -> - ?LOG(error, "Unexpected frame error: ~p", [Reason]), + ?SLOG(error, #{msg => "malformed_mqtt_message", reason => Reason}), {ok, Channel}; handle_in(Packet, Channel) -> - ?LOG(error, "Unexpected incoming: ~p", [Packet]), + ?SLOG(error, #{msg => "disconnecting_due_to_unexpected_message", packet => Packet}), handle_out(disconnect, ?RC_PROTOCOL_ERROR, Channel). %%-------------------------------------------------------------------- @@ -529,7 +529,7 @@ process_connect(AckProps, Channel = #channel{conninfo = ConnInfo, {error, client_id_unavailable} -> handle_out(connack, ?RC_CLIENT_IDENTIFIER_NOT_VALID, Channel); {error, Reason} -> - ?LOG(error, "Failed to open session due to ~p", [Reason]), + ?SLOG(error, #{msg => "failed_to_open_session", reason => Reason}), handle_out(connack, ?RC_UNSPECIFIED_ERROR, Channel) end. @@ -548,8 +548,11 @@ process_publish(Packet = ?PUBLISH_PACKET(QoS, Topic, PacketId), Channel) -> Msg = packet_to_message(NPacket, NChannel), do_publish(PacketId, Msg, NChannel); {error, Rc = ?RC_NOT_AUTHORIZED, NChannel} -> - ?LOG(warning, "Cannot publish message to ~s due to ~s.", - [Topic, emqx_reason_codes:text(Rc)]), + ?SLOG(warning, #{ + msg => "cannot_publish_to_topic", + topic => Topic, + reason => emqx_reason_codes:name(Rc) + }), case emqx:get_config([authorization, deny_action], ignore) of ignore -> case QoS of @@ -563,8 +566,11 @@ process_publish(Packet = ?PUBLISH_PACKET(QoS, Topic, PacketId), Channel) -> handle_out(disconnect, Rc, NChannel) end; {error, Rc = ?RC_QUOTA_EXCEEDED, NChannel} -> - ?LOG(warning, "Cannot publish messages to ~s due to ~s.", - [Topic, emqx_reason_codes:text(Rc)]), + ?SLOG(warning, #{ + msg => "cannot_publish_to_topic", + topic => Topic, + reason => emqx_reason_codes:name(Rc) + }), case QoS of ?QOS_0 -> ok = emqx_metrics:inc('packets.publish.dropped'), @@ -575,8 +581,11 @@ process_publish(Packet = ?PUBLISH_PACKET(QoS, Topic, PacketId), Channel) -> handle_out(pubrec, {PacketId, Rc}, NChannel) end; {error, Rc, NChannel} -> - ?LOG(warning, "Cannot publish message to ~s due to ~s.", - [Topic, emqx_reason_codes:text(Rc)]), + ?SLOG(warning, #{ + msg => "cannot_publish_to_topic", + topic => Topic, + reason => emqx_reason_codes:name(Rc) + }), handle_out(disconnect, Rc, NChannel) end. @@ -621,8 +630,11 @@ do_publish(PacketId, Msg = #message{qos = ?QOS_2}, ok = emqx_metrics:inc('packets.publish.inuse'), handle_out(pubrec, {PacketId, RC}, Channel); {error, RC = ?RC_RECEIVE_MAXIMUM_EXCEEDED} -> - ?LOG(warning, "Dropped the qos2 packet ~w " - "due to awaiting_rel is full.", [PacketId]), + ?SLOG(warning, #{ + msg => "dropped_qos2_packet", + reason => emqx_reason_codes:name(RC), + packetId => PacketId + }), ok = emqx_metrics:inc('packets.publish.dropped'), handle_out(pubrec, {PacketId, RC}, Channel) end. @@ -671,8 +683,10 @@ process_subscribe([Topic = {TopicFilter, SubOpts}|More], SubProps, Channel, Acc) Channel), process_subscribe(More, SubProps, NChannel, [{Topic, ReasonCode} | Acc]); {error, ReasonCode} -> - ?LOG(warning, "Cannot subscribe ~s due to ~s.", - [TopicFilter, emqx_reason_codes:text(ReasonCode)]), + ?SLOG(warning, #{ + msg => "cannot_subscribe_topic_filter", + reason => emqx_reason_codes:name(ReasonCode) + }), process_subscribe(More, SubProps, Channel, [{Topic, ReasonCode} | Acc]) end. @@ -685,8 +699,10 @@ do_subscribe(TopicFilter, SubOpts = #{qos := QoS}, Channel = {ok, NSession} -> {QoS, Channel#channel{session = NSession}}; {error, RC} -> - ?LOG(warning, "Cannot subscribe ~s due to ~s.", - [TopicFilter, emqx_reason_codes:text(RC)]), + ?SLOG(warning, #{ + msg => "cannot_subscribe_topic_filter", + reason => emqx_reason_codes:text(RC) + }), {RC, Channel} end. @@ -869,7 +885,7 @@ handle_out(auth, {ReasonCode, Properties}, Channel) -> {ok, ?AUTH_PACKET(ReasonCode, Properties), Channel}; handle_out(Type, Data, Channel) -> - ?LOG(error, "Unexpected outgoing: ~s, ~p", [Type, Data]), + ?SLOG(error, #{msg => "unexpected_outgoing", type => Type, data => Data}), {ok, Channel}. %%-------------------------------------------------------------------- @@ -964,7 +980,7 @@ handle_call({quota, Policy}, Channel) -> reply(ok, Channel#channel{quota = Quota}); handle_call(Req, Channel) -> - ?LOG(error, "Unexpected call: ~p", [Req]), + ?SLOG(error, #{msg => "unexpected_call", call => Req}), reply(ignored, Channel). %%-------------------------------------------------------------------- @@ -1004,7 +1020,7 @@ handle_info({sock_closed, Reason}, Channel = end; handle_info({sock_closed, Reason}, Channel = #channel{conn_state = disconnected}) -> - ?LOG(error, "Unexpected sock_closed: ~p", [Reason]), + ?SLOG(error, #{msg => "unexpected_sock_close", reason => Reason}), {ok, Channel}; handle_info(clean_authz_cache, Channel) -> @@ -1012,7 +1028,7 @@ handle_info(clean_authz_cache, Channel) -> {ok, Channel}; handle_info(Info, Channel) -> - ?LOG(error, "Unexpected info: ~p", [Info]), + ?SLOG(error, #{msg => "unexpected_info", info => Info}), {ok, Channel}. %%-------------------------------------------------------------------- @@ -1075,7 +1091,7 @@ handle_timeout(_TRef, expire_quota_limit, Channel) -> {ok, clean_timer(quota_timer, Channel)}; handle_timeout(_TRef, Msg, Channel) -> - ?LOG(error, "Unexpected timeout: ~p~n", [Msg]), + ?SLOG(error, #{msg => "unexpected_timeout", timeout_message => Msg}), {ok, Channel}. %%-------------------------------------------------------------------- diff --git a/apps/emqx/src/emqx_cm.erl b/apps/emqx/src/emqx_cm.erl index f4f5f3981..c76c8d396 100644 --- a/apps/emqx/src/emqx_cm.erl +++ b/apps/emqx/src/emqx_cm.erl @@ -266,9 +266,8 @@ get_mqtt_conf(Zone, Key) -> emqx_config:get_zone_conf(Zone, [mqtt, Key]). %% @doc Try to takeover a session. --spec(takeover_session(emqx_types:clientid()) - -> {error, term()} - | {ok, atom(), pid(), emqx_session:session()}). +-spec(takeover_session(emqx_types:clientid()) -> + {error, term()} | {ok, atom(), pid(), emqx_session:session()}). takeover_session(ClientId) -> case lookup_channels(ClientId) of [] -> {error, not_found}; @@ -276,7 +275,7 @@ takeover_session(ClientId) -> takeover_session(ClientId, ChanPid); ChanPids -> [ChanPid|StalePids] = lists:reverse(ChanPids), - ?LOG(error, "More than one channel found: ~p", [ChanPids]), + ?SLOG(warning, #{msg => "more_than_one_channel_found", chan_pids => ChanPids}), lists:foreach(fun(StalePid) -> catch discard_session(ClientId, StalePid) end, StalePids), @@ -341,7 +340,7 @@ kick_session(ClientId) -> kick_session(ClientId, ChanPid); ChanPids -> [ChanPid|StalePids] = lists:reverse(ChanPids), - ?LOG(error, "More than one channel found: ~p", [ChanPids]), + ?SLOG(warning, #{msg => "more_than_one_channel_found", chan_pids => ChanPids}), lists:foreach(fun(StalePid) -> catch discard_session(ClientId, StalePid) end, StalePids), @@ -409,14 +408,14 @@ cast(Msg) -> gen_server:cast(?CM, Msg). init([]) -> TabOpts = [public, {write_concurrency, true}], - ok = emqx_tables:new(?CHAN_TAB, [bag, {read_concurrency, true}|TabOpts]), + ok = emqx_tables:new(?CHAN_TAB, [bag, {read_concurrency, true} | TabOpts]), ok = emqx_tables:new(?CHAN_CONN_TAB, [bag | TabOpts]), ok = emqx_tables:new(?CHAN_INFO_TAB, [set, compressed | TabOpts]), ok = emqx_stats:update_interval(chan_stats, fun ?MODULE:stats_fun/0), {ok, #{chan_pmon => emqx_pmon:new()}}. handle_call(Req, _From, State) -> - ?LOG(error, "Unexpected call: ~p", [Req]), + ?SLOG(error, #{msg => "unexpected_call", call => Req}), {reply, ignored, State}. handle_cast({registered, {ClientId, ChanPid}}, State = #{chan_pmon := PMon}) -> @@ -424,7 +423,7 @@ handle_cast({registered, {ClientId, ChanPid}}, State = #{chan_pmon := PMon}) -> {noreply, State#{chan_pmon := PMon1}}; handle_cast(Msg, State) -> - ?LOG(error, "Unexpected cast: ~p", [Msg]), + ?SLOG(error, #{msg => "unexpected_cast", cast => Msg}), {noreply, State}. handle_info({'DOWN', _MRef, process, Pid, _Reason}, State = #{chan_pmon := PMon}) -> @@ -434,7 +433,8 @@ handle_info({'DOWN', _MRef, process, Pid, _Reason}, State = #{chan_pmon := PMon} {noreply, State#{chan_pmon := PMon1}}; handle_info(Info, State) -> - ?LOG(error, "Unexpected info: ~p", [Info]), + ?SLOG(error, #{msg => "unexpected_info", info => Info}), + {noreply, State}. terminate(_Reason, _State) -> diff --git a/apps/emqx/src/emqx_cm_registry.erl b/apps/emqx/src/emqx_cm_registry.erl index 6fc34dee8..ef7ad6131 100644 --- a/apps/emqx/src/emqx_cm_registry.erl +++ b/apps/emqx/src/emqx_cm_registry.erl @@ -114,11 +114,11 @@ init([]) -> {ok, #{}}. handle_call(Req, _From, State) -> - ?LOG(error, "Unexpected call: ~p", [Req]), + ?SLOG(error, #{msg => "unexpected_call", call => Req}), {reply, ignored, State}. handle_cast(Msg, State) -> - ?LOG(error, "Unexpected cast: ~p", [Msg]), + ?SLOG(error, #{msg => "unexpected_cast", cast => Msg}), {noreply, State}. handle_info({membership, {mnesia, down, Node}}, State) -> @@ -132,7 +132,7 @@ handle_info({membership, _Event}, State) -> {noreply, State}; handle_info(Info, State) -> - ?LOG(error, "Unexpected info: ~p", [Info]), + ?SLOG(error, #{msg => "unexpected_info", info => Info}), {noreply, State}. terminate(_Reason, _State) -> diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index 98466d3df..05dd3d122 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -66,6 +66,8 @@ , find_listener_conf/3 ]). +-include("logger.hrl"). + -define(CONF, conf). -define(RAW_CONF, raw_conf). -define(PERSIS_SCHEMA_MODS, {?MODULE, schema_mods}). @@ -250,7 +252,7 @@ init_load(SchemaMod, Conf) when is_list(Conf) orelse is_binary(Conf) -> {ok, RawRichConf} -> init_load(SchemaMod, RawRichConf); {error, Reason} -> - logger:error(#{msg => failed_to_load_hocon_conf, + ?SLOG(error, #{msg => failed_to_load_hocon_conf, reason => Reason }), error(failed_to_load_hocon_conf) @@ -294,7 +296,8 @@ fill_defaults(RawConf) -> -spec fill_defaults(module(), raw_config()) -> map(). fill_defaults(SchemaMod, RawConf) -> hocon_schema:check_plain(SchemaMod, RawConf, - #{nullable => true, no_conversion => true}, root_names_from_conf(RawConf)). + #{nullable => true, only_fill_defaults => true}, + root_names_from_conf(RawConf)). -spec read_override_conf() -> raw_config(). read_override_conf() -> @@ -358,7 +361,9 @@ save_to_override_conf(RawConf) -> case file:write_file(FileName, hocon_pp:do(RawConf, #{})) of ok -> ok; {error, Reason} -> - logger:error("write to ~s failed, ~p", [FileName, Reason]), + ?SLOG(error, #{msg => failed_to_write_override_file, + filename => FileName, + reason => Reason}), {error, Reason} end end. diff --git a/apps/emqx/src/emqx_config_handler.erl b/apps/emqx/src/emqx_config_handler.erl index e47bb489e..83db8e480 100644 --- a/apps/emqx/src/emqx_config_handler.erl +++ b/apps/emqx/src/emqx_config_handler.erl @@ -77,7 +77,7 @@ stop() -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. update_config(SchemaModule, ConfKeyPath, UpdateArgs) -> ?ATOM_CONF_PATH(ConfKeyPath, gen_server:call(?MODULE, {change_config, SchemaModule, - AtomKeyPath, UpdateArgs}), {error, ConfKeyPath}). + AtomKeyPath, UpdateArgs}), {error, {not_found, ConfKeyPath}}). -spec add_handler(emqx_config:config_key_path(), handler_name()) -> ok. add_handler(ConfKeyPath, HandlerName) -> @@ -117,7 +117,12 @@ handle_call({change_config, SchemaModule, ConfKeyPath, UpdateArgs}, _From, {error, Result} end catch Error:Reason:ST -> - ?LOG(error, "change_config failed: ~p", [{Error, Reason, ST}]), + ?SLOG(error, #{ + msg => "change_config_failed", + exception => Error, + reason => Reason, + stacktrace => ST + }), {error, Reason} end, {reply, Reply, State}; diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl index 26eb346a4..cb6e2ce8f 100644 --- a/apps/emqx/src/emqx_connection.erl +++ b/apps/emqx/src/emqx_connection.erl @@ -417,14 +417,14 @@ handle_msg({'$gen_cast', Req}, State) -> {ok, NewState}; handle_msg({Inet, _Sock, Data}, State) when Inet == tcp; Inet == ssl -> - ?LOG(debug, "RECV ~0p", [Data]), + ?SLOG(debug, #{msg => "RECV_data", data => Data, transport => Inet}), Oct = iolist_size(Data), inc_counter(incoming_bytes, Oct), ok = emqx_metrics:inc('bytes.received', Oct), parse_incoming(Data, State); handle_msg({quic, Data, _Sock, _, _, _}, State) -> - ?LOG(debug, "RECV ~0p", [Data]), + ?SLOG(debug, #{msg => "RECV_data", data => Data, transport => quic}), Oct = iolist_size(Data), inc_counter(incoming_bytes, Oct), ok = emqx_metrics:inc('bytes.received', Oct), @@ -489,7 +489,7 @@ handle_msg({connack, ConnAck}, State) -> handle_outgoing(ConnAck, State); handle_msg({close, Reason}, State) -> - ?LOG(debug, "Force to close the socket due to ~p", [Reason]), + ?SLOG(debug, #{msg => "force_socket_close", reason => Reason}), handle_info({sock_closed, Reason}, close_socket(State)); handle_msg({event, connected}, State = #state{channel = Channel}) -> @@ -644,10 +644,21 @@ parse_incoming(Data, Packets, State = #state{parse_state = ParseState}) -> NState = State#state{parse_state = NParseState}, parse_incoming(Rest, [Packet|Packets], NState) catch - error:Reason:Stk -> - ?LOG(error, "~nParse failed for ~0p~n~0p~nFrame data:~0p", - [Reason, Stk, Data]), - {[{frame_error, Reason}|Packets], State} + throw : ?FRAME_PARSE_ERROR(Reason) -> + ?SLOG(info, #{ reason => Reason + , at_state => emqx_frame:describe_state(ParseState) + , input_bytes => Data + , parsed_packets => Packets + }), + {[{frame_error, Reason} | Packets], State}; + error : Reason : Stacktrace -> + ?SLOG(error, #{ at_state => emqx_frame:describe_state(ParseState) + , input_bytes => Data + , parsed_packets => Packets + , reason => Reason + , stacktrace => Stacktrace + }), + {[{frame_error, Reason} | Packets], State} end. -compile({inline, [next_incoming_msgs/1]}). @@ -661,7 +672,7 @@ next_incoming_msgs(Packets) -> handle_incoming(Packet, State) when is_record(Packet, mqtt_packet) -> ok = inc_incoming_stats(Packet), - ?LOG(debug, "RECV ~s", [emqx_packet:format(Packet)]), + ?SLOG(debug, #{msg => "RECV_packet", packet => Packet}), with_channel(handle_in, [Packet], State); handle_incoming(FrameError, State) -> @@ -696,15 +707,32 @@ handle_outgoing(Packet, State) -> serialize_and_inc_stats_fun(#state{serialize = Serialize}) -> fun(Packet) -> - case emqx_frame:serialize_pkt(Packet, Serialize) of - <<>> -> ?LOG(warning, "~s is discarded due to the frame is too large!", - [emqx_packet:format(Packet)]), + try emqx_frame:serialize_pkt(Packet, Serialize) of + <<>> -> ?SLOG(warning, #{ + msg => "packet_is_discarded", + reason => "frame_is_too_large", + packet => emqx_packet:format(Packet) + }), ok = emqx_metrics:inc('delivery.dropped.too_large'), ok = emqx_metrics:inc('delivery.dropped'), <<>>; - Data -> ?LOG(debug, "SEND ~s", [emqx_packet:format(Packet)]), + Data -> ?SLOG(debug, #{ + msg => "SEND_packet", + packet => emqx_packet:format(Packet) + }), ok = inc_outgoing_stats(Packet), Data + catch + %% Maybe Never happen. + throw : ?FRAME_SERIALIZE_ERROR(Reason) -> + ?SLOG(info, #{ reason => Reason + , input_packet => Packet}), + erlang:error(?FRAME_SERIALIZE_ERROR(Reason)); + error : Reason : Stacktrace -> + ?SLOG(error, #{ input_packet => Packet + , exception => Reason + , stacktrace => Stacktrace}), + erlang:error(frame_serialize_error) end end. @@ -741,7 +769,7 @@ handle_info(activate_socket, State = #state{sockstate = OldSst}) -> handle_info({sock_error, Reason}, State) -> case Reason =/= closed andalso Reason =/= einval of - true -> ?LOG(warning, "socket_error: ~p", [Reason]); + true -> ?SLOG(warning, #{msg => "socket_error", reason => Reason}); false -> ok end, handle_info({sock_closed, Reason}, close_socket(State)); @@ -783,7 +811,7 @@ ensure_rate_limit(Stats, State = #state{limiter = Limiter}) -> {ok, Limiter1} -> State#state{limiter = Limiter1}; {pause, Time, Limiter1} -> - ?LOG(warning, "Pause ~pms due to rate limit", [Time]), + ?SLOG(warning, #{msg => "pause_time_due_to_rate_limit", time_in_ms => Time}), TRef = start_timer(Time, limit_timeout), State#state{sockstate = blocked, limiter = Limiter1, diff --git a/apps/emqx/src/emqx_ctl.erl b/apps/emqx/src/emqx_ctl.erl index a71398095..52930e714 100644 --- a/apps/emqx/src/emqx_ctl.erl +++ b/apps/emqx/src/emqx_ctl.erl @@ -185,13 +185,13 @@ handle_call({register_command, Cmd, MF, Opts}, _From, State = #state{seq = Seq}) case ets:match(?CMD_TAB, {{'$1', Cmd}, '_', '_'}) of [] -> ets:insert(?CMD_TAB, {{Seq, Cmd}, MF, Opts}); [[OriginSeq] | _] -> - ?LOG(warning, "CMD ~s is overidden by ~p", [Cmd, MF]), + ?SLOG(warning, #{msg => "CMD_overidden", cmd => Cmd, mf => MF}), true = ets:insert(?CMD_TAB, {{OriginSeq, Cmd}, MF, Opts}) end, {reply, ok, next_seq(State)}; handle_call(Req, _From, State) -> - ?LOG(error, "Unexpected call: ~p", [Req]), + ?SLOG(error, #{msg => "unexpected_call", call => Req}), {reply, ignored, State}. handle_cast({unregister_command, Cmd}, State) -> @@ -199,11 +199,11 @@ handle_cast({unregister_command, Cmd}, State) -> noreply(State); handle_cast(Msg, State) -> - ?LOG(error, "Unexpected cast: ~p", [Msg]), + ?SLOG(error, #{msg => "unexpected_cast", cast => Msg}), noreply(State). handle_info(Info, State) -> - ?LOG(error, "Unexpected info: ~p", [Info]), + ?SLOG(error, #{msg => "unexpected_info", info => Info}), noreply(State). terminate(_Reason, _State) -> diff --git a/apps/emqx/src/emqx_flapping.erl b/apps/emqx/src/emqx_flapping.erl index 1908430be..0b4611c4c 100644 --- a/apps/emqx/src/emqx_flapping.erl +++ b/apps/emqx/src/emqx_flapping.erl @@ -106,7 +106,7 @@ init([]) -> {ok, #{}, hibernate}. handle_call(Req, _From, State) -> - ?LOG(error, "Unexpected call: ~p", [Req]), + ?SLOG(error, #{msg => "unexpected_call", call => Req}), {reply, ignored, State}. handle_cast({detected, #flapping{clientid = ClientId, @@ -116,8 +116,13 @@ handle_cast({detected, #flapping{clientid = ClientId, #{window_time := WindTime, ban_time := Interval}}, State) -> case now_diff(StartedAt) < WindTime of true -> %% Flapping happened:( - ?LOG(error, "Flapping detected: ~s(~s) disconnected ~w times in ~wms", - [ClientId, inet:ntoa(PeerHost), DetectCnt, WindTime]), + ?SLOG(warning, #{ + msg => "flapping_detected", + client_id => ClientId, + peer_host => fmt_host(PeerHost), + detect_cnt => DetectCnt, + wind_time_in_ms => WindTime + }), Now = erlang:system_time(second), Banned = #banned{who = {clientid, ClientId}, by = <<"flapping detector">>, @@ -126,13 +131,18 @@ handle_cast({detected, #flapping{clientid = ClientId, until = Now + (Interval div 1000)}, emqx_banned:create(Banned); false -> - ?LOG(warning, "~s(~s) disconnected ~w times in ~wms", - [ClientId, inet:ntoa(PeerHost), DetectCnt, Interval]) + ?SLOG(warning, #{ + msg => "client_disconnected", + client_id => ClientId, + peer_host => fmt_host(PeerHost), + detect_cnt => DetectCnt, + interval => Interval + }) end, {noreply, State}; handle_cast(Msg, State) -> - ?LOG(error, "Unexpected cast: ~p", [Msg]), + ?SLOG(error, #{msg => "unexpected_cast", cast => Msg}), {noreply, State}. handle_info({timeout, _TRef, {garbage_collect, Zone}}, State) -> @@ -144,7 +154,7 @@ handle_info({timeout, _TRef, {garbage_collect, Zone}}, State) -> {noreply, State, hibernate}; handle_info(Info, State) -> - ?LOG(error, "Unexpected info: ~p", [Info]), + ?SLOG(error, #{msg => "unexpected_info", info => Info}), {noreply, State}. terminate(_Reason, _State) -> @@ -161,3 +171,8 @@ start_timers() -> lists:foreach(fun({Zone, _ZoneConf}) -> start_timer(Zone) end, maps:to_list(emqx:get_config([zones], #{}))). + +fmt_host(PeerHost) -> + try inet:ntoa(PeerHost) + catch _:_ -> PeerHost + end. diff --git a/apps/emqx/src/emqx_frame.erl b/apps/emqx/src/emqx_frame.erl index cea94eec8..2fe1b6d1a 100644 --- a/apps/emqx/src/emqx_frame.erl +++ b/apps/emqx/src/emqx_frame.erl @@ -34,6 +34,10 @@ , serialize/2 ]). + +-export([ describe_state/1 + ]). + -export_type([ options/0 , parse_state/0 , parse_result/0 @@ -47,7 +51,9 @@ version => emqx_types:proto_ver() }). --type(parse_state() :: {none, options()} | {cont_state(), options()}). +-define(NONE(Options), {none, Options}). + +-type(parse_state() :: ?NONE(options()) | {cont_state(), options()}). -type(parse_result() :: {more, parse_state()} | {ok, emqx_types:packet(), binary(), parse_state()}). @@ -61,27 +67,45 @@ -type(serialize_opts() :: options()). --define(none(Options), {none, Options}). - -define(DEFAULT_OPTIONS, #{strict_mode => false, max_size => ?MAX_PACKET_SIZE, version => ?MQTT_PROTO_V4 }). +-define(PARSE_ERR(Reason), ?THROW_FRAME_ERROR(Reason)). +-define(SERIALIZE_ERR(Reason), ?THROW_SERIALIZE_ERROR(Reason)). + +-define(MULTIPLIER_MAX, 16#200000). + -dialyzer({no_match, [serialize_utf8_string/2]}). +%% @doc Describe state for logging. +describe_state(?NONE(_Opts)) -> <<"clean">>; +describe_state({{len, _}, _Opts}) -> <<"parsing_varint_length">>; +describe_state({{body, State}, _Opts}) -> + #{ hdr := Hdr + , len := Len + } = State, + Desc = #{ parsed_header => Hdr + , expected_bytes => Len + }, + case maps:get(rest, State, undefined) of + undefined -> Desc; + Body -> Desc#{received_bytes => body_bytes(Body)} + end. + %%-------------------------------------------------------------------- %% Init Parse State %%-------------------------------------------------------------------- --spec(initial_parse_state() -> {none, options()}). +-spec(initial_parse_state() -> ?NONE(options())). initial_parse_state() -> initial_parse_state(#{}). --spec(initial_parse_state(options()) -> {none, options()}). +-spec(initial_parse_state(options()) -> ?NONE(options())). initial_parse_state(Options) when is_map(Options) -> - ?none(maps:merge(?DEFAULT_OPTIONS, Options)). + ?NONE(maps:merge(?DEFAULT_OPTIONS, Options)). %%-------------------------------------------------------------------- %% Parse MQTT Frame @@ -92,10 +116,10 @@ parse(Bin) -> parse(Bin, initial_parse_state()). -spec(parse(binary(), parse_state()) -> parse_result()). -parse(<<>>, {none, Options}) -> - {more, {none, Options}}; +parse(<<>>, ?NONE(Options)) -> + {more, ?NONE(Options)}; parse(<>, - {none, Options = #{strict_mode := StrictMode}}) -> + ?NONE(Options = #{strict_mode := StrictMode})) -> %% Validate header if strict mode. StrictMode andalso validate_header(Type, Dup, QoS, Retain), Header = #mqtt_packet_header{type = Type, @@ -123,14 +147,14 @@ parse_remaining_len(Rest, Header, Options) -> parse_remaining_len(_Bin, _Header, _Multiplier, Length, #{max_size := MaxSize}) when Length > MaxSize -> - error(frame_too_large); + ?PARSE_ERR(frame_too_large); parse_remaining_len(<<>>, Header, Multiplier, Length, Options) -> {more, {{len, #{hdr => Header, len => {Multiplier, Length}}}, Options}}; %% Match DISCONNECT without payload parse_remaining_len(<<0:8, Rest/binary>>, Header = #mqtt_packet_header{type = ?DISCONNECT}, 1, 0, Options) -> Packet = packet(Header, #mqtt_packet_disconnect{reason_code = ?RC_SUCCESS}), - {ok, Packet, Rest, ?none(Options)}; + {ok, Packet, Rest, ?NONE(Options)}; %% Match PINGREQ. parse_remaining_len(<<0:8, Rest/binary>>, Header, 1, 0, Options) -> parse_frame(Rest, Header, 0, Options); @@ -138,21 +162,22 @@ parse_remaining_len(<<0:8, Rest/binary>>, Header, 1, 0, Options) -> parse_remaining_len(<<0:1, 2:7, Rest/binary>>, Header, 1, 0, Options) -> parse_frame(Rest, Header, 2, Options); parse_remaining_len(<<1:1, _Len:7, _Rest/binary>>, _Header, Multiplier, _Value, _Options) - when Multiplier > 2097152 -> - error(malformed_variable_byte_integer); + when Multiplier > ?MULTIPLIER_MAX -> + ?PARSE_ERR(malformed_variable_byte_integer); parse_remaining_len(<<1:1, Len:7, Rest/binary>>, Header, Multiplier, Value, Options) -> parse_remaining_len(Rest, Header, Multiplier * ?HIGHBIT, Value + Len * Multiplier, Options); parse_remaining_len(<<0:1, Len:7, Rest/binary>>, Header, Multiplier, Value, Options = #{max_size := MaxSize}) -> FrameLen = Value + Len * Multiplier, case FrameLen > MaxSize of - true -> error(frame_too_large); + true -> ?PARSE_ERR(frame_too_large); false -> parse_frame(Rest, Header, FrameLen, Options) end. body_bytes(B) when is_binary(B) -> size(B); body_bytes(?Q(Bytes, _)) -> Bytes. +append_body(H, <<>>) -> H; append_body(H, T) when is_binary(H) andalso size(H) < 1024 -> <>; append_body(H, T) when is_binary(H) -> @@ -165,18 +190,18 @@ flatten_body(Body) when is_binary(Body) -> Body; flatten_body(?Q(_, Q)) -> iolist_to_binary(queue:to_list(Q)). parse_frame(Body, Header, 0, Options) -> - {ok, packet(Header), flatten_body(Body), ?none(Options)}; + {ok, packet(Header), flatten_body(Body), ?NONE(Options)}; parse_frame(Body, Header, Length, Options) -> case body_bytes(Body) >= Length of true -> <> = flatten_body(Body), case parse_packet(Header, FrameBin, Options) of {Variable, Payload} -> - {ok, packet(Header, Variable, Payload), Rest, ?none(Options)}; + {ok, packet(Header, Variable, Payload), Rest, ?NONE(Options)}; Variable = #mqtt_packet_connect{proto_ver = Ver} -> - {ok, packet(Header, Variable), Rest, ?none(Options#{version := Ver})}; + {ok, packet(Header, Variable), Rest, ?NONE(Options#{version := Ver})}; Variable -> - {ok, packet(Header, Variable), Rest, ?none(Options)} + {ok, packet(Header, Variable), Rest, ?NONE(Options)} end; false -> {more, {{body, #{hdr => Header, @@ -420,10 +445,16 @@ parse_property(<<16#28, Val, Bin/binary>>, Props) -> parse_property(<<16#29, Val, Bin/binary>>, Props) -> parse_property(Bin, Props#{'Subscription-Identifier-Available' => Val}); parse_property(<<16#2A, Val, Bin/binary>>, Props) -> - parse_property(Bin, Props#{'Shared-Subscription-Available' => Val}). + parse_property(Bin, Props#{'Shared-Subscription-Available' => Val}); +parse_property(<>, _Props) -> + ?PARSE_ERR(#{invalid_property_code => Property}). +%% TODO: invalid property in specific packet. parse_variable_byte_integer(Bin) -> parse_variable_byte_integer(Bin, 1, 0). +parse_variable_byte_integer(<<1:1, _Len:7, _Rest/binary>>, Multiplier, _Value) + when Multiplier > ?MULTIPLIER_MAX -> + ?PARSE_ERR(malformed_variable_byte_integer); parse_variable_byte_integer(<<1:1, Len:7, Rest/binary>>, Multiplier, Value) -> parse_variable_byte_integer(Rest, Multiplier * ?HIGHBIT, Value + Len * Multiplier); parse_variable_byte_integer(<<0:1, Len:7, Rest/binary>>, Multiplier, Value) -> @@ -441,7 +472,23 @@ parse_reason_codes(Bin) -> parse_utf8_pair(<>) -> - {{Key, Val}, Rest}. + {{Key, Val}, Rest}; +parse_utf8_pair(<>) + when LenK > byte_size(Rest) -> + ?PARSE_ERR(#{ hint => user_property_not_enough_bytes + , parsed_key_length => LenK + , remaining_bytes_length => byte_size(Rest)}); +parse_utf8_pair(<>) + when LenV > byte_size(Rest) -> + ?PARSE_ERR(#{ hint => malformed_user_property_value + , parsed_key_length => LenK + , parsed_value_length => LenV + , remaining_bytes_length => byte_size(Rest)}); +parse_utf8_pair(Bin) + when 4 > byte_size(Bin) -> + ?PARSE_ERR(#{ hint => user_property_not_enough_bytes + , total_bytes => byte_size(Bin)}). parse_utf8_string(Bin, false) -> {undefined, Bin}; @@ -449,10 +496,26 @@ parse_utf8_string(Bin, true) -> parse_utf8_string(Bin). parse_utf8_string(<>) -> - {Str, Rest}. + {Str, Rest}; +parse_utf8_string(<>) + when Len > byte_size(Rest) -> + ?PARSE_ERR(#{ hint => malformed_utf8_string + , parsed_length => Len + , remaining_bytes_length => byte_size(Rest)}); +parse_utf8_string(Bin) + when 2 > byte_size(Bin) -> + ?PARSE_ERR(malformed_utf8_string_length). parse_binary_data(<>) -> - {Data, Rest}. + {Data, Rest}; +parse_binary_data(<>) + when Len > byte_size(Rest) -> + ?PARSE_ERR(#{ hint => malformed_binary_data + , parsed_length => Len + , remaining_bytes_length => byte_size(Rest)}); +parse_binary_data(Bin) + when 2 > byte_size(Bin) -> + ?PARSE_ERR(malformed_binary_data_length). %%-------------------------------------------------------------------- %% Serialize MQTT Packet @@ -719,7 +782,7 @@ serialize_binary_data(Bin) -> [<<(byte_size(Bin)):16/big-unsigned-integer>>, Bin]. serialize_utf8_string(undefined, false) -> - error(utf8_string_undefined); + ?SERIALIZE_ERR(utf8_string_undefined); serialize_utf8_string(undefined, true) -> <<>>; serialize_utf8_string(String, _AllowNull) -> @@ -767,13 +830,13 @@ validate_header(?PINGREQ, 0, 0, 0) -> ok; validate_header(?PINGRESP, 0, 0, 0) -> ok; validate_header(?DISCONNECT, 0, 0, 0) -> ok; validate_header(?AUTH, 0, 0, 0) -> ok; -validate_header(_Type, _Dup, _QoS, _Rt) -> error(bad_frame_header). +validate_header(_Type, _Dup, _QoS, _Rt) -> ?PARSE_ERR(bad_frame_header). -compile({inline, [validate_packet_id/1]}). -validate_packet_id(0) -> error(bad_packet_id); +validate_packet_id(0) -> ?PARSE_ERR(bad_packet_id); validate_packet_id(_) -> ok. -validate_subqos([3|_]) -> error(bad_subqos); +validate_subqos([3|_]) -> ?PARSE_ERR(bad_subqos); validate_subqos([_|T]) -> validate_subqos(T); validate_subqos([]) -> ok. diff --git a/apps/emqx/src/emqx_hooks.erl b/apps/emqx/src/emqx_hooks.erl index 088bb4085..7817a9b2d 100644 --- a/apps/emqx/src/emqx_hooks.erl +++ b/apps/emqx/src/emqx_hooks.erl @@ -67,7 +67,7 @@ %% - The execution order is the adding order of callbacks if they have %% equal priority values. --type(hookpoint() :: atom()). +-type(hookpoint() :: atom() | binary()). -type(action() :: {module(), atom(), [term()] | undefined}). -type(filter() :: {module(), atom(), [term()] | undefined}). @@ -158,12 +158,12 @@ del(HookPoint, Action) -> gen_server:cast(?SERVER, {del, HookPoint, Action}). %% @doc Run hooks. --spec(run(atom(), list(Arg::term())) -> ok). +-spec(run(hookpoint(), list(Arg::term())) -> ok). run(HookPoint, Args) -> do_run(lookup(HookPoint), Args). %% @doc Run hooks with Accumulator. --spec(run_fold(atom(), list(Arg::term()), Acc::term()) -> Acc::term()). +-spec(run_fold(hookpoint(), list(Arg::term()), Acc::term()) -> Acc::term()). run_fold(HookPoint, Args, Acc) -> do_run_fold(lookup(HookPoint), Args, Acc). @@ -206,8 +206,13 @@ safe_execute({M, F, A}, Args) -> Result -> Result catch Error:Reason:Stacktrace -> - ?LOG(error, "Failed to execute ~0p: ~0p", [{M, F, A}, {Error, Reason, Stacktrace}]), - ok + ?SLOG(error, #{ + msg => "failed_to_execute", + exception => Error, + reason => Reason, + stacktrace => Stacktrace, + failed_call => {M, F, A} + }) end. %% @doc execute a function. @@ -246,7 +251,7 @@ handle_call({put, HookPoint, Callback = #callback{action = {M, F, _}}}, _From, S {reply, Reply, State}; handle_call(Req, _From, State) -> - ?LOG(error, "Unexpected call: ~p", [Req]), + ?SLOG(error, #{msg => "unexpected_call", req => Req}), {reply, ignored, State}. handle_cast({del, HookPoint, Action}, State) -> @@ -259,11 +264,11 @@ handle_cast({del, HookPoint, Action}, State) -> {noreply, State}; handle_cast(Msg, State) -> - ?LOG(error, "Unexpected msg: ~p", [Msg]), + ?SLOG(error, #{msg => "unexpected_cast", req => Msg}), {noreply, State}. handle_info(Info, State) -> - ?LOG(error, "Unexpected info: ~p", [Info]), + ?SLOG(error, #{msg => "unexpected_info", info => Info}), {noreply, State}. terminate(_Reason, _State) -> diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 06d89c86d..5c4776207 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -46,6 +46,7 @@ -export([post_config_update/4]). -define(CONF_KEY_PATH, [listeners]). +-define(TYPES_STRING, ["tcp","ssl","ws","wss","quic"]). %% @doc List configured listeners. -spec(list() -> [{ListenerId :: atom(), ListenerConf :: map()}]). @@ -349,11 +350,10 @@ listener_id(Type, ListenerName) -> list_to_atom(lists:append([str(Type), ":", str(ListenerName)])). parse_listener_id(Id) -> - try - [Type, Name] = string:split(str(Id), ":", leading), - {list_to_existing_atom(Type), list_to_atom(Name)} - catch - _ : _ -> error({invalid_listener_id, Id}) + [Type, Name] = string:split(str(Id), ":", leading), + case lists:member(Type, ?TYPES_STRING) of + true -> {list_to_existing_atom(Type), list_to_atom(Name)}; + false -> {error, {invalid_listener_id, Id}} end. zone(Opts) -> diff --git a/apps/emqx/src/emqx_message.erl b/apps/emqx/src/emqx_message.erl index b70655fc5..57571eab0 100644 --- a/apps/emqx/src/emqx_message.erl +++ b/apps/emqx/src/emqx_message.erl @@ -66,6 +66,7 @@ -export([ to_packet/2 , to_map/1 + , to_log_map/1 , to_list/1 , from_map/1 ]). @@ -79,11 +80,10 @@ headers := emqx_types:headers(), topic := emqx_types:topic(), payload := emqx_types:payload(), - timestamp := integer()} + timestamp := integer(), + extra := _} ). --export([format/1]). - -elvis([{elvis_style, god_modules, disable}]). -spec(make(emqx_types:topic(), emqx_types:payload()) -> emqx_types:message()). @@ -292,7 +292,8 @@ to_map(#message{ headers = Headers, topic = Topic, payload = Payload, - timestamp = Timestamp + timestamp = Timestamp, + extra = Extra }) -> #{id => Id, qos => QoS, @@ -301,9 +302,13 @@ to_map(#message{ headers => Headers, topic => Topic, payload => Payload, - timestamp => Timestamp + timestamp => Timestamp, + extra => Extra }. +%% @doc To map for logging, with payload dropped. +to_log_map(Msg) -> maps:without([payload], to_map(Msg)). + %% @doc Message to tuple list -spec(to_list(emqx_types:message()) -> list()). to_list(Msg) -> @@ -318,7 +323,8 @@ from_map(#{id := Id, headers := Headers, topic := Topic, payload := Payload, - timestamp := Timestamp + timestamp := Timestamp, + extra := Extra }) -> #message{ id = Id, @@ -328,24 +334,10 @@ from_map(#{id := Id, headers = Headers, topic = Topic, payload = Payload, - timestamp = Timestamp + timestamp = Timestamp, + extra = Extra }. %% MilliSeconds elapsed(Since) -> max(0, erlang:system_time(millisecond) - Since). - -format(#message{id = Id, - qos = QoS, - topic = Topic, - from = From, - flags = Flags, - headers = Headers}) -> - io_lib:format("Message(Id=~s, QoS=~w, Topic=~s, From=~p, Flags=~s, Headers=~s)", - [Id, QoS, Topic, From, format(flags, Flags), format(headers, Headers)]). - -format(flags, Flags) -> - io_lib:format("~p", [[Flag || {Flag, true} <- maps:to_list(Flags)]]); -format(headers, Headers) -> - io_lib:format("~p", [Headers]). - diff --git a/apps/emqx/src/emqx_metrics.erl b/apps/emqx/src/emqx_metrics.erl index 282b8b5f3..740c29290 100644 --- a/apps/emqx/src/emqx_metrics.erl +++ b/apps/emqx/src/emqx_metrics.erl @@ -442,13 +442,17 @@ init([]) -> {ok, #state{next_idx = ?RESERVED_IDX + 1}, hibernate}. handle_call({create, Type, Name}, _From, State = #state{next_idx = ?MAX_SIZE}) -> - ?LOG(error, "Failed to create ~s:~s for index exceeded.", [Type, Name]), + ?SLOG(error, #{ + msg => "failed_to_create_type_name_for_index_exceeded", + type => Type, + name => Name + }), {reply, {error, metric_index_exceeded}, State}; handle_call({create, Type, Name}, _From, State = #state{next_idx = NextIdx}) -> case ets:lookup(?TAB, Name) of [#metric{idx = Idx}] -> - ?LOG(info, "~s already exists.", [Name]), + ?SLOG(info, #{msg => "name_already_exists", name => Name}), {reply, {ok, Idx}, State}; [] -> Metric = #metric{name = Name, type = Type, idx = NextIdx}, @@ -464,15 +468,15 @@ handle_call({set_type_to_counter, Keys}, _From, State) -> {reply, ok, State}; handle_call(Req, _From, State) -> - ?LOG(error, "Unexpected call: ~p", [Req]), + ?SLOG(error, #{msg => "unexpected_call", req => Req}), {reply, ignored, State}. handle_cast(Msg, State) -> - ?LOG(error, "Unexpected cast: ~p", [Msg]), + ?SLOG(error, #{msg => "unexpected_cast", req => Msg}), {noreply, State}. handle_info(Info, State) -> - ?LOG(error, "Unexpected info: ~p", [Info]), + ?SLOG(error, #{msg => "unexpected_info", info => Info}), {noreply, State}. terminate(_Reason, _State) -> diff --git a/apps/emqx/src/emqx_os_mon.erl b/apps/emqx/src/emqx_os_mon.erl index 85e448f41..24795c7ba 100644 --- a/apps/emqx/src/emqx_os_mon.erl +++ b/apps/emqx/src/emqx_os_mon.erl @@ -87,7 +87,7 @@ handle_call(Req, _From, State) -> {reply, {error, {unexpected_call, Req}}, State}. handle_cast(Msg, State) -> - ?LOG(error, "unexpected_cast_discarded: ~p", [Msg]), + ?SLOG(error, #{msg => "unexpected_cast", cast=> Msg}), {noreply, State}. handle_info({timeout, _Timer, check}, State) -> @@ -109,7 +109,7 @@ handle_info({timeout, _Timer, check}, State) -> {noreply, State}; handle_info(Info, State) -> - ?LOG(info, "unexpected_info_discarded: ~p", [Info]), + ?SLOG(error, #{msg => "unexpected_info", info => Info}), {noreply, State}. terminate(_Reason, _State) -> diff --git a/apps/emqx/src/emqx_passwd.erl b/apps/emqx/src/emqx_passwd.erl index f6c382ef9..ff3b40f9f 100644 --- a/apps/emqx/src/emqx_passwd.erl +++ b/apps/emqx/src/emqx_passwd.erl @@ -20,6 +20,8 @@ , check_pass/2 ]). +-include("logger.hrl"). + -type(hash_type() :: plain | md5 | sha | sha256 | pbkdf2 | bcrypt). -export_type([hash_type/0]). @@ -67,8 +69,8 @@ hash(pbkdf2, {Salt, Password, Macfun, Iterations, Dklen}) -> case pbkdf2:pbkdf2(Macfun, Password, Salt, Iterations, Dklen) of {ok, Hexstring} -> pbkdf2:to_hex(Hexstring); - {error, Error} -> - error_logger:error_msg("pbkdf2 hash error:~p", [Error]), + {error, Reason} -> + ?SLOG(error, #{msg => "pbkdf2_hash_error", reason => Reason}), <<>> end; hash(bcrypt, {Salt, Password}) -> @@ -76,8 +78,8 @@ hash(bcrypt, {Salt, Password}) -> case bcrypt:hashpw(Password, Salt) of {ok, HashPasswd} -> list_to_binary(HashPasswd); - {error, Error}-> - error_logger:error_msg("bcrypt hash error:~p", [Error]), + {error, Reason}-> + ?SLOG(error, #{msg => "bcrypt_hash_error", reason => Reason}), <<>> end. diff --git a/apps/emqx/src/emqx_plugins.erl b/apps/emqx/src/emqx_plugins.erl index 7bb9c084b..e334bdb4a 100644 --- a/apps/emqx/src/emqx_plugins.erl +++ b/apps/emqx/src/emqx_plugins.erl @@ -29,8 +29,6 @@ , find_plugin/1 ]). --export([funlog/2]). - -ifdef(TEST). -compile(export_all). -compile(nowarn_export_all). @@ -50,10 +48,14 @@ load() -> load(PluginName) when is_atom(PluginName) -> case {lists:member(PluginName, names(plugin)), lists:member(PluginName, names(started_app))} of {false, _} -> - ?LOG(alert, "Plugin ~s not found, cannot load it", [PluginName]), + ?SLOG(alert, #{msg => "failed_to_load_plugin", + plugin_name => PluginName, + reason => not_found}), {error, not_found}; {_, true} -> - ?LOG(notice, "Plugin ~s is already started", [PluginName]), + ?SLOG(notice, #{msg => "plugin_already_loaded", + plugin_name => PluginName, + reason => already_loaded}), {error, already_started}; {_, false} -> load_plugin(PluginName) @@ -69,10 +71,14 @@ unload() -> unload(PluginName) when is_atom(PluginName) -> case {lists:member(PluginName, names(plugin)), lists:member(PluginName, names(started_app))} of {false, _} -> - ?LOG(error, "Plugin ~s is not found, cannot unload it", [PluginName]), + ?SLOG(error, #{msg => "fialed_to_unload_plugin", + plugin_name => PluginName, + reason => not_found}), {error, not_found}; {_, false} -> - ?LOG(error, "Plugin ~s is not started", [PluginName]), + ?SLOG(error, #{msg => "failed_to_unload_plugin", + plugin_name => PluginName, + reason => not_loaded}), {error, not_started}; {_, _} -> unload_plugin(PluginName) @@ -81,7 +87,9 @@ unload(PluginName) when is_atom(PluginName) -> reload(PluginName) when is_atom(PluginName)-> case {lists:member(PluginName, names(plugin)), lists:member(PluginName, names(started_app))} of {false, _} -> - ?LOG(error, "Plugin ~s is not found, cannot reload it", [PluginName]), + ?SLOG(error, #{msg => "failed_to_reload_plugin", + plugin_name => PluginName, + reason => not_found}), {error, not_found}; {_, false} -> load(PluginName); @@ -127,14 +135,14 @@ load_ext_plugins(Dir) -> end, filelib:wildcard("*", Dir)). load_ext_plugin(PluginDir) -> - ?LOG(debug, "loading_extra_plugin: ~s", [PluginDir]), + ?SLOG(debug, #{msg => "loading_extra_plugin", plugin_dir => PluginDir}), Ebin = filename:join([PluginDir, "ebin"]), AppFile = filename:join([Ebin, "*.app"]), AppName = case filelib:wildcard(AppFile) of [App] -> list_to_atom(filename:basename(App, ".app")); [] -> - ?LOG(alert, "plugin_app_file_not_found: ~s", [AppFile]), + ?SLOG(alert, #{msg => "plugin_app_file_not_found", app_file => AppFile}), error({plugin_app_file_not_found, AppFile}) end, ok = load_plugin_app(AppName, Ebin). @@ -184,8 +192,14 @@ load_plugin(Name) -> {error, Error0} -> {error, Error0} end - catch _ : Error : Stacktrace -> - ?LOG(alert, "Plugin ~s load failed with ~p", [Name, {Error, Stacktrace}]), + catch Error : Reason : Stacktrace -> + ?SLOG(alert, #{ + msg => "plugin_load_failed", + name => Name, + exception => Error, + reason => Reason, + stacktrace => Stacktrace + }), {error, parse_config_file_failed} end. @@ -202,12 +216,19 @@ load_app(App) -> start_app(App) -> case application:ensure_all_started(App) of {ok, Started} -> - ?LOG(info, "Started plugins: ~p", [Started]), - ?LOG(info, "Load plugin ~s successfully", [App]), + case Started =/= [] of + true -> ?SLOG(info, #{msg => "started_plugin_dependency_apps", apps => Started}); + false -> ok + end, + ?SLOG(info, #{msg => "started_plugin_app", app => App}), ok; {error, {ErrApp, Reason}} -> - ?LOG(error, "Load plugin ~s failed, cannot start plugin ~s for ~0p", [App, ErrApp, Reason]), - {error, {ErrApp, Reason}} + ?SLOG(error, #{msg => failed_to_start_plugin_app, + app => App, + err_app => ErrApp, + reason => Reason + }), + {error, failed_to_start_plugin_app} end. unload_plugin(App) -> @@ -221,11 +242,17 @@ unload_plugin(App) -> stop_app(App) -> case application:stop(App) of ok -> - ?LOG(info, "Stop plugin ~s successfully", [App]), ok; + ?SLOG(info, #{msg => "stop_plugin_successfully", app => App}), + ok; {error, {not_started, App}} -> - ?LOG(error, "Plugin ~s is not started", [App]), ok; + ?SLOG(info, #{msg => "plugin_not_started", app => App}), + ok; {error, Reason} -> - ?LOG(error, "Stop plugin ~s error: ~p", [App]), {error, Reason} + ?SLOG(error, #{msg => "failed_to_stop_plugin_app", + app => App, + error => Reason + }), + {error, Reason} end. names(plugin) -> @@ -236,6 +263,3 @@ names(started_app) -> names(Plugins) -> [Name || #plugin{name = Name} <- Plugins]. - -funlog(Key, Value) -> - ?LOG(info, "~s = ~p", [string:join(Key, "."), Value]). diff --git a/apps/emqx/src/emqx_pool.erl b/apps/emqx/src/emqx_pool.erl index 8fa950fe3..8b9508768 100644 --- a/apps/emqx/src/emqx_pool.erl +++ b/apps/emqx/src/emqx_pool.erl @@ -100,22 +100,26 @@ handle_call({submit, Task}, _From, State) -> {reply, catch run(Task), State}; handle_call(Req, _From, State) -> - ?LOG(error, "Unexpected call: ~p", [Req]), + ?SLOG(error, #{msg => "unexpected_call", call => Req}), {reply, ignored, State}. handle_cast({async_submit, Task}, State) -> try run(Task) - catch _:Error:Stacktrace -> - ?LOG(error, "Error: ~0p, ~0p", [Error, Stacktrace]) + catch Error:Reason:Stacktrace -> + ?SLOG(error, #{msg => "async_submit_error", + exception => Error, + reason => Reason, + stacktrace => Stacktrace + }) end, {noreply, State}; handle_cast(Msg, State) -> - ?LOG(error, "Unexpected cast: ~p", [Msg]), + ?SLOG(error, #{msg => "unexpected_cast", cast => Msg}), {noreply, State}. handle_info(Info, State) -> - ?LOG(error, "Unexpected info: ~p", [Info]), + ?SLOG(error, #{msg => "unexpected_info", info => Info}), {noreply, State}. terminate(_Reason, #{pool := Pool, id := Id}) -> diff --git a/apps/emqx/src/emqx_router.erl b/apps/emqx/src/emqx_router.erl index d25a8bec6..afc1c3f87 100644 --- a/apps/emqx/src/emqx_router.erl +++ b/apps/emqx/src/emqx_router.erl @@ -203,15 +203,15 @@ handle_call({delete_route, Topic, Dest}, _From, State) -> {reply, Ok, State}; handle_call(Req, _From, State) -> - ?LOG(error, "Unexpected call: ~p", [Req]), + ?SLOG(error, #{msg => "unexpected_call", call => Req}), {reply, ignored, State}. handle_cast(Msg, State) -> - ?LOG(error, "Unexpected cast: ~p", [Msg]), + ?SLOG(error, #{msg => "unexpected_cast", cast => Msg}), {noreply, State}. handle_info(Info, State) -> - ?LOG(error, "Unexpected info: ~p", [Info]), + ?SLOG(error, #{msg => "unexpected_info", info => Info}), {noreply, State}. terminate(_Reason, #{pool := Pool, id := Id}) -> diff --git a/apps/emqx/src/emqx_router_helper.erl b/apps/emqx/src/emqx_router_helper.erl index 78d763cac..a88e82d8d 100644 --- a/apps/emqx/src/emqx_router_helper.erl +++ b/apps/emqx/src/emqx_router_helper.erl @@ -109,11 +109,11 @@ init([]) -> {ok, #{nodes => Nodes}, hibernate}. handle_call(Req, _From, State) -> - ?LOG(error, "Unexpected call: ~p", [Req]), + ?SLOG(error, #{msg => "unexpected_call", call => Req}), {reply, ignored, State}. handle_cast(Msg, State) -> - ?LOG(error, "Unexpected cast: ~p", [Msg]), + ?SLOG(error, #{msg => "unexpected_cast", cast => Msg}), {noreply, State}. handle_info({mnesia_table_event, {write, {?ROUTING_NODE, Node, _}, _}}, @@ -130,7 +130,7 @@ handle_info({mnesia_table_event, {delete, {?ROUTING_NODE, _Node}, _}}, State) -> {noreply, State}; handle_info({mnesia_table_event, Event}, State) -> - ?LOG(error, "Unexpected mnesia_table_event: ~p", [Event]), + ?SLOG(error,#{msg => "unexpected_mnesia_table_event", event => Event}), {noreply, State}; handle_info({nodedown, Node}, State = #{nodes := Nodes}) -> @@ -148,7 +148,7 @@ handle_info({membership, _Event}, State) -> {noreply, State}; handle_info(Info, State) -> - ?LOG(error, "Unexpected info: ~p", [Info]), + ?SLOG(error, #{msg => "unexpected_info", info => Info}), {noreply, State}. terminate(_Reason, _State) -> diff --git a/apps/emqx/src/emqx_rule_actions_trans.erl b/apps/emqx/src/emqx_rule_actions_trans.erl deleted file mode 100644 index df1e58797..000000000 --- a/apps/emqx/src/emqx_rule_actions_trans.erl +++ /dev/null @@ -1,66 +0,0 @@ --module(emqx_rule_actions_trans). - --include_lib("syntax_tools/include/merl.hrl"). - --export([parse_transform/2]). - -parse_transform(Forms, _Options) -> - trans(Forms, []). - -trans([], ResAST) -> - lists:reverse(ResAST); -trans([{eof, L} | AST], ResAST) -> - lists:reverse([{eof, L} | ResAST]) ++ AST; -trans([{function, LineNo, FuncName, Arity, Clauses} | AST], ResAST) -> - NewClauses = trans_func_clauses(atom_to_list(FuncName), Clauses), - trans(AST, [{function, LineNo, FuncName, Arity, NewClauses} | ResAST]); -trans([Form | AST], ResAST) -> - trans(AST, [Form | ResAST]). - -trans_func_clauses("on_action_create_" ++ _ = _FuncName , Clauses) -> - NewClauses = [ - begin - Bindings = lists:flatten(get_vars(Args) ++ get_vars(Body, lefth)), - Body2 = append_to_result(Bindings, Body), - {clause, LineNo, Args, Guards, Body2} - end || {clause, LineNo, Args, Guards, Body} <- Clauses], - NewClauses; -trans_func_clauses(_FuncName, Clauses) -> - Clauses. - -get_vars(Exprs) -> - get_vars(Exprs, all). -get_vars(Exprs, Type) -> - do_get_vars(Exprs, [], Type). - -do_get_vars([], Vars, _Type) -> Vars; -do_get_vars([Line | Expr], Vars, all) -> - do_get_vars(Expr, [syntax_vars(erl_syntax:form_list([Line])) | Vars], all); -do_get_vars([Line | Expr], Vars, lefth) -> - do_get_vars(Expr, - case (Line) of - ?Q("_@LeftV = _@@_") -> Vars ++ syntax_vars(LeftV); - _ -> Vars - end, lefth). - -syntax_vars(Line) -> - sets:to_list(erl_syntax_lib:variables(Line)). - -%% append bindings to the return value as the first tuple element. -%% e.g. if the original result is R, then the new result will be {[binding()], R}. -append_to_result(Bindings, Exprs) -> - erl_syntax:revert_forms(do_append_to_result(to_keyword(Bindings), Exprs, [])). - -do_append_to_result(KeyWordVars, [Line], Res) -> - case Line of - ?Q("_@LeftV = _@RightV") -> - lists:reverse([?Q("{[_@KeyWordVars], _@LeftV}"), Line | Res]); - _ -> - lists:reverse([?Q("{[_@KeyWordVars], _@Line}") | Res]) - end; -do_append_to_result(KeyWordVars, [Line | Exprs], Res) -> - do_append_to_result(KeyWordVars, Exprs, [Line | Res]). - -to_keyword(Vars) -> - [erl_syntax:tuple([erl_syntax:atom(Var), merl:var(Var)]) - || Var <- Vars]. diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 66db17e81..a2fb13bab 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -55,7 +55,7 @@ % workaround: prevent being recognized as unused functions -export([to_duration/1, to_duration_s/1, to_duration_ms/1, - to_bytesize/1, to_wordsize/1, + mk_duration/2, to_bytesize/1, to_wordsize/1, to_percent/1, to_comma_separated_list/1, to_bar_separated_list/1, to_ip_port/1, to_erl_cipher_suite/1, @@ -71,7 +71,7 @@ -export([namespace/0, roots/0, roots/1, fields/1]). -export([conf_get/2, conf_get/3, keys/2, filter/1]). --export([ssl/1]). +-export([server_ssl_opts_schema/2, client_ssl_opts_schema/1, ciphers_schema/1, default_ciphers/1]). namespace() -> undefined. @@ -87,23 +87,26 @@ roots(high) -> } , {"zones", sc(map("name", ref("zone")), - #{ desc => "A zone is a set of configs grouped by the zone name.
" - "For flexible configuration mapping, the name " - "can be set to a listener's zone config.
" - "NOTE: A builtin zone named default is auto created " - "and can not be deleted." + #{ desc => +"""A zone is a set of configs grouped by the zone name.
+For flexible configuration mapping, the name +can be set to a listener's zone config.
+NOTE: A builtin zone named default is auto created +and can not be deleted.""" })} , {"mqtt", sc(ref("mqtt"), - #{ desc => "Global MQTT configuration.
" - "The configs here work as default values which can be overriden " - "in zone configs" + #{ desc => +"""Global MQTT configuration.
+The configs here work as default values which can be overriden +in zone configs""" })} , {"authentication", sc(hoconsc:lazy(hoconsc:array(map())), - #{ desc => "Default authentication configs for all MQTT listeners.
" - "For per-listener overrides see authentication " - "in listener configs" + #{ desc => +"""Default authentication configs for all MQTT listeners.
+For per-listener overrides see authentication +in listener configs""" })} , {"authorization", sc(ref("authorization"), @@ -156,11 +159,11 @@ fields("stats") -> fields("authorization") -> [ {"no_match", - sc(hoconsc:union([allow, deny]), + sc(hoconsc:enum([allow, deny]), #{ default => allow })} , {"deny_action", - sc(hoconsc:union([ignore, disconnect]), + sc(hoconsc:enum([ignore, disconnect]), #{ default => ignore })} , {"cache", @@ -294,7 +297,7 @@ fields("mqtt") -> }) } , {"mqueue_default_priority", - sc(union(highest, lowest), + sc(hoconsc:enum([highest, lowest]), #{ default => lowest }) } @@ -309,11 +312,11 @@ fields("mqtt") -> }) } , {"peer_cert_as_username", - sc(hoconsc:union([disabled, cn, dn, crt, pem, md5]), + sc(hoconsc:enum([disabled, cn, dn, crt, pem, md5]), #{ default => disabled })} , {"peer_cert_as_clientid", - sc(hoconsc:union([disabled, cn, dn, crt, pem, md5]), + sc(hoconsc:enum([disabled, cn, dn, crt, pem, md5]), #{ default => disabled })} ]; @@ -483,7 +486,7 @@ fields("mqtt_wss_listener") -> #{}) } , {"ssl", - sc(ref("listener_ssl_opts"), + sc(ref("listener_wss_opts"), #{}) } , {"websocket", @@ -498,6 +501,7 @@ fields("mqtt_quic_listener") -> #{ default => true }) } + %% TODO: ensure cacertfile is configurable , {"certfile", sc(string(), #{}) @@ -506,11 +510,7 @@ fields("mqtt_quic_listener") -> sc(string(), #{}) } - , {"ciphers", - sc(comma_separated_list(), - #{ default => "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256," - "TLS_CHACHA20_POLY1305_SHA256" - })} + , {"ciphers", ciphers_schema(quic)} , {"idle_timeout", sc(duration(), #{ default => "15s" @@ -525,7 +525,7 @@ fields("ws_opts") -> }) } , {"mqtt_piggyback", - sc(hoconsc:union([single, multiple]), + sc(hoconsc:enum([single, multiple]), #{ default => multiple }) } @@ -634,16 +634,26 @@ fields("tcp_opts") -> ]; fields("listener_ssl_opts") -> - ssl(#{handshake_timeout => "15s" - , depth => 10 - , reuse_sessions => true - , versions => default_tls_vsns() - , ciphers => default_ciphers() - }); + server_ssl_opts_schema( + #{ depth => 10 + , reuse_sessions => true + , versions => tls_all_available + , ciphers => tls_all_available + }, false); + +fields("listener_wss_opts") -> + server_ssl_opts_schema( + #{ depth => 10 + , reuse_sessions => true + , versions => tls_all_available + , ciphers => tls_all_available + }, true); +fields(ssl_client_opts) -> + client_ssl_opts_schema(#{}); fields("deflate_opts") -> [ {"level", - sc(hoconsc:union([none, default, best_compression, best_speed]), + sc(hoconsc:enum([none, default, best_compression, best_speed]), #{}) } , {"mem_level", @@ -652,15 +662,15 @@ fields("deflate_opts") -> }) } , {"strategy", - sc(hoconsc:union([default, filtered, huffman_only, rle]), + sc(hoconsc:enum([default, filtered, huffman_only, rle]), #{}) } , {"server_context_takeover", - sc(hoconsc:union([takeover, no_takeover]), + sc(hoconsc:enum([takeover, no_takeover]), #{}) } , {"client_context_takeover", - sc(hoconsc:union([takeover, no_takeover]), + sc(hoconsc:enum([takeover, no_takeover]), #{}) } , {"server_max_window_bits", @@ -699,12 +709,12 @@ fields("broker") -> }) } , {"session_locking_strategy", - sc(hoconsc:union([local, leader, quorum, all]), + sc(hoconsc:enum([local, leader, quorum, all]), #{ default => quorum }) } , {"shared_subscription_strategy", - sc(hoconsc:union([random, round_robin]), + sc(hoconsc:enum([random, round_robin]), #{ default => round_robin }) } @@ -726,7 +736,7 @@ fields("broker") -> fields("broker_perf") -> [ {"route_lock_type", - sc(hoconsc:union([key, tab, global]), + sc(hoconsc:enum([key, tab, global]), #{ default => key })} , {"trie_compaction", @@ -902,7 +912,10 @@ conf_get(Key, Conf, Default) -> filter(Opts) -> [{K, V} || {K, V} <- Opts, V =/= undefined]. -ssl(Defaults) -> +%% @private This function defines the SSL opts which are commonly used by +%% SSL listener and client. +-spec common_ssl_opts_schema(map()) -> hocon_schema:field_schema(). +common_ssl_opts_schema(Defaults) -> D = fun (Field) -> maps:get(to_atom(Field), Defaults, undefined) end, Df = fun (Field, Default) -> maps:get(to_atom(Field), Defaults, Default) end, [ {"enable", @@ -913,69 +926,51 @@ ssl(Defaults) -> , {"cacertfile", sc(string(), #{ default => D("cacertfile") + , nullable => true + , desc => +"""Trusted PEM format CA certificates bundle file.
+The certificates in this file are used to verify the TLS peer's certificates. +Append new certificates to the file if new CAs are to be trusted. +There is no need to restart EMQ X to have the updated file loaded, because +the system regularly checks if file has been updated (and reload).
+NOTE: invalidating (deleting) a certificate from the file will not affect +already established connections. +""" }) } , {"certfile", sc(string(), #{ default => D("certfile") + , nullable => true + , desc => +"""PEM format certificates chain file.
+The certificates in this file should be in reversed order of the certificate +issue chain. That is, the host's certificate should be placed in the beginning +of the file, followed by the immediate issuer certificate and so on. +Although the root CA certificate is optional, it should placed at the end of +the file if it is to be added. +""" }) } , {"keyfile", sc(string(), #{ default => D("keyfile") + , nullable => true + , desc => +"""PEM format private key file.
+""" }) } , {"verify", - sc(hoconsc:union([verify_peer, verify_none]), + sc(hoconsc:enum([verify_peer, verify_none]), #{ default => Df("verify", verify_none) }) } - , {"fail_if_no_peer_cert", - sc(boolean(), - #{ default => Df("fail_if_no_peer_cert", false) - }) - } - , {"secure_renegotiate", - sc(boolean(), - #{ default => Df("secure_renegotiate", true) - , desc => """ -SSL parameter renegotiation is a feature that allows a client and a server -to renegotiate the parameters of the SSL connection on the fly. -RFC 5746 defines a more secure way of doing this. By enabling secure renegotiation, -you drop support for the insecure renegotiation, prone to MitM attacks. -""" - }) - } - , {"client_renegotiation", - sc(boolean(), - #{ default => Df("client_renegotiation", true) - , desc => """ -In protocols that support client-initiated renegotiation, -the cost of resources of such an operation is higher for the server than the client. -This can act as a vector for denial of service attacks. -The SSL application already takes measures to counter-act such attempts, -but client-initiated renegotiation can be strictly disabled by setting this option to false. -The default value is true. Note that disabling renegotiation can result in -long-lived connections becoming unusable due to limits on -the number of messages the underlying cipher suite can encipher. -""" - }) - } , {"reuse_sessions", sc(boolean(), #{ default => Df("reuse_sessions", true) }) } - , {"honor_cipher_order", - sc(boolean(), - #{ default => Df("honor_cipher_order", true) - }) - } - , {"handshake_timeout", - sc(duration(), - #{ default => Df("handshake_timeout", "15s") - }) - } , {"depth", sc(integer(), #{default => Df("depth", 10) @@ -983,74 +978,196 @@ the number of messages the underlying cipher suite can encipher. } , {"password", sc(string(), - #{ default => D("key_password") - , sensitive => true - }) - } - , {"dhfile", - sc(string(), - #{ default => D("dhfile") - }) - } - , {"server_name_indication", - sc(hoconsc:union([disable, string()]), - #{ default => D("server_name_indication") + #{ sensitive => true + , nullable => true + , desc => +"""String containing the user's password. Only used if the private +keyfile is password-protected.""" }) } , {"versions", - sc(typerefl:alias("string", list(atom())), - #{ default => maps:get(versions, Defaults, default_tls_vsns()) - , converter => fun (Vsns) -> [tls_vsn(iolist_to_binary(V)) || V <- Vsns] end + sc(hoconsc:array(typerefl:atom()), + #{ default => default_tls_vsns(maps:get(versions, Defaults, tls_all_available)) + , desc => +"""All TLS/DTLS versions to be supported.
+NOTE: PSK ciphers are suppresed by 'tlsv1.3' version config
+In case PSK cipher suites are intended, make sure to configured +['tlsv1.2', 'tlsv1.1'] here. +""" + , validator => fun validate_tls_versions/1 }) } - , {"ciphers", - sc(hoconsc:array(string()), - #{ default => D("ciphers") - }) - } - , {"user_lookup_fun", + , {"ciphers", ciphers_schema(D("ciphers"))} + , {user_lookup_fun, sc(typerefl:alias("string", any()), - #{ default => "emqx_psk:lookup" + #{ default => "emqx_tls_psk:lookup" , converter => fun ?MODULE:parse_user_lookup_fun/1 }) } + , {"secure_renegotiate", + sc(boolean(), + #{ default => Df("secure_renegotiate", true) + , desc => """ +SSL parameter renegotiation is a feature that allows a client and a server +to renegotiate the parameters of the SSL connection on the fly. +RFC 5746 defines a more secure way of doing this. By enabling secure renegotiation, +you drop support for the insecure renegotiation, prone to MitM attacks. +""" + }) + } ]. -%% on erl23.2.7.2-emqx-2, sufficient_crypto_support('tlsv1.3') -> false -default_tls_vsns() -> [<<"tlsv1.2">>, <<"tlsv1.1">>, <<"tlsv1">>]. - -tls_vsn(<<"tlsv1.3">>) -> 'tlsv1.3'; -tls_vsn(<<"tlsv1.2">>) -> 'tlsv1.2'; -tls_vsn(<<"tlsv1.1">>) -> 'tlsv1.1'; -tls_vsn(<<"tlsv1">>) -> 'tlsv1'. - -default_ciphers() -> [ - "TLS_AES_256_GCM_SHA384", "TLS_AES_128_GCM_SHA256", "TLS_CHACHA20_POLY1305_SHA256", - "TLS_AES_128_CCM_SHA256", "TLS_AES_128_CCM_8_SHA256", "ECDHE-ECDSA-AES256-GCM-SHA384", - "ECDHE-RSA-AES256-GCM-SHA384", "ECDHE-ECDSA-AES256-SHA384", "ECDHE-RSA-AES256-SHA384", - "ECDHE-ECDSA-DES-CBC3-SHA", "ECDH-ECDSA-AES256-GCM-SHA384", "ECDH-RSA-AES256-GCM-SHA384", - "ECDH-ECDSA-AES256-SHA384", "ECDH-RSA-AES256-SHA384", "DHE-DSS-AES256-GCM-SHA384", - "DHE-DSS-AES256-SHA256", "AES256-GCM-SHA384", "AES256-SHA256", - "ECDHE-ECDSA-AES128-GCM-SHA256", "ECDHE-RSA-AES128-GCM-SHA256", - "ECDHE-ECDSA-AES128-SHA256", "ECDHE-RSA-AES128-SHA256", "ECDH-ECDSA-AES128-GCM-SHA256", - "ECDH-RSA-AES128-GCM-SHA256", "ECDH-ECDSA-AES128-SHA256", "ECDH-RSA-AES128-SHA256", - "DHE-DSS-AES128-GCM-SHA256", "DHE-DSS-AES128-SHA256", "AES128-GCM-SHA256", "AES128-SHA256", - "ECDHE-ECDSA-AES256-SHA", "ECDHE-RSA-AES256-SHA", "DHE-DSS-AES256-SHA", - "ECDH-ECDSA-AES256-SHA", "ECDH-RSA-AES256-SHA", "AES256-SHA", "ECDHE-ECDSA-AES128-SHA", - "ECDHE-RSA-AES128-SHA", "DHE-DSS-AES128-SHA", "ECDH-ECDSA-AES128-SHA", - "ECDH-RSA-AES128-SHA", "AES128-SHA" - ] ++ psk_ciphers(). - -psk_ciphers() -> [ - "PSK-AES128-CBC-SHA", "PSK-AES256-CBC-SHA", "PSK-3DES-EDE-CBC-SHA", "PSK-RC4-SHA" +%% @doc Make schema for SSL listener options. +%% When it's for ranch listener, an extra field `handshake_timeout' is added. +-spec server_ssl_opts_schema(map(), boolean()) -> hocon_schema:field_schema(). +server_ssl_opts_schema(Defaults, IsRanchListener) -> + D = fun (Field) -> maps:get(to_atom(Field), Defaults, undefined) end, + Df = fun (Field, Default) -> maps:get(to_atom(Field), Defaults, Default) end, + common_ssl_opts_schema(Defaults) ++ + [ {"dhfile", + sc(string(), + #{ default => D("dhfile") + , nullable => true + , desc => +"""Path to a file containing PEM-encoded Diffie Hellman parameters +to be used by the server if a cipher suite using Diffie Hellman +key exchange is negotiated. If not specified, default parameters +are used.
+NOTE: The dhfile option is not supported by TLS 1.3.""" + }) + } + , {"fail_if_no_peer_cert", + sc(boolean(), + #{ default => Df("fail_if_no_peer_cert", false) + , desc => +""" +Used together with {verify, verify_peer} by an TLS/DTLS server. +If set to true, the server fails if the client does not have a +certificate to send, that is, sends an empty certificate. +If set to false, it fails only if the client sends an invalid +certificate (an empty certificate is considered valid). +""" + }) + } + , {"honor_cipher_order", + sc(boolean(), + #{ default => Df("honor_cipher_order", true) + }) + } + , {"client_renegotiation", + sc(boolean(), + #{ default => Df("client_renegotiation", true) + , desc => """ +In protocols that support client-initiated renegotiation, +the cost of resources of such an operation is higher for the server than the client. +This can act as a vector for denial of service attacks. +The SSL application already takes measures to counter-act such attempts, +but client-initiated renegotiation can be strictly disabled by setting this option to false. +The default value is true. Note that disabling renegotiation can result in +long-lived connections becoming unusable due to limits on +the number of messages the underlying cipher suite can encipher. +""" + }) + } + | [ {"handshake_timeout", + sc(duration(), + #{ default => Df("handshake_timeout", "15s") + , desc => "Maximum time duration allowed for the handshake to complete" + })} + || IsRanchListener] ]. +%% @doc Make schema for SSL client. +-spec client_ssl_opts_schema(map()) -> hocon_schema:field_schema(). +client_ssl_opts_schema(Defaults) -> + common_ssl_opts_schema(Defaults) ++ + [ { "server_name_indication", + sc(hoconsc:union([disable, string()]), + #{ default => disable + , desc => +"""Specify the host name to be used in TLS Server Name Indication extension.
+For instance, when connecting to \"server.example.net\", the genuine server +which accedpts the connection and performs TLS handshake may differ from the +host the TLS client initially connects to, e.g. when connecting to an IP address +or when the host has multiple resolvable DNS records
+If not specified, it will default to the host name string which is used +to establish the connection, unless it is IP addressed used.
+The host name is then also used in the host name verification of the peer +certificate.
The special value 'disable' prevents the Server Name +Indication extension from being sent and disables the hostname +verification check.""" + })} + ]. + + +default_tls_vsns(dtls_all_available) -> + proplists:get_value(available_dtls, ssl:versions()); +default_tls_vsns(tls_all_available) -> + emqx_tls_lib:default_versions(). + +-spec ciphers_schema(quic | dtls_all_available | tls_all_available | undefined) -> hocon_schema:field_schema(). +ciphers_schema(Default) -> + sc(hoconsc:array(string()), + #{ default => default_ciphers(Default) + , converter => fun(Ciphers) when is_binary(Ciphers) -> + binary:split(Ciphers, <<",">>, [global]); + (Ciphers) when is_list(Ciphers) -> + Ciphers + end + , validator => case Default =:= quic of + true -> undefined; %% quic has openssl statically linked + false -> fun validate_ciphers/1 + end + , desc => +"""TLS cipher suite names separated by comma, or as an array of strings +\"TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256\" or +[\"TLS_AES_256_GCM_SHA384\",\"TLS_AES_128_GCM_SHA256\"] +Ciphers (and their ordering) define the way in which the +client and server encrypts information over the wire. +Selecting a good cipher suite is critical for the +application's data security, confidentiality and performance. +The names should be in OpenSSL sting format (not RFC format). +Default values and examples proveded by EMQ X config +documentation are all in OpenSSL format.
+ +NOTE: Certain cipher suites are only compatible with +specific TLS versions ('tlsv1.1', 'tlsv1.2' or 'tlsv1.3') +incompatible cipher suites will be silently dropped. +For instance, if only 'tlsv1.3' is given in the versions, +configuring cipher suites for other versions will have no effect. +
+ +NOTE: PSK ciphers are suppresed by 'tlsv1.3' version config
+If PSK cipher suites are intended, 'tlsv1.3' should be disabled from versions.
+PSK cipher suites: \"RSA-PSK-AES256-GCM-SHA384,RSA-PSK-AES256-CBC-SHA384, +RSA-PSK-AES128-GCM-SHA256,RSA-PSK-AES128-CBC-SHA256, +RSA-PSK-AES256-CBC-SHA,RSA-PSK-AES128-CBC-SHA, +RSA-PSK-DES-CBC3-SHA,RSA-PSK-RC4-SHA\"
+""" ++ case Default of + quic -> "NOTE: QUIC listener supports only 'tlsv1.3' ciphers
"; + _ -> "" + end}). + +default_ciphers(undefined) -> + default_ciphers(tls_all_available); +default_ciphers(quic) -> [ + "TLS_AES_256_GCM_SHA384", + "TLS_AES_128_GCM_SHA256", + "TLS_CHACHA20_POLY1305_SHA256" + ]; +default_ciphers(dtls_all_available) -> + %% as of now, dtls does not support tlsv1.3 ciphers + emqx_tls_lib:selected_ciphers(['dtlsv1.2', 'dtlsv1']); +default_ciphers(tls_all_available) -> + emqx_tls_lib:default_ciphers(). + %% @private return a list of keys in a parent field -spec(keys(string(), hocon:config()) -> [string()]). keys(Parent, Conf) -> [binary_to_list(B) || B <- maps:keys(conf_get(Parent, Conf, #{}))]. --spec ceiling(float()) -> integer(). +-spec ceiling(number()) -> integer(). ceiling(X) -> T = erlang:trunc(X), case (X - T) of @@ -1069,6 +1186,15 @@ ref(Field) -> hoconsc:ref(?MODULE, Field). ref(Module, Field) -> hoconsc:ref(Module, Field). +mk_duration(Desc, OverrideMeta) -> + DefaultMeta = #{desc => Desc ++ " Time span. A text string with number followed by time units: + `ms` for milli-seconds, + `s` for seconds, + `m` for minutes, + `h` for hours; + or combined representation like `1h5m0s`"}, + hoconsc:mk(typerefl:alias("string", duration()), maps:merge(DefaultMeta, OverrideMeta)). + to_duration(Str) -> case hocon_postprocess:duration(Str) of I when is_integer(I) -> {ok, I}; @@ -1077,13 +1203,15 @@ to_duration(Str) -> to_duration_s(Str) -> case hocon_postprocess:duration(Str) of - I when is_integer(I) -> {ok, ceiling(I / 1000)}; + I when is_number(I) -> {ok, ceiling(I / 1000)}; _ -> {error, Str} end. +-spec to_duration_ms(Input) -> {ok, integer()} | {error, Input} + when Input :: string() | binary(). to_duration_ms(Str) -> case hocon_postprocess:duration(Str) of - I when is_integer(I) -> {ok, ceiling(I)}; + I when is_number(I) -> {ok, ceiling(I)}; _ -> {error, Str} end. @@ -1159,4 +1287,19 @@ parse_user_lookup_fun(StrConf) -> [ModStr, FunStr] = string:tokens(StrConf, ":"), Mod = list_to_atom(ModStr), Fun = list_to_atom(FunStr), - {fun Mod:Fun/3, <<>>}. + {fun Mod:Fun/3, undefined}. + +validate_ciphers(Ciphers) -> + All = emqx_tls_lib:all_ciphers(), + case lists:filter(fun(Cipher) -> not lists:member(Cipher, All) end, Ciphers) of + [] -> ok; + Bad -> {error, {bad_ciphers, Bad}} + end. + +validate_tls_versions(Versions) -> + AvailableVersions = proplists:get_value(available, ssl:versions()) ++ + proplists:get_value(available_dtls, ssl:versions()), + case lists:filter(fun(V) -> not lists:member(V, AvailableVersions) end, Versions) of + [] -> ok; + Vs -> {error, {unsupported_ssl_versions, Vs}} + end. diff --git a/apps/emqx/src/emqx_session.erl b/apps/emqx/src/emqx_session.erl index f915155cb..408435006 100644 --- a/apps/emqx/src/emqx_session.erl +++ b/apps/emqx/src/emqx_session.erl @@ -479,11 +479,12 @@ log_dropped(Msg = #message{qos = QoS}, #session{mqueue = Q}) -> case (QoS == ?QOS_0) andalso (not emqx_mqueue:info(store_qos0, Q)) of true -> ok = emqx_metrics:inc('delivery.dropped.qos0_msg'), - ?LOG(warning, "Dropped qos0 msg: ~s", [emqx_message:format(Msg)]); + ?SLOG(warning, #{msg => "dropped_qos0_msg", + payload => emqx_message:to_log_map(Msg)}); false -> ok = emqx_metrics:inc('delivery.dropped.queue_full'), - ?LOG(warning, "Dropped msg due to mqueue is full: ~s", - [emqx_message:format(Msg)]) + ?SLOG(warning, #{msg => "dropped_msg_due_to_mqueue_is_full", + payload => emqx_message:to_log_map(Msg)}) end. enrich_fun(Session = #session{subscriptions = Subs}) -> diff --git a/apps/emqx/src/emqx_shared_sub.erl b/apps/emqx/src/emqx_shared_sub.erl index 59e364f58..ef8e3d288 100644 --- a/apps/emqx/src/emqx_shared_sub.erl +++ b/apps/emqx/src/emqx_shared_sub.erl @@ -325,11 +325,11 @@ handle_call({unsubscribe, Group, Topic, SubPid}, _From, State) -> {reply, ok, State}; handle_call(Req, _From, State) -> - ?LOG(error, "Unexpected call: ~p", [Req]), + ?SLOG(error, #{msg => "unexpected_call", req => Req}), {reply, ignored, State}. handle_cast(Msg, State) -> - ?LOG(error, "Unexpected cast: ~p", [Msg]), + ?SLOG(error, #{msg => "unexpected_cast", req => Msg}), {noreply, State}. handle_info({mnesia_table_event, {write, NewRecord, _}}, State = #state{pmon = PMon}) -> @@ -347,8 +347,8 @@ handle_info({mnesia_table_event, {write, NewRecord, _}}, State = #state{pmon = P handle_info({mnesia_table_event, _Event}, State) -> {noreply, State}; -handle_info({'DOWN', _MRef, process, SubPid, _Reason}, State = #state{pmon = PMon}) -> - ?LOG(info, "Shared subscriber down: ~p", [SubPid]), +handle_info({'DOWN', _MRef, process, SubPid, Reason}, State = #state{pmon = PMon}) -> + ?SLOG(info, #{msg => "shared_subscriber_down", sub_pid => SubPid, reason => Reason}), cleanup_down(SubPid), {noreply, update_stats(State#state{pmon = emqx_pmon:erase(SubPid, PMon)})}; diff --git a/apps/emqx/src/emqx_stats.erl b/apps/emqx/src/emqx_stats.erl index 51ba72155..0d2b1a1fd 100644 --- a/apps/emqx/src/emqx_stats.erl +++ b/apps/emqx/src/emqx_stats.erl @@ -202,7 +202,7 @@ handle_call(stop, _From, State) -> {stop, normal, ok, State}; handle_call(Req, _From, State) -> - ?LOG(error, "Unexpected call: ~p", [Req]), + ?SLOG(error, #{msg => "unexpected_call", call => Req}), {reply, ignored, State}. handle_cast({setstat, Stat, MaxStat, Val}, State) -> @@ -221,7 +221,9 @@ handle_cast({update_interval, Update = #update{name = Name}}, State = #state{updates = Updates}) -> NState = case lists:keyfind(Name, #update.name, Updates) of #update{} -> - ?LOG(warning, "Duplicated update: ~s", [Name]), + ?SLOG(warning, #{msg => "duplicated_update", + name => Name + }), State; false -> State#state{updates = [Update|Updates]} end, @@ -232,7 +234,7 @@ handle_cast({cancel_update, Name}, State = #state{updates = Updates}) -> {noreply, State#state{updates = Updates1}}; handle_cast(Msg, State) -> - ?LOG(error, "Unexpected cast: ~p", [Msg]), + ?SLOG(error, #{msg => "unexpected_cast", cast => Msg}), {noreply, State}. handle_info({timeout, TRef, tick}, State = #state{timer = TRef, updates = Updates}) -> @@ -241,8 +243,13 @@ handle_info({timeout, TRef, tick}, State = #state{timer = TRef, updates = Update func = UpFun}, Acc) when C =< 0 -> try UpFun() catch - _:Error -> - ?LOG(error, "Update ~s failed: ~0p", [Name, Error]) + Error : Reason : Stacktrace -> + ?SLOG(error, #{msg => "update_name_failed", + name => Name, + exception => Error, + reason => Reason, + stacktrace => Stacktrace + }) end, [Update#update{countdown = I} | Acc]; (Update = #update{countdown = C}, Acc) -> @@ -251,7 +258,7 @@ handle_info({timeout, TRef, tick}, State = #state{timer = TRef, updates = Update {noreply, start_timer(State#state{updates = Updates1}), hibernate}; handle_info(Info, State) -> - ?LOG(error, "Unexpected info: ~p", [Info]), + ?SLOG(error, #{msg => "unexpected_info", info => Info}), {noreply, State}. terminate(_Reason, #state{timer = TRef}) -> @@ -271,6 +278,9 @@ safe_update_element(Key, Val) -> true -> true catch error:badarg -> - ?LOG(warning, "Failed to update ~0p to ~0p", [Key, Val]) + ?SLOG(warning, #{ + msg => "failed_to_update", + key => Key, + val => Val + }) end. - diff --git a/apps/emqx/src/emqx_sys.erl b/apps/emqx/src/emqx_sys.erl index 70043e2bb..692d2bd0a 100644 --- a/apps/emqx/src/emqx_sys.erl +++ b/apps/emqx/src/emqx_sys.erl @@ -134,11 +134,11 @@ handle_call(uptime, _From, State) -> {reply, uptime(State), State}; handle_call(Req, _From, State) -> - ?LOG(error, "Unexpected call: ~p", [Req]), + ?SLOG(error, #{msg => "unexpected_call", call => Req}), {reply, ignored, State}. handle_cast(Msg, State) -> - ?LOG(error, "Unexpected cast: ~p", [Msg]), + ?SLOG(error, #{msg => "unexpected_cast", cast => Msg}), {noreply, State}. handle_info({timeout, TRef, heartbeat}, State = #state{heartbeat = TRef}) -> @@ -156,7 +156,7 @@ handle_info({timeout, TRef, tick}, {noreply, tick(State), hibernate}; handle_info(Info, State) -> - ?LOG(error, "Unexpected info: ~p", [Info]), + ?SLOG(error, #{msg => "unexpected_info", info => Info}), {noreply, State}. terminate(_Reason, #state{heartbeat = TRef1, ticker = TRef2}) -> diff --git a/apps/emqx/src/emqx_sys_mon.erl b/apps/emqx/src/emqx_sys_mon.erl index 80f5e49ec..3d47038c6 100644 --- a/apps/emqx/src/emqx_sys_mon.erl +++ b/apps/emqx/src/emqx_sys_mon.erl @@ -83,42 +83,51 @@ sysm_opts([_Opt|Opts], Acc) -> sysm_opts(Opts, Acc). handle_call(Req, _From, State) -> - ?LOG(error, "Unexpected call: ~p", [Req]), + ?SLOG(error, #{msg => "unexpected_call", req => Req}), {reply, ignored, State}. handle_cast(Msg, State) -> - ?LOG(error, "Unexpected cast: ~p", [Msg]), + ?SLOG(error, #{msg => "unexpected_cast", req => Msg}), {noreply, State}. handle_info({monitor, Pid, long_gc, Info}, State) -> suppress({long_gc, Pid}, fun() -> - WarnMsg = io_lib:format("long_gc warning: pid = ~p, info: ~p", [Pid, Info]), - ?LOG(warning, "~s~n~p", [WarnMsg, procinfo(Pid)]), + WarnMsg = io_lib:format("long_gc warning: pid = ~p", [Pid]), + ?SLOG(warning, #{msg => long_gc, + info => Info, + porcinfo => procinfo(Pid) + }), safe_publish(long_gc, WarnMsg) end, State); handle_info({monitor, Pid, long_schedule, Info}, State) when is_pid(Pid) -> suppress({long_schedule, Pid}, fun() -> - WarnMsg = io_lib:format("long_schedule warning: pid = ~p, info: ~p", [Pid, Info]), - ?LOG(warning, "~s~n~p", [WarnMsg, procinfo(Pid)]), + WarnMsg = io_lib:format("long_schedule warning: pid = ~p", [Pid]), + ?SLOG(warning, #{msg => long_schedule, + info => Info, + procinfo => procinfo(Pid)}), safe_publish(long_schedule, WarnMsg) end, State); handle_info({monitor, Port, long_schedule, Info}, State) when is_port(Port) -> suppress({long_schedule, Port}, fun() -> - WarnMsg = io_lib:format("long_schedule warning: port = ~p, info: ~p", [Port, Info]), - ?LOG(warning, "~s~n~p", [WarnMsg, erlang:port_info(Port)]), + WarnMsg = io_lib:format("long_schedule warning: port = ~p", [Port]), + ?SLOG(warning, #{msg => long_schedule, + info => Info, + portinfo => portinfo(Port)}), safe_publish(long_schedule, WarnMsg) end, State); handle_info({monitor, Pid, large_heap, Info}, State) -> suppress({large_heap, Pid}, fun() -> - WarnMsg = io_lib:format("large_heap warning: pid = ~p, info: ~p", [Pid, Info]), - ?LOG(warning, "~s~n~p", [WarnMsg, procinfo(Pid)]), + WarnMsg = io_lib:format("large_heap warning: pid = ~p", [Pid]), + ?SLOG(warning, #{msg => large_heap, + info => Info, + procinfo => procinfo(Pid)}), safe_publish(large_heap, WarnMsg) end, State); @@ -126,7 +135,10 @@ handle_info({monitor, SusPid, busy_port, Port}, State) -> suppress({busy_port, Port}, fun() -> WarnMsg = io_lib:format("busy_port warning: suspid = ~p, port = ~p", [SusPid, Port]), - ?LOG(warning, "~s~n~p~n~p", [WarnMsg, procinfo(SusPid), erlang:port_info(Port)]), + ?SLOG(warning, #{msg => busy_port, + portinfo => portinfo(Port), + procinfo => procinfo(SusPid) + }), safe_publish(busy_port, WarnMsg) end, State); @@ -134,7 +146,9 @@ handle_info({monitor, SusPid, busy_dist_port, Port}, State) -> suppress({busy_dist_port, Port}, fun() -> WarnMsg = io_lib:format("busy_dist_port warning: suspid = ~p, port = ~p", [SusPid, Port]), - ?LOG(warning, "~s~n~p~n~p", [WarnMsg, procinfo(SusPid), erlang:port_info(Port)]), + ?SLOG(warning, #{msg => busy_dist_port, + portinfo => portinfo(Port), + procinfo => procinfo(SusPid)}), safe_publish(busy_dist_port, WarnMsg) end, State); @@ -142,7 +156,7 @@ handle_info({timeout, _Ref, reset}, State) -> {noreply, State#{events := []}, hibernate}; handle_info(Info, State) -> - ?LOG(error, "Unexpected Info: ~p", [Info]), + ?SLOG(error, #{msg => "unexpected_info", info => Info}), {noreply, State}. terminate(_Reason, #{timer := TRef}) -> @@ -170,11 +184,14 @@ suppress(Key, SuccFun, State = #{events := Events}) -> end. procinfo(Pid) -> - case {emqx_vm:get_process_info(Pid), emqx_vm:get_process_gc_info(Pid)} of - {undefined, _} -> undefined; - {_, undefined} -> undefined; - {Info, GcInfo} -> Info ++ GcInfo - end. + [{pid, Pid} | procinfo_l(emqx_vm:get_process_gc_info(Pid))] ++ + procinfo_l(emqx_vm:get_process_info(Pid)). + +procinfo_l(undefined) -> []; +procinfo_l(List) -> List. + +portinfo(Port) -> + [{port, Port} | erlang:port_info(Port)]. safe_publish(Event, WarnMsg) -> Topic = emqx_topic:systop(lists:concat(['sysmon/', Event])), diff --git a/apps/emqx/src/emqx_tls_lib.erl b/apps/emqx/src/emqx_tls_lib.erl index 24a9a15cf..3b3953b83 100644 --- a/apps/emqx/src/emqx_tls_lib.erl +++ b/apps/emqx/src/emqx_tls_lib.erl @@ -19,9 +19,10 @@ -export([ default_versions/0 , integral_versions/1 , default_ciphers/0 - , default_ciphers/1 + , selected_ciphers/1 , integral_ciphers/2 , drop_tls13_for_old_otp/1 + , all_ciphers/0 ]). %% non-empty string @@ -31,9 +32,7 @@ %% @doc Returns the default supported tls versions. -spec default_versions() -> [atom()]. -default_versions() -> - OtpRelease = list_to_integer(erlang:system_info(otp_release)), - integral_versions(default_versions(OtpRelease)). +default_versions() -> available_versions(). %% @doc Validate a given list of desired tls versions. %% raise an error exception if non of them are available. @@ -51,7 +50,7 @@ integral_versions(Desired) when ?IS_STRING(Desired) -> integral_versions(Desired) when is_binary(Desired) -> integral_versions(parse_versions(Desired)); integral_versions(Desired) -> - {_, Available} = lists:keyfind(available, 1, ssl:versions()), + Available = available_versions(), case lists:filter(fun(V) -> lists:member(V, Available) end, Desired) of [] -> erlang:error(#{ reason => no_available_tls_version , desired => Desired @@ -61,27 +60,63 @@ integral_versions(Desired) -> Filtered end. -%% @doc Return a list of default (openssl string format) cipher suites. --spec default_ciphers() -> [string()]. -default_ciphers() -> default_ciphers(default_versions()). +%% @doc Return a list of all supported ciphers. +all_ciphers() -> all_ciphers(default_versions()). %% @doc Return a list of (openssl string format) cipher suites. --spec default_ciphers([ssl:tls_version()]) -> [string()]. -default_ciphers(['tlsv1.3']) -> +-spec all_ciphers([ssl:tls_version()]) -> [string()]. +all_ciphers(['tlsv1.3']) -> %% When it's only tlsv1.3 wanted, use 'exclusive' here %% because 'all' returns legacy cipher suites too, %% which does not make sense since tlsv1.3 can not use %% legacy cipher suites. ssl:cipher_suites(exclusive, 'tlsv1.3', openssl); -default_ciphers(Versions) -> +all_ciphers(Versions) -> %% assert non-empty [_ | _] = dedup(lists:append([ssl:cipher_suites(all, V, openssl) || V <- Versions])). + +%% @doc All Pre-selected TLS ciphers. +default_ciphers() -> + selected_ciphers(available_versions()). + +%% @doc Pre-selected TLS ciphers for given versions.. +selected_ciphers(Vsns) -> + All = all_ciphers(Vsns), + dedup(lists:filter(fun(Cipher) -> lists:member(Cipher, All) end, + lists:flatmap(fun do_selected_ciphers/1, Vsns))). + +do_selected_ciphers('tlsv1.3') -> + case lists:member('tlsv1.3', proplists:get_value(available, ssl:versions())) of + true -> ssl:cipher_suites(exclusive, 'tlsv1.3', openssl); + false -> [] + end ++ do_selected_ciphers('tlsv1.2'); +do_selected_ciphers(_) -> + [ "ECDHE-ECDSA-AES256-GCM-SHA384", + "ECDHE-RSA-AES256-GCM-SHA384", "ECDHE-ECDSA-AES256-SHA384", "ECDHE-RSA-AES256-SHA384", + "ECDH-ECDSA-AES256-GCM-SHA384", "ECDH-RSA-AES256-GCM-SHA384", + "ECDH-ECDSA-AES256-SHA384", "ECDH-RSA-AES256-SHA384", "DHE-DSS-AES256-GCM-SHA384", + "DHE-DSS-AES256-SHA256", "AES256-GCM-SHA384", "AES256-SHA256", + "ECDHE-ECDSA-AES128-GCM-SHA256", "ECDHE-RSA-AES128-GCM-SHA256", + "ECDHE-ECDSA-AES128-SHA256", "ECDHE-RSA-AES128-SHA256", "ECDH-ECDSA-AES128-GCM-SHA256", + "ECDH-RSA-AES128-GCM-SHA256", "ECDH-ECDSA-AES128-SHA256", "ECDH-RSA-AES128-SHA256", + "DHE-DSS-AES128-GCM-SHA256", "DHE-DSS-AES128-SHA256", "AES128-GCM-SHA256", "AES128-SHA256", + "ECDHE-ECDSA-AES256-SHA", "ECDHE-RSA-AES256-SHA", "DHE-DSS-AES256-SHA", + "ECDH-ECDSA-AES256-SHA", "ECDH-RSA-AES256-SHA", "ECDHE-ECDSA-AES128-SHA", + "ECDHE-RSA-AES128-SHA", "DHE-DSS-AES128-SHA", "ECDH-ECDSA-AES128-SHA", + "ECDH-RSA-AES128-SHA", + + %% psk + "RSA-PSK-AES256-GCM-SHA384","RSA-PSK-AES256-CBC-SHA384", + "RSA-PSK-AES128-GCM-SHA256","RSA-PSK-AES128-CBC-SHA256", + "RSA-PSK-AES256-CBC-SHA","RSA-PSK-AES128-CBC-SHA" + ]. + %% @doc Ensure version & cipher-suites integrity. -spec integral_ciphers([ssl:tls_version()], binary() | string() | [string()]) -> [string()]. integral_ciphers(Versions, Ciphers) when Ciphers =:= [] orelse Ciphers =:= undefined -> %% not configured - integral_ciphers(Versions, default_ciphers(Versions)); + integral_ciphers(Versions, selected_ciphers(Versions)); integral_ciphers(Versions, Ciphers) when ?IS_STRING_LIST(Ciphers) -> %% ensure tlsv1.3 ciphers if none of them is found in Ciphers dedup(ensure_tls13_cipher(lists:member('tlsv1.3', Versions), Ciphers)); @@ -95,7 +130,7 @@ integral_ciphers(Versions, Ciphers) -> %% In case tlsv1.3 is present, ensure tlsv1.3 cipher is added if user %% did not provide it from config --- which is a common mistake ensure_tls13_cipher(true, Ciphers) -> - Tls13Ciphers = default_ciphers(['tlsv1.3']), + Tls13Ciphers = selected_ciphers(['tlsv1.3']), case lists:any(fun(C) -> lists:member(C, Tls13Ciphers) end, Ciphers) of true -> Ciphers; false -> Tls13Ciphers ++ Ciphers @@ -103,11 +138,17 @@ ensure_tls13_cipher(true, Ciphers) -> ensure_tls13_cipher(false, Ciphers) -> Ciphers. +%% default ssl versions based on available versions. +-spec available_versions() -> [atom()]. +available_versions() -> + OtpRelease = list_to_integer(erlang:system_info(otp_release)), + default_versions(OtpRelease). + %% tlsv1.3 is available from OTP-22 but we do not want to use until 23. default_versions(OtpRelease) when OtpRelease >= 23 -> - ['tlsv1.3' | default_versions(22)]; + proplists:get_value(available, ssl:versions()); default_versions(_) -> - ['tlsv1.2', 'tlsv1.1', tlsv1]. + lists:delete('tlsv1.3', proplists:get_value(available, ssl:versions())). %% Deduplicate a list without re-ordering the elements. dedup([]) -> []; @@ -178,7 +219,7 @@ drop_tls13(SslOpts0) -> drop_tls13_test() -> Versions = default_versions(), ?assert(lists:member('tlsv1.3', Versions)), - Ciphers = default_ciphers(), + Ciphers = all_ciphers(), ?assert(has_tlsv13_cipher(Ciphers)), Opts0 = #{versions => Versions, ciphers => Ciphers, other => true}, Opts = drop_tls13(Opts0), diff --git a/apps/emqx/src/emqx_psk.erl b/apps/emqx/src/emqx_tls_psk.erl similarity index 56% rename from apps/emqx/src/emqx_psk.erl rename to apps/emqx/src/emqx_tls_psk.erl index 0c5ca2964..baad60385 100644 --- a/apps/emqx/src/emqx_psk.erl +++ b/apps/emqx/src/emqx_tls_psk.erl @@ -14,11 +14,10 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_psk). +-module(emqx_tls_psk). -include("logger.hrl"). - %% SSL PSK Callbacks -export([lookup/3]). @@ -26,14 +25,24 @@ -type psk_user_state() :: term(). -spec lookup(psk, psk_identity(), psk_user_state()) -> {ok, SharedSecret :: binary()} | error. -lookup(psk, ClientPSKID, _UserState) -> - try emqx_hooks:run_fold('tls_handshake.psk_lookup', [ClientPSKID], not_found) of - SharedSecret when is_binary(SharedSecret) -> {ok, SharedSecret}; - Error -> - ?LOG(error, "Look PSK for PSKID ~p error: ~p", [ClientPSKID, Error]), +lookup(psk, PSKIdentity, _UserState) -> + try emqx_hooks:run_fold('tls_handshake.psk_lookup', [PSKIdentity], normal) of + {ok, SharedSecret} when is_binary(SharedSecret) -> + {ok, SharedSecret}; + normal -> + ?SLOG(info, #{msg => "psk_identity_not_found", + psk_identity => PSKIdentity}), + error; + {error, Reason} -> + ?SLOG(warning, #{msg => "psk_identity_not_found", + psk_identity => PSKIdentity, + reason => Reason}), error catch - Except:Error:Stacktrace -> - ?LOG(error, "Lookup PSK failed, ~0p: ~0p", [{Except,Error}, Stacktrace]), + Class:Reason:Stacktrace -> + ?SLOG(error, #{msg => "lookup_psk_failed", + class => Class, + reason => Reason, + stacktrace => Stacktrace}), error end. diff --git a/apps/emqx/src/emqx_tracer.erl b/apps/emqx/src/emqx_tracer.erl index d05840433..ab354ae21 100644 --- a/apps/emqx/src/emqx_tracer.erl +++ b/apps/emqx/src/emqx_tracer.erl @@ -115,18 +115,18 @@ install_trace_handler(Who, Level, LogFile) -> {fun filter_by_meta_key/2, Who}}]}) of ok -> - ?LOG(info, "Start trace for ~p", [Who]); + ?SLOG(info, #{msg => "start_trace", who => Who}); {error, Reason} -> - ?LOG(error, "Start trace for ~p failed, error: ~p", [Who, Reason]), + ?SLOG(error, #{msg => "failed_to_trace", who => Who, reason => Reason}), {error, Reason} end. uninstall_trance_handler(Who) -> case logger:remove_handler(handler_id(Who)) of ok -> - ?LOG(info, "Stop trace for ~p", [Who]); + ?SLOG(info, #{msg => "stop_trace", who => Who}); {error, Reason} -> - ?LOG(error, "Stop trace for ~p failed, error: ~p", [Who, Reason]), + ?SLOG(error, #{msg => "failed_to_stop_trace", who => Who, reason => Reason}), {error, Reason} end. diff --git a/apps/emqx/src/emqx_vm_mon.erl b/apps/emqx/src/emqx_vm_mon.erl index 51710b5b5..703aca52f 100644 --- a/apps/emqx/src/emqx_vm_mon.erl +++ b/apps/emqx/src/emqx_vm_mon.erl @@ -49,11 +49,11 @@ init([]) -> {ok, #{}}. handle_call(Req, _From, State) -> - ?LOG(error, "[VM_MON] Unexpected call: ~p", [Req]), + ?SLOG(error, #{msg => "unexpected_call", call => Req}), {reply, ignored, State}. handle_cast(Msg, State) -> - ?LOG(error, "[VM_MON] Unexpected cast: ~p", [Msg]), + ?SLOG(error, #{msg => "unexpected_cast", cast => Msg}), {noreply, State}. handle_info({timeout, _Timer, check}, State) -> @@ -75,7 +75,7 @@ handle_info({timeout, _Timer, check}, State) -> {noreply, State}; handle_info(Info, State) -> - ?LOG(error, "[VM_MON] Unexpected info: ~p", [Info]), + ?SLOG(error, #{msg => "unexpected_info", info => Info}), {noreply, State}. terminate(_Reason, _State) -> diff --git a/apps/emqx/src/emqx_ws_connection.erl b/apps/emqx/src/emqx_ws_connection.erl index 32a81c26a..9ac8a03d0 100644 --- a/apps/emqx/src/emqx_ws_connection.erl +++ b/apps/emqx/src/emqx_ws_connection.erl @@ -181,10 +181,11 @@ init(Req, #{listener := {Type, Listener}} = Opts) -> idle_timeout => get_ws_opts(Type, Listener, idle_timeout) }, case check_origin_header(Req, Opts) of - {error, Message} -> - ?LOG(error, "Invalid Origin Header ~p~n", [Message]), + {error, Reason} -> + ?SLOG(error, #{msg => "invalid_origin_header", reason => Reason}), {ok, cowboy_req:reply(403, Req), WsOpts}; - ok -> parse_sec_websocket_protocol(Req, Opts, WsOpts) + ok -> + parse_sec_websocket_protocol(Req, Opts, WsOpts) end. parse_sec_websocket_protocol(Req, #{listener := {Type, Listener}} = Opts, WsOpts) -> @@ -231,7 +232,7 @@ parse_header_fun_origin(Req, #{listener := {Type, Listener}}) -> Value -> case lists:member(Value, get_ws_opts(Type, Listener, check_origins)) of true -> ok; - false -> {origin_not_allowed, Value} + false -> {error, #{bad_origin => Value}} end end. @@ -263,11 +264,12 @@ websocket_init([Req, #{zone := Zone, listener := {Type, Listener}} = Opts]) -> WsCookie = try cowboy_req:parse_cookies(Req) catch error:badarg -> - ?LOG(error, "Illegal cookie"), + ?SLOG(error, #{msg => "bad_cookie"}), undefined; Error:Reason -> - ?LOG(error, "Failed to parse cookie, Error: ~0p, Reason ~0p", - [Error, Reason]), + ?SLOG(error, #{msg => "failed_to_parse_cookie", + exception => Error, + reason => Reason}), undefined end, ConnInfo = #{socktype => ws, @@ -324,7 +326,7 @@ websocket_handle({binary, Data}, State) when is_list(Data) -> websocket_handle({binary, iolist_to_binary(Data)}, State); websocket_handle({binary, Data}, State) -> - ?LOG(debug, "RECV ~0p", [Data]), + ?SLOG(debug, #{msg => "RECV_data", data => Data, transport => websocket}), ok = inc_recv_stats(1, iolist_size(Data)), NState = ensure_stats_timer(State), return(parse_incoming(Data, NState)); @@ -339,7 +341,7 @@ websocket_handle({Frame, _}, State) when Frame =:= ping; Frame =:= pong -> websocket_handle({Frame, _}, State) -> %% TODO: should not close the ws connection - ?LOG(error, "Unexpected frame - ~p", [Frame]), + ?SLOG(error, #{msg => "unexpected_frame", frame => Frame}), shutdown(unexpected_ws_frame, State). websocket_info({call, From, Req}, State) -> @@ -397,11 +399,11 @@ websocket_info(Info, State) -> websocket_close({_, ReasonCode, _Payload}, State) when is_integer(ReasonCode) -> websocket_close(ReasonCode, State); websocket_close(Reason, State) -> - ?LOG(debug, "Websocket closed due to ~p~n", [Reason]), + ?SLOG(debug, #{msg => "websocket_closed", reason => Reason}), handle_info({sock_closed, Reason}, State). terminate(Reason, _Req, #state{channel = Channel}) -> - ?LOG(debug, "Terminated due to ~p", [Reason]), + ?SLOG(debug, #{msg => "terminated", reason => Reason}), emqx_channel:terminate(Reason, Channel); terminate(_Reason, _Req, _UnExpectedState) -> @@ -446,7 +448,7 @@ handle_info({connack, ConnAck}, State) -> return(enqueue(ConnAck, State)); handle_info({close, Reason}, State) -> - ?LOG(debug, "Force to close the socket due to ~p", [Reason]), + ?SLOG(debug, #{msg => "force_socket_close", reason => Reason}), return(enqueue({close, Reason}, State)); handle_info({event, connected}, State = #state{channel = Channel}) -> @@ -499,7 +501,7 @@ ensure_rate_limit(Stats, State = #state{limiter = Limiter}) -> {ok, Limiter1} -> State#state{limiter = Limiter1}; {pause, Time, Limiter1} -> - ?LOG(warning, "Pause ~pms due to rate limit", [Time]), + ?SLOG(warning, #{msg => "pause_due_to_rate_limit", time => Time}), TRef = start_timer(Time, limit_timeout), NState = State#state{sockstate = blocked, limiter = Limiter1, @@ -547,9 +549,19 @@ parse_incoming(Data, State = #state{parse_state = ParseState}) -> NState = State#state{parse_state = NParseState}, parse_incoming(Rest, postpone({incoming, Packet}, NState)) catch - error:Reason:Stk -> - ?LOG(error, "~nParse failed for ~0p~n~0p~nFrame data: ~0p", - [Reason, Stk, Data]), + throw : ?FRAME_PARSE_ERROR(Reason) -> + ?SLOG(info, #{ reason => Reason + , at_state => emqx_frame:describe_state(ParseState) + , input_bytes => Data + }), + FrameError = {frame_error, Reason}, + postpone({incoming, FrameError}, State); + error : Reason : Stacktrace -> + ?SLOG(error, #{ at_state => emqx_frame:describe_state(ParseState) + , input_bytes => Data + , exception => Reason + , stacktrace => Stacktrace + }), FrameError = {frame_error, Reason}, postpone({incoming, FrameError}, State) end. @@ -560,7 +572,7 @@ parse_incoming(Data, State = #state{parse_state = ParseState}) -> handle_incoming(Packet, State = #state{listener = {Type, Listener}}) when is_record(Packet, mqtt_packet) -> - ?LOG(debug, "RECV ~s", [emqx_packet:format(Packet)]), + ?SLOG(debug, #{msg => "RECV", packet => emqx_packet:format(Packet)}), ok = inc_incoming_stats(Packet), NState = case emqx_pd:get_counter(incoming_pubs) > get_active_n(Type, Listener) of @@ -617,15 +629,27 @@ handle_outgoing(Packets, State = #state{mqtt_piggyback = MQTTPiggyback, serialize_and_inc_stats_fun(#state{serialize = Serialize}) -> fun(Packet) -> - case emqx_frame:serialize_pkt(Packet, Serialize) of - <<>> -> ?LOG(warning, "~s is discarded due to the frame is too large.", - [emqx_packet:format(Packet)]), + try emqx_frame:serialize_pkt(Packet, Serialize) of + <<>> -> ?SLOG(warning, #{msg => "packet_discarded", + reason => "frame_too_large", + packet => emqx_packet:format(Packet)}), ok = emqx_metrics:inc('delivery.dropped.too_large'), ok = emqx_metrics:inc('delivery.dropped'), <<>>; - Data -> ?LOG(debug, "SEND ~s", [emqx_packet:format(Packet)]), + Data -> ?SLOG(debug, #{msg => "SEND", packet => Packet}), ok = inc_outgoing_stats(Packet), Data + catch + %% Maybe Never happen. + throw : ?FRAME_SERIALIZE_ERROR(Reason) -> + ?SLOG(info, #{ reason => Reason + , input_packet => Packet}), + erlang:error(?FRAME_SERIALIZE_ERROR(Reason)); + error : Reason : Stacktrace -> + ?SLOG(error, #{ input_packet => Packet + , exception => Reason + , stacktrace => Stacktrace}), + erlang:error(frame_serialize_error) end end. @@ -791,4 +815,4 @@ get_ws_opts(Type, Listener, Key) -> emqx_config:get_listener_conf(Type, Listener, [websocket, Key]). get_active_n(Type, Listener) -> - emqx_config:get_listener_conf(Type, Listener, [tcp, active_n]). \ No newline at end of file + emqx_config:get_listener_conf(Type, Listener, [tcp, active_n]). diff --git a/apps/emqx/test/emqx_alarm_SUITE.erl b/apps/emqx/test/emqx_alarm_SUITE.erl index 605300d2f..453061f9f 100644 --- a/apps/emqx/test/emqx_alarm_SUITE.erl +++ b/apps/emqx/test/emqx_alarm_SUITE.erl @@ -90,7 +90,7 @@ t_validity_period(_) -> ok = emqx_alarm:activate(a), ok = emqx_alarm:deactivate(a), ?assertNotEqual({error, not_found}, get_alarm(a, emqx_alarm:get_alarms(deactivated))), - ct:sleep(2000), + ct:sleep(3000), ?assertEqual({error, not_found}, get_alarm(a, emqx_alarm:get_alarms(deactivated))). get_alarm(Name, [Alarm = #{name := Name} | _More]) -> diff --git a/apps/emqx/test/emqx_authentication_SUITE.erl b/apps/emqx/test/emqx_authentication_SUITE.erl index 5fd2e47af..e4684649d 100644 --- a/apps/emqx/test/emqx_authentication_SUITE.erl +++ b/apps/emqx/test/emqx_authentication_SUITE.erl @@ -236,6 +236,9 @@ t_update_config(Config) when is_list(Config) -> ?assertMatch({ok, _}, update_config([authentication], {delete_authenticator, Global, ID1})), ?assertEqual({error, {not_found, {authenticator, ID1}}}, ?AUTHN:lookup_authenticator(Global, ID1)), + ?assertMatch({ok, _}, update_config([authentication], {delete_authenticator, Global, ID2})), + ?assertEqual({error, {not_found, {authenticator, ID2}}}, ?AUTHN:lookup_authenticator(Global, ID2)), + ListenerID = 'tcp:default', ConfKeyPath = [listeners, tcp, default, authentication], ?assertMatch({ok, _}, update_config(ConfKeyPath, {create_authenticator, ListenerID, AuthenticatorConfig1})), diff --git a/apps/emqx/test/emqx_broker_SUITE.erl b/apps/emqx/test/emqx_broker_SUITE.erl index fe754e9df..fbc374f90 100644 --- a/apps/emqx/test/emqx_broker_SUITE.erl +++ b/apps/emqx/test/emqx_broker_SUITE.erl @@ -37,38 +37,82 @@ init_per_suite(Config) -> end_per_suite(_Config) -> emqx_ct_helpers:stop_apps([]). +init_per_testcase(Case, Config) -> + ?MODULE:Case({init, Config}). + +end_per_testcase(Case, Config) -> + ?MODULE:Case({'end', Config}). + %%-------------------------------------------------------------------- %% PubSub Test %%-------------------------------------------------------------------- -t_stats_fun(_) -> - Subscribers = emqx_stats:getstat('subscribers.count'), - Subscriptions = emqx_stats:getstat('subscriptions.count'), - Subopts = emqx_stats:getstat('suboptions.count'), +t_stats_fun({init, Config}) -> + Parent = self(), + F = fun Loop() -> + N1 = emqx_stats:getstat('subscribers.count'), + N2 = emqx_stats:getstat('subscriptions.count'), + N3 = emqx_stats:getstat('suboptions.count'), + case N1 + N2 + N3 =:= 0 of + true -> + Parent ! {ready, self()}, + exit(normal); + false -> + receive + stop -> + exit(normal) + after + 100 -> + Loop() + end + end + end, + Pid = spawn_link(F), + receive + {ready, P} when P =:= Pid-> + Config + after + 5000 -> + Pid ! stop, + ct:fail("timedout_waiting_for_sub_stats_to_reach_zero") + end; +t_stats_fun(Config) when is_list(Config) -> ok = emqx_broker:subscribe(<<"topic">>, <<"clientid">>), ok = emqx_broker:subscribe(<<"topic2">>, <<"clientid">>), + %% ensure stats refreshed emqx_broker:stats_fun(), - ct:sleep(10), - ?assertEqual(Subscribers + 2, emqx_stats:getstat('subscribers.count')), - ?assertEqual(Subscribers + 2, emqx_stats:getstat('subscribers.max')), - ?assertEqual(Subscriptions + 2, emqx_stats:getstat('subscriptions.count')), - ?assertEqual(Subscriptions + 2, emqx_stats:getstat('subscriptions.max')), - ?assertEqual(Subopts + 2, emqx_stats:getstat('suboptions.count')), - ?assertEqual(Subopts + 2, emqx_stats:getstat('suboptions.max')). + %% emqx_stats:set_stat is a gen_server cast + %% make a synced call sync + ignored = gen_server:call(emqx_stats, call, infinity), + ?assertEqual(2, emqx_stats:getstat('subscribers.count')), + ?assertEqual(2, emqx_stats:getstat('subscribers.max')), + ?assertEqual(2, emqx_stats:getstat('subscriptions.count')), + ?assertEqual(2, emqx_stats:getstat('subscriptions.max')), + ?assertEqual(2, emqx_stats:getstat('suboptions.count')), + ?assertEqual(2, emqx_stats:getstat('suboptions.max')); +t_stats_fun({'end', _Config}) -> + ok = emqx_broker:unsubscribe(<<"topic">>), + ok = emqx_broker:unsubscribe(<<"topic2">>). -t_subscribed(_) -> +t_subscribed({init, Config}) -> emqx_broker:subscribe(<<"topic">>), + Config; +t_subscribed(Config) when is_list(Config) -> ?assertEqual(false, emqx_broker:subscribed(undefined, <<"topic">>)), - ?assertEqual(true, emqx_broker:subscribed(self(), <<"topic">>)), + ?assertEqual(true, emqx_broker:subscribed(self(), <<"topic">>)); +t_subscribed({'end', _Config}) -> emqx_broker:unsubscribe(<<"topic">>). -t_subscribed_2(_) -> +t_subscribed_2({init, Config}) -> emqx_broker:subscribe(<<"topic">>, <<"clientid">>), - %?assertEqual(true, emqx_broker:subscribed(<<"clientid">>, <<"topic">>)), - ?assertEqual(true, emqx_broker:subscribed(self(), <<"topic">>)), + Config; +t_subscribed_2(Config) when is_list(Config) -> + ?assertEqual(true, emqx_broker:subscribed(self(), <<"topic">>)); +t_subscribed_2({'end', _Config}) -> emqx_broker:unsubscribe(<<"topic">>). -t_subopts(_) -> +t_subopts({init, Config}) -> Config; +t_subopts(Config) when is_list(Config) -> ?assertEqual(false, emqx_broker:set_subopts(<<"topic">>, #{qos => 1})), ?assertEqual(undefined, emqx_broker:get_subopts(self(), <<"topic">>)), ?assertEqual(undefined, emqx_broker:get_subopts(<<"clientid">>, <<"topic">>)), @@ -85,42 +129,54 @@ t_subopts(_) -> ?assertEqual(true, emqx_broker:set_subopts(<<"topic">>, #{qos => 0})), ?assertEqual(#{nl => 0, qos => 0, rap => 0, rh => 0, subid => <<"clientid">>}, - emqx_broker:get_subopts(self(), <<"topic">>)), + emqx_broker:get_subopts(self(), <<"topic">>)); +t_subopts({'end', _Config}) -> emqx_broker:unsubscribe(<<"topic">>). -t_topics(_) -> +t_topics({init, Config}) -> Topics = [<<"topic">>, <<"topic/1">>, <<"topic/2">>], - ok = emqx_broker:subscribe(lists:nth(1, Topics), <<"clientId">>), - ok = emqx_broker:subscribe(lists:nth(2, Topics), <<"clientId">>), - ok = emqx_broker:subscribe(lists:nth(3, Topics), <<"clientId">>), + [{topics, Topics} | Config]; +t_topics(Config) when is_list(Config) -> + Topics = [T1, T2, T3] = proplists:get_value(topics, Config), + ok = emqx_broker:subscribe(T1, <<"clientId">>), + ok = emqx_broker:subscribe(T2, <<"clientId">>), + ok = emqx_broker:subscribe(T3, <<"clientId">>), Topics1 = emqx_broker:topics(), ?assertEqual(true, lists:foldl(fun(Topic, Acc) -> case lists:member(Topic, Topics1) of true -> Acc; false -> false end - end, true, Topics)), - emqx_broker:unsubscribe(lists:nth(1, Topics)), - emqx_broker:unsubscribe(lists:nth(2, Topics)), - emqx_broker:unsubscribe(lists:nth(3, Topics)). + end, true, Topics)); +t_topics({'end', Config}) -> + Topics = proplists:get_value(topics, Config), + lists:foreach(fun(T) -> emqx_broker:unsubscribe(T) end, Topics). -t_subscribers(_) -> +t_subscribers({init, Config}) -> emqx_broker:subscribe(<<"topic">>, <<"clientid">>), - ?assertEqual([self()], emqx_broker:subscribers(<<"topic">>)), + Config; +t_subscribers(Config) when is_list(Config) -> + ?assertEqual([self()], emqx_broker:subscribers(<<"topic">>)); +t_subscribers({'end', _Config}) -> emqx_broker:unsubscribe(<<"topic">>). -t_subscriptions(_) -> +t_subscriptions({init, Config}) -> emqx_broker:subscribe(<<"topic">>, <<"clientid">>, #{qos => 1}), - ok = timer:sleep(100), + Config; +t_subscriptions(Config) when is_list(Config) -> + ct:sleep(100), ?assertEqual(#{nl => 0, qos => 1, rap => 0, rh => 0, subid => <<"clientid">>}, proplists:get_value(<<"topic">>, emqx_broker:subscriptions(self()))), ?assertEqual(#{nl => 0, qos => 1, rap => 0, rh => 0, subid => <<"clientid">>}, - proplists:get_value(<<"topic">>, emqx_broker:subscriptions(<<"clientid">>))), + proplists:get_value(<<"topic">>, emqx_broker:subscriptions(<<"clientid">>))); +t_subscriptions({'end', _Config}) -> emqx_broker:unsubscribe(<<"topic">>). -t_sub_pub(_) -> +t_sub_pub({init, Config}) -> ok = emqx_broker:subscribe(<<"topic">>), - ct:sleep(10), + Config; +t_sub_pub(Config) when is_list(Config) -> + ct:sleep(100), emqx_broker:safe_publish(emqx_message:make(ct, <<"topic">>, <<"hello">>)), ?assert( receive @@ -130,16 +186,22 @@ t_sub_pub(_) -> false after 100 -> false - end). + end); +t_sub_pub({'end', _Config}) -> + ok = emqx_broker:unsubscribe(<<"topic">>). -t_nosub_pub(_) -> +t_nosub_pub({init, Config}) -> Config; +t_nosub_pub({'end', _Config}) -> ok; +t_nosub_pub(Config) when is_list(Config) -> ?assertEqual(0, emqx_metrics:val('messages.dropped')), emqx_broker:publish(emqx_message:make(ct, <<"topic">>, <<"hello">>)), ?assertEqual(1, emqx_metrics:val('messages.dropped')). -t_shared_subscribe(_) -> +t_shared_subscribe({init, Config}) -> emqx_broker:subscribe(<<"topic">>, <<"clientid">>, #{share => <<"group">>}), - ct:sleep(10), + ct:sleep(100), + Config; +t_shared_subscribe(Config) when is_list(Config) -> emqx_broker:safe_publish(emqx_message:make(ct, <<"topic">>, <<"hello">>)), ?assert(receive {deliver, <<"topic">>, #message{payload = <<"hello">>}} -> @@ -149,9 +211,12 @@ t_shared_subscribe(_) -> false after 100 -> false - end), + end); +t_shared_subscribe({'end', _Config}) -> emqx_broker:unsubscribe(<<"$share/group/topic">>). +t_shared_subscribe_2({init, Config}) -> Config; +t_shared_subscribe_2({'end', _Config}) -> ok; t_shared_subscribe_2(_) -> {ok, ConnPid} = emqtt:start_link([{clean_start, true}, {clientid, <<"clientid">>}]), {ok, _} = emqtt:connect(ConnPid), @@ -173,6 +238,8 @@ t_shared_subscribe_2(_) -> emqtt:disconnect(ConnPid), emqtt:disconnect(ConnPid2). +t_shared_subscribe_3({init, Config}) -> Config; +t_shared_subscribe_3({'end', _Config}) -> ok; t_shared_subscribe_3(_) -> {ok, ConnPid} = emqtt:start_link([{clean_start, true}, {clientid, <<"clientid">>}]), {ok, _} = emqtt:connect(ConnPid), @@ -189,11 +256,13 @@ t_shared_subscribe_3(_) -> emqtt:disconnect(ConnPid), emqtt:disconnect(ConnPid2). -t_shard(_) -> +t_shard({init, Config}) -> ok = meck:new(emqx_broker_helper, [passthrough, no_history]), ok = meck:expect(emqx_broker_helper, get_sub_shard, fun(_, _) -> 1 end), emqx_broker:subscribe(<<"topic">>, <<"clientid">>), - ct:sleep(10), + Config; +t_shard(Config) when is_list(Config) -> + ct:sleep(100), emqx_broker:safe_publish(emqx_message:make(ct, <<"topic">>, <<"hello">>)), ?assert( receive @@ -203,7 +272,9 @@ t_shard(_) -> false after 100 -> false - end), + end); +t_shard({'end', _Config}) -> + emqx_broker:unsubscribe(<<"topic">>), ok = meck:unload(emqx_broker_helper). recv_msgs(Count) -> diff --git a/apps/emqx/test/emqx_cm_SUITE.erl b/apps/emqx/test/emqx_cm_SUITE.erl index d492edd0e..512e07c9c 100644 --- a/apps/emqx/test/emqx_cm_SUITE.erl +++ b/apps/emqx/test/emqx_cm_SUITE.erl @@ -141,11 +141,8 @@ t_open_session_race_condition(_) -> end end, N = 1000, - [spawn( - fun() -> - spawn(OpenASession), - spawn(OpenASession) - end) || _ <- lists:seq(1, N)], + Pids = lists:flatten([[spawn_monitor(OpenASession), spawn_monitor(OpenASession)] || + _ <- lists:seq(1, N)]), WaitingRecv = fun _Wr(N1, N2, 0) -> {N1, N2}; @@ -158,14 +155,29 @@ t_open_session_race_condition(_) -> {Succeeded, Failed} = WaitingRecv(0, 0, 2 * N), ct:pal("Race condition status: succeeded=~p failed=~p~n", [Succeeded, Failed]), + ?assertEqual(2 * N, length(Pids)), + WaitForDowns = + fun _Wd([{Pid, _Ref}]) -> Pid; + _Wd(Pids0) -> + receive + {'DOWN', DownRef, process, DownPid, _} -> + ?assert(lists:member({DownPid, DownRef}, Pids0)), + _Wd(lists:delete({DownPid, DownRef}, Pids0)) + after + 10000 -> + exit(timeout) + end + end, + Winner = WaitForDowns(Pids), ?assertMatch([_], ets:lookup(emqx_channel, ClientId)), - [Pid] = emqx_cm:lookup_channels(ClientId), - ?assertMatch([_], ets:lookup(emqx_channel_conn, {ClientId, Pid})), + ?assertEqual([Winner], emqx_cm:lookup_channels(ClientId)), + ?assertMatch([_], ets:lookup(emqx_channel_conn, {ClientId, Winner})), ?assertMatch([_], ets:lookup(emqx_channel_registry, ClientId)), - exit(Pid, kill), - timer:sleep(100), %% TODO deterministic + exit(Winner, kill), + receive {'DOWN', _, process, Winner, _} -> ok end, + ignored = gen_server:call(emqx_cm, ignore, infinity), %% sync ?assertEqual([], emqx_cm:lookup_channels(ClientId)). t_discard_session(_) -> diff --git a/apps/emqx/test/emqx_frame_SUITE.erl b/apps/emqx/test/emqx_frame_SUITE.erl index 09206cee1..6d3bccd99 100644 --- a/apps/emqx/test/emqx_frame_SUITE.erl +++ b/apps/emqx/test/emqx_frame_SUITE.erl @@ -22,7 +22,9 @@ -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). --include_lib("emqx_ct_helpers/include/emqx_ct.hrl"). + +-define(ASSERT_FRAME_THROW(Reason, Expr), + ?assertThrow(?FRAME_PARSE_ERROR(Reason), Expr)). all() -> [{group, parse}, @@ -113,7 +115,7 @@ init_per_group(_Group, Config) -> Config. end_per_group(_Group, _Config) -> - ok. + ok. t_parse_cont(_) -> Packet = ?CONNECT_PACKET(#mqtt_packet_connect{}), @@ -127,15 +129,15 @@ t_parse_cont(_) -> t_parse_frame_too_large(_) -> Packet = ?PUBLISH_PACKET(?QOS_1, <<"t">>, 1, payload(1000)), - ?catch_error(frame_too_large, parse_serialize(Packet, #{max_size => 256})), - ?catch_error(frame_too_large, parse_serialize(Packet, #{max_size => 512})), + ?ASSERT_FRAME_THROW(frame_too_large, parse_serialize(Packet, #{max_size => 256})), + ?ASSERT_FRAME_THROW(frame_too_large, parse_serialize(Packet, #{max_size => 512})), ?assertEqual(Packet, parse_serialize(Packet, #{max_size => 2048, version => ?MQTT_PROTO_V4})). t_parse_frame_malformed_variable_byte_integer(_) -> - MalformedPayload = << <<16#80>> || _ <- lists:seq(1, 4) >>, + MalformedPayload = << <<16#80>> || _ <- lists:seq(1, 6) >>, ParseState = emqx_frame:initial_parse_state(#{}), - ?catch_error(malformed_variable_byte_integer, - emqx_frame:parse(MalformedPayload, ParseState)). + ?ASSERT_FRAME_THROW(malformed_variable_byte_integer, + emqx_frame:parse(MalformedPayload, ParseState)). t_serialize_parse_v3_connect(_) -> Bin = <<16,37,0,6,77,81,73,115,100,112,3,2,0,60,0,23,109,111,115, @@ -329,7 +331,7 @@ t_serialize_parse_qos1_publish(_) -> ?assertEqual(Bin, serialize_to_binary(Packet)), ?assertMatch(Packet, parse_to_packet(Bin, #{strict_mode => true})), %% strict_mode = true - ?catch_error(bad_packet_id, parse_serialize(?PUBLISH_PACKET(?QOS_1, <<"Topic">>, 0, <<>>))), + ?ASSERT_FRAME_THROW(bad_packet_id, parse_serialize(?PUBLISH_PACKET(?QOS_1, <<"Topic">>, 0, <<>>))), %% strict_mode = false _ = parse_serialize(?PUBLISH_PACKET(?QOS_1, <<"Topic">>, 0, <<>>), #{strict_mode => false}). @@ -340,7 +342,7 @@ t_serialize_parse_qos2_publish(_) -> ?assertEqual(Bin, serialize_to_binary(Packet)), ?assertMatch(Packet, parse_to_packet(Bin, #{strict_mode => true})), %% strict_mode = true - ?catch_error(bad_packet_id, parse_serialize(?PUBLISH_PACKET(?QOS_2, <<"Topic">>, 0, <<>>))), + ?ASSERT_FRAME_THROW(bad_packet_id, parse_serialize(?PUBLISH_PACKET(?QOS_2, <<"Topic">>, 0, <<>>))), %% strict_mode = false _ = parse_serialize(?PUBLISH_PACKET(?QOS_2, <<"Topic">>, 0, <<>>), #{strict_mode => false}). @@ -360,7 +362,7 @@ t_serialize_parse_puback(_) -> ?assertEqual(<<64,2,0,1>>, serialize_to_binary(Packet)), ?assertEqual(Packet, parse_serialize(Packet)), %% strict_mode = true - ?catch_error(bad_packet_id, parse_serialize(?PUBACK_PACKET(0))), + ?ASSERT_FRAME_THROW(bad_packet_id, parse_serialize(?PUBACK_PACKET(0))), %% strict_mode = false ?PUBACK_PACKET(0) = parse_serialize(?PUBACK_PACKET(0), #{strict_mode => false}). @@ -381,7 +383,7 @@ t_serialize_parse_pubrec(_) -> ?assertEqual(<<5:4,0:4,2,0,1>>, serialize_to_binary(Packet)), ?assertEqual(Packet, parse_serialize(Packet)), %% strict_mode = true - ?catch_error(bad_packet_id, parse_serialize(?PUBREC_PACKET(0))), + ?ASSERT_FRAME_THROW(bad_packet_id, parse_serialize(?PUBREC_PACKET(0))), %% strict_mode = false ?PUBREC_PACKET(0) = parse_serialize(?PUBREC_PACKET(0), #{strict_mode => false}). @@ -397,11 +399,11 @@ t_serialize_parse_pubrel(_) -> %% PUBREL with bad qos 0 Bin0 = <<6:4,0:4,2,0,1>>, ?assertMatch(Packet, parse_to_packet(Bin0, #{strict_mode => false})), - ?catch_error(bad_frame_header, parse_to_packet(Bin0, #{strict_mode => true})), + ?ASSERT_FRAME_THROW(bad_frame_header, parse_to_packet(Bin0, #{strict_mode => true})), %% strict_mode = false ?PUBREL_PACKET(0) = parse_serialize(?PUBREL_PACKET(0), #{strict_mode => false}), %% strict_mode = true - ?catch_error(bad_packet_id, parse_serialize(?PUBREL_PACKET(0))). + ?ASSERT_FRAME_THROW(bad_packet_id, parse_serialize(?PUBREL_PACKET(0))). t_serialize_parse_pubrel_v5(_) -> Packet = ?PUBREL_PACKET(16, ?RC_SUCCESS, #{'Reason-String' => <<"success">>}), @@ -415,7 +417,7 @@ t_serialize_parse_pubcomp(_) -> %% strict_mode = false ?PUBCOMP_PACKET(0) = parse_serialize(?PUBCOMP_PACKET(0), #{strict_mode => false}), %% strict_mode = true - ?catch_error(bad_packet_id, parse_serialize(?PUBCOMP_PACKET(0))). + ?ASSERT_FRAME_THROW(bad_packet_id, parse_serialize(?PUBCOMP_PACKET(0))). t_serialize_parse_pubcomp_v5(_) -> Packet = ?PUBCOMP_PACKET(16, ?RC_SUCCESS, #{'Reason-String' => <<"success">>}), @@ -434,12 +436,12 @@ t_serialize_parse_subscribe(_) -> ?assertMatch(Packet, parse_to_packet(Bin0, #{strict_mode => false})), %% strict_mode = false _ = parse_to_packet(Bin0, #{strict_mode => false}), - ?catch_error(bad_frame_header, parse_to_packet(Bin0, #{strict_mode => true})), + ?ASSERT_FRAME_THROW(bad_frame_header, parse_to_packet(Bin0, #{strict_mode => true})), %% strict_mode = false _ = parse_serialize(?SUBSCRIBE_PACKET(0, TopicFilters), #{strict_mode => false}), %% strict_mode = true - ?catch_error(bad_packet_id, parse_serialize(?SUBSCRIBE_PACKET(0, TopicFilters))), - ?catch_error(bad_subqos, parse_serialize(?SUBSCRIBE_PACKET(1, [{<<"t">>, #{qos => 3}}]))). + ?ASSERT_FRAME_THROW(bad_packet_id, parse_serialize(?SUBSCRIBE_PACKET(0, TopicFilters))), + ?ASSERT_FRAME_THROW(bad_subqos, parse_serialize(?SUBSCRIBE_PACKET(1, [{<<"t">>, #{qos => 3}}]))). t_serialize_parse_subscribe_v5(_) -> TopicFilters = [{<<"TopicQos0">>, #{rh => 1, qos => ?QOS_2, rap => 0, nl => 0}}, @@ -453,7 +455,7 @@ t_serialize_parse_suback(_) -> %% strict_mode = false _ = parse_serialize(?SUBACK_PACKET(0, [?QOS_0]), #{strict_mode => false}), %% strict_mode = true - ?catch_error(bad_packet_id, parse_serialize(?SUBACK_PACKET(0, [?QOS_0]))). + ?ASSERT_FRAME_THROW(bad_packet_id, parse_serialize(?SUBACK_PACKET(0, [?QOS_0]))). t_serialize_parse_suback_v5(_) -> Packet = ?SUBACK_PACKET(1, #{'Reason-String' => <<"success">>, @@ -471,11 +473,11 @@ t_serialize_parse_unsubscribe(_) -> %% UNSUBSCRIBE(Q1, R0, D0, PacketId=2, TopicTable=[<<"TopicA">>]) Bin0 = <>, ?assertMatch(Packet, parse_to_packet(Bin0, #{strict_mode => false})), - ?catch_error(bad_frame_header, parse_to_packet(Bin0, #{strict_mode => true})), + ?ASSERT_FRAME_THROW(bad_frame_header, parse_to_packet(Bin0, #{strict_mode => true})), %% strict_mode = false _ = parse_serialize(?UNSUBSCRIBE_PACKET(0, [<<"TopicA">>]), #{strict_mode => false}), %% strict_mode = true - ?catch_error(bad_packet_id, parse_serialize(?UNSUBSCRIBE_PACKET(0, [<<"TopicA">>]))). + ?ASSERT_FRAME_THROW(bad_packet_id, parse_serialize(?UNSUBSCRIBE_PACKET(0, [<<"TopicA">>]))). t_serialize_parse_unsubscribe_v5(_) -> Props = #{'User-Property' => [{<<"key">>, <<"val">>}]}, @@ -550,4 +552,3 @@ parse_to_packet(Bin, Opts) -> Packet. payload(Len) -> iolist_to_binary(lists:duplicate(Len, 1)). - diff --git a/apps/emqx/test/emqx_inflight_SUITE.erl b/apps/emqx/test/emqx_inflight_SUITE.erl index a819e788b..6e5d8f5cc 100644 --- a/apps/emqx/test/emqx_inflight_SUITE.erl +++ b/apps/emqx/test/emqx_inflight_SUITE.erl @@ -20,10 +20,9 @@ -compile(nowarn_export_all). -include_lib("eunit/include/eunit.hrl"). --include_lib("emqx_ct_helpers/include/emqx_ct.hrl"). all() -> emqx_ct:all(?MODULE). - + t_contain(_) -> Inflight = emqx_inflight:insert(k, v, emqx_inflight:new()), ?assert(emqx_inflight:contain(k, Inflight)), @@ -41,12 +40,12 @@ t_insert(_) -> ?assertEqual(2, emqx_inflight:size(Inflight)), ?assertEqual({value, 1}, emqx_inflight:lookup(a, Inflight)), ?assertEqual({value, 2}, emqx_inflight:lookup(b, Inflight)), - ?catch_error({key_exists, a}, emqx_inflight:insert(a, 1, Inflight)). + ?assertError({key_exists, a}, emqx_inflight:insert(a, 1, Inflight)). t_update(_) -> Inflight = emqx_inflight:insert(k, v, emqx_inflight:new()), ?assertEqual(Inflight, emqx_inflight:update(k, v, Inflight)), - ?catch_error(function_clause, emqx_inflight:update(badkey, v, Inflight)). + ?assertError(function_clause, emqx_inflight:update(badkey, v, Inflight)). t_resize(_) -> Inflight = emqx_inflight:insert(k, v, emqx_inflight:new(2)), diff --git a/apps/emqx/test/emqx_message_SUITE.erl b/apps/emqx/test/emqx_message_SUITE.erl index 63db85664..7c8435a8d 100644 --- a/apps/emqx/test/emqx_message_SUITE.erl +++ b/apps/emqx/test/emqx_message_SUITE.erl @@ -141,13 +141,6 @@ t_undefined_headers(_) -> Msg2 = emqx_message:set_header(c, 3, Msg), ?assertEqual(3, emqx_message:get_header(c, Msg2)). -t_format(_) -> - Msg = emqx_message:make(<<"clientid">>, <<"topic">>, <<"payload">>), - io:format("~s~n", [emqx_message:format(Msg)]), - Msg1 = emqx_message:set_header(properties, #{'Subscription-Identifier' => 1}, - emqx_message:set_flag(dup, Msg)), - io:format("~s~n", [emqx_message:format(Msg1)]). - t_is_expired(_) -> Msg = emqx_message:make(<<"clientid">>, <<"topic">>, <<"payload">>), ?assertNot(emqx_message:is_expired(Msg)), @@ -206,7 +199,9 @@ t_to_map(_) -> {headers, #{}}, {topic, <<"topic">>}, {payload, <<"payload">>}, - {timestamp, emqx_message:timestamp(Msg)}], + {timestamp, emqx_message:timestamp(Msg)}, + {extra, []} + ], ?assertEqual(List, emqx_message:to_list(Msg)), ?assertEqual(maps:from_list(List), emqx_message:to_map(Msg)). @@ -219,6 +214,8 @@ t_from_map(_) -> headers => #{}, topic => <<"topic">>, payload => <<"payload">>, - timestamp => emqx_message:timestamp(Msg)}, + timestamp => emqx_message:timestamp(Msg), + extra => [] + }, ?assertEqual(Map, emqx_message:to_map(Msg)), ?assertEqual(Msg, emqx_message:from_map(emqx_message:to_map(Msg))). diff --git a/apps/emqx/test/emqx_mqtt_props_SUITE.erl b/apps/emqx/test/emqx_mqtt_props_SUITE.erl index 2e96182b0..b4dcd2f18 100644 --- a/apps/emqx/test/emqx_mqtt_props_SUITE.erl +++ b/apps/emqx/test/emqx_mqtt_props_SUITE.erl @@ -21,7 +21,6 @@ -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("eunit/include/eunit.hrl"). --include_lib("emqx_ct_helpers/include/emqx_ct.hrl"). all() -> emqx_ct:all(?MODULE). @@ -30,14 +29,14 @@ t_id(_) -> fun({Id, Prop}) -> ?assertEqual(Id, emqx_mqtt_props:id(element(1, Prop))) end), - ?catch_error({bad_property, 'Bad-Property'}, emqx_mqtt_props:id('Bad-Property')). + ?assertError({bad_property, 'Bad-Property'}, emqx_mqtt_props:id('Bad-Property')). t_name(_) -> foreach_prop( fun({Id, Prop}) -> ?assertEqual(emqx_mqtt_props:name(Id), element(1, Prop)) end), - ?catch_error({unsupported_property, 16#FF}, emqx_mqtt_props:name(16#FF)). + ?assertError({unsupported_property, 16#FF}, emqx_mqtt_props:name(16#FF)). t_filter(_) -> ConnProps = #{'Session-Expiry-Interval' => 1, @@ -60,7 +59,7 @@ t_validate(_) -> }, ok = emqx_mqtt_props:validate(ConnProps), BadProps = #{'Unknown-Property' => 10}, - ?catch_error({bad_property,'Unknown-Property'}, + ?assertError({bad_property,'Unknown-Property'}, emqx_mqtt_props:validate(BadProps)). t_validate_value(_) -> @@ -68,11 +67,11 @@ t_validate_value(_) -> ok = emqx_mqtt_props:validate(#{'Reason-String' => <<"Unknown Reason">>}), ok = emqx_mqtt_props:validate(#{'User-Property' => {<<"Prop">>, <<"Val">>}}), ok = emqx_mqtt_props:validate(#{'User-Property' => [{<<"Prop">>, <<"Val">>}]}), - ?catch_error({bad_property_value, {'Payload-Format-Indicator', 16#FFFF}}, + ?assertError({bad_property_value, {'Payload-Format-Indicator', 16#FFFF}}, emqx_mqtt_props:validate(#{'Payload-Format-Indicator' => 16#FFFF})), - ?catch_error({bad_property_value, {'Server-Keep-Alive', 16#FFFFFF}}, + ?assertError({bad_property_value, {'Server-Keep-Alive', 16#FFFFFF}}, emqx_mqtt_props:validate(#{'Server-Keep-Alive' => 16#FFFFFF})), - ?catch_error({bad_property_value, {'Will-Delay-Interval', -16#FF}}, + ?assertError({bad_property_value, {'Will-Delay-Interval', -16#FF}}, emqx_mqtt_props:validate(#{'Will-Delay-Interval' => -16#FF})). foreach_prop(Fun) -> @@ -86,4 +85,4 @@ foreach_prop(Fun) -> % error('TODO'). % t_get(_) -> -% error('TODO'). \ No newline at end of file +% error('TODO'). diff --git a/apps/emqx/test/emqx_proper_types.erl b/apps/emqx/test/emqx_proper_types.erl new file mode 100644 index 000000000..4b072e2d7 --- /dev/null +++ b/apps/emqx/test/emqx_proper_types.erl @@ -0,0 +1,483 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2019 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% The proper types extension for EMQ X + +-module(emqx_proper_types). + +-include_lib("proper/include/proper.hrl"). +-include("emqx.hrl"). + +%% High level Types +-export([ conninfo/0 + , clientinfo/0 + , sessioninfo/0 + , connack_return_code/0 + , message/0 + , topictab/0 + , topic/0 + , systopic/0 + , subopts/0 + , nodename/0 + , normal_topic/0 + , normal_topic_filter/0 + ]). + +%% Basic Types +-export([ url/0 + , ip/0 + , port/0 + , limited_atom/0 + , limited_latin_atom/0 + ]). + +%% Iterators +-export([ nof/1 + ]). + +%%-------------------------------------------------------------------- +%% Types High level +%%-------------------------------------------------------------------- + +%% Type defined emqx_types.erl - conninfo() +conninfo() -> + Keys = [{socktype, socktype()}, + {sockname, peername()}, + {peername, peername()}, + {peercert, peercert()}, + {conn_mod, conn_mod()}, + {proto_name, proto_name()}, + {proto_ver, non_neg_integer()}, + {clean_start, boolean()}, + {clientid, clientid()}, + {username, username()}, + {conn_props, properties()}, + {connected, boolean()}, + {connected_at, timestamp()}, + {keepalive, range(0, 16#ffff)}, + {receive_maximum, non_neg_integer()}, + {expiry_interval, non_neg_integer()}], + ?LET({Ks, M}, {Keys, map(limited_atom(), limited_any_term())}, + begin + maps:merge(maps:from_list(Ks), M) + end). + +clientinfo() -> + Keys = [{zone, zone()}, + {protocol, protocol()}, + {peerhost, ip()}, + {sockport, port()}, + {clientid, clientid()}, + {username, username()}, + {is_bridge, boolean()}, + {is_supuser, boolean()}, + {mountpoint, maybe(utf8())}, + {ws_cookie, maybe(list())} + % password, + % auth_result, + % anonymous, + % cn, + % dn, + ], + ?LET({Ks, M}, {Keys, map(limited_atom(), limited_any_term())}, + begin + maps:merge(maps:from_list(Ks), M) + end). + +%% See emqx_session:session() type define +sessioninfo() -> + ?LET(Session, {session, + subscriptions(), % subscriptions + non_neg_integer(), % max_subscriptions + boolean(), % upgrade_qos + inflight(), % emqx_inflight:inflight() + mqueue(), % emqx_mqueue:mqueue() + packet_id(), % next_pkt_id + safty_timeout(), % retry_interval + awaiting_rel(), % awaiting_rel + non_neg_integer(), % max_awaiting_rel + safty_timeout(), % await_rel_timeout + timestamp() % created_at + }, + emqx_session:info(Session)). + +subscriptions() -> + ?LET(L, list({topic(), subopts()}), maps:from_list(L)). + +inflight() -> + ?LET(MaxLen, non_neg_integer(), + begin + ?LET(Msgs, limited_list(MaxLen, {packet_id(), message(), timestamp()}), + begin + lists:foldl(fun({PktId, Msg, Ts}, Ift) -> + try + emqx_inflight:insert(PktId, {Msg, Ts}, Ift) + catch _:_ -> + Ift + end + end, emqx_inflight:new(MaxLen), Msgs) + end) + end). + +mqueue() -> + ?LET({MaxLen, IsStoreQos0}, {non_neg_integer(), boolean()}, + begin + ?LET(Msgs, limited_list(MaxLen, message()), + begin + Q = emqx_mqueue:init(#{max_len => MaxLen, store_qos0 => IsStoreQos0}), + lists:foldl(fun(Msg, Acc) -> + {_Dropped, NQ} = emqx_mqueue:in(Msg, Acc), + NQ + end, Q, Msgs) + end) + end). + +message() -> + #message{ + id = emqx_guid:gen(), + qos = qos(), + from = from(), + flags = flags(), + headers = map(limited_latin_atom(), limited_any_term()), %% headers + topic = topic(), + payload = payload(), + timestamp = timestamp(), + extra = [] + }. + +%% @private +flags() -> + ?LET({Dup, Retain}, {boolean(), boolean()}, #{dup => Dup, retain => Retain}). + +packet_id() -> + range(1, 16#ffff). + +awaiting_rel() -> + ?LET(L, list({packet_id(), timestamp()}), maps:from_list(L)). + +connack_return_code() -> + oneof([ success + , protocol_error + , client_identifier_not_valid + , bad_username_or_password + , bad_clientid_or_password + , username_or_password_undefined + , password_error + , not_authorized + , server_unavailable + , server_busy + , banned + , bad_authentication_method + ]). + +topictab() -> + non_empty(list({topic(), subopts()})). + +topic() -> + oneof([normal_topic(), + normal_topic_filter(), + systopic_broker(), systopic_present(), systopic_stats(), + systopic_metrics(), systopic_alarms(), systopic_mon(), + sharetopic()]). + +subopts() -> + ?LET({Nl, Qos, Rap, Rh}, + {range(0, 1), qos(), + range(0, 1), range(0, 1)}, + #{nl => Nl, qos => Qos, rap => Rap, rh => Rh}). + +qos() -> + range(0, 2). + +from() -> + oneof([limited_latin_atom()]). + +payload() -> + binary(). + +safty_timeout() -> + non_neg_integer(). + +nodename() -> + ?LET({Name, Ip}, {non_empty(list(latin_char())), ip()}, + begin + binary_to_atom(iolist_to_binary([Name, "@", inet:ntoa(Ip)]), utf8) + end). + +systopic() -> + oneof( + [systopic_broker(), systopic_present(), systopic_stats(), + systopic_metrics(), systopic_alarms(), systopic_mon()]). + +systopic_broker() -> + Topics = [<<"">>, <<"version">>, <<"uptime">>, <<"datetime">>, <<"sysdescr">>], + ?LET({Nodename, T}, + {nodename(), oneof(Topics)}, + begin + case byte_size(T) of + 0 -> <<"$SYS/brokers">>; + _ -> + <<"$SYS/brokers/", (ensure_bin(Nodename))/binary, "/", T/binary>> + end + end). + +systopic_present() -> + ?LET({Nodename, ClientId, T}, + {nodename(), clientid(), oneof([<<"connected">>, <<"disconnected">>])}, + begin + <<"$SYS/brokers/", (ensure_bin(Nodename))/binary, "/clients/", (ensure_bin(ClientId))/binary, "/", T/binary>> + end). + +systopic_stats() -> + Topics = [<<"connections/max">>, <<"connections/count">>, + <<"suboptions/max">>, <<"suboptions/count">>, + <<"subscribers/max">>, <<"subscribers/count">>, + <<"subscriptions/max">>, <<"subscriptions/count">>, + <<"subscriptions/shared/max">>, <<"subscriptions/shared/count">>, + <<"topics/max">>, <<"topics/count">>, + <<"routes/max">>, <<"routes/count">> + ], + ?LET({Nodename, T}, + {nodename(), oneof(Topics)}, + <<"$SYS/brokers/", (ensure_bin(Nodename))/binary, "/stats/", T/binary>>). + +systopic_metrics() -> + Topics = [<<"bytes/received">>, <<"bytes/sent">>, + <<"packets/received">>, <<"packets/sent">>, + <<"packets/connect/received">>, <<"packets/connack/sent">>, + <<"packets/publish/received">>, <<"packets/publish/sent">>, + <<"packets/publish/error">>, <<"packets/publish/auth_error">>, + <<"packets/publish/dropped">>, + <<"packets/puback/received">>, <<"packets/puback/sent">>, + <<"packets/puback/inuse">>, <<"packets/puback/missed">>, + <<"packets/pubrec/received">>, <<"packets/pubrec/sent">>, + <<"packets/pubrec/inuse">>, <<"packets/pubrec/missed">>, + <<"packets/pubrel/received">>, <<"packets/pubrel/sent">>, + <<"packets/pubrel/missed">>, + <<"packets/pubcomp/received">>, <<"packets/pubcomp/sent">>, + <<"packets/pubcomp/inuse">>, <<"packets/pubcomp/missed">>, + <<"packets/subscribe/received">>, <<"packets/subscribe/error">>, + <<"packets/subscribe/auth_error">>, <<"packets/suback/sent">>, + <<"packets/unsubscribe/received">>, <<"packets/unsuback/sent">>, + <<"packets/pingreq/received">>, <<"packets/pingresp/sent">>, + <<"packets/disconnect/received">>, <<"packets/disconnect/sent">>, + <<"packets/auth/received">>, <<"packets/auth/sent">>, + <<"messages/received">>, <<"messages/sent">>, + <<"messages/qos0/received">>, <<"messages/qos0/sent">>, + <<"messages/qos1/received">>, <<"messages/qos1/sent">>, + <<"messages/qos2/received">>, <<"messages/qos2/sent">>, + <<"messages/publish">>, <<"messages/dropped">>, + <<"messages/dropped/expired">>, <<"messages/dropped/no_subscribers">>, + <<"messages/forward">>, <<"messages/retained">>, + <<"messages/delayed">>, <<"messages/delivered">>, + <<"messages/acked">>], + ?LET({Nodename, T}, + {nodename(), oneof(Topics)}, + <<"$SYS/brokers/", (ensure_bin(Nodename))/binary, "/metrics/", T/binary>>). + +systopic_alarms() -> + ?LET({Nodename, T}, + {nodename(), oneof([<<"alert">>, <<"clear">>])}, + <<"$SYS/brokers/", (ensure_bin(Nodename))/binary, "/alarms/", T/binary>>). + +systopic_mon() -> + Topics = [<<"long_gc">>, <<"long_schedule">>, + <<"large_heap">>, <<"busy_port">>, <<"busy_dist_port">>], + ?LET({Nodename, T}, + {nodename(), oneof(Topics)}, + <<"$SYS/brokers/", (ensure_bin(Nodename))/binary, "/sysmon/", T/binary>>). + +sharetopic() -> + ?LET({Type, Grp, T}, + {oneof([<<"$queue">>, <<"$share">>]), list(latin_char()), normal_topic()}, + <>). + +normal_topic() -> + ?LET(L, list(frequency([{3, latin_char()}, {1, $/}])), + list_to_binary(L)). + +normal_topic_filter() -> + ?LET({L, Wild}, {list(list(latin_char())), oneof(['#', '+'])}, + begin + case Wild of + '#' -> + case L of + [] -> <<"#">>; + _ -> iolist_to_binary([lists:join("/", L), "/#"]) + end; + '+' -> + case L of + [] -> <<"+">>; + _ -> + L1 = [case rand:uniform(3) == 1 of + true -> "+"; + _ -> E + end || E <- L], + iolist_to_binary(lists:join("/", L1)) + end + end + end). + +%%-------------------------------------------------------------------- +%% Basic Types +%%-------------------------------------------------------------------- + +maybe(T) -> + oneof([undefined, T]). + +socktype() -> + oneof([tcp, udp, ssl, proxy]). + +peername() -> + {ip(), port()}. + +peercert() -> + %% TODO: cert? + oneof([nossl, undefined]). + +conn_mod() -> + oneof([emqx_connection, emqx_ws_connection, emqx_coap_mqtt_adapter, + emqx_sn_gateway, emqx_lwm2m_protocol, emqx_gbt32960_conn, + emqx_jt808_connection, emqx_tcp_connection]). + +proto_name() -> + oneof([<<"MQTT">>, <<"MQTT-SN">>, <<"CoAP">>, <<"LwM2M">>, utf8()]). + +clientid() -> + utf8(). + +username() -> + maybe(utf8()). + +properties() -> + map(limited_latin_atom(), binary()). + +%% millisecond +timestamp() -> + %% 12h <- Now -> 12h + ?LET(Offset, range(-43200, 43200), erlang:system_time(millisecond) + Offset). + +zone() -> + oneof([external, internal, limited_latin_atom()]). + +protocol() -> + oneof([mqtt, 'mqtt-sn', coap, lwm2m, limited_latin_atom()]). + +url() -> + ?LET({Schema, IP, Port, Path}, {oneof(["http://", "https://"]), ip(), port(), http_path()}, + begin + IP1 = case tuple_size(IP) == 8 of + true -> "[" ++ inet:ntoa(IP) ++ "]"; + false -> inet:ntoa(IP) + end, + lists:concat([Schema, IP1, ":", integer_to_list(Port), "/", Path]) + end). + +ip() -> + oneof([ipv4(), ipv6(), ipv6_from_v4()]). + +ipv4() -> + ?LET(IP, {range(1, 16#ff), range(0, 16#ff), + range(0, 16#ff), range(0, 16#ff)}, IP). + +ipv6() -> + ?LET(IP, {range(0, 16#ff), range(0, 16#ff), + range(0, 16#ff), range(0, 16#ff), + range(0, 16#ff), range(0, 16#ff), + range(0, 16#ff), range(0, 16#ff)}, IP). + +ipv6_from_v4() -> + ?LET(IP, {range(1, 16#ff), range(0, 16#ff), + range(0, 16#ff), range(0, 16#ff)}, + inet:ipv4_mapped_ipv6_address(IP)). + +port() -> + ?LET(Port, range(1, 16#ffff), Port). + +http_path() -> + list(frequency([{3, latin_char()}, + {1, $/}])). + +latin_char() -> + oneof([integer($0, $9), integer($A, $Z), integer($a, $z)]). + +limited_latin_atom() -> + oneof([ 'abc_atom' + , '0123456789' + , 'ABC-ATOM' + , 'abc123ABC' + ]). + +%% Avoid generating a lot of atom and causing atom table overflows +limited_atom() -> + oneof([ 'a_normal_atom' + , '10123_num_prefixed_atom' + , '___dash_prefixed_atom' + , '123' + , binary_to_atom(<<"你好_utf8_atom"/utf8>>) + , '_', ' ', '""', '#$%^&*' + %% The longest atom with 255 chars + , list_to_atom( + lists:append([ "so" + , [ $o || _ <- lists:seq(1, 243)] + , "-long-atom"] + ) + ) + ]). + +limited_any_term() -> + oneof([binary(), number(), string()]). + +%%-------------------------------------------------------------------- +%% Iterators +%%-------------------------------------------------------------------- + +nof(Ls) when is_list(Ls) -> + Len = length(Ls), + ?LET(N, range(0, Len), + begin + Ns = rand_nl(N, Len, []), + [lists:nth(I, Ls) || I <- Ns] + end). + +limited_list(0, T) -> + list(T); + +limited_list(N, T) -> + ?LET(N2, range(0, N), + begin + [T || _ <- lists:seq(1, N2)] + end). + +%%-------------------------------------------------------------------- +%% Internal funcs +%%-------------------------------------------------------------------- + +-compile({inline, rand_nl/3}). + +rand_nl(0, _, Acc) -> + Acc; +rand_nl(N, L, Acc) -> + R = rand:uniform(L), + case lists:member(R, Acc) of + true -> rand_nl(N, L, Acc); + _ -> rand_nl(N-1, L, [R|Acc]) + end. + +ensure_bin(A) when is_atom(A) -> + atom_to_binary(A, utf8); +ensure_bin(B) when is_binary(B) -> + B. diff --git a/apps/emqx/test/emqx_schema_tests.erl b/apps/emqx/test/emqx_schema_tests.erl new file mode 100644 index 000000000..e2825498d --- /dev/null +++ b/apps/emqx/test/emqx_schema_tests.erl @@ -0,0 +1,99 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_schema_tests). + +-include_lib("eunit/include/eunit.hrl"). + +ssl_opts_dtls_test() -> + Sc = emqx_schema:server_ssl_opts_schema(#{versions => dtls_all_available, + ciphers => dtls_all_available}, false), + Checked = validate(Sc, #{<<"versions">> => [<<"dtlsv1.2">>, <<"dtlsv1">>]}), + ?assertMatch(#{versions := ['dtlsv1.2', 'dtlsv1'], + ciphers := ["ECDHE-ECDSA-AES256-GCM-SHA384" | _] + }, Checked). + +ssl_opts_tls_1_3_test() -> + Sc = emqx_schema:server_ssl_opts_schema(#{}, false), + Checked = validate(Sc, #{<<"versions">> => [<<"tlsv1.3">>]}), + ?assertNot(maps:is_key(handshake_timeout, Checked)), + ?assertMatch(#{versions := ['tlsv1.3'], + ciphers := [_ | _] + }, Checked). + +ssl_opts_tls_for_ranch_test() -> + Sc = emqx_schema:server_ssl_opts_schema(#{}, true), + Checked = validate(Sc, #{<<"versions">> => [<<"tlsv1.3">>]}), + ?assertMatch(#{versions := ['tlsv1.3'], + ciphers := [_ | _], + handshake_timeout := _ + }, Checked). + +ssl_opts_cipher_array_test() -> + Sc = emqx_schema:server_ssl_opts_schema(#{}, false), + Checked = validate(Sc, #{<<"versions">> => [<<"tlsv1.3">>], + <<"ciphers">> => [<<"TLS_AES_256_GCM_SHA384">>, + <<"ECDHE-ECDSA-AES256-GCM-SHA384">>]}), + ?assertMatch(#{versions := ['tlsv1.3'], + ciphers := ["TLS_AES_256_GCM_SHA384", "ECDHE-ECDSA-AES256-GCM-SHA384"] + }, Checked). + +ssl_opts_cipher_comma_separated_string_test() -> + Sc = emqx_schema:server_ssl_opts_schema(#{}, false), + Checked = validate(Sc, #{<<"versions">> => [<<"tlsv1.3">>], + <<"ciphers">> => <<"TLS_AES_256_GCM_SHA384,ECDHE-ECDSA-AES256-GCM-SHA384">>}), + ?assertMatch(#{versions := ['tlsv1.3'], + ciphers := ["TLS_AES_256_GCM_SHA384", "ECDHE-ECDSA-AES256-GCM-SHA384"] + }, Checked). + +ssl_opts_tls_psk_test() -> + Sc = emqx_schema:server_ssl_opts_schema(#{}, false), + Checked = validate(Sc, #{<<"versions">> => [<<"tlsv1.2">>]}), + ?assertMatch(#{versions := ['tlsv1.2']}, Checked). + +bad_cipher_test() -> + Sc = emqx_schema:server_ssl_opts_schema(#{}, false), + Reason = {bad_ciphers, ["foo"]}, + ?assertThrow({_Sc, [{validation_error, #{reason := Reason}}]}, + validate(Sc, #{<<"versions">> => [<<"tlsv1.2">>], + <<"ciphers">> => [<<"foo">>]})), + ok. + +validate(Schema, Data0) -> + Sc = #{ roots => [ssl_opts] + , fields => #{ssl_opts => Schema} + }, + Data = Data0#{ cacertfile => <<"cacertfile">> + , certfile => <<"certfile">> + , keyfile => <<"keyfile">> + }, + #{ssl_opts := Checked} = + hocon_schema:check_plain(Sc, #{<<"ssl_opts">> => Data}, + #{atom_key => true}), + Checked. + +ciperhs_schema_test() -> + Sc = emqx_schema:ciphers_schema(undefined), + WSc = #{roots => [{ciphers, Sc}]}, + ?assertThrow({_, [{validation_error, _}]}, + hocon_schema:check_plain(WSc, #{<<"ciphers">> => <<"foo,bar">>})). + +bad_tls_version_test() -> + Sc = emqx_schema:server_ssl_opts_schema(#{}, false), + Reason = {unsupported_ssl_versions, [foo]}, + ?assertThrow({_Sc, [{validation_error, #{reason := Reason}}]}, + validate(Sc, #{<<"versions">> => [<<"foo">>]})), + ok. diff --git a/apps/emqx/test/emqx_sys_mon_SUITE.erl b/apps/emqx/test/emqx_sys_mon_SUITE.erl index 70f518ad5..53770f7e2 100644 --- a/apps/emqx/test/emqx_sys_mon_SUITE.erl +++ b/apps/emqx/test/emqx_sys_mon_SUITE.erl @@ -24,21 +24,22 @@ -define(SYSMON, emqx_sys_mon). +-define(FAKE_PORT, hd(erlang:ports())). +-define(FAKE_INFO, [{timeout, 100}, {in, foo}, {out, {?MODULE, bar, 1}}]). -define(INPUTINFO, [{self(), long_gc, - concat_str("long_gc warning: pid = ~p, info: ~p", self(), "hello"), "hello"}, + fmt("long_gc warning: pid = ~p", [self()]), ?FAKE_INFO}, {self(), long_schedule, - concat_str("long_schedule warning: pid = ~p, info: ~p", self(), "hello"), "hello"}, + fmt("long_schedule warning: pid = ~p", [self()]), ?FAKE_INFO}, {self(), large_heap, - concat_str("large_heap warning: pid = ~p, info: ~p", self(), "hello"), "hello"}, + fmt("large_heap warning: pid = ~p", [self()]), ?FAKE_INFO}, {self(), busy_port, - concat_str("busy_port warning: suspid = ~p, port = ~p", - self(), list_to_port("#Port<0.4>")), list_to_port("#Port<0.4>")}, + fmt("busy_port warning: suspid = ~p, port = ~p", + [self(), ?FAKE_PORT]), ?FAKE_PORT}, {self(), busy_dist_port, - concat_str("busy_dist_port warning: suspid = ~p, port = ~p", - self(), list_to_port("#Port<0.4>")),list_to_port("#Port<0.4>")}, - {list_to_port("#Port<0.4>"), long_schedule, - concat_str("long_schedule warning: port = ~p, info: ~p", - list_to_port("#Port<0.4>"), "hello"), "hello"} + fmt("busy_dist_port warning: suspid = ~p, port = ~p", + [self(), ?FAKE_PORT]), ?FAKE_PORT}, + {?FAKE_PORT, long_schedule, + fmt("long_schedule warning: port = ~p", [?FAKE_PORT]), ?FAKE_INFO} ]). all() -> emqx_ct:all(?MODULE). @@ -82,16 +83,16 @@ t_procinfo(_) -> ok = meck:new(emqx_vm, [passthrough, no_history]), ok = meck:expect(emqx_vm, get_process_info, fun(_) -> [] end), ok = meck:expect(emqx_vm, get_process_gc_info, fun(_) -> [] end), - ?assertEqual([], emqx_sys_mon:procinfo([])), - ok = meck:expect(emqx_vm, get_process_info, fun(_) -> ok end), + ?assertEqual([{pid, undefined}], emqx_sys_mon:procinfo(undefined)), + ok = meck:expect(emqx_vm, get_process_info, fun(_) -> [] end), ok = meck:expect(emqx_vm, get_process_gc_info, fun(_) -> undefined end), - ?assertEqual(undefined, emqx_sys_mon:procinfo([])), + ?assertEqual([{pid, self()}], emqx_sys_mon:procinfo(self())), ok = meck:unload(emqx_vm). t_sys_mon(_Config) -> lists:foreach( - fun({PidOrPort, SysMonName,ValidateInfo, InfoOrPort}) -> - validate_sys_mon_info(PidOrPort, SysMonName,ValidateInfo, InfoOrPort) + fun({PidOrPort, SysMonName, ValidateInfo, InfoOrPort}) -> + validate_sys_mon_info(PidOrPort, SysMonName, ValidateInfo, InfoOrPort) end, ?INPUTINFO). t_sys_mon2(_Config) -> @@ -101,7 +102,7 @@ t_sys_mon2(_Config) -> ?assertEqual(ok, gen_server:cast(?SYSMON, ignored)), gen_server:stop(?SYSMON). -validate_sys_mon_info(PidOrPort, SysMonName,ValidateInfo, InfoOrPort) -> +validate_sys_mon_info(PidOrPort, SysMonName, ValidateInfo, InfoOrPort) -> {ok, C} = emqtt:start_link([{host, "localhost"}]), {ok, _} = emqtt:connect(C), emqtt:subscribe(C, emqx_topic:systop(lists:concat(['sysmon/', SysMonName])), qos1), @@ -117,6 +118,4 @@ validate_sys_mon_info(PidOrPort, SysMonName,ValidateInfo, InfoOrPort) -> end, emqtt:stop(C). -concat_str(ValidateInfo, InfoOrPort, Info) -> - WarnInfo = io_lib:format(ValidateInfo, [InfoOrPort, Info]), - lists:flatten(WarnInfo). +fmt(Fmt, Args) -> lists:flatten(io_lib:format(Fmt, Args)). diff --git a/apps/emqx/test/emqx_topic_SUITE.erl b/apps/emqx/test/emqx_topic_SUITE.erl index 0cccb74bb..e8262a8ec 100644 --- a/apps/emqx/test/emqx_topic_SUITE.erl +++ b/apps/emqx/test/emqx_topic_SUITE.erl @@ -20,7 +20,6 @@ -compile(nowarn_export_all). -include_lib("eunit/include/eunit.hrl"). --include_lib("emqx_ct_helpers/include/emqx_ct.hrl"). -import(emqx_topic, [ wildcard/1 @@ -126,21 +125,21 @@ t_validate(_) -> true = validate({filter, <<"abc/#">>}), true = validate({filter, <<"x">>}), true = validate({name, <<"x//y">>}), - true = validate({filter, <<"sport/tennis/#">>}), - ok = ?catch_error(empty_topic, validate({name, <<>>})), - ok = ?catch_error(topic_name_error, validate({name, <<"abc/#">>})), - ok = ?catch_error(topic_too_long, validate({name, long_topic()})), - ok = ?catch_error('topic_invalid_#', validate({filter, <<"abc/#/1">>})), - ok = ?catch_error(topic_invalid_char, validate({filter, <<"abc/#xzy/+">>})), - ok = ?catch_error(topic_invalid_char, validate({filter, <<"abc/xzy/+9827">>})), - ok = ?catch_error(topic_invalid_char, validate({filter, <<"sport/tennis#">>})), - ok = ?catch_error('topic_invalid_#', validate({filter, <<"sport/tennis/#/ranking">>})). + true = validate({filter, <<"sport/tennis/#">>}), + ?assertError(empty_topic, validate({name, <<>>})), + ?assertError(topic_name_error, validate({name, <<"abc/#">>})), + ?assertError(topic_too_long, validate({name, long_topic()})), + ?assertError('topic_invalid_#', validate({filter, <<"abc/#/1">>})), + ?assertError(topic_invalid_char, validate({filter, <<"abc/#xzy/+">>})), + ?assertError(topic_invalid_char, validate({filter, <<"abc/xzy/+9827">>})), + ?assertError(topic_invalid_char, validate({filter, <<"sport/tennis#">>})), + ?assertError('topic_invalid_#', validate({filter, <<"sport/tennis/#/ranking">>})). t_sigle_level_validate(_) -> true = validate({filter, <<"+">>}), true = validate({filter, <<"+/tennis/#">>}), true = validate({filter, <<"sport/+/player1">>}), - ok = ?catch_error(topic_invalid_char, validate({filter, <<"sport+">>})). + ?assertError(topic_invalid_char, validate({filter, <<"sport+">>})). t_prepend(_) -> ?assertEqual(<<"ab">>, prepend(undefined, <<"ab">>)), @@ -192,14 +191,14 @@ long_topic() -> iolist_to_binary([[integer_to_list(I), "/"] || I <- lists:seq(0, 66666)]). t_parse(_) -> - ok = ?catch_error({invalid_topic_filter, <<"$queue/t">>}, - parse(<<"$queue/t">>, #{share => <<"g">>})), - ok = ?catch_error({invalid_topic_filter, <<"$share/g/t">>}, - parse(<<"$share/g/t">>, #{share => <<"g">>})), - ok = ?catch_error({invalid_topic_filter, <<"$share/t">>}, - parse(<<"$share/t">>)), - ok = ?catch_error({invalid_topic_filter, <<"$share/+/t">>}, - parse(<<"$share/+/t">>)), + ?assertError({invalid_topic_filter, <<"$queue/t">>}, + parse(<<"$queue/t">>, #{share => <<"g">>})), + ?assertError({invalid_topic_filter, <<"$share/g/t">>}, + parse(<<"$share/g/t">>, #{share => <<"g">>})), + ?assertError({invalid_topic_filter, <<"$share/t">>}, + parse(<<"$share/t">>)), + ?assertError({invalid_topic_filter, <<"$share/+/t">>}, + parse(<<"$share/+/t">>)), ?assertEqual({<<"a/b/+/#">>, #{}}, parse(<<"a/b/+/#">>)), ?assertEqual({<<"a/b/+/#">>, #{qos => 1}}, parse({<<"a/b/+/#">>, #{qos => 1}})), ?assertEqual({<<"topic">>, #{share => <<"$queue">>}}, parse(<<"$queue/topic">>)), diff --git a/apps/emqx/test/emqx_ws_connection_SUITE.erl b/apps/emqx/test/emqx_ws_connection_SUITE.erl index 767a7994e..b7484ba90 100644 --- a/apps/emqx/test/emqx_ws_connection_SUITE.erl +++ b/apps/emqx/test/emqx_ws_connection_SUITE.erl @@ -229,7 +229,7 @@ t_ws_check_origin(_) -> ?assertMatch({gun_upgrade, _}, start_ws_client(#{protocols => [<<"mqtt">>], headers => [{<<"origin">>, <<"http://localhost:18083">>}]})), - ?assertMatch({gun_response, {_, 500, _}}, + ?assertMatch({gun_response, {_, 403, _}}, start_ws_client(#{protocols => [<<"mqtt">>], headers => [{<<"origin">>, <<"http://localhost:18080">>}]})). diff --git a/apps/emqx/test/props/prop_emqx_json.erl b/apps/emqx/test/props/prop_emqx_json.erl index 819b029d2..23f656f64 100644 --- a/apps/emqx/test/props/prop_emqx_json.erl +++ b/apps/emqx/test/props/prop_emqx_json.erl @@ -135,7 +135,7 @@ json_basic() -> oneof([true, false, null, number(), json_string()]). latin_atom() -> - emqx_ct_proper_types:limited_latin_atom(). + emqx_proper_types:limited_latin_atom(). json_string() -> utf8(). diff --git a/apps/emqx/test/props/prop_emqx_psk.erl b/apps/emqx/test/props/prop_emqx_psk.erl index 106de3fda..1d2ce5a12 100644 --- a/apps/emqx/test/props/prop_emqx_psk.erl +++ b/apps/emqx/test/props/prop_emqx_psk.erl @@ -33,7 +33,7 @@ prop_lookup() -> ?ALL({ClientPSKID, UserState}, {client_pskid(), user_state()}, begin - case emqx_psk:lookup(psk, ClientPSKID, UserState) of + case emqx_tls_psk:lookup(psk, ClientPSKID, UserState) of {ok, _Result} -> true; error -> true; _Other -> false diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index 540cf86e3..827e08dab 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -91,7 +91,7 @@ enable => true})). -define(INSTANCE_EXAMPLE_2, maps:merge(?EXAMPLE_2, #{id => <<"password-based:http-server">>, - connect_timeout => 5000, + connect_timeout => "5s", enable_pipelining => true, headers => #{ <<"accept">> => <<"application/json">>, @@ -102,8 +102,8 @@ }, max_retries => 5, pool_size => 8, - request_timeout => 5000, - retry_interval => 1000, + request_timeout => "5s", + retry_interval => "1s", enable => true})). -define(INSTANCE_EXAMPLE_3, maps:merge(?EXAMPLE_3, #{id => <<"jwt">>, @@ -1259,9 +1259,9 @@ definitions() -> example => <<"SELECT password_hash FROM mqtt_user WHERE username = ${mqtt-username}">> }, query_timeout => #{ - type => integer, - description => <<"Query timeout, Unit: Milliseconds">>, - default => 5000 + type => string, + description => <<"Query timeout">>, + default => "5s" } } }, @@ -1528,16 +1528,16 @@ definitions() -> type => object }, connect_timeout => #{ - type => integer, - default => 5000 + type => string, + default => <<"5s">> }, max_retries => #{ type => integer, default => 5 }, retry_interval => #{ - type => integer, - default => 1000 + type => string, + default => <<"1s">> }, request_timout => #{ type => integer, @@ -1970,8 +1970,9 @@ find_config(AuthenticatorID, AuthenticatorsConfig) -> end. fill_defaults(Config) -> - #{<<"authentication">> := CheckedConfig} = hocon_schema:check_plain( - ?AUTHN, #{<<"authentication">> => Config}, #{no_conversion => true}), + #{<<"authentication">> := CheckedConfig} = + hocon_schema:check_plain(?AUTHN, #{<<"authentication">> => Config}, + #{only_fill_defaults => true}), CheckedConfig. convert_certs(#{<<"ssl">> := SSLOpts} = Config) -> @@ -2070,4 +2071,4 @@ to_list(L) when is_list(L) -> to_atom(B) when is_binary(B) -> binary_to_atom(B); to_atom(A) when is_atom(A) -> - A. \ No newline at end of file + A. diff --git a/apps/emqx_authn/src/emqx_authn_utils.erl b/apps/emqx_authn/src/emqx_authn_utils.erl index c0ba8a549..4784c91c7 100644 --- a/apps/emqx_authn/src/emqx_authn_utils.erl +++ b/apps/emqx_authn/src/emqx_authn_utils.erl @@ -18,6 +18,8 @@ -export([ replace_placeholders/2 , replace_placeholder/2 + , check_password/3 + , is_superuser/1 , hash/4 , gen_salt/0 , bin/1 @@ -55,6 +57,28 @@ replace_placeholder(<<"${cert-common-name}">>, Credential) -> replace_placeholder(Constant, _) -> Constant. +check_password(undefined, _Selected, _State) -> + {error, bad_username_or_password}; +check_password(Password, + #{<<"password_hash">> := Hash}, + #{password_hash_algorithm := bcrypt}) -> + case {ok, Hash} =:= bcrypt:hashpw(Password, Hash) of + true -> ok; + false -> {error, bad_username_or_password} + end; +check_password(Password, + #{<<"password_hash">> := Hash} = Selected, + #{password_hash_algorithm := Algorithm, + salt_position := SaltPosition}) -> + Salt = maps:get(<<"salt">>, Selected, <<>>), + case Hash =:= hash(Algorithm, Password, Salt, SaltPosition) of + true -> ok; + false -> {error, bad_username_or_password} + end. + +is_superuser(Selected) -> + #{is_superuser => maps:get(<<"is_superuser">>, Selected, false)}. + hash(Algorithm, Password, Salt, prefix) -> emqx_passwd:hash(Algorithm, <>); hash(Algorithm, Password, Salt, suffix) -> @@ -75,4 +99,4 @@ bin(X) -> X. convert_to_sql_param(undefined) -> null; convert_to_sql_param(V) -> - bin(V). \ No newline at end of file + bin(V). diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl index 2fa29d2df..22f701220 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl @@ -100,8 +100,8 @@ body(type) -> map(); body(validator) -> [fun check_body/1]; body(_) -> undefined. -request_timeout(type) -> non_neg_integer(); -request_timeout(default) -> 5000; +request_timeout(type) -> emqx_schema:duration_ms(); +request_timeout(default) -> "5s"; request_timeout(_) -> undefined. %%------------------------------------------------------------------------------ @@ -156,26 +156,23 @@ authenticate(#{auth_method := _}, _) -> authenticate(Credential, #{'_unique' := Unique, method := Method, request_timeout := RequestTimeout} = State) -> - try - Request = generate_request(Credential, State), - case emqx_resource:query(Unique, {Method, Request, RequestTimeout}) of - {ok, 204, _Headers} -> {ok, #{is_superuser => false}}; - {ok, 200, Headers, Body} -> - ContentType = proplists:get_value(<<"content-type">>, Headers, <<"application/json">>), - case safely_parse_body(ContentType, Body) of - {ok, NBody} -> - %% TODO: Return by user property - {ok, #{is_superuser => maps:get(<<"is_superuser">>, NBody, false), - user_property => NBody}}; - {error, _Reason} -> - {ok, #{is_superuser => false}} - end; - {error, _Reason} -> - ignore - end - catch - error:Reason -> - ?LOG(warning, "The following error occurred in '~s' during authentication: ~p", [Unique, Reason]), + Request = generate_request(Credential, State), + case emqx_resource:query(Unique, {Method, Request, RequestTimeout}) of + {ok, 204, _Headers} -> {ok, #{is_superuser => false}}; + {ok, 200, Headers, Body} -> + ContentType = proplists:get_value(<<"content-type">>, Headers, <<"application/json">>), + case safely_parse_body(ContentType, Body) of + {ok, NBody} -> + %% TODO: Return by user property + {ok, #{is_superuser => maps:get(<<"is_superuser">>, NBody, false), + user_property => NBody}}; + {error, _Reason} -> + {ok, #{is_superuser => false}} + end; + {error, Reason} -> + ?SLOG(error, #{msg => "http_server_query_failed", + resource => Unique, + reason => Reason}), ignore end. @@ -194,9 +191,9 @@ check_url(URL) -> end. check_body(Body) -> - lists:any(fun({_, V}) -> - not is_binary(V) - end, maps:to_list(Body)). + maps:fold(fun(_K, _V, false) -> false; + (_K, V, true) -> is_binary(V) + end, true, Body). default_headers() -> maps:put(<<"content-type">>, diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_connector.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_connector.erl index d6e977be6..89bc565c6 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_connector.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_connector.erl @@ -94,7 +94,9 @@ handle_info({http, {RequestID, Result}}, State1 = State0#{request_id := undefined}, case Result of {error, Reason} -> - ?LOG(error, "Failed to request jwks endpoint(~s): ~p", [Endpoint, Reason]), + ?SLOG(warning, #{msg => "failed_to_request_jwks_endpoint", + endpoint => Endpoint, + reason => Reason}), State1; {_StatusLine, _Headers, Body} -> try @@ -102,7 +104,9 @@ handle_info({http, {RequestID, Result}}, {_, JWKs} = JWKS#jose_jwk.keys, State1#{jwks := JWKs} catch _:_ -> - ?LOG(error, "Invalid jwks returned from jwks endpoint(~s): ~p~n", [Endpoint, Body]), + ?SLOG(warning, #{msg => "invalid_jwks_returned", + endpoint => Endpoint, + body => Body}), State1 end end; @@ -136,11 +140,16 @@ handle_options(#{endpoint := Endpoint, refresh_jwks(#{endpoint := Endpoint, ssl_opts := SSLOpts} = State) -> - HTTPOpts = [{timeout, 5000}, {connect_timeout, 5000}, {ssl, SSLOpts}], + HTTPOpts = [ {timeout, 5000} + , {connect_timeout, 5000} + , {ssl, SSLOpts} + ], NState = case httpc:request(get, {Endpoint, [{"Accept", "application/json"}]}, HTTPOpts, [{body_format, binary}, {sync, false}, {receiver, self()}]) of {error, Reason} -> - ?LOG(error, "Failed to request jwks endpoint(~s): ~p", [Endpoint, Reason]), + ?SLOG(warning, #{msg => "failed_to_request_jwks_endpoint", + endpoint => Endpoint, + reason => Reason}), State; {ok, RequestID} -> State#{request_id := RequestID} diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl index 5ad148009..e2459ffe8 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl @@ -141,29 +141,27 @@ authenticate(#{password := Password} = Credential, , selector := Selector0 , '_unique' := Unique } = State) -> - try - Selector1 = replace_placeholders(Selector0, Credential), - Selector2 = normalize_selector(Selector1), - case emqx_resource:query(Unique, {find_one, Collection, Selector2, #{}}) of - undefined -> ignore; - {error, Reason} -> - ?LOG(error, "['~s'] Query failed: ~p", [Unique, Reason]), - ignore; - Doc -> - case check_password(Password, Doc, State) of - ok -> - {ok, #{is_superuser => is_superuser(Doc, State)}}; - {error, {cannot_find_password_hash_field, PasswordHashField}} -> - ?LOG(error, "['~s'] Can't find password hash field: ~s", [Unique, PasswordHashField]), - {error, bad_username_or_password}; - {error, Reason} -> - {error, Reason} - end - end - catch - error:Error -> - ?LOG(warning, "The following error occurred in '~s' during authentication: ~p", [Unique, Error]), - ignore + Selector1 = replace_placeholders(Selector0, Credential), + Selector2 = normalize_selector(Selector1), + case emqx_resource:query(Unique, {find_one, Collection, Selector2, #{}}) of + undefined -> ignore; + {error, Reason} -> + ?SLOG(error, #{msg => "mongodb_query_failed", + resource => Unique, + reason => Reason}), + ignore; + Doc -> + case check_password(Password, Doc, State) of + ok -> + {ok, #{is_superuser => is_superuser(Doc, State)}}; + {error, {cannot_find_password_hash_field, PasswordHashField}} -> + ?SLOG(error, #{msg => "cannot_find_password_hash_field", + resource => Unique, + password_hash_field => PasswordHashField}), + ignore; + {error, Reason} -> + {error, Reason} + end end. destroy(#{'_unique' := Unique}) -> diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl index 87c61da1e..43a9bd252 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl @@ -65,8 +65,8 @@ salt_position(_) -> undefined. query(type) -> string(); query(_) -> undefined. -query_timeout(type) -> integer(); -query_timeout(default) -> 5000; +query_timeout(type) -> emqx_schema:duration_ms(); +query_timeout(default) -> "5s"; query_timeout(_) -> undefined. %%------------------------------------------------------------------------------ @@ -114,24 +114,21 @@ authenticate(#{password := Password} = Credential, query := Query, query_timeout := Timeout, '_unique' := Unique} = State) -> - try - Params = emqx_authn_utils:replace_placeholders(PlaceHolders, Credential), - case emqx_resource:query(Unique, {sql, Query, Params, Timeout}) of - {ok, _Columns, []} -> ignore; - {ok, Columns, Rows} -> - Selected = maps:from_list(lists:zip(Columns, Rows)), - case check_password(Password, Selected, State) of - ok -> - {ok, #{is_superuser => maps:get(<<"is_superuser">>, Selected, false)}}; - {error, Reason} -> - {error, Reason} - end; - {error, _Reason} -> - ignore - end - catch - error:Error -> - ?LOG(warning, "The following error occurred in '~s' during authentication: ~p", [Unique, Error]), + Params = emqx_authn_utils:replace_placeholders(PlaceHolders, Credential), + case emqx_resource:query(Unique, {sql, Query, Params, Timeout}) of + {ok, _Columns, []} -> ignore; + {ok, Columns, Rows} -> + Selected = maps:from_list(lists:zip(Columns, Rows)), + case emqx_authn_utils:check_password(Password, Selected, State) of + ok -> + {ok, emqx_authn_utils:is_superuser(Selected)}; + {error, Reason} -> + {error, Reason} + end; + {error, Reason} -> + ?SLOG(error, #{msg => "mysql_query_failed", + resource => Unique, + reason => Reason}), ignore end. @@ -143,25 +140,6 @@ destroy(#{'_unique' := Unique}) -> %% Internal functions %%------------------------------------------------------------------------------ -check_password(undefined, _Selected, _State) -> - {error, bad_username_or_password}; -check_password(Password, - #{<<"password_hash">> := Hash}, - #{password_hash_algorithm := bcrypt}) -> - case {ok, Hash} =:= bcrypt:hashpw(Password, Hash) of - true -> ok; - false -> {error, bad_username_or_password} - end; -check_password(Password, - #{<<"password_hash">> := Hash} = Selected, - #{password_hash_algorithm := Algorithm, - salt_position := SaltPosition}) -> - Salt = maps:get(<<"salt">>, Selected, <<>>), - case Hash =:= emqx_authn_utils:hash(Algorithm, Password, Salt, SaltPosition) of - true -> ok; - false -> {error, bad_username_or_password} - end. - %% TODO: Support prepare parse_query(Query) -> case re:run(Query, ?RE_PLACEHOLDER, [global, {capture, all, binary}]) of diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl index 940c50519..99b83844b 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl @@ -103,25 +103,22 @@ authenticate(#{password := Password} = Credential, #{query := Query, placeholders := PlaceHolders, '_unique' := Unique} = State) -> - try - Params = emqx_authn_utils:replace_placeholders(PlaceHolders, Credential), - case emqx_resource:query(Unique, {sql, Query, Params}) of - {ok, _Columns, []} -> ignore; - {ok, Columns, Rows} -> - NColumns = [Name || #column{name = Name} <- Columns], - Selected = maps:from_list(lists:zip(NColumns, Rows)), - case check_password(Password, Selected, State) of - ok -> - {ok, #{is_superuser => maps:get(<<"is_superuser">>, Selected, false)}}; - {error, Reason} -> - {error, Reason} - end; - {error, _Reason} -> - ignore - end - catch - error:Error -> - ?LOG(warning, "The following error occurred in '~s' during authentication: ~p", [Unique, Error]), + Params = emqx_authn_utils:replace_placeholders(PlaceHolders, Credential), + case emqx_resource:query(Unique, {sql, Query, Params}) of + {ok, _Columns, []} -> ignore; + {ok, Columns, Rows} -> + NColumns = [Name || #column{name = Name} <- Columns], + Selected = maps:from_list(lists:zip(NColumns, Rows)), + case emqx_authn_utils:check_password(Password, Selected, State) of + ok -> + {ok, emqx_authn_utils:is_superuser(Selected)}; + {error, Reason} -> + {error, Reason} + end; + {error, Reason} -> + ?SLOG(error, #{msg => "postgresql_query_failed", + resource => Unique, + reason => Reason}), ignore end. @@ -133,30 +130,11 @@ destroy(#{'_unique' := Unique}) -> %% Internal functions %%------------------------------------------------------------------------------ -check_password(undefined, _Selected, _State) -> - {error, bad_username_or_password}; -check_password(Password, - #{<<"password_hash">> := Hash}, - #{password_hash_algorithm := bcrypt}) -> - case {ok, Hash} =:= bcrypt:hashpw(Password, Hash) of - true -> ok; - false -> {error, bad_username_or_password} - end; -check_password(Password, - #{<<"password_hash">> := Hash} = Selected, - #{password_hash_algorithm := Algorithm, - salt_position := SaltPosition}) -> - Salt = maps:get(<<"salt">>, Selected, <<>>), - case Hash =:= emqx_authn_utils:hash(Algorithm, Password, Salt, SaltPosition) of - true -> ok; - false -> {error, bad_username_or_password} - end. - %% TODO: Support prepare parse_query(Query) -> case re:run(Query, ?RE_PLACEHOLDER, [global, {capture, all, binary}]) of {match, Captured} -> - PlaceHolders = [PlaceHolder || PlaceHolder <- Captured], + PlaceHolders = [PlaceHolder || [PlaceHolder] <- Captured], Replacements = ["$" ++ integer_to_list(I) || I <- lists:seq(1, length(Captured))], NQuery = lists:foldl(fun({PlaceHolder, Replacement}, Query0) -> re:replace(Query0, <<"'\\", PlaceHolder/binary, "'">>, Replacement, [{return, binary}]) diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl index 5926740a8..9b8dbefbf 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl @@ -127,24 +127,26 @@ authenticate(#{password := Password} = Credential, #{ query := {Command, Key, Fields} , '_unique' := Unique } = State) -> - try - NKey = binary_to_list(iolist_to_binary(replace_placeholders(Key, Credential))), - case emqx_resource:query(Unique, {cmd, [Command, NKey | Fields]}) of - {ok, Values} -> - Selected = merge(Fields, Values), - case check_password(Password, Selected, State) of - ok -> - {ok, #{is_superuser => maps:get("is_superuser", Selected, false)}}; - {error, Reason} -> - {error, Reason} - end; - {error, Reason} -> - ?LOG(error, "['~s'] Query failed: ~p", [Unique, Reason]), - ignore - end - catch - error:{cannot_get_variable, Placeholder} -> - ?LOG(warning, "The following error occurred in '~s' during authentication: ~p", [Unique, {cannot_get_variable, Placeholder}]), + NKey = binary_to_list(iolist_to_binary(replace_placeholders(Key, Credential))), + case emqx_resource:query(Unique, {cmd, [Command, NKey | Fields]}) of + {ok, Values} -> + case merge(Fields, Values) of + #{<<"password_hash">> := _} = Selected -> + case emqx_authn_utils:check_password(Password, Selected, State) of + ok -> + {ok, emqx_authn_utils:is_superuser(Selected)}; + {error, Reason} -> + {error, Reason} + end; + _ -> + ?SLOG(error, #{msg => "cannot_find_password_hash_field", + resource => Unique}), + ignore + end; + {error, Reason} -> + ?SLOG(error, #{msg => "redis_query_failed", + resource => Unique, + reason => Reason}), ignore end. @@ -209,27 +211,5 @@ merge(Fields, Value) when not is_list(Value) -> merge(Fields, [Value]); merge(Fields, Values) -> maps:from_list( - lists:filter(fun({_, V}) -> - V =/= undefined - end, lists:zip(Fields, Values))). - -check_password(undefined, _Selected, _State) -> - {error, bad_username_or_password}; -check_password(Password, - #{"password_hash" := PasswordHash}, - #{password_hash_algorithm := bcrypt}) -> - case {ok, PasswordHash} =:= bcrypt:hashpw(Password, PasswordHash) of - true -> ok; - false -> {error, bad_username_or_password} - end; -check_password(Password, - #{"password_hash" := PasswordHash} = Selected, - #{password_hash_algorithm := Algorithm, - salt_position := SaltPosition}) -> - Salt = maps:get("salt", Selected, <<>>), - case PasswordHash =:= emqx_authn_utils:hash(Algorithm, Password, Salt, SaltPosition) of - true -> ok; - false -> {error, bad_username_or_password} - end; -check_password(_Password, _Selected, _State) -> - ignore. + [{list_to_binary(K), V} + || {K, V} <- lists:zip(Fields, Values), V =/= undefined]). diff --git a/apps/emqx_authz/etc/acl.conf b/apps/emqx_authz/etc/acl.conf index 2948f2af7..a1cfd41d3 100644 --- a/apps/emqx_authz/etc/acl.conf +++ b/apps/emqx_authz/etc/acl.conf @@ -3,9 +3,9 @@ %% %% -type(ipaddrs() :: {ipaddrs, string()}). %% -%% -type(username() :: {username, regex()}). +%% -type(username() :: {user | username, string()} | {user | username, {re, regex()}}). %% -%% -type(clientid() :: {clientid, regex()}). +%% -type(clientid() :: {client | clientid, string()} | {client | clientid, {re, regex()}}). %% %% -type(who() :: ipaddr() | ipaddrs() |username() | clientid() | %% {'and', [ipaddr() | ipaddrs()| username() | clientid()]} | @@ -20,7 +20,7 @@ %% %% -type(permission() :: allow | deny). %% -%% -type(rule() :: {permission(), who(), access(), topics()}). +%% -type(rule() :: {permission(), who(), access(), topics()} | {permission(), all}). %%-------------------------------------------------------------------- {allow, {username, "^dashboard?"}, subscribe, ["$SYS/#"]}. diff --git a/apps/emqx_authz/etc/emqx_authz.conf b/apps/emqx_authz/etc/emqx_authz.conf index 096f3d314..3469aad3a 100644 --- a/apps/emqx_authz/etc/emqx_authz.conf +++ b/apps/emqx_authz/etc/emqx_authz.conf @@ -1,63 +1,67 @@ authorization { - # sources = [ - # # { - # # type: http - # # url: "https://emqx.com" - # # headers: { - # # Accept: "application/json" - # # Content-Type: "application/json" - # # } - # # }, - # # { - # # type: mysql - # # server: "127.0.0.1:3306" - # # database: mqtt - # # pool_size: 1 - # # username: root - # # password: public - # # auto_reconnect: true - # # ssl: { - # # enable: true - # # cacertfile: "{{ platform_etc_dir }}/certs/cacert.pem" - # # certfile: "{{ platform_etc_dir }}/certs/client-cert.pem" - # # keyfile: "{{ platform_etc_dir }}/certs/client-key.pem" - # # } - # # query: "select ipaddress, username, clientid, action, permission, topic from mqtt_authz where ipaddr = '%a' or username = '%u' or clientid = '%c'" - # # }, - # # { - # # type: postgresql - # # server: "127.0.0.1:5432" - # # database: mqtt - # # pool_size: 1 - # # username: root - # # password: public - # # auto_reconnect: true - # # ssl: {enable: false} - # # query: "select ipaddress, username, clientid, action, permission, topic from mqtt_authz where ipaddr = '%a' or username = '%u' or username = '$all' or clientid = '%c'" - # # }, - # # { - # # type: redis - # # server: "127.0.0.1:6379" - # # database: 0 - # # pool_size: 1 - # # password: public - # # auto_reconnect: true - # # ssl: {enable: false} - # # cmd: "HGETALL mqtt_authz:%u" - # # }, - # # { - # # type: mongodb - # # mongo_type: single - # # server: "127.0.0.1:27017" - # # pool_size: 1 - # # database: mqtt - # # ssl: {enable: false} - # # collection: mqtt_authz - # # selector: { "$or": [ { "username": "%u" }, { "clientid": "%c" } ] } - # # }, - # { - # type: file - # path: "{{ platform_etc_dir }}/acl.conf" - # } - # ] + sources = [ + # { + # type: http + # url: "https://emqx.com" + # headers: { + # Accept: "application/json" + # Content-Type: "application/json" + # } + # }, + # { + # type: mysql + # server: "127.0.0.1:3306" + # database: mqtt + # pool_size: 1 + # username: root + # password: public + # auto_reconnect: true + # ssl: { + # enable: true + # cacertfile: "{{ platform_etc_dir }}/certs/cacert.pem" + # certfile: "{{ platform_etc_dir }}/certs/client-cert.pem" + # keyfile: "{{ platform_etc_dir }}/certs/client-key.pem" + # } + # query: "select ipaddress, username, clientid, action, permission, topic from mqtt_authz where ipaddr = '%a' or username = '%u' or clientid = '%c'" + # }, + # { + # type: postgresql + # server: "127.0.0.1:5432" + # database: mqtt + # pool_size: 1 + # username: root + # password: public + # auto_reconnect: true + # ssl: {enable: false} + # query: "select ipaddress, username, clientid, action, permission, topic from mqtt_authz where ipaddr = '%a' or username = '%u' or username = '$all' or clientid = '%c'" + # }, + # { + # type: redis + # server: "127.0.0.1:6379" + # database: 0 + # pool_size: 1 + # password: public + # auto_reconnect: true + # ssl: {enable: false} + # cmd: "HGETALL mqtt_authz:%u" + # }, + # { + # type: mongodb + # mongo_type: single + # server: "127.0.0.1:27017" + # pool_size: 1 + # database: mqtt + # ssl: {enable: false} + # collection: mqtt_authz + # selector: { "$or": [ { "username": "%u" }, { "clientid": "%c" } ] } + # }, + { + type: built-in-database + }, + { + type: file + # file is loaded into cache + path: "{{ platform_etc_dir }}/acl.conf" + } + ] } diff --git a/apps/emqx_authz/include/emqx_authz.hrl b/apps/emqx_authz/include/emqx_authz.hrl index c83dfde0d..d86f08888 100644 --- a/apps/emqx_authz/include/emqx_authz.hrl +++ b/apps/emqx_authz/include/emqx_authz.hrl @@ -29,12 +29,37 @@ (A =:= all) orelse (A =:= <<"all">>) )). +-define(ACL_SHARDED, emqx_acl_sharded). + +-define(ACL_TABLE, emqx_acl). + +%% To save some space, use an integer for label, 0 for 'all', {1, Username} and {2, ClientId}. +-define(ACL_TABLE_ALL, 0). +-define(ACL_TABLE_USERNAME, 1). +-define(ACL_TABLE_CLIENTID, 2). + +-record(emqx_acl, { + who :: ?ACL_TABLE_ALL| {?ACL_TABLE_USERNAME, binary()} | {?ACL_TABLE_CLIENTID, binary()}, + rules :: [ {permission(), action(), emqx_topic:topic()} ] + }). + -record(authz_metrics, { allow = 'client.authorize.allow', deny = 'client.authorize.deny', ignore = 'client.authorize.ignore' }). +-define(CMD_REPLCAE, replace). +-define(CMD_DELETE, delete). +-define(CMD_PREPEND, prepend). +-define(CMD_APPEND, append). +-define(CMD_MOVE, move). + +-define(CMD_MOVE_TOP, <<"top">>). +-define(CMD_MOVE_BOTTOM, <<"bottom">>). +-define(CMD_MOVE_BEFORE(Before), {<<"before">>, Before}). +-define(CMD_MOVE_AFTER(After), {<<"after">>, After}). + -define(METRICS(Type), tl(tuple_to_list(#Type{}))). -define(METRICS(Type, K), #Type{}#Type.K). diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl index e7ccbe5b0..d8ab36b32 100644 --- a/apps/emqx_authz/src/emqx_authz.erl +++ b/apps/emqx_authz/src/emqx_authz.erl @@ -39,7 +39,6 @@ -export([post_config_update/4, pre_config_update/2]). -define(CONF_KEY_PATH, [authorization, sources]). --define(SOURCE_TYPES, [file, http, mongodb, mysql, postgresql, redis]). -spec(register_metrics() -> ok). register_metrics() -> @@ -50,228 +49,153 @@ init() -> emqx_config_handler:add_handler(?CONF_KEY_PATH, ?MODULE), Sources = emqx:get_config(?CONF_KEY_PATH, []), ok = check_dup_types(Sources), - NSources = [init_source(Source) || Source <- Sources], + NSources = init_sources(Sources), ok = emqx_hooks:add('client.authorize', {?MODULE, authorize, [NSources]}, -1). lookup() -> {_M, _F, [A]}= find_action_in_hooks(), A. + lookup(Type) -> - try find_source_by_type(atom(Type), lookup()) of - {_, Source} -> Source - catch - error:Reason -> {error, Reason} - end. + {Source, _Front, _Rear} = take(Type), + Source. move(Type, Cmd) -> move(Type, Cmd, #{}). move(Type, #{<<"before">> := Before}, Opts) -> - emqx:update_config(?CONF_KEY_PATH, {move, atom(Type), #{<<"before">> => atom(Before)}}, Opts); + emqx:update_config(?CONF_KEY_PATH, {?CMD_MOVE, type(Type), ?CMD_MOVE_BEFORE(type(Before))}, Opts); move(Type, #{<<"after">> := After}, Opts) -> - emqx:update_config(?CONF_KEY_PATH, {move, atom(Type), #{<<"after">> => atom(After)}}, Opts); + emqx:update_config(?CONF_KEY_PATH, {?CMD_MOVE, type(Type), ?CMD_MOVE_AFTER(type(After))}, Opts); move(Type, Position, Opts) -> - emqx:update_config(?CONF_KEY_PATH, {move, atom(Type), Position}, Opts). + emqx:update_config(?CONF_KEY_PATH, {?CMD_MOVE, type(Type), Position}, Opts). update(Cmd, Sources) -> update(Cmd, Sources, #{}). -update({replace_once, Type}, Sources, Opts) -> - emqx:update_config(?CONF_KEY_PATH, {{replace_once, atom(Type)}, Sources}, Opts); -update({delete_once, Type}, Sources, Opts) -> - emqx:update_config(?CONF_KEY_PATH, {{delete_once, atom(Type)}, Sources}, Opts); +update({replace, Type}, Sources, Opts) -> + emqx:update_config(?CONF_KEY_PATH, {{replace, type(Type)}, Sources}, Opts); +update({delete, Type}, Sources, Opts) -> + emqx:update_config(?CONF_KEY_PATH, {{delete, type(Type)}, Sources}, Opts); update(Cmd, Sources, Opts) -> emqx:update_config(?CONF_KEY_PATH, {Cmd, Sources}, Opts). -pre_config_update({move, Type, <<"top">>}, Conf) when is_list(Conf) -> - {Index, _} = find_source_by_type(Type), - {List1, List2} = lists:split(Index, Conf), - NConf = [lists:nth(Index, Conf)] ++ lists:droplast(List1) ++ List2, - case check_dup_types(NConf) of - ok -> {ok, NConf}; - Error -> Error - end; - -pre_config_update({move, Type, <<"bottom">>}, Conf) when is_list(Conf) -> - {Index, _} = find_source_by_type(Type), - {List1, List2} = lists:split(Index, Conf), - NConf = lists:droplast(List1) ++ List2 ++ [lists:nth(Index, Conf)], - case check_dup_types(NConf) of - ok -> {ok, NConf}; - Error -> Error - end; - -pre_config_update({move, Type, #{<<"before">> := Before}}, Conf) when is_list(Conf) -> - {Index1, _} = find_source_by_type(Type), - Conf1 = lists:nth(Index1, Conf), - {Index2, _} = find_source_by_type(Before), - Conf2 = lists:nth(Index2, Conf), - - {List1, List2} = lists:split(Index2, Conf), - NConf = lists:delete(Conf1, lists:droplast(List1)) - ++ [Conf1] ++ [Conf2] - ++ lists:delete(Conf1, List2), - case check_dup_types(NConf) of - ok -> {ok, NConf}; - Error -> Error - end; - -pre_config_update({move, Type, #{<<"after">> := After}}, Conf) when is_list(Conf) -> - {Index1, _} = find_source_by_type(Type), - Conf1 = lists:nth(Index1, Conf), - {Index2, _} = find_source_by_type(After), - - {List1, List2} = lists:split(Index2, Conf), - NConf = lists:delete(Conf1, List1) - ++ [Conf1] - ++ lists:delete(Conf1, List2), - case check_dup_types(NConf) of - ok -> {ok, NConf}; - Error -> Error - end; - -pre_config_update({head, Sources}, Conf) when is_list(Sources), is_list(Conf) -> +do_update({?CMD_MOVE, Type, ?CMD_MOVE_TOP}, Conf) when is_list(Conf) -> + {Source, Front, Rear} = take(Type, Conf), + [Source | Front] ++ Rear; +do_update({?CMD_MOVE, Type, ?CMD_MOVE_BOTTOM}, Conf) when is_list(Conf) -> + {Source, Front, Rear} = take(Type, Conf), + Front ++ Rear ++ [Source]; +do_update({?CMD_MOVE, Type, ?CMD_MOVE_BEFORE(Before)}, Conf) when is_list(Conf) -> + {S1, Front1, Rear1} = take(Type, Conf), + {S2, Front2, Rear2} = take(Before, Front1 ++ Rear1), + Front2 ++ [S1, S2] ++ Rear2; +do_update({?CMD_MOVE, Type, ?CMD_MOVE_AFTER(After)}, Conf) when is_list(Conf) -> + {S1, Front1, Rear1} = take(Type, Conf), + {S2, Front2, Rear2} = take(After, Front1 ++ Rear1), + Front2 ++ [S2, S1] ++ Rear2; +do_update({?CMD_PREPEND, Sources}, Conf) when is_list(Sources), is_list(Conf) -> NConf = Sources ++ Conf, - case check_dup_types(NConf) of - ok -> {ok, Sources ++ Conf}; - Error -> Error - end; -pre_config_update({tail, Sources}, Conf) when is_list(Sources), is_list(Conf) -> + ok = check_dup_types(NConf), + NConf; +do_update({?CMD_APPEND, Sources}, Conf) when is_list(Sources), is_list(Conf) -> NConf = Conf ++ Sources, - case check_dup_types(NConf) of - ok -> {ok, Conf ++ Sources}; - Error -> Error - end; -pre_config_update({{replace_once, Type}, Source}, Conf) when is_map(Source), is_list(Conf) -> - {Index, _} = find_source_by_type(Type), - {List1, List2} = lists:split(Index, Conf), - NConf = lists:droplast(List1) ++ [Source] ++ List2, - case check_dup_types(NConf) of - ok -> {ok, NConf}; - Error -> Error - end; -pre_config_update({{delete_once, Type}, _Source}, Conf) when is_list(Conf) -> - {Index, _} = find_source_by_type(Type), - {List1, List2} = lists:split(Index, Conf), - NConf = lists:droplast(List1) ++ List2, - case check_dup_types(NConf) of - ok -> {ok, NConf}; - Error -> Error - end; -pre_config_update({_, Sources}, _Conf) when is_list(Sources)-> + ok = check_dup_types(NConf), + NConf; +do_update({{replace, Type}, Source}, Conf) when is_map(Source), is_list(Conf) -> + {_Old, Front, Rear} = take(Type, Conf), + NConf = Front ++ [Source | Rear], + ok = check_dup_types(NConf), + NConf; +do_update({{delete, Type}, _Source}, Conf) when is_list(Conf) -> + {_Old, Front, Rear} = take(Type, Conf), + NConf = Front ++ Rear, + NConf; +do_update({_, Sources}, _Conf) when is_list(Sources)-> %% overwrite the entire config! - {ok, Sources}. + Sources. + +pre_config_update(Cmd, Conf) -> + {ok, do_update(Cmd, Conf)}. + post_config_update(_, undefined, _Conf, _AppEnvs) -> ok; -post_config_update({move, Type, <<"top">>}, _NewSources, _OldSources, _AppEnvs) -> - InitedSources = lookup(), - {Index, Source} = find_source_by_type(Type, InitedSources), - {Sources1, Sources2 } = lists:split(Index, InitedSources), - Sources3 = [Source] ++ lists:droplast(Sources1) ++ Sources2, - ok = emqx_hooks:put('client.authorize', {?MODULE, authorize, [Sources3]}, -1), - ok = emqx_authz_cache:drain_cache(); -post_config_update({move, Type, <<"bottom">>}, _NewSources, _OldSources, _AppEnvs) -> - InitedSources = lookup(), - {Index, Source} = find_source_by_type(Type, InitedSources), - {Sources1, Sources2 } = lists:split(Index, InitedSources), - Sources3 = lists:droplast(Sources1) ++ Sources2 ++ [Source], - ok = emqx_hooks:put('client.authorize', {?MODULE, authorize, [Sources3]}, -1), - ok = emqx_authz_cache:drain_cache(); -post_config_update({move, Type, #{<<"before">> := Before}}, _NewSources, _OldSources, _AppEnvs) -> - InitedSources = lookup(), - {_, Source0} = find_source_by_type(Type, InitedSources), - {Index, Source1} = find_source_by_type(Before, InitedSources), - {Sources1, Sources2} = lists:split(Index, InitedSources), - Sources3 = lists:delete(Source0, lists:droplast(Sources1)) - ++ [Source0] ++ [Source1] - ++ lists:delete(Source0, Sources2), - ok = emqx_hooks:put('client.authorize', {?MODULE, authorize, [Sources3]}, -1), - ok = emqx_authz_cache:drain_cache(); - -post_config_update({move, Type, #{<<"after">> := After}}, _NewSources, _OldSources, _AppEnvs) -> - InitedSources = lookup(), - {_, Source} = find_source_by_type(Type, InitedSources), - {Index, _} = find_source_by_type(After, InitedSources), - {Sources1, Sources2} = lists:split(Index, InitedSources), - Sources3 = lists:delete(Source, Sources1) - ++ [Source] - ++ lists:delete(Source, Sources2), - ok = emqx_hooks:put('client.authorize', {?MODULE, authorize, [Sources3]}, -1), - ok = emqx_authz_cache:drain_cache(); - -post_config_update({head, Sources}, _NewSources, _OldConf, _AppEnvs) -> - InitedSources = [init_source(R) || R <- check_sources(Sources)], - ok = emqx_hooks:put('client.authorize', {?MODULE, authorize, [InitedSources ++ lookup()]}, -1), - ok = emqx_authz_cache:drain_cache(); - -post_config_update({tail, Sources}, _NewSources, _OldConf, _AppEnvs) -> - InitedSources = [init_source(R) || R <- check_sources(Sources)], - emqx_hooks:put('client.authorize', {?MODULE, authorize, [lookup() ++ InitedSources]}, -1), - ok = emqx_authz_cache:drain_cache(); - -post_config_update({{replace_once, Type}, #{type := Type} = Source}, _NewSources, _OldConf, _AppEnvs) when is_map(Source) -> - OldInitedSources = lookup(), - {Index, OldSource} = find_source_by_type(Type, OldInitedSources), - case maps:get(type, OldSource, undefined) of - undefined -> ok; - file -> ok; - _ -> - #{annotations := #{id := Id}} = OldSource, - ok = emqx_resource:remove(Id) - end, - {OldSources1, OldSources2 } = lists:split(Index, OldInitedSources), - InitedSources = [init_source(R) || R <- check_sources([Source])], - ok = emqx_hooks:put('client.authorize', {?MODULE, authorize, [lists:droplast(OldSources1) ++ InitedSources ++ OldSources2]}, -1), - ok = emqx_authz_cache:drain_cache(); -post_config_update({{delete_once, Type}, _Source}, _NewSources, _OldConf, _AppEnvs) -> - OldInitedSources = lookup(), - {_, OldSource} = find_source_by_type(Type, OldInitedSources), - case OldSource of - #{annotations := #{id := Id}} -> - ok = emqx_resource:remove(Id); - _ -> ok - end, - ok = emqx_hooks:put('client.authorize', {?MODULE, authorize, [lists:delete(OldSource, OldInitedSources)]}, -1), - ok = emqx_authz_cache:drain_cache(); -post_config_update(_, NewSources, _OldConf, _AppEnvs) -> - %% overwrite the entire config! - OldInitedSources = lookup(), - InitedSources = [init_source(Source) || Source <- NewSources], - ok = emqx_hooks:put('client.authorize', {?MODULE, authorize, [InitedSources]}, -1), - lists:foreach(fun (#{type := _Type, enable := true, annotations := #{id := Id}}) -> - ok = emqx_resource:remove(Id); - (_) -> ok - end, OldInitedSources), +post_config_update(Cmd, NewSources, _OldSource, _AppEnvs) -> + ok = do_post_update(Cmd, NewSources), ok = emqx_authz_cache:drain_cache(). -%%-------------------------------------------------------------------- -%% Initialize source -%%-------------------------------------------------------------------- +do_post_update({?CMD_MOVE, _Type, _Where} = Cmd, _NewSources) -> + InitedSources = lookup(), + MovedSources = do_update(Cmd, InitedSources), + ok = emqx_hooks:put('client.authorize', {?MODULE, authorize, [MovedSources]}, -1), + ok = emqx_authz_cache:drain_cache(); +do_post_update({?CMD_PREPEND, Sources}, _NewSources) -> + InitedSources = init_sources(check_sources(Sources)), + ok = emqx_hooks:put('client.authorize', {?MODULE, authorize, [InitedSources ++ lookup()]}, -1), + ok = emqx_authz_cache:drain_cache(); +do_post_update({?CMD_APPEND, Sources}, _NewSources) -> + InitedSources = init_sources(check_sources(Sources)), + emqx_hooks:put('client.authorize', {?MODULE, authorize, [lookup() ++ InitedSources]}, -1), + ok = emqx_authz_cache:drain_cache(); +do_post_update({{replace, Type}, #{type := Type} = Source}, _NewSources) when is_map(Source) -> + OldInitedSources = lookup(), + {OldSource, Front, Rear} = take(Type, OldInitedSources), + ok = ensure_resource_deleted(OldSource), + InitedSources = init_sources(check_sources([Source])), + ok = emqx_hooks:put('client.authorize', {?MODULE, authorize, [Front ++ InitedSources ++ Rear]}, -1), + ok = emqx_authz_cache:drain_cache(); +do_post_update({{delete, Type}, _Source}, _NewSources) -> + OldInitedSources = lookup(), + {OldSource, Front, Rear} = take(Type, OldInitedSources), + ok = ensure_resource_deleted(OldSource), + ok = emqx_hooks:put('client.authorize', {?MODULE, authorize, [Front ++ Rear]}, -1), + ok = emqx_authz_cache:drain_cache(); +do_post_update(_, NewSources) -> + %% overwrite the entire config! + OldInitedSources = lookup(), + InitedSources = init_sources(NewSources), + ok = emqx_hooks:put('client.authorize', {?MODULE, authorize, [InitedSources]}, -1), + lists:foreach(fun ensure_resource_deleted/1, OldInitedSources), + ok = emqx_authz_cache:drain_cache(). + +ensure_resource_deleted(#{enable := false}) -> ok; +ensure_resource_deleted(#{type := file}) -> ok; +ensure_resource_deleted(#{type := 'built-in-database'}) -> ok; +ensure_resource_deleted(#{annotations := #{id := Id}}) -> ok = emqx_resource:remove(Id). check_dup_types(Sources) -> - check_dup_types(Sources, ?SOURCE_TYPES). -check_dup_types(_Sources, []) -> ok; -check_dup_types(Sources, [T0 | Tail]) -> - case lists:foldl(fun (#{type := T1}, AccIn) -> - case T0 =:= T1 of - true -> AccIn + 1; - false -> AccIn - end; - (#{<<"type">> := T1}, AccIn) -> - case T0 =:= atom(T1) of - true -> AccIn + 1; - false -> AccIn - end - end, 0, Sources) > 1 of + check_dup_types(Sources, []). + +check_dup_types([], _Checked) -> ok; +check_dup_types([Source | Sources], Checked) -> + %% the input might be raw or type-checked result, so lookup both 'type' and <<"type">> + %% TODO: check: really? + Type = case maps:get(<<"type">>, Source, maps:get(type, Source, undefined)) of + undefined -> + %% this should never happen if the value is type checked by honcon schema + error({bad_source_input, Source}); + Type0 -> + type(Type0) + end, + case lists:member(Type, Checked) of true -> - ?LOG(error, "The type is duplicated in the Authorization source"), - {error, 'The type is duplicated in the Authorization source'}; - false -> check_dup_types(Sources, Tail) + %% we have made it clear not to support more than one authz instance for each type + error({duplicated_authz_source_type, Type}); + false -> + check_dup_types(Sources, [Type | Checked]) end. -init_source(#{enable := true, - type := file, +init_sources(Sources) -> + {_Enabled, Disabled} = lists:partition(fun(#{enable := Enable}) -> Enable end, Sources), + case Disabled =/= [] of + true -> ?SLOG(info, #{msg => "disabled_sources_ignored", sources => Disabled}); + false -> ok + end, + lists:map(fun init_source/1, Sources). + +init_source(#{enable := false} = Source) -> Source; +init_source(#{type := file, path := Path } = Source) -> Rules = case file:consult(Path) of @@ -288,8 +212,7 @@ init_source(#{enable := true, error(Reason) end, Source#{annotations => #{rules => Rules}}; -init_source(#{enable := true, - type := http, +init_source(#{type := http, url := Url } = Source) -> NSource= maps:put(base_url, maps:remove(query, Url), Source), @@ -297,16 +220,17 @@ init_source(#{enable := true, {error, Reason} -> error({load_config_error, Reason}); Id -> Source#{annotations => #{id => Id}} end; -init_source(#{enable := true, - type := DB +init_source(#{type := 'built-in-database' + } = Source) -> + Source; +init_source(#{type := DB } = Source) when DB =:= redis; DB =:= mongodb -> case create_resource(Source) of {error, Reason} -> error({load_config_error, Reason}); Id -> Source#{annotations => #{id => Id}} end; -init_source(#{enable := true, - type := DB, +init_source(#{type := DB, query := SQL } = Source) when DB =:= mysql; DB =:= postgresql -> @@ -318,8 +242,7 @@ init_source(#{enable := true, query => Mod:parse_query(SQL) } } - end; -init_source(#{enable := false} = Source) ->Source. + end. %%-------------------------------------------------------------------- %% AuthZ callbacks @@ -373,13 +296,17 @@ check_sources(RawSources) -> #{sources := Sources} = hocon_schema:check_plain(Schema, Conf, #{atom_key => true}), Sources. -find_source_by_type(Type) -> find_source_by_type(Type, lookup()). -find_source_by_type(Type, Sources) -> find_source_by_type(Type, Sources, 1). -find_source_by_type(_, [], _N) -> error(not_found_source); -find_source_by_type(Type, [ Source = #{type := T} | Tail], N) -> - case Type =:= T of - true -> {N, Source}; - false -> find_source_by_type(Type, Tail, N + 1) +take(Type) -> take(Type, lookup()). + +%% Take the source of give type, the sources list is split into two parts +%% front part and rear part. +take(Type, Sources) -> + {Front, Rear} = lists:splitwith(fun(T) -> type(T) =/= type(Type) end, Sources), + case Rear =:= [] of + true -> + error({authz_source_of_type_not_found, Type}); + _ -> + {hd(Rear), Front, tl(Rear)} end. find_action_in_hooks() -> @@ -404,6 +331,8 @@ create_resource(#{type := DB} = Source) -> {error, Reason} -> {error, Reason} end. +authz_module('built-in-database') -> + emqx_authz_mnesia; authz_module(Type) -> list_to_existing_atom("emqx_authz_" ++ atom_to_list(Type)). @@ -414,9 +343,20 @@ connector_module(postgresql) -> connector_module(Type) -> list_to_existing_atom("emqx_connector_" ++ atom_to_list(Type)). -atom(B) when is_binary(B) -> - try binary_to_existing_atom(B, utf8) - catch - _ -> binary_to_atom(B) - end; -atom(A) when is_atom(A) -> A. +type(#{type := Type}) -> type(Type); +type(#{<<"type">> := Type}) -> type(Type); +type(file) -> file; +type(<<"file">>) -> file; +type(http) -> http; +type(<<"http">>) -> http; +type(mongodb) -> mongodb; +type(<<"mongodb">>) -> mongodb; +type(mysql) -> mysql; +type(<<"mysql">>) -> mysql; +type(redis) -> redis; +type(<<"redis">>) -> redis; +type(postgresql) -> postgresql; +type(<<"postgresql">>) -> postgresql; +type('built-in-database') -> 'built-in-database'; +type(<<"built-in-database">>) -> 'built-in-database'; +type(Unknown) -> error({unknown_authz_source_type, Unknown}). % should never happend if the input is type-checked by hocon schema diff --git a/apps/emqx_authz/src/emqx_authz_api_mnesia.erl b/apps/emqx_authz/src/emqx_authz_api_mnesia.erl new file mode 100644 index 000000000..c29f90f43 --- /dev/null +++ b/apps/emqx_authz/src/emqx_authz_api_mnesia.erl @@ -0,0 +1,659 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authz_api_mnesia). + +-behavior(minirest_api). + +-include("emqx_authz.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("stdlib/include/ms_transform.hrl"). + +-define(EXAMPLE_USERNAME, #{username => user1, + rules => [ #{topic => <<"test/toopic/1">>, + permission => <<"allow">>, + action => <<"publish">> + } + , #{topic => <<"test/toopic/2">>, + permission => <<"allow">>, + action => <<"subscribe">> + } + , #{topic => <<"eq test/#">>, + permission => <<"deny">>, + action => <<"all">> + } + ] + }). +-define(EXAMPLE_CLIENTID, #{clientid => client1, + rules => [ #{topic => <<"test/toopic/1">>, + permission => <<"allow">>, + action => <<"publish">> + } + , #{topic => <<"test/toopic/2">>, + permission => <<"allow">>, + action => <<"subscribe">> + } + , #{topic => <<"eq test/#">>, + permission => <<"deny">>, + action => <<"all">> + } + ] + }). +-define(EXAMPLE_ALL , #{rules => [ #{topic => <<"test/toopic/1">>, + permission => <<"allow">>, + action => <<"publish">> + } + , #{topic => <<"test/toopic/2">>, + permission => <<"allow">>, + action => <<"subscribe">> + } + , #{topic => <<"eq test/#">>, + permission => <<"deny">>, + action => <<"all">> + } + ] + }). + +-export([ api_spec/0 + , purge/2 + , users/2 + , user/2 + , clients/2 + , client/2 + , all/2 + ]). + +api_spec() -> + {[ purge_api() + , users_api() + , user_api() + , clients_api() + , client_api() + , all_api() + ], definitions()}. + +definitions() -> + Rules = #{ + type => array, + items => #{ + type => object, + required => [topic, permission, action], + properties => #{ + topic => #{ + type => string, + example => <<"test/topic/1">> + }, + permission => #{ + type => string, + enum => [<<"allow">>, <<"deny">>], + example => <<"allow">> + }, + action => #{ + type => string, + enum => [<<"publish">>, <<"subscribe">>, <<"all">>], + example => <<"publish">> + } + } + } + }, + Username = #{ + type => object, + required => [username, rules], + properties => #{ + username => #{ + type => string, + example => <<"username">> + }, + rules => minirest:ref(<<"rules">>) + } + }, + Clientid = #{ + type => object, + required => [clientid, rules], + properties => #{ + clientid => #{ + type => string, + example => <<"clientid">> + }, + rules => minirest:ref(<<"rules">>) + } + }, + ALL = #{ + type => object, + required => [rules], + properties => #{ + rules => minirest:ref(<<"rules">>) + } + }, + [ #{<<"rules">> => Rules} + , #{<<"username">> => Username} + , #{<<"clientid">> => Clientid} + , #{<<"all">> => ALL} + ]. + +users_api() -> + Metadata = #{ + get => #{ + description => "Show the list of record for username", + parameters => [ + #{ + name => page, + in => query, + required => false, + description => <<"Page Index">>, + schema => #{type => integer} + }, + #{ + name => limit, + in => query, + required => false, + description => <<"Page limit">>, + schema => #{type => integer} + } + ], + responses => #{ + <<"200">> => #{ + description => <<"OK">>, + content => #{ + 'application/json' => #{ + schema => #{ + type => array, + items => minirest:ref(<<"username">>) + }, + examples => #{ + username => #{ + summary => <<"Username">>, + value => jsx:encode([?EXAMPLE_USERNAME]) + } + } + } + } + } + } + }, + post => #{ + description => "Add new records for username", + requestBody => #{ + content => #{ + 'application/json' => #{ + schema => #{ + type => array, + items => #{ + oneOf => [ minirest:ref(<<"username">>) + ] + } + }, + examples => #{ + username => #{ + summary => <<"Username">>, + value => jsx:encode([?EXAMPLE_USERNAME]) + } + } + } + } + }, + responses => #{ + <<"204">> => #{description => <<"Created">>}, + <<"400">> => emqx_mgmt_util:bad_request() + } + } + }, + {"/authorization/sources/built-in-database/username", Metadata, users}. + +clients_api() -> + Metadata = #{ + get => #{ + description => "Show the list of record for clientid", + parameters => [ + #{ + name => page, + in => query, + required => false, + description => <<"Page Index">>, + schema => #{type => integer} + }, + #{ + name => limit, + in => query, + required => false, + description => <<"Page limit">>, + schema => #{type => integer} + } + ], + responses => #{ + <<"200">> => #{ + description => <<"OK">>, + content => #{ + 'application/json' => #{ + schema => #{ + type => array, + items => minirest:ref(<<"clientid">>) + }, + examples => #{ + clientid => #{ + summary => <<"Clientid">>, + value => jsx:encode([?EXAMPLE_CLIENTID]) + } + } + } + } + } + } + }, + post => #{ + description => "Add new records for clientid", + requestBody => #{ + content => #{ + 'application/json' => #{ + schema => #{ + type => array, + items => #{ + oneOf => [ minirest:ref(<<"clientid">>) + ] + } + }, + examples => #{ + clientid => #{ + summary => <<"Clientid">>, + value => jsx:encode([?EXAMPLE_CLIENTID]) + } + } + } + } + }, + responses => #{ + <<"204">> => #{description => <<"Created">>}, + <<"400">> => emqx_mgmt_util:bad_request() + } + } + }, + {"/authorization/sources/built-in-database/clientid", Metadata, clients}. + +user_api() -> + Metadata = #{ + get => #{ + description => "Get record info for username", + parameters => [ + #{ + name => username, + in => path, + schema => #{ + type => string + }, + required => true + } + ], + responses => #{ + <<"200">> => #{ + description => <<"OK">>, + content => #{ + 'application/json' => #{ + schema => minirest:ref(<<"username">>), + examples => #{ + username => #{ + summary => <<"Username">>, + value => jsx:encode(?EXAMPLE_USERNAME) + } + } + } + } + }, + <<"404">> => emqx_mgmt_util:bad_request(<<"Not Found">>) + } + }, + put => #{ + description => "Set record for username", + parameters => [ + #{ + name => username, + in => path, + schema => #{ + type => string + }, + required => true + } + ], + requestBody => #{ + content => #{ + 'application/json' => #{ + schema => minirest:ref(<<"username">>), + examples => #{ + username => #{ + summary => <<"Username">>, + value => jsx:encode(?EXAMPLE_USERNAME) + } + } + } + } + }, + responses => #{ + <<"204">> => #{description => <<"Updated">>}, + <<"400">> => emqx_mgmt_util:bad_request() + } + }, + delete => #{ + description => "Delete one record for username", + parameters => [ + #{ + name => username, + in => path, + schema => #{ + type => string + }, + required => true + } + ], + responses => #{ + <<"204">> => #{description => <<"No Content">>}, + <<"400">> => emqx_mgmt_util:bad_request() + } + } + }, + {"/authorization/sources/built-in-database/username/:username", Metadata, user}. + +client_api() -> + Metadata = #{ + get => #{ + description => "Get record info for clientid", + parameters => [ + #{ + name => clientid, + in => path, + schema => #{ + type => string + }, + required => true + } + ], + responses => #{ + <<"200">> => #{ + description => <<"OK">>, + content => #{ + 'application/json' => #{ + schema => minirest:ref(<<"clientid">>), + examples => #{ + clientid => #{ + summary => <<"Clientid">>, + value => jsx:encode(?EXAMPLE_CLIENTID) + } + } + } + } + }, + <<"404">> => emqx_mgmt_util:bad_request(<<"Not Found">>) + } + }, + put => #{ + description => "Set record for clientid", + parameters => [ + #{ + name => clientid, + in => path, + schema => #{ + type => string + }, + required => true + } + ], + requestBody => #{ + content => #{ + 'application/json' => #{ + schema => minirest:ref(<<"clientid">>), + examples => #{ + clientid => #{ + summary => <<"Clientid">>, + value => jsx:encode(?EXAMPLE_CLIENTID) + } + } + } + } + }, + responses => #{ + <<"204">> => #{description => <<"Updated">>}, + <<"400">> => emqx_mgmt_util:bad_request() + } + }, + delete => #{ + description => "Delete one record for clientid", + parameters => [ + #{ + name => clientid, + in => path, + schema => #{ + type => string + }, + required => true + } + ], + responses => #{ + <<"204">> => #{description => <<"No Content">>}, + <<"400">> => emqx_mgmt_util:bad_request() + } + } + }, + {"/authorization/sources/built-in-database/clientid/:clientid", Metadata, client}. + +all_api() -> + Metadata = #{ + get => #{ + description => "Show the list of rules for all", + responses => #{ + <<"200">> => #{ + description => <<"OK">>, + content => #{ + 'application/json' => #{ + schema => minirest:ref(<<"clientid">>), + examples => #{ + clientid => #{ + summary => <<"All">>, + value => jsx:encode(?EXAMPLE_ALL) + } + } + } + } + } + } + }, + put => #{ + description => "Set the list of rules for all", + requestBody => #{ + content => #{ + 'application/json' => #{ + schema => minirest:ref(<<"all">>), + examples => #{ + all => #{ + summary => <<"All">>, + value => jsx:encode(?EXAMPLE_ALL) + } + } + } + } + }, + responses => #{ + <<"204">> => #{description => <<"Created">>}, + <<"400">> => emqx_mgmt_util:bad_request() + } + } + }, + {"/authorization/sources/built-in-database/all", Metadata, all}. + +purge_api() -> + Metadata = #{ + delete => #{ + description => "Purge all records", + responses => #{ + <<"204">> => #{description => <<"No Content">>}, + <<"400">> => emqx_mgmt_util:bad_request() + } + } + }, + {"/authorization/sources/built-in-database/purge-all", Metadata, purge}. + +users(get, #{query_string := Qs}) -> + MatchSpec = ets:fun2ms( + fun({?ACL_TABLE, {?ACL_TABLE_USERNAME, Username}, Rules}) -> + [{username, Username}, {rules, Rules}] + end), + Format = fun ([{username, Username}, {rules, Rules}]) -> + #{username => Username, + rules => [ #{topic => Topic, + action => Action, + permission => Permission + } || {Permission, Action, Topic} <- Rules] + } + end, + case Qs of + #{<<"limit">> := _, <<"page">> := _} = Page -> + {200, emqx_mgmt_api:paginate(?ACL_TABLE, MatchSpec, Page, Format)}; + #{<<"limit">> := Limit} -> + case ets:select(?ACL_TABLE, MatchSpec, binary_to_integer(Limit)) of + {Rows, _Continuation} -> {200, [Format(Row) || Row <- Rows ]}; + '$end_of_table' -> {404, #{code => <<"NOT_FOUND">>, message => <<"Not Found">>}} + end; + _ -> + {200, [Format(Row) || Row <- ets:select(?ACL_TABLE, MatchSpec)]} + end; +users(post, #{body := Body}) when is_list(Body) -> + lists:foreach(fun(#{<<"username">> := Username, <<"rules">> := Rules}) -> + ekka_mnesia:dirty_write(#emqx_acl{ + who = {?ACL_TABLE_USERNAME, Username}, + rules = format_rules(Rules) + }) + end, Body), + {204}. + +clients(get, #{query_string := Qs}) -> + MatchSpec = ets:fun2ms( + fun({?ACL_TABLE, {?ACL_TABLE_CLIENTID, Clientid}, Rules}) -> + [{clientid, Clientid}, {rules, Rules}] + end), + Format = fun ([{clientid, Clientid}, {rules, Rules}]) -> + #{clientid => Clientid, + rules => [ #{topic => Topic, + action => Action, + permission => Permission + } || {Permission, Action, Topic} <- Rules] + } + end, + case Qs of + #{<<"limit">> := _, <<"page">> := _} = Page -> + {200, emqx_mgmt_api:paginate(?ACL_TABLE, MatchSpec, Page, Format)}; + #{<<"limit">> := Limit} -> + case ets:select(?ACL_TABLE, MatchSpec, binary_to_integer(Limit)) of + {Rows, _Continuation} -> {200, [Format(Row) || Row <- Rows ]}; + '$end_of_table' -> {404, #{code => <<"NOT_FOUND">>, message => <<"Not Found">>}} + end; + _ -> + {200, [Format(Row) || Row <- ets:select(?ACL_TABLE, MatchSpec)]} + end; +clients(post, #{body := Body}) when is_list(Body) -> + lists:foreach(fun(#{<<"clientid">> := Clientid, <<"rules">> := Rules}) -> + ekka_mnesia:dirty_write(#emqx_acl{ + who = {?ACL_TABLE_CLIENTID, Clientid}, + rules = format_rules(Rules) + }) + end, Body), + {204}. + +user(get, #{bindings := #{username := Username}}) -> + case mnesia:dirty_read(?ACL_TABLE, {?ACL_TABLE_USERNAME, Username}) of + [] -> {404, #{code => <<"NOT_FOUND">>, message => <<"Not Found">>}}; + [#emqx_acl{who = {?ACL_TABLE_USERNAME, Username}, rules = Rules}] -> + {200, #{username => Username, + rules => [ #{topic => Topic, + action => Action, + permission => Permission + } || {Permission, Action, Topic} <- Rules]} + } + end; +user(put, #{bindings := #{username := Username}, + body := #{<<"username">> := Username, <<"rules">> := Rules}}) -> + ekka_mnesia:dirty_write(#emqx_acl{ + who = {?ACL_TABLE_USERNAME, Username}, + rules = format_rules(Rules) + }), + {204}; +user(delete, #{bindings := #{username := Username}}) -> + ekka_mnesia:dirty_delete({?ACL_TABLE, {?ACL_TABLE_USERNAME, Username}}), + {204}. + +client(get, #{bindings := #{clientid := Clientid}}) -> + case mnesia:dirty_read(?ACL_TABLE, {?ACL_TABLE_CLIENTID, Clientid}) of + [] -> {404, #{code => <<"NOT_FOUND">>, message => <<"Not Found">>}}; + [#emqx_acl{who = {?ACL_TABLE_CLIENTID, Clientid}, rules = Rules}] -> + {200, #{clientid => Clientid, + rules => [ #{topic => Topic, + action => Action, + permission => Permission + } || {Permission, Action, Topic} <- Rules]} + } + end; +client(put, #{bindings := #{clientid := Clientid}, + body := #{<<"clientid">> := Clientid, <<"rules">> := Rules}}) -> + ekka_mnesia:dirty_write(#emqx_acl{ + who = {?ACL_TABLE_CLIENTID, Clientid}, + rules = format_rules(Rules) + }), + {204}; +client(delete, #{bindings := #{clientid := Clientid}}) -> + ekka_mnesia:dirty_delete({?ACL_TABLE, {?ACL_TABLE_CLIENTID, Clientid}}), + {204}. + +all(get, _) -> + case mnesia:dirty_read(?ACL_TABLE, ?ACL_TABLE_ALL) of + [] -> + {200, #{rules => []}}; + [#emqx_acl{who = ?ACL_TABLE_ALL, rules = Rules}] -> + {200, #{rules => [ #{topic => Topic, + action => Action, + permission => Permission + } || {Permission, Action, Topic} <- Rules]} + } + end; +all(put, #{body := #{<<"rules">> := Rules}}) -> + ekka_mnesia:dirty_write(#emqx_acl{ + who = ?ACL_TABLE_ALL, + rules = format_rules(Rules) + }), + {204}. + +purge(delete, _) -> + case emqx_authz_api_sources:get_raw_source(<<"built-in-database">>) of + [#{enable := false}] -> + ok = lists:foreach(fun(Key) -> + ok = ekka_mnesia:dirty_delete(?ACL_TABLE, Key) + end, mnesia:dirty_all_keys(?ACL_TABLE)), + {204}; + _ -> + {400, #{code => <<"BAD_REQUEST">>, + message => <<"'built-in-database' type source must be disabled before purge.">>}} + end. + +format_rules(Rules) when is_list(Rules) -> + lists:foldl(fun(#{<<"topic">> := Topic, + <<"action">> := Action, + <<"permission">> := Permission + }, AccIn) when ?PUBSUB(Action) + andalso ?ALLOW_DENY(Permission) -> + AccIn ++ [{ atom(Permission), atom(Action), Topic }] + end, [], Rules). + +atom(B) when is_binary(B) -> + try binary_to_existing_atom(B, utf8) + catch + _ -> binary_to_atom(B) + end; +atom(A) when is_atom(A) -> A. diff --git a/apps/emqx_authz/src/emqx_authz_api_schema.erl b/apps/emqx_authz/src/emqx_authz_api_schema.erl index d99c5acb5..b05476b03 100644 --- a/apps/emqx_authz/src/emqx_authz_api_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_api_schema.erl @@ -19,29 +19,9 @@ -export([definitions/0]). definitions() -> - RetruenedSources = #{ - allOf => [ #{type => object, - properties => #{ - annotations => #{ - type => object, - required => [status], - properties => #{ - id => #{ - type => string - }, - status => #{ - type => string, - example => <<"healthy">> - } - } - } - } - } - , minirest:ref(<<"sources">>) - ] - }, Sources = #{ oneOf => [ minirest:ref(<<"http">>) + , minirest:ref(<<"built-in-database">>) , minirest:ref(<<"mongo_single">>) , minirest:ref(<<"mongo_rs">>) , minirest:ref(<<"mongo_sharded">>) @@ -100,9 +80,9 @@ definitions() -> }, headers => #{type => object}, body => #{type => object}, - connect_timeout => #{type => integer}, + connect_timeout => #{type => string}, max_retries => #{type => integer}, - retry_interval => #{type => integer}, + retry_interval => #{type => string}, pool_type => #{ type => string, enum => [<<"random">>, <<"hash">>], @@ -154,8 +134,8 @@ definitions() -> properties => #{ pool_size => #{type => integer}, max_overflow => #{type => integer}, - overflow_ttl => #{type => integer}, - overflow_check_period => #{type => integer}, + overflow_ttl => #{type => string}, + overflow_check_period => #{type => string}, local_threshold_ms => #{type => integer}, connect_timeout_ms => #{type => integer}, socket_timeout_ms => #{type => integer}, @@ -212,8 +192,8 @@ definitions() -> properties => #{ pool_size => #{type => integer}, max_overflow => #{type => integer}, - overflow_ttl => #{type => integer}, - overflow_check_period => #{type => integer}, + overflow_ttl => #{type => string}, + overflow_check_period => #{type => string}, local_threshold_ms => #{type => integer}, connect_timeout_ms => #{type => integer}, socket_timeout_ms => #{type => integer}, @@ -268,8 +248,8 @@ definitions() -> properties => #{ pool_size => #{type => integer}, max_overflow => #{type => integer}, - overflow_ttl => #{type => integer}, - overflow_check_period => #{type => integer}, + overflow_ttl => #{type => string}, + overflow_check_period => #{type => string}, local_threshold_ms => #{type => integer}, connect_timeout_ms => #{type => integer}, socket_timeout_ms => #{type => integer}, @@ -467,6 +447,21 @@ definitions() -> ssl => minirest:ref(<<"ssl">>) } }, + Mnesia = #{ + type => object, + required => [type, enable], + properties => #{ + type => #{ + type => string, + enum => [<<"redis">>], + example => <<"redis">> + }, + enable => #{ + type => boolean, + example => true + } + } + }, File = #{ type => object, required => [type, enable, rules], @@ -493,10 +488,10 @@ definitions() -> } } }, - [ #{<<"returned_sources">> => RetruenedSources} - , #{<<"sources">> => Sources} + [ #{<<"sources">> => Sources} , #{<<"ssl">> => SSL} , #{<<"http">> => HTTP} + , #{<<"built-in-database">> => Mnesia} , #{<<"mongo_single">> => MongoSingle} , #{<<"mongo_rs">> => MongoRs} , #{<<"mongo_sharded">> => MongoSharded} diff --git a/apps/emqx_authz/src/emqx_authz_api_sources.erl b/apps/emqx_authz/src/emqx_authz_api_sources.erl index 1586b1c88..e0d4667cb 100644 --- a/apps/emqx_authz/src/emqx_authz_api_sources.erl +++ b/apps/emqx_authz/src/emqx_authz_api_sources.erl @@ -35,19 +35,16 @@ rules => <<"{allow,{username,\"^dashboard?\"},subscribe,[\"$SYS/#\"]}.\n{allow,{ipaddr,\"127.0.0.1\"},all,[\"$SYS/#\",\"#\"]}.">> }). --define(EXAMPLE_RETURNED_REDIS, - maps:put(annotations, #{status => healthy}, ?EXAMPLE_REDIS) - ). --define(EXAMPLE_RETURNED_FILE, - maps:put(annotations, #{status => healthy}, ?EXAMPLE_FILE) - ). - -define(EXAMPLE_RETURNED, - #{sources => [ ?EXAMPLE_RETURNED_REDIS - , ?EXAMPLE_RETURNED_FILE + #{sources => [ ?EXAMPLE_REDIS + , ?EXAMPLE_FILE ] }). +-export([ get_raw_sources/0 + , get_raw_source/1 + ]). + -export([ api_spec/0 , sources/2 , source/2 @@ -76,7 +73,7 @@ sources_api() -> required => [sources], properties => #{sources => #{ type => array, - items => minirest:ref(<<"returned_sources">>) + items => minirest:ref(<<"sources">>) } } }, @@ -122,7 +119,7 @@ sources_api() -> 'application/json' => #{ schema => #{ type => array, - items => minirest:ref(<<"returned_sources">>) + items => minirest:ref(<<"sources">>) }, examples => #{ redis => #{ @@ -154,7 +151,15 @@ source_api() -> name => type, in => path, schema => #{ - type => string + type => string, + enum => [ <<"file">> + , <<"http">> + , <<"mongodb">> + , <<"mysql">> + , <<"postgresql">> + , <<"redis">> + , <<"built-in-database">> + ] }, required => true } @@ -164,15 +169,15 @@ source_api() -> description => <<"OK">>, content => #{ 'application/json' => #{ - schema => minirest:ref(<<"returned_sources">>), + schema => minirest:ref(<<"sources">>), examples => #{ redis => #{ summary => <<"Redis">>, - value => jsx:encode(?EXAMPLE_RETURNED_REDIS) + value => jsx:encode(?EXAMPLE_REDIS) }, file => #{ summary => <<"File">>, - value => jsx:encode(?EXAMPLE_RETURNED_FILE) + value => jsx:encode(?EXAMPLE_FILE) } } } @@ -188,7 +193,15 @@ source_api() -> name => type, in => path, schema => #{ - type => string + type => string, + enum => [ <<"file">> + , <<"http">> + , <<"mongodb">> + , <<"mysql">> + , <<"postgresql">> + , <<"redis">> + , <<"built-in-database">> + ] }, required => true } @@ -223,7 +236,15 @@ source_api() -> name => type, in => path, schema => #{ - type => string + type => string, + enum => [ <<"file">> + , <<"http">> + , <<"mongodb">> + , <<"mysql">> + , <<"postgresql">> + , <<"redis">> + , <<"built-in-database">> + ] }, required => true } @@ -245,7 +266,15 @@ move_source_api() -> name => type, in => path, schema => #{ - type => string + type => string, + enum => [ <<"file">> + , <<"http">> + , <<"mongodb">> + , <<"mysql">> + , <<"postgresql">> + , <<"redis">> + , <<"built-in-database">> + ] }, required => true } @@ -297,109 +326,69 @@ move_source_api() -> {"/authorization/sources/:type/move", Metadata, move_source}. sources(get, _) -> - Sources = lists:foldl(fun (#{type := file, enable := Enable, path := Path}, AccIn) -> + Sources = lists:foldl(fun (#{<<"type">> := <<"file">>, <<"enable">> := Enable, <<"path">> := Path}, AccIn) -> case file:read_file(Path) of {ok, Rules} -> lists:append(AccIn, [#{type => file, enable => Enable, - rules => Rules, - annotations => #{status => healthy} + rules => Rules }]); {error, _} -> lists:append(AccIn, [#{type => file, enable => Enable, - rules => <<"">>, - annotations => #{status => unhealthy} + rules => <<"">> }]) end; - (#{enable := false} = Source, AccIn) -> - lists:append(AccIn, [Source#{annotations => #{status => unhealthy}}]); - (#{type := _Type, annotations := #{id := Id}} = Source, AccIn) -> - NSource0 = case maps:get(server, Source, undefined) of - undefined -> Source; - Server -> - Source#{server => emqx_connector_schema_lib:ip_port_to_string(Server)} - end, - NSource1 = case maps:get(servers, Source, undefined) of - undefined -> NSource0; - Servers -> - NSource0#{servers => [emqx_connector_schema_lib:ip_port_to_string(Server) || Server <- Servers]} - end, - NSource2 = case emqx_resource:health_check(Id) of - ok -> - NSource1#{annotations => #{status => healthy}}; - _ -> - NSource1#{annotations => #{status => unhealthy}} - end, - lists:append(AccIn, [read_cert(NSource2)]); (Source, AccIn) -> - lists:append(AccIn, [Source#{annotations => #{status => healthy}}]) - end, [], emqx_authz:lookup()), + lists:append(AccIn, [read_cert(Source)]) + end, [], get_raw_sources()), {200, #{sources => Sources}}; sources(post, #{body := #{<<"type">> := <<"file">>, <<"rules">> := Rules}}) -> {ok, Filename} = write_file(filename:join([emqx:get_config([node, data_dir]), "acl.conf"]), Rules), - update_config(head, [#{type => file, enable => true, path => Filename}]); + update_config(?CMD_PREPEND, [#{<<"type">> => <<"file">>, <<"enable">> => true, <<"path">> => Filename}]); sources(post, #{body := Body}) when is_map(Body) -> - update_config(head, [write_cert(Body)]); + update_config(?CMD_PREPEND, [write_cert(Body)]); sources(put, #{body := Body}) when is_list(Body) -> NBody = [ begin case Source of #{<<"type">> := <<"file">>, <<"rules">> := Rules, <<"enable">> := Enable} -> {ok, Filename} = write_file(filename:join([emqx:get_config([node, data_dir]), "acl.conf"]), Rules), - #{type => file, enable => Enable, path => Filename}; + #{<<"type">> => <<"file">>, <<"enable">> => Enable, <<"path">> => Filename}; _ -> write_cert(Source) end end || Source <- Body], - update_config(replace, NBody). + update_config(?CMD_REPLCAE, NBody). source(get, #{bindings := #{type := Type}}) -> - case emqx_authz:lookup(Type) of - {error, Reason} -> {404, #{message => atom_to_binary(Reason)}}; - #{type := file, enable := Enable, path := Path}-> + case get_raw_source(Type) of + [] -> {404, #{message => <<"Not found ", Type/binary>>}}; + [#{<<"type">> := <<"file">>, <<"enable">> := Enable, <<"path">> := Path}] -> case file:read_file(Path) of {ok, Rules} -> {200, #{type => file, enable => Enable, - rules => Rules, - annotations => #{status => healthy} + rules => Rules } }; {error, Reason} -> {400, #{code => <<"BAD_REQUEST">>, - message => atom_to_binary(Reason)}} + message => bin(Reason)}} end; - #{enable := false} = Source -> {200, Source#{annotations => #{status => unhealthy}}}; - #{annotations := #{id := Id}} = Source -> - NSource0 = case maps:get(server, Source, undefined) of - undefined -> Source; - Server -> - Source#{server => emqx_connector_schema_lib:ip_port_to_string(Server)} - end, - NSource1 = case maps:get(servers, Source, undefined) of - undefined -> NSource0; - Servers -> - NSource0#{servers => [emqx_connector_schema_lib:ip_port_to_string(Server) || Server <- Servers]} - end, - NSource2 = case emqx_resource:health_check(Id) of - ok -> - NSource1#{annotations => #{status => healthy}}; - _ -> - NSource1#{annotations => #{status => unhealthy}} - end, - {200, read_cert(NSource2)} + [Source] -> + {200, read_cert(Source)} end; source(put, #{bindings := #{type := <<"file">>}, body := #{<<"type">> := <<"file">>, <<"rules">> := Rules, <<"enable">> := Enable}}) -> {ok, Filename} = write_file(maps:get(path, emqx_authz:lookup(file), ""), Rules), - case emqx_authz:update({replace_once, file}, #{type => file, enable => Enable, path => Filename}) of + case emqx_authz:update({?CMD_REPLCAE, file}, #{<<"type">> => file, <<"enable">> => Enable, <<"path">> => Filename}) of {ok, _} -> {204}; {error, Reason} -> {400, #{code => <<"BAD_REQUEST">>, - message => atom_to_binary(Reason)}} + message => bin(Reason)}} end; source(put, #{bindings := #{type := Type}, body := Body}) when is_map(Body) -> - update_config({replace_once, Type}, write_cert(Body)); + update_config({?CMD_REPLCAE, Type}, write_cert(Body)); source(delete, #{bindings := #{type := Type}}) -> - update_config({delete_once, Type}, #{}). + update_config({?CMD_DELETE, Type}, #{}). move_source(post, #{bindings := #{type := Type}, body := #{<<"position">> := Position}}) -> case emqx_authz:move(Type, Position) of @@ -409,40 +398,52 @@ move_source(post, #{bindings := #{type := Type}, body := #{<<"position">> := Pos message => <<"source ", Type/binary, " not found">>}}; {error, Reason} -> {400, #{code => <<"BAD_REQUEST">>, - message => atom_to_binary(Reason)}} + message => bin(Reason)}} end. +get_raw_sources() -> + RawSources = emqx:get_raw_config([authorization, sources]), + Schema = #{roots => emqx_authz_schema:fields("authorization"), fields => #{}}, + Conf = #{<<"sources">> => RawSources}, + #{<<"sources">> := Sources} = hocon_schema:check_plain(Schema, Conf, #{only_fill_defaults => true}), + Sources. + +get_raw_source(Type) -> + lists:filter(fun (#{<<"type">> := T}) -> + T =:= Type + end, get_raw_sources()). + update_config(Cmd, Sources) -> case emqx_authz:update(Cmd, Sources) of {ok, _} -> {204}; {error, {pre_config_update, emqx_authz, Reason}} -> {400, #{code => <<"BAD_REQUEST">>, - message => atom_to_binary(Reason)}}; + message => bin(Reason)}}; {error, {post_config_update, emqx_authz, Reason}} -> {400, #{code => <<"BAD_REQUEST">>, - message => atom_to_binary(Reason)}}; + message => bin(Reason)}}; {error, Reason} -> {400, #{code => <<"BAD_REQUEST">>, - message => atom_to_binary(Reason)}} + message => bin(Reason)}} end. -read_cert(#{ssl := #{enable := true} = SSL} = Source) -> - CaCert = case file:read_file(maps:get(cacertfile, SSL, "")) of +read_cert(#{<<"ssl">> := #{<<"enable">> := true} = SSL} = Source) -> + CaCert = case file:read_file(maps:get(<<"cacertfile">>, SSL, "")) of {ok, CaCert0} -> CaCert0; _ -> "" end, - Cert = case file:read_file(maps:get(certfile, SSL, "")) of + Cert = case file:read_file(maps:get(<<"certfile">>, SSL, "")) of {ok, Cert0} -> Cert0; _ -> "" end, - Key = case file:read_file(maps:get(keyfile, SSL, "")) of + Key = case file:read_file(maps:get(<<"keyfile">>, SSL, "")) of {ok, Key0} -> Key0; _ -> "" end, - Source#{ssl => SSL#{cacertfile => CaCert, - certfile => Cert, - keyfile => Key - } + Source#{<<"ssl">> => SSL#{<<"cacertfile">> => CaCert, + <<"certfile">> => Cert, + <<"keyfile">> => Key + } }; read_cert(Source) -> Source. @@ -494,3 +495,6 @@ do_write_file(Filename, Bytes) -> ?LOG(error, "Write File ~p Error: ~p", [Filename, Reason]), error(Reason) end. + +bin(Term) -> + erlang:iolist_to_binary(io_lib:format("~p", [Term])). diff --git a/apps/emqx_authz/src/emqx_authz_app.erl b/apps/emqx_authz/src/emqx_authz_app.erl index 460d7cbf9..f868ac342 100644 --- a/apps/emqx_authz/src/emqx_authz_app.erl +++ b/apps/emqx_authz/src/emqx_authz_app.erl @@ -7,9 +7,12 @@ -behaviour(application). +-include("emqx_authz.hrl"). + -export([start/2, stop/1]). start(_StartType, _StartArgs) -> + ok = ekka_rlog:wait_for_shards([?ACL_SHARDED], infinity), {ok, Sup} = emqx_authz_sup:start_link(), ok = emqx_authz:init(), {ok, Sup}. diff --git a/apps/emqx_authz/src/emqx_authz_mnesia.erl b/apps/emqx_authz/src/emqx_authz_mnesia.erl new file mode 100644 index 000000000..ab755403e --- /dev/null +++ b/apps/emqx_authz/src/emqx_authz_mnesia.erl @@ -0,0 +1,76 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authz_mnesia). + +-include("emqx_authz.hrl"). +-include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/logger.hrl"). + +%% AuthZ Callbacks +-export([ mnesia/1 + , authorize/4 + , description/0 + ]). + +-ifdef(TEST). +-compile(export_all). +-compile(nowarn_export_all). +-endif. + +-boot_mnesia({mnesia, [boot]}). +-copy_mnesia({mnesia, [copy]}). + +-spec(mnesia(boot | copy) -> ok). +mnesia(boot) -> + ok = ekka_mnesia:create_table(?ACL_TABLE, [ + {type, ordered_set}, + {rlog_shard, ?ACL_SHARDED}, + {disc_copies, [node()]}, + {attributes, record_info(fields, ?ACL_TABLE)}, + {storage_properties, [{ets, [{read_concurrency, true}]}]}]); +mnesia(copy) -> + ok = ekka_mnesia:copy_table(?ACL_TABLE, disc_copies). + +description() -> + "AuthZ with Mnesia". + +authorize(#{username := Username, + clientid := Clientid + } = Client, PubSub, Topic, #{type := 'built-in-database'}) -> + + Rules = case mnesia:dirty_read(?ACL_TABLE, {?ACL_TABLE_CLIENTID, Clientid}) of + [] -> []; + [#emqx_acl{rules = Rules0}] when is_list(Rules0) -> Rules0 + end + ++ case mnesia:dirty_read(?ACL_TABLE, {?ACL_TABLE_USERNAME, Username}) of + [] -> []; + [#emqx_acl{rules = Rules1}] when is_list(Rules1) -> Rules1 + end + ++ case mnesia:dirty_read(?ACL_TABLE, ?ACL_TABLE_ALL) of + [] -> []; + [#emqx_acl{rules = Rules2}] when is_list(Rules2) -> Rules2 + end, + do_authorize(Client, PubSub, Topic, Rules). + +do_authorize(_Client, _PubSub, _Topic, []) -> nomatch; +do_authorize(Client, PubSub, Topic, [ {Permission, Action, TopicFilter} | Tail]) -> + case emqx_authz_rule:match(Client, PubSub, Topic, + emqx_authz_rule:compile({Permission, all, Action, [TopicFilter]}) + ) of + {matched, Permission} -> {matched, Permission}; + nomatch -> do_authorize(Client, PubSub, Topic, Tail) + end. diff --git a/apps/emqx_authz/src/emqx_authz_mongodb.erl b/apps/emqx_authz/src/emqx_authz_mongodb.erl index 6c0fb126a..ca65e6f53 100644 --- a/apps/emqx_authz/src/emqx_authz_mongodb.erl +++ b/apps/emqx_authz/src/emqx_authz_mongodb.erl @@ -58,9 +58,9 @@ do_authorize(Client, PubSub, Topic, [Rule | Tail]) -> end. replvar(Selector, #{clientid := Clientid, - username := Username, - peerhost := IpAddress - }) -> + username := Username, + peerhost := IpAddress + }) -> Fun = fun _Fun(K, V, AccIn) when is_map(V) -> maps:put(K, maps:fold(_Fun, AccIn, V), AccIn); _Fun(K, V, AccIn) when is_list(V) -> diff --git a/apps/emqx_authz/src/emqx_authz_mysql.erl b/apps/emqx_authz/src/emqx_authz_mysql.erl index ac8f04f32..6ad206d90 100644 --- a/apps/emqx_authz/src/emqx_authz_mysql.erl +++ b/apps/emqx_authz/src/emqx_authz_mysql.erl @@ -69,7 +69,6 @@ do_authorize(Client, PubSub, Topic, Columns, [Row | Tail]) -> nomatch -> do_authorize(Client, PubSub, Topic, Columns, Tail) end. - format_result(Columns, Row) -> Permission = lists:nth(index(<<"permission">>, Columns), Row), Action = lists:nth(index(<<"action">>, Columns), Row), diff --git a/apps/emqx_authz/src/emqx_authz_rule.erl b/apps/emqx_authz/src/emqx_authz_rule.erl index deb8968c6..5f4dcfcab 100644 --- a/apps/emqx_authz/src/emqx_authz_rule.erl +++ b/apps/emqx_authz/src/emqx_authz_rule.erl @@ -32,16 +32,21 @@ -export_type([rule/0]). +compile({Permission, all}) when ?ALLOW_DENY(Permission) -> {Permission, all, all, [compile_topic(<<"#">>)]}; compile({Permission, Who, Action, TopicFilters}) when ?ALLOW_DENY(Permission), ?PUBSUB(Action), is_list(TopicFilters) -> {atom(Permission), compile_who(Who), atom(Action), [compile_topic(Topic) || Topic <- TopicFilters]}. compile_who(all) -> all; -compile_who({username, Username}) -> +compile_who({user, Username}) -> compile_who({username, Username}); +compile_who({username, {re, Username}}) -> {ok, MP} = re:compile(bin(Username)), {username, MP}; -compile_who({clientid, Clientid}) -> +compile_who({username, Username}) -> {username, {eq, bin(Username)}}; +compile_who({client, Clientid}) -> compile_who({clientid, Clientid}); +compile_who({clientid, {re, Clientid}}) -> {ok, MP} = re:compile(bin(Clientid)), {clientid, MP}; +compile_who({clientid, Clientid}) -> {clientid, {eq, bin(Clientid)}}; compile_who({ipaddr, CIDR}) -> {ipaddr, esockd_cidr:parse(CIDR, true)}; compile_who({ipaddrs, CIDRs}) -> @@ -102,14 +107,16 @@ match_action(_, all) -> true; match_action(_, _) -> false. match_who(_, all) -> true; -match_who(#{username := undefined}, {username, _MP}) -> +match_who(#{username := undefined}, {username, _}) -> false; -match_who(#{username := Username}, {username, MP}) -> +match_who(#{username := Username}, {username, {eq, Username}}) -> true; +match_who(#{username := Username}, {username, {re_pattern, _, _, _, _} = MP}) -> case re:run(Username, MP) of {match, _} -> true; _ -> false end; -match_who(#{clientid := Clientid}, {clientid, MP}) -> +match_who(#{clientid := Clientid}, {clientid, {eq, Clientid}}) -> true; +match_who(#{clientid := Clientid}, {clientid, {re_pattern, _, _, _, _} = MP}) -> case re:run(Clientid, MP) of {match, _} -> true; _ -> false diff --git a/apps/emqx_authz/src/emqx_authz_schema.erl b/apps/emqx_authz/src/emqx_authz_schema.erl index c880b7669..50c2ec3ff 100644 --- a/apps/emqx_authz/src/emqx_authz_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_schema.erl @@ -18,6 +18,8 @@ , fields/1 ]). +-import(emqx_schema, [mk_duration/2]). + namespace() -> authz. %% @doc authorization schema is not exported @@ -29,6 +31,7 @@ fields("authorization") -> [ hoconsc:ref(?MODULE, file) , hoconsc:ref(?MODULE, http_get) , hoconsc:ref(?MODULE, http_post) + , hoconsc:ref(?MODULE, mnesia) , hoconsc:ref(?MODULE, mongo_single) , hoconsc:ref(?MODULE, mongo_rs) , hoconsc:ref(?MODULE, mongo_sharded) @@ -45,11 +48,7 @@ fields(file) -> , {enable, #{type => boolean(), default => true}} , {path, #{type => string(), - validator => fun(S) -> case filelib:is_file(S) of - true -> ok; - _ -> {error, "File does not exist"} - end - end + desc => "Path to the file which contains the ACL rules." }} ]; fields(http_get) -> @@ -77,7 +76,7 @@ fields(http_get) -> end } } - , {request_timeout, #{type => timeout(), default => 30000 }} + , {request_timeout, mk_duration("request timeout", #{default => "30s"})} ] ++ proplists:delete(base_url, emqx_connector_http:fields(config)); fields(http_post) -> [ {type, #{type => http}} @@ -107,12 +106,17 @@ fields(http_post) -> end } } - , {request_timeout, #{type => timeout(), default => 30000 }} + , {request_timeout, mk_duration("request timeout", #{default => "30s"})} , {body, #{type => map(), nullable => true } } ] ++ proplists:delete(base_url, emqx_connector_http:fields(config)); +fields(mnesia) -> + [ {type, #{type => 'built-in-database'}} + , {enable, #{type => boolean(), + default => true}} + ]; fields(mongo_single) -> [ {collection, #{type => atom()}} , {selector, #{type => map()}} diff --git a/apps/emqx_authz/test/emqx_authz_SUITE.erl b/apps/emqx_authz/test/emqx_authz_SUITE.erl index bfdc131a0..16bf39d49 100644 --- a/apps/emqx_authz/test/emqx_authz_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_SUITE.erl @@ -50,14 +50,14 @@ init_per_suite(Config) -> Config. end_per_suite(_Config) -> - {ok, _} = emqx_authz:update(replace, []), + {ok, _} = emqx_authz:update(?CMD_REPLCAE, []), emqx_ct_helpers:stop_apps([emqx_authz, emqx_resource]), meck:unload(emqx_resource), meck:unload(emqx_schema), ok. init_per_testcase(_, Config) -> - {ok, _} = emqx_authz:update(replace, []), + {ok, _} = emqx_authz:update(?CMD_REPLCAE, []), Config. -define(SOURCE1, #{<<"type">> => <<"http">>, @@ -120,12 +120,12 @@ init_per_testcase(_, Config) -> %%------------------------------------------------------------------------------ t_update_source(_) -> - {ok, _} = emqx_authz:update(replace, [?SOURCE3]), - {ok, _} = emqx_authz:update(head, [?SOURCE2]), - {ok, _} = emqx_authz:update(head, [?SOURCE1]), - {ok, _} = emqx_authz:update(tail, [?SOURCE4]), - {ok, _} = emqx_authz:update(tail, [?SOURCE5]), - {ok, _} = emqx_authz:update(tail, [?SOURCE6]), + {ok, _} = emqx_authz:update(?CMD_REPLCAE, [?SOURCE3]), + {ok, _} = emqx_authz:update(?CMD_PREPEND, [?SOURCE2]), + {ok, _} = emqx_authz:update(?CMD_PREPEND, [?SOURCE1]), + {ok, _} = emqx_authz:update(?CMD_APPEND, [?SOURCE4]), + {ok, _} = emqx_authz:update(?CMD_APPEND, [?SOURCE5]), + {ok, _} = emqx_authz:update(?CMD_APPEND, [?SOURCE6]), ?assertMatch([ #{type := http, enable := true} , #{type := mongodb, enable := true} @@ -135,12 +135,12 @@ t_update_source(_) -> , #{type := file, enable := true} ], emqx:get_config([authorization, sources], [])), - {ok, _} = emqx_authz:update({replace_once, http}, ?SOURCE1#{<<"enable">> := false}), - {ok, _} = emqx_authz:update({replace_once, mongodb}, ?SOURCE2#{<<"enable">> := false}), - {ok, _} = emqx_authz:update({replace_once, mysql}, ?SOURCE3#{<<"enable">> := false}), - {ok, _} = emqx_authz:update({replace_once, postgresql}, ?SOURCE4#{<<"enable">> := false}), - {ok, _} = emqx_authz:update({replace_once, redis}, ?SOURCE5#{<<"enable">> := false}), - {ok, _} = emqx_authz:update({replace_once, file}, ?SOURCE6#{<<"enable">> := false}), + {ok, _} = emqx_authz:update({?CMD_REPLCAE, http}, ?SOURCE1#{<<"enable">> := false}), + {ok, _} = emqx_authz:update({?CMD_REPLCAE, mongodb}, ?SOURCE2#{<<"enable">> := false}), + {ok, _} = emqx_authz:update({?CMD_REPLCAE, mysql}, ?SOURCE3#{<<"enable">> := false}), + {ok, _} = emqx_authz:update({?CMD_REPLCAE, postgresql}, ?SOURCE4#{<<"enable">> := false}), + {ok, _} = emqx_authz:update({?CMD_REPLCAE, redis}, ?SOURCE5#{<<"enable">> := false}), + {ok, _} = emqx_authz:update({?CMD_REPLCAE, file}, ?SOURCE6#{<<"enable">> := false}), ?assertMatch([ #{type := http, enable := false} , #{type := mongodb, enable := false} @@ -150,10 +150,10 @@ t_update_source(_) -> , #{type := file, enable := false} ], emqx:get_config([authorization, sources], [])), - {ok, _} = emqx_authz:update(replace, []). + {ok, _} = emqx_authz:update(?CMD_REPLCAE, []). t_move_source(_) -> - {ok, _} = emqx_authz:update(replace, [?SOURCE1, ?SOURCE2, ?SOURCE3, ?SOURCE4, ?SOURCE5, ?SOURCE6]), + {ok, _} = emqx_authz:update(?CMD_REPLCAE, [?SOURCE1, ?SOURCE2, ?SOURCE3, ?SOURCE4, ?SOURCE5, ?SOURCE6]), ?assertMatch([ #{type := http} , #{type := mongodb} , #{type := mysql} diff --git a/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl new file mode 100644 index 000000000..1ea942c10 --- /dev/null +++ b/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl @@ -0,0 +1,224 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authz_api_mnesia_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include("emqx_authz.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-define(CONF_DEFAULT, <<"authorization: {sources: []}">>). + +-import(emqx_ct_http, [ request_api/3 + , request_api/5 + , get_http_data/1 + , create_default_app/0 + , delete_default_app/0 + , default_auth_header/0 + , auth_header/2 + ]). + +-define(HOST, "http://127.0.0.1:18083/"). +-define(API_VERSION, "v5"). +-define(BASE_PATH, "api"). + +-define(EXAMPLE_USERNAME, #{username => user1, + rules => [ #{topic => <<"test/toopic/1">>, + permission => <<"allow">>, + action => <<"publish">> + } + , #{topic => <<"test/toopic/2">>, + permission => <<"allow">>, + action => <<"subscribe">> + } + , #{topic => <<"eq test/#">>, + permission => <<"deny">>, + action => <<"all">> + } + ] + }). +-define(EXAMPLE_CLIENTID, #{clientid => client1, + rules => [ #{topic => <<"test/toopic/1">>, + permission => <<"allow">>, + action => <<"publish">> + } + , #{topic => <<"test/toopic/2">>, + permission => <<"allow">>, + action => <<"subscribe">> + } + , #{topic => <<"eq test/#">>, + permission => <<"deny">>, + action => <<"all">> + } + ] + }). +-define(EXAMPLE_ALL , #{rules => [ #{topic => <<"test/toopic/1">>, + permission => <<"allow">>, + action => <<"publish">> + } + , #{topic => <<"test/toopic/2">>, + permission => <<"allow">>, + action => <<"subscribe">> + } + , #{topic => <<"eq test/#">>, + permission => <<"deny">>, + action => <<"all">> + } + ] + }). + +all() -> + []. %% Todo: Waiting for @terry-xiaoyu to fix the config_not_found error + % emqx_ct:all(?MODULE). + +groups() -> + []. + +init_per_suite(Config) -> + meck:new(emqx_schema, [non_strict, passthrough, no_history, no_link]), + meck:expect(emqx_schema, fields, fun("authorization") -> + meck:passthrough(["authorization"]) ++ + emqx_authz_schema:fields("authorization"); + (F) -> meck:passthrough([F]) + end), + + ok = emqx_config:init_load(emqx_authz_schema, ?CONF_DEFAULT), + + ok = emqx_ct_helpers:start_apps([emqx_authz, emqx_dashboard], fun set_special_configs/1), + {ok, _} = emqx:update_config([authorization, cache, enable], false), + {ok, _} = emqx:update_config([authorization, no_match], deny), + + Config. + +end_per_suite(_Config) -> + {ok, _} = emqx_authz:update(replace, []), + emqx_ct_helpers:stop_apps([emqx_authz, emqx_dashboard]), + meck:unload(emqx_schema), + ok. + +set_special_configs(emqx_dashboard) -> + Config = #{ + default_username => <<"admin">>, + default_password => <<"public">>, + listeners => [#{ + protocol => http, + port => 18083 + }] + }, + emqx_config:put([emqx_dashboard], Config), + ok; +set_special_configs(emqx_authz) -> + emqx_config:put([authorization], #{sources => [#{type => 'built-in-database', + enable => true} + ]}), + ok; +set_special_configs(_App) -> + ok. + +%%------------------------------------------------------------------------------ +%% Testcases +%%------------------------------------------------------------------------------ + +t_api(_) -> + {ok, 204, _} = request(post, uri(["authorization", "sources", "built-in-database", "username"]), [?EXAMPLE_USERNAME]), + {ok, 200, Request1} = request(get, uri(["authorization", "sources", "built-in-database", "username"]), []), + {ok, 200, Request2} = request(get, uri(["authorization", "sources", "built-in-database", "username", "user1"]), []), + [#{<<"username">> := <<"user1">>, <<"rules">> := Rules1}] = jsx:decode(Request1), + #{<<"username">> := <<"user1">>, <<"rules">> := Rules1} = jsx:decode(Request2), + ?assertEqual(3, length(Rules1)), + + {ok, 204, _} = request(put, uri(["authorization", "sources", "built-in-database", "username", "user1"]), ?EXAMPLE_USERNAME#{rules => []}), + {ok, 200, Request3} = request(get, uri(["authorization", "sources", "built-in-database", "username", "user1"]), []), + #{<<"username">> := <<"user1">>, <<"rules">> := Rules2} = jsx:decode(Request3), + ?assertEqual(0, length(Rules2)), + + {ok, 204, _} = request(delete, uri(["authorization", "sources", "built-in-database", "username", "user1"]), []), + {ok, 404, _} = request(get, uri(["authorization", "sources", "built-in-database", "username", "user1"]), []), + + {ok, 204, _} = request(post, uri(["authorization", "sources", "built-in-database", "clientid"]), [?EXAMPLE_CLIENTID]), + {ok, 200, Request4} = request(get, uri(["authorization", "sources", "built-in-database", "clientid"]), []), + {ok, 200, Request5} = request(get, uri(["authorization", "sources", "built-in-database", "clientid", "client1"]), []), + [#{<<"clientid">> := <<"client1">>, <<"rules">> := Rules3}] = jsx:decode(Request4), + #{<<"clientid">> := <<"client1">>, <<"rules">> := Rules3} = jsx:decode(Request5), + ?assertEqual(3, length(Rules3)), + + {ok, 204, _} = request(put, uri(["authorization", "sources", "built-in-database", "clientid", "client1"]), ?EXAMPLE_CLIENTID#{rules => []}), + {ok, 200, Request6} = request(get, uri(["authorization", "sources", "built-in-database", "clientid", "client1"]), []), + #{<<"clientid">> := <<"client1">>, <<"rules">> := Rules4} = jsx:decode(Request6), + ?assertEqual(0, length(Rules4)), + + {ok, 204, _} = request(delete, uri(["authorization", "sources", "built-in-database", "clientid", "client1"]), []), + {ok, 404, _} = request(get, uri(["authorization", "sources", "built-in-database", "clientid", "client1"]), []), + + {ok, 204, _} = request(put, uri(["authorization", "sources", "built-in-database", "all"]), ?EXAMPLE_ALL), + {ok, 200, Request7} = request(get, uri(["authorization", "sources", "built-in-database", "all"]), []), + [#{<<"rules">> := Rules5}] = jsx:decode(Request7), + ?assertEqual(3, length(Rules5)), + + {ok, 204, _} = request(put, uri(["authorization", "sources", "built-in-database", "all"]), ?EXAMPLE_ALL#{rules => []}), + {ok, 200, Request8} = request(get, uri(["authorization", "sources", "built-in-database", "all"]), []), + [#{<<"rules">> := Rules6}] = jsx:decode(Request8), + ?assertEqual(0, length(Rules6)), + + {ok, 204, _} = request(post, uri(["authorization", "sources", "built-in-database", "username"]), [ #{username => N, rules => []} || N <- lists:seq(1, 20) ]), + {ok, 200, Request9} = request(get, uri(["authorization", "sources", "built-in-database", "username?page=2&limit=5"]), []), + #{<<"data">> := Data1} = jsx:decode(Request9), + ?assertEqual(5, length(Data1)), + + {ok, 204, _} = request(post, uri(["authorization", "sources", "built-in-database", "clientid"]), [ #{clientid => N, rules => []} || N <- lists:seq(1, 20) ]), + {ok, 200, Request10} = request(get, uri(["authorization", "sources", "built-in-database", "clientid?limit=5"]), []), + ?assertEqual(5, length(jsx:decode(Request10))), + + {ok, 400, _} = request(delete, uri(["authorization", "sources", "built-in-database", "purge-all"]), []), + {ok, 204, _} = request(put, uri(["authorization", "sources", "built-in-database"]), #{<<"enable">> => false}), + {ok, 204, _} = request(delete, uri(["authorization", "sources", "built-in-database", "purge-all"]), []), + ?assertEqual([], mnesia:dirty_all_keys(?ACL_TABLE)), + + ok. + +%%-------------------------------------------------------------------- +%% HTTP Request +%%-------------------------------------------------------------------- + +request(Method, Url, Body) -> + Request = case Body of + [] -> {Url, [auth_header_()]}; + _ -> {Url, [auth_header_()], "application/json", jsx:encode(Body)} + end, + ct:pal("Method: ~p, Request: ~p", [Method, Request]), + case httpc:request(Method, Request, [], [{body_format, binary}]) of + {error, socket_closed_remotely} -> + {error, socket_closed_remotely}; + {ok, {{"HTTP/1.1", Code, _}, _Headers, Return} } -> + {ok, Code, Return}; + {ok, {Reason, _, _}} -> + {error, Reason} + end. + +uri() -> uri([]). +uri(Parts) when is_list(Parts) -> + NParts = [E || E <- Parts], + ?HOST ++ filename:join([?BASE_PATH, ?API_VERSION | NParts]). + +get_sources(Result) -> jsx:decode(Result). + +auth_header_() -> + Username = <<"admin">>, + Password = <<"public">>, + {ok, Token} = emqx_dashboard_admin:sign_token(Username, Password), + {"Authorization", "Bearer " ++ binary_to_list(Token)}. diff --git a/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl index a3c6e6e50..86b347b98 100644 --- a/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl @@ -42,7 +42,7 @@ <<"url">> => <<"https://fake.com:443/">>, <<"headers">> => #{}, <<"method">> => <<"get">>, - <<"request_timeout">> => 5000 + <<"request_timeout">> => <<"5s">> }). -define(SOURCE2, #{<<"type">> => <<"mongodb">>, <<"enable">> => true, @@ -96,7 +96,8 @@ }). all() -> - emqx_ct:all(?MODULE). + []. %% Todo: Waiting for @terry-xiaoyu to fix the config_not_found error + % emqx_ct:all(?MODULE). groups() -> []. diff --git a/apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl b/apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl new file mode 100644 index 000000000..8b221d3e7 --- /dev/null +++ b/apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl @@ -0,0 +1,109 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authz_mnesia_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include("emqx_authz.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-define(CONF_DEFAULT, <<"authorization: {sources: []}">>). + +all() -> + emqx_ct:all(?MODULE). + +groups() -> + []. + +init_per_suite(Config) -> + meck:new(emqx_schema, [non_strict, passthrough, no_history, no_link]), + meck:expect(emqx_schema, fields, fun("authorization") -> + meck:passthrough(["authorization"]) ++ + emqx_authz_schema:fields("authorization"); + (F) -> meck:passthrough([F]) + end), + + ok = emqx_config:init_load(emqx_authz_schema, ?CONF_DEFAULT), + ok = emqx_ct_helpers:start_apps([emqx_authz]), + + {ok, _} = emqx:update_config([authorization, cache, enable], false), + {ok, _} = emqx:update_config([authorization, no_match], deny), + Rules = [#{<<"type">> => <<"built-in-database">>}], + {ok, _} = emqx_authz:update(replace, Rules), + Config. + +end_per_suite(_Config) -> + {ok, _} = emqx_authz:update(replace, []), + emqx_ct_helpers:stop_apps([emqx_authz]), + meck:unload(emqx_schema), + ok. + +init_per_testcase(t_authz, Config) -> + mnesia:transaction(fun ekka_mnesia:dirty_write/1, [#emqx_acl{who = {?ACL_TABLE_USERNAME, <<"test_username">>}, + rules = [{allow, publish, <<"test/%u">>}, + {allow, subscribe, <<"eq #">>} + ] + }]), + mnesia:transaction(fun ekka_mnesia:dirty_write/1, [#emqx_acl{who = {?ACL_TABLE_CLIENTID, <<"test_clientid">>}, + rules = [{allow, publish, <<"test/%c">>}, + {deny, subscribe, <<"eq #">>} + ] + }]), + mnesia:transaction(fun ekka_mnesia:dirty_write/1, [#emqx_acl{who = ?ACL_TABLE_ALL, + rules = [{deny, all, <<"#">>}] + }]), + Config; +init_per_testcase(_, Config) -> Config. + +end_per_testcase(t_authz, Config) -> + [ ekka_mnesia:dirty_delete(?ACL_TABLE, K) || K <- mnesia:dirty_all_keys(?ACL_TABLE)], + Config; +end_per_testcase(_, Config) -> Config. + +%%------------------------------------------------------------------------------ +%% Testcases +%%------------------------------------------------------------------------------ + +t_authz(_) -> + ClientInfo1 = #{clientid => <<"test">>, + username => <<"test">>, + peerhost => {127,0,0,1}, + listener => {tcp, default} + }, + ClientInfo2 = #{clientid => <<"fake_clientid">>, + username => <<"test_username">>, + peerhost => {127,0,0,1}, + listener => {tcp, default} + }, + ClientInfo3 = #{clientid => <<"test_clientid">>, + username => <<"fake_username">>, + peerhost => {127,0,0,1}, + listener => {tcp, default} + }, + + ?assertEqual(deny, emqx_access_control:authorize(ClientInfo1, subscribe, <<"#">>)), + ?assertEqual(deny, emqx_access_control:authorize(ClientInfo1, publish, <<"#">>)), + + ?assertEqual(allow, emqx_access_control:authorize(ClientInfo2, publish, <<"test/test_username">>)), + ?assertEqual(allow, emqx_access_control:authorize(ClientInfo2, subscribe, <<"#">>)), + + ?assertEqual(allow, emqx_access_control:authorize(ClientInfo3, publish, <<"test/test_clientid">>)), + ?assertEqual(deny, emqx_access_control:authorize(ClientInfo3, subscribe, <<"#">>)), + + ok. + diff --git a/apps/emqx_authz/test/emqx_authz_rule_SUITE.erl b/apps/emqx_authz/test/emqx_authz_rule_SUITE.erl index c38d99cba..3c7e314cd 100644 --- a/apps/emqx_authz/test/emqx_authz_rule_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_rule_SUITE.erl @@ -22,11 +22,11 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). --define(SOURCE1, {deny, all, all, ["#"]}). +-define(SOURCE1, {deny, all}). -define(SOURCE2, {allow, {ipaddr, "127.0.0.1"}, all, [{eq, "#"}, {eq, "+"}]}). -define(SOURCE3, {allow, {ipaddrs, ["127.0.0.1", "192.168.1.0/24"]}, subscribe, ["%c"]}). --define(SOURCE4, {allow, {'and', [{clientid, "^test?"}, {username, "^test?"}]}, publish, ["topic/test"]}). --define(SOURCE5, {allow, {'or', [{username, "^test"}, {clientid, "test?"}]}, publish, ["%u", "%c"]}). +-define(SOURCE4, {allow, {'and', [{client, "test"}, {user, "test"}]}, publish, ["topic/test"]}). +-define(SOURCE5, {allow, {'or', [{username, {re, "^test"}}, {clientid, {re, "test?"}}]}, publish, ["%u", "%c"]}). all() -> emqx_ct:all(?MODULE). @@ -52,7 +52,7 @@ t_compile(_) -> }, emqx_authz_rule:compile(?SOURCE3)), ?assertMatch({allow, - {'and', [{clientid, {re_pattern, _, _, _, _}}, {username, {re_pattern, _, _, _, _}}]}, + {'and', [{clientid, {eq, <<"test">>}}, {username, {eq, <<"test">>}}]}, publish, [[<<"topic">>, <<"test">>]] }, emqx_authz_rule:compile(?SOURCE4)), diff --git a/apps/emqx_bridge/etc/emqx_bridge.conf b/apps/emqx_bridge/etc/emqx_bridge.conf index e8af40341..f26172ef6 100644 --- a/apps/emqx_bridge/etc/emqx_bridge.conf +++ b/apps/emqx_bridge/etc/emqx_bridge.conf @@ -45,3 +45,30 @@ # retain = false # } #} +# +#bridges.http.my_http_bridge { +# base_url: "http://localhost:9901" +# connect_timeout: "30s" +# max_retries: 3 +# retry_interval = "10s" +# pool_type = "random" +# pool_size = 4 +# enable_pipelining = true +# ssl { +# enable = false +# keyfile = "{{ platform_etc_dir }}/certs/client-key.pem" +# certfile = "{{ platform_etc_dir }}/certs/client-cert.pem" +# cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" +# } +# egress_channels.post_messages { +# subscribe_local_topic = "emqx_http/#" +# request_timeout: "30s" +# ## following config entries can use placehodler variables +# method = post +# path = "/messages/${topic}" +# body = "${payload}" +# headers { +# "content-type": "application/json" +# } +# } +#} diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index e3458adca..351e6aeca 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -15,9 +15,15 @@ %%-------------------------------------------------------------------- -module(emqx_bridge). -behaviour(emqx_config_handler). +-include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/logger.hrl"). -export([post_config_update/4]). +-export([reload_hook/0, unload_hook/0]). + +-export([on_message_publish/1]). + -export([ load_bridges/0 , get_bridge/2 , get_bridge/3 @@ -28,6 +34,7 @@ , start_bridge/2 , stop_bridge/2 , restart_bridge/2 + , send_message/2 ]). -export([ config_key_path/0 @@ -38,24 +45,57 @@ , resource_id/1 , resource_id/2 , parse_bridge_id/1 + , channel_id/4 + , parse_channel_id/1 ]). +reload_hook() -> + unload_hook(), + Bridges = emqx:get_config([bridges], #{}), + lists:foreach(fun({_Type, Bridge}) -> + lists:foreach(fun({_Name, BridgeConf}) -> + load_hook(BridgeConf) + end, maps:to_list(Bridge)) + end, maps:to_list(Bridges)). + +load_hook(#{egress_channels := Channels}) -> + case has_subscribe_local_topic(Channels) of + true -> ok; + false -> emqx_hooks:put('message.publish', {?MODULE, on_message_publish, []}) + end; +load_hook(_Conf) -> ok. + +unload_hook() -> + ok = emqx_hooks:del('message.publish', {?MODULE, on_message_publish}). + +on_message_publish(Message = #message{topic = Topic, flags = Flags}) -> + case maps:get(sys, Flags, false) of + false -> + ChannelIds = get_matched_channels(Topic), + lists:foreach(fun(ChannelId) -> + send_message(ChannelId, emqx_message:to_map(Message)) + end, ChannelIds); + true -> ok + end, + {ok, Message}. + +%% TODO: remove this clause, treat mqtt bridges the same as other bridges +send_message(ChannelId, Message) -> + {BridgeType, BridgeName, _, _} = parse_channel_id(ChannelId), + ResId = emqx_bridge:resource_id(BridgeType, BridgeName), + do_send_message(ResId, ChannelId, Message). + +do_send_message(ResId, ChannelId, Message) -> + emqx_resource:query(ResId, {send_message, ChannelId, Message}). + config_key_path() -> [bridges]. resource_type(mqtt) -> emqx_connector_mqtt; -resource_type(mysql) -> emqx_connector_mysql; -resource_type(pgsql) -> emqx_connector_pgsql; -resource_type(mongo) -> emqx_connector_mongo; -resource_type(redis) -> emqx_connector_redis; -resource_type(ldap) -> emqx_connector_ldap. +resource_type(http) -> emqx_connector_http. bridge_type(emqx_connector_mqtt) -> mqtt; -bridge_type(emqx_connector_mysql) -> mysql; -bridge_type(emqx_connector_pgsql) -> pgsql; -bridge_type(emqx_connector_mongo) -> mongo; -bridge_type(emqx_connector_redis) -> redis; -bridge_type(emqx_connector_ldap) -> ldap. +bridge_type(emqx_connector_http) -> http. post_config_update(_Req, NewConf, OldConf, _AppEnv) -> #{added := Added, removed := Removed, changed := Updated} @@ -100,11 +140,23 @@ bridge_id(BridgeType, BridgeName) -> <>. parse_bridge_id(BridgeId) -> - try - [Type, Name] = string:split(str(BridgeId), ":", leading), - {list_to_existing_atom(Type), list_to_atom(Name)} - catch - _ : _ -> error({invalid_bridge_id, BridgeId}) + case string:split(bin(BridgeId), ":", all) of + [Type, Name] -> {binary_to_atom(Type, utf8), binary_to_atom(Name, utf8)}; + _ -> error({invalid_bridge_id, BridgeId}) + end. + +channel_id(BridgeType, BridgeName, ChannelType, ChannelName) -> + BType = bin(BridgeType), + BName = bin(BridgeName), + CType = bin(ChannelType), + CName = bin(ChannelName), + <>. + +parse_channel_id(ChannelId) -> + case string:split(bin(ChannelId), ":", all) of + [BridgeType, BridgeName, ChannelType, ChannelName] -> + {BridgeType, BridgeName, ChannelType, ChannelName}; + _ -> error({invalid_bridge_id, ChannelId}) end. list_bridges() -> @@ -137,7 +189,8 @@ restart_bridge(Type, Name) -> emqx_resource:restart(resource_id(Type, Name)). create_bridge(Type, Name, Conf) -> - logger:info("create ~p bridge ~p use config: ~p", [Type, Name, Conf]), + ?SLOG(info, #{msg => "create bridge", type => Type, name => Name, + config => Conf}), ResId = resource_id(Type, Name), case emqx_resource:create(ResId, emqx_bridge:resource_type(Type), Conf) of @@ -158,12 +211,13 @@ update_bridge(Type, Name, {_OldConf, Conf}) -> %% `egress_channels` are changed, then we should not restart the bridge, we only restart/start %% the channels. %% - logger:info("update ~p bridge ~p use config: ~p", [Type, Name, Conf]), + ?SLOG(info, #{msg => "update bridge", type => Type, name => Name, + config => Conf}), emqx_resource:recreate(resource_id(Type, Name), emqx_bridge:resource_type(Type), Conf, []). remove_bridge(Type, Name, _Conf) -> - logger:info("remove ~p bridge ~p", [Type, Name]), + ?SLOG(info, #{msg => "remove bridge", type => Type, name => Name}), case emqx_resource:remove(resource_id(Type, Name)) of ok -> ok; {error, not_found} -> ok; @@ -184,13 +238,35 @@ flatten_confs(Conf0) -> do_flatten_confs(Type, Conf0) -> [{{Type, Name}, Conf} || {Name, Conf} <- maps:to_list(Conf0)]. +has_subscribe_local_topic(Channels) -> + lists:any(fun (#{subscribe_local_topic := _}) -> true; + (_) -> false + end, maps:to_list(Channels)). + +get_matched_channels(Topic) -> + Bridges = emqx:get_config([bridges], #{}), + maps:fold(fun + %% TODO: also trigger 'message.publish' for mqtt bridges. + (mqtt, _Conf, Acc0) -> Acc0; + (BType, Conf, Acc0) -> + maps:fold(fun + (BName, #{egress_channels := Channels}, Acc1) -> + do_get_matched_channels(Topic, Channels, BType, BName, egress_channels) + ++ Acc1; + (_Name, _BridgeConf, Acc1) -> Acc1 + end, Acc0, Conf) + end, [], Bridges). + +do_get_matched_channels(Topic, Channels, BType, BName, CType) -> + maps:fold(fun + (ChannName, #{subscribe_local_topic := Filter}, Acc) -> + case emqx_topic:match(Topic, Filter) of + true -> [channel_id(BType, BName, CType, ChannName) | Acc]; + false -> Acc + end; + (_ChannName, _ChannConf, Acc) -> Acc + end, [], Channels). + bin(Bin) when is_binary(Bin) -> Bin; bin(Str) when is_list(Str) -> list_to_binary(Str); bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8). - -str(A) when is_atom(A) -> - atom_to_list(A); -str(B) when is_binary(B) -> - binary_to_list(B); -str(S) when is_list(S) -> - S. diff --git a/apps/emqx_bridge/src/emqx_bridge_app.erl b/apps/emqx_bridge/src/emqx_bridge_app.erl index 004b32787..8cb325e20 100644 --- a/apps/emqx_bridge/src/emqx_bridge_app.erl +++ b/apps/emqx_bridge/src/emqx_bridge_app.erl @@ -22,10 +22,12 @@ start(_StartType, _StartArgs) -> {ok, Sup} = emqx_bridge_sup:start_link(), ok = emqx_bridge:load_bridges(), + ok = emqx_bridge:reload_hook(), emqx_config_handler:add_handler(emqx_bridge:config_key_path(), emqx_bridge), {ok, Sup}. stop(_State) -> + ok = emqx_bridge:unload_hook(), ok. %% internal functions \ No newline at end of file diff --git a/apps/emqx_bridge/src/emqx_bridge_schema.erl b/apps/emqx_bridge/src/emqx_bridge_schema.erl index 87eb40372..2072d15ec 100644 --- a/apps/emqx_bridge/src/emqx_bridge_schema.erl +++ b/apps/emqx_bridge/src/emqx_bridge_schema.erl @@ -1,5 +1,7 @@ -module(emqx_bridge_schema). +-include_lib("typerefl/include/types.hrl"). + -export([roots/0, fields/1]). %%====================================================================================== @@ -8,7 +10,16 @@ roots() -> [bridges]. fields(bridges) -> - [{mqtt, hoconsc:mk(hoconsc:map(name, hoconsc:ref(?MODULE, "mqtt_bridge")))}]; + [ {mqtt, hoconsc:mk(hoconsc:map(name, hoconsc:ref(?MODULE, "mqtt_bridge")))} + , {http, hoconsc:mk(hoconsc:map(name, hoconsc:ref(?MODULE, "http_bridge")))} + ]; fields("mqtt_bridge") -> - emqx_connector_mqtt:fields("config"). + emqx_connector_mqtt:fields("config"); + +fields("http_bridge") -> + emqx_connector_http:fields(config) ++ http_channels(). + +http_channels() -> + [{egress_channels, hoconsc:mk(hoconsc:map(id, + hoconsc:ref(emqx_connector_http, "http_request")))}]. diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl index 0f8c23986..92c7c6d64 100644 --- a/apps/emqx_connector/src/emqx_connector_http.erl +++ b/apps/emqx_connector/src/emqx_connector_http.erl @@ -21,6 +21,8 @@ -include_lib("typerefl/include/types.hrl"). -include_lib("emqx_resource/include/emqx_resource_behaviour.hrl"). +-include_lib("emqx/include/logger.hrl"). + %% callbacks of behaviour emqx_resource -export([ on_start/2 , on_stop/2 @@ -38,7 +40,7 @@ -export([ check_ssl_opts/2 ]). --type connect_timeout() :: non_neg_integer() | infinity. +-type connect_timeout() :: emqx_schema:duration() | infinity. -type pool_type() :: random | hash. -reflect_type([ connect_timeout/0 @@ -50,6 +52,22 @@ roots() -> [{config, #{type => hoconsc:ref(?MODULE, config)}}]. +fields("http_request") -> + [ {subscribe_local_topic, hoconsc:mk(binary())} + , {method, hoconsc:mk(method(), #{default => post})} + , {path, hoconsc:mk(binary(), #{default => <<"">>})} + , {headers, hoconsc:mk(map(), + #{default => #{ + <<"accept">> => <<"application/json">>, + <<"cache-control">> => <<"no-cache">>, + <<"connection">> => <<"keep-alive">>, + <<"content-type">> => <<"application/json">>, + <<"keep-alive">> => <<"timeout=5">>}}) + } + , {body, hoconsc:mk(binary(), #{default => <<"${payload}">>})} + , {request_timeout, hoconsc:mk(emqx_schema:duration_ms(), #{default => <<"30s">>})} + ]; + fields(config) -> [ {base_url, fun base_url/1} , {connect_timeout, fun connect_timeout/1} @@ -60,6 +78,13 @@ fields(config) -> , {enable_pipelining, fun enable_pipelining/1} ] ++ emqx_connector_schema_lib:ssl_fields(). +method() -> + hoconsc:union([ typerefl:atom(post) + , typerefl:atom(put) + , typerefl:atom(get) + , typerefl:atom(delete) + ]). + validations() -> [ {check_ssl_opts, fun check_ssl_opts/1} ]. @@ -71,16 +96,16 @@ base_url(validator) -> fun(#{query := _Query}) -> end; base_url(_) -> undefined. -connect_timeout(type) -> connect_timeout(); -connect_timeout(default) -> 5000; +connect_timeout(type) -> emqx_schema:duration_ms(); +connect_timeout(default) -> "5s"; connect_timeout(_) -> undefined. max_retries(type) -> non_neg_integer(); max_retries(default) -> 5; max_retries(_) -> undefined. -retry_interval(type) -> non_neg_integer(); -retry_interval(default) -> 1000; +retry_interval(type) -> emqx_schema:duration(); +retry_interval(default) -> "1s"; retry_interval(_) -> undefined. pool_type(type) -> pool_type(); @@ -105,13 +130,14 @@ on_start(InstId, #{base_url := #{scheme := Scheme, retry_interval := RetryInterval, pool_type := PoolType, pool_size := PoolSize} = Config) -> - logger:info("starting http connector: ~p, config: ~p", [InstId, Config]), + ?SLOG(info, #{msg => "starting http connector", + connector => InstId, config => Config}), {Transport, TransportOpts} = case Scheme of http -> {tcp, []}; https -> SSLOpts = emqx_plugin_libs_ssl:save_files_return_opts( - maps:get(ssl_opts, Config), "connectors", InstId), + maps:get(ssl, Config), "connectors", InstId), {tls, SSLOpts} end, NTransportOpts = emqx_misc:ipv6_probe(TransportOpts), @@ -126,30 +152,51 @@ on_start(InstId, #{base_url := #{scheme := Scheme, , {transport, Transport} , {transport_opts, NTransportOpts}], PoolName = emqx_plugin_libs_pool:pool_name(InstId), - {ok, _} = ehttpc_sup:start_pool(PoolName, PoolOpts), - {ok, #{pool_name => PoolName, - host => Host, - port => Port, - base_path => BasePath}}. + State = #{ + pool_name => PoolName, + host => Host, + port => Port, + base_path => BasePath, + channels => preproc_channels(InstId, Config) + }, + case ehttpc_sup:start_pool(PoolName, PoolOpts) of + {ok, _} -> {ok, State}; + {error, {already_started, _}} -> {ok, State}; + {error, Reason} -> + {error, Reason} + end. on_stop(InstId, #{pool_name := PoolName}) -> - logger:info("stopping http connector: ~p", [InstId]), + ?SLOG(info, #{msg => "stopping http connector", + connector => InstId}), ehttpc_sup:stop_pool(PoolName). +on_query(InstId, {send_message, ChannelId, Msg}, AfterQuery, #{channels := Channels} = State) -> + case maps:find(ChannelId, Channels) of + error -> ?SLOG(error, #{msg => "channel not found", channel_id => ChannelId}); + {ok, ChannConf} -> + #{method := Method, path := Path, body := Body, headers := Headers, + request_timeout := Timeout} = proc_channel_conf(ChannConf, Msg), + on_query(InstId, {Method, {Path, Headers, Body}, Timeout}, AfterQuery, State) + end; on_query(InstId, {Method, Request}, AfterQuery, State) -> on_query(InstId, {undefined, Method, Request, 5000}, AfterQuery, State); on_query(InstId, {Method, Request, Timeout}, AfterQuery, State) -> on_query(InstId, {undefined, Method, Request, Timeout}, AfterQuery, State); -on_query(InstId, {KeyOrNum, Method, Request, Timeout}, AfterQuery, #{pool_name := PoolName, - base_path := BasePath} = State) -> - logger:debug("http connector ~p received request: ~p, at state: ~p", [InstId, Request, State]), +on_query(InstId, {KeyOrNum, Method, Request, Timeout}, AfterQuery, + #{pool_name := PoolName, base_path := BasePath} = State) -> + ?SLOG(debug, #{msg => "http connector received request", + request => Request, connector => InstId, + state => State}), NRequest = update_path(BasePath, Request), case Result = ehttpc:request(case KeyOrNum of undefined -> PoolName; _ -> {PoolName, KeyOrNum} end, Method, NRequest, Timeout) of {error, Reason} -> - logger:debug("http connector ~p do reqeust failed, sql: ~p, reason: ~p", [InstId, NRequest, Reason]), + ?SLOG(error, #{msg => "http connector do reqeust failed", + request => NRequest, reason => Reason, + connector => InstId}), emqx_resource:query_failed(AfterQuery); _ -> emqx_resource:query_success(AfterQuery) @@ -169,6 +216,54 @@ on_health_check(_InstId, #{host := Host, port := Port} = State) -> %% Internal functions %%-------------------------------------------------------------------- +preproc_channels(<<"bridge:", BridgeId/binary>>, Config) -> + {BridgeType, BridgeName} = emqx_bridge:parse_bridge_id(BridgeId), + maps:fold(fun(ChannName, ChannConf, Acc) -> + Acc#{emqx_bridge:channel_id(BridgeType, BridgeName, egress_channels, ChannName) => + preproc_channel_conf(ChannConf)} + end, #{}, maps:get(egress_channels, Config, #{})); +preproc_channels(_InstId, _Config) -> + #{}. + +preproc_channel_conf(#{ + method := Method, + path := Path, + body := Body, + headers := Headers} = Conf) -> + Conf#{ method => emqx_plugin_libs_rule:preproc_tmpl(bin(Method)) + , path => emqx_plugin_libs_rule:preproc_tmpl(Path) + , body => emqx_plugin_libs_rule:preproc_tmpl(Body) + , headers => preproc_headers(Headers) + }. + +preproc_headers(Headers) -> + maps:fold(fun(K, V, Acc) -> + Acc#{emqx_plugin_libs_rule:preproc_tmpl(bin(K)) => + emqx_plugin_libs_rule:preproc_tmpl(bin(V))} + end, #{}, Headers). + +proc_channel_conf(#{ + method := MethodTks, + path := PathTks, + body := BodyTks, + headers := HeadersTks} = Conf, Msg) -> + Conf#{ method => make_method(emqx_plugin_libs_rule:proc_tmpl(MethodTks, Msg)) + , path => emqx_plugin_libs_rule:proc_tmpl(PathTks, Msg) + , body => emqx_plugin_libs_rule:proc_tmpl(BodyTks, Msg) + , headers => maps:to_list(proc_headers(HeadersTks, Msg)) + }. + +proc_headers(HeaderTks, Msg) -> + maps:fold(fun(K, V, Acc) -> + Acc#{emqx_plugin_libs_rule:proc_tmpl(K, Msg) => + emqx_plugin_libs_rule:proc_tmpl(V, Msg)} + end, #{}, HeaderTks). + +make_method(M) when M == <<"POST">>; M == <<"post">> -> post; +make_method(M) when M == <<"PUT">>; M == <<"put">> -> put; +make_method(M) when M == <<"GET">>; M == <<"get">> -> get; +make_method(M) when M == <<"DELETE">>; M == <<"delete">> -> delete. + check_ssl_opts(Conf) -> check_ssl_opts("base_url", Conf). @@ -185,3 +280,10 @@ update_path(BasePath, {Path, Headers}) -> {filename:join(BasePath, Path), Headers}; update_path(BasePath, {Path, Headers, Body}) -> {filename:join(BasePath, Path), Headers, Body}. + +bin(Bin) when is_binary(Bin) -> + Bin; +bin(Str) when is_list(Str) -> + list_to_binary(Str); +bin(Atom) when is_atom(Atom) -> + atom_to_binary(Atom, utf8). diff --git a/apps/emqx_connector/src/emqx_connector_ldap.erl b/apps/emqx_connector/src/emqx_connector_ldap.erl index fadf7f56f..85e42b0f3 100644 --- a/apps/emqx_connector/src/emqx_connector_ldap.erl +++ b/apps/emqx_connector/src/emqx_connector_ldap.erl @@ -18,6 +18,7 @@ -include("emqx_connector.hrl"). -include_lib("typerefl/include/types.hrl"). -include_lib("emqx_resource/include/emqx_resource_behaviour.hrl"). +-include_lib("emqx/include/logger.hrl"). -export([roots/0, fields/1]). @@ -53,7 +54,8 @@ on_start(InstId, #{servers := Servers0, pool_size := PoolSize, auto_reconnect := AutoReconn, ssl := SSL} = Config) -> - logger:info("starting ldap connector: ~p, config: ~p", [InstId, Config]), + ?SLOG(info, #{msg => "starting ldap connector", + connector => InstId, config => Config}), Servers = [begin proplists:get_value(host, S) end || S <- Servers0], SslOpts = case maps:get(enable, SSL) of true -> @@ -75,14 +77,20 @@ on_start(InstId, #{servers := Servers0, {ok, #{poolname => PoolName}}. on_stop(InstId, #{poolname := PoolName}) -> - logger:info("stopping ldap connector: ~p", [InstId]), + ?SLOG(info, #{msg => "stopping ldap connector", + connector => InstId}), emqx_plugin_libs_pool:stop_pool(PoolName). on_query(InstId, {search, Base, Filter, Attributes}, AfterQuery, #{poolname := PoolName} = State) -> - logger:debug("ldap connector ~p received request: ~p, at state: ~p", [InstId, {Base, Filter, Attributes}, State]), + Request = {Base, Filter, Attributes}, + ?SLOG(debug, #{msg => "ldap connector received request", + request => Request, connector => InstId, + state => State}), case Result = ecpool:pick_and_do(PoolName, {?MODULE, search, [Base, Filter, Attributes]}, no_handover) of {error, Reason} -> - logger:debug("ldap connector ~p do request failed, request: ~p, reason: ~p", [InstId, {Base, Filter, Attributes}, Reason]), + ?SLOG(error, #{msg => "ldap connector do request failed", + request => Request, connector => InstId, + reason => Reason}), emqx_resource:query_failed(AfterQuery); _ -> emqx_resource:query_success(AfterQuery) diff --git a/apps/emqx_connector/src/emqx_connector_mongo.erl b/apps/emqx_connector/src/emqx_connector_mongo.erl index 906b57fb3..0cb40adbb 100644 --- a/apps/emqx_connector/src/emqx_connector_mongo.erl +++ b/apps/emqx_connector/src/emqx_connector_mongo.erl @@ -18,6 +18,7 @@ -include("emqx_connector.hrl"). -include_lib("typerefl/include/types.hrl"). -include_lib("emqx_resource/include/emqx_resource_behaviour.hrl"). +-include_lib("emqx/include/logger.hrl"). -type server() :: emqx_schema:ip_port(). -reflect_type([server/0]). @@ -93,7 +94,8 @@ on_jsonify(Config) -> %% =================================================================== on_start(InstId, Config = #{server := Server, mongo_type := single}) -> - logger:info("starting mongodb connector: ~p, config: ~p", [InstId, Config]), + ?SLOG(info, #{msg => "starting mongodb single connector", + connector => InstId, config => Config}), Opts = [{type, single}, {hosts, [emqx_connector_schema_lib:ip_port_to_string(Server)]} ], @@ -102,7 +104,8 @@ on_start(InstId, Config = #{server := Server, on_start(InstId, Config = #{servers := Servers, mongo_type := rs, replica_set_name := RsName}) -> - logger:info("starting mongodb connector: ~p, config: ~p", [InstId, Config]), + ?SLOG(info, #{msg => "starting mongodb rs connector", + connector => InstId, config => Config}), Opts = [{type, {rs, RsName}}, {hosts, [emqx_connector_schema_lib:ip_port_to_string(S) || S <- Servers]} @@ -111,7 +114,8 @@ on_start(InstId, Config = #{servers := Servers, on_start(InstId, Config = #{servers := Servers, mongo_type := sharded}) -> - logger:info("starting mongodb connector: ~p, config: ~p", [InstId, Config]), + ?SLOG(info, #{msg => "starting mongodb sharded connector", + connector => InstId, config => Config}), Opts = [{type, sharded}, {hosts, [emqx_connector_schema_lib:ip_port_to_string(S) || S <- Servers]} @@ -119,14 +123,20 @@ on_start(InstId, Config = #{servers := Servers, do_start(InstId, Opts, Config). on_stop(InstId, #{poolname := PoolName}) -> - logger:info("stopping mongodb connector: ~p", [InstId]), + ?SLOG(info, #{msg => "stopping mongodb connector", + connector => InstId}), emqx_plugin_libs_pool:stop_pool(PoolName). on_query(InstId, {Action, Collection, Selector, Docs}, AfterQuery, #{poolname := PoolName} = State) -> - logger:debug("mongodb connector ~p received request: ~p, at state: ~p", [InstId, {Action, Collection, Selector, Docs}, State]), + Request = {Action, Collection, Selector, Docs}, + ?SLOG(debug, #{msg => "mongodb connector received request", + request => Request, connector => InstId, + state => State}), case ecpool:pick_and_do(PoolName, {?MODULE, mongo_query, [Action, Collection, Selector, Docs]}, no_handover) of {error, Reason} -> - logger:debug("mongodb connector ~p do sql query failed, request: ~p, reason: ~p", [InstId, {Action, Collection, Selector, Docs}, Reason]), + ?SLOG(error, #{msg => "mongodb connector do query failed", + request => Request, reason => Reason, + connector => InstId}), emqx_resource:query_failed(AfterQuery), {error, Reason}; {ok, Cursor} when is_pid(Cursor) -> diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl index a03c888d3..a4527984a 100644 --- a/apps/emqx_connector/src/emqx_connector_mqtt.erl +++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl @@ -17,6 +17,7 @@ -include_lib("typerefl/include/types.hrl"). -include_lib("emqx_resource/include/emqx_resource_behaviour.hrl"). +-include_lib("emqx/include/logger.hrl"). -behaviour(supervisor). @@ -88,13 +89,15 @@ drop_bridge(Name) -> %% =================================================================== %% When use this bridge as a data source, ?MODULE:on_message_received/2 will be called %% if the bridge received msgs from the remote broker. -on_message_received(Msg, ChannelName) -> - emqx:run_hook(ChannelName, [Msg]). +on_message_received(Msg, ChannId) -> + Name = atom_to_binary(ChannId, utf8), + emqx:run_hook(<<"$bridges/", Name/binary>>, [Msg]). %% =================================================================== on_start(InstId, Conf) -> - logger:info("starting mqtt connector: ~p, ~p", [InstId, Conf]), - NamePrefix = binary_to_list(InstId), + ?SLOG(info, #{msg => "starting mqtt connector", + connector => InstId, config => Conf}), + "bridge:" ++ NamePrefix = binary_to_list(InstId), BasicConf = basic_config(Conf), InitRes = {ok, #{name_prefix => NamePrefix, baisc_conf => BasicConf, channels => []}}, InOutConfigs = taged_map_list(ingress_channels, maps:get(ingress_channels, Conf, #{})) @@ -110,7 +113,8 @@ on_start(InstId, Conf) -> end, InitRes, InOutConfigs). on_stop(InstId, #{channels := NameList}) -> - logger:info("stopping mqtt connector: ~p", [InstId]), + ?SLOG(info, #{msg => "stopping mqtt connector", + connector => InstId}), lists:foreach(fun(Name) -> remove_channel(Name) end, NameList). @@ -120,9 +124,10 @@ on_stop(InstId, #{channels := NameList}) -> on_query(_InstId, {create_channel, Conf}, _AfterQuery, #{name_prefix := Prefix, baisc_conf := BasicConf}) -> create_channel(Conf, Prefix, BasicConf); -on_query(_InstId, {send_to_remote, ChannelName, Msg}, _AfterQuery, _State) -> - logger:debug("send msg to remote node on channel: ~p, msg: ~p", [ChannelName, Msg]), - emqx_connector_mqtt_worker:send_to_remote(ChannelName, Msg). +on_query(_InstId, {send_message, ChannelId, Msg}, _AfterQuery, _State) -> + ?SLOG(debug, #{msg => "send msg to remote node", message => Msg, + channel_id => ChannelId}), + emqx_connector_mqtt_worker:send_to_remote(ChannelId, Msg). on_health_check(_InstId, #{channels := NameList} = State) -> Results = [{Name, emqx_connector_mqtt_worker:ping(Name)} || Name <- NameList], @@ -134,35 +139,43 @@ on_health_check(_InstId, #{channels := NameList} = State) -> create_channel({{ingress_channels, Id}, #{subscribe_remote_topic := RemoteT} = Conf}, NamePrefix, BasicConf) -> LocalT = maps:get(local_topic, Conf, undefined), - Name = ingress_channel_name(NamePrefix, Id), - logger:info("creating ingress channel ~p, remote ~s -> local ~s", [Name, RemoteT, LocalT]), + ChannId = ingress_channel_id(NamePrefix, Id), + ?SLOG(info, #{msg => "creating ingress channel", + remote_topic => RemoteT, + local_topic => LocalT, + channel_id => ChannId}), do_create_channel(BasicConf#{ - name => Name, - clientid => clientid(Name), + name => ChannId, + clientid => clientid(ChannId), subscriptions => Conf#{ local_topic => LocalT, - on_message_received => {fun ?MODULE:on_message_received/2, [Name]} + on_message_received => {fun ?MODULE:on_message_received/2, [ChannId]} }, forwards => undefined}); create_channel({{egress_channels, Id}, #{remote_topic := RemoteT} = Conf}, NamePrefix, BasicConf) -> LocalT = maps:get(subscribe_local_topic, Conf, undefined), - Name = egress_channel_name(NamePrefix, Id), - logger:info("creating egress channel ~p, local ~s -> remote ~s", [Name, LocalT, RemoteT]), + ChannId = egress_channel_id(NamePrefix, Id), + ?SLOG(info, #{msg => "creating egress channel", + remote_topic => RemoteT, + local_topic => LocalT, + channel_id => ChannId}), do_create_channel(BasicConf#{ - name => Name, - clientid => clientid(Name), + name => ChannId, + clientid => clientid(ChannId), subscriptions => undefined, forwards => Conf#{subscribe_local_topic => LocalT}}). -remove_channel(ChannelName) -> - logger:info("removing channel ~p", [ChannelName]), - case ?MODULE:drop_bridge(ChannelName) of +remove_channel(ChannId) -> + ?SLOG(info, #{msg => "removing channel", + channel_id => ChannId}), + case ?MODULE:drop_bridge(ChannId) of ok -> ok; {error, not_found} -> ok; {error, Reason} -> - logger:error("stop channel ~p failed, error: ~p", [ChannelName, Reason]) + ?SLOG(error, #{msg => "stop channel failed", + channel_id => ChannId, reason => Reason}) end. do_create_channel(#{name := Name} = Conf) -> @@ -215,9 +228,9 @@ basic_config(#{ taged_map_list(Tag, Map) -> [{{Tag, K}, V} || {K, V} <- maps:to_list(Map)]. -ingress_channel_name(Prefix, Id) -> +ingress_channel_id(Prefix, Id) -> channel_name("ingress_channels", Prefix, Id). -egress_channel_name(Prefix, Id) -> +egress_channel_id(Prefix, Id) -> channel_name("egress_channels", Prefix, Id). channel_name(Type, Prefix, Id) -> diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl index 9dc194c55..845c96161 100644 --- a/apps/emqx_connector/src/emqx_connector_mysql.erl +++ b/apps/emqx_connector/src/emqx_connector_mysql.erl @@ -17,6 +17,7 @@ -include_lib("typerefl/include/types.hrl"). -include_lib("emqx_resource/include/emqx_resource_behaviour.hrl"). +-include_lib("emqx/include/logger.hrl"). %% callbacks of behaviour emqx_resource -export([ on_start/2 @@ -54,7 +55,8 @@ on_start(InstId, #{server := {Host, Port}, auto_reconnect := AutoReconn, pool_size := PoolSize, ssl := SSL } = Config) -> - logger:info("starting mysql connector: ~p, config: ~p", [InstId, Config]), + ?SLOG(info, #{msg => "starting mysql connector", + connector => InstId, config => Config}), SslOpts = case maps:get(enable, SSL) of true -> [{ssl, [{server_name_indication, disable} | @@ -73,16 +75,21 @@ on_start(InstId, #{server := {Host, Port}, {ok, #{poolname => PoolName}}. on_stop(InstId, #{poolname := PoolName}) -> - logger:info("stopping mysql connector: ~p", [InstId]), + ?SLOG(info, #{msg => "stopping mysql connector", + connector => InstId}), emqx_plugin_libs_pool:stop_pool(PoolName). -on_query(InstId, {sql, SQL}, AfterQuery, #{poolname := PoolName} = State) -> - on_query(InstId, {sql, SQL, []}, AfterQuery, #{poolname := PoolName} = State); -on_query(InstId, {sql, SQL, Params}, AfterQuery, #{poolname := PoolName} = State) -> - logger:debug("mysql connector ~p received sql query: ~p, at state: ~p", [InstId, SQL, State]), - case Result = ecpool:pick_and_do(PoolName, {mysql, query, [SQL, Params]}, no_handover) of +on_query(InstId, {sql, SQL}, AfterQuery, #{poolname := _PoolName} = State) -> + on_query(InstId, {sql, SQL, [], default_timeout}, AfterQuery, State); +on_query(InstId, {sql, SQL, Params}, AfterQuery, #{poolname := _PoolName} = State) -> + on_query(InstId, {sql, SQL, Params, default_timeout}, AfterQuery, State); +on_query(InstId, {sql, SQL, Params, Timeout}, AfterQuery, #{poolname := PoolName} = State) -> + ?SLOG(debug, #{msg => "mysql connector received sql query", + connector => InstId, sql => SQL, state => State}), + case Result = ecpool:pick_and_do(PoolName, {mysql, query, [SQL, Params, Timeout]}, no_handover) of {error, Reason} -> - logger:debug("mysql connector ~p do sql query failed, sql: ~p, reason: ~p", [InstId, SQL, Reason]), + ?SLOG(error, #{msg => "mysql connector do sql query failed", + connector => InstId, sql => SQL, reason => Reason}), emqx_resource:query_failed(AfterQuery); _ -> emqx_resource:query_success(AfterQuery) diff --git a/apps/emqx_connector/src/emqx_connector_pgsql.erl b/apps/emqx_connector/src/emqx_connector_pgsql.erl index 8472c661e..5b0adbeb9 100644 --- a/apps/emqx_connector/src/emqx_connector_pgsql.erl +++ b/apps/emqx_connector/src/emqx_connector_pgsql.erl @@ -17,6 +17,7 @@ -include_lib("typerefl/include/types.hrl"). -include_lib("emqx_resource/include/emqx_resource_behaviour.hrl"). +-include_lib("emqx/include/logger.hrl"). -export([roots/0, fields/1]). @@ -54,7 +55,8 @@ on_start(InstId, #{server := {Host, Port}, auto_reconnect := AutoReconn, pool_size := PoolSize, ssl := SSL } = Config) -> - logger:info("starting postgresql connector: ~p, config: ~p", [InstId, Config]), + ?SLOG(info, #{msg => "starting postgresql connector", + connector => InstId, config => Config}), SslOpts = case maps:get(enable, SSL) of true -> [{ssl, [{server_name_indication, disable} | @@ -73,16 +75,20 @@ on_start(InstId, #{server := {Host, Port}, {ok, #{poolname => PoolName}}. on_stop(InstId, #{poolname := PoolName}) -> - logger:info("stopping postgresql connector: ~p", [InstId]), + ?SLOG(info, #{msg => "stopping postgresql connector", + connector => InstId}), emqx_plugin_libs_pool:stop_pool(PoolName). -on_query(InstId, {sql, SQL}, AfterQuery, #{poolname := PoolName} = State) -> - on_query(InstId, {sql, SQL, []}, AfterQuery, #{poolname := PoolName} = State); +on_query(InstId, {sql, SQL}, AfterQuery, #{poolname := _PoolName} = State) -> + on_query(InstId, {sql, SQL, []}, AfterQuery, State); on_query(InstId, {sql, SQL, Params}, AfterQuery, #{poolname := PoolName} = State) -> - logger:debug("postgresql connector ~p received sql query: ~p, at state: ~p", [InstId, SQL, State]), + ?SLOG(debug, #{msg => "postgresql connector received sql query", + connector => InstId, sql => SQL, state => State}), case Result = ecpool:pick_and_do(PoolName, {?MODULE, query, [SQL, Params]}, no_handover) of {error, Reason} -> - logger:debug("postgresql connector ~p do sql query failed, sql: ~p, reason: ~p", [InstId, SQL, Reason]), + ?SLOG(error, #{ + msg => "postgresql connector do sql query failed", + connector => InstId, sql => SQL, reason => Reason}), emqx_resource:query_failed(AfterQuery); _ -> emqx_resource:query_success(AfterQuery) diff --git a/apps/emqx_connector/src/emqx_connector_redis.erl b/apps/emqx_connector/src/emqx_connector_redis.erl index 44b036f39..aed06e724 100644 --- a/apps/emqx_connector/src/emqx_connector_redis.erl +++ b/apps/emqx_connector/src/emqx_connector_redis.erl @@ -18,6 +18,7 @@ -include("emqx_connector.hrl"). -include_lib("typerefl/include/types.hrl"). -include_lib("emqx_resource/include/emqx_resource_behaviour.hrl"). +-include_lib("emqx/include/logger.hrl"). -type server() :: tuple(). @@ -85,7 +86,8 @@ on_start(InstId, #{redis_type := Type, pool_size := PoolSize, auto_reconnect := AutoReconn, ssl := SSL } = Config) -> - logger:info("starting redis connector: ~p, config: ~p", [InstId, Config]), + ?SLOG(info, #{msg => "starting redis connector", + connector => InstId, config => Config}), Servers = case Type of single -> [{servers, [maps:get(server, Config)]}]; _ ->[{servers, maps:get(servers, Config)}] @@ -116,18 +118,21 @@ on_start(InstId, #{redis_type := Type, {ok, #{poolname => PoolName, type => Type}}. on_stop(InstId, #{poolname := PoolName}) -> - logger:info("stopping redis connector: ~p", [InstId]), + ?SLOG(info, #{msg => "stopping redis connector", + connector => InstId}), emqx_plugin_libs_pool:stop_pool(PoolName). on_query(InstId, {cmd, Command}, AfterCommand, #{poolname := PoolName, type := Type} = State) -> - logger:debug("redis connector ~p received cmd query: ~p, at state: ~p", [InstId, Command, State]), + ?SLOG(debug, #{msg => "redis connector received cmd query", + connector => InstId, sql => Command, state => State}), Result = case Type of cluster -> eredis_cluster:q(PoolName, Command); _ -> ecpool:pick_and_do(PoolName, {?MODULE, cmd, [Type, Command]}, no_handover) end, case Result of {error, Reason} -> - logger:debug("redis connector ~p do cmd query failed, cmd: ~p, reason: ~p", [InstId, Command, Reason]), + ?SLOG(error, #{msg => "redis connector do cmd query failed", + connector => InstId, sql => Command, reason => Reason}), emqx_resource:query_failed(AfterCommand); _ -> emqx_resource:query_success(AfterCommand) diff --git a/apps/emqx_connector/src/emqx_connector_schema_lib.erl b/apps/emqx_connector/src/emqx_connector_schema_lib.erl index ecdfb1416..9ecfb56b3 100644 --- a/apps/emqx_connector/src/emqx_connector_schema_lib.erl +++ b/apps/emqx_connector/src/emqx_connector_schema_lib.erl @@ -53,18 +53,12 @@ -export([roots/0, fields/1]). -roots() -> ["ssl"]. +roots() -> []. -fields("ssl") -> - [ {enable, #{type => boolean(), default => false}} - , {cacertfile, fun cacertfile/1} - , {keyfile, fun keyfile/1} - , {certfile, fun certfile/1} - , {verify, fun verify/1} - ]. +fields(_) -> []. ssl_fields() -> - [ {ssl, #{type => hoconsc:ref(?MODULE, "ssl"), + [ {ssl, #{type => hoconsc:ref(emqx_schema, ssl_client_opts), default => #{<<"enable">> => false} } } @@ -106,22 +100,6 @@ auto_reconnect(type) -> boolean(); auto_reconnect(default) -> true; auto_reconnect(_) -> undefined. -cacertfile(type) -> string(); -cacertfile(nullable) -> true; -cacertfile(_) -> undefined. - -keyfile(type) -> string(); -keyfile(nullable) -> true; -keyfile(_) -> undefined. - -certfile(type) -> string(); -certfile(nullable) -> true; -certfile(_) -> undefined. - -verify(type) -> boolean(); -verify(default) -> false; -verify(_) -> undefined. - servers(type) -> servers(); servers(validator) -> [?NOT_EMPTY("the value of the field 'servers' cannot be empty")]; servers(_) -> undefined. diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl index 8b0aa5051..853221eec 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl @@ -155,14 +155,18 @@ handle_puback(#{packet_id := PktId, reason_code := RC}, Parent) RC =:= ?RC_NO_MATCHING_SUBSCRIBERS -> Parent ! {batch_ack, PktId}, ok; handle_puback(#{packet_id := PktId, reason_code := RC}, _Parent) -> - ?LOG(warning, "publish ~p to remote node falied, reason_code: ~p", [PktId, RC]). + ?SLOG(warning, #{msg => "publish to remote node falied", + packet_id => PktId, reason_code => RC}). handle_publish(Msg, undefined) -> - ?LOG(error, "cannot publish to local broker as 'bridge.mqtt..in' not configured, msg: ~p", [Msg]); + ?SLOG(error, #{msg => "cannot publish to local broker as" + " ingress_channles' is not configured", + message => Msg}); handle_publish(Msg, #{on_message_received := {OnMsgRcvdFunc, Args}} = Vars) -> - ?LOG(debug, "publish to local broker, msg: ~p, vars: ~p", [Msg, Vars]), + ?SLOG(debug, #{msg => "publish to local broker", + message => Msg, vars => Vars}), emqx_metrics:inc('bridge.mqtt.message_received_from_remote', 1), - _ = erlang:apply(OnMsgRcvdFunc, [Msg, Args]), + _ = erlang:apply(OnMsgRcvdFunc, [Msg | Args]), case maps:get(local_topic, Vars, undefined) of undefined -> ok; _Topic -> diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl index a00b76b97..b0aaeb8b6 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl @@ -23,19 +23,21 @@ -export([ roots/0 , fields/1]). +-import(emqx_schema, [mk_duration/2]). + roots() -> [{config, #{type => hoconsc:ref(?MODULE, "config")}}]. fields("config") -> [ {server, hoconsc:mk(emqx_schema:ip_port(), #{default => "127.0.0.1:1883"})} - , {reconnect_interval, hoconsc:mk(emqx_schema:duration_ms(), #{default => "30s"})} + , {reconnect_interval, mk_duration("reconnect interval", #{default => "30s"})} , {proto_ver, fun proto_ver/1} , {bridge_mode, hoconsc:mk(boolean(), #{default => true})} , {username, hoconsc:mk(string())} , {password, hoconsc:mk(string())} , {clean_start, hoconsc:mk(boolean(), #{default => true})} - , {keepalive, hoconsc:mk(integer(), #{default => 300})} - , {retry_interval, hoconsc:mk(emqx_schema:duration_ms(), #{default => "30s"})} + , {keepalive, mk_duration("keepalive", #{default => "300s"})} + , {retry_interval, mk_duration("retry interval", #{default => "30s"})} , {max_inflight, hoconsc:mk(integer(), #{default => 32})} , {replayq, hoconsc:mk(hoconsc:ref(?MODULE, "replayq"))} , {ingress_channels, hoconsc:mk(hoconsc:map(id, hoconsc:ref(?MODULE, "ingress_channels")), #{default => []})} diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl index c98efd322..990d15ef5 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl @@ -63,6 +63,7 @@ -behaviour(gen_statem). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). +-include_lib("emqx/include/logger.hrl"). %% APIs -export([ start_link/1 @@ -189,7 +190,8 @@ callback_mode() -> [state_functions]. %% @doc Config should be a map(). init(#{name := Name} = ConnectOpts) -> - ?LOG(debug, "starting bridge worker for ~p", [Name]), + ?SLOG(debug, #{msg => "starting bridge worker", + name => Name}), erlang:process_flag(trap_exit, true), Queue = open_replayq(Name, maps:get(replayq, ConnectOpts, #{})), State = init_state(ConnectOpts), @@ -335,8 +337,9 @@ common(_StateName, cast, {send_to_remote, Msg}, #{replayq := Q} = State) -> NewQ = replayq:append(Q, [Msg]), {keep_state, State#{replayq => NewQ}, {next_event, internal, maybe_send}}; common(StateName, Type, Content, #{name := Name} = State) -> - ?LOG(notice, "Bridge ~p discarded ~p type event at state ~p:~p", - [Name, Type, StateName, Content]), + ?SLOG(notice, #{msg => "Bridge discarded event", + name => Name, type => Type, state_name => StateName, + content => Content}), {keep_state, State}. do_connect(#{connect_opts := ConnectOpts = #{forwards := Forwards}, @@ -352,8 +355,8 @@ do_connect(#{connect_opts := ConnectOpts = #{forwards := Forwards}, {ok, State#{connection => Conn}}; {error, Reason} -> ConnectOpts1 = obfuscate(ConnectOpts), - ?LOG(error, "Failed to connect \n" - "config=~p\nreason:~p", [ConnectOpts1, Reason]), + ?SLOG(error, #{msg => "Failed to connect", + config => ConnectOpts1, reason => Reason}), {error, Reason, State} end. @@ -399,7 +402,9 @@ pop_and_send_loop(#{replayq := Q} = State, N) -> %% Assert non-empty batch because we have a is_empty check earlier. do_send(#{connect_opts := #{forwards := undefined}}, _QAckRef, Batch) -> - ?LOG(error, "cannot forward messages to remote broker as 'bridge.mqtt..in' not configured, msg: ~p", [Batch]); + ?SLOG(error, #{msg => "cannot forward messages to remote broker" + " as egress_channel is not configured", + messages => Batch}); do_send(#{inflight := Inflight, connection := Connection, mountpoint := Mountpoint, @@ -409,14 +414,16 @@ do_send(#{inflight := Inflight, emqx_metrics:inc('bridge.mqtt.message_sent_to_remote'), emqx_connector_mqtt_msg:to_remote_msg(Message, Vars) end, - ?LOG(debug, "publish to remote broker, msg: ~p, vars: ~p", [Batch, Vars]), + ?SLOG(debug, #{msg => "publish to remote broker", + message => Batch, vars => Vars}), case emqx_connector_mqtt_mod:send(Connection, [ExportMsg(M) || M <- Batch]) of {ok, Refs} -> {ok, State#{inflight := Inflight ++ [#{q_ack_ref => QAckRef, send_ack_ref => map_set(Refs), batch => Batch}]}}; {error, Reason} -> - ?LOG(info, "mqtt_bridge_produce_failed ~p", [Reason]), + ?SLOG(info, #{msg => "mqtt_bridge_produce_failed", + reason => Reason}), {error, State} end. @@ -436,7 +443,8 @@ handle_batch_ack(#{inflight := Inflight0, replayq := Q} = State, Ref) -> State#{inflight := Inflight}. do_ack([], Ref) -> - ?LOG(debug, "stale_batch_ack_reference ~p", [Ref]), + ?SLOG(debug, #{msg => "stale_batch_ack_reference", + ref => Ref}), []; do_ack([#{send_ack_ref := Refs} = First | Rest], Ref) -> case maps:is_key(Ref, Refs) of diff --git a/apps/emqx_dashboard/src/emqx_dashboard.erl b/apps/emqx_dashboard/src/emqx_dashboard.erl index d109dd445..f1f5ab133 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard.erl @@ -47,11 +47,17 @@ start_listeners() -> type => apiKey, name => "authorization", in => header}}}}, - Dispatch = [ - {"/", cowboy_static, {priv_file, emqx_dashboard, "www/index.html"}}, - {"/static/[...]", cowboy_static, {priv_dir, emqx_dashboard, "www/static"}}, - {'_', cowboy_static, {priv_file, emqx_dashboard, "www/index.html"}} - ], + %% TODO: make it permanent when it's ready to release + Dispatch = + case os:getenv("_EMQX_ENABLE_DASHBOARD") of + V when V =:= "true" orelse V =:= "1" -> + [{"/", cowboy_static, {priv_file, emqx_dashboard, "www/index.html"}}, + {"/static/[...]", cowboy_static, {priv_dir, emqx_dashboard, "www/static"}}, + {'_', cowboy_static, {priv_file, emqx_dashboard, "www/index.html"}} + ]; + _ -> + [] + end, BaseMinirest = #{ base_path => ?BASE_PATH, modules => minirest_api:find_api_modules(apps()), diff --git a/apps/emqx_dashboard/src/emqx_dashboard_admin.erl b/apps/emqx_dashboard/src/emqx_dashboard_admin.erl index b477bd779..5af983b4d 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_admin.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_admin.erl @@ -139,7 +139,10 @@ update_pwd(Username, Fun) -> -spec(lookup_user(binary()) -> [mqtt_admin()]). -lookup_user(Username) when is_binary(Username) -> mnesia:dirty_read(mqtt_admin, Username). +lookup_user(Username) when is_binary(Username) -> + Fun = fun() -> mnesia:read(mqtt_admin, Username) end, + {atomic, User} = ekka_mnesia:ro_transaction(?DASHBOARD_SHARD, Fun), + User. -spec(all_users() -> [#mqtt_admin{}]). all_users() -> ets:tab2list(mqtt_admin). diff --git a/apps/emqx_dashboard/src/emqx_dashboard_collection.erl b/apps/emqx_dashboard/src/emqx_dashboard_collection.erl index 8b0576342..0e2adf7c3 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_collection.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_collection.erl @@ -162,7 +162,8 @@ flush({Connection, Route, Subscription}, {Received0, Sent0, Dropped0}) -> diff(Sent, Sent0), diff(Dropped, Dropped0)}, Ts = get_local_time(), - _ = mnesia:dirty_write(emqx_collect, #mqtt_collect{timestamp = Ts, collect = Collect}), + ekka_mnesia:transaction(ekka_mnesia:local_content_shard(), + fun mnesia:write/1, [#mqtt_collect{timestamp = Ts, collect = Collect}]), {Received, Sent, Dropped}. avg(Items) -> diff --git a/apps/emqx_dashboard/src/emqx_dashboard_schema.erl b/apps/emqx_dashboard/src/emqx_dashboard_schema.erl index 94cfaddad..ff3be9320 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_schema.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_schema.erl @@ -45,7 +45,9 @@ fields("http") -> ]; fields("https") -> - proplists:delete("fail_if_no_peer_cert", emqx_schema:ssl(#{})) ++ fields("http"). + fields("http") ++ + proplists:delete("fail_if_no_peer_cert", + emqx_schema:server_ssl_opts_schema(#{}, true)). default_username(type) -> string(); default_username(default) -> "admin"; diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 9033a9b40..75b3ab201 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -6,6 +6,11 @@ %% API -export([spec/1, spec/2]). -export([translate_req/2]). +-export([namespace/0, fields/1]). +-export([error_codes/1, error_codes/2]). +-define(MAX_ROW_LIMIT, 100). + +%% API -ifdef(TEST). -compile(export_all). @@ -22,7 +27,8 @@ -define(INIT_SCHEMA, #{fields => #{}, translations => #{}, validations => [], namespace => undefined}). -define(TO_REF(_N_, _F_), iolist_to_binary([to_bin(_N_), ".", to_bin(_F_)])). --define(TO_COMPONENTS(_M_, _F_), iolist_to_binary([<<"#/components/schemas/">>, ?TO_REF(namespace(_M_), _F_)])). +-define(TO_COMPONENTS_SCHEMA(_M_, _F_), iolist_to_binary([<<"#/components/schemas/">>, ?TO_REF(namespace(_M_), _F_)])). +-define(TO_COMPONENTS_PARAM(_M_, _F_), iolist_to_binary([<<"#/components/parameters/">>, ?TO_REF(namespace(_M_), _F_)])). %% @equiv spec(Module, #{check_schema => false}) -spec(spec(module()) -> @@ -54,7 +60,6 @@ spec(Module, Options) -> end, {[], []}, Paths), {ApiSpec, components(lists:usort(AllRefs))}. - -spec(translate_req(#{binding => list(), query_string => list(), body => map()}, #{module => module(), path => string(), method => atom()}) -> {ok, #{binding => list(), query_string => list(), body => map()}}| @@ -64,7 +69,7 @@ translate_req(Request, #{module := Module, path := Path, method := Method}) -> try Params = maps:get(parameters, Spec, []), Body = maps:get(requestBody, Spec, []), - {Bindings, QueryStr} = check_parameters(Request, Params), + {Bindings, QueryStr} = check_parameters(Request, Params, Module), NewBody = check_requestBody(Request, Body, Module, hoconsc:is_schema(Body)), {ok, Request#{bindings => Bindings, query_string => QueryStr, body => NewBody}} catch throw:Error -> @@ -73,6 +78,30 @@ translate_req(Request, #{module := Module, path := Path, method := Method}) -> {400, 'BAD_REQUEST', iolist_to_binary(io_lib:format("~s : ~p", [Key, Reason]))} end. +namespace() -> "public". + +fields(page) -> + Desc = <<"Page number of the results to fetch.">>, + Meta = #{in => query, desc => Desc, default => 1, example => 1}, + [{page, hoconsc:mk(integer(), Meta)}]; +fields(limit) -> + Desc = iolist_to_binary([<<"Results per page(max ">>, + integer_to_binary(?MAX_ROW_LIMIT), <<")">>]), + Meta = #{in => query, desc => Desc, default => ?MAX_ROW_LIMIT, example => 50}, + [{limit, hoconsc:mk(range(1, ?MAX_ROW_LIMIT), Meta)}]. + +error_codes(Codes) -> + error_codes(Codes, <<"Error code to troubleshoot problems.">>). + +error_codes(Codes = [_ | _], MsgExample) -> + [ + {code, hoconsc:mk(hoconsc:enum(Codes))}, + {message, hoconsc:mk(string(), #{ + desc => <<"Details description of the error.">>, + example => MsgExample + })} + ]. + support_check_schema(#{check_schema := true}) -> ?DEFAULT_FILTER; support_check_schema(#{check_schema := Func})when is_function(Func, 2) -> #{filter => Func}; support_check_schema(_) -> #{filter => undefined}. @@ -93,23 +122,28 @@ parse_spec_ref(Module, Path) -> maps:without([operationId], Schema)), {maps:get(operationId, Schema), Specs, Refs}. -check_parameters(Request, Spec) -> +check_parameters(Request, Spec, Module) -> #{bindings := Bindings, query_string := QueryStr} = Request, BindingsBin = maps:fold(fun(Key, Value, Acc) -> Acc#{atom_to_binary(Key) => Value} end, #{}, Bindings), - check_parameter(Spec, BindingsBin, QueryStr, #{}, #{}). + check_parameter(Spec, BindingsBin, QueryStr, Module, #{}, #{}). -check_parameter([], _Bindings, _QueryStr, NewBindings, NewQueryStr) -> {NewBindings, NewQueryStr}; -check_parameter([{Name, Type} | Spec], Bindings, QueryStr, BindingsAcc, QueryStrAcc) -> +check_parameter([?REF(Fields) | Spec], Bindings, QueryStr, LocalMod, BindingsAcc, QueryStrAcc) -> + check_parameter([?R_REF(LocalMod, Fields) | Spec], Bindings, QueryStr, LocalMod, BindingsAcc, QueryStrAcc); +check_parameter([?R_REF(Module, Fields) | Spec], Bindings, QueryStr, LocalMod, BindingsAcc, QueryStrAcc) -> + Params = apply(Module, fields, [Fields]), + check_parameter(Params ++ Spec, Bindings, QueryStr, LocalMod, BindingsAcc, QueryStrAcc); +check_parameter([], _Bindings, _QueryStr, _Module, NewBindings, NewQueryStr) -> {NewBindings, NewQueryStr}; +check_parameter([{Name, Type} | Spec], Bindings, QueryStr, Module, BindingsAcc, QueryStrAcc) -> Schema = ?INIT_SCHEMA#{roots => [{Name, Type}]}, case hocon_schema:field_schema(Type, in) of path -> NewBindings = hocon_schema:check_plain(Schema, Bindings, #{atom_key => true, override_env => false}), NewBindingsAcc = maps:merge(BindingsAcc, NewBindings), - check_parameter(Spec, Bindings, QueryStr, NewBindingsAcc, QueryStrAcc); + check_parameter(Spec, Bindings, QueryStr, Module, NewBindingsAcc, QueryStrAcc); query -> NewQueryStr = hocon_schema:check_plain(Schema, QueryStr, #{override_env => false}), NewQueryStrAcc = maps:merge(QueryStrAcc, NewQueryStr), - check_parameter(Spec, Bindings, QueryStr, BindingsAcc, NewQueryStrAcc) + check_parameter(Spec, Bindings, QueryStr, Module, BindingsAcc, NewQueryStrAcc) end. check_requestBody(#{body := Body}, Schema, Module, true) -> @@ -154,19 +188,28 @@ to_spec(Meta, Params, RequestBody, Responses) -> parameters(Params, Module) -> {SpecList, AllRefs} = - lists:foldl(fun({Name, Type}, {Acc, RefsAcc}) -> - In = hocon_schema:field_schema(Type, in), - In =:= undefined andalso throw({error, <<"missing in:path/query field in parameters">>}), - Nullable = hocon_schema:field_schema(Type, nullable), - Default = hocon_schema:field_schema(Type, default), - HoconType = hocon_schema:field_schema(Type, type), - Meta = init_meta(Nullable, Default), - {ParamType, Refs} = hocon_schema_to_spec(HoconType, Module), - Spec0 = init_prop([required | ?DEFAULT_FIELDS], - #{schema => maps:merge(ParamType, Meta), name => Name, in => In}, Type), - Spec1 = trans_required(Spec0, Nullable, In), - Spec2 = trans_desc(Spec1, Type), - {[Spec2 | Acc], Refs ++ RefsAcc} + lists:foldl(fun(Param, {Acc, RefsAcc}) -> + case Param of + ?REF(StructName) -> + {[#{<<"$ref">> => ?TO_COMPONENTS_PARAM(Module, StructName)} |Acc], + [{Module, StructName, parameter}|RefsAcc]}; + ?R_REF(RModule, StructName) -> + {[#{<<"$ref">> => ?TO_COMPONENTS_PARAM(RModule, StructName)} |Acc], + [{RModule, StructName, parameter}|RefsAcc]}; + {Name, Type} -> + In = hocon_schema:field_schema(Type, in), + In =:= undefined andalso throw({error, <<"missing in:path/query field in parameters">>}), + Nullable = hocon_schema:field_schema(Type, nullable), + Default = hocon_schema:field_schema(Type, default), + HoconType = hocon_schema:field_schema(Type, type), + Meta = init_meta(Nullable, Default), + {ParamType, Refs} = hocon_schema_to_spec(HoconType, Module), + Spec0 = init_prop([required | ?DEFAULT_FIELDS], + #{schema => maps:merge(ParamType, Meta), name => Name, in => In}, Type), + Spec1 = trans_required(Spec0, Nullable, In), + Spec2 = trans_desc(Spec1, Type), + {[Spec2 | Acc], Refs ++ RefsAcc} + end end, {[], []}, Params), {lists:reverse(SpecList), AllRefs}. @@ -196,7 +239,7 @@ trans_required(Spec, _, _) -> Spec. trans_desc(Spec, Hocon) -> case hocon_schema:field_schema(Hocon, desc) of undefined -> Spec; - Desc -> Spec#{description => Desc} + Desc -> Spec#{description => to_bin(Desc)} end. requestBody([], _Module) -> {[], []}; @@ -248,6 +291,13 @@ components([{Module, Field} | Refs], SpecAcc, SubRefsAcc) -> Namespace = namespace(Module), {Object, SubRefs} = parse_object(Props, Module), NewSpecAcc = SpecAcc#{?TO_REF(Namespace, Field) => Object}, + components(Refs, NewSpecAcc, SubRefs ++ SubRefsAcc); +%% parameters in ref only have one value, not array +components([{Module, Field, parameter} | Refs], SpecAcc, SubRefsAcc) -> + Props = apply(Module, fields, [Field]), + {[Param], SubRefs} = parameters(Props, Module), + Namespace = namespace(Module), + NewSpecAcc = SpecAcc#{?TO_REF(Namespace, Field) => Param}, components(Refs, NewSpecAcc, SubRefs ++ SubRefsAcc). namespace(Module) -> @@ -257,10 +307,10 @@ namespace(Module) -> end. hocon_schema_to_spec(?R_REF(Module, StructName), _LocalModule) -> - {#{<<"$ref">> => ?TO_COMPONENTS(Module, StructName)}, + {#{<<"$ref">> => ?TO_COMPONENTS_SCHEMA(Module, StructName)}, [{Module, StructName}]}; hocon_schema_to_spec(?REF(StructName), LocalModule) -> - {#{<<"$ref">> => ?TO_COMPONENTS(LocalModule, StructName)}, + {#{<<"$ref">> => ?TO_COMPONENTS_SCHEMA(LocalModule, StructName)}, [{LocalModule, StructName}]}; hocon_schema_to_spec(Type, _LocalModule) when ?IS_TYPEREFL(Type) -> {typename_to_spec(typerefl:name(Type)), []}; diff --git a/apps/emqx_dashboard/src/emqx_dashboard_token.erl b/apps/emqx_dashboard/src/emqx_dashboard_token.erl index 2acf00f13..c1ca15cb3 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_token.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_token.erl @@ -103,7 +103,8 @@ do_sign(Username, Password) -> }, Signed = jose_jwt:sign(JWK, JWS, JWT), {_, Token} = jose_jws:compact(Signed), - ok = ekka_mnesia:dirty_write(format(Token, Username, ExpTime)), + JWTRec = format(Token, Username, ExpTime), + ekka_mnesia:transaction(?DASHBOARD_SHARD, fun mnesia:write/1, [JWTRec]), {ok, Token}. do_verify(Token)-> @@ -111,8 +112,9 @@ do_verify(Token)-> {ok, JWT = #mqtt_admin_jwt{exptime = ExpTime}} -> case ExpTime > erlang:system_time(millisecond) of true -> - ekka_mnesia:dirty_write(JWT#mqtt_admin_jwt{exptime = jwt_expiration_time()}), - ok; + NewJWT = JWT#mqtt_admin_jwt{exptime = jwt_expiration_time()}, + {atomic, Res} = ekka_mnesia:transaction(?DASHBOARD_SHARD, fun mnesia:write/1, [NewJWT]), + Res; _ -> {error, token_timeout} end; @@ -132,14 +134,18 @@ do_destroy_by_username(Username) -> %% jwt internal util function -spec(lookup(Token :: binary()) -> {ok, #mqtt_admin_jwt{}} | {error, not_found}). lookup(Token) -> - case mnesia:dirty_read(?TAB, Token) of - [JWT] -> {ok, JWT}; - [] -> {error, not_found} + Fun = fun() -> mnesia:read(?TAB, Token) end, + case ekka_mnesia:ro_transaction(?DASHBOARD_SHARD, Fun) of + {atomic, [JWT]} -> {ok, JWT}; + {atomic, []} -> {error, not_found} end. lookup_by_username(Username) -> Spec = [{{mqtt_admin_jwt, '_', Username, '_'}, [], ['$_']}], - mnesia:dirty_select(?TAB, Spec). + Fun = fun() -> mnesia:select(?TAB, Spec) end, + {atomic, List} = ekka_mnesia:ro_transaction(?DASHBOARD_SHARD, Fun), + List. + jwk(Username, Password, Salt) -> Key = erlang:md5(<>), @@ -187,7 +193,8 @@ handle_info(clean_jwt, State) -> timer_clean(self()), Now = erlang:system_time(millisecond), Spec = [{{mqtt_admin_jwt, '_', '_', '$1'}, [{'<', '$1', Now}], ['$_']}], - JWTList = mnesia:dirty_select(?TAB, Spec), + {atomic, JWTList} = ekka_mnesia:ro_transaction(?DASHBOARD_SHARD, + fun() -> mnesia:select(?TAB, Spec) end), destroy(JWTList), {noreply, State}; handle_info(_Info, State) -> diff --git a/apps/emqx_dashboard/src/emqx_swagger_util.erl b/apps/emqx_dashboard/src/emqx_swagger_util.erl deleted file mode 100644 index e2f279941..000000000 --- a/apps/emqx_dashboard/src/emqx_swagger_util.erl +++ /dev/null @@ -1,13 +0,0 @@ -%%%------------------------------------------------------------------- -%%% @author zhongwen -%%% @copyright (C) 2021, -%%% @doc -%%% -%%% @end -%%% Created : 22. 9月 2021 13:38 -%%%------------------------------------------------------------------- --module(emqx_swagger_util). --author("zhongwen"). - -%% API --export([]). diff --git a/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl index a5c458ffa..cea0a915d 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl @@ -3,10 +3,10 @@ -behaviour(hocon_schema). %% API --export([paths/0, api_spec/0, schema/1]). --export([t_in_path/1, t_in_query/1, t_in_mix/1, t_without_in/1]). +-export([paths/0, api_spec/0, schema/1, fields/1]). +-export([t_in_path/1, t_in_query/1, t_in_mix/1, t_without_in/1, t_ref/1, t_public_ref/1]). -export([t_require/1, t_nullable/1, t_method/1, t_api_spec/1]). --export([t_in_path_trans/1, t_in_query_trans/1, t_in_mix_trans/1]). +-export([t_in_path_trans/1, t_in_query_trans/1, t_in_mix_trans/1, t_ref_trans/1]). -export([t_in_path_trans_error/1, t_in_query_trans_error/1, t_in_mix_trans_error/1]). -export([all/0, suite/0, groups/0]). @@ -20,9 +20,9 @@ all() -> [{group, spec}, {group, validation}]. suite() -> [{timetrap, {minutes, 1}}]. groups() -> [ - {spec, [parallel], [t_api_spec, t_in_path, t_in_query, t_in_mix, - t_without_in, t_require, t_nullable, t_method]}, - {validation, [parallel], [t_in_path_trans, t_in_query_trans, t_in_mix_trans, + {spec, [parallel], [t_api_spec, t_in_path, t_ref, t_in_query, t_in_mix, + t_without_in, t_require, t_nullable, t_method, t_public_ref]}, + {validation, [parallel], [t_in_path_trans, t_ref_trans, t_in_query_trans, t_in_mix_trans, t_in_path_trans_error, t_in_query_trans_error, t_in_mix_trans_error]} ]. @@ -44,6 +44,41 @@ t_in_query(_Config) -> validate("/test/in/query", Expect), ok. +t_ref(_Config) -> + LocalPath = "/test/in/ref/local", + Path = "/test/in/ref", + Expect = [#{<<"$ref">> => <<"#/components/parameters/emqx_swagger_parameter_SUITE.page">>}], + {OperationId, Spec, Refs} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path), + {OperationId, Spec, Refs} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, LocalPath), + ?assertEqual(test, OperationId), + Params = maps:get(parameters, maps:get(post, Spec)), + ?assertEqual(Expect, Params), + ?assertEqual([{?MODULE, page, parameter}], Refs), + ok. + +t_public_ref(_Config) -> + Path = "/test/in/ref/public", + Expect = [ + #{<<"$ref">> => <<"#/components/parameters/public.page">>}, + #{<<"$ref">> => <<"#/components/parameters/public.limit">>} + ], + {OperationId, Spec, Refs} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path), + ?assertEqual(test, OperationId), + Params = maps:get(parameters, maps:get(post, Spec)), + ?assertEqual(Expect, Params), + ?assertEqual([ + {emqx_dashboard_swagger, limit, parameter}, + {emqx_dashboard_swagger, page, parameter} + ], Refs), + ExpectRefs = [ + #{<<"public.limit">> => #{description => <<"Results per page(max 100)">>, example => 50,in => query,name => limit, + schema => #{default => 100,example => 1,maximum => 100, minimum => 1,type => integer}}}, + #{<<"public.page">> => #{description => <<"Page number of the results to fetch.">>, + example => 1,in => query,name => page, + schema => #{default => 1,example => 100,type => integer}}}], + ?assertEqual(ExpectRefs, emqx_dashboard_swagger:components(Refs)), + ok. + t_in_mix(_Config) -> Expect = [#{description => <<"Indicates which sorts of issues to return">>, @@ -115,6 +150,18 @@ t_in_query_trans(_Config) -> ?assertEqual(Expect, trans_parameters(Path, #{}, #{<<"per_page">> => 100})), ok. +t_ref_trans(_Config) -> + LocalPath = "/test/in/ref/local", + Path = "/test/in/ref", + Expect = {ok, #{bindings => #{},body => #{}, + query_string => #{<<"per_page">> => 100}}}, + ?assertEqual(Expect, trans_parameters(Path, #{}, #{<<"per_page">> => 100})), + ?assertEqual(Expect, trans_parameters(LocalPath, #{}, #{<<"per_page">> => 100})), + {400,'BAD_REQUEST', Reason} = trans_parameters(Path, #{}, #{<<"per_page">> => 1010}), + ?assertNotEqual(nomatch, binary:match(Reason, [<<"per_page">>])), + {400,'BAD_REQUEST', Reason} = trans_parameters(LocalPath, #{}, #{<<"per_page">> => 1010}), + ok. + t_in_mix_trans(_Config) -> Path = "/test/in/mix/:state", Bindings = #{ @@ -186,7 +233,7 @@ trans_parameters(Path, Bindings, QueryStr) -> api_spec() -> emqx_dashboard_swagger:spec(?MODULE). -paths() -> ["/test/in/:filter", "/test/in/query", "/test/in/mix/:state", +paths() -> ["/test/in/:filter", "/test/in/query", "/test/in/mix/:state", "/test/in/ref", "/required/false", "/nullable/false", "/nullable/true", "/method/ok"]. schema("/test/in/:filter") -> @@ -213,6 +260,33 @@ schema("/test/in/query") -> responses => #{200 => <<"ok">>} } }; +schema("/test/in/ref/local") -> + #{ + operationId => test, + post => #{ + parameters => [hoconsc:ref(page)], + responses => #{200 => <<"ok">>} + } + }; +schema("/test/in/ref") -> + #{ + operationId => test, + post => #{ + parameters => [hoconsc:ref(?MODULE, page)], + responses => #{200 => <<"ok">>} + } + }; +schema("/test/in/ref/public") -> + #{ + operationId => test, + post => #{ + parameters => [ + hoconsc:ref(emqx_dashboard_swagger, page), + hoconsc:ref(emqx_dashboard_swagger, limit) + ], + responses => #{200 => <<"ok">>} + } + }; schema("/test/in/mix/:state") -> #{ operationId => test, @@ -257,6 +331,13 @@ schema("/method/ok") -> #{operationId => test}, ?METHODS); schema("/method/error") -> #{operationId => test, bar => #{200 => <<"ok">>}}. + +fields(page) -> + [ + {per_page, + mk(range(1, 100), + #{in => query, desc => <<"results per page (max 100)">>, example => 1})} + ]. to_schema(Params) -> #{ operationId => test, diff --git a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl index 49dca926f..84cbfe5fb 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl @@ -101,7 +101,7 @@ t_remote_ref(_Config) -> {<<"another_ref">>, #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_remote_schema.ref3">>}}], <<"type">> => object}}, #{<<"emqx_swagger_remote_schema.ref3">> => #{<<"properties">> => [ {<<"ip">>, #{description => <<"IP:Port">>, example => <<"127.0.0.1:80">>,type => string}}, - {<<"version">>, #{description => "a good version", example => <<"1.0.0">>,type => string}}], + {<<"version">>, #{description => <<"a good version">>, example => <<"1.0.0">>,type => string}}], <<"type">> => object}}], ?assertEqual(ExpectComponents, Components), ok. @@ -116,7 +116,7 @@ t_nest_ref(_Config) -> ExpectComponents = lists:sort([ #{<<"emqx_swagger_requestBody_SUITE.nest_ref">> => #{<<"properties">> => [ {<<"env">>, #{enum => [test,dev,prod],type => string}}, - {<<"another_ref">>, #{description => "nest ref", <<"$ref">> => <<"#/components/schemas/emqx_swagger_requestBody_SUITE.good_ref">>}}], + {<<"another_ref">>, #{description => <<"nest ref">>, <<"$ref">> => <<"#/components/schemas/emqx_swagger_requestBody_SUITE.good_ref">>}}], <<"type">> => object}}, #{<<"emqx_swagger_requestBody_SUITE.good_ref">> => #{<<"properties">> => [ {<<"webhook-host">>, #{default => <<"127.0.0.1:80">>, example => <<"127.0.0.1:80">>,type => string}}, diff --git a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl index c2140d2c0..fd6920549 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl @@ -12,7 +12,7 @@ -export([all/0, suite/0, groups/0]). -export([paths/0, api_spec/0, schema/1, fields/1]). --export([t_simple_binary/1, t_object/1, t_nest_object/1, t_empty/1, +-export([t_simple_binary/1, t_object/1, t_nest_object/1, t_empty/1, t_error/1, t_raw_local_ref/1, t_raw_remote_ref/1, t_hocon_schema_function/1, t_local_ref/1, t_remote_ref/1, t_bad_ref/1, t_none_ref/1, t_nest_ref/1, t_ref_array_with_key/1, t_ref_array_without_key/1, t_api_spec/1]). @@ -21,7 +21,7 @@ all() -> [{group, spec}]. suite() -> [{timetrap, {minutes, 1}}]. groups() -> [ {spec, [parallel], [ - t_api_spec, t_simple_binary, t_object, t_nest_object, + t_api_spec, t_simple_binary, t_object, t_nest_object, t_error, t_raw_local_ref, t_raw_remote_ref, t_empty, t_hocon_schema_function, t_local_ref, t_remote_ref, t_bad_ref, t_none_ref, t_ref_array_with_key, t_ref_array_without_key, t_nest_ref]} @@ -48,6 +48,33 @@ t_object(_config) -> validate(Path, Object, ExpectRefs), ok. +t_error(_Config) -> + Path = "/error", + Error400 = #{<<"content">> => + #{<<"application/json">> => #{<<"schema">> => #{<<"type">> => object, + <<"properties">> => + [ + {<<"code">>, #{enum => ['Bad1','Bad2'], type => string}}, + {<<"message">>, #{description => <<"Details description of the error.">>, + example => <<"Bad request desc">>, type => string}}] + }}}}, + Error404 = #{<<"content">> => + #{<<"application/json">> => #{<<"schema">> => #{<<"type">> => object, + <<"properties">> => + [ + {<<"code">>, #{enum => ['Not-Found'], type => string}}, + {<<"message">>, #{description => <<"Details description of the error.">>, + example => <<"Error code to troubleshoot problems.">>, type => string}}] + }}}}, + {OperationId, Spec, Refs} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path), + ?assertEqual(test, OperationId), + Response = maps:get(responses, maps:get(get, Spec)), + ?assertEqual(Error400, maps:get(<<"400">>, Response)), + ?assertEqual(Error404, maps:get(<<"404">>, Response)), + ?assertEqual(#{}, maps:without([<<"400">>, <<"404">>], Response)), + ?assertEqual([], Refs), + ok. + t_nest_object(_Config) -> Path = "/nest/object", Object = @@ -175,7 +202,7 @@ t_hocon_schema_function(_Config) -> #{<<"emqx_swagger_remote_schema.ref3">> => #{<<"type">> => object, <<"properties">> => [ {<<"ip">>, #{description => <<"IP:Port">>, example => <<"127.0.0.1:80">>,type => string}}, - {<<"version">>, #{description => "a good version", example => <<"1.0.0">>, type => string}}] + {<<"version">>, #{description => <<"a good version">>, example => <<"1.0.0">>, type => string}}] }}, #{<<"emqx_swagger_remote_schema.root">> => #{required => [<<"default_password">>, <<"default_username">>], <<"properties">> => [{<<"listeners">>, #{items => @@ -255,7 +282,15 @@ schema("/ref/array/with/key") -> schema("/ref/array/without/key") -> to_schema(mk(hoconsc:array(hoconsc:ref(?MODULE, good_ref)), #{})); schema("/ref/hocon/schema/function") -> - to_schema(mk(hoconsc:ref(emqx_swagger_remote_schema, "root"), #{})). + to_schema(mk(hoconsc:ref(emqx_swagger_remote_schema, "root"), #{})); +schema("/error") -> + #{ + operationId => test, + get => #{responses => #{ + 400 => emqx_dashboard_swagger:error_codes(['Bad1', 'Bad2'], <<"Bad request desc">>), + 404 => emqx_dashboard_swagger:error_codes(['Not-Found']) + }} + }. validate(Path, ExpectObject, ExpectRefs) -> {OperationId, Spec, Refs} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path), diff --git a/apps/emqx_exhook/src/emqx_exhook_server.erl b/apps/emqx_exhook/src/emqx_exhook_server.erl index 924e5d7ba..e776f8c62 100644 --- a/apps/emqx_exhook/src/emqx_exhook_server.erl +++ b/apps/emqx_exhook/src/emqx_exhook_server.erl @@ -153,7 +153,7 @@ do_init(ChannName, ReqOpts) -> case do_call(ChannName, 'on_provider_loaded', Req, ReqOpts) of {ok, InitialResp} -> try - {ok, resovle_hookspec(maps:get(hooks, InitialResp, []))} + {ok, resolve_hookspec(maps:get(hooks, InitialResp, []))} catch _:Reason:Stk -> ?LOG(error, "try to init ~p failed, reason: ~p, stacktrace: ~0p", [ChannName, Reason, Stk]), @@ -164,7 +164,7 @@ do_init(ChannName, ReqOpts) -> end. %% @private -resovle_hookspec(HookSpecs) when is_list(HookSpecs) -> +resolve_hookspec(HookSpecs) when is_list(HookSpecs) -> MessageHooks = message_hooks(), AvailableHooks = available_hooks(), lists:foldr(fun(HookSpec, Acc) -> diff --git a/apps/emqx_exhook/test/props/prop_exhook_hooks.erl b/apps/emqx_exhook/test/props/prop_exhook_hooks.erl index a57e0b49c..74db72022 100644 --- a/apps/emqx_exhook/test/props/prop_exhook_hooks.erl +++ b/apps/emqx_exhook/test/props/prop_exhook_hooks.erl @@ -19,7 +19,7 @@ -include_lib("proper/include/proper.hrl"). -include_lib("eunit/include/eunit.hrl"). --import(emqx_ct_proper_types, +-import(emqx_proper_types, [ conninfo/0 , clientinfo/0 , sessioninfo/0 @@ -503,7 +503,7 @@ unsub_properties() -> #{}. shutdown_reason() -> - oneof([utf8(), {shutdown, emqx_ct_proper_types:limited_atom()}]). + oneof([utf8(), {shutdown, emqx_proper_types:limited_atom()}]). authresult() -> ?LET(RC, connack_return_code(), diff --git a/apps/emqx_gateway/etc/emqx_gateway.conf b/apps/emqx_gateway/etc/emqx_gateway.conf index 2ce48bf75..f6b0aaa37 100644 --- a/apps/emqx_gateway/etc/emqx_gateway.conf +++ b/apps/emqx_gateway/etc/emqx_gateway.conf @@ -2,299 +2,6 @@ ## EMQ X Gateway configurations ##-------------------------------------------------------------------- -## TODO: These configuration options are temporary example here. -## In the final version, it will be commented out. - -gateway.stomp { - - ## How long time the connection will be disconnected if the - ## connection is established but no bytes received - idle_timeout = 30s - - ## To control whether write statistics data into ETS table - ## for dashbord to read. - enable_stats = true - - ## When publishing or subscribing, prefix all topics with a mountpoint string. - mountpoint = "" - - frame { - max_headers = 10 - max_headers_length = 1024 - max_body_length = 8192 - } - - clientinfo_override { - username = "${Packet.headers.login}" - password = "${Packet.headers.passcode}" - } - - authentication: [ - # { - # name = "authenticator1" - # type = "password-based:built-in-database" - # user_id_type = clientid - # } - ] - - listeners.tcp.default { - bind = 61613 - acceptors = 16 - max_connections = 1024000 - max_conn_rate = 1000 - - access_rules = [ - "allow all" - ] - - ## TCP options - ## See ${example_common_tcp_options} for more information - tcp.active_n = 100 - tcp.backlog = 1024 - tcp.buffer = 4KB - } - - listeners.ssl.default { - bind = 61614 - acceptors = 16 - max_connections = 1024000 - max_conn_rate = 1000 - - ## TCP options - ## See ${example_common_tcp_options} for more information - tcp.active_n = 100 - tcp.backlog = 1024 - tcp.buffer = 4KB - - ## SSL options - ## See ${example_common_ssl_options} for more information - ssl.versions = ["tlsv1.3", "tlsv1.2", "tlsv1.1", "tlsv1"] - ssl.keyfile = "{{ platform_etc_dir }}/certs/key.pem" - ssl.certfile = "{{ platform_etc_dir }}/certs/cert.pem" - ssl.cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" - #ssl.verify = verify_none - #ssl.fail_if_no_peer_cert = false - #ssl.server_name_indication = disable - #ssl.secure_renegotiate = false - #ssl.reuse_sessions = false - #ssl.honor_cipher_order = false - #ssl.handshake_timeout = 15s - #ssl.depth = 10 - #ssl.password = foo - #ssl.dhfile = path-to-your-file - } -} - -gateway.coap { - - ## How long time the connection will be disconnected if the - ## connection is established but no bytes received - idle_timeout = 30s - - ## To control whether write statistics data into ETS table - ## for dashbord to read. - enable_stats = true - - ## When publishing or subscribing, prefix all topics with a mountpoint string. - mountpoint = "" - - notify_type = qos - - ## if true, you need to establish a connection before use - connection_required = false - subscribe_qos = qos0 - publish_qos = qos1 - - listeners.udp.default { - bind = 5683 - acceptors = 4 - max_connections = 102400 - max_conn_rate = 1000 - - ## UDP Options - ## See ${example_common_udp_options} for more information - udp.active_n = 100 - udp.buffer = 16KB - } - listeners.dtls.default { - bind = 5684 - acceptors = 4 - max_connections = 102400 - max_conn_rate = 1000 - - ## UDP Options - ## See ${example_common_udp_options} for more information - udp.active_n = 100 - udp.buffer = 16KB - - ## DTLS Options - ## See #{example_common_dtls_options} for more information - dtls.versions = ["dtlsv1.2", "dtlsv1"] - dtls.keyfile = "{{ platform_etc_dir }}/certs/key.pem" - dtls.certfile = "{{ platform_etc_dir }}/certs/cert.pem" - dtls.cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" - } -} - -gateway.mqttsn { - - ## How long time the connection will be disconnected if the - ## connection is established but no bytes received - idle_timeout = 30s - - ## To control whether write statistics data into ETS table - ## for dashbord to read. - enable_stats = true - - ## When publishing or subscribing, prefix all topics with a mountpoint string. - mountpoint = "" - - ## The MQTT-SN Gateway ID in ADVERTISE message. - gateway_id = 1 - - ## Enable broadcast this gateway to WLAN - broadcast = true - - ## To control whether accept and process the received - ## publish message with qos=-1. - enable_qos3 = true - - ## The pre-defined topic name corresponding to the pre-defined topic - ## id of N. - ## Note that the pre-defined topic id of 0 is reserved. - predefined = [ - { id = 1 - topic = "/predefined/topic/name/hello" - }, - { id = 2 - topic = "/predefined/topic/name/nice" - } - ] - - ### ClientInfo override - clientinfo_override { - username = "mqtt_sn_user" - password = "abc" - } - - listeners.udp.default { - bind = 1884 - max_connections = 10240000 - max_conn_rate = 1000 - } - - listeners.dtls.default { - bind = 1885 - acceptors = 4 - max_connections = 102400 - max_conn_rate = 1000 - - ## UDP Options - ## See ${example_common_udp_options} for more information - udp.active_n = 100 - udp.buffer = 16KB - - ## DTLS Options - ## See #{example_common_dtls_options} for more information - dtls.versions = ["dtlsv1.2", "dtlsv1"] - dtls.keyfile = "{{ platform_etc_dir }}/certs/key.pem" - dtls.certfile = "{{ platform_etc_dir }}/certs/cert.pem" - dtls.cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" - } - -} - -gateway.lwm2m { - - ## How long time the connection will be disconnected if the - ## connection is established but no bytes received - idle_timeout = 30s - - ## To control whether write statistics data into ETS table - ## for dashbord to read. - enable_stats = true - - ## When publishing or subscribing, prefix all topics with a mountpoint string. - mountpoint = "lwm2m/%u" - - xml_dir = "{{ platform_etc_dir }}/lwm2m_xml" - - lifetime_min = 1s - lifetime_max = 86400s - qmode_time_windonw = 22 - auto_observe = false - - ## always | contains_object_list - update_msg_publish_condition = contains_object_list - - - translators { - command { - topic = "/dn/#" - qos = 0 - } - - response { - topic = "/up/resp" - qos = 0 - } - - notify { - topic = "/up/notify" - qos = 0 - } - - register { - topic = "/up/resp" - qos = 0 - } - - update { - topic = "/up/resp" - qos = 0 - } - } - - listeners.udp.default { - bind = 5783 - } -} - -gateway.exproto { - - ## How long time the connection will be disconnected if the - ## connection is established but no bytes received - idle_timeout = 30s - - ## To control whether write statistics data into ETS table - ## for dashbord to read. - enable_stats = true - - ## When publishing or subscribing, prefix all topics with a mountpoint string. - mountpoint = "" - - ## The gRPC server to accept requests - server { - bind = 9100 - #ssl.keyfile: - #ssl.certfile: - #ssl.cacertfile: - } - - handler { - address = "http://127.0.0.1:9001" - #ssl.keyfile: - #ssl.certfile: - #ssl.cacertfile: - } - - listeners.tcp.default { - bind = 7993 - acceptors = 8 - max_connections = 10240 - max_conn_rate = 1000 - } - #listeners.ssl.default: {} - #listeners.udp.default: {} - #listeners.dtls.default: {} -} +## No gateway by default. +## +## If you want to get how to config it, please see emqx_gateway.conf.example. diff --git a/apps/emqx_gateway/etc/emqx_gateway.conf.example b/apps/emqx_gateway/etc/emqx_gateway.conf.example new file mode 100644 index 000000000..184d998dc --- /dev/null +++ b/apps/emqx_gateway/etc/emqx_gateway.conf.example @@ -0,0 +1,346 @@ +##-------------------------------------------------------------------- +## EMQ X Gateway configurations +##-------------------------------------------------------------------- + +gateway.stomp { + + ## How long time the connection will be disconnected if the + ## connection is established but no bytes received + idle_timeout = 30s + + ## To control whether write statistics data into ETS table + ## for dashbord to read. + enable_stats = true + + ## When publishing or subscribing, prefix all topics with a mountpoint string. + mountpoint = "" + + frame { + max_headers = 10 + max_headers_length = 1024 + max_body_length = 8192 + } + + clientinfo_override { + username = "${Packet.headers.login}" + password = "${Packet.headers.passcode}" + } + + authentication: { + mechanism = password-based + backend = built-in-database + user_id_type = clientid + } + + listeners.tcp.default { + bind = 61613 + acceptors = 16 + max_connections = 1024000 + max_conn_rate = 1000 + + access_rules = [ + "allow all" + ] + + authentication: { + mechanism = password-based + backend = built-in-database + user_id_type = username + } + + ## TCP options + ## See ${example_common_tcp_options} for more information + tcp.active_n = 100 + tcp.backlog = 1024 + tcp.buffer = 4KB + } + + listeners.ssl.default { + bind = 61614 + acceptors = 16 + max_connections = 1024000 + max_conn_rate = 1000 + + ## TCP options + ## See ${example_common_tcp_options} for more information + tcp.active_n = 100 + tcp.backlog = 1024 + tcp.buffer = 4KB + + ## SSL options + ## See ${example_common_ssl_options} for more information + ssl.versions = ["tlsv1.3", "tlsv1.2", "tlsv1.1", "tlsv1"] + ssl.keyfile = "{{ platform_etc_dir }}/certs/key.pem" + ssl.certfile = "{{ platform_etc_dir }}/certs/cert.pem" + ssl.cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" + #ssl.verify = verify_none + #ssl.fail_if_no_peer_cert = false + #ssl.server_name_indication = disable + #ssl.secure_renegotiate = false + #ssl.reuse_sessions = false + #ssl.honor_cipher_order = false + #ssl.handshake_timeout = 15s + #ssl.depth = 10 + #ssl.password = foo + #ssl.dhfile = path-to-your-file + } +} + +gateway.coap { + + ## How long time the connection will be disconnected if the + ## connection is established but no bytes received + idle_timeout = 30s + + ## To control whether write statistics data into ETS table + ## for dashbord to read. + enable_stats = true + + ## When publishing or subscribing, prefix all topics with a mountpoint string. + mountpoint = "" + + ## Enable or disable connection mode + ## If true, you need to establish a connection before send any publish/subscribe + ## requests + ## + ## Default: false + #connection_required = false + + ## The Notification Message Type. + ## The notification message will be delivered to the CoAP client if a new + ## message received on an observed topic. + ## The type of delivered coap message can be set to: + ## - non: Non-confirmable + ## - con: Confirmable + ## - qos: Mapping from QoS type of the recevied message. + ## QoS0 -> non, QoS1,2 -> con. + ## + ## Enum: non | con | qos + ## Default: qos + #notify_type = qos + + ## The *Default QoS Level* indicator for subscribe request. + ## This option specifies the QoS level for the CoAP Client when establishing + ## a subscription membership, if the subscribe request is not carried `qos` + ## option. + ## The indicator can be set to: + ## - qos0, qos1, qos2: Fixed default QoS level + ## - coap: Dynamic QoS level by the message type of subscribe request + ## * qos0: If the subscribe request is non-confirmable + ## * qos1: If the subscribe request is confirmable + ## + ## Enum: qos0 | qos1 | qos2 | coap + ## Default: coap + #subscribe_qos = coap + + ## The *Default QoS Level* indicator for publish request. + ## This option specifies the QoS level for the CoAP Client when publishing a + ## message to EMQ X PUB/SUB system, if the publish request is not carried `qos` + ## option. + ## The indicator can be set to: + ## - qos0, qos1, qos2: Fixed default QoS level + ## - coap: Dynamic QoS level by the message type of publish request + ## * qos0: If the publish request is non-confirmable + ## * qos1: If the publish request is confirmable + ## + ## Enum: qos0 | qos1 | qos2 | coap + #publish_qos = coap + + listeners.udp.default { + bind = 5683 + max_connections = 102400 + max_conn_rate = 1000 + + ## UDP Options + ## See ${example_common_udp_options} for more information + udp.active_n = 100 + udp.buffer = 16KB + } + listeners.dtls.default { + bind = 5684 + acceptors = 4 + max_connections = 102400 + max_conn_rate = 1000 + + ## UDP Options + ## See ${example_common_udp_options} for more information + udp.active_n = 100 + udp.buffer = 16KB + + ## DTLS Options + ## See #{example_common_dtls_options} for more information + dtls.versions = ["dtlsv1.2", "dtlsv1"] + dtls.keyfile = "{{ platform_etc_dir }}/certs/key.pem" + dtls.certfile = "{{ platform_etc_dir }}/certs/cert.pem" + dtls.cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" + dtls.handshake_timeout = 15s + } +} + +gateway.mqttsn { + + ## How long time the connection will be disconnected if the + ## connection is established but no bytes received + idle_timeout = 30s + + ## To control whether write statistics data into ETS table + ## for dashbord to read. + enable_stats = true + + ## When publishing or subscribing, prefix all topics with a mountpoint string. + mountpoint = "" + + ## The MQTT-SN Gateway ID in ADVERTISE message. + gateway_id = 1 + + ## Enable broadcast this gateway to WLAN + broadcast = true + + ## To control whether accept and process the received + ## publish message with qos=-1. + enable_qos3 = true + + ## The pre-defined topic name corresponding to the pre-defined topic + ## id of N. + ## Note that the pre-defined topic id of 0 is reserved. + predefined = [ + { id = 1 + topic = "/predefined/topic/name/hello" + }, + { id = 2 + topic = "/predefined/topic/name/nice" + } + ] + + ### ClientInfo override + clientinfo_override { + username = "mqtt_sn_user" + password = "abc" + } + + listeners.udp.default { + bind = 1884 + max_connections = 10240000 + max_conn_rate = 1000 + } + + listeners.dtls.default { + bind = 1885 + acceptors = 4 + max_connections = 102400 + max_conn_rate = 1000 + + ## UDP Options + ## See ${example_common_udp_options} for more information + udp.active_n = 100 + udp.buffer = 16KB + + ## DTLS Options + ## See #{example_common_dtls_options} for more information + dtls.versions = ["dtlsv1.2", "dtlsv1"] + dtls.keyfile = "{{ platform_etc_dir }}/certs/key.pem" + dtls.certfile = "{{ platform_etc_dir }}/certs/cert.pem" + dtls.cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" + } + +} + +gateway.lwm2m { + + ## How long time the connection will be disconnected if the + ## connection is established but no bytes received + idle_timeout = 30s + + ## To control whether write statistics data into ETS table + ## for dashbord to read. + enable_stats = true + + ## When publishing or subscribing, prefix all topics with a mountpoint string. + mountpoint = "lwm2m/%u" + + xml_dir = "{{ platform_etc_dir }}/lwm2m_xml" + + ## + ## + lifetime_min = 1s + + lifetime_max = 86400s + + qmode_time_window = 22 + + auto_observe = false + + ## always | contains_object_list + update_msg_publish_condition = contains_object_list + + + translators { + command { + topic = "/dn/#" + qos = 0 + } + + response { + topic = "/up/resp" + qos = 0 + } + + notify { + topic = "/up/notify" + qos = 0 + } + + register { + topic = "/up/resp" + qos = 0 + } + + update { + topic = "/up/resp" + qos = 0 + } + } + + listeners.udp.default { + bind = 5783 + } +} + +gateway.exproto { + + ## How long time the connection will be disconnected if the + ## connection is established but no bytes received + idle_timeout = 30s + + ## To control whether write statistics data into ETS table + ## for dashbord to read. + enable_stats = true + + ## When publishing or subscribing, prefix all topics with a mountpoint string. + mountpoint = "" + + ## The gRPC server to accept requests + server { + bind = 9100 + #ssl.keyfile: + #ssl.certfile: + #ssl.cacertfile: + } + + handler { + address = "http://127.0.0.1:9001" + #ssl.keyfile: + #ssl.certfile: + #ssl.cacertfile: + } + + listeners.tcp.default { + bind = 7993 + acceptors = 8 + max_connections = 10240 + max_conn_rate = 1000 + } + #listeners.ssl.default: {} + #listeners.udp.default: {} + #listeners.dtls.default: {} +} diff --git a/apps/emqx_gateway/include/emqx_gateway.hrl b/apps/emqx_gateway/include/emqx_gateway.hrl index 5c0893cb2..8b2081a90 100644 --- a/apps/emqx_gateway/include/emqx_gateway.hrl +++ b/apps/emqx_gateway/include/emqx_gateway.hrl @@ -19,8 +19,6 @@ -type gateway_name() :: atom(). --type listener() :: #{}. - %% @doc The Gateway defination -type gateway() :: #{ name := gateway_name() diff --git a/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl b/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl index 543b2e169..af34a1754 100644 --- a/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl +++ b/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl @@ -81,10 +81,13 @@ %% Frame Module frame_mod :: atom(), %% Channel Module - chann_mod :: atom() + chann_mod :: atom(), + %% Listener Tag + listener :: listener() | undefined }). --type(state() :: #state{}). +-type listener() :: {GwName :: atom(), LisType :: atom(), LisName :: atom()}. +-type state() :: #state{}. -define(INFO_KEYS, [socktype, peername, sockname, sockstate, active_n]). -define(CONN_STATS, [recv_pkt, recv_msg, send_pkt, send_msg]). @@ -279,7 +282,8 @@ init_state(WrappedSock, Peername, Options, FrameMod, ChannMod) -> idle_timer = IdleTimer, oom_policy = OomPolicy, frame_mod = FrameMod, - chann_mod = ChannMod + chann_mod = ChannMod, + listener = maps:get(listener, Options, undefined) }. run_loop(Parent, State = #state{socket = Socket, diff --git a/apps/emqx_gateway/src/coap/emqx_coap_api.erl b/apps/emqx_gateway/src/coap/emqx_coap_api.erl index ab04269b8..e84a67860 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_api.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_api.erl @@ -52,8 +52,8 @@ request(post, #{body := Body, bindings := Bindings}) -> CT = maps:get(<<"content_type">>, Body, <<"text/plain">>), Token = maps:get(<<"token">>, Body, <<>>), Payload = maps:get(<<"payload">>, Body, <<>>), - WaitTime = maps:get(<<"timeout">>, Body, ?DEF_WAIT_TIME), - + BinWaitTime = maps:get(<<"timeout">>, Body, <<"10s">>), + {ok, WaitTime} = emqx_schema:to_duration_ms(BinWaitTime), Payload2 = parse_payload(CT, Payload), ReqType = erlang:binary_to_atom(Method), @@ -83,7 +83,7 @@ request_parameters() -> request_properties() -> properties([ {token, string, "message token, can be empty"} , {method, string, "request method type", ["get", "put", "post", "delete"]} - , {timeout, integer, "timespan for response"} + , {timeout, string, "timespan for response", "10s"} , {content_type, string, "payload type", [<<"text/plain">>, <<"application/json">>, <<"application/octet-stream">>]} , {payload, string, "payload"}]). diff --git a/apps/emqx_gateway/src/coap/emqx_coap_channel.erl b/apps/emqx_gateway/src/coap/emqx_coap_channel.erl index 7e69067c6..b6808c896 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_channel.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_channel.erl @@ -103,9 +103,15 @@ init(ConnInfo = #{peername := {PeerHost, _}, #{ctx := Ctx} = Config) -> Peercert = maps:get(peercert, ConnInfo, undefined), Mountpoint = maps:get(mountpoint, Config, <<>>), + ListenerId = case maps:get(listener, Config, undefined) of + undefined -> undefined; + {GwName, Type, LisName} -> + emqx_gateway_utils:listener_id(GwName, Type, LisName) + end, ClientInfo = set_peercert_infos( Peercert, #{ zone => default + , listener => ListenerId , protocol => 'coap' , peerhost => PeerHost , sockport => SockPort diff --git a/apps/emqx_gateway/src/coap/emqx_coap_impl.erl b/apps/emqx_gateway/src/coap/emqx_coap_impl.erl index 055eab759..5fd557def 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_impl.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_impl.erl @@ -100,8 +100,8 @@ start_listener(GwName, Ctx, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) -> Name = emqx_gateway_utils:listener_id(GwName, Type, LisName), - NCfg = Cfg#{ - ctx => Ctx, + NCfg = Cfg#{ctx => Ctx, + listener => {GwName, Type, LisName}, frame_mod => emqx_coap_frame, chann_mod => emqx_coap_channel }, diff --git a/apps/emqx_gateway/src/emqx_gateway.erl b/apps/emqx_gateway/src/emqx_gateway.erl index 596b47547..96cc5d4ae 100644 --- a/apps/emqx_gateway/src/emqx_gateway.erl +++ b/apps/emqx_gateway/src/emqx_gateway.erl @@ -20,11 +20,6 @@ -include("include/emqx_gateway.hrl"). -%% callbacks for emqx_config_handler --export([ pre_config_update/2 - , post_config_update/4 - ]). - %% Gateway APIs -export([ registered_gateway/0 , load/2 @@ -36,8 +31,6 @@ , list/0 ]). --export([update_rawconf/2]). - %%-------------------------------------------------------------------- %% APIs %%-------------------------------------------------------------------- @@ -73,6 +66,7 @@ lookup(Name) -> emqx_gateway_sup:lookup_gateway(Name). -spec update(gateway_name(), emqx_config:config()) -> ok | {error, any()}. +%% @doc This function only supports full configuration updates update(Name, Config) -> emqx_gateway_sup:update_gateway(Name, Config). @@ -84,37 +78,6 @@ start(Name) -> stop(Name) -> emqx_gateway_sup:stop_gateway_insta(Name). --spec update_rawconf(binary(), emqx_config:raw_config()) - -> ok - | {error, any()}. -update_rawconf(RawName, RawConfDiff) -> - case emqx:update_config([gateway], {RawName, RawConfDiff}) of - {ok, _Result} -> ok; - {error, Reason} -> {error, Reason} - end. - -%%-------------------------------------------------------------------- -%% Config Handler - --spec pre_config_update(emqx_config:update_request(), - emqx_config:raw_config()) -> - {ok, emqx_config:update_request()} | {error, term()}. -pre_config_update({RawName, RawConfDiff}, RawConf) -> - {ok, emqx_map_lib:deep_merge(RawConf, #{RawName => RawConfDiff})}. - --spec post_config_update(emqx_config:update_request(), emqx_config:config(), - emqx_config:config(), emqx_config:app_envs()) - -> ok | {ok, Result::any()} | {error, Reason::term()}. -post_config_update({RawName, _}, NewConfig, OldConfig, _AppEnvs) -> - GwName = binary_to_existing_atom(RawName), - SubConf = maps:get(GwName, NewConfig), - case maps:get(GwName, OldConfig, undefined) of - undefined -> - emqx_gateway:load(GwName, SubConf); - _ -> - emqx_gateway:update(GwName, SubConf) - end. - %%-------------------------------------------------------------------- %% Internal funcs %%-------------------------------------------------------------------- diff --git a/apps/emqx_gateway/src/emqx_gateway_api.erl b/apps/emqx_gateway/src/emqx_gateway_api.erl index 9037518c5..a517d1b83 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api.erl @@ -48,6 +48,7 @@ apis() -> , {"/gateway/:name", gateway_insta} , {"/gateway/:name/stats", gateway_insta_stats} ]. + %%-------------------------------------------------------------------- %% http handlers @@ -57,30 +58,51 @@ gateway(get, Request) -> undefined -> all; S0 -> binary_to_existing_atom(S0, utf8) end, - {200, emqx_gateway_http:gateways(Status)}. + {200, emqx_gateway_http:gateways(Status)}; +gateway(post, Request) -> + Body = maps:get(body, Request, #{}), + try + Name0 = maps:get(<<"name">>, Body), + GwName = binary_to_existing_atom(Name0), + case emqx_gateway_registry:lookup(GwName) of + undefined -> error(badarg); + _ -> + GwConf = maps:without([<<"name">>], Body), + case emqx_gateway_conf:load_gateway(GwName, GwConf) of + ok -> + {204}; + {error, Reason} -> + return_http_error(500, Reason) + end + end + catch + error : {badkey, K} -> + return_http_error(400, [K, " is required"]); + error : badarg -> + return_http_error(404, "Bad gateway name") + end. gateway_insta(delete, #{bindings := #{name := Name0}}) -> with_gateway(Name0, fun(GwName, _) -> - _ = emqx_gateway:unload(GwName), - {204} + case emqx_gateway_conf:unload_gateway(GwName) of + ok -> + {204}; + {error, Reason} -> + return_http_error(400, Reason) + end end); gateway_insta(get, #{bindings := #{name := Name0}}) -> with_gateway(Name0, fun(_, _) -> - GwConf = filled_raw_confs([<<"gateway">>, Name0]), - LisConf = maps:get(<<"listeners">>, GwConf, #{}), - NLisConf = emqx_gateway_http:mapping_listener_m2l(Name0, LisConf), - {200, GwConf#{<<"listeners">> => NLisConf}} + GwConf = emqx_gateway_conf:gateway(Name0), + {200, GwConf#{<<"name">> => Name0}} end); -gateway_insta(put, #{body := GwConf0, +gateway_insta(put, #{body := GwConf, bindings := #{name := Name0} }) -> - with_gateway(Name0, fun(_, _) -> - GwConf = maps:without([<<"authentication">>, <<"listeners">>], GwConf0), - case emqx_gateway:update_rawconf(Name0, GwConf) of + with_gateway(Name0, fun(GwName, _) -> + case emqx_gateway_conf:update_gateway(GwName, GwConf) of ok -> {200}; - {error, not_found} -> - return_http_error(404, "Gateway not found"); {error, Reason} -> return_http_error(500, Reason) end @@ -89,13 +111,6 @@ gateway_insta(put, #{body := GwConf0, gateway_insta_stats(get, _Req) -> return_http_error(401, "Implement it later (maybe 5.1)"). -filled_raw_confs(Path) -> - RawConf = emqx_config:fill_defaults( - emqx_config:get_root_raw(Path) - ), - Confs = emqx_map_lib:deep_get(Path, RawConf), - emqx_map_lib:jsonable_map(Confs). - %%-------------------------------------------------------------------- %% Swagger defines %%-------------------------------------------------------------------- @@ -122,6 +137,16 @@ swagger("/gateway", get) -> , responses => #{ <<"200">> => schema_gateway_overview_list() } }; +swagger("/gateway", post) -> + #{ description => <<"Load a gateway">> + , requestBody => schema_gateway_conf() + , responses => + #{ <<"400">> => schema_bad_request() + , <<"404">> => schema_not_found() + , <<"500">> => schema_internal_error() + , <<"204">> => schema_no_content() + } + }; swagger("/gateway/:name", get) -> #{ description => <<"Get the gateway configurations">> , parameters => params_gateway_name_in_path() @@ -189,7 +214,7 @@ schema_gateway_overview_list() -> #{ type => object , properties => properties_gateway_overview() }, - <<"Gateway Overview list">> + <<"Gateway list">> ). %% XXX: This is whole confs for all type gateways. It is used to fill the @@ -202,6 +227,7 @@ schema_gateway_overview_list() -> <<"name">> => <<"authenticator1">>, <<"server_type">> => <<"built-in-database">>, <<"user_id_type">> => <<"clientid">>}, + <<"name">> => <<"coap">>, <<"enable">> => true, <<"enable_stats">> => true,<<"heartbeat">> => <<"30s">>, <<"idle_timeout">> => <<"30s">>, @@ -219,6 +245,7 @@ schema_gateway_overview_list() -> -define(EXPROTO_GATEWAY_CONFS, #{<<"enable">> => true, + <<"name">> => <<"exproto">>, <<"enable_stats">> => true, <<"handler">> => #{<<"address">> => <<"http://127.0.0.1:9001">>}, @@ -236,6 +263,7 @@ schema_gateway_overview_list() -> -define(LWM2M_GATEWAY_CONFS, #{<<"auto_observe">> => false, + <<"name">> => <<"lwm2m">>, <<"enable">> => true, <<"enable_stats">> => true, <<"idle_timeout">> => <<"30s">>, @@ -264,6 +292,7 @@ schema_gateway_overview_list() -> #{<<"password">> => <<"abc">>, <<"username">> => <<"mqtt_sn_user">>}, <<"enable">> => true, + <<"name">> => <<"mqtt-sn">>, <<"enable_qos3">> => true,<<"enable_stats">> => true, <<"gateway_id">> => 1,<<"idle_timeout">> => <<"30s">>, <<"listeners">> => [ @@ -290,6 +319,7 @@ schema_gateway_overview_list() -> #{<<"password">> => <<"${Packet.headers.passcode}">>, <<"username">> => <<"${Packet.headers.login}">>}, <<"enable">> => true, + <<"name">> => <<"stomp">>, <<"enable_stats">> => true, <<"frame">> => #{<<"max_body_length">> => 8192,<<"max_headers">> => 10, diff --git a/apps/emqx_gateway/src/emqx_gateway_api_authn.erl b/apps/emqx_gateway/src/emqx_gateway_api_authn.erl index 85eb4ddc7..ac5ee17a5 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_authn.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_authn.erl @@ -18,8 +18,144 @@ -behaviour(minirest_api). +-import(emqx_gateway_http, + [ return_http_error/2 + , schema_bad_request/0 + , schema_not_found/0 + , schema_internal_error/0 + , schema_no_content/0 + , with_gateway/2 + , checks/2 + ]). + %% minirest behaviour callbacks -export([api_spec/0]). +%% http handlers +-export([authn/2]). + +%% internal export for emqx_gateway_api_listeners module +-export([schema_authn/0]). + +%%-------------------------------------------------------------------- +%% minirest behaviour callbacks +%%-------------------------------------------------------------------- + api_spec() -> - {[], []}. + {metadata(apis()), []}. + +apis() -> + [ {"/gateway/:name/authentication", authn} + ]. + +%%-------------------------------------------------------------------- +%% http handlers + +authn(get, #{bindings := #{name := Name0}}) -> + with_gateway(Name0, fun(GwName, _) -> + {200, emqx_gateway_http:authn(GwName)} + end); + +authn(put, #{bindings := #{name := Name0}, + body := Body}) -> + with_gateway(Name0, fun(GwName, _) -> + ok = emqx_gateway_http:update_authn(GwName, Body), + {204} + end); + +authn(post, #{bindings := #{name := Name0}, + body := Body}) -> + with_gateway(Name0, fun(GwName, _) -> + ok = emqx_gateway_http:add_authn(GwName, Body), + {204} + end); + +authn(delete, #{bindings := #{name := Name0}}) -> + with_gateway(Name0, fun(GwName, _) -> + ok = emqx_gateway_http:remove_authn(GwName), + {204} + end). + +%%-------------------------------------------------------------------- +%% Swagger defines +%%-------------------------------------------------------------------- + +metadata(APIs) -> + metadata(APIs, []). +metadata([], APIAcc) -> + lists:reverse(APIAcc); +metadata([{Path, Fun}|More], APIAcc) -> + Methods = [get, post, put, delete, patch], + Mds = lists:foldl(fun(M, Acc) -> + try + Acc#{M => swagger(Path, M)} + catch + error : function_clause -> + Acc + end + end, #{}, Methods), + metadata(More, [{Path, Mds, Fun} | APIAcc]). + +swagger("/gateway/:name/authentication", get) -> + #{ description => <<"Get the gateway authentication">> + , parameters => params_gateway_name_in_path() + , responses => + #{ <<"400">> => schema_bad_request() + , <<"404">> => schema_not_found() + , <<"500">> => schema_internal_error() + , <<"200">> => schema_authn() + } + }; +swagger("/gateway/:name/authentication", put) -> + #{ description => <<"Create the gateway authentication">> + , parameters => params_gateway_name_in_path() + , requestBody => schema_authn() + , responses => + #{ <<"400">> => schema_bad_request() + , <<"404">> => schema_not_found() + , <<"500">> => schema_internal_error() + , <<"204">> => schema_no_content() + } + }; +swagger("/gateway/:name/authentication", post) -> + #{ description => <<"Add authentication for the gateway">> + , parameters => params_gateway_name_in_path() + , requestBody => schema_authn() + , responses => + #{ <<"400">> => schema_bad_request() + , <<"404">> => schema_not_found() + , <<"500">> => schema_internal_error() + , <<"204">> => schema_no_content() + } + }; +swagger("/gateway/:name/authentication", delete) -> + #{ description => <<"Remove the gateway authentication">> + , parameters => params_gateway_name_in_path() + , responses => + #{ <<"400">> => schema_bad_request() + , <<"404">> => schema_not_found() + , <<"500">> => schema_internal_error() + , <<"204">> => schema_no_content() + } + }. + +%%-------------------------------------------------------------------- +%% params defines + +params_gateway_name_in_path() -> + [#{ name => name + , in => path + , schema => #{type => string} + , required => true + }]. + +%%-------------------------------------------------------------------- +%% schemas + +schema_authn() -> + #{ description => <<"OK">> + , content => #{ + 'application/json' => #{ + schema => minirest:ref(<<"AuthenticatorInstance">>) + }} + }. diff --git a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl index 386d6e1ea..41cc1fa26 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl @@ -108,7 +108,7 @@ clients_insta(get, #{ bindings := #{name := Name0, {?MODULE, format_channel_info}) of [ClientInfo] -> {200, ClientInfo}; - [ClientInfo|_More] -> + [ClientInfo | _More] -> ?LOG(warning, "More than one client info was returned on ~s", [ClientId]), {200, ClientInfo}; @@ -194,16 +194,16 @@ extra_sub_props(Props) -> %%-------------------------------------------------------------------- %% query funcs -query(Tab, {Qs, []}, Start, Limit) -> +query(Tab, {Qs, []}, Continuation, Limit) -> Ms = qs2ms(Qs), - emqx_mgmt_api:select_table(Tab, Ms, Start, Limit, - fun format_channel_info/1); + emqx_mgmt_api:select_table_with_count(Tab, Ms, Continuation, Limit, + fun format_channel_info/1); -query(Tab, {Qs, Fuzzy}, Start, Limit) -> +query(Tab, {Qs, Fuzzy}, Continuation, Limit) -> Ms = qs2ms(Qs), - MatchFun = match_fun(Ms, Fuzzy), - emqx_mgmt_api:traverse_table(Tab, MatchFun, Start, Limit, - fun format_channel_info/1). + FuzzyFilterFun = fuzzy_filter_fun(Fuzzy), + emqx_mgmt_api:select_table_with_count(Tab, {Ms, FuzzyFilterFun}, Continuation, Limit, + fun format_channel_info/1). qs2ms(Qs) -> {MtchHead, Conds} = qs2ms(Qs, 2, {#{}, []}), @@ -247,32 +247,26 @@ ms(created_at, X) -> #{session => #{created_at => X}}. %%-------------------------------------------------------------------- -%% Match funcs +%% Fuzzy filter funcs -match_fun(Ms, Fuzzy) -> - MsC = ets:match_spec_compile(Ms), +fuzzy_filter_fun(Fuzzy) -> REFuzzy = lists:map(fun({K, like, S}) -> {ok, RE} = re:compile(S), {K, like, RE} end, Fuzzy), - fun(Rows) -> - case ets:match_spec_run(Rows, MsC) of - [] -> []; - Ls -> - lists:filter(fun(E) -> - run_fuzzy_match(E, REFuzzy) - end, Ls) - end + fun(MsRaws) when is_list(MsRaws) -> + lists:filter( fun(E) -> run_fuzzy_filter(E, REFuzzy) end + , MsRaws) end. -run_fuzzy_match(_, []) -> +run_fuzzy_filter(_, []) -> true; -run_fuzzy_match(E = {_, #{clientinfo := ClientInfo}, _}, [{Key, _, RE}|Fuzzy]) -> +run_fuzzy_filter(E = {_, #{clientinfo := ClientInfo}, _}, [{Key, _, RE} | Fuzzy]) -> Val = case maps:get(Key, ClientInfo, "") of undefined -> ""; V -> V end, - re:run(Val, RE, [{capture, none}]) == match andalso run_fuzzy_match(E, Fuzzy). + re:run(Val, RE, [{capture, none}]) == match andalso run_fuzzy_filter(E, Fuzzy). %%-------------------------------------------------------------------- %% format funcs @@ -325,12 +319,12 @@ eval(Ls) -> eval(Ls, #{}). eval([], AccMap) -> AccMap; -eval([{K, Vx}|More], AccMap) -> +eval([{K, Vx} | More], AccMap) -> case valuex_get(K, Vx) of undefined -> eval(More, AccMap#{K => null}); Value -> eval(More, AccMap#{K => Value}) end; -eval([{K, Vx, Default}|More], AccMap) -> +eval([{K, Vx, Default} | More], AccMap) -> case valuex_get(K, Vx) of undefined -> eval(More, AccMap#{K => Default}); Value -> eval(More, AccMap#{K => Value}) @@ -369,7 +363,7 @@ metadata(APIs) -> metadata(APIs, []). metadata([], APIAcc) -> lists:reverse(APIAcc); -metadata([{Path, Fun}|More], APIAcc) -> +metadata([{Path, Fun} | More], APIAcc) -> Methods = [get, post, put, delete, patch], Mds = lists:foldl(fun(M, Acc) -> try diff --git a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl index 374f2841d..b3df7bb35 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl @@ -20,20 +20,23 @@ -import(emqx_gateway_http, [ return_http_error/2 - , with_gateway/2 - , checks/2 , schema_bad_request/0 , schema_not_found/0 , schema_internal_error/0 , schema_no_content/0 + , with_gateway/2 + , checks/2 ]). +-import(emqx_gateway_api_authn, [schema_authn/0]). + %% minirest behaviour callbacks -export([api_spec/0]). %% http handlers -export([ listeners/2 , listeners_insta/2 + , listeners_insta_authn/2 ]). %%-------------------------------------------------------------------- @@ -46,13 +49,15 @@ api_spec() -> apis() -> [ {"/gateway/:name/listeners", listeners} , {"/gateway/:name/listeners/:id", listeners_insta} + , {"/gateway/:name/listeners/:id/authentication", listeners_insta_authn} ]. + %%-------------------------------------------------------------------- %% http handlers listeners(get, #{bindings := #{name := Name0}}) -> with_gateway(Name0, fun(GwName, _) -> - {200, emqx_gateway_http:listeners(GwName)} + {200, emqx_gateway_conf:listeners(GwName)} end); listeners(post, #{bindings := #{name := Name0}, body := LConf}) -> @@ -69,13 +74,8 @@ listeners(post, #{bindings := #{name := Name0}, body := LConf}) -> undefined -> ListenerId = emqx_gateway_utils:listener_id( GwName, Type, LName), - case emqx_gateway_http:update_listener( - ListenerId, LConf) of - ok -> - {204}; - {error, Reason} -> - return_http_error(500, Reason) - end; + ok = emqx_gateway_http:add_listener(ListenerId, LConf), + {204}; _ -> return_http_error(400, "Listener name has occupied") end @@ -84,17 +84,13 @@ listeners(post, #{bindings := #{name := Name0}, body := LConf}) -> listeners_insta(delete, #{bindings := #{name := Name0, id := ListenerId0}}) -> ListenerId = emqx_mgmt_util:urldecode(ListenerId0), with_gateway(Name0, fun(_GwName, _) -> - case emqx_gateway_http:remove_listener(ListenerId) of - ok -> {204}; - {error, not_found} -> {204}; - {error, Reason} -> - return_http_error(500, Reason) - end + ok = emqx_gateway_http:remove_listener(ListenerId), + {204} end); listeners_insta(get, #{bindings := #{name := Name0, id := ListenerId0}}) -> ListenerId = emqx_mgmt_util:urldecode(ListenerId0), with_gateway(Name0, fun(_GwName, _) -> - case emqx_gateway_http:listener(ListenerId) of + case emqx_gateway_conf:listener(ListenerId) of {ok, Listener} -> {200, Listener}; {error, not_found} -> @@ -108,12 +104,38 @@ listeners_insta(put, #{body := LConf, }) -> ListenerId = emqx_mgmt_util:urldecode(ListenerId0), with_gateway(Name0, fun(_GwName, _) -> - case emqx_gateway_http:update_listener(ListenerId, LConf) of - ok -> - {204}; - {error, Reason} -> - return_http_error(500, Reason) - end + ok = emqx_gateway_http:update_listener(ListenerId, LConf), + {204} + end). + +listeners_insta_authn(get, #{bindings := #{name := Name0, + id := ListenerId0}}) -> + ListenerId = emqx_mgmt_util:urldecode(ListenerId0), + with_gateway(Name0, fun(GwName, _) -> + {200, emqx_gateway_http:authn(GwName, ListenerId)} + end); +listeners_insta_authn(post, #{body := Conf, + bindings := #{name := Name0, + id := ListenerId0}}) -> + ListenerId = emqx_mgmt_util:urldecode(ListenerId0), + with_gateway(Name0, fun(GwName, _) -> + ok = emqx_gateway_http:add_authn(GwName, ListenerId, Conf), + {204} + end); +listeners_insta_authn(put, #{body := Conf, + bindings := #{name := Name0, + id := ListenerId0}}) -> + ListenerId = emqx_mgmt_util:urldecode(ListenerId0), + with_gateway(Name0, fun(GwName, _) -> + ok = emqx_gateway_http:update_authn(GwName, ListenerId, Conf), + {204} + end); +listeners_insta_authn(delete, #{bindings := #{name := Name0, + id := ListenerId0}}) -> + ListenerId = emqx_mgmt_util:urldecode(ListenerId0), + with_gateway(Name0, fun(GwName, _) -> + ok = emqx_gateway_http:remove_authn(GwName, ListenerId), + {204} end). %%-------------------------------------------------------------------- @@ -190,6 +212,52 @@ swagger("/gateway/:name/listeners/:id", put) -> , <<"500">> => schema_internal_error() , <<"200">> => schema_no_content() } + }; +swagger("/gateway/:name/listeners/:id/authentication", get) -> + #{ description => <<"Get the listener's authentication info">> + , parameters => params_gateway_name_in_path() + ++ params_listener_id_in_path() + , responses => + #{ <<"400">> => schema_bad_request() + , <<"404">> => schema_not_found() + , <<"500">> => schema_internal_error() + , <<"200">> => schema_authn() + } + }; +swagger("/gateway/:name/listeners/:id/authentication", post) -> + #{ description => <<"Add authentication for the listener">> + , parameters => params_gateway_name_in_path() + ++ params_listener_id_in_path() + , requestBody => schema_authn() + , responses => + #{ <<"400">> => schema_bad_request() + , <<"404">> => schema_not_found() + , <<"500">> => schema_internal_error() + , <<"204">> => schema_no_content() + } + }; +swagger("/gateway/:name/listeners/:id/authentication", put) -> + #{ description => <<"Update authentication for the listener">> + , parameters => params_gateway_name_in_path() + ++ params_listener_id_in_path() + , requestBody => schema_authn() + , responses => + #{ <<"400">> => schema_bad_request() + , <<"404">> => schema_not_found() + , <<"500">> => schema_internal_error() + , <<"204">> => schema_no_content() + } + }; +swagger("/gateway/:name/listeners/:id/authentication", delete) -> + #{ description => <<"Remove authentication for the listener">> + , parameters => params_gateway_name_in_path() + ++ params_listener_id_in_path() + , responses => + #{ <<"400">> => schema_bad_request() + , <<"404">> => schema_not_found() + , <<"500">> => schema_internal_error() + , <<"204">> => schema_no_content() + } }. %%-------------------------------------------------------------------- @@ -301,7 +369,6 @@ raw_properties_common_listener() -> <<"Listener type. Enum: tcp, udp, ssl, dtls">>, [<<"tcp">>, <<"ssl">>, <<"udp">>, <<"dtls">>]} , {running, boolean, <<"Listener running status">>} - %% FIXME: , {bind, string, <<"Listener bind address or port">>} , {acceptors, integer, <<"Listener acceptors number">>} , {access_rules, {array, string}, <<"Listener Access rules for client">>} diff --git a/apps/emqx_gateway/src/emqx_gateway_app.erl b/apps/emqx_gateway/src/emqx_gateway_app.erl index d90942220..1f8d226e2 100644 --- a/apps/emqx_gateway/src/emqx_gateway_app.erl +++ b/apps/emqx_gateway/src/emqx_gateway_app.erl @@ -22,20 +22,17 @@ -export([start/2, stop/1]). --define(CONF_CALLBACK_MODULE, emqx_gateway). - start(_StartType, _StartArgs) -> {ok, Sup} = emqx_gateway_sup:start_link(), emqx_gateway_cli:load(), load_default_gateway_applications(), load_gateway_by_default(), - emqx_config_handler:add_handler([gateway], ?CONF_CALLBACK_MODULE), + emqx_gateway_conf:load(), {ok, Sup}. stop(_State) -> + emqx_gateway_conf:unload(), emqx_gateway_cli:unload(), - %% XXX: No api now - %emqx_config_handler:remove_handler([gateway], ?MODULE), ok. %%-------------------------------------------------------------------- diff --git a/apps/emqx_gateway/src/emqx_gateway_cli.erl b/apps/emqx_gateway/src/emqx_gateway_cli.erl index 6ccb444f0..fce969130 100644 --- a/apps/emqx_gateway/src/emqx_gateway_cli.erl +++ b/apps/emqx_gateway/src/emqx_gateway_cli.erl @@ -53,50 +53,77 @@ gateway(["list"]) -> lists:foreach(fun(#{name := Name} = Gateway) -> %% TODO: More infos: listeners?, connected? Status = maps:get(status, Gateway, stopped), - emqx_ctl:print("Gateway(name=~s, status=~s)~n", - [Name, Status]) + print("Gateway(name=~s, status=~s)~n", [Name, Status]) end, emqx_gateway:list()); gateway(["lookup", Name]) -> case emqx_gateway:lookup(atom(Name)) of undefined -> - emqx_ctl:print("undefined~n"); + print("undefined~n"); Info -> - emqx_ctl:print("~p~n", [Info]) + print("~p~n", [Info]) + end; + +gateway(["load", Name, Conf]) -> + case emqx_gateway_conf:load_gateway( + bin(Name), + emqx_json:decode(Conf, [return_maps]) + ) of + ok -> + print("ok~n"); + {error, Reason} -> + print("Error: ~p~n", [Reason]) + end; + +gateway(["unload", Name]) -> + case emqx_gateway_conf:unload_gateway(bin(Name)) of + ok -> + print("ok~n"); + {error, Reason} -> + print("Error: ~p~n", [Reason]) end; gateway(["stop", Name]) -> - case emqx_gateway:stop(atom(Name)) of + case emqx_gateway_conf:update_gateway( + bin(Name), + #{<<"enable">> => <<"false">>} + ) of ok -> - emqx_ctl:print("ok~n"); + print("ok~n"); {error, Reason} -> - emqx_ctl:print("Error: ~p~n", [Reason]) + print("Error: ~p~n", [Reason]) end; gateway(["start", Name]) -> - case emqx_gateway:start(atom(Name)) of + case emqx_gateway_conf:update_gateway( + bin(Name), + #{<<"enable">> => <<"true">>} + ) of ok -> - emqx_ctl:print("ok~n"); + print("ok~n"); {error, Reason} -> - emqx_ctl:print("Error: ~p~n", [Reason]) + print("Error: ~p~n", [Reason]) end; gateway(_) -> - %% TODO: create/remove APIs emqx_ctl:usage([ {"gateway list", "List all gateway"} , {"gateway lookup ", "Lookup a gateway detailed informations"} + , {"gateway load ", + "Load a gateway with config"} + , {"gateway unload ", + "Unload the gateway"} , {"gateway stop ", - "Stop a gateway instance"} + "Stop the gateway"} , {"gateway start ", - "Start a gateway instance"} + "Start the gateway"} ]). 'gateway-registry'(["list"]) -> lists:foreach( fun({Name, #{cbkmod := CbMod}}) -> - emqx_ctl:print("Registered Name: ~s, Callback Module: ~s~n", [Name, CbMod]) + print("Registered Name: ~s, Callback Module: ~s~n", [Name, CbMod]) end, emqx_gateway_registry:list()); @@ -106,11 +133,11 @@ gateway(_) -> ]). 'gateway-clients'(["list", Name]) -> - %% FIXME: page me. for example: --limit 100 --page 10 ??? + %% FIXME: page me? InfoTab = emqx_gateway_cm:tabname(info, Name), case ets:info(InfoTab) of undefined -> - emqx_ctl:print("Bad Gateway Name.~n"); + print("Bad Gateway Name.~n"); _ -> dump(InfoTab, client) end; @@ -118,17 +145,17 @@ gateway(_) -> 'gateway-clients'(["lookup", Name, ClientId]) -> ChanTab = emqx_gateway_cm:tabname(chan, Name), case ets:lookup(ChanTab, bin(ClientId)) of - [] -> emqx_ctl:print("Not Found.~n"); + [] -> print("Not Found.~n"); [Chann] -> InfoTab = emqx_gateway_cm:tabname(info, Name), [ChannInfo] = ets:lookup(InfoTab, Chann), - print({client, ChannInfo}) + print_record({client, ChannInfo}) end; 'gateway-clients'(["kick", Name, ClientId]) -> case emqx_gateway_cm:kick_session(Name, bin(ClientId)) of - ok -> emqx_ctl:print("ok~n"); - _ -> emqx_ctl:print("Not Found.~n") + ok -> print("ok~n"); + _ -> print("Not Found.~n") end; 'gateway-clients'(_) -> @@ -144,11 +171,11 @@ gateway(_) -> Tab = emqx_gateway_metrics:tabname(Name), case ets:info(Tab) of undefined -> - emqx_ctl:print("Bad Gateway Name.~n"); + print("Bad Gateway Name.~n"); _ -> lists:foreach( fun({K, V}) -> - emqx_ctl:print("~-30s: ~w~n", [K, V]) + print("~-30s: ~w~n", [K, V]) end, lists:sort(ets:tab2list(Tab))) end; @@ -177,10 +204,10 @@ dump(_Table, _, '$end_of_table', Result) -> lists:reverse(Result); dump(Table, Tag, Key, Result) -> - PrintValue = [print({Tag, Record}) || Record <- ets:lookup(Table, Key)], + PrintValue = [print_record({Tag, Record}) || Record <- ets:lookup(Table, Key)], dump(Table, Tag, ets:next(Table, Key), [PrintValue | Result]). -print({client, {_, Infos, Stats}}) -> +print_record({client, {_, Infos, Stats}}) -> ClientInfo = maps:get(clientinfo, Infos, #{}), ConnInfo = maps:get(conninfo, Infos, #{}), _Session = maps:get(session, Infos, #{}), @@ -202,11 +229,14 @@ print({client, {_, Infos, Stats}}) -> connected_at => ConnectedAt }, - emqx_ctl:print("Client(~s, username=~s, peername=~s, " - "clean_start=~s, keepalive=~w, " - "subscriptions=~w, delivered_msgs=~w, " - "connected=~s, created_at=~w, connected_at=~w)~n", - [format(K, maps:get(K, Info)) || K <- InfoKeys]). + print("Client(~s, username=~s, peername=~s, " + "clean_start=~s, keepalive=~w, " + "subscriptions=~w, delivered_msgs=~w, " + "connected=~s, created_at=~w, connected_at=~w)~n", + [format(K, maps:get(K, Info)) || K <- InfoKeys]). + +print(S) -> emqx_ctl:print(S). +print(S, A) -> emqx_ctl:print(S, A). format(_, undefined) -> undefined; diff --git a/apps/emqx_gateway/src/emqx_gateway_conf.erl b/apps/emqx_gateway/src/emqx_gateway_conf.erl new file mode 100644 index 000000000..d89c518a1 --- /dev/null +++ b/apps/emqx_gateway/src/emqx_gateway_conf.erl @@ -0,0 +1,382 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @doc The gateway configuration management module +-module(emqx_gateway_conf). + +%% Load/Unload +-export([ load/0 + , unload/0 + ]). + +%% APIs +-export([ gateway/1 + , load_gateway/2 + , update_gateway/2 + , unload_gateway/1 + ]). + +-export([ listeners/1 + , listener/1 + , add_listener/3 + , update_listener/3 + , remove_listener/2 + ]). + +-export([ add_authn/2 + , add_authn/3 + , update_authn/2 + , update_authn/3 + , remove_authn/1 + , remove_authn/2 + ]). + +%% internal exports +-export([ unconvert_listeners/1 + , convert_listeners/2 + ]). + +%% callbacks for emqx_config_handler +-export([ pre_config_update/2 + , post_config_update/4 + ]). + +-type atom_or_bin() :: atom() | binary(). +-type ok_or_err() :: ok_or_err(). +-type listener_ref() :: {ListenerType :: atom_or_bin(), + ListenerName :: atom_or_bin()}. + +%%-------------------------------------------------------------------- +%% Load/Unload +%%-------------------------------------------------------------------- + +-spec load() -> ok. +load() -> + emqx_config_handler:add_handler([gateway], ?MODULE). + +-spec unload() -> ok. +unload() -> + emqx_config_handler:remove_handler([gateway]). + +%%-------------------------------------------------------------------- +%% APIs + +-spec load_gateway(atom_or_bin(), map()) -> ok_or_err(). +load_gateway(GwName, Conf) -> + NConf = case maps:take(<<"listeners">>, Conf) of + error -> Conf; + {Ls, Conf1} -> + Conf1#{<<"listeners">> => unconvert_listeners(Ls)} + end, + update({?FUNCTION_NAME, bin(GwName), NConf}). + +%% @doc convert listener array to map +unconvert_listeners(Ls) when is_list(Ls) -> + lists:foldl(fun(Lis, Acc) -> + {[Type, Name], Lis1} = maps_key_take([<<"type">>, <<"name">>], Lis), + emqx_map_lib:deep_merge(Acc, #{Type => #{Name => Lis1}}) + end, #{}, Ls). + +maps_key_take(Ks, M) -> + maps_key_take(Ks, M, []). +maps_key_take([], M, Acc) -> + {lists:reverse(Acc), M}; +maps_key_take([K|Ks], M, Acc) -> + case maps:take(K, M) of + error -> throw(bad_key); + {V, M1} -> + maps_key_take(Ks, M1, [V|Acc]) + end. + +-spec update_gateway(atom_or_bin(), map()) -> ok_or_err(). +update_gateway(GwName, Conf0) -> + Conf = maps:without([listeners, authentication, + <<"listeners">>, <<"authentication">>], Conf0), + update({?FUNCTION_NAME, bin(GwName), Conf}). + +-spec unload_gateway(atom_or_bin()) -> ok_or_err(). +unload_gateway(GwName) -> + update({?FUNCTION_NAME, bin(GwName)}). + +%% @doc Get the gateway configurations. +%% Missing fields are filled with default values. This function is typically +%% used to show the user what configuration value is currently in effect. +-spec gateway(atom_or_bin()) -> map(). +gateway(GwName0) -> + GwName = bin(GwName0), + Path = [<<"gateway">>, GwName], + RawConf = emqx_config:fill_defaults( + emqx_config:get_root_raw(Path) + ), + Confs = emqx_map_lib:jsonable_map( + emqx_map_lib:deep_get(Path, RawConf)), + LsConf = maps:get(<<"listeners">>, Confs, #{}), + Confs#{<<"listeners">> => convert_listeners(GwName, LsConf)}. + +%% @doc convert listeners map to array +convert_listeners(GwName, Ls) when is_map(Ls) -> + lists:append([do_convert_listener(GwName, Type, maps:to_list(Conf)) + || {Type, Conf} <- maps:to_list(Ls)]). + +do_convert_listener(GwName, Type, Conf) -> + [begin + ListenerId = emqx_gateway_utils:listener_id(GwName, Type, LName), + Running = emqx_gateway_utils:is_running(ListenerId, LConf), + bind2str( + LConf#{ + id => ListenerId, + type => Type, + name => LName, + running => Running + }) + end || {LName, LConf} <- Conf, is_map(LConf)]. + +bind2str(LConf = #{bind := Bind}) when is_integer(Bind) -> + maps:put(bind, integer_to_binary(Bind), LConf); +bind2str(LConf = #{<<"bind">> := Bind}) when is_integer(Bind) -> + maps:put(<<"bind">>, integer_to_binary(Bind), LConf); +bind2str(LConf = #{bind := Bind}) when is_binary(Bind) -> + LConf; +bind2str(LConf = #{<<"bind">> := Bind}) when is_binary(Bind) -> + LConf. + +-spec listeners(atom_or_bin()) -> [map()]. +listeners(GwName0) -> + GwName = bin(GwName0), + RawConf = emqx_config:fill_defaults( + emqx_config:get_root_raw([<<"gateway">>])), + Listeners = emqx_map_lib:jsonable_map( + emqx_map_lib:deep_get( + [<<"gateway">>, GwName, <<"listeners">>], RawConf)), + convert_listeners(GwName, Listeners). + +-spec listener(binary()) -> {ok, map()} | {error, not_found} | {error, any()}. +listener(ListenerId) -> + {GwName, Type, LName} = emqx_gateway_utils:parse_listener_id(ListenerId), + RootConf = emqx_config:fill_defaults( + emqx_config:get_root_raw([<<"gateway">>])), + try + Path = [<<"gateway">>, GwName, <<"listeners">>, Type, LName], + LConf = emqx_map_lib:deep_get(Path, RootConf), + Running = emqx_gateway_utils:is_running( + binary_to_existing_atom(ListenerId), LConf), + {ok, emqx_map_lib:jsonable_map( + LConf#{ + id => ListenerId, + type => Type, + name => LName, + running => Running})} + catch + error : {config_not_found, _} -> + {error, not_found}; + _Class : Reason -> + {error, Reason} + end. + +-spec add_listener(atom_or_bin(), listener_ref(), map()) -> ok_or_err(). +add_listener(GwName, ListenerRef, Conf) -> + update({?FUNCTION_NAME, bin(GwName), bin(ListenerRef), Conf}). + +-spec update_listener(atom_or_bin(), listener_ref(), map()) -> ok_or_err(). +update_listener(GwName, ListenerRef, Conf) -> + update({?FUNCTION_NAME, bin(GwName), bin(ListenerRef), Conf}). + +-spec remove_listener(atom_or_bin(), listener_ref()) -> ok_or_err(). +remove_listener(GwName, ListenerRef) -> + update({?FUNCTION_NAME, bin(GwName), bin(ListenerRef)}). + +-spec add_authn(atom_or_bin(), map()) -> ok_or_err(). +add_authn(GwName, Conf) -> + update({?FUNCTION_NAME, bin(GwName), Conf}). + +-spec add_authn(atom_or_bin(), listener_ref(), map()) -> ok_or_err(). +add_authn(GwName, ListenerRef, Conf) -> + update({?FUNCTION_NAME, bin(GwName), bin(ListenerRef), Conf}). + +-spec update_authn(atom_or_bin(), map()) -> ok_or_err(). +update_authn(GwName, Conf) -> + update({?FUNCTION_NAME, bin(GwName), Conf}). + +-spec update_authn(atom_or_bin(), listener_ref(), map()) -> ok_or_err(). +update_authn(GwName, ListenerRef, Conf) -> + update({?FUNCTION_NAME, bin(GwName), bin(ListenerRef), Conf}). + +-spec remove_authn(atom_or_bin()) -> ok_or_err(). +remove_authn(GwName) -> + update({?FUNCTION_NAME, bin(GwName)}). + +-spec remove_authn(atom_or_bin(), listener_ref()) -> ok_or_err(). +remove_authn(GwName, ListenerRef) -> + update({?FUNCTION_NAME, bin(GwName), bin(ListenerRef)}). + +%% @private +update(Req) -> + res(emqx:update_config([gateway], Req)). + +res({ok, _Result}) -> ok; +res({error, {pre_config_update,emqx_gateway_conf,Reason}}) -> {error, Reason}; +res({error, Reason}) -> {error, Reason}. + +bin({LType, LName}) -> + {bin(LType), bin(LName)}; +bin(A) when is_atom(A) -> + atom_to_binary(A); +bin(B) when is_binary(B) -> + B. + +%%-------------------------------------------------------------------- +%% Config Handler +%%-------------------------------------------------------------------- + +-spec pre_config_update(emqx_config:update_request(), + emqx_config:raw_config()) -> + {ok, emqx_config:update_request()} | {error, term()}. +pre_config_update({load_gateway, GwName, Conf}, RawConf) -> + case maps:get(GwName, RawConf, undefined) of + undefined -> + {ok, emqx_map_lib:deep_merge(RawConf, #{GwName => Conf})}; + _ -> + {error, already_exist} + end; +pre_config_update({update_gateway, GwName, Conf}, RawConf) -> + case maps:get(GwName, RawConf, undefined) of + undefined -> + {error, not_found}; + _ -> + NConf = maps:without([<<"listeners">>, + <<"authentication">>], Conf), + {ok, emqx_map_lib:deep_merge(RawConf, #{GwName => NConf})} + end; +pre_config_update({unload_gateway, GwName}, RawConf) -> + {ok, maps:remove(GwName, RawConf)}; + +pre_config_update({add_listener, GwName, {LType, LName}, Conf}, RawConf) -> + case emqx_map_lib:deep_get( + [GwName, <<"listeners">>, LType, LName], RawConf, undefined) of + undefined -> + NListener = #{LType => #{LName => Conf}}, + {ok, emqx_map_lib:deep_merge( + RawConf, + #{GwName => #{<<"listeners">> => NListener}})}; + _ -> + {error, already_exist} + end; +pre_config_update({update_listener, GwName, {LType, LName}, Conf}, RawConf) -> + case emqx_map_lib:deep_get( + [GwName, <<"listeners">>, LType, LName], RawConf, undefined) of + undefined -> + {error, not_found}; + _OldConf -> + NListener = #{LType => #{LName => Conf}}, + {ok, emqx_map_lib:deep_merge( + RawConf, + #{GwName => #{<<"listeners">> => NListener}})} + + end; +pre_config_update({remove_listener, GwName, {LType, LName}}, RawConf) -> + {ok, emqx_map_lib:deep_remove( + [GwName, <<"listeners">>, LType, LName], RawConf)}; + +pre_config_update({add_authn, GwName, Conf}, RawConf) -> + case emqx_map_lib:deep_get( + [GwName, <<"authentication">>], RawConf, undefined) of + undefined -> + {ok, emqx_map_lib:deep_merge( + RawConf, + #{GwName => #{<<"authentication">> => Conf}})}; + _ -> + {error, already_exist} + end; +pre_config_update({add_authn, GwName, {LType, LName}, Conf}, RawConf) -> + case emqx_map_lib:deep_get( + [GwName, <<"listeners">>, LType, LName], + RawConf, undefined) of + undefined -> + {error, not_found}; + Listener -> + case maps:get(<<"authentication">>, Listener, undefined) of + undefined -> + NListener = maps:put(<<"authentication">>, Conf, Listener), + NGateway = #{GwName => + #{<<"listeners">> => + #{LType => #{LName => NListener}}}}, + {ok, emqx_map_lib:deep_merge(RawConf, NGateway)}; + _ -> + {error, already_exist} + end + end; +pre_config_update({update_authn, GwName, Conf}, RawConf) -> + case emqx_map_lib:deep_get( + [GwName, <<"authentication">>], RawConf, undefined) of + undefined -> + {error, not_found}; + _ -> + {ok, emqx_map_lib:deep_merge( + RawConf, + #{GwName => #{<<"authentication">> => Conf}})} + end; +pre_config_update({update_authn, GwName, {LType, LName}, Conf}, RawConf) -> + case emqx_map_lib:deep_get( + [GwName, <<"listeners">>, LType, LName], + RawConf, undefined) of + undefined -> + {error, not_found}; + Listener -> + case maps:get(<<"authentication">>, Listener, undefined) of + undefined -> + {error, not_found}; + Auth -> + NListener = maps:put( + <<"authentication">>, + emqx_map_lib:deep_merge(Auth, Conf), + Listener + ), + NGateway = #{GwName => + #{<<"listeners">> => + #{LType => #{LName => NListener}}}}, + {ok, emqx_map_lib:deep_merge(RawConf, NGateway)} + end + end; +pre_config_update({remove_authn, GwName}, RawConf) -> + {ok, emqx_map_lib:deep_remove( + [GwName, <<"authentication">>], RawConf)}; +pre_config_update({remove_authn, GwName, {LType, LName}}, RawConf) -> + Path = [GwName, <<"listeners">>, LType, LName, <<"authentication">>], + {ok, emqx_map_lib:deep_remove(Path, RawConf)}; + +pre_config_update(UnknownReq, _RawConf) -> + logger:error("Unknown configuration update request: ~0p", [UnknownReq]), + {error, badreq}. + +-spec post_config_update(emqx_config:update_request(), emqx_config:config(), + emqx_config:config(), emqx_config:app_envs()) + -> ok | {ok, Result::any()} | {error, Reason::term()}. + +post_config_update(Req, NewConfig, OldConfig, _AppEnvs) -> + [_Tag, GwName0|_] = tuple_to_list(Req), + GwName = binary_to_existing_atom(GwName0), + + case {maps:get(GwName, NewConfig, undefined), + maps:get(GwName, OldConfig, undefined)} of + {undefined, undefined} -> + ok; %% nothing to change + {undefined, Old} when is_map(Old) -> + emqx_gateway:unload(GwName); + {New, undefined} when is_map(New) -> + emqx_gateway:load(GwName, New); + {New, Old} when is_map(New), is_map(Old) -> + emqx_gateway:update(GwName, New) + end. diff --git a/apps/emqx_gateway/src/emqx_gateway_ctx.erl b/apps/emqx_gateway/src/emqx_gateway_ctx.erl index 8022c3797..54714974a 100644 --- a/apps/emqx_gateway/src/emqx_gateway_ctx.erl +++ b/apps/emqx_gateway/src/emqx_gateway_ctx.erl @@ -29,8 +29,8 @@ -type context() :: #{ %% Gateway Name gwname := gateway_name() - %% Autenticator - , auth := emqx_authn:chain_id() | undefined + %% Authentication chains + , auth := [emqx_authentication:chain_name()] | undefined %% The ConnectionManager PID , cm := pid() }. @@ -66,12 +66,8 @@ | {error, any()}. authenticate(_Ctx = #{auth := undefined}, ClientInfo) -> {ok, mountpoint(ClientInfo)}; -authenticate(_Ctx = #{auth := ChainId}, ClientInfo0) -> - ClientInfo = ClientInfo0#{ - zone => default, - listener => {tcp, default}, - chain_id => ChainId - }, +authenticate(_Ctx = #{auth := _ChainName}, ClientInfo0) -> + ClientInfo = ClientInfo0#{zone => default}, case emqx_access_control:authenticate(ClientInfo) of {ok, _} -> {ok, mountpoint(ClientInfo)}; diff --git a/apps/emqx_gateway/src/emqx_gateway_http.erl b/apps/emqx_gateway/src/emqx_gateway_http.erl index ed8e511c7..e5c927a1a 100644 --- a/apps/emqx_gateway/src/emqx_gateway_http.erl +++ b/apps/emqx_gateway/src/emqx_gateway_http.erl @@ -24,12 +24,20 @@ -export([ gateways/1 ]). -%% Mgmt APIs - listeners --export([ listeners/1 - , listener/1 +%% Mgmt APIs +-export([ add_listener/2 , remove_listener/1 , update_listener/2 - , mapping_listener_m2l/2 + ]). + +-export([ authn/1 + , authn/2 + , add_authn/2 + , add_authn/3 + , update_authn/2 + , update_authn/3 + , remove_authn/1 + , remove_authn/2 ]). %% Mgmt APIs - clients @@ -129,86 +137,69 @@ get_listeners_status(GwName, Config) -> %% Mgmt APIs - listeners %%-------------------------------------------------------------------- --spec listeners(atom() | binary()) -> list(). -listeners(GwName) when is_atom(GwName) -> - listeners(atom_to_binary(GwName)); -listeners(GwName) -> - RawConf = emqx_config:fill_defaults( - emqx_config:get_root_raw([<<"gateway">>])), - Listeners = emqx_map_lib:jsonable_map( - emqx_map_lib:deep_get( - [<<"gateway">>, GwName, <<"listeners">>], RawConf)), - mapping_listener_m2l(GwName, Listeners). - --spec listener(binary()) -> {ok, map()} | {error, not_found} | {error, any()}. -listener(ListenerId) -> - {GwName, Type, LName} = emqx_gateway_utils:parse_listener_id(ListenerId), - RootConf = emqx_config:fill_defaults( - emqx_config:get_root_raw([<<"gateway">>])), - try - Path = [<<"gateway">>, GwName, <<"listeners">>, Type, LName], - LConf = emqx_map_lib:deep_get(Path, RootConf), - Running = is_running(binary_to_existing_atom(ListenerId), LConf), - {ok, emqx_map_lib:jsonable_map( - LConf#{ - id => ListenerId, - type => Type, - name => LName, - running => Running})} - catch - error : {config_not_found, _} -> - {error, not_found}; - _Class : Reason -> - {error, Reason} - end. - -mapping_listener_m2l(GwName, Listeners0) -> - Listeners = maps:to_list(Listeners0), - lists:append([listener(GwName, Type, maps:to_list(Conf)) - || {Type, Conf} <- Listeners]). - -listener(GwName, Type, Conf) -> - [begin - ListenerId = emqx_gateway_utils:listener_id(GwName, Type, LName), - Running = is_running(ListenerId, LConf), - LConf#{ - id => ListenerId, - type => Type, - name => LName, - running => Running - } - end || {LName, LConf} <- Conf, is_map(LConf)]. - -is_running(ListenerId, #{<<"bind">> := ListenOn0}) -> - ListenOn = emqx_gateway_utils:parse_listenon(ListenOn0), - try esockd:listener({ListenerId, ListenOn}) of - Pid when is_pid(Pid)-> - true - catch _:_ -> - false - end. - --spec remove_listener(binary()) -> ok | {error, not_found} | {error, any()}. -remove_listener(ListenerId) -> - {GwName, Type, Name} = emqx_gateway_utils:parse_listener_id(ListenerId), - LConf = emqx:get_raw_config( - [<<"gateway">>, GwName, <<"listeners">>, Type] - ), - NLConf = maps:remove(Name, LConf), - emqx_gateway:update_rawconf( - GwName, - #{<<"listeners">> => #{Type => NLConf}} - ). - --spec update_listener(atom() | binary(), map()) -> ok | {error, any()}. -update_listener(ListenerId, NewConf0) -> +-spec add_listener(atom() | binary(), map()) -> ok. +add_listener(ListenerId, NewConf0) -> {GwName, Type, Name} = emqx_gateway_utils:parse_listener_id(ListenerId), NewConf = maps:without([<<"id">>, <<"name">>, <<"type">>, <<"running">>], NewConf0), - emqx_gateway:update_rawconf( - GwName, - #{<<"listeners">> => #{Type => #{Name => NewConf}} - }). + confexp(emqx_gateway_conf:add_listener(GwName, {Type, Name}, NewConf)). + +-spec update_listener(atom() | binary(), map()) -> ok. +update_listener(ListenerId, NewConf0) -> + {GwName, Type, Name} = emqx_gateway_utils:parse_listener_id(ListenerId), + + NewConf = maps:without([<<"id">>, <<"name">>, + <<"type">>, <<"running">>], NewConf0), + confexp(emqx_gateway_conf:update_listener(GwName, {Type, Name}, NewConf)). + +-spec remove_listener(binary()) -> ok. +remove_listener(ListenerId) -> + {GwName, Type, Name} = emqx_gateway_utils:parse_listener_id(ListenerId), + confexp(emqx_gateway_conf:remove_listener(GwName, {Type, Name})). + +-spec authn(gateway_name()) -> map(). +authn(GwName) -> + Path = [gateway, GwName, authentication], + emqx_map_lib:jsonable_map(emqx:get_config(Path)). + +-spec authn(gateway_name(), binary()) -> map(). +authn(GwName, ListenerId) -> + {_, Type, Name} = emqx_gateway_utils:parse_listener_id(ListenerId), + Path = [gateway, GwName, listeners, Type, Name, authentication], + emqx_map_lib:jsonable_map(emqx:get_config(Path)). + +-spec add_authn(gateway_name(), map()) -> ok. +add_authn(GwName, AuthConf) -> + confexp(emqx_gateway_conf:add_authn(GwName, AuthConf)). + +-spec add_authn(gateway_name(), binary(), map()) -> ok. +add_authn(GwName, ListenerId, AuthConf) -> + {_, Type, Name} = emqx_gateway_utils:parse_listener_id(ListenerId), + confexp(emqx_gateway_conf:add_authn(GwName, {Type, Name}, AuthConf)). + +-spec update_authn(gateway_name(), map()) -> ok. +update_authn(GwName, AuthConf) -> + confexp(emqx_gateway_conf:update_authn(GwName, AuthConf)). + +-spec update_authn(gateway_name(), binary(), map()) -> ok. +update_authn(GwName, ListenerId, AuthConf) -> + {_, Type, Name} = emqx_gateway_utils:parse_listener_id(ListenerId), + confexp(emqx_gateway_conf:update_authn(GwName, {Type, Name}, AuthConf)). + +-spec remove_authn(gateway_name()) -> ok. +remove_authn(GwName) -> + confexp(emqx_gateway_conf:remove_authn(GwName)). + +-spec remove_authn(gateway_name(), binary()) -> ok. +remove_authn(GwName, ListenerId) -> + {_, Type, Name} = emqx_gateway_utils:parse_listener_id(ListenerId), + confexp(emqx_gateway_conf:remove_authn(GwName, {Type, Name})). + +confexp(ok) -> ok; +confexp({error, not_found}) -> + error({update_conf_error, not_found}); +confexp({error, already_exist}) -> + error({update_conf_error, already_exist}). %%-------------------------------------------------------------------- %% Mgmt APIs - clients @@ -328,10 +319,22 @@ with_gateway(GwName0, Fun) -> catch error : badname -> return_http_error(404, "Bad gateway name"); + %% Exceptions from: checks/2 error : {miss_param, K} -> return_http_error(400, [K, " is required"]); + %% Exceptions from emqx_gateway_utils:parse_listener_id/1 error : {invalid_listener_id, Id} -> return_http_error(400, ["invalid listener id: ", Id]); + %% Exceptions from: emqx:get_config/1 + error : {config_not_found, Path0} -> + Path = lists:concat( + lists:join(".", lists:map(fun to_list/1, Path0))), + return_http_error(404, "Resource not found. path: " ++ Path); + %% Exceptions from: confexp/1 + error : {update_conf_error, not_found} -> + return_http_error(404, "Resource not found"); + error : {update_conf_error, already_exist} -> + return_http_error(400, "Resource already exist"); Class : Reason : Stk -> ?LOG(error, "Uncatched error: {~p, ~p}, stacktrace: ~0p", [Class, Reason, Stk]), @@ -348,6 +351,11 @@ checks([K|Ks], Map) -> error({miss_param, K}) end. +to_list(A) when is_atom(A) -> + atom_to_list(A); +to_list(B) when is_binary(B) -> + binary_to_list(B). + %%-------------------------------------------------------------------- %% common schemas diff --git a/apps/emqx_gateway/src/emqx_gateway_insta_sup.erl b/apps/emqx_gateway/src/emqx_gateway_insta_sup.erl index 39115f114..172193e28 100644 --- a/apps/emqx_gateway/src/emqx_gateway_insta_sup.erl +++ b/apps/emqx_gateway/src/emqx_gateway_insta_sup.erl @@ -43,6 +43,7 @@ name :: gateway_name(), config :: emqx_config:config(), ctx :: emqx_gateway_ctx:context(), + authns :: [emqx_authentication:chain_name()], status :: stopped | running, child_pids :: [pid()], gw_state :: emqx_gateway_impl:state() | undefined, @@ -94,16 +95,23 @@ init([Gateway, Ctx, _GwDscrptr]) -> State = #state{ ctx = Ctx, name = GwName, + authns = [], config = Config, child_pids = [], status = stopped, created_at = erlang:system_time(millisecond) }, - case cb_gateway_load(State) of - {error, Reason} -> - {stop, {load_gateway_failure, Reason}}; - {ok, NState} -> - {ok, NState} + case maps:get(enable, Config, true) of + false -> + ?LOG(info, "Skipp to start ~s gateway due to disabled", [GwName]), + {ok, State}; + true -> + case cb_gateway_load(State) of + {error, Reason} -> + {stop, {load_gateway_failure, Reason}}; + {ok, NState} -> + {ok, NState} + end end. handle_call(info, _From, State) -> @@ -174,9 +182,9 @@ handle_info(Info, State) -> ?LOG(warning, "Unexcepted info: ~p", [Info]), {noreply, State}. -terminate(_Reason, State = #state{ctx = Ctx, child_pids = Pids}) -> +terminate(_Reason, State = #state{child_pids = Pids}) -> Pids /= [] andalso (_ = cb_gateway_unload(State)), - _ = do_deinit_authn(maps:get(auth, Ctx, undefined)), + _ = do_deinit_authn(State#state.authns), ok. code_change(_OldVsn, State, _Extra) -> @@ -197,60 +205,108 @@ detailed_gateway_info(State) -> %% Internal funcs %%-------------------------------------------------------------------- -do_init_authn(GwName, Config) -> - case maps:get(authentication, Config, #{enable => false}) of - #{enable := false} -> undefined; - AuthCfg when is_map(AuthCfg) -> - case maps:get(enable, AuthCfg, true) of - false -> - undefined; - _ -> - %% TODO: Implement Authentication - GwName - %case emqx_authn:create_chain(#{id => ChainId}) of - % {ok, _ChainInfo} -> - % case emqx_authn:create_authenticator(ChainId, AuthCfg) of - % {ok, _} -> ChainId; - % {error, Reason} -> - % ?LOG(error, "Failed to create authentication ~p", [Reason]), - % throw({bad_authentication, Reason}) - % end; - % {error, Reason} -> - % ?LOG(error, "Failed to create authentication chain: ~p", [Reason]), - % throw({bad_chain, {ChainId, Reason}}) - %end. - end; - _ -> - undefined +%% same with emqx_authentication:global_chain/1 +global_chain(mqtt) -> + 'mqtt:global'; +global_chain('mqtt-sn') -> + 'mqtt-sn:global'; +global_chain(coap) -> + 'coap:global'; +global_chain(lwm2m) -> + 'lwm2m:global'; +global_chain(stomp) -> + 'stomp:global'; +global_chain(_) -> + 'unknown:global'. + +listener_chain(GwName, Type, LisName) -> + emqx_gateway_utils:listener_id(GwName, Type, LisName). + +%% There are two layer authentication configs +%% stomp.authn +%% / \ +%% listeners.tcp.defautl.authn *.ssl.default.authn +%% + +init_authn(GwName, Config) -> + Authns = authns(GwName, Config), + try + do_init_authn(Authns, []) + catch + throw : Reason = {badauth, _} -> + do_deinit_authn(proplists:get_keys(Authns)), + throw(Reason) end. -do_deinit_authn(undefined) -> - ok; -do_deinit_authn(AuthnRef) -> - %% TODO: - ?LOG(warning, "Failed to clean authn ~p, not suppported now", [AuthnRef]). - %case emqx_authn:delete_chain(AuthnRef) of - % ok -> ok; - % {error, {not_found, _}} -> - % ?LOG(warning, "Failed to clean authentication chain: ~s, " - % "reason: not_found", [AuthnRef]); - % {error, Reason} -> - % ?LOG(error, "Failed to clean authentication chain: ~s, " - % "reason: ~p", [AuthnRef, Reason]) - %end. +do_init_authn([], Names) -> + Names; +do_init_authn([{_ChainName, _AuthConf = #{enable := false}}|More], Names) -> + do_init_authn(More, Names); +do_init_authn([{ChainName, AuthConf}|More], Names) when is_map(AuthConf) -> + _ = application:ensure_all_started(emqx_authn), + do_create_authn_chain(ChainName, AuthConf), + do_init_authn(More, [ChainName|Names]); +do_init_authn([_BadConf|More], Names) -> + do_init_authn(More, Names). -do_update_one_by_one(NCfg0, State = #state{ - ctx = Ctx, +authns(GwName, Config) -> + Listeners = maps:to_list(maps:get(listeners, Config, #{})), + lists:append( + [ [{listener_chain(GwName, LisType, LisName), authn_conf(Opts)} + || {LisName, Opts} <- maps:to_list(LisNames) ] + || {LisType, LisNames} <- Listeners]) + ++ [{global_chain(GwName), authn_conf(Config)}]. + +authn_conf(Conf) -> + maps:get(authentication, Conf, #{enable => false}). + +do_create_authn_chain(ChainName, AuthConf) -> + case ensure_chain(ChainName) of + ok -> + case emqx_authentication:create_authenticator(ChainName, AuthConf) of + {ok, _} -> ok; + {error, Reason} -> + ?LOG(error, "Failed to create authenticator chain ~s, " + "reason: ~p, config: ~p", + [ChainName, Reason, AuthConf]), + throw({badauth, Reason}) + end; + {error, Reason} -> + ?LOG(error, "Falied to create authn chain ~s, reason ~p", + [ChainName, Reason]), + throw({badauth, Reason}) + end. + +ensure_chain(ChainName) -> + case emqx_authentication:create_chain(ChainName) of + {ok, _ChainInfo} -> + ok; + {error, {already_exists, _}} -> + ok; + {error, Reason} -> + {error, Reason} + end. + +do_deinit_authn(Names) -> + lists:foreach(fun(ChainName) -> + case emqx_authentication:delete_chain(ChainName) of + ok -> ok; + {error, {not_found, _}} -> ok; + {error, Reason} -> + ?LOG(error, "Failed to clean authentication chain: ~s, " + "reason: ~p", [ChainName, Reason]) + end + end, Names). + +do_update_one_by_one(NCfg, State = #state{ + name = GwName, config = OCfg, status = Status}) -> - - NCfg = emqx_map_lib:deep_merge(OCfg, NCfg0), - OEnable = maps:get(enable, OCfg, true), - NEnable = maps:get(enable, NCfg0, OEnable), + NEnable = maps:get(enable, NCfg, OEnable), - OAuth = maps:get(authentication, OCfg, undefined), - NAuth = maps:get(authentication, NCfg0, OAuth), + OAuths = authns(GwName, OCfg), + NAuths = authns(GwName, NCfg), if Status == stopped, NEnable == true -> @@ -259,19 +315,15 @@ do_update_one_by_one(NCfg0, State = #state{ Status == stopped, NEnable == false -> {ok, State#state{config = NCfg}}; Status == running, NEnable == true -> - NState = case NAuth == OAuth of + NState = case NAuths == OAuths of true -> State; false -> %% Reset Authentication first - _ = do_deinit_authn(maps:get(auth, Ctx, undefined)), - NCtx = Ctx#{ - auth => do_init_authn( - State#state.name, - NCfg - ) - }, - State#state{ctx = NCtx} + _ = do_deinit_authn(State#state.authns), + AuthnNames = init_authn(State#state.name, NCfg), + State#state{authns = AuthnNames} end, + %% XXX: minimum impact update ??? cb_gateway_update(NCfg, NState); Status == running, NEnable == false -> case cb_gateway_unload(State) of @@ -289,6 +341,7 @@ cb_gateway_unload(State = #state{name = GwName, #{cbkmod := CbMod} = emqx_gateway_registry:lookup(GwName), CbMod:on_gateway_unload(Gateway, GwState), {ok, State#state{child_pids = [], + authns = [], status = stopped, gw_state = undefined, started_at = undefined, @@ -300,6 +353,8 @@ cb_gateway_unload(State = #state{name = GwName, [GwName, GwState, Class, Reason, Stk]), {error, {Class, Reason, Stk}} + after + _ = do_deinit_authn(State#state.authns) end. %% @doc 1. Create Authentcation Context @@ -311,38 +366,33 @@ cb_gateway_load(State = #state{name = GwName, ctx = Ctx}) -> Gateway = detailed_gateway_info(State), - - case maps:get(enable, Config, true) of - false -> - ?LOG(info, "Skipp to start ~s gateway due to disabled", [GwName]); - true -> - try - AuthnRef = do_init_authn(GwName, Config), - NCtx = Ctx#{auth => AuthnRef}, - #{cbkmod := CbMod} = emqx_gateway_registry:lookup(GwName), - case CbMod:on_gateway_load(Gateway, NCtx) of - {error, Reason} -> - do_deinit_authn(AuthnRef), - throw({callback_return_error, Reason}); - {ok, ChildPidOrSpecs, GwState} -> - ChildPids = start_child_process(ChildPidOrSpecs), - {ok, State#state{ - ctx = NCtx, - status = running, - child_pids = ChildPids, - gw_state = GwState, - stopped_at = undefined, - started_at = erlang:system_time(millisecond) - }} - end - catch - Class : Reason1 : Stk -> - ?LOG(error, "Failed to load ~s gateway (~0p, ~0p) " - "crashed: {~p, ~p}, stacktrace: ~0p", - [GwName, Gateway, Ctx, - Class, Reason1, Stk]), - {error, {Class, Reason1, Stk}} - end + try + AuthnNames = init_authn(GwName, Config), + NCtx = Ctx#{auth => AuthnNames}, + #{cbkmod := CbMod} = emqx_gateway_registry:lookup(GwName), + case CbMod:on_gateway_load(Gateway, NCtx) of + {error, Reason} -> + do_deinit_authn(AuthnNames), + throw({callback_return_error, Reason}); + {ok, ChildPidOrSpecs, GwState} -> + ChildPids = start_child_process(ChildPidOrSpecs), + {ok, State#state{ + ctx = NCtx, + authns = AuthnNames, + status = running, + child_pids = ChildPids, + gw_state = GwState, + stopped_at = undefined, + started_at = erlang:system_time(millisecond) + }} + end + catch + Class : Reason1 : Stk -> + ?LOG(error, "Failed to load ~s gateway (~0p, ~0p) " + "crashed: {~p, ~p}, stacktrace: ~0p", + [GwName, Gateway, Ctx, + Class, Reason1, Stk]), + {error, {Class, Reason1, Stk}} end. cb_gateway_update(Config, diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl index 3811d56c6..bb0bf9dbe 100644 --- a/apps/emqx_gateway/src/emqx_gateway_schema.erl +++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl @@ -50,11 +50,11 @@ namespace() -> gateway. roots() -> [gateway]. fields(gateway) -> - [{stomp, sc(ref(stomp))}, - {mqttsn, sc(ref(mqttsn))}, - {coap, sc(ref(coap))}, - {lwm2m, sc(ref(lwm2m))}, - {exproto, sc(ref(exproto))} + [{stomp, sc_meta(ref(stomp) , #{nullable => {true, recursively}})}, + {mqttsn, sc_meta(ref(mqttsn) , #{nullable => {true, recursively}})}, + {coap, sc_meta(ref(coap) , #{nullable => {true, recursively}})}, + {lwm2m, sc_meta(ref(lwm2m) , #{nullable => {true, recursively}})}, + {exproto, sc_meta(ref(exproto), #{nullable => {true, recursively}})} ]; fields(stomp) -> @@ -92,10 +92,10 @@ fields(coap) -> fields(lwm2m) -> [ {xml_dir, sc(binary())} - , {lifetime_min, sc(duration())} - , {lifetime_max, sc(duration())} - , {qmode_time_windonw, sc(integer())} - , {auto_observe, sc(boolean())} + , {lifetime_min, sc(duration(), "1s")} + , {lifetime_max, sc(duration(), "86400s")} + , {qmode_time_window, sc(integer(), 22)} + , {auto_observe, sc(boolean(), false)} , {update_msg_publish_condition, sc(hoconsc:union([always, contains_object_list]))} , {translators, sc(ref(translators))} , {listeners, sc(ref(udp_listeners))} @@ -154,8 +154,8 @@ fields(udp_tcp_listeners) -> ]; fields(tcp_listener) -> - [ - %% some special confs for tcp listener + [ %% some special confs for tcp listener + {acceptors, sc(integer(), 16)} ] ++ tcp_opts() ++ proxy_protocol_opts() ++ @@ -163,7 +163,9 @@ fields(tcp_listener) -> fields(ssl_listener) -> fields(tcp_listener) ++ - ssl_opts(); + [{ssl, sc_meta(hoconsc:ref(emqx_schema, "listener_ssl_opts"), + #{desc => "SSL listener options"})}]; + fields(udp_listener) -> [ @@ -173,8 +175,11 @@ fields(udp_listener) -> common_listener_opts(); fields(dtls_listener) -> + [ {acceptors, sc(integer(), 16)} + ] ++ fields(udp_listener) ++ - dtls_opts(); + [{dtls, sc_meta(ref(dtls_opts), + #{desc => "DTLS listener options"})}]; fields(udp_opts) -> [ {active_n, sc(integer(), 100)} @@ -184,65 +189,37 @@ fields(udp_opts) -> , {reuseaddr, sc(boolean(), true)} ]; -fields(dtls_listener_ssl_opts) -> - Base = emqx_schema:fields("listener_ssl_opts"), - DtlsVers = hoconsc:mk( - typerefl:alias("string", list(atom())), - #{ default => default_dtls_vsns(), - converter => fun (Vsns) -> - [dtls_vsn(iolist_to_binary(V)) || V <- Vsns] - end - }), - Ciphers = sc(hoconsc:array(string()), default_ciphers()), - lists:keydelete( - "handshake_timeout", 1, - lists:keyreplace( - "ciphers", 1, - lists:keyreplace("versions", 1, Base, {"versions", DtlsVers}), - {"ciphers", Ciphers} - ) - ). +fields(dtls_opts) -> + emqx_schema:server_ssl_opts_schema( + #{ depth => 10 + , reuse_sessions => true + , versions => dtls_all_available + , ciphers => dtls_all_available + }, false). -default_ciphers() -> - ["ECDHE-ECDSA-AES256-GCM-SHA384", - "ECDHE-RSA-AES256-GCM-SHA384", "ECDHE-ECDSA-AES256-SHA384", "ECDHE-RSA-AES256-SHA384", - "ECDHE-ECDSA-DES-CBC3-SHA", "ECDH-ECDSA-AES256-GCM-SHA384", "ECDH-RSA-AES256-GCM-SHA384", - "ECDH-ECDSA-AES256-SHA384", "ECDH-RSA-AES256-SHA384", "DHE-DSS-AES256-GCM-SHA384", - "DHE-DSS-AES256-SHA256", "AES256-GCM-SHA384", "AES256-SHA256", - "ECDHE-ECDSA-AES128-GCM-SHA256", "ECDHE-RSA-AES128-GCM-SHA256", - "ECDHE-ECDSA-AES128-SHA256", "ECDHE-RSA-AES128-SHA256", "ECDH-ECDSA-AES128-GCM-SHA256", - "ECDH-RSA-AES128-GCM-SHA256", "ECDH-ECDSA-AES128-SHA256", "ECDH-RSA-AES128-SHA256", - "DHE-DSS-AES128-GCM-SHA256", "DHE-DSS-AES128-SHA256", "AES128-GCM-SHA256", "AES128-SHA256", - "ECDHE-ECDSA-AES256-SHA", "ECDHE-RSA-AES256-SHA", "DHE-DSS-AES256-SHA", - "ECDH-ECDSA-AES256-SHA", "ECDH-RSA-AES256-SHA", "AES256-SHA", "ECDHE-ECDSA-AES128-SHA", - "ECDHE-RSA-AES128-SHA", "DHE-DSS-AES128-SHA", "ECDH-ECDSA-AES128-SHA", - "ECDH-RSA-AES128-SHA", "AES128-SHA" - ] ++ psk_ciphers(). - -psk_ciphers() -> - ["PSK-AES128-CBC-SHA", "PSK-AES256-CBC-SHA", - "PSK-3DES-EDE-CBC-SHA", "PSK-RC4-SHA" - ]. - -% authentication() -> -% hoconsc:union( -% [ undefined -% , hoconsc:ref(emqx_authn_mnesia, config) -% , hoconsc:ref(emqx_authn_mysql, config) -% , hoconsc:ref(emqx_authn_pgsql, config) -% , hoconsc:ref(emqx_authn_mongodb, standalone) -% , hoconsc:ref(emqx_authn_mongodb, 'replica-set') -% , hoconsc:ref(emqx_authn_mongodb, 'sharded-cluster') -% , hoconsc:ref(emqx_authn_redis, standalone) -% , hoconsc:ref(emqx_authn_redis, cluster) -% , hoconsc:ref(emqx_authn_redis, sentinel) -% , hoconsc:ref(emqx_authn_http, get) -% , hoconsc:ref(emqx_authn_http, post) -% , hoconsc:ref(emqx_authn_jwt, 'hmac-based') -% , hoconsc:ref(emqx_authn_jwt, 'public-key') -% , hoconsc:ref(emqx_authn_jwt, 'jwks') -% , hoconsc:ref(emqx_enhanced_authn_scram_mnesia, config) -% ]). +authentication() -> + sc_meta(hoconsc:union( + [ hoconsc:ref(emqx_authn_mnesia, config) + , hoconsc:ref(emqx_authn_mysql, config) + , hoconsc:ref(emqx_authn_pgsql, config) + , hoconsc:ref(emqx_authn_mongodb, standalone) + , hoconsc:ref(emqx_authn_mongodb, 'replica-set') + , hoconsc:ref(emqx_authn_mongodb, 'sharded-cluster') + , hoconsc:ref(emqx_authn_redis, standalone) + , hoconsc:ref(emqx_authn_redis, cluster) + , hoconsc:ref(emqx_authn_redis, sentinel) + , hoconsc:ref(emqx_authn_http, get) + , hoconsc:ref(emqx_authn_http, post) + , hoconsc:ref(emqx_authn_jwt, 'hmac-based') + , hoconsc:ref(emqx_authn_jwt, 'public-key') + , hoconsc:ref(emqx_authn_jwt, 'jwks') + , hoconsc:ref(emqx_enhanced_authn_scram_mnesia, config) + ]), + #{nullable => {true, recursively}, + desc => +"""Default authentication configs for all of the gateway listeners.
+For per-listener overrides see authentication +in listener configs"""}). gateway_common_options() -> [ {enable, sc(boolean(), true)} @@ -250,16 +227,15 @@ gateway_common_options() -> , {idle_timeout, sc(duration(), <<"30s">>)} , {mountpoint, sc(binary(), <<>>)} , {clientinfo_override, sc(ref(clientinfo_override))} - , {authentication, sc(hoconsc:lazy(map()))} + , {authentication, authentication()} ]. common_listener_opts() -> [ {enable, sc(boolean(), true)} , {bind, sc(union(ip_port(), integer()))} - , {acceptors, sc(integer(), 16)} , {max_connections, sc(integer(), 1024)} , {max_conn_rate, sc(integer())} - %, {rate_limit, sc(comma_separated_list())} + , {authentication, authentication()} , {mountpoint, sc(binary(), undefined)} , {access_rules, sc(hoconsc:array(string()), [])} ]. @@ -270,23 +246,11 @@ tcp_opts() -> udp_opts() -> [{udp, sc_meta(ref(udp_opts), #{})}]. -ssl_opts() -> - [{ssl, sc_meta(ref(emqx_schema, "listener_ssl_opts"), #{})}]. - -dtls_opts() -> - [{dtls, sc_meta(ref(dtls_listener_ssl_opts), #{})}]. - proxy_protocol_opts() -> - [ {proxy_protocol, sc(boolean())} - , {proxy_protocol_timeout, sc(duration())} + [ {proxy_protocol, sc(boolean(), false)} + , {proxy_protocol_timeout, sc(duration(), "15s")} ]. -default_dtls_vsns() -> - [<<"dtlsv1.2">>, <<"dtlsv1">>]. - -dtls_vsn(<<"dtlsv1.2">>) -> 'dtlsv1.2'; -dtls_vsn(<<"dtlsv1">>) -> 'dtlsv1'. - sc(Type) -> sc_meta(Type, #{}). diff --git a/apps/emqx_gateway/src/emqx_gateway_utils.erl b/apps/emqx_gateway/src/emqx_gateway_utils.erl index 6d19cbbcf..0f8ee99b0 100644 --- a/apps/emqx_gateway/src/emqx_gateway_utils.erl +++ b/apps/emqx_gateway/src/emqx_gateway_utils.erl @@ -33,6 +33,7 @@ , unix_ts_to_rfc3339/2 , listener_id/3 , parse_listener_id/1 + , is_running/2 ]). -export([ stringfy/1 @@ -117,13 +118,18 @@ format_listenon({Addr, Port}) when is_tuple(Addr) -> parse_listenon(Port) when is_integer(Port) -> Port; +parse_listenon(IpPort) when is_tuple(IpPort) -> + IpPort; parse_listenon(Str) when is_binary(Str) -> parse_listenon(binary_to_list(Str)); parse_listenon(Str) when is_list(Str) -> - case emqx_schema:to_ip_port(Str) of - {ok, R} -> R; - {error, _} -> - error({invalid_listenon_name, Str}) + try list_to_integer(Str) + catch _ : _ -> + case emqx_schema:to_ip_port(Str) of + {ok, R} -> R; + {error, _} -> + error({invalid_listenon_name, Str}) + end end. listener_id(GwName, Type, LisName) -> @@ -143,6 +149,15 @@ parse_listener_id(Id) -> _ : _ -> error({invalid_listener_id, Id}) end. +is_running(ListenerId, #{<<"bind">> := ListenOn0}) -> + ListenOn = emqx_gateway_utils:parse_listenon(ListenOn0), + try esockd:listener({ListenerId, ListenOn}) of + Pid when is_pid(Pid)-> + true + catch _:_ -> + false + end. + bin(A) when is_atom(A) -> atom_to_binary(A); bin(L) when is_list(L); is_binary(L) -> @@ -226,11 +241,7 @@ sock_opts(Name, Opts) -> %% Envs active_n(Options) -> - maps:get( - active_n, - maps:get(listener, Options, #{active_n => ?ACTIVE_N}), - ?ACTIVE_N - ). + maps:get(active_n, Options, ?ACTIVE_N). -spec idle_timeout(map()) -> pos_integer(). idle_timeout(Options) -> diff --git a/apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl b/apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl index 4893f6d00..e1dacab7f 100644 --- a/apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl +++ b/apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl @@ -139,7 +139,12 @@ init(ConnInfo = #{socktype := Socktype, GRpcChann = maps:get(handler, Options), PoolName = maps:get(pool_name, Options), NConnInfo = default_conninfo(ConnInfo), - ClientInfo = default_clientinfo(ConnInfo), + ListenerId = case maps:get(listener, Options, undefined) of + undefined -> undefined; + {GwName, Type, LisName} -> + emqx_gateway_utils:listener_id(GwName, Type, LisName) + end, + ClientInfo = maps:put(listener, ListenerId, default_clientinfo(ConnInfo)), Channel = #channel{ ctx = Ctx, gcli = #{channel => GRpcChann, pool_name => PoolName}, diff --git a/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl b/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl index 3e142f3dc..87170fff2 100644 --- a/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl +++ b/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl @@ -156,6 +156,7 @@ start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) -> Name = emqx_gateway_utils:listener_id(GwName, Type, LisName), NCfg = Cfg#{ ctx => Ctx, + listener => {GwName, Type, LisName}, frame_mod => emqx_exproto_frame, chann_mod => emqx_exproto_channel }, diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl index e25be88fc..c1648a037 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl @@ -89,9 +89,15 @@ init(ConnInfo = #{peername := {PeerHost, _}, #{ctx := Ctx} = Config) -> Peercert = maps:get(peercert, ConnInfo, undefined), Mountpoint = maps:get(mountpoint, Config, undefined), + ListenerId = case maps:get(listener, Config, undefined) of + undefined -> undefined; + {GwName, Type, LisName} -> + emqx_gateway_utils:listener_id(GwName, Type, LisName) + end, ClientInfo = set_peercert_infos( Peercert, #{ zone => default + , listener => ListenerId , protocol => lwm2m , peerhost => PeerHost , sockport => SockPort diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl index 649a14643..b5cce573f 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl @@ -102,6 +102,7 @@ start_listener(GwName, Ctx, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) -> Name = emqx_gateway_utils:listener_id(GwName, Type, LisName), NCfg = Cfg#{ ctx => Ctx + , listener => {GwName, Type, LisName} , frame_mod => emqx_coap_frame , chann_mod => emqx_lwm2m_channel }, diff --git a/apps/emqx_gateway/src/mqttsn/emqx_sn_channel.erl b/apps/emqx_gateway/src/mqttsn/emqx_sn_channel.erl index 4476abfea..98aaf3d31 100644 --- a/apps/emqx_gateway/src/mqttsn/emqx_sn_channel.erl +++ b/apps/emqx_gateway/src/mqttsn/emqx_sn_channel.erl @@ -116,9 +116,15 @@ init(ConnInfo = #{peername := {PeerHost, _}, Registry = maps:get(registry, Option), GwId = maps:get(gateway_id, Option), EnableQoS3 = maps:get(enable_qos3, Option, true), + ListenerId = case maps:get(listener, Option, undefined) of + undefined -> undefined; + {GwName, Type, LisName} -> + emqx_gateway_utils:listener_id(GwName, Type, LisName) + end, ClientInfo = set_peercert_infos( Peercert, #{ zone => default + , listener => ListenerId , protocol => 'mqtt-sn' , peerhost => PeerHost , sockport => SockPort diff --git a/apps/emqx_gateway/src/mqttsn/emqx_sn_impl.erl b/apps/emqx_gateway/src/mqttsn/emqx_sn_impl.erl index a79173cff..f5660e0dc 100644 --- a/apps/emqx_gateway/src/mqttsn/emqx_sn_impl.erl +++ b/apps/emqx_gateway/src/mqttsn/emqx_sn_impl.erl @@ -121,6 +121,7 @@ start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) -> Name = emqx_gateway_utils:listener_id(GwName, Type, LisName), NCfg = Cfg#{ ctx => Ctx, + listene => {GwName, Type, LisName}, frame_mod => emqx_sn_frame, chann_mod => emqx_sn_channel }, @@ -138,13 +139,13 @@ merge_default(Options) -> end. stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> - StopRet = stop_listener(GwName, LisName, Type, ListenOn, SocketOpts, Cfg), + StopRet = stop_listener(GwName, Type, LisName, ListenOn, SocketOpts, Cfg), ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), case StopRet of ok -> ?ULOG("Gateway ~s:~s:~s on ~s stopped.~n", [GwName, Type, LisName, ListenOnStr]); {error, Reason} -> - ?ELOG("Failed to stop gatewat ~s:~s:~s on ~s: ~0p~n", + ?ELOG("Failed to stop gateway ~s:~s:~s on ~s: ~0p~n", [GwName, Type, LisName, ListenOnStr, Reason]) end, StopRet. diff --git a/apps/emqx_gateway/src/stomp/emqx_stomp_channel.erl b/apps/emqx_gateway/src/stomp/emqx_stomp_channel.erl index 31b1904bb..4e045ac3c 100644 --- a/apps/emqx_gateway/src/stomp/emqx_stomp_channel.erl +++ b/apps/emqx_gateway/src/stomp/emqx_stomp_channel.erl @@ -109,10 +109,15 @@ init(ConnInfo = #{peername := {PeerHost, _}, sockname := {_, SockPort}}, Option) -> Peercert = maps:get(peercert, ConnInfo, undefined), Mountpoint = maps:get(mountpoint, Option, undefined), + ListenerId = case maps:get(listener, Option, undefined) of + undefined -> undefined; + {GwName, Type, LisName} -> + emqx_gateway_utils:listener_id(GwName, Type, LisName) + end, ClientInfo = setting_peercert_infos( Peercert, #{ zone => default - , listener => {tcp, default} + , listener => ListenerId , protocol => stomp , peerhost => PeerHost , sockport => SockPort diff --git a/apps/emqx_gateway/src/stomp/emqx_stomp_impl.erl b/apps/emqx_gateway/src/stomp/emqx_stomp_impl.erl index 9599ef6e3..a93240207 100644 --- a/apps/emqx_gateway/src/stomp/emqx_stomp_impl.erl +++ b/apps/emqx_gateway/src/stomp/emqx_stomp_impl.erl @@ -106,6 +106,7 @@ start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) -> Name = emqx_gateway_utils:listener_id(GwName, Type, LisName), NCfg = Cfg#{ ctx => Ctx, + listener => {GwName, Type, LisName}, %% Used for authn frame_mod => emqx_stomp_frame, chann_mod => emqx_stomp_channel }, diff --git a/apps/emqx_gateway/test/emqx_coap_SUITE.erl b/apps/emqx_gateway/test/emqx_coap_SUITE.erl index 8e7352a74..f55fdf88c 100644 --- a/apps/emqx_gateway/test/emqx_coap_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_coap_SUITE.erl @@ -34,7 +34,6 @@ gateway.coap connection_required = true subscribe_qos = qos1 publish_qos = qos1 - authentication = undefined listeners.udp.default {bind = 5683} @@ -113,24 +112,24 @@ t_publish(_Config) -> with_connection(Action). -t_publish_authz_deny(_Config) -> - Action = fun(Channel, Token) -> - Topic = <<"/abc">>, - Payload = <<"123">>, - InvalidToken = lists:reverse(Token), - - TopicStr = binary_to_list(Topic), - URI = ?PS_PREFIX ++ TopicStr ++ "?clientid=client1&token=" ++ InvalidToken, - - %% Sub topic first - emqx:subscribe(Topic), - - Req = make_req(post, Payload), - Result = do_request(Channel, URI, Req), - ?assertEqual({error, reset}, Result) - end, - - with_connection(Action). +%t_publish_authz_deny(_Config) -> +% Action = fun(Channel, Token) -> +% Topic = <<"/abc">>, +% Payload = <<"123">>, +% InvalidToken = lists:reverse(Token), +% +% TopicStr = binary_to_list(Topic), +% URI = ?PS_PREFIX ++ TopicStr ++ "?clientid=client1&token=" ++ InvalidToken, +% +% %% Sub topic first +% emqx:subscribe(Topic), +% +% Req = make_req(post, Payload), +% Result = do_request(Channel, URI, Req), +% ?assertEqual({error, reset}, Result) +% end, +% +% with_connection(Action). t_subscribe(_Config) -> Topic = <<"/abc">>, diff --git a/apps/emqx_gateway/test/emqx_coap_api_SUITE.erl b/apps/emqx_gateway/test/emqx_coap_api_SUITE.erl index 83521f5cd..935a4ec53 100644 --- a/apps/emqx_gateway/test/emqx_coap_api_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_coap_api_SUITE.erl @@ -25,20 +25,18 @@ -define(CONF_DEFAULT, <<" gateway.coap { - idle_timeout = 30s - enable_stats = false - mountpoint = \"\" - notify_type = qos - connection_required = true - subscribe_qos = qos1 - publish_qos = qos1 - authentication = undefined - - listeners.udp.default { - bind = 5683 - } - } - ">>). + idle_timeout = 30s + enable_stats = false + mountpoint = \"\" + notify_type = qos + connection_required = true + subscribe_qos = qos1 + publish_qos = qos1 + listeners.udp.default { + bind = 5683 + } +} +">>). -define(HOST, "127.0.0.1"). -define(PORT, 5683). @@ -73,7 +71,7 @@ t_send_request_api(_) -> Payload = <<"simple echo this">>, Req = #{token => Token, payload => Payload, - timeout => 10, + timeout => <<"10s">>, content_type => <<"text/plain">>, method => <<"get">>}, Auth = emqx_mgmt_api_test_util:auth_header_(), diff --git a/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl new file mode 100644 index 000000000..295df5737 --- /dev/null +++ b/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl @@ -0,0 +1,260 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_gateway_conf_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +%%-------------------------------------------------------------------- +%% Setups +%%-------------------------------------------------------------------- + +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Conf) -> + %% FIXME: Magic line. for saving gateway schema name for emqx_config + emqx_config:init_load(emqx_gateway_schema, <<"gateway {}">>), + emqx_ct_helpers:start_apps([emqx_gateway]), + Conf. + +end_per_suite(_Conf) -> + emqx_ct_helpers:stop_apps([emqx_gateway]). + +init_per_testcase(_CaseName, Conf) -> + _ = emqx_gateway_conf:unload_gateway(stomp), + Conf. + +%%-------------------------------------------------------------------- +%% Cases +%%-------------------------------------------------------------------- + +-define(CONF_STOMP_BAISC_1, + #{ <<"idle_timeout">> => <<"10s">>, + <<"mountpoint">> => <<"t/">>, + <<"frame">> => + #{ <<"max_headers">> => 20, + <<"max_headers_length">> => 2000, + <<"max_body_length">> => 2000 + } + }). +-define(CONF_STOMP_BAISC_2, + #{ <<"idle_timeout">> => <<"20s">>, + <<"mountpoint">> => <<"t2/">>, + <<"frame">> => + #{ <<"max_headers">> => 30, + <<"max_headers_length">> => 3000, + <<"max_body_length">> => 3000 + } + }). +-define(CONF_STOMP_LISTENER_1, + #{ <<"bind">> => <<"61613">> + }). +-define(CONF_STOMP_LISTENER_2, + #{ <<"bind">> => <<"61614">> + }). +-define(CONF_STOMP_AUTHN_1, + #{ <<"mechanism">> => <<"password-based">>, + <<"backend">> => <<"built-in-database">>, + <<"user_id_type">> => <<"clientid">> + }). +-define(CONF_STOMP_AUTHN_2, + #{ <<"mechanism">> => <<"password-based">>, + <<"backend">> => <<"built-in-database">>, + <<"user_id_type">> => <<"username">> + }). + +t_load_unload_gateway(_) -> + StompConf1 = compose(?CONF_STOMP_BAISC_1, + ?CONF_STOMP_AUTHN_1, + ?CONF_STOMP_LISTENER_1 + ), + StompConf2 = compose(?CONF_STOMP_BAISC_2, + ?CONF_STOMP_AUTHN_1, + ?CONF_STOMP_LISTENER_1), + + ok = emqx_gateway_conf:load_gateway(stomp, StompConf1), + {error, already_exist} = + emqx_gateway_conf:load_gateway(stomp, StompConf1), + assert_confs(StompConf1, emqx:get_raw_config([gateway, stomp])), + + ok = emqx_gateway_conf:update_gateway(stomp, StompConf2), + assert_confs(StompConf2, emqx:get_raw_config([gateway, stomp])), + + ok = emqx_gateway_conf:unload_gateway(stomp), + ok = emqx_gateway_conf:unload_gateway(stomp), + + {error, not_found} = + emqx_gateway_conf:update_gateway(stomp, StompConf2), + + ?assertException(error, {config_not_found, [gateway, stomp]}, + emqx:get_raw_config([gateway, stomp])), + ok. + +t_load_remove_authn(_) -> + StompConf = compose_listener(?CONF_STOMP_BAISC_1, ?CONF_STOMP_LISTENER_1), + + ok = emqx_gateway_conf:load_gateway(<<"stomp">>, StompConf), + assert_confs(StompConf, emqx:get_raw_config([gateway, stomp])), + + ok = emqx_gateway_conf:add_authn(<<"stomp">>, ?CONF_STOMP_AUTHN_1), + assert_confs( + maps:put(<<"authentication">>, ?CONF_STOMP_AUTHN_1, StompConf), + emqx:get_raw_config([gateway, stomp])), + + ok = emqx_gateway_conf:update_authn(<<"stomp">>, ?CONF_STOMP_AUTHN_2), + assert_confs( + maps:put(<<"authentication">>, ?CONF_STOMP_AUTHN_2, StompConf), + emqx:get_raw_config([gateway, stomp])), + + ok = emqx_gateway_conf:remove_authn(<<"stomp">>), + + {error, not_found} = + emqx_gateway_conf:update_authn(<<"stomp">>, ?CONF_STOMP_AUTHN_2), + + ?assertException( + error, {config_not_found, [gateway, stomp, authentication]}, + emqx:get_raw_config([gateway, stomp, authentication]) + ), + ok. + +t_load_remove_listeners(_) -> + StompConf = compose_authn(?CONF_STOMP_BAISC_1, ?CONF_STOMP_AUTHN_1), + + ok = emqx_gateway_conf:load_gateway(<<"stomp">>, StompConf), + assert_confs(StompConf, emqx:get_raw_config([gateway, stomp])), + + ok = emqx_gateway_conf:add_listener( + <<"stomp">>, {<<"tcp">>, <<"default">>}, ?CONF_STOMP_LISTENER_1), + assert_confs( + maps:merge(StompConf, listener(?CONF_STOMP_LISTENER_1)), + emqx:get_raw_config([gateway, stomp])), + + ok = emqx_gateway_conf:update_listener( + <<"stomp">>, {<<"tcp">>, <<"default">>}, ?CONF_STOMP_LISTENER_2), + assert_confs( + maps:merge(StompConf, listener(?CONF_STOMP_LISTENER_2)), + emqx:get_raw_config([gateway, stomp])), + + ok = emqx_gateway_conf:remove_listener( + <<"stomp">>, {<<"tcp">>, <<"default">>}), + + {error, not_found} = + emqx_gateway_conf:update_listener( + <<"stomp">>, {<<"tcp">>, <<"default">>}, ?CONF_STOMP_LISTENER_2), + + ?assertException( + error, {config_not_found, [gateway, stomp, listeners, tcp, default]}, + emqx:get_raw_config([gateway, stomp, listeners, tcp, default]) + ), + ok. + +t_load_remove_listener_authn(_) -> + StompConf = compose_listener( + ?CONF_STOMP_BAISC_1, + ?CONF_STOMP_LISTENER_1 + ), + StompConf1 = compose_listener_authn( + ?CONF_STOMP_BAISC_1, + ?CONF_STOMP_LISTENER_1, + ?CONF_STOMP_AUTHN_1 + ), + StompConf2 = compose_listener_authn( + ?CONF_STOMP_BAISC_1, + ?CONF_STOMP_LISTENER_1, + ?CONF_STOMP_AUTHN_2 + ), + + ok = emqx_gateway_conf:load_gateway(<<"stomp">>, StompConf), + assert_confs(StompConf, emqx:get_raw_config([gateway, stomp])), + + ok = emqx_gateway_conf:add_authn( + <<"stomp">>, {<<"tcp">>, <<"default">>}, ?CONF_STOMP_AUTHN_1), + assert_confs(StompConf1, emqx:get_raw_config([gateway, stomp])), + + ok = emqx_gateway_conf:update_authn( + <<"stomp">>, {<<"tcp">>, <<"default">>}, ?CONF_STOMP_AUTHN_2), + assert_confs(StompConf2, emqx:get_raw_config([gateway, stomp])), + + ok = emqx_gateway_conf:remove_authn( + <<"stomp">>, {<<"tcp">>, <<"default">>}), + + {error, not_found} = + emqx_gateway_conf:update_authn( + <<"stomp">>, {<<"tcp">>, <<"default">>}, ?CONF_STOMP_AUTHN_2), + + Path = [gateway, stomp, listeners, tcp, default, authentication], + ?assertException( + error, {config_not_found, Path}, + emqx:get_raw_config(Path) + ), + ok. + +%%-------------------------------------------------------------------- +%% Utils + +compose(Basic, Authn, Listener) -> + maps:merge( + maps:merge(Basic, #{<<"authentication">> => Authn}), + listener(Listener)). + +compose_listener(Basic, Listener) -> + maps:merge(Basic, listener(Listener)). + +compose_authn(Basic, Authn) -> + maps:merge(Basic, #{<<"authentication">> => Authn}). + +compose_listener_authn(Basic, Listener, Authn) -> + maps:merge( + Basic, + listener(maps:put(<<"authentication">>, Authn, Listener))). + +listener(L) -> + #{<<"listeners">> => [L#{<<"type">> => <<"tcp">>, + <<"name">> => <<"default">>}]}. + +assert_confs(Expected0, Effected) -> + Expected = maybe_unconvert_listeners(Expected0), + case do_assert_confs(Expected, Effected) of + false -> + io:format(standard_error, "Expected config: ~p,\n" + "Effected config: ~p", + [Expected, Effected]), + exit(conf_not_match); + true -> + ok + end. + +do_assert_confs(Expected, Effected) when is_map(Expected), + is_map(Effected) -> + Ks1 = maps:keys(Expected), + lists:all(fun(K) -> + do_assert_confs(maps:get(K, Expected), + maps:get(K, Effected, undefined)) + end, Ks1); +do_assert_confs(Expected, Effected) -> + Expected =:= Effected. + +maybe_unconvert_listeners(Conf) -> + case maps:take(<<"listeners">>, Conf) of + error -> Conf; + {Ls, Conf1} -> + Conf1#{<<"listeners">> => + emqx_gateway_conf:unconvert_listeners(Ls)} + end. diff --git a/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl b/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl index 2ee2312df..f36c1e816 100644 --- a/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl @@ -33,7 +33,7 @@ gateway.lwm2m { xml_dir = \"../../lib/emqx_gateway/src/lwm2m/lwm2m_xml\" lifetime_min = 1s lifetime_max = 86400s - qmode_time_windonw = 22 + qmode_time_window = 22 auto_observe = false mountpoint = \"lwm2m/%u\" update_msg_publish_condition = contains_object_list diff --git a/apps/emqx_gateway/test/emqx_lwm2m_api_SUITE.erl b/apps/emqx_gateway/test/emqx_lwm2m_api_SUITE.erl index 081f11005..a875aceb6 100644 --- a/apps/emqx_gateway/test/emqx_lwm2m_api_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_lwm2m_api_SUITE.erl @@ -33,7 +33,7 @@ gateway.lwm2m { xml_dir = \"../../lib/emqx_gateway/src/lwm2m/lwm2m_xml\" lifetime_min = 1s lifetime_max = 86400s - qmode_time_windonw = 22 + qmode_time_window = 22 auto_observe = false mountpoint = \"lwm2m/%u\" update_msg_publish_condition = contains_object_list diff --git a/apps/emqx_gateway/test/props/emqx_sn_proper_types.erl b/apps/emqx_gateway/test/props/emqx_sn_proper_types.erl index 96318788d..95cfc6d20 100644 --- a/apps/emqx_gateway/test/props/emqx_sn_proper_types.erl +++ b/apps/emqx_gateway/test/props/emqx_sn_proper_types.erl @@ -24,8 +24,7 @@ -compile(export_all). -compile(nowarn_export_all). --import(emqx_ct_proper_types, - [topic/0]). +-import(emqx_proper_types, [topic/0]). %%-------------------------------------------------------------------- %% Messages diff --git a/apps/emqx_machine/include/emqx_cluster_rpc.hrl b/apps/emqx_machine/include/emqx_machine.hrl similarity index 95% rename from apps/emqx_machine/include/emqx_cluster_rpc.hrl rename to apps/emqx_machine/include/emqx_machine.hrl index 046331871..cea62c5c3 100644 --- a/apps/emqx_machine/include/emqx_cluster_rpc.hrl +++ b/apps/emqx_machine/include/emqx_machine.hrl @@ -20,6 +20,8 @@ -define(CLUSTER_MFA, cluster_rpc_mfa). -define(CLUSTER_COMMIT, cluster_rpc_commit). +-define(EMQX_MACHINE_SHARD, emqx_machine_shard). + -record(cluster_rpc_mfa, { tnx_id :: pos_integer(), mfa :: mfa(), diff --git a/apps/emqx_machine/src/emqx_cluster_rpc.erl b/apps/emqx_machine/src/emqx_cluster_rpc.erl index d4a7bfee8..66616f3ea 100644 --- a/apps/emqx_machine/src/emqx_cluster_rpc.erl +++ b/apps/emqx_machine/src/emqx_cluster_rpc.erl @@ -32,12 +32,11 @@ -boot_mnesia({mnesia, [boot]}). -copy_mnesia({mnesia, [copy]}). --include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/logger.hrl"). --include("emqx_cluster_rpc.hrl"). +-include("emqx_machine.hrl"). --rlog_shard({?COMMON_SHARD, ?CLUSTER_MFA}). --rlog_shard({?COMMON_SHARD, ?CLUSTER_COMMIT}). +-rlog_shard({?EMQX_MACHINE_SHARD, ?CLUSTER_MFA}). +-rlog_shard({?EMQX_MACHINE_SHARD, ?CLUSTER_COMMIT}). -define(CATCH_UP, catch_up). -define(TIMEOUT, timer:minutes(1)). @@ -48,13 +47,13 @@ mnesia(boot) -> ok = ekka_mnesia:create_table(?CLUSTER_MFA, [ {type, ordered_set}, - {rlog_shard, ?COMMON_SHARD}, + {rlog_shard, ?EMQX_MACHINE_SHARD}, {disc_copies, [node()]}, {record_name, cluster_rpc_mfa}, {attributes, record_info(fields, cluster_rpc_mfa)}]), ok = ekka_mnesia:create_table(?CLUSTER_COMMIT, [ {type, set}, - {rlog_shard, ?COMMON_SHARD}, + {rlog_shard, ?EMQX_MACHINE_SHARD}, {disc_copies, [node()]}, {record_name, cluster_rpc_commit}, {attributes, record_info(fields, cluster_rpc_commit)}]); @@ -95,7 +94,7 @@ multicall(M, F, A, RequireNum, Timeout) when RequireNum =:= all orelse RequireNu %% the initiate transaction must happened on core node %% make sure MFA(in the transaction) and the transaction on the same node %% don't need rpc again inside transaction. - case ekka_rlog_status:upstream_node(?COMMON_SHARD) of + case ekka_rlog_status:upstream_node(?EMQX_MACHINE_SHARD) of {ok, Node} -> gen_server:call({?MODULE, Node}, MFA, Timeout); disconnected -> {error, disconnected} end @@ -281,7 +280,7 @@ do_catch_up_in_one_trans(LatestId, Node) -> end. transaction(Func, Args) -> - ekka_mnesia:transaction(?COMMON_SHARD, Func, Args). + ekka_mnesia:transaction(?EMQX_MACHINE_SHARD, Func, Args). trans_status() -> mnesia:foldl(fun(Rec, Acc) -> diff --git a/apps/emqx_machine/src/emqx_cluster_rpc_handler.erl b/apps/emqx_machine/src/emqx_cluster_rpc_handler.erl index 803b7f9fc..6dcbd3d25 100644 --- a/apps/emqx_machine/src/emqx_cluster_rpc_handler.erl +++ b/apps/emqx_machine/src/emqx_cluster_rpc_handler.erl @@ -17,9 +17,8 @@ -behaviour(gen_server). --include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/logger.hrl"). --include("emqx_cluster_rpc.hrl"). +-include("emqx_machine.hrl"). -export([start_link/0, start_link/2]). -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, @@ -50,7 +49,7 @@ handle_cast(Msg, State) -> {noreply, State}. handle_info({timeout, TRef, del_stale_mfa}, State = #{timer := TRef, max_history := MaxHistory}) -> - case ekka_mnesia:transaction(?COMMON_SHARD, fun del_stale_mfa/1, [MaxHistory]) of + case ekka_mnesia:transaction(?EMQX_MACHINE_SHARD, fun del_stale_mfa/1, [MaxHistory]) of {atomic, ok} -> ok; Error -> ?LOG(error, "del_stale_cluster_rpc_mfa error:~p", [Error]) end, diff --git a/apps/emqx_machine/src/emqx_machine.erl b/apps/emqx_machine/src/emqx_machine.erl index 97125d79f..0f8208b46 100644 --- a/apps/emqx_machine/src/emqx_machine.erl +++ b/apps/emqx_machine/src/emqx_machine.erl @@ -21,34 +21,23 @@ , is_ready/0 ]). --export([ stop_apps/1 - , ensure_apps_started/0 - ]). - --export([sorted_reboot_apps/0]). - --ifdef(TEST). --export([sorted_reboot_apps/1]). --endif. - -include_lib("emqx/include/logger.hrl"). +-include("emqx_machine.hrl"). %% @doc EMQ X boot entrypoint. start() -> - os:set_signal(sighup, ignore), - os:set_signal(sigterm, handle), %% default is handle + case os:type() of + {win32, nt} -> ok; + _nix -> + os:set_signal(sighup, ignore), + os:set_signal(sigterm, handle) %% default is handle + end, ok = set_backtrace_depth(), ok = print_otp_version_warning(), - ok = load_config_files(), - - ok = ensure_apps_started(), - - _ = emqx_plugins:load(), - - ok = print_vsn(), - - ok = start_autocluster(). + ekka:start(), + ekka_rlog:wait_for_shards([?EMQX_MACHINE_SHARD], infinity), + ok. graceful_shutdown() -> emqx_machine_terminator:graceful_wait(). @@ -70,13 +59,6 @@ print_otp_version_warning() -> [?OTP_RELEASE]). -endif. % OTP_RELEASE > 22 --ifdef(TEST). -print_vsn() -> ok. --else. % TEST -print_vsn() -> - ?ULOG("~s ~s is running now!~n", [emqx_app:get_description(), emqx_app:get_release()]). --endif. % TEST - load_config_files() -> %% the app env 'config_files' for 'emqx` app should be set %% in app.time.config by boot script before starting Erlang VM @@ -85,114 +67,3 @@ load_config_files() -> ok = emqx_config:init_load(emqx_machine_schema, ConfFiles), %% to avoid config being loaded again when emqx app starts. ok = emqx_app:set_init_config_load_done(). - -start_autocluster() -> - ekka:callback(prepare, fun ?MODULE:stop_apps/1), - ekka:callback(reboot, fun ?MODULE:ensure_apps_started/0), - _ = ekka:autocluster(emqx), %% returns 'ok' or a pid or 'any()' as in spec - ok. - -stop_apps(Reason) -> - ?SLOG(info, #{msg => "stopping_apps", reason => Reason}), - _ = emqx_alarm_handler:unload(), - lists:foreach(fun stop_one_app/1, lists:reverse(sorted_reboot_apps())). - -stop_one_app(App) -> - ?SLOG(debug, #{msg => "stopping_app", app => App}), - try - _ = application:stop(App) - catch - C : E -> - ?SLOG(error, #{msg => "failed_to_stop_app", - app => App, - exception => C, - reason => E}) - end. - - -ensure_apps_started() -> - lists:foreach(fun start_one_app/1, sorted_reboot_apps()). - -start_one_app(App) -> - ?SLOG(debug, #{msg => "starting_app", app => App}), - case application:ensure_all_started(App) of - {ok, Apps} -> - ?SLOG(debug, #{msg => "started_apps", apps => Apps}); - {error, Reason} -> - ?SLOG(critical, #{msg => "failed_to_start_app", app => App, reason => Reason}), - error({failed_to_start_app, App, Reason}) - end. - -%% list of app names which should be rebooted when: -%% 1. due to static static config change -%% 2. after join a cluster -reboot_apps() -> - [ gproc - , esockd - , ranch - , cowboy - , emqx - , emqx_prometheus - , emqx_modules - , emqx_dashboard - , emqx_connector - , emqx_gateway - , emqx_statsd - , emqx_resource - , emqx_rule_engine - , emqx_bridge - , emqx_bridge_mqtt - , emqx_plugin_libs - , emqx_management - , emqx_retainer - , emqx_exhook - , emqx_rule_actions - , emqx_authn - , emqx_authz - ]. - -sorted_reboot_apps() -> - Apps = [{App, app_deps(App)} || App <- reboot_apps()], - sorted_reboot_apps(Apps). - -app_deps(App) -> - case application:get_key(App, applications) of - undefined -> []; - {ok, List} -> lists:filter(fun(A) -> lists:member(A, reboot_apps()) end, List) - end. - -sorted_reboot_apps(Apps) -> - G = digraph:new(), - try - lists:foreach(fun({App, Deps}) -> add_app(G, App, Deps) end, Apps), - case digraph_utils:topsort(G) of - Sorted when is_list(Sorted) -> - Sorted; - false -> - Loops = find_loops(G), - error({circular_application_dependency, Loops}) - end - after - digraph:delete(G) - end. - -add_app(G, App, undefined) -> - ?SLOG(debug, #{msg => "app_is_not_loaded", app => App}), - %% not loaded - add_app(G, App, []); -add_app(_G, _App, []) -> - ok; -add_app(G, App, [Dep | Deps]) -> - digraph:add_vertex(G, App), - digraph:add_vertex(G, Dep), - digraph:add_edge(G, Dep, App), %% dep -> app as dependency - add_app(G, App, Deps). - -find_loops(G) -> - lists:filtermap( - fun (App) -> - case digraph:get_short_cycle(G, App) of - false -> false; - Apps -> {true, Apps} - end - end, digraph:vertices(G)). diff --git a/apps/emqx_machine/src/emqx_machine_boot.erl b/apps/emqx_machine/src/emqx_machine_boot.erl new file mode 100644 index 000000000..8fc3a14f4 --- /dev/null +++ b/apps/emqx_machine/src/emqx_machine_boot.erl @@ -0,0 +1,152 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_machine_boot). + +-include_lib("emqx/include/logger.hrl"). + +-export([post_boot/0]). +-export([stop_apps/1, ensure_apps_started/0]). +-export([sorted_reboot_apps/0]). +-export([start_autocluster/0]). + +-ifdef(TEST). +-export([sorted_reboot_apps/1]). +-endif. + +post_boot() -> + ok = ensure_apps_started(), + _ = emqx_plugins:load(), + ok = print_vsn(), + ok = start_autocluster(), + ignore. + +-ifdef(TEST). +print_vsn() -> ok. +-else. % TEST +print_vsn() -> + ?ULOG("~s ~s is running now!~n", [emqx_app:get_description(), emqx_app:get_release()]). +-endif. % TEST + + +start_autocluster() -> + ekka:callback(prepare, fun ?MODULE:stop_apps/1), + ekka:callback(reboot, fun ?MODULE:ensure_apps_started/0), + _ = ekka:autocluster(emqx), %% returns 'ok' or a pid or 'any()' as in spec + ok. + +stop_apps(Reason) -> + ?SLOG(info, #{msg => "stopping_apps", reason => Reason}), + _ = emqx_alarm_handler:unload(), + lists:foreach(fun stop_one_app/1, lists:reverse(sorted_reboot_apps())). + +stop_one_app(App) -> + ?SLOG(debug, #{msg => "stopping_app", app => App}), + try + _ = application:stop(App) + catch + C : E -> + ?SLOG(error, #{msg => "failed_to_stop_app", + app => App, + exception => C, + reason => E}) + end. + + +ensure_apps_started() -> + lists:foreach(fun start_one_app/1, sorted_reboot_apps()). + +start_one_app(App) -> + ?SLOG(debug, #{msg => "starting_app", app => App}), + case application:ensure_all_started(App) of + {ok, Apps} -> + ?SLOG(debug, #{msg => "started_apps", apps => Apps}); + {error, Reason} -> + ?SLOG(critical, #{msg => "failed_to_start_app", app => App, reason => Reason}), + error({failed_to_start_app, App, Reason}) + end. + +%% list of app names which should be rebooted when: +%% 1. due to static static config change +%% 2. after join a cluster +reboot_apps() -> + [ gproc + , esockd + , ranch + , cowboy + , emqx + , emqx_prometheus + , emqx_modules + , emqx_dashboard + , emqx_connector + , emqx_gateway + , emqx_statsd + , emqx_resource + , emqx_rule_engine + , emqx_bridge + , emqx_bridge_mqtt + , emqx_plugin_libs + , emqx_management + , emqx_retainer + , emqx_exhook + , emqx_authn + , emqx_authz + ]. + +sorted_reboot_apps() -> + Apps = [{App, app_deps(App)} || App <- reboot_apps()], + sorted_reboot_apps(Apps). + +app_deps(App) -> + case application:get_key(App, applications) of + undefined -> []; + {ok, List} -> lists:filter(fun(A) -> lists:member(A, reboot_apps()) end, List) + end. + +sorted_reboot_apps(Apps) -> + G = digraph:new(), + try + lists:foreach(fun({App, Deps}) -> add_app(G, App, Deps) end, Apps), + case digraph_utils:topsort(G) of + Sorted when is_list(Sorted) -> + Sorted; + false -> + Loops = find_loops(G), + error({circular_application_dependency, Loops}) + end + after + digraph:delete(G) + end. + +add_app(G, App, undefined) -> + ?SLOG(debug, #{msg => "app_is_not_loaded", app => App}), + %% not loaded + add_app(G, App, []); +add_app(_G, _App, []) -> + ok; +add_app(G, App, [Dep | Deps]) -> + digraph:add_vertex(G, App), + digraph:add_vertex(G, Dep), + digraph:add_edge(G, Dep, App), %% dep -> app as dependency + add_app(G, App, Deps). + +find_loops(G) -> + lists:filtermap( + fun (App) -> + case digraph:get_short_cycle(G, App) of + false -> false; + Apps -> {true, Apps} + end + end, digraph:vertices(G)). diff --git a/apps/emqx_machine/src/emqx_machine_schema.erl b/apps/emqx_machine/src/emqx_machine_schema.erl index aca518b0d..369d7b3c5 100644 --- a/apps/emqx_machine/src/emqx_machine_schema.erl +++ b/apps/emqx_machine/src/emqx_machine_schema.erl @@ -53,6 +53,7 @@ , emqx_prometheus_schema , emqx_rule_engine_schema , emqx_exhook_schema + , emqx_psk_schema ]). namespace() -> undefined. @@ -102,7 +103,7 @@ fields("cluster") -> , default => emqxcl })} , {"discovery_strategy", - sc(union([manual, static, mcast, dns, etcd, k8s]), + sc(hoconsc:enum([manual, static, mcast, dns, etcd, k8s]), #{ default => manual })} , {"autoclean", @@ -122,7 +123,7 @@ fields("cluster") -> sc(ref(cluster_mcast), #{})} , {"proto_dist", - sc(union([inet_tcp, inet6_tcp, inet_tls]), + sc(hoconsc:enum([inet_tcp, inet6_tcp, inet_tls]), #{ mapping => "ekka.proto_dist" , default => inet_tcp })} @@ -136,7 +137,7 @@ fields("cluster") -> sc(ref(cluster_k8s), #{})} , {"db_backend", - sc(union([mnesia, rlog]), + sc(hoconsc:enum([mnesia, rlog]), #{ mapping => "ekka.db_backend" , default => mnesia })} @@ -211,13 +212,10 @@ fields(cluster_etcd) -> #{ default => "1m" })} , {"ssl", - sc(ref(etcd_ssl_opts), + sc(hoconsc:ref(emqx_schema, ssl_client_opts), #{})} ]; -fields(etcd_ssl_opts) -> - emqx_schema:ssl(#{}); - fields(cluster_k8s) -> [ {"apiserver", sc(string(), @@ -227,7 +225,7 @@ fields(cluster_k8s) -> #{ default => "emqx" })} , {"address_type", - sc(union([ip, dns, hostname]), + sc(hoconsc:enum([ip, dns, hostname]), #{})} , {"app_name", sc(string(), @@ -245,7 +243,7 @@ fields(cluster_k8s) -> fields("rlog") -> [ {"role", - sc(union([core, replicant]), + sc(hoconsc:enum([core, replicant]), #{ mapping => "ekka.node_role" , default => core })} @@ -312,24 +310,8 @@ fields("node") -> )} , {"etc_dir", sc(string(), - #{ - converter => fun(EtcDir) -> - case filename:absname(EtcDir) =:= EtcDir of - true -> - unicode:characters_to_list(EtcDir); - false -> - unicode:characters_to_list(filename:join([code:lib_dir(), "..", EtcDir])) - end - end, - validator => fun(Path) -> - case filelib:is_dir(Path) of - true -> - ok; - false -> - error({not_dir, Path}) - end - end - } + #{ desc => "`etc` dir for the node" + } )} ]; @@ -353,7 +335,7 @@ fields("cluster_call") -> fields("rpc") -> [ {"mode", - sc(union(sync, async), + sc(hoconsc:enum([sync, async]), #{ default => async })} , {"async_batch_size", @@ -362,7 +344,7 @@ fields("rpc") -> , default => 256 })} , {"port_discovery", - sc(union(manual, stateless), + sc(hoconsc:enum([manual, stateless]), #{ mapping => "gen_rpc.port_discovery" , default => stateless })} @@ -453,7 +435,7 @@ fields("log_file_handler") -> sc(ref("log_rotation"), #{})} , {"max_size", - sc(union([infinity, emqx_schema:bytesize()]), + sc(hoconsc:union([infinity, emqx_schema:bytesize()]), #{ default => "10MB" })} ] ++ log_handler_common_confs(); @@ -483,7 +465,7 @@ fields("log_overload_kill") -> #{ default => 20000 })} , {"restart_after", - sc(union(emqx_schema:duration(), infinity), + sc(hoconsc:union([emqx_schema:duration(), infinity]), #{ default => "5s" })} ]; @@ -601,7 +583,7 @@ log_handler_common_confs() -> #{ default => unlimited })} , {"formatter", - sc(union([text, json]), + sc(hoconsc:enum([text, json]), #{ default => text })} , {"single_line", @@ -627,11 +609,11 @@ log_handler_common_confs() -> sc(ref("log_burst_limit"), #{})} , {"supervisor_reports", - sc(union([error, progress]), + sc(hoconsc:enum([error, progress]), #{ default => error })} , {"max_depth", - sc(union([unlimited, integer()]), + sc(hoconsc:union([unlimited, integer()]), #{ default => 100 })} ]. diff --git a/apps/emqx_machine/src/emqx_machine_sup.erl b/apps/emqx_machine/src/emqx_machine_sup.erl index 798beee1c..406e1d483 100644 --- a/apps/emqx_machine/src/emqx_machine_sup.erl +++ b/apps/emqx_machine/src/emqx_machine_sup.erl @@ -33,7 +33,8 @@ init([]) -> Terminator = child_worker(emqx_machine_terminator, [], transient), ClusterRpc = child_worker(emqx_cluster_rpc, [], permanent), ClusterHandler = child_worker(emqx_cluster_rpc_handler, [], permanent), - Children = [GlobalGC, Terminator, ClusterRpc, ClusterHandler], + BootApps = child_worker(emqx_machine_boot, post_boot, [], temporary), + Children = [GlobalGC, Terminator, ClusterRpc, ClusterHandler, BootApps], SupFlags = #{strategy => one_for_one, intensity => 100, period => 10 @@ -41,8 +42,11 @@ init([]) -> {ok, {SupFlags, Children}}. child_worker(M, Args, Restart) -> + child_worker(M, start_link, Args, Restart). + +child_worker(M, Func, Args, Restart) -> #{id => M, - start => {M, start_link, Args}, + start => {M, Func, Args}, restart => Restart, shutdown => 5000, type => worker, diff --git a/apps/emqx_machine/src/emqx_machine_terminator.erl b/apps/emqx_machine/src/emqx_machine_terminator.erl index 74479a6d9..733c1a5dc 100644 --- a/apps/emqx_machine/src/emqx_machine_terminator.erl +++ b/apps/emqx_machine/src/emqx_machine_terminator.erl @@ -80,7 +80,7 @@ handle_cast(_Cast, State) -> handle_call(?DO_IT, _From, State) -> try - emqx_machine:stop_apps(normal) + emqx_machine_boot:stop_apps(normal) catch C : E : St -> Apps = [element(1, A) || A <- application:which_applications()], diff --git a/apps/emqx_machine/test/emqx_cluster_rpc_SUITE.erl b/apps/emqx_machine/test/emqx_cluster_rpc_SUITE.erl index d39dc3c6e..4e3b2d2c2 100644 --- a/apps/emqx_machine/test/emqx_cluster_rpc_SUITE.erl +++ b/apps/emqx_machine/test/emqx_cluster_rpc_SUITE.erl @@ -20,6 +20,7 @@ -compile(nowarn_export_all). -include_lib("emqx/include/emqx.hrl"). +-include("emqx_machine.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). -define(NODE1, emqx_cluster_rpc). @@ -42,7 +43,7 @@ init_per_suite(Config) -> application:load(emqx), application:load(emqx_machine), ok = ekka:start(), - ok = ekka_rlog:wait_for_shards([emqx_common_shard], infinity), + ok = ekka_rlog:wait_for_shards([?EMQX_MACHINE_SHARD], infinity), application:set_env(emqx_machine, cluster_call_max_history, 100), application:set_env(emqx_machine, cluster_call_clean_interval, 1000), application:set_env(emqx_machine, cluster_call_retry_interval, 900), @@ -124,7 +125,7 @@ t_catch_up_status_handle_next_commit(_Config) -> t_commit_ok_apply_fail_on_other_node_then_recover(_Config) -> emqx_cluster_rpc:reset(), {atomic, []} = emqx_cluster_rpc:status(), - Now = erlang:system_time(second), + Now = erlang:system_time(millisecond), {M, F, A} = {?MODULE, failed_on_other_recover_after_5_second, [erlang:whereis(?NODE1), Now]}, {ok, _, ok} = emqx_cluster_rpc:multicall(M, F, A, 1, 1000), {ok, _, ok} = emqx_cluster_rpc:multicall(io, format, ["test"], 1, 1000), @@ -132,10 +133,10 @@ t_commit_ok_apply_fail_on_other_node_then_recover(_Config) -> ?assertEqual([], L), ?assertEqual({io, format, ["test"]}, maps:get(mfa, Status)), ?assertEqual(node(), maps:get(node, Status)), - sleep(3000), + sleep(2300), {atomic, [Status1]} = emqx_cluster_rpc:status(), ?assertEqual(Status, Status1), - sleep(2600), + sleep(3600), {atomic, NewStatus} = emqx_cluster_rpc:status(), ?assertEqual(3, length(NewStatus)), Pid = self(), @@ -243,11 +244,11 @@ failed_on_node_by_odd(Pid) -> end. failed_on_other_recover_after_5_second(Pid, CreatedAt) -> - Now = erlang:system_time(second), + Now = erlang:system_time(millisecond), case Pid =:= self() of true -> ok; false -> - case Now < CreatedAt + 5 of + case Now < CreatedAt + 5001 of true -> "MFA return not ok"; false -> ok end diff --git a/apps/emqx_machine/test/emqx_machine_SUITE.erl b/apps/emqx_machine/test/emqx_machine_SUITE.erl index 51cf4f8b4..1e90d867b 100644 --- a/apps/emqx_machine/test/emqx_machine_SUITE.erl +++ b/apps/emqx_machine/test/emqx_machine_SUITE.erl @@ -33,9 +33,9 @@ end_per_suite(_Config) -> emqx_ct_helpers:stop_apps([]). t_shutdown_reboot(_Config) -> - emqx_machine:stop_apps(normal), + emqx_machine_boot:stop_apps(normal), false = emqx:is_running(node()), - emqx_machine:ensure_apps_started(), + emqx_machine_boot:ensure_apps_started(), true = emqx:is_running(node()), - ok = emqx_machine:stop_apps(for_test), + ok = emqx_machine_boot:stop_apps(for_test), false = emqx:is_running(node()). diff --git a/apps/emqx_machine/test/emqx_machine_tests.erl b/apps/emqx_machine/test/emqx_machine_tests.erl index dded07570..1a562b815 100644 --- a/apps/emqx_machine/test/emqx_machine_tests.erl +++ b/apps/emqx_machine/test/emqx_machine_tests.erl @@ -38,7 +38,7 @@ sorted_reboot_apps_cycle_test() -> check_order(Apps) -> AllApps = lists:usort(lists:append([[A | Deps] || {A, Deps} <- Apps])), - Sorted = emqx_machine:sorted_reboot_apps(Apps), + Sorted = emqx_machine_boot:sorted_reboot_apps(Apps), case length(AllApps) =:= length(Sorted) of true -> ok; false -> error({AllApps, Sorted}) diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl index 3cc31b47f..3b1fd5903 100644 --- a/apps/emqx_management/src/emqx_mgmt.erl +++ b/apps/emqx_management/src/emqx_mgmt.erl @@ -508,12 +508,16 @@ update_listener(Id, Config) -> [update_listener(Node, Id, Config) || Node <- ekka_mnesia:running_nodes()]. update_listener(Node, Id, Config) when Node =:= node() -> - {Type, Name} = emqx_listeners:parse_listener_id(Id), - case emqx:update_config([listeners, Type, Name], Config, #{}) of - {ok, #{raw_config := RawConf}} -> - RawConf#{node => Node, id => Id, running => true}; - {error, Reason} -> - error(Reason) + case emqx_listeners:parse_listener_id(Id) of + {error, {invalid_listener_id, Id}} -> + {error, {invalid_listener_id, Id}}; + {Type, Name} -> + case emqx:update_config([listeners, Type, Name], Config, #{}) of + {ok, #{raw_config := RawConf}} -> + RawConf#{node => Node, id => Id, running => true}; + {error, Reason} -> + {error, Reason} + end end; update_listener(Node, Id, Config) -> rpc_call(Node, update_listener, [Node, Id, Config]). diff --git a/apps/emqx_management/src/emqx_mgmt_api.erl b/apps/emqx_management/src/emqx_mgmt_api.erl index 8cf2fa1cb..ef8d7c70c 100644 --- a/apps/emqx_management/src/emqx_mgmt_api.erl +++ b/apps/emqx_management/src/emqx_mgmt_api.erl @@ -18,14 +18,16 @@ -include_lib("stdlib/include/qlc.hrl"). --export([paginate/3]). +-define(FRESH_SELECT, fresh_select). + +-export([ paginate/3 + , paginate/4 + ]). %% first_next query APIs --export([ params2qs/2 - , node_query/5 +-export([ node_query/5 , cluster_query/4 - , traverse_table/5 - , select_table/5 + , select_table_with_count/5 ]). -export([do_query/6]). @@ -47,13 +49,40 @@ paginate(Tables, Params, RowFun) -> #{meta => #{page => Page, limit => Limit, count => Count}, data => [RowFun(Row) || Row <- Rows]}. +paginate(Tables, MatchSpec, Params, RowFun) -> + Qh = query_handle(Tables, MatchSpec), + Count = count(Tables, MatchSpec), + Page = b2i(page(Params)), + Limit = b2i(limit(Params)), + Cursor = qlc:cursor(Qh), + case Page > 1 of + true -> + _ = qlc:next_answers(Cursor, (Page - 1) * Limit), + ok; + false -> ok + end, + Rows = qlc:next_answers(Cursor, Limit), + qlc:delete_cursor(Cursor), + #{meta => #{page => Page, limit => Limit, count => Count}, + data => [RowFun(Row) || Row <- Rows]}. + query_handle(Table) when is_atom(Table) -> - qlc:q([R|| R <- ets:table(Table)]); + qlc:q([R || R <- ets:table(Table)]); query_handle([Table]) when is_atom(Table) -> - qlc:q([R|| R <- ets:table(Table)]); + qlc:q([R || R <- ets:table(Table)]); query_handle(Tables) -> qlc:append([qlc:q([E || E <- ets:table(T)]) || T <- Tables]). +query_handle(Table, MatchSpec) when is_atom(Table) -> + Options = {traverse, {select, MatchSpec}}, + qlc:q([R || R <- ets:table(Table, Options)]); +query_handle([Table], MatchSpec) when is_atom(Table) -> + Options = {traverse, {select, MatchSpec}}, + qlc:q([R || R <- ets:table(Table, Options)]); +query_handle(Tables, MatchSpec) -> + Options = {traverse, {select, MatchSpec}}, + qlc:append([qlc:q([E || E <- ets:table(T, Options)]) || T <- Tables]). + count(Table) when is_atom(Table) -> ets:info(Table, size); count([Table]) when is_atom(Table) -> @@ -61,8 +90,16 @@ count([Table]) when is_atom(Table) -> count(Tables) -> lists:sum([count(T) || T <- Tables]). -count(Table, Nodes) -> - lists:sum([rpc_call(Node, ets, info, [Table, size], 5000) || Node <- Nodes]). +count(Table, MatchSpec) when is_atom(Table) -> + [{MatchPattern, Where, _Re}] = MatchSpec, + NMatchSpec = [{MatchPattern, Where, [true]}], + ets:select_count(Table, NMatchSpec); +count([Table], MatchSpec) when is_atom(Table) -> + [{MatchPattern, Where, _Re}] = MatchSpec, + NMatchSpec = [{MatchPattern, Where, [true]}], + ets:select_count(Table, NMatchSpec); +count(Tables, MatchSpec) -> + lists:sum([count(T, MatchSpec) || T <- Tables]). page(Params) when is_map(Params) -> maps:get(<<"page">>, Params, 1); @@ -79,26 +116,110 @@ limit(Params) -> %%-------------------------------------------------------------------- node_query(Node, Params, Tab, QsSchema, QueryFun) -> - {CodCnt, Qs} = params2qs(Params, QsSchema), + {_CodCnt, Qs} = params2qs(Params, QsSchema), Limit = b2i(limit(Params)), Page = b2i(page(Params)), - Start = if Page > 1 -> (Page-1) * Limit; - true -> 0 - end, - {_, Rows} = do_query(Node, Tab, Qs, QueryFun, Start, Limit+1), - Meta = #{page => Page, limit => Limit}, - NMeta = case CodCnt =:= 0 of - true -> Meta#{count => count(Tab)}; - _ -> Meta#{count => length(Rows)} - end, - #{meta => NMeta, data => lists:sublist(Rows, Limit)}. + Meta = #{page => Page, limit => Limit, count => 0}, + do_node_query(Node, Tab, Qs, QueryFun, Meta). %% @private -do_query(Node, Tab, Qs, {M,F}, Start, Limit) when Node =:= node() -> - M:F(Tab, Qs, Start, Limit); -do_query(Node, Tab, Qs, QueryFun, Start, Limit) -> +do_node_query(Node, Tab, Qs, QueryFun, Meta) -> + do_node_query(Node, Tab, Qs, QueryFun, _Continuation = ?FRESH_SELECT, Meta, _Results = []). + +do_node_query( Node, Tab, Qs, QueryFun, Continuation + , Meta = #{limit := Limit} + , Results) -> + {Len, Rows, NContinuation} = do_query(Node, Tab, Qs, QueryFun, Continuation, Limit), + case judge_page_with_counting(Len, Meta) of + {more, NMeta} -> + case NContinuation of + ?FRESH_SELECT -> + #{meta => NMeta, data => []}; %% page and limit too big + _ -> + do_node_query(Node, Tab, Qs, QueryFun, NContinuation, NMeta, []) + end; + {cutrows, NMeta} -> + {SubStart, NeedNowNum} = rows_sub_params(Len, NMeta), + ThisRows = lists:sublist(Rows, SubStart, NeedNowNum), + NResults = lists:sublist( lists:append(Results, ThisRows) + , SubStart, Limit), + case NContinuation of + ?FRESH_SELECT -> + #{meta => NMeta, data => NResults}; + _ -> + do_node_query(Node, Tab, Qs, QueryFun, NContinuation, NMeta, NResults) + end; + {enough, NMeta} -> + NResults = lists:sublist(lists:append(Results, Rows), 1, Limit), + case NContinuation of + ?FRESH_SELECT -> + #{meta => NMeta, data => NResults}; + _ -> + do_node_query(Node, Tab, Qs, QueryFun, NContinuation, NMeta, NResults) + end + end. + +%%-------------------------------------------------------------------- +%% Cluster Query +%%-------------------------------------------------------------------- + +cluster_query(Params, Tab, QsSchema, QueryFun) -> + {_CodCnt, Qs} = params2qs(Params, QsSchema), + Limit = b2i(limit(Params)), + Page = b2i(page(Params)), + Nodes = ekka_mnesia:running_nodes(), + Meta = #{page => Page, limit => Limit, count => 0}, + do_cluster_query(Nodes, Tab, Qs, QueryFun, Meta). + +%% @private +do_cluster_query(Nodes, Tab, Qs, QueryFun, Meta) -> + do_cluster_query(Nodes, Tab, Qs, QueryFun, _Continuation = ?FRESH_SELECT, Meta, _Results = []). + +do_cluster_query([], _Tab, _Qs, _QueryFun, _Continuation, Meta, Results) -> + #{meta => Meta, data => Results}; +do_cluster_query( [Node | Nodes], Tab, Qs, QueryFun, Continuation + , Meta = #{limit := Limit} + , Results) -> + {Len, Rows, NContinuation} = do_query(Node, Tab, Qs, QueryFun, Continuation, Limit), + case judge_page_with_counting(Len, Meta) of + {more, NMeta} -> + case NContinuation of + ?FRESH_SELECT -> + do_cluster_query(Nodes, Tab, Qs, QueryFun, NContinuation, NMeta, []); %% next node with parts of results + _ -> + do_cluster_query([Node | Nodes], Tab, Qs, QueryFun, NContinuation, NMeta, []) %% continue this node + end; + {cutrows, NMeta} -> + {SubStart, NeedNowNum} = rows_sub_params(Len, NMeta), + ThisRows = lists:sublist(Rows, SubStart, NeedNowNum), + NResults = lists:sublist( lists:append(Results, ThisRows) + , SubStart, Limit), + case NContinuation of + ?FRESH_SELECT -> + do_cluster_query(Nodes, Tab, Qs, QueryFun, NContinuation, NMeta, NResults); %% next node with parts of results + _ -> + do_cluster_query([Node | Nodes], Tab, Qs, QueryFun, NContinuation, NMeta, NResults) %% continue this node + end; + {enough, NMeta} -> + NResults = lists:sublist(lists:append(Results, Rows), 1, Limit), + case NContinuation of + ?FRESH_SELECT -> + do_cluster_query(Nodes, Tab, Qs, QueryFun, NContinuation, NMeta, NResults); %% next node with parts of results + _ -> + do_cluster_query([Node | Nodes], Tab, Qs, QueryFun, NContinuation, NMeta, NResults) %% continue this node + end + end. + +%%-------------------------------------------------------------------- +%% Do Query (or rpc query) +%%-------------------------------------------------------------------- + +%% @private +do_query(Node, Tab, Qs, {M,F}, Continuation, Limit) when Node =:= node() -> + M:F(Tab, Qs, Continuation, Limit); +do_query(Node, Tab, Qs, QueryFun, Continuation, Limit) -> rpc_call(Node, ?MODULE, do_query, - [Node, Tab, Qs, QueryFun, Start, Limit], 50000). + [Node, Tab, Qs, QueryFun, Continuation, Limit], 50000). %% @private rpc_call(Node, M, F, A, T) -> @@ -108,108 +229,53 @@ rpc_call(Node, M, F, A, T) -> end. %%-------------------------------------------------------------------- -%% Cluster Query +%% Table Select %%-------------------------------------------------------------------- -cluster_query(Params, Tab, QsSchema, QueryFun) -> - {CodCnt, Qs} = params2qs(Params, QsSchema), - Limit = b2i(limit(Params)), - Page = b2i(page(Params)), - Start = if Page > 1 -> (Page-1) * Limit; - true -> 0 - end, - Nodes = ekka_mnesia:running_nodes(), - Rows = do_cluster_query(Nodes, Tab, Qs, QueryFun, Start, Limit+1, []), - Meta = #{page => Page, limit => Limit}, - NMeta = case CodCnt =:= 0 of - true -> Meta#{count => count(Tab, Nodes)}; - _ -> Meta#{count => length(Rows)} - end, - #{meta => NMeta, data => lists:sublist(Rows, Limit)}. - -%% @private -do_cluster_query([], _, _, _, _, _, Acc) -> - lists:append(lists:reverse(Acc)); -do_cluster_query([Node|Nodes], Tab, Qs, QueryFun, Start, Limit, Acc) -> - {NStart, Rows} = do_query(Node, Tab, Qs, QueryFun, Start, Limit), - case Limit - length(Rows) of - Rest when Rest > 0 -> - do_cluster_query(Nodes, Tab, Qs, QueryFun, NStart, Limit, [Rows|Acc]); - 0 -> - lists:append(lists:reverse([Rows|Acc])) - end. - -traverse_table(Tab, MatchFun, Start, Limit, FmtFun) -> - ets:safe_fixtable(Tab, true), - {NStart, Rows} = traverse_n_by_one(Tab, ets:first(Tab), MatchFun, Start, Limit, []), - ets:safe_fixtable(Tab, false), - {NStart, lists:map(FmtFun, Rows)}. - -%% @private -traverse_n_by_one(_, '$end_of_table', _, Start, _, Acc) -> - {Start, lists:flatten(lists:reverse(Acc))}; -traverse_n_by_one(_, _, _, Start, _Limit=0, Acc) -> - {Start, lists:flatten(lists:reverse(Acc))}; -traverse_n_by_one(Tab, K, MatchFun, Start, Limit, Acc) -> - GetRows = fun _GetRows('$end_of_table', _, Ks) -> - {'$end_of_table', Ks}; - _GetRows(Kn, 1, Ks) -> - {ets:next(Tab, Kn), [ets:lookup(Tab, Kn) | Ks]}; - _GetRows(Kn, N, Ks) -> - _GetRows(ets:next(Tab, Kn), N-1, [ets:lookup(Tab, Kn) | Ks]) - end, - {K2, Rows} = GetRows(K, 100, []), - case MatchFun(lists:flatten(lists:reverse(Rows))) of - [] -> - traverse_n_by_one(Tab, K2, MatchFun, Start, Limit, Acc); - Ls -> - case Start - length(Ls) of - N when N > 0 -> %% Skip - traverse_n_by_one(Tab, K2, MatchFun, N, Limit, Acc); - _ -> - Got = lists:sublist(Ls, Start+1, Limit), - NLimit = Limit - length(Got), - traverse_n_by_one(Tab, K2, MatchFun, 0, NLimit, [Got|Acc]) - end - end. - -select_table(Tab, Ms, 0, Limit, FmtFun) -> +select_table_with_count(Tab, {Ms, FuzzyFilterFun}, ?FRESH_SELECT, Limit, FmtFun) + when is_function(FuzzyFilterFun) andalso Limit > 0 -> case ets:select(Tab, Ms, Limit) of '$end_of_table' -> - {0, []}; - {Rows, _} -> - {0, lists:map(FmtFun, lists:reverse(Rows))} + {0, [], ?FRESH_SELECT}; + {RawResult, NContinuation} -> + Rows = FuzzyFilterFun(RawResult), + {length(Rows), lists:map(FmtFun, Rows), NContinuation} end; - -select_table(Tab, Ms, Start, Limit, FmtFun) -> - {NStart, Rows} = select_n_by_one(ets:select(Tab, Ms, Limit), Start, Limit, []), - {NStart, lists:map(FmtFun, Rows)}. - -select_n_by_one('$end_of_table', Start, _Limit, Acc) -> - {Start, lists:flatten(lists:reverse(Acc))}; -select_n_by_one(_, Start, _Limit = 0, Acc) -> - {Start, lists:flatten(lists:reverse(Acc))}; - -select_n_by_one({Rows0, Cons}, Start, Limit, Acc) -> - Rows = lists:reverse(Rows0), - case Start - length(Rows) of - N when N > 0 -> %% Skip - select_n_by_one(ets:select(Cons), N, Limit, Acc); - _ -> - Got = lists:sublist(Rows, Start+1, Limit), - NLimit = Limit - length(Got), - select_n_by_one(ets:select(Cons), 0, NLimit, [Got|Acc]) +select_table_with_count(_Tab, {_Ms, FuzzyFilterFun}, Continuation, _Limit, FmtFun) + when is_function(FuzzyFilterFun) -> + case ets:select(Continuation) of + '$end_of_table' -> + {0, [], ?FRESH_SELECT}; + {RawResult, NContinuation} -> + Rows = FuzzyFilterFun(RawResult), + {length(Rows), lists:map(FmtFun, Rows), NContinuation} + end; +select_table_with_count(Tab, Ms, ?FRESH_SELECT, Limit, FmtFun) + when Limit > 0 -> + case ets:select(Tab, Ms, Limit) of + '$end_of_table' -> + {0, [], ?FRESH_SELECT}; + {RawResult, NContinuation} -> + {length(RawResult), lists:map(FmtFun, RawResult), NContinuation} + end; +select_table_with_count(_Tab, _Ms, Continuation, _Limit, FmtFun) -> + case ets:select(Continuation) of + '$end_of_table' -> + {0, [], ?FRESH_SELECT}; + {RawResult, NContinuation} -> + {length(RawResult), lists:map(FmtFun, RawResult), NContinuation} end. +%%-------------------------------------------------------------------- +%% Internal funcs +%%-------------------------------------------------------------------- + params2qs(Params, QsSchema) when is_map(Params) -> params2qs(maps:to_list(Params), QsSchema); params2qs(Params, QsSchema) -> {Qs, Fuzzy} = pick_params_to_qs(Params, QsSchema, [], []), {length(Qs) + length(Fuzzy), {Qs, Fuzzy}}. -%%-------------------------------------------------------------------- -%% Internal funcs - pick_params_to_qs([], _, Acc1, Acc2) -> NAcc2 = [E || E <- Acc2, not lists:keymember(element(1, E), 1, Acc1)], {lists:reverse(Acc1), lists:reverse(NAcc2)}; @@ -274,8 +340,36 @@ is_fuzzy_key(<<"match_", _/binary>>) -> is_fuzzy_key(_) -> false. +page_start(Page, Limit) -> + if Page > 1 -> (Page-1) * Limit + 1; + true -> 1 + end. + +judge_page_with_counting(Len, Meta = #{page := Page, limit := Limit, count := Count}) -> + PageStart = page_start(Page, Limit), + PageEnd = Page * Limit, + case Count + Len of + NCount when NCount < PageStart -> + {more, Meta#{count => NCount}}; + NCount when NCount < PageEnd -> + {cutrows, Meta#{count => NCount}}; + NCount when NCount >= PageEnd -> + {enough, Meta#{count => NCount}} + end. + +rows_sub_params(Len, _Meta = #{page := Page, limit := Limit, count := Count}) -> + PageStart = page_start(Page, Limit), + if Count - Len < PageStart -> + NeedNowNum = Count - PageStart + 1, + SubStart = Len - NeedNowNum + 1, + {SubStart, NeedNowNum}; + true -> + {_SubStart = 1, _NeedNowNum = Len} + end. + %%-------------------------------------------------------------------- %% Types +%%-------------------------------------------------------------------- to_type(V, TargetType) -> try diff --git a/apps/emqx_management/src/emqx_mgmt_api_alarms.erl b/apps/emqx_management/src/emqx_mgmt_api_alarms.erl index db6484060..3641f82b0 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_alarms.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_alarms.erl @@ -88,9 +88,9 @@ alarms(delete, _Params) -> %%%============================================================================================== %% internal -query(Table, _QsSpec, Start, Limit) -> +query(Table, _QsSpec, Continuation, Limit) -> Ms = [{'$1',[],['$1']}], - emqx_mgmt_api:select_table(Table, Ms, Start, Limit, fun format_alarm/1). + emqx_mgmt_api:select_table_with_count(Table, Ms, Continuation, Limit, fun format_alarm/1). format_alarm(Alarms) when is_list(Alarms) -> [emqx_alarm:format(Alarm) || Alarm <- Alarms]; diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl index 16d2d99af..618976cb5 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl @@ -502,7 +502,7 @@ unsubscribe(#{clientid := ClientID, topic := Topic}) -> end. subscribe_batch(#{clientid := ClientID, topics := Topics}) -> - ArgList = [[ClientID, Topic, Qos]|| #{topic := Topic, qos := Qos} <- Topics], + ArgList = [[ClientID, Topic, Qos] || #{topic := Topic, qos := Qos} <- Topics], emqx_mgmt_util:batch_operation(?MODULE, do_subscribe, ArgList). %%-------------------------------------------------------------------- @@ -555,16 +555,16 @@ generate_qs(Qs) -> %%-------------------------------------------------------------------- %% Query Functions -query(Tab, {Qs, []}, Start, Limit) -> +query(Tab, {Qs, []}, Continuation, Limit) -> Ms = qs2ms(Qs), - emqx_mgmt_api:select_table(Tab, Ms, Start, Limit, - fun format_channel_info/1); + emqx_mgmt_api:select_table_with_count(Tab, Ms, Continuation, Limit, + fun format_channel_info/1); -query(Tab, {Qs, Fuzzy}, Start, Limit) -> +query(Tab, {Qs, Fuzzy}, Continuation, Limit) -> Ms = qs2ms(Qs), - MatchFun = match_fun(Ms, Fuzzy), - emqx_mgmt_api:traverse_table(Tab, MatchFun, Start, Limit, - fun format_channel_info/1). + FuzzyFilterFun = fuzzy_filter_fun(Fuzzy), + emqx_mgmt_api:select_table_with_count(Tab, {Ms, FuzzyFilterFun}, Continuation, Limit, + fun format_channel_info/1). %%-------------------------------------------------------------------- %% QueryString to Match Spec @@ -616,30 +616,24 @@ ms(created_at, X) -> %%-------------------------------------------------------------------- %% Match funcs -match_fun(Ms, Fuzzy) -> - MsC = ets:match_spec_compile(Ms), +fuzzy_filter_fun(Fuzzy) -> REFuzzy = lists:map(fun({K, like, S}) -> {ok, RE} = re:compile(S), {K, like, RE} end, Fuzzy), - fun(Rows) -> - case ets:match_spec_run(Rows, MsC) of - [] -> []; - Ls -> - lists:filter(fun(E) -> - run_fuzzy_match(E, REFuzzy) - end, Ls) - end + fun(MsRaws) when is_list(MsRaws) -> + lists:filter( fun(E) -> run_fuzzy_filter(E, REFuzzy) end + , MsRaws) end. -run_fuzzy_match(_, []) -> +run_fuzzy_filter(_, []) -> true; -run_fuzzy_match(E = {_, #{clientinfo := ClientInfo}, _}, [{Key, _, RE}|Fuzzy]) -> +run_fuzzy_filter(E = {_, #{clientinfo := ClientInfo}, _}, [{Key, _, RE} | Fuzzy]) -> Val = case maps:get(Key, ClientInfo, "") of undefined -> ""; V -> V end, - re:run(Val, RE, [{capture, none}]) == match andalso run_fuzzy_match(E, Fuzzy). + re:run(Val, RE, [{capture, none}]) == match andalso run_fuzzy_filter(E, Fuzzy). %%-------------------------------------------------------------------- %% format funcs diff --git a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl index ad8ce0f67..9e12cbe3a 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl @@ -33,7 +33,13 @@ -include_lib("emqx/include/emqx.hrl"). -define(NODE_LISTENER_NOT_FOUND, <<"Node name or listener id not found">>). +-define(NODE_NOT_FOUND_OR_DOWN, <<"Node not found or Down">>). -define(LISTENER_NOT_FOUND, <<"Listener id not found">>). +-define(ADDR_PORT_INUSE, <<"Addr port in use">>). +-define(CONFIG_SCHEMA_ERROR, <<"Config schema error">>). +-define(INVALID_LISTENER_PROTOCOL, <<"Invalid listener type">>). +-define(UPDATE_CONFIG_FAILED, <<"Update configuration failed">>). +-define(OPERATION_FAILED, <<"Operation failed">>). api_spec() -> { @@ -48,11 +54,11 @@ api_spec() -> [] }. --define(TYPES, [tcp, ssl, ws, wss, quic]). +-define(TYPES_ATOM, [tcp, ssl, ws, wss, quic]). req_schema() -> Schema = [emqx_mgmt_api_configs:gen_schema( emqx:get_raw_config([listeners, T, default], #{})) - || T <- ?TYPES], + || T <- ?TYPES_ATOM], #{oneOf => Schema}. resp_schema() -> @@ -90,8 +96,12 @@ api_list_update_listeners_by_id() -> parameters => [param_path_id()], requestBody => emqx_mgmt_util:schema(req_schema(), <<"Listener Config">>), responses => #{ + <<"400">> => + emqx_mgmt_util:error_schema(?UPDATE_CONFIG_FAILED, ['BAD_LISTENER_ID', 'BAD_CONFIG_SCHEMA']), <<"404">> => emqx_mgmt_util:error_schema(?LISTENER_NOT_FOUND, ['BAD_LISTENER_ID']), + <<"500">> => + emqx_mgmt_util:error_schema(?OPERATION_FAILED, ['INTERNAL_ERROR']), <<"200">> => emqx_mgmt_util:array_schema(resp_schema(), <<"Create or update listener successfully">>)}}, delete => #{ @@ -111,7 +121,12 @@ api_list_listeners_on_node() -> description => <<"List listeners in one node">>, parameters => [param_path_node()], responses => #{ - <<"200">> => emqx_mgmt_util:schema(resp_schema(), <<"List listeners successfully">>)}}}, + <<"404">> => + emqx_mgmt_util:error_schema(?NODE_NOT_FOUND_OR_DOWN, ['RESOURCE_NOT_FOUND']), + <<"500">> => + emqx_mgmt_util:error_schema(?OPERATION_FAILED, ['INTERNAL_ERROR']), + <<"200">> => + emqx_mgmt_util:schema(resp_schema(), <<"List listeners successfully">>)}}}, {"/nodes/:node/listeners", Metadata, list_listeners_on_node}. api_get_update_listener_by_id_on_node() -> @@ -130,9 +145,13 @@ api_get_update_listener_by_id_on_node() -> parameters => [param_path_node(), param_path_id()], requestBody => emqx_mgmt_util:schema(req_schema(), <<"Listener Config">>), responses => #{ + <<"400">> => + emqx_mgmt_util:error_schema(?UPDATE_CONFIG_FAILED, ['BAD_LISTENER_ID', 'BAD_CONFIG_SCHEMA']), <<"404">> => emqx_mgmt_util:error_schema(?NODE_LISTENER_NOT_FOUND, ['BAD_NODE_NAME', 'BAD_LISTENER_ID']), + <<"500">> => + emqx_mgmt_util:error_schema(?OPERATION_FAILED, ['INTERNAL_ERROR']), <<"200">> => emqx_mgmt_util:schema(resp_schema(), <<"Get listener successfully">>)}}, delete => #{ @@ -154,7 +173,7 @@ api_manage_listeners() -> param_path_id(), param_path_operation()], responses => #{ - <<"500">> => emqx_mgmt_util:error_schema(<<"Operation Failed">>, ['INTERNAL_ERROR']), + <<"500">> => emqx_mgmt_util:error_schema(?OPERATION_FAILED, ['INTERNAL_ERROR']), <<"200">> => emqx_mgmt_util:schema(<<"Operation success">>)}}}, {"/listeners/:id/operation/:operation", Metadata, manage_listeners}. @@ -167,7 +186,7 @@ api_manage_listeners_on_node() -> param_path_id(), param_path_operation()], responses => #{ - <<"500">> => emqx_mgmt_util:error_schema(<<"Operation Failed">>, ['INTERNAL_ERROR']), + <<"500">> => emqx_mgmt_util:error_schema(?OPERATION_FAILED, ['INTERNAL_ERROR']), <<"200">> => emqx_mgmt_util:schema(<<"Operation success">>)}}}, {"/nodes/:node/listeners/:id/operation/:operation", Metadata, manage_listeners}. @@ -215,16 +234,31 @@ crud_listeners_by_id(get, #{bindings := #{id := Id}}) -> {200, format(Listeners)} end; crud_listeners_by_id(put, #{bindings := #{id := Id}, body := Conf}) -> - return_listeners(emqx_mgmt:update_listener(Id, Conf)); + Results = format(emqx_mgmt:update_listener(Id, Conf)), + case lists:filter(fun filter_errors/1, Results) of + [{error, {invalid_listener_id, Id}} | _] -> + {400, #{code => 'BAD_REQUEST', message => ?INVALID_LISTENER_PROTOCOL}}; + [{error, {emqx_machine_schema, _}} | _] -> + {400, #{code => 'BAD_REQUEST', message => ?CONFIG_SCHEMA_ERROR}}; + [{error, {eaddrinuse, _}} | _] -> + {400, #{code => 'BAD_REQUEST', message => ?ADDR_PORT_INUSE}}; + [{error, Reason} | _] -> + {500, #{code => 'UNKNOWN_ERROR', message => err_msg(Reason)}}; + [] -> + {200, Results} + end; + crud_listeners_by_id(delete, #{bindings := #{id := Id}}) -> Results = emqx_mgmt:remove_listener(Id), - case lists:filter(fun({error, _}) -> true; (_) -> false end, Results) of + case lists:filter(fun filter_errors/1, Results) of [] -> {200}; Errors -> {500, #{code => 'UNKNOW_ERROR', message => err_msg(Errors)}} end. list_listeners_on_node(get, #{bindings := #{node := Node}}) -> case emqx_mgmt:list_listeners(atom(Node)) of + {error, nodedown} -> + {404, #{code => 'RESOURCE_NOT_FOUND', message => ?NODE_NOT_FOUND_OR_DOWN}}; {error, Reason} -> {500, #{code => 'UNKNOW_ERROR', message => err_msg(Reason)}}; Listener -> @@ -240,8 +274,21 @@ crud_listener_by_id_on_node(get, #{bindings := #{id := Id, node := Node}}) -> Listener -> {200, format(Listener)} end; -crud_listener_by_id_on_node(put, #{bindings := #{id := Id, node := Node, body := Conf}}) -> - return_listeners(emqx_mgmt:update_listener(atom(Node), Id, Conf)); +crud_listener_by_id_on_node(put, #{bindings := #{id := Id, node := Node}, body := Conf}) -> + case emqx_mgmt:update_listener(atom(Node), Id, Conf) of + {error, nodedown} -> + {404, #{code => 'RESOURCE_NOT_FOUND', message => ?NODE_NOT_FOUND_OR_DOWN}}; + {error, {invalid_listener_id, _}} -> + {400, #{code => 'BAD_REQUEST', message => ?INVALID_LISTENER_PROTOCOL}}; + {error, {emqx_machine_schema, _}} -> + {400, #{code => 'BAD_REQUEST', message => ?CONFIG_SCHEMA_ERROR}}; + {error, {eaddrinuse, _}} -> + {400, #{code => 'BAD_REQUEST', message => ?ADDR_PORT_INUSE}}; + {error, Reason} -> + {500, #{code => 'UNKNOW_ERROR', message => err_msg(Reason)}}; + Listener -> + {200, format(Listener)} + end; crud_listener_by_id_on_node(delete, #{bindings := #{id := Id, node := Node}}) -> case emqx_mgmt:remove_listener(atom(Node), Id) of ok -> {200}; @@ -288,13 +335,6 @@ do_manage_listeners2(<<"restart">>, Param) -> {500, #{code => 'UNKNOW_ERROR', message => err_msg(Reason)}} end. -return_listeners(Listeners) -> - Results = format(Listeners), - case lists:filter(fun({error, _}) -> true; (_) -> false end, Results) of - [] -> {200, Results}; - Errors -> {500, #{code => 'UNKNOW_ERROR', message => manage_listeners_err(Errors)}} - end. - manage_listeners_err(Errors) -> list_to_binary(lists:foldl(fun({Node, Err}, Str) -> err_msg_str(#{node => Node, error => Err}) ++ "; " ++ Str @@ -319,6 +359,11 @@ trans_running(Conf) -> Running end. +filter_errors({error, _}) -> + true; +filter_errors(_) -> + false. + jsonable_resp(bind, Port) when is_integer(Port) -> {bind, Port}; jsonable_resp(bind, {Addr, Port}) when is_tuple(Addr); is_integer(Port)-> diff --git a/apps/emqx_management/src/emqx_mgmt_api_routes.erl b/apps/emqx_management/src/emqx_mgmt_api_routes.erl index 870f66892..e95679400 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_routes.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_routes.erl @@ -105,9 +105,9 @@ generate_topic(Params = #{topic := Topic}) -> Params#{topic => uri_string:percent_decode(Topic)}; generate_topic(Params) -> Params. -query(Tab, {Qs, _}, Start, Limit) -> +query(Tab, {Qs, _}, Continuation, Limit) -> Ms = qs2ms(Qs, [{{route, '_', '_'}, [], ['$_']}]), - emqx_mgmt_api:select_table(Tab, Ms, Start, Limit, fun format/1). + emqx_mgmt_api:select_table_with_count(Tab, Ms, Continuation, Limit, fun format/1). qs2ms([], Res) -> Res; qs2ms([{topic,'=:=', T} | Qs], [{{route, _, N}, [], ['$_']}]) -> diff --git a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl index 3afd52050..2be5773f3 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl @@ -148,28 +148,25 @@ format({_Subscriber, Topic, Options}) -> %% Query Function %%-------------------------------------------------------------------- -query(Tab, {Qs, []}, Start, Limit) -> +query(Tab, {Qs, []}, Continuation, Limit) -> Ms = qs2ms(Qs), - emqx_mgmt_api:select_table(Tab, Ms, Start, Limit, fun format/1); + emqx_mgmt_api:select_table_with_count(Tab, Ms, Continuation, Limit, fun format/1); -query(Tab, {Qs, Fuzzy}, Start, Limit) -> +query(Tab, {Qs, Fuzzy}, Continuation, Limit) -> Ms = qs2ms(Qs), - MatchFun = match_fun(Ms, Fuzzy), - emqx_mgmt_api:traverse_table(Tab, MatchFun, Start, Limit, fun format/1). + FuzzyFilterFun = fuzzy_filter_fun(Fuzzy), + emqx_mgmt_api:select_table_with_count(Tab, {Ms, FuzzyFilterFun}, Continuation, Limit, fun format/1). -match_fun(Ms, Fuzzy) -> - MsC = ets:match_spec_compile(Ms), - fun(Rows) -> - case ets:match_spec_run(Rows, MsC) of - [] -> []; - Ls -> lists:filter(fun(E) -> run_fuzzy_match(E, Fuzzy) end, Ls) - end +fuzzy_filter_fun(Fuzzy) -> + fun(MsRaws) when is_list(MsRaws) -> + lists:filter( fun(E) -> run_fuzzy_filter(E, Fuzzy) end + , MsRaws) end. -run_fuzzy_match(_, []) -> +run_fuzzy_filter(_, []) -> true; -run_fuzzy_match(E = {{_, Topic}, _}, [{topic, match, TopicFilter}|Fuzzy]) -> - emqx_topic:match(Topic, TopicFilter) andalso run_fuzzy_match(E, Fuzzy). +run_fuzzy_filter(E = {{_, Topic}, _}, [{topic, match, TopicFilter} | Fuzzy]) -> + emqx_topic:match(Topic, TopicFilter) andalso run_fuzzy_filter(E, Fuzzy). %%-------------------------------------------------------------------- %% Query String to Match Spec diff --git a/apps/emqx_modules/src/emqx_delayed.erl b/apps/emqx_modules/src/emqx_delayed.erl index 21757b528..86ef97dac 100644 --- a/apps/emqx_modules/src/emqx_delayed.erl +++ b/apps/emqx_modules/src/emqx_delayed.erl @@ -164,13 +164,17 @@ to_rfc3339(Timestamp) -> list_to_binary(calendar:system_time_to_rfc3339(Timestamp, [{unit, second}])). get_delayed_message(Id0) -> - Id = emqx_guid:from_hexstr(Id0), - case ets:select(?TAB, ?QUERY_MS(Id)) of - [] -> - {error, not_found}; - Rows -> - Message = hd(Rows), - {ok, format_delayed(Message, true)} + try emqx_guid:from_hexstr(Id0) of + Id -> + case ets:select(?TAB, ?QUERY_MS(Id)) of + [] -> + {error, not_found}; + Rows -> + Message = hd(Rows), + {ok, format_delayed(Message, true)} + end + catch + error:function_clause -> {error, id_schema_error} end. delete_delayed_message(Id0) -> diff --git a/apps/emqx_modules/src/emqx_delayed_api.erl b/apps/emqx_modules/src/emqx_delayed_api.erl index 96589bf06..94a388767 100644 --- a/apps/emqx_modules/src/emqx_delayed_api.erl +++ b/apps/emqx_modules/src/emqx_delayed_api.erl @@ -18,22 +18,19 @@ -behavior(minirest_api). --import(emqx_mgmt_util, [ page_params/0 - , schema/1 - , schema/2 - , object_schema/2 - , error_schema/2 - , page_object_schema/1 - , properties/1 - ]). +-include_lib("typerefl/include/types.hrl"). + +-import(hoconsc, [mk/2, ref/1, ref/2]). -define(MAX_PAYLOAD_LENGTH, 2048). -define(PAYLOAD_TOO_LARGE, 'PAYLOAD_TOO_LARGE'). --export([ status/2 - , delayed_messages/2 - , delayed_message/2 - ]). +-export([status/2 + , delayed_messages/2 + , delayed_message/2 +]). + +-export([paths/0, fields/1, schema/1]). %% for rpc -export([update_config_/1]). @@ -46,90 +43,97 @@ -define(BAD_REQUEST, 'BAD_REQUEST'). -define(MESSAGE_ID_NOT_FOUND, 'MESSAGE_ID_NOT_FOUND'). +-define(MESSAGE_ID_SCHEMA_ERROR, 'MESSAGE_ID_SCHEMA_ERROR'). api_spec() -> - { - [status_api(), delayed_messages_api(), delayed_message_api()], - [] - }. + emqx_dashboard_swagger:spec(?MODULE). -conf_schema() -> - emqx_mgmt_api_configs:gen_schema(emqx:get_raw_config([delayed])). -properties() -> - PayloadDesc = io_lib:format("Payload, base64 encode. Payload will be ~p if length large than ~p", - [?PAYLOAD_TOO_LARGE, ?MAX_PAYLOAD_LENGTH]), - properties([ - {msgid, integer, <<"Message Id">>}, - {publish_at, string, <<"Client publish message time, rfc 3339">>}, - {delayed_interval, integer, <<"Delayed interval, second">>}, - {delayed_remaining, integer, <<"Delayed remaining, second">>}, - {expected_at, string, <<"Expect publish time, rfc 3339">>}, - {topic, string, <<"Topic">>}, - {qos, string, <<"QoS">>}, - {payload, string, iolist_to_binary(PayloadDesc)}, - {from_clientid, string, <<"From ClientId">>}, - {from_username, string, <<"From Username">>} - ]). +paths() -> ["/mqtt/delayed", "/mqtt/delayed/messages", "/mqtt/delayed/messages/:msgid"]. -parameters() -> - [#{ - name => msgid, - in => path, - schema => #{type => string}, - required => true - }]. - -status_api() -> - Metadata = #{ +schema("/mqtt/delayed") -> + #{ + operationId => status, get => #{ + tags => [<<"mqtt">>], description => <<"Get delayed status">>, + summary => <<"Get delayed status">>, responses => #{ - <<"200">> => schema(conf_schema())} - }, + 200 => ref(emqx_modules_schema, "delayed") + } + }, put => #{ + tags => [<<"mqtt">>], description => <<"Enable or disable delayed, set max delayed messages">>, - 'requestBody' => schema(conf_schema()), + requestBody => ref(emqx_modules_schema, "delayed"), responses => #{ - <<"200">> => - schema(conf_schema(), <<"Enable or disable delayed successfully">>), - <<"400">> => - error_schema(<<"Max limit illegality">>, [?BAD_REQUEST]) + 200 => mk(ref(emqx_modules_schema, "delayed"), + #{desc => <<"Enable or disable delayed successfully">>}), + 400 => emqx_dashboard_swagger:error_codes([?BAD_REQUEST], <<"Max limit illegality">>) } } - }, - {"/mqtt/delayed", Metadata, status}. + }; -delayed_messages_api() -> - Metadata = #{ - get => #{ - description => "List delayed messages", - parameters => page_params(), - responses => #{ - <<"200">> => page_object_schema(properties()) - } - } - }, - {"/mqtt/delayed/messages", Metadata, delayed_messages}. - -delayed_message_api() -> - Metadata = #{ +schema("/mqtt/delayed/messages/:msgid") -> + #{operationId => delayed_message, get => #{ + tags => [<<"mqtt">>], description => <<"Get delayed message">>, - parameters => parameters(), + parameters => [{msgid, mk(binary(), #{in => path, desc => <<"delay message ID">>})}], responses => #{ - <<"200">> => object_schema(maps:without([payload], properties()), <<"Get delayed message success">>), - <<"404">> => error_schema(<<"Message ID not found">>, [?MESSAGE_ID_NOT_FOUND]) + 200 => ref("message_without_payload"), + 400 => emqx_dashboard_swagger:error_codes([?MESSAGE_ID_SCHEMA_ERROR], <<"Bad MsgId format">>), + 404 => emqx_dashboard_swagger:error_codes([?MESSAGE_ID_NOT_FOUND], <<"MsgId not found">>) } }, delete => #{ + tags => [<<"mqtt">>], description => <<"Delete delayed message">>, - parameters => parameters(), + parameters => [{msgid, mk(binary(), #{in => path, desc => <<"delay message ID">>})}], responses => #{ - <<"200">> => schema(<<"Delete delayed message success">>) + 200 => <<"Delete delayed message success">>, + 400 => emqx_dashboard_swagger:error_codes([?MESSAGE_ID_SCHEMA_ERROR], <<"Bad MsgId format">>), + 404 => emqx_dashboard_swagger:error_codes([?MESSAGE_ID_NOT_FOUND], <<"MsgId not found">>) } } - }, - {"/mqtt/delayed/messages/:msgid", Metadata, delayed_message}. + }; +schema("/mqtt/delayed/messages") -> + #{ + operationId => delayed_messages, + get => #{ + tags => [<<"mqtt">>], + description => <<"List delayed messages">>, + parameters => [ref(emqx_dashboard_swagger, page), ref(emqx_dashboard_swagger, limit)], + responses => #{ + 200 => + [ + {data, mk(hoconsc:array(ref("message")), #{})}, + {meta, [ + {page, mk(integer(), #{})}, + {limit, mk(integer(), #{})}, + {count, mk(integer(), #{})} + ]} + ] + } + } + }. + +fields("message_without_payload") -> + [ + {msgid, mk(integer(), #{desc => <<"Message Id (MQTT message id hash)">>})}, + {publish_at, mk(binary(), #{desc => <<"Client publish message time, rfc 3339">>})}, + {delayed_interval, mk(integer(), #{desc => <<"Delayed interval, second">>})}, + {delayed_remaining, mk(integer(), #{desc => <<"Delayed remaining, second">>})}, + {expected_at, mk(binary(), #{desc => <<"Expect publish time, rfc 3339">>})}, + {topic, mk(binary(), #{desc => <<"Topic">>, example => <<"/sys/#">>})}, + {qos, mk(binary(), #{desc => <<"QoS">>})}, + {from_clientid, mk(binary(), #{desc => <<"From ClientId">>})}, + {from_username, mk(binary(), #{desc => <<"From Username">>})} + ]; +fields("message") -> + PayloadDesc = io_lib:format("Payload, base64 encode. Payload will be ~p if length large than ~p", + [?PAYLOAD_TOO_LARGE, ?MAX_PAYLOAD_LENGTH]), + fields("message_without_payload") ++ + [{payload, mk(binary(), #{desc => iolist_to_binary(PayloadDesc)})}]. %%-------------------------------------------------------------------- %% HTTP API @@ -151,15 +155,23 @@ delayed_message(get, #{bindings := #{msgid := Id}}) -> true -> {200, Message#{payload => ?PAYLOAD_TOO_LARGE}}; _ -> - {200, Message#{payload => base64:encode(Payload)}} + {200, Message#{payload => Payload}} end; + {error, id_schema_error} -> + {400, generate_http_code_map(id_schema_error, Id)}; {error, not_found} -> - Message = iolist_to_binary(io_lib:format("Message ID ~p not found", [Id])), - {404, #{code => ?MESSAGE_ID_NOT_FOUND, message => Message}} + {404, generate_http_code_map(not_found, Id)} end; delayed_message(delete, #{bindings := #{msgid := Id}}) -> - _ = emqx_delayed:delete_delayed_message(Id), - {200}. + case emqx_delayed:get_delayed_message(Id) of + {ok, _Message} -> + _ = emqx_delayed:delete_delayed_message(Id), + {200}; + {error, id_schema_error} -> + {400, generate_http_code_map(id_schema_error, Id)}; + {error, not_found} -> + {404, generate_http_code_map(not_found, Id)} + end. %%-------------------------------------------------------------------- %% internal function @@ -198,7 +210,7 @@ generate_max_delayed_messages(Config) -> update_config_(Config) -> lists:foreach(fun(Node) -> update_config_(Node, Config) - end, ekka_mnesia:running_nodes()). + end, ekka_mnesia:running_nodes()). update_config_(Node, Config) when Node =:= node() -> _ = emqx_delayed:update_config(Config), @@ -220,6 +232,11 @@ update_config_(Node, Config) when Node =:= node() -> update_config_(Node, Config) -> rpc_call(Node, ?MODULE, ?FUNCTION_NAME, [Node, Config]). +generate_http_code_map(id_schema_error, Id) -> + #{code => ?MESSAGE_ID_SCHEMA_ERROR, message => iolist_to_binary(io_lib:format("Message ID ~p schema error", [Id]))}; +generate_http_code_map(not_found, Id) -> + #{code => ?MESSAGE_ID_NOT_FOUND, message => iolist_to_binary(io_lib:format("Message ID ~p not found", [Id]))}. + rpc_call(Node, Module, Fun, Args) -> case rpc:call(Node, Module, Fun, Args) of {badrpc, Reason} -> {error, Reason}; diff --git a/apps/emqx_modules/src/emqx_topic_metrics.erl b/apps/emqx_modules/src/emqx_topic_metrics.erl index 00e1e4bbc..05c45f469 100644 --- a/apps/emqx_modules/src/emqx_topic_metrics.erl +++ b/apps/emqx_modules/src/emqx_topic_metrics.erl @@ -304,8 +304,8 @@ do_register(Topic, Speeds) -> true -> {error, already_existed}; false -> - case number_of_registered_topics() < ?MAX_TOPICS of - true -> + case {number_of_registered_topics() < ?MAX_TOPICS, emqx_topic:wildcard(Topic)} of + {true, false} -> CreateTime = emqx_rule_funcs:now_rfc3339(), CRef = counters:new(counters_size(), [write_concurrency]), ok = reset_counter(CRef), @@ -318,8 +318,12 @@ do_register(Topic, Speeds) -> end, Speeds, ?TOPIC_METRICS), add_topic_config(Topic), {ok, NSpeeds}; - false -> - {error, quota_exceeded} + {true, true} -> + {error, bad_topic}; + {false, false} -> + {error, quota_exceeded}; + {false, true} -> + {error, {quota_exceeded, bad_topic}} end end. diff --git a/apps/emqx_modules/src/emqx_topic_metrics_api.erl b/apps/emqx_modules/src/emqx_topic_metrics_api.erl index 98ae249cb..fa2547e58 100644 --- a/apps/emqx_modules/src/emqx_topic_metrics_api.erl +++ b/apps/emqx_modules/src/emqx_topic_metrics_api.erl @@ -36,6 +36,8 @@ -define(EXCEED_LIMIT, 'EXCEED_LIMIT'). +-define(BAD_TOPIC, 'BAD_TOPIC'). + -define(BAD_REQUEST, 'BAD_REQUEST'). api_spec() -> @@ -94,7 +96,7 @@ topic_metrics_api() -> responses => #{ <<"200">> => schema(<<"Create topic metrics success">>), <<"409">> => error_schema(<<"Topic metrics max limit">>, [?EXCEED_LIMIT]), - <<"400">> => error_schema(<<"Topic metrics already exist or bad topic">>, [?BAD_REQUEST]) + <<"400">> => error_schema(<<"Topic metrics already exist or bad topic">>, [?BAD_REQUEST, ?BAD_TOPIC]) } } }, @@ -164,12 +166,20 @@ list_metrics() -> register(Topic) -> case emqx_topic_metrics:register(Topic) of {error, quota_exceeded} -> - Message = list_to_binary(io_lib:format("Max topic metrics count is ~p", + Message = list_to_binary(io_lib:format("Max topic metrics count is ~p", [emqx_topic_metrics:max_limit()])), {409, #{code => ?EXCEED_LIMIT, message => Message}}; + {error, bad_topic} -> + Message = list_to_binary(io_lib:format("Bad Topic, topic cannot have wildcard ~p", + [Topic])), + {400, #{code => ?BAD_TOPIC, message => Message}}; + {error, {quota_exceeded, bad_topic}} -> + Message = list_to_binary(io_lib:format("Max topic metrics count is ~p, and topic cannot have wildcard ~p", + [emqx_topic_metrics:max_limit(), Topic])), + {400, #{code => ?BAD_REQUEST, message => Message}}; {error, already_existed} -> Message = list_to_binary(io_lib:format("Topic ~p already registered", [Topic])), - {400, #{code => ?BAD_REQUEST, message => Message}}; + {400, #{code => ?BAD_TOPIC, message => Message}}; ok -> {200} end. diff --git a/apps/emqx_psk/data/init.psk b/apps/emqx_psk/data/init.psk new file mode 100644 index 000000000..7e1a1edf5 --- /dev/null +++ b/apps/emqx_psk/data/init.psk @@ -0,0 +1,2 @@ +myclient1:8c701116e9127c57a99d5563709af3deaca75563e2c4dd0865701ae839fb6d79 +myclient2:d1e617d3b963757bfc21dad3fea169716c3a2f053f23decaea5cdfaabd04bfc4 diff --git a/apps/emqx_psk/etc/emqx_psk.conf b/apps/emqx_psk/etc/emqx_psk.conf new file mode 100644 index 000000000..80b29bfd4 --- /dev/null +++ b/apps/emqx_psk/etc/emqx_psk.conf @@ -0,0 +1,22 @@ +##-------------------------------------------------------------------- +## EMQ X PSK +##-------------------------------------------------------------------- + +psk { + ## Whether to enable the PSK feature. + enable = false + + ## If init file is specified, emqx will import PSKs from the file + ## into the built-in database at startup for use by the runtime. + ## + ## The file has to be structured line-by-line, each line must be in + ## the format: : + ## init_file = "{{ platform_data_dir }}/init.psk" + + ## Specifies the separator for PSKIdentity and SharedSecret in the init file. + ## The default is colon (:) + ## separator = ":" + + ## The size of each chunk used to import to the built-in database from psk file + ## chunk_size = 50 +} diff --git a/apps/emqx_rule_actions/mix.exs b/apps/emqx_psk/mix.exs similarity index 90% rename from apps/emqx_rule_actions/mix.exs rename to apps/emqx_psk/mix.exs index 40b658f50..2472b4a31 100644 --- a/apps/emqx_rule_actions/mix.exs +++ b/apps/emqx_psk/mix.exs @@ -18,6 +18,8 @@ defmodule EmqxRuleActions.MixProject do def application do [ + registered: [:emqx_psk_sup], + mod: {:emqx_psk_app, []}, extra_applications: [:logger] ] end diff --git a/apps/emqx_rule_actions/rebar.config b/apps/emqx_psk/rebar.config similarity index 57% rename from apps/emqx_rule_actions/rebar.config rename to apps/emqx_psk/rebar.config index 097c18a3d..73696b033 100644 --- a/apps/emqx_rule_actions/rebar.config +++ b/apps/emqx_psk/rebar.config @@ -1,25 +1,18 @@ {deps, []}. +{edoc_opts, [{preprocess, true}]}. {erl_opts, [warn_unused_vars, warn_shadow_vars, + warnings_as_errors, warn_unused_import, warn_obsolete_guard, - no_debug_info, - compressed, %% for edge - {parse_transform} - ]}. - -{overrides, [{add, [{erl_opts, [no_debug_info, compressed]}]}]}. - -{edoc_opts, [{preprocess, true}]}. + debug_info, + {parse_transform}]}. {xref_checks, [undefined_function_calls, undefined_functions, locals_not_used, deprecated_function_calls, - warnings_as_errors, deprecated_functions - ]}. + warnings_as_errors, deprecated_functions]}. {cover_enabled, true}. {cover_opts, [verbose]}. {cover_export_enabled, true}. - -{plugins, [rebar3_proper]}. diff --git a/apps/emqx_psk/src/emqx_psk.app.src b/apps/emqx_psk/src/emqx_psk.app.src new file mode 100644 index 000000000..3d749abf6 --- /dev/null +++ b/apps/emqx_psk/src/emqx_psk.app.src @@ -0,0 +1,15 @@ +%% -*- mode: erlang -*- +{application, emqx_psk, + [{description, "EMQ X PSK"}, + {vsn, "5.0.0"}, % strict semver, bump manually! + {modules, []}, + {registered, [emqx_psk_sup]}, + {applications, [kernel,stdlib]}, + {mod, {emqx_psk_app,[]}}, + {env, []}, + {licenses, ["Apache-2.0"]}, + {maintainers, ["EMQ X Team "]}, + {links, [{"Homepage", "https://emqx.io/"}, + {"Github", "https://github.com/emqx/emqx"} + ]} + ]}. diff --git a/apps/emqx_psk/src/emqx_psk.erl b/apps/emqx_psk/src/emqx_psk.erl new file mode 100644 index 000000000..9ea25cdeb --- /dev/null +++ b/apps/emqx_psk/src/emqx_psk.erl @@ -0,0 +1,251 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_psk). + +-behaviour(gen_server). + +-include_lib("emqx/include/logger.hrl"). + +-export([ load/0 + , unload/0 + , on_psk_lookup/2 + , import/1 + ]). + +-export([ start_link/0 + , stop/0 + ]). + +%% gen_server callbacks +-export([ init/1 + , handle_call/3 + , handle_cast/2 + , handle_info/2 + , terminate/2 + , code_change/3 + ]). + +-record(psk_entry, {psk_id :: binary(), + shared_secret :: binary(), + extra :: term() + }). + +-export([mnesia/1]). + +-boot_mnesia({mnesia, [boot]}). +-copy_mnesia({mnesia, [copy]}). + +-define(TAB, ?MODULE). +-define(PSK_SHARD, emqx_psk_shard). + +-define(DEFAULT_DELIMITER, <<":">>). + +-define(CR, 13). +-define(LF, 10). + +%%------------------------------------------------------------------------------ +%% Mnesia bootstrap +%%------------------------------------------------------------------------------ + +%% @doc Create or replicate tables. +-spec(mnesia(boot | copy) -> ok). +mnesia(boot) -> + ok = ekka_mnesia:create_table(?TAB, [ + {rlog_shard, ?PSK_SHARD}, + {type, ordered_set}, + {disc_copies, [node()]}, + {record_name, psk_entry}, + {attributes, record_info(fields, psk_entry)}, + {storage_properties, [{ets, [{read_concurrency, true}]}]}]); + +mnesia(copy) -> + ok = ekka_mnesia:copy_table(?TAB, disc_copies). + +%%------------------------------------------------------------------------------ +%% APIs +%%------------------------------------------------------------------------------ + +load() -> + emqx:hook('tls_handshake.psk_lookup', {?MODULE, on_psk_lookup, []}). + +unload() -> + emqx:unhook('tls_handshake.psk_lookup', {?MODULE, on_psk_lookup, []}). + +on_psk_lookup(PSKIdentity, _UserState) -> + case mnesia:dirty_read(?TAB, PSKIdentity) of + [#psk_entry{shared_secret = SharedSecret}] -> + {stop, {ok, SharedSecret}}; + _ -> + ignore + end. + +import(SrcFile) -> + call({import, SrcFile}). + +-spec start_link() -> {ok, pid()} | ignore | {error, term()}. +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +-spec stop() -> ok. +stop() -> + gen_server:stop(?MODULE). + +%%-------------------------------------------------------------------- +%% gen_server callbacks +%%-------------------------------------------------------------------- + +init(_Opts) -> + _ = case get_config(enable) of + true -> load(); + false -> ?SLOG(info, #{msg => "emqx_psk_disabled"}) + end, + _ = case get_config(init_file) of + undefined -> ok; + InitFile -> import_psks(InitFile) + end, + {ok, #{}}. + +handle_call({import, SrcFile}, _From, State) -> + {reply, import_psks(SrcFile), State}; + +handle_call(Req, _From, State) -> + ?SLOG(info, #{msg => "unexpected_call_discarded", req => Req}), + {reply, {error, unexecpted}, State}. + +handle_cast(Req, State) -> + ?SLOG(info, #{msg => "unexpected_cast_discarded", req => Req}), + {noreply, State}. + +handle_info(Info, State) -> + ?SLOG(info, #{msg => "unexpected_info_discarded", info => Info}), + {noreply, State}. + +terminate(_Reason, _State) -> + unload(), + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%------------------------------------------------------------------------------ +%% Internal functions +%%------------------------------------------------------------------------------ + +get_config(enable) -> + emqx_config:get([psk, enable]); +get_config(init_file) -> + emqx_config:get([psk, init_file], undefined); +get_config(separator) -> + emqx_config:get([psk, separator], ?DEFAULT_DELIMITER); +get_config(chunk_size) -> + emqx_config:get([psk, chunk_size]). + +import_psks(SrcFile) -> + case file:open(SrcFile, [read, raw, binary, read_ahead]) of + {error, Reason} -> + ?SLOG(error, #{msg => "failed_to_open_psk_file", + file => SrcFile, + reason => Reason}), + {error, Reason}; + {ok, Io} -> + try import_psks(Io, get_config(separator), get_config(chunk_size), 0) of + ok -> ok; + {error, Reason} -> + ?SLOG(error, #{msg => "failed_to_import_psk_file", + file => SrcFile, + reason => Reason}), + {error, Reason} + catch + Exception:Reason:Stacktrace -> + ?SLOG(error, #{msg => "failed_to_import_psk_file", + file => SrcFile, + exception => Exception, + reason => Reason, + stacktrace => Stacktrace}), + {error, Reason} + after + _ = file:close(Io) + end + end. + +import_psks(Io, Delimiter, ChunkSize, NChunk) -> + case get_psks(Io, Delimiter, ChunkSize) of + {ok, Entries} -> + _ = trans(fun insert_psks/1, [Entries]), + import_psks(Io, Delimiter, ChunkSize, NChunk + 1); + {eof, Entries} -> + _ = trans(fun insert_psks/1, [Entries]), + ok; + {error, {bad_format, {line, N}}} -> + {error, {bad_format, {line, NChunk * ChunkSize + N}}}; + {error, Reaosn} -> + {error, Reaosn} + end. + +get_psks(Io, Delimiter, Max) -> + get_psks(Io, Delimiter, Max, []). + +get_psks(_Io, _Delimiter, 0, Acc) -> + {ok, Acc}; +get_psks(Io, Delimiter, Remaining, Acc) -> + case file:read_line(Io) of + {ok, Line} -> + case binary:split(Line, Delimiter) of + [PSKIdentity, SharedSecret] -> + NSharedSecret = trim_crlf(SharedSecret), + get_psks(Io, Delimiter, Remaining - 1, [{PSKIdentity, NSharedSecret} | Acc]); + _ -> + {error, {bad_format, {line, length(Acc) + 1}}} + end; + eof -> + {eof, Acc}; + {error, Reason} -> + {error, Reason} + end. + +insert_psks(Entries) -> + lists:foreach(fun(Entry) -> + insert_psk(Entry) + end, Entries). + +insert_psk({PSKIdentity, SharedSecret}) -> + mnesia:write(?TAB, #psk_entry{psk_id = PSKIdentity, shared_secret = SharedSecret}, write). + +trim_crlf(Bin) -> + Size = byte_size(Bin), + case binary:at(Bin, Size - 1) of + ?LF -> + case binary:at(Bin, Size - 2) of + ?CR -> binary:part(Bin, 0, Size - 2); + _ -> binary:part(Bin, 0, Size - 1) + end; + _ -> Bin + end. + +trans(Fun, Args) -> + case ekka_mnesia:transaction(?PSK_SHARD, Fun, Args) of + {atomic, Res} -> Res; + {aborted, Reason} -> {error, Reason} + end. + +call(Request) -> + try + gen_server:call(?MODULE, Request, 10000) + catch + exit:{timeout, _Details} -> + {error, timeout} + end. diff --git a/apps/emqx_rule_engine/src/emqx_rule_locker.erl b/apps/emqx_psk/src/emqx_psk_app.erl similarity index 68% rename from apps/emqx_rule_engine/src/emqx_rule_locker.erl rename to apps/emqx_psk/src/emqx_psk_app.erl index 9e45b8c09..95e947291 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_locker.erl +++ b/apps/emqx_psk/src/emqx_psk_app.erl @@ -14,21 +14,17 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_rule_locker). +-module(emqx_psk_app). --export([start_link/0]). +-behaviour(application). --export([ lock/1 - , unlock/1 +-export([ start/2 + , stop/1 ]). -start_link() -> - ekka_locker:start_link(?MODULE). +start(_Type, _Args) -> + {ok, Sup} = emqx_psk_sup:start_link(), + {ok, Sup}. --spec(lock(binary()) -> ekka_locker:lock_result()). -lock(Id) -> - ekka_locker:acquire(?MODULE, Id, local). - --spec(unlock(binary()) -> {boolean(), [node()]}). -unlock(Id) -> - ekka_locker:release(?MODULE, Id, local). +stop(_State) -> + ok. diff --git a/apps/emqx_psk/src/emqx_psk_schema.erl b/apps/emqx_psk/src/emqx_psk_schema.erl new file mode 100644 index 000000000..cce51d3fa --- /dev/null +++ b/apps/emqx_psk/src/emqx_psk_schema.erl @@ -0,0 +1,60 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_psk_schema). + +-behaviour(hocon_schema). + +-include_lib("typerefl/include/types.hrl"). + +-export([ roots/0 + , fields/1 + ]). + +roots() -> ["psk"]. + +fields("psk") -> + [ {enable, fun enable/1} + , {init_file, fun init_file/1} + , {separator, fun separator/1} + , {chunk_size, fun chunk_size/1} + ]. + +enable(type) -> boolean(); +enable(desc) -> <<"Whether to enable tls psk support">>; +enable(default) -> false; +enable(_) -> undefined. + +init_file(type) -> binary(); +init_file(desc) -> + <<"If init_file is specified, emqx will import PSKs from the file ", + "into the built-in database at startup for use by the runtime. ", + "The file has to be structured line-by-line, each line must be in ", + "the format: :">>; +init_file(nullable) -> true; +init_file(_) -> undefined. + +separator(type) -> binary(); +separator(desc) -> + <<"The separator between PSKIdentity and SharedSecret in the psk file">>; +separator(default) -> <<":">>; +separator(_) -> undefined. + +chunk_size(type) -> integer(); +chunk_size(desc) -> + <<"The size of each chunk used to import to the built-in database from psk file">>; +chunk_size(default) -> 50; +chunk_size(_) -> undefined. diff --git a/apps/emqx_psk/src/emqx_psk_sup.erl b/apps/emqx_psk/src/emqx_psk_sup.erl new file mode 100644 index 000000000..2c5181735 --- /dev/null +++ b/apps/emqx_psk/src/emqx_psk_sup.erl @@ -0,0 +1,35 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_psk_sup). + +-behaviour(supervisor). + +-export([start_link/0]). + +-export([init/1]). + +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +init([]) -> + {ok, {{one_for_one, 10, 3600}, + [#{id => emqx_psk, + start => {emqx_psk, start_link, []}, + restart => permanent, + shutdown => 5000, + type => worker, + modules => [emqx_psk]}]}}. diff --git a/apps/emqx_psk/test/data/init.psk b/apps/emqx_psk/test/data/init.psk new file mode 100644 index 000000000..7e1a1edf5 --- /dev/null +++ b/apps/emqx_psk/test/data/init.psk @@ -0,0 +1,2 @@ +myclient1:8c701116e9127c57a99d5563709af3deaca75563e2c4dd0865701ae839fb6d79 +myclient2:d1e617d3b963757bfc21dad3fea169716c3a2f053f23decaea5cdfaabd04bfc4 diff --git a/apps/emqx_psk/test/emqx_psk_SUITE.erl b/apps/emqx_psk/test/emqx_psk_SUITE.erl new file mode 100644 index 000000000..fa24322a2 --- /dev/null +++ b/apps/emqx_psk/test/emqx_psk_SUITE.erl @@ -0,0 +1,85 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_psk_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + meck:new(emqx_config, [non_strict, passthrough, no_history, no_link]), + meck:expect(emqx_config, get, fun([psk, enable]) -> true; + ([psk, chunk_size]) -> 50; + (KeyPath) -> meck:passthrough([KeyPath]) + end), + meck:expect(emqx_config, get, fun([psk, init_file], _) -> + filename:join([code:lib_dir(emqx_psk, test), "data/init.psk"]); + ([psk, separator], _) -> <<":">>; + (KeyPath, Default) -> meck:passthrough([KeyPath, Default]) + end), + emqx_ct_helpers:start_apps([emqx_psk]), + Config. + +end_per_suite(_) -> + meck:unload(emqx_config), + emqx_ct_helpers:stop_apps([emqx_psk]), + ok. + +t_psk_lookup(_) -> + + PSKIdentity1 = <<"myclient1">>, + SharedSecret1 = <<"8c701116e9127c57a99d5563709af3deaca75563e2c4dd0865701ae839fb6d79">>, + ?assertEqual({stop, {ok, SharedSecret1}}, emqx_psk:on_psk_lookup(PSKIdentity1, any)), + + PSKIdentity2 = <<"myclient2">>, + SharedSecret2 = <<"d1e617d3b963757bfc21dad3fea169716c3a2f053f23decaea5cdfaabd04bfc4">>, + ?assertEqual({stop, {ok, SharedSecret2}}, emqx_psk:on_psk_lookup(PSKIdentity2, any)), + + ?assertEqual(ignore, emqx_psk:on_psk_lookup(<<"myclient3">>, any)), + + ClientLookup = fun(psk, undefined, _) -> {ok, SharedSecret1}; + (psk, _, _) -> error + end, + + ClientTLSOpts = #{ versions => ['tlsv1.2'] + , ciphers => ["PSK-AES256-CBC-SHA"] + , psk_identity => "myclient1" + , verify => verify_none + , user_lookup_fun => {ClientLookup, undefined} + }, + + ServerTLSOpts = #{ versions => ['tlsv1.2'] + , ciphers => ["PSK-AES256-CBC-SHA"] + , verify => verify_none + , reuseaddr => true + , user_lookup_fun => {fun emqx_tls_psk:lookup/3, undefined} + }, + emqx_config:put([listeners, ssl ,default, ssl], ServerTLSOpts), + emqx_listeners:restart_listener('ssl:default'), + + {ok, Socket} = ssl:connect("127.0.0.1", 8883, maps:to_list(ClientTLSOpts)), + ssl:close(Socket), + + ClientTLSOpts1 = ClientTLSOpts#{psk_identity => "myclient2"}, + ?assertMatch({error, _}, ssl:connect("127.0.0.1", 8883, maps:to_list(ClientTLSOpts1))), + + ok. + diff --git a/apps/emqx_retainer/src/emqx_retainer.erl b/apps/emqx_retainer/src/emqx_retainer.erl index 3df52b7a7..147e61ab7 100644 --- a/apps/emqx_retainer/src/emqx_retainer.erl +++ b/apps/emqx_retainer/src/emqx_retainer.erl @@ -187,8 +187,9 @@ init([]) -> end}. handle_call({update_config, Conf}, _, State) -> - {ok, Config} = emqx:update_config([?APP], Conf), - State2 = update_config(State, maps:get(config, Config)), + OldConf = emqx:get_config([?APP]), + {ok, #{config := NewConf}} = emqx:update_config([?APP], Conf), + State2 = update_config(State, NewConf, OldConf), {reply, ok, State2}; handle_call({wait_semaphore, Id}, From, #{wait_quotas := Waits} = State) -> @@ -343,34 +344,43 @@ insert_shared_context(Key, Term) -> get_msg_deliver_quota() -> emqx:get_config([?APP, flow_control, msg_deliver_quota]). --spec update_config(state(), hocons:config()) -> state(). -update_config(#{clear_timer := ClearTimer, - release_quota_timer := QuotaTimer} = State, Conf) -> - #{enable := Enable, - config := Config, +-spec update_config(state(), hocons:config(), hocons:config()) -> state(). +update_config(State, Conf, OldConf) -> + update_config(maps:get(enable, Conf), + maps:get(enable, OldConf), + State, + Conf, + OldConf). + +-spec update_config(boolean(), boolean(), state(), hocons:config(), hocons:config()) -> state(). +update_config(false, _, State, _, _) -> + disable_retainer(State); + +update_config(true, false, State, NewConf, _) -> + enable_retainer(State, NewConf); + +update_config(true, true, + #{clear_timer := ClearTimer, + release_quota_timer := QuotaTimer} = State, NewConf, OldConf) -> + #{config := Cfg, flow_control := #{quota_release_interval := QuotaInterval}, - msg_clear_interval := ClearInterval} = Conf, + msg_clear_interval := ClearInterval} = NewConf, - #{config := OldConfig} = emqx:get_config([?APP]), + #{config := OldCfg} = OldConf, - case Enable of - true -> - StorageType = maps:get(type, Config), - OldStrorageType = maps:get(type, OldConfig), - case OldStrorageType of - StorageType -> - State#{clear_timer := check_timer(ClearTimer, - ClearInterval, - clear_expired), - release_quota_timer := check_timer(QuotaTimer, - QuotaInterval, - release_deliver_quota)}; - _ -> - State2 = disable_retainer(State), - enable_retainer(State2, Conf) - end; + StorageType = maps:get(type, Cfg), + OldStrorageType = maps:get(type, OldCfg), + case OldStrorageType of + StorageType -> + State#{clear_timer := check_timer(ClearTimer, + ClearInterval, + clear_expired), + release_quota_timer := check_timer(QuotaTimer, + QuotaInterval, + release_deliver_quota)}; _ -> - disable_retainer(State) + State2 = disable_retainer(State), + enable_retainer(State2, NewConf) end. -spec enable_retainer(state(), hocon:config()) -> state(). diff --git a/apps/emqx_rule_actions/README.md b/apps/emqx_rule_actions/README.md deleted file mode 100644 index c17e1a34a..000000000 --- a/apps/emqx_rule_actions/README.md +++ /dev/null @@ -1,11 +0,0 @@ -# emqx_rule_actions - -This project contains a collection of rule actions/resources. It is mainly for - making unit test easier. Also it's easier for us to create utils that many - modules depends on it. - -## Build ------ - - $ rebar3 compile - diff --git a/apps/emqx_rule_actions/src/emqx_bridge_mqtt_actions.erl b/apps/emqx_rule_actions/src/emqx_bridge_mqtt_actions.erl deleted file mode 100644 index ce1192579..000000000 --- a/apps/emqx_rule_actions/src/emqx_bridge_mqtt_actions.erl +++ /dev/null @@ -1,576 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - -%% @doc This module implements EMQX Bridge transport layer on top of MQTT protocol - --module(emqx_bridge_mqtt_actions). - --include_lib("emqx/include/emqx.hrl"). --include_lib("emqx/include/logger.hrl"). --include_lib("emqx_rule_engine/include/rule_actions.hrl"). - --import(emqx_plugin_libs_rule, [str/1]). - --export([ on_resource_create/2 - , on_get_resource_status/2 - , on_resource_destroy/2 - ]). - -%% Callbacks of ecpool Worker --export([connect/1]). - --export([subscriptions/1]). - --export([ on_action_create_data_to_mqtt_broker/2 - , on_action_data_to_mqtt_broker/2 - ]). - --define(RESOURCE_TYPE_MQTT, 'bridge_mqtt'). --define(RESOURCE_TYPE_RPC, 'bridge_rpc'). - --define(RESOURCE_CONFIG_SPEC_MQTT, #{ - address => #{ - order => 1, - type => string, - required => true, - default => <<"127.0.0.1:1883">>, - title => #{en => <<" Broker Address">>, - zh => <<"远程 broker 地址"/utf8>>}, - description => #{en => <<"The MQTT Remote Address">>, - zh => <<"远程 MQTT Broker 的地址"/utf8>>} - }, - pool_size => #{ - order => 2, - type => number, - required => true, - default => 8, - title => #{en => <<"Pool Size">>, - zh => <<"连接池大小"/utf8>>}, - description => #{en => <<"MQTT Connection Pool Size">>, - zh => <<"连接池大小"/utf8>>} - }, - clientid => #{ - order => 3, - type => string, - required => true, - default => <<"client">>, - title => #{en => <<"ClientId">>, - zh => <<"客户端 Id"/utf8>>}, - description => #{en => <<"ClientId for connecting to remote MQTT broker">>, - zh => <<"连接远程 Broker 的 ClientId"/utf8>>} - }, - append => #{ - order => 4, - type => boolean, - required => false, - default => true, - title => #{en => <<"Append GUID">>, - zh => <<"附加 GUID"/utf8>>}, - description => #{en => <<"Append GUID to MQTT ClientId?">>, - zh => <<"是否将GUID附加到 MQTT ClientId 后"/utf8>>} - }, - username => #{ - order => 5, - type => string, - required => false, - default => <<"">>, - title => #{en => <<"Username">>, zh => <<"用户名"/utf8>>}, - description => #{en => <<"Username for connecting to remote MQTT Broker">>, - zh => <<"连接远程 Broker 的用户名"/utf8>>} - }, - password => #{ - order => 6, - type => password, - required => false, - default => <<"">>, - title => #{en => <<"Password">>, - zh => <<"密码"/utf8>>}, - description => #{en => <<"Password for connecting to remote MQTT Broker">>, - zh => <<"连接远程 Broker 的密码"/utf8>>} - }, - mountpoint => #{ - order => 7, - type => string, - required => false, - default => <<"bridge/aws/${node}/">>, - title => #{en => <<"Bridge MountPoint">>, - zh => <<"桥接挂载点"/utf8>>}, - description => #{ - en => <<"MountPoint for bridge topic:
" - "Example: The topic of messages sent to `topic1` on local node " - "will be transformed to `bridge/aws/${node}/topic1`">>, - zh => <<"桥接主题的挂载点:
" - "示例: 本地节点向 `topic1` 发消息，远程桥接节点的主题" - "会变换为 `bridge/aws/${node}/topic1`"/utf8>> - } - }, - disk_cache => #{ - order => 8, - type => boolean, - required => false, - default => false, - title => #{en => <<"Disk Cache">>, - zh => <<"磁盘缓存"/utf8>>}, - description => #{en => <<"The flag which determines whether messages " - "can be cached on local disk when bridge is " - "disconnected">>, - zh => <<"当桥接断开时用于控制是否将消息缓存到本地磁" - "盘队列上"/utf8>>} - }, - proto_ver => #{ - order => 9, - type => string, - required => false, - default => <<"mqttv4">>, - enum => [<<"mqttv3">>, <<"mqttv4">>, <<"mqttv5">>], - title => #{en => <<"Protocol Version">>, - zh => <<"协议版本"/utf8>>}, - description => #{en => <<"MQTTT Protocol version">>, - zh => <<"MQTT 协议版本"/utf8>>} - }, - keepalive => #{ - order => 10, - type => string, - required => false, - default => <<"60s">> , - title => #{en => <<"Keepalive">>, - zh => <<"心跳间隔"/utf8>>}, - description => #{en => <<"Keepalive">>, - zh => <<"心跳间隔"/utf8>>} - }, - reconnect_interval => #{ - order => 11, - type => string, - required => false, - default => <<"30s">>, - title => #{en => <<"Reconnect Interval">>, - zh => <<"重连间隔"/utf8>>}, - description => #{en => <<"Reconnect interval of bridge:
">>, - zh => <<"重连间隔"/utf8>>} - }, - retry_interval => #{ - order => 12, - type => string, - required => false, - default => <<"20s">>, - title => #{en => <<"Retry interval">>, - zh => <<"重传间隔"/utf8>>}, - description => #{en => <<"Retry interval for bridge QoS1 message delivering">>, - zh => <<"消息重传间隔"/utf8>>} - }, - bridge_mode => #{ - order => 13, - type => boolean, - required => false, - default => false, - title => #{en => <<"Bridge Mode">>, - zh => <<"桥接模式"/utf8>>}, - description => #{en => <<"Bridge mode for MQTT bridge connection">>, - zh => <<"MQTT 连接是否为桥接模式"/utf8>>} - }, - ssl => #{ - order => 14, - type => boolean, - default => false, - title => #{en => <<"Enable SSL">>, - zh => <<"开启SSL链接"/utf8>>}, - description => #{en => <<"Enable SSL or not">>, - zh => <<"是否开启 SSL"/utf8>>} - }, - cacertfile => #{ - order => 15, - type => file, - required => false, - default => <<"etc/certs/cacert.pem">>, - title => #{en => <<"CA certificates">>, - zh => <<"CA 证书"/utf8>>}, - description => #{en => <<"The file path of the CA certificates">>, - zh => <<"CA 证书路径"/utf8>>} - }, - certfile => #{ - order => 16, - type => file, - required => false, - default => <<"etc/certs/client-cert.pem">>, - title => #{en => <<"SSL Certfile">>, - zh => <<"SSL 客户端证书"/utf8>>}, - description => #{en => <<"The file path of the client certfile">>, - zh => <<"客户端证书路径"/utf8>>} - }, - keyfile => #{ - order => 17, - type => file, - required => false, - default => <<"etc/certs/client-key.pem">>, - title => #{en => <<"SSL Keyfile">>, - zh => <<"SSL 密钥文件"/utf8>>}, - description => #{en => <<"The file path of the client keyfile">>, - zh => <<"客户端密钥路径"/utf8>>} - }, - ciphers => #{ - order => 18, - type => string, - required => false, - default => <<"ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,", - "ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,", - "ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,", - "ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,", - "AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,", - "ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,", - "ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,", - "DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,", - "ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,", - "ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,", - "DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA">>, - title => #{en => <<"SSL Ciphers">>, - zh => <<"SSL 加密算法"/utf8>>}, - description => #{en => <<"SSL Ciphers">>, - zh => <<"SSL 加密算法"/utf8>>} - } - }). - --define(RESOURCE_CONFIG_SPEC_RPC, #{ - address => #{ - order => 1, - type => string, - required => true, - default => <<"emqx2@127.0.0.1">>, - title => #{en => <<"EMQ X Node Name">>, - zh => <<"EMQ X 节点名称"/utf8>>}, - description => #{en => <<"EMQ X Remote Node Name">>, - zh => <<"远程 EMQ X 节点名称 "/utf8>>} - }, - mountpoint => #{ - order => 2, - type => string, - required => false, - default => <<"bridge/emqx/${node}/">>, - title => #{en => <<"Bridge MountPoint">>, - zh => <<"桥接挂载点"/utf8>>}, - description => #{en => <<"MountPoint for bridge topic
" - "Example: The topic of messages sent to `topic1` on local node " - "will be transformed to `bridge/aws/${node}/topic1`">>, - zh => <<"桥接主题的挂载点
" - "示例: 本地节点向 `topic1` 发消息，远程桥接节点的主题" - "会变换为 `bridge/aws/${node}/topic1`"/utf8>>} - }, - pool_size => #{ - order => 3, - type => number, - required => true, - default => 8, - title => #{en => <<"Pool Size">>, - zh => <<"连接池大小"/utf8>>}, - description => #{en => <<"MQTT/RPC Connection Pool Size">>, - zh => <<"连接池大小"/utf8>>} - }, - reconnect_interval => #{ - order => 4, - type => string, - required => false, - default => <<"30s">>, - title => #{en => <<"Reconnect Interval">>, - zh => <<"重连间隔"/utf8>>}, - description => #{en => <<"Reconnect Interval of bridge">>, - zh => <<"重连间隔"/utf8>>} - }, - batch_size => #{ - order => 5, - type => number, - required => false, - default => 32, - title => #{en => <<"Batch Size">>, - zh => <<"批处理大小"/utf8>>}, - description => #{en => <<"Batch Size">>, - zh => <<"批处理大小"/utf8>>} - }, - disk_cache => #{ - order => 6, - type => boolean, - required => false, - default => false, - title => #{en => <<"Disk Cache">>, - zh => <<"磁盘缓存"/utf8>>}, - description => #{en => <<"The flag which determines whether messages " - "can be cached on local disk when bridge is " - "disconnected">>, - zh => <<"当桥接断开时用于控制是否将消息缓存到本地磁" - "盘队列上"/utf8>>} - } - }). - --define(ACTION_PARAM_RESOURCE, #{ - type => string, - required => true, - title => #{en => <<"Resource ID">>, zh => <<"资源 ID"/utf8>>}, - description => #{en => <<"Bind a resource to this action">>, - zh => <<"给动作绑定一个资源"/utf8>>} - }). - --resource_type(#{ - name => ?RESOURCE_TYPE_MQTT, - create => on_resource_create, - status => on_get_resource_status, - destroy => on_resource_destroy, - params => ?RESOURCE_CONFIG_SPEC_MQTT, - title => #{en => <<"MQTT Bridge">>, zh => <<"MQTT Bridge"/utf8>>}, - description => #{en => <<"MQTT Message Bridge">>, zh => <<"MQTT 消息桥接"/utf8>>} - }). - - --resource_type(#{ - name => ?RESOURCE_TYPE_RPC, - create => on_resource_create, - status => on_get_resource_status, - destroy => on_resource_destroy, - params => ?RESOURCE_CONFIG_SPEC_RPC, - title => #{en => <<"EMQX Bridge">>, zh => <<"EMQX Bridge"/utf8>>}, - description => #{en => <<"EMQ X RPC Bridge">>, zh => <<"EMQ X RPC 消息桥接"/utf8>>} - }). - --rule_action(#{ - name => data_to_mqtt_broker, - category => data_forward, - for => 'message.publish', - types => [?RESOURCE_TYPE_MQTT, ?RESOURCE_TYPE_RPC], - create => on_action_create_data_to_mqtt_broker, - params => #{'$resource' => ?ACTION_PARAM_RESOURCE, - forward_topic => #{ - order => 1, - type => string, - required => false, - default => <<"">>, - title => #{en => <<"Forward Topic">>, - zh => <<"转发消息主题"/utf8>>}, - description => #{en => <<"The topic used when forwarding the message. " - "Defaults to the topic of the bridge message if not provided.">>, - zh => <<"转发消息时使用的主题。如果未提供，则默认为桥接消息的主题。"/utf8>>} - }, - payload_tmpl => #{ - order => 2, - type => string, - input => textarea, - required => false, - default => <<"">>, - title => #{en => <<"Payload Template">>, - zh => <<"消息内容模板"/utf8>>}, - description => #{en => <<"The payload template, variable interpolation is supported. " - "If using empty template (default), then the payload will be " - "all the available vars in JSON format">>, - zh => <<"消息内容模板，支持变量。" - "若使用空模板（默认），消息内容为 JSON 格式的所有字段"/utf8>>} - } - }, - title => #{en => <<"Data bridge to MQTT Broker">>, - zh => <<"桥接数据到 MQTT Broker"/utf8>>}, - description => #{en => <<"Bridge Data to MQTT Broker">>, - zh => <<"桥接数据到 MQTT Broker"/utf8>>} - }). - -on_resource_create(ResId, Params) -> - ?LOG(info, "Initiating Resource ~p, ResId: ~p", [?RESOURCE_TYPE_MQTT, ResId]), - {ok, _} = application:ensure_all_started(ecpool), - PoolName = pool_name(ResId), - Options = options(Params, PoolName, ResId), - start_resource(ResId, PoolName, Options), - case test_resource_status(PoolName) of - true -> ok; - false -> - on_resource_destroy(ResId, #{<<"pool">> => PoolName}), - error({{?RESOURCE_TYPE_MQTT, ResId}, connection_failed}) - end, - #{<<"pool">> => PoolName}. - -start_resource(ResId, PoolName, Options) -> - case ecpool:start_sup_pool(PoolName, ?MODULE, Options) of - {ok, _} -> - ?LOG(info, "Initiated Resource ~p Successfully, ResId: ~p", [?RESOURCE_TYPE_MQTT, ResId]); - {error, {already_started, _Pid}} -> - on_resource_destroy(ResId, #{<<"pool">> => PoolName}), - start_resource(ResId, PoolName, Options); - {error, Reason} -> - ?LOG(error, "Initiate Resource ~p failed, ResId: ~p, ~p", [?RESOURCE_TYPE_MQTT, ResId, Reason]), - on_resource_destroy(ResId, #{<<"pool">> => PoolName}), - error({{?RESOURCE_TYPE_MQTT, ResId}, create_failed}) - end. - -test_resource_status(PoolName) -> - IsConnected = fun(Worker) -> - case ecpool_worker:client(Worker) of - {ok, Bridge} -> - try emqx_connector_mqtt_worker:status(Bridge) of - connected -> true; - _ -> false - catch _Error:_Reason -> - false - end; - {error, _} -> - false - end - end, - Status = [IsConnected(Worker) || {_WorkerName, Worker} <- ecpool:workers(PoolName)], - lists:any(fun(St) -> St =:= true end, Status). - --spec(on_get_resource_status(ResId::binary(), Params::map()) -> Status::map()). -on_get_resource_status(_ResId, #{<<"pool">> := PoolName}) -> - IsAlive = test_resource_status(PoolName), - #{is_alive => IsAlive}. - -on_resource_destroy(ResId, #{<<"pool">> := PoolName}) -> - ?LOG(info, "Destroying Resource ~p, ResId: ~p", [?RESOURCE_TYPE_MQTT, ResId]), - case ecpool:stop_sup_pool(PoolName) of - ok -> - ?LOG(info, "Destroyed Resource ~p Successfully, ResId: ~p", [?RESOURCE_TYPE_MQTT, ResId]); - {error, Reason} -> - ?LOG(error, "Destroy Resource ~p failed, ResId: ~p, ~p", [?RESOURCE_TYPE_MQTT, ResId, Reason]), - error({{?RESOURCE_TYPE_MQTT, ResId}, destroy_failed}) - end. - -on_action_create_data_to_mqtt_broker(ActId, Opts = #{<<"pool">> := PoolName, - <<"forward_topic">> := ForwardTopic, - <<"payload_tmpl">> := PayloadTmpl}) -> - ?LOG(info, "Initiating Action ~p.", [?FUNCTION_NAME]), - PayloadTks = emqx_plugin_libs_rule:preproc_tmpl(PayloadTmpl), - TopicTks = case ForwardTopic == <<"">> of - true -> undefined; - false -> emqx_plugin_libs_rule:preproc_tmpl(ForwardTopic) - end, - Opts. - -on_action_data_to_mqtt_broker(Msg, _Env = - #{id := Id, clientid := From, flags := Flags, - topic := Topic, timestamp := TimeStamp, qos := QoS, - ?BINDING_KEYS := #{ - 'ActId' := ActId, - 'PoolName' := PoolName, - 'TopicTks' := TopicTks, - 'PayloadTks' := PayloadTks - }}) -> - Topic1 = case TopicTks =:= undefined of - true -> Topic; - false -> emqx_plugin_libs_rule:proc_tmpl(TopicTks, Msg) - end, - BrokerMsg = #message{id = Id, - qos = QoS, - from = From, - flags = Flags, - topic = Topic1, - payload = format_data(PayloadTks, Msg), - timestamp = TimeStamp}, - ecpool:with_client(PoolName, - fun(BridgePid) -> - BridgePid ! {deliver, rule_engine, BrokerMsg} - end), - emqx_rule_metrics:inc_actions_success(ActId). - -format_data([], Msg) -> - emqx_json:encode(Msg); - -format_data(Tokens, Msg) -> - emqx_plugin_libs_rule:proc_tmpl(Tokens, Msg). - -subscriptions(Subscriptions) -> - scan_binary(<<"[", Subscriptions/binary, "].">>). - -is_node_addr(Addr0) -> - Addr = binary_to_list(Addr0), - case string:tokens(Addr, "@") of - [_NodeName, _Hostname] -> true; - _ -> false - end. - -scan_binary(Bin) -> - TermString = binary_to_list(Bin), - scan_string(TermString). - -scan_string(TermString) -> - {ok, Tokens, _} = erl_scan:string(TermString), - {ok, Term} = erl_parse:parse_term(Tokens), - Term. - -connect(Options) when is_list(Options) -> - connect(maps:from_list(Options)); -connect(Options = #{disk_cache := DiskCache, ecpool_worker_id := Id, pool_name := Pool}) -> - Options0 = case DiskCache of - true -> - DataDir = filename:join([emqx:get_config([node, data_dir]), replayq, Pool, integer_to_list(Id)]), - QueueOption = #{replayq_dir => DataDir}, - Options#{queue => QueueOption}; - false -> - Options - end, - Options1 = case maps:is_key(append, Options0) of - false -> Options0; - true -> - case maps:get(append, Options0, false) of - true -> - ClientId = lists:concat([str(maps:get(clientid, Options0)), "_", str(emqx_guid:to_hexstr(emqx_guid:gen()))]), - Options0#{clientid => ClientId}; - false -> - Options0 - end - end, - Options2 = maps:without([ecpool_worker_id, pool_name, append], Options1), - emqx_connector_mqtt_worker:start_link(Options2#{name => name(Pool, Id)}). -name(Pool, Id) -> - list_to_atom(atom_to_list(Pool) ++ ":" ++ integer_to_list(Id)). -pool_name(ResId) -> - list_to_atom("bridge_mqtt:" ++ str(ResId)). - -options(Options, PoolName, ResId) -> - GetD = fun(Key, Default) -> maps:get(Key, Options, Default) end, - Get = fun(Key) -> GetD(Key, undefined) end, - Address = Get(<<"address">>), - [{max_inflight_batches, 32}, - {forward_mountpoint, str(Get(<<"mountpoint">>))}, - {disk_cache, GetD(<<"disk_cache">>, false)}, - {start_type, auto}, - {reconnect_delay_ms, hocon_postprocess:duration(str(Get(<<"reconnect_interval">>)))}, - {if_record_metrics, false}, - {pool_size, GetD(<<"pool_size">>, 1)}, - {pool_name, PoolName} - ] ++ case is_node_addr(Address) of - true -> - [{address, binary_to_atom(Get(<<"address">>), utf8)}, - {connect_module, emqx_bridge_rpc}, - {batch_size, Get(<<"batch_size">>)}]; - false -> - [{address, binary_to_list(Address)}, - {bridge_mode, GetD(<<"bridge_mode">>, true)}, - {clean_start, true}, - {clientid, str(Get(<<"clientid">>))}, - {append, Get(<<"append">>)}, - {connect_module, emqx_bridge_mqtt}, - {keepalive, hocon_postprocess:duration(str(Get(<<"keepalive">>))) div 1000}, - {username, str(Get(<<"username">>))}, - {password, str(Get(<<"password">>))}, - {proto_ver, mqtt_ver(Get(<<"proto_ver">>))}, - {retry_interval, hocon_postprocess:duration(str(GetD(<<"retry_interval">>, "30s"))) div 1000} - | maybe_ssl(Options, Get(<<"ssl">>), ResId)] - end. - -maybe_ssl(_Options, false, _ResId) -> - []; -maybe_ssl(Options, true, ResId) -> - [{ssl, true}, {ssl_opts, emqx_plugin_libs_ssl:save_files_return_opts(Options, "rules", ResId)}]. - -mqtt_ver(ProtoVer) -> - case ProtoVer of - <<"mqttv3">> -> v3; - <<"mqttv4">> -> v4; - <<"mqttv5">> -> v5; - _ -> v4 - end. diff --git a/apps/emqx_rule_actions/src/emqx_rule_actions.app.src b/apps/emqx_rule_actions/src/emqx_rule_actions.app.src deleted file mode 100644 index 8c2b8d247..000000000 --- a/apps/emqx_rule_actions/src/emqx_rule_actions.app.src +++ /dev/null @@ -1,12 +0,0 @@ -%% -*- mode: erlang -*- -{application, emqx_rule_actions, - [{description, "Rule actions"}, - {vsn, "5.0.0"}, - {registered, []}, - {applications, - [kernel,stdlib,emqx]}, - {env,[]}, - {modules, []}, - {licenses, ["Apache 2.0"]}, - {links, []} - ]}. diff --git a/apps/emqx_rule_actions/src/emqx_web_hook_actions.erl b/apps/emqx_rule_actions/src/emqx_web_hook_actions.erl deleted file mode 100644 index 1b68ad5b0..000000000 --- a/apps/emqx_rule_actions/src/emqx_web_hook_actions.erl +++ /dev/null @@ -1,379 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - -%% Define the default actions. --module(emqx_web_hook_actions). - --export([ on_resource_create/2 - , on_get_resource_status/2 - , on_resource_destroy/2 - ]). - --export([ on_action_create_data_to_webserver/2 - , on_action_data_to_webserver/2 - ]). - --export_type([action_fun/0]). - --include_lib("emqx/include/emqx.hrl"). --include_lib("emqx/include/logger.hrl"). --include_lib("emqx_rule_engine/include/rule_actions.hrl"). - --type(action_fun() :: fun((Data :: map(), Envs :: map()) -> Result :: any())). - --type(url() :: binary()). - --define(RESOURCE_TYPE_WEBHOOK, 'web_hook'). --define(RESOURCE_CONFIG_SPEC, #{ - url => #{order => 1, - type => string, - format => url, - required => true, - title => #{en => <<"Request URL">>, - zh => <<"请求 URL"/utf8>>}, - description => #{en => <<"The URL of the server that will receive the Webhook requests.">>, - zh => <<"用于接收 Webhook 请求的服务器的 URL。"/utf8>>}}, - connect_timeout => #{order => 2, - type => string, - default => <<"5s">>, - title => #{en => <<"Connect Timeout">>, - zh => <<"连接超时时间"/utf8>>}, - description => #{en => <<"Connect Timeout In Seconds">>, - zh => <<"连接超时时间"/utf8>>}}, - request_timeout => #{order => 3, - type => string, - default => <<"5s">>, - title => #{en => <<"Request Timeout">>, - zh => <<"请求超时时间时间"/utf8>>}, - description => #{en => <<"Request Timeout In Seconds">>, - zh => <<"请求超时时间"/utf8>>}}, - pool_size => #{order => 4, - type => number, - default => 8, - title => #{en => <<"Pool Size">>, zh => <<"连接池大小"/utf8>>}, - description => #{en => <<"Connection Pool">>, - zh => <<"连接池大小"/utf8>>} - }, - cacertfile => #{order => 5, - type => file, - default => <<"">>, - title => #{en => <<"CA Certificate File">>, - zh => <<"CA 证书文件"/utf8>>}, - description => #{en => <<"CA Certificate file">>, - zh => <<"CA 证书文件"/utf8>>}}, - keyfile => #{order => 6, - type => file, - default => <<"">>, - title =>#{en => <<"SSL Key">>, - zh => <<"SSL Key"/utf8>>}, - description => #{en => <<"Your ssl keyfile">>, - zh => <<"SSL 私钥"/utf8>>}}, - certfile => #{order => 7, - type => file, - default => <<"">>, - title => #{en => <<"SSL Cert">>, - zh => <<"SSL Cert"/utf8>>}, - description => #{en => <<"Your ssl certfile">>, - zh => <<"SSL 证书"/utf8>>}}, - verify => #{order => 8, - type => boolean, - default => false, - title => #{en => <<"Verify Server Certfile">>, - zh => <<"校验服务器证书"/utf8>>}, - description => #{en => <<"Whether to verify the server certificate. By default, the client will not verify the server's certificate. If verification is required, please set it to true.">>, - zh => <<"是否校验服务器证书。 默认客户端不会去校验服务器的证书，如果需要校验，请设置成true。"/utf8>>}}, - server_name_indication => #{order => 9, - type => string, - title => #{en => <<"Server Name Indication">>, - zh => <<"服务器名称指示"/utf8>>}, - description => #{en => <<"Specify the hostname used for peer certificate verification, or set to disable to turn off this verification.">>, - zh => <<"指定用于对端证书验证时使用的主机名，或者设置为 disable 以关闭此项验证。"/utf8>>}} -}). - --define(ACTION_PARAM_RESOURCE, #{ - order => 0, - type => string, - required => true, - title => #{en => <<"Resource ID">>, - zh => <<"资源 ID"/utf8>>}, - description => #{en => <<"Bind a resource to this action">>, - zh => <<"给动作绑定一个资源"/utf8>>} -}). - --define(ACTION_DATA_SPEC, #{ - '$resource' => ?ACTION_PARAM_RESOURCE, - method => #{ - order => 1, - type => string, - enum => [<<"POST">>, <<"DELETE">>, <<"PUT">>, <<"GET">>], - default => <<"POST">>, - title => #{en => <<"Method">>, - zh => <<"Method"/utf8>>}, - description => #{en => <<"HTTP Method.\n" - "Note that: the Body option in the Action will be discarded in case of GET or DELETE method.">>, - zh => <<"HTTP Method。\n" - "注意：当方法为 GET 或 DELETE 时，动作中的 Body 选项会被忽略。"/utf8>>}}, - path => #{ - order => 2, - type => string, - required => false, - default => <<"">>, - title => #{en => <<"Path">>, - zh => <<"Path"/utf8>>}, - description => #{en => <<"The path part of the URL, support using ${Var} to get the field value output by the rule.">>, - zh => <<"URL 的路径部分，支持使用 ${Var} 获取规则输出的字段值。\n"/utf8>>} - }, - headers => #{ - order => 3, - type => object, - schema => #{}, - default => #{<<"content-type">> => <<"application/json">>}, - title => #{en => <<"Headers">>, - zh => <<"Headers"/utf8>>}, - description => #{en => <<"HTTP headers.">>, - zh => <<"HTTP headers。"/utf8>>}}, - body => #{ - order => 4, - type => string, - input => textarea, - required => false, - default => <<"">>, - title => #{en => <<"Body">>, - zh => <<"Body"/utf8>>}, - description => #{en => <<"The HTTP body supports the use of ${Var} to obtain the field value output by the rule.\n" - "The content of the default HTTP request body is a JSON string composed of the keys and values of all fields output by the rule.">>, - zh => <<"HTTP 请求体，支持使用 ${Var} 获取规则输出的字段值\n" - "默认 HTTP 请求体的内容为规则输出的所有字段的键和值构成的 JSON 字符串。"/utf8>>}} - }). - --resource_type( - #{name => ?RESOURCE_TYPE_WEBHOOK, - create => on_resource_create, - status => on_get_resource_status, - destroy => on_resource_destroy, - params => ?RESOURCE_CONFIG_SPEC, - title => #{en => <<"WebHook">>, - zh => <<"WebHook"/utf8>>}, - description => #{en => <<"WebHook">>, - zh => <<"WebHook"/utf8>>} -}). - --rule_action(#{name => data_to_webserver, - category => data_forward, - for => '$any', - create => on_action_create_data_to_webserver, - params => ?ACTION_DATA_SPEC, - types => [?RESOURCE_TYPE_WEBHOOK], - title => #{en => <<"Data to Web Server">>, - zh => <<"发送数据到 Web 服务"/utf8>>}, - description => #{en => <<"Forward Messages to Web Server">>, - zh => <<"将数据转发给 Web 服务"/utf8>>} -}). - -%%------------------------------------------------------------------------------ -%% Actions for web hook -%%------------------------------------------------------------------------------ - --spec(on_resource_create(binary(), map()) -> map()). -on_resource_create(ResId, Conf) -> - {ok, _} = application:ensure_all_started(ehttpc), - Options = pool_opts(Conf, ResId), - PoolName = pool_name(ResId), - case test_http_connect(Conf) of - true -> ok; - false -> error({error, check_http_connectivity_failed}) - end, - start_resource(ResId, PoolName, Options), - Conf#{<<"pool">> => PoolName, options => Options}. - -start_resource(ResId, PoolName, Options) -> - case ehttpc_pool:start_pool(PoolName, Options) of - {ok, _} -> - ?LOG(info, "Initiated Resource ~p Successfully, ResId: ~p", - [?RESOURCE_TYPE_WEBHOOK, ResId]); - {error, {already_started, _Pid}} -> - on_resource_destroy(ResId, #{<<"pool">> => PoolName}), - start_resource(ResId, PoolName, Options); - {error, Reason} -> - ?LOG(error, "Initiate Resource ~p failed, ResId: ~p, ~0p", - [?RESOURCE_TYPE_WEBHOOK, ResId, Reason]), - error({{?RESOURCE_TYPE_WEBHOOK, ResId}, create_failed}) - end. - --spec(on_get_resource_status(binary(), map()) -> map()). -on_get_resource_status(_ResId, Conf) -> - #{is_alive => test_http_connect(Conf)}. - --spec(on_resource_destroy(binary(), map()) -> ok | {error, Reason::term()}). -on_resource_destroy(ResId, #{<<"pool">> := PoolName}) -> - ?LOG(info, "Destroying Resource ~p, ResId: ~p", [?RESOURCE_TYPE_WEBHOOK, ResId]), - case ehttpc_pool:stop_pool(PoolName) of - ok -> - ?LOG(info, "Destroyed Resource ~p Successfully, ResId: ~p", [?RESOURCE_TYPE_WEBHOOK, ResId]); - {error, Reason} -> - ?LOG(error, "Destroy Resource ~p failed, ResId: ~p, ~p", [?RESOURCE_TYPE_WEBHOOK, ResId, Reason]), - error({{?RESOURCE_TYPE_WEBHOOK, ResId}, destroy_failed}) - end. - -%% An action that forwards publish messages to a remote web server. --spec(on_action_create_data_to_webserver(Id::binary(), #{url() := string()}) -> {bindings(), NewParams :: map()}). -on_action_create_data_to_webserver(Id, Params) -> - #{method := Method, - path := Path, - headers := Headers, - body := Body, - pool := Pool, - request_timeout := RequestTimeout} = parse_action_params(Params), - BodyTokens = emqx_plugin_libs_rule:preproc_tmpl(Body), - PathTokens = emqx_plugin_libs_rule:preproc_tmpl(Path), - Params. - -on_action_data_to_webserver(Selected, _Envs = - #{?BINDING_KEYS := #{ - 'Id' := Id, - 'Method' := Method, - 'Headers' := Headers, - 'PathTokens' := PathTokens, - 'BodyTokens' := BodyTokens, - 'Pool' := Pool, - 'RequestTimeout' := RequestTimeout}, - clientid := ClientID}) -> - NBody = format_msg(BodyTokens, Selected), - NPath = emqx_plugin_libs_rule:proc_tmpl(PathTokens, Selected), - Req = create_req(Method, NPath, Headers, NBody), - case ehttpc:request(ehttpc_pool:pick_worker(Pool, ClientID), Method, Req, RequestTimeout) of - {ok, StatusCode, _} when StatusCode >= 200 andalso StatusCode < 300 -> - emqx_rule_metrics:inc_actions_success(Id); - {ok, StatusCode, _, _} when StatusCode >= 200 andalso StatusCode < 300 -> - emqx_rule_metrics:inc_actions_success(Id); - {ok, StatusCode, _} -> - ?LOG(warning, "[WebHook Action] HTTP request failed with status code: ~p", [StatusCode]), - emqx_rule_metrics:inc_actions_error(Id); - {ok, StatusCode, _, _} -> - ?LOG(warning, "[WebHook Action] HTTP request failed with status code: ~p", [StatusCode]), - emqx_rule_metrics:inc_actions_error(Id); - {error, Reason} -> - ?LOG(error, "[WebHook Action] HTTP request error: ~p", [Reason]), - emqx_rule_metrics:inc_actions_error(Id) - end. - -format_msg([], Data) -> - emqx_json:encode(Data); -format_msg(Tokens, Data) -> - emqx_plugin_libs_rule:proc_tmpl(Tokens, Data). - -%%------------------------------------------------------------------------------ -%% Internal functions -%%------------------------------------------------------------------------------ - -create_req(Method, Path, Headers, _Body) - when Method =:= get orelse Method =:= delete -> - {Path, Headers}; -create_req(_, Path, Headers, Body) -> - {Path, Headers, Body}. - -parse_action_params(Params = #{<<"url">> := URL}) -> - try - {ok, #{path := CommonPath}} = emqx_http_lib:uri_parse(URL), - Method = method(maps:get(<<"method">>, Params, <<"POST">>)), - Headers = headers(maps:get(<<"headers">>, Params, undefined)), - NHeaders = ensure_content_type_header(Headers, Method), - #{method => Method, - path => merge_path(CommonPath, maps:get(<<"path">>, Params, <<>>)), - headers => NHeaders, - body => maps:get(<<"body">>, Params, <<>>), - request_timeout => hocon_postprocess:duration(str(maps:get(<<"request_timeout">>, Params, <<"5s">>))), - pool => maps:get(<<"pool">>, Params)} - catch _:_ -> - throw({invalid_params, Params}) - end. - -ensure_content_type_header(Headers, Method) when Method =:= post orelse Method =:= put -> - Headers; -ensure_content_type_header(Headers, _Method) -> - lists:keydelete("content-type", 1, Headers). - -merge_path(CommonPath, <<>>) -> - CommonPath; -merge_path(CommonPath, Path0) -> - case emqx_http_lib:uri_parse(Path0) of - {ok, #{path := Path1, 'query' := Query}} -> - Path2 = filename:join(CommonPath, Path1), - <>; - {ok, #{path := Path1}} -> - filename:join(CommonPath, Path1) - end. - -method(GET) when GET == <<"GET">>; GET == <<"get">> -> get; -method(POST) when POST == <<"POST">>; POST == <<"post">> -> post; -method(PUT) when PUT == <<"PUT">>; PUT == <<"put">> -> put; -method(DEL) when DEL == <<"DELETE">>; DEL == <<"delete">> -> delete. - -headers(undefined) -> []; -headers(Headers) when is_map(Headers) -> - headers(maps:to_list(Headers)); -headers(Headers) when is_list(Headers) -> - [{string:to_lower(str(K)), str(V)} || {K, V} <- Headers]. - -str(Str) when is_list(Str) -> Str; -str(Atom) when is_atom(Atom) -> atom_to_list(Atom); -str(Bin) when is_binary(Bin) -> binary_to_list(Bin). - -pool_opts(Params = #{<<"url">> := URL}, ResId) -> - {ok, #{host := Host, - port := Port, - scheme := Scheme}} = emqx_http_lib:uri_parse(URL), - PoolSize = maps:get(<<"pool_size">>, Params, 32), - ConnectTimeout = - hocon_postprocess:duration(str(maps:get(<<"connect_timeout">>, Params, <<"5s">>))), - TransportOpts0 = - case Scheme =:= https of - true -> [get_ssl_opts(Params, ResId)]; - false -> [] - end, - TransportOpts = emqx_misc:ipv6_probe(TransportOpts0), - Opts = case Scheme =:= https of - true -> [{transport_opts, TransportOpts}, {transport, ssl}]; - false -> [{transport_opts, TransportOpts}] - end, - [{host, Host}, - {port, Port}, - {pool_size, PoolSize}, - {pool_type, hash}, - {connect_timeout, ConnectTimeout}, - {retry, 5}, - {retry_timeout, 1000} | Opts]. - -pool_name(ResId) -> - list_to_atom("webhook:" ++ str(ResId)). - -get_ssl_opts(Opts, ResId) -> - emqx_plugin_libs_ssl:save_files_return_opts(Opts, "rules", ResId). - -test_http_connect(Conf) -> - Url = fun() -> maps:get(<<"url">>, Conf) end, - try - emqx_plugin_libs_rule:http_connectivity(Url()) - of - ok -> true; - {error, _Reason} -> - ?LOG(error, "check http_connectivity failed: ~p", [Url()]), - false - catch - Err:Reason:ST -> - ?LOG(error, "check http_connectivity failed: ~p, ~0p", [Conf, {Err, Reason, ST}]), - false - end. diff --git a/apps/emqx_rule_engine/docs/api_examples.md b/apps/emqx_rule_engine/docs/api_examples.md deleted file mode 100644 index f546a3a57..000000000 --- a/apps/emqx_rule_engine/docs/api_examples.md +++ /dev/null @@ -1,197 +0,0 @@ -#Rule-Engine-APIs - -## ENVs - -APPSECRET="88ebdd6569afc:Mjg3MzUyNTI2Mjk2NTcyOTEwMDEwMDMzMTE2NTM1MTkzNjA" - -## Rules - -### test sql - -$ curl -v --basic -u $APPSECRET -k 'http://localhost:8081/api/v4/rules?test' -d \ -'{"rawsql":"select * from \"message.publish\" where topic=\"t/a\"","ctx":{}}' - - - -### create - -```shell -$ curl -v --basic -u $APPSECRET -k 'http://localhost:8081/api/v4/rules' -d \ -'{"rawsql":"select * from \"t/a\"","actions":[{"name":"inspect","params":{"a":1}}],"description":"test-rule"}' - -{"code":0,"data":{"actions":[{"name":"inspect","params":{"a":1}}],"description":"test-rule","enabled":true,"id":"rule:bc987915","rawsql":"select * from \"t/a\""}} - -## with a resource id in the action args -$ curl -v --basic -u $APPSECRET -k 'http://localhost:8081/api/v4/rules' -d \ -'{"rawsql":"select * from \"t/a\"","actions":[{"name":"inspect","params":{"$resource":"resource:3a7b44a1"}}],"description":"test-rule"}' - -{"code":0,"data":{"actions":[{"name":"inspect","params":{"$resource":"resource:3a7b44a1","a":1}}],"description":"test-rule","enabled":true,"id":"rule:6fce0ca9","rawsql":"select * from \"t/a\""}} -``` - -### modify - -```shell -## modify all of the params -$ curl -XPUT -v --basic -u $APPSECRET -k 'http://localhost:8081/api/v4/rules/rule:bc987915' -d \ -'{"rawsql":"select * from \"t/a\"","actions":[{"name":"inspect","params":{"a":1}}],"description":"test-rule"}' - -## modify some of the params: disable it -$ curl -XPUT -v --basic -u $APPSECRET -k 'http://localhost:8081/api/v4/rules/rule:bc987915' -d \ -'{"enabled": false}' - -## modify some of the params: add fallback actions -$ curl -XPUT -v --basic -u $APPSECRET -k 'http://localhost:8081/api/v4/rules/rule:bc987915' -d \ -'{"actions":[{"name":"inspect","params":{"a":1}, "fallbacks": [{"name":"donothing"}]}]}' -``` - -### show - -```shell -$ curl -v --basic -u $APPSECRET -k 'http://localhost:8081/api/v4/rules/rule:bc987915' - -{"code":0,"data":{"actions":[{"name":"inspect","params":{"a":1}}],"description":"test-rule","enabled":true,"id":"rule:bc987915","rawsql":"select * from \"t/a\""}} -``` - -### list - -```shell -$ curl -v --basic -u $APPSECRET -k http://localhost:8081/api/v4/rules - -{"code":0,"data":[{"actions":[{"name":"inspect","params":{"a":1}}],"description":"test-rule","enabled":true,"id":"rule:bc987915","rawsql":"select * from \"t/a\""},{"actions":[{"name":"inspect","params":{"$resource":"resource:3a7b44a1","a":1}}],"description":"test-rule","enabled":true,"id":"rule:6fce0ca9","rawsql":"select * from \"t/a\""}]} -``` - -### delete - -```shell -$ curl -XDELETE -v --basic -u $APPSECRET -k 'http://localhost:8081/api/v4/rules/rule:bc987915' - -{"code":0} -``` - -## Actions - -### list - -```shell -$ curl -v --basic -u $APPSECRET -k http://localhost:8081/api/v4/actions - -{"code":0,"data":[{"app":"emqx_rule_engine","description":"Republish a MQTT message to a another topic","name":"republish","params":{...},"types":[]},{"app":"emqx_rule_engine","description":"Inspect the details of action params for debug purpose","name":"inspect","params":{},"types":[]},{"app":"emqx_web_hook","description":"Forward Messages to Web Server","name":"data_to_webserver","params":{...},"types":["web_hook"]}]} -``` - -### show - -```shell -$ curl -v --basic -u $APPSECRET -k 'http://localhost:8081/api/v4/actions/inspect' - -{"code":0,"data":{"app":"emqx_rule_engine","description":"Debug Action","name":"inspect","params":{"$resource":"built_in"}}} -``` - -## Resource Types - -### list - -```shell -$ curl -v --basic -u $APPSECRET -k 'http://localhost:8081/api/v4/resource_types' - -{"code":0,"data":[{"description":"Debug resource type","name":"built_in","params":{},"provider":"emqx_rule_engine"}]} -``` - -### list all resources of a type - -```shell -$ curl -v --basic -u $APPSECRET -k 'http://localhost:8081/api/v4/resource_types/built_in/resources' - -{"code":0,"data":[{"attrs":"undefined","config":{"a":1},"description":"test-rule","id":"resource:71df3086","type":"built_in"}]} -``` - -### show - -```shell -$ curl -v --basic -u $APPSECRET -k 'http://localhost:8081/api/v4/resource_types/built_in' - -{"code":0,"data":{"description":"Debug resource type","name":"built_in","params":{},"provider":"emqx_rule_engine"}} -``` - - - -## Resources - -### create - -```shell -$ curl -v --basic -u $APPSECRET -k 'http://localhost:8081/api/v4/resources' -d \ -'{"type": "built_in", "config": {"a":1}, "description": "test-resource"}' - -{"code":0,"data":{"attrs":"undefined","config":{"a":1},"description":"test-resource","id":"resource:71df3086","type":"built_in"}} -``` - -### start - -```shell -$ curl -XPOST -v --basic -u $APPSECRET -k 'http://localhost:8081/api/v4/resources/resource:71df3086' - -{"code":0} -``` - -### list - -```shell -$ curl -v --basic -u $APPSECRET -k 'http://localhost:8081/api/v4/resources' - -{"code":0,"data":[{"attrs":"undefined","config":{"a":1},"description":"test-resource","id":"resource:71df3086","type":"built_in"}]} -``` - -### show - -```shell -$ curl -v --basic -u $APPSECRET -k 'http://localhost:8081/api/v4/resources/resource:71df3086' - -{"code":0,"data":{"attrs":"undefined","config":{"a":1},"description":"test-resource","id":"resource:71df3086","type":"built_in"}} -``` - -### get resource status - -```shell -curl -v --basic -u $APPSECRET -k 'http://localhost:8081/api/v4/resource_status/resource:71df3086' - -{"code":0,"data":{"is_alive":true}} -``` - -### delete - -```shell -$ curl -XDELETE -v --basic -u $APPSECRET -k 'http://localhost:8081/api/v4/resources/resource:71df3086' - -{"code":0} -``` - -## Rule example using webhook - -``` shell - -$ curl -v --basic -u $APPSECRET -k 'http://localhost:8081/api/v4/resources' -d \ -'{"type": "web_hook", "config": {"url": "http://127.0.0.1:9910", "headers": {"token":"axfw34y235wrq234t4ersgw4t"}, "method": "POST"}, "description": "web hook resource-1"}' - -{"code":0,"data":{"attrs":"undefined","config":{"headers":{"token":"axfw34y235wrq234t4ersgw4t"},"method":"POST","url":"http://127.0.0.1:9910"},"description":"web hook resource-1","id":"resource:8531a11f","type":"web_hook"}} - -curl -v --basic -u $APPSECRET -k 'http://localhost:8081/api/v4/rules' -d \ -'{"rawsql":"SELECT clientid as c, username as u.name FROM \"#\"","actions":[{"name":"data_to_webserver","params":{"$resource": "resource:8531a11f"}}],"description":"Forward connected events to webhook"}' - -{"code":0,"data":{"actions":[{"name":"data_to_webserver","params":{"$resource":"resource:8531a11f","headers":{"token":"axfw34y235wrq234t4ersgw4t"},"method":"POST","url":"http://127.0.0.1:9910"}}],"description":"Forward connected events to webhook","enabled":true,"id":"rule:4fe05936","rawsql":"select * from \"#\""}} -``` - -Start a `web server` using `nc`, and then connect to emqx broker using a mqtt client with username = 'Shawn': - -```shell -$ echo -e "HTTP/1.1 200 OK\n\n $(date)" | nc -l 127.0.0.1 9910 - -POST / HTTP/1.1 -content-type: application/json -content-length: 48 -te: -host: 127.0.0.1:9910 -connection: keep-alive -token: axfw34y235wrq234t4ersgw4t - -{"c":"clientId-bP70ymeIyo","u":{"name":"Shawn"} -``` diff --git a/apps/emqx_rule_engine/docs/cli_examples.md b/apps/emqx_rule_engine/docs/cli_examples.md deleted file mode 100644 index 3d854129c..000000000 --- a/apps/emqx_rule_engine/docs/cli_examples.md +++ /dev/null @@ -1,164 +0,0 @@ -#Rule-Engine-CLIs - -## Rules - -### create - -```shell - $ ./bin/emqx_ctl rules create 'SELECT payload FROM "t/#" username="Steven"' '[{"name":"data_to_webserver", "params": {"$resource": "resource:9093f1cb"}}]' --descr="Msg From Steven to WebServer" - -Rule rule:98a75239 created -``` - -### modify - - -```shell - ## update sql, action, description - $ ./bin/emqx_ctl rules update 'rule:98a75239' \ - -s "select * from \"t/a\" " \ - -a '[{"name":"do_nothing", "fallbacks": []' -g continue \ - -d 'Rule for debug2' \ - - ## update sql only - $ ./bin/emqx_ctl rules update 'rule:98a75239' -s 'SELECT * FROM "t/a"' - - ## disable the rule - $ ./bin/emqx_ctl rules update 'rule:98a75239' -e false - -``` - -### show - -```shell -$ ./bin/emqx_ctl rules show rule:98a75239 - -rule(id='rule:98a75239', rawsql='SELECT payload FROM "t/#" username="Steven"', actions=[{"name":"data_to_webserver","params":{"$resource":"resource:9093f1cb","url":"http://host-name/chats"}}], enabled='true', description='Msg From Steven to WebServer') -``` - -### list - -```shell -$ ./bin/emqx_ctl rules list - -rule(id='rule:98a75239', rawsql='SELECT payload FROM "t/#" username="Steven"', actions=[{"name":"data_to_webserver","params":{"$resource":"resource:9093f1cb","url":"http://host-name/chats"}}], enabled='true', description='Msg From Steven to WebServer') - -``` - -### delete - -```shell -$ ./bin/emqx_ctl rules delete 'rule:98a75239' - -ok -``` - -## Actions - -### list - -```shell -$ ./bin/emqx_ctl rule-actions list - -action(name='republish', app='emqx_rule_engine', types=[], params=#{...}, description='Republish a MQTT message to a another topic') -action(name='inspect', app='emqx_rule_engine', types=[], params=#{...}, description='Inspect the details of action params for debug purpose') -action(name='data_to_webserver', app='emqx_web_hook', types=[], params=#{...}, description='Forward Messages to Web Server') -``` - -### show - -```shell -$ ./bin/emqx_ctl rule-actions show 'data_to_webserver' - -action(name='data_to_webserver', app='emqx_web_hook', types=['web_hook'], params=#{...}, description='Forward Messages to Web Server') -``` - -## Resource - -### create - -```shell -$ ./bin/emqx_ctl resources create 'web_hook' -c '{"url": "http://host-name/chats"}' --descr 'Resource towards http://host-name/chats' - -Resource resource:19addfef created -``` - -### list - -```shell -$ ./bin/emqx_ctl resources list - -resource(id='resource:19addfef', type='web_hook', config=#{<<"url">> => <<"http://host-name/chats">>}, attrs=undefined, description='Resource towards http://host-name/chats') - -``` - -### list all resources of a type - -```shell -$ ./bin/emqx_ctl resources list -t 'web_hook' - -resource(id='resource:19addfef', type='web_hook', config=#{<<"url">> => <<"http://host-name/chats">>}, attrs=undefined, description='Resource towards http://host-name/chats') -``` - -### show - -```shell -$ ./bin/emqx_ctl resources show 'resource:19addfef' - -resource(id='resource:19addfef', type='web_hook', config=#{<<"url">> => <<"http://host-name/chats">>}, attrs=undefined, description='Resource towards http://host-name/chats') -``` - -### delete - -```shell -$ ./bin/emqx_ctl resources delete 'resource:19addfef' - -ok -``` - -## Resources Types - -### list - -```shell -$ ./bin/emqx_ctl resource-types list - -resource_type(name='built_in', provider='emqx_rule_engine', params=#{...}, on_create={emqx_rule_actions,on_resource_create}, description='The built in resource type for debug purpose') -resource_type(name='web_hook', provider='emqx_web_hook', params=#{...}, on_create={emqx_web_hook_actions,on_resource_create}, description='WebHook Resource') -``` - -### show - -```shell -$ ./bin/emqx_ctl resource-types show built_in - -resource_type(name='built_in', provider='emqx_rule_engine', params=#{}, description='The built in resource type for debug purpose') -``` - -## Rule example using webhook - -``` shell -1. Create a webhook resource to URL http://127.0.0.1:9910. -./bin/emqx_ctl resources create 'web_hook' --config '{"url": "http://127.0.0.1:9910", "headers": {"token":"axfw34y235wrq234t4ersgw4t"}, "method": "POST"}' -Resource resource:3128243e created - -2. Create a rule using action data_to_webserver, and bind above resource to that action. -./bin/emqx_ctl rules create 'client.connected' 'SELECT clientid as c, username as u.name FROM "#"' '[{"name":"data_to_webserver", "params": {"$resource": "resource:3128243e"}}]' --descr "Forward Connected Events to WebServer" -Rule rule:222b59f7 created -``` - -Start a simple `Web Server` using `nc`, and then connect to emqx broker using a mqtt client with username = 'Shawn': - -```shell -$ echo -e "HTTP/1.1 200 OK\n\n $(date)" | nc -l 127.0.0.1 9910 - -POST / HTTP/1.1 -content-type: application/json -content-length: 48 -te: -host: 127.0.0.1:9910 -connection: keep-alive -token: axfw34y235wrq234t4ersgw4t - -{"c":"clientId-bP70ymeIyo","u":{"name":"Shawn"} -``` diff --git a/apps/emqx_rule_engine/docs/design.md b/apps/emqx_rule_engine/docs/design.md deleted file mode 100644 index 3e2c60c41..000000000 --- a/apps/emqx_rule_engine/docs/design.md +++ /dev/null @@ -1,188 +0,0 @@ - -# EMQ X Rule Engine - -This is the design guide of message routing rule engine for the EMQ X Broker. - -## Concept - -A rule is: - -``` -when - Match | -then - Select and Take ; -``` - -or: - -``` -rule "Rule Name" - when - rule match - select - para1 = val1 - para2 = val2 - then - action(#{para2 => val1, #para2 => val2}) -``` - -## Architecture - -``` - |-----------------| - P ---->| Message Routing |----> S - |-----------------| - | /|\ - \|/ | - |-----------------| - | Rule Engine | - |-----------------| - | | - Backends Services Bridges -``` - -## Design - -``` -Event | Message -> Rules -> Actions -> Resources -``` - -``` - P -> |--------------------| |---------------------------------------| - | Messages (Routing) | -> | Rules (Select Data, Match Conditions) | - S <- |--------------------| |---------------------------------------| - |---------| |-----------| |-------------------------------| - ->| Actions | -> | Resources | -> | (Backends, Bridges, WebHooks) | - |---------| |-----------| |-------------------------------| -``` - - - -## Rule - -A rule consists of a SELECT statement, a topic filter, and a rule action - -Rules consist of the following: - -- Id -- Name -- Topic -- Description -- Action -- Enabled - -The operations on a rule: - -- Create -- Enable -- Disable -- Delete - - - -## Action - -Actions consist of the following: - -- Id -- Name -- For -- App -- Module -- Func -- Args -- Descr - -Define a rule action in ADT: - -``` -action :: Application -> Resource -> Params -> IO () -``` - -A rule action: - -Module:function(Args) - - - -## Resource - -### Resource Name - -``` -backend:mysql:localhost:port:db -backend:redis:localhost: -webhook:url -bridge:kafka: -bridge:rabbit:localhost -``` - -### Resource Properties - -- Name -- Descr or Description -- Config #{} -- Instances -- State: Running | Stopped - -### Resource Management - -1. Create Resource -2. List Resources -3. Lookup Resource -4. Delete Resource -5. Test Resource - -### Resource State (Lifecircle) - -0. Create Resource and Validate a Resource -1. Start/Connect Resource -2. Bind resource name to instance -3. Stop/Disconnect Resource -4. Unbind resource name with instance -5. Is Resource Alive? - -### Resource Type - -The properties and behaviors of resources is defined by resource types. A resoure type is provided(contributed) by a plugin. - -### Resource Type Provider - -Provider of resource type is a EMQ X Plugin. - -### Resource Manager - -``` - Supervisor - | - \|/ -Action ----> Proxy(Batch|Write) ----> Connection -----> ExternalResource - | /|\ - |------------------Fetch----------------| -``` - - - -## REST API - -Rules API -Actions API -Resources API - -## CLI - -``` -rules list -rules show - -rule-actions list -rule-actions show - -resources list -resources show - -resource_templates list -resource_templates show -``` - diff --git a/apps/emqx_rule_engine/etc/emqx_rule_engine.conf b/apps/emqx_rule_engine/etc/emqx_rule_engine.conf index 22543a977..a4344cda8 100644 --- a/apps/emqx_rule_engine/etc/emqx_rule_engine.conf +++ b/apps/emqx_rule_engine/etc/emqx_rule_engine.conf @@ -1,6 +1,6 @@ ##==================================================================== ## Rule Engine for EMQ X R5.0 ##==================================================================== -emqx_rule_engine { +rule_engine { ignore_sys_message = true } diff --git a/apps/emqx_rule_engine/include/rule_actions.hrl b/apps/emqx_rule_engine/include/rule_actions.hrl deleted file mode 100644 index e432c4399..000000000 --- a/apps/emqx_rule_engine/include/rule_actions.hrl +++ /dev/null @@ -1,11 +0,0 @@ --compile({parse_transform, emqx_rule_actions_trans}). - --type selected_data() :: map(). --type env_vars() :: map(). --type bindings() :: list({atom(), term()}). - --define(BINDING_KEYS, '__bindings__'). - --define(bound_v(Key, ENVS0), - maps:get(Key, - maps:get(?BINDING_KEYS, ENVS0, #{}))). diff --git a/apps/emqx_rule_engine/include/rule_engine.hrl b/apps/emqx_rule_engine/include/rule_engine.hrl index 760495f6b..c230aa8c3 100644 --- a/apps/emqx_rule_engine/include/rule_engine.hrl +++ b/apps/emqx_rule_engine/include/rule_engine.hrl @@ -18,109 +18,46 @@ -define(KV_TAB, '@rule_engine_db'). --type(maybe(T) :: T | undefined). +-type maybe(T) :: T | undefined. --type(rule_id() :: binary()). --type(rule_name() :: binary()). +-type rule_id() :: binary(). +-type rule_name() :: binary(). --type(resource_id() :: binary()). --type(action_instance_id() :: binary()). +-type mf() :: {Module::atom(), Fun::atom()}. --type(action_name() :: atom()). --type(resource_type_name() :: atom()). +-type hook() :: atom() | 'any'. --type(category() :: data_persist| data_forward | offline_msgs | debug | other). +-type topic() :: binary(). +-type bridge_channel_id() :: binary(). +-type selected_data() :: map(). +-type envs() :: map(). +-type output_type() :: bridge | builtin | func. +-type output_target() :: bridge_channel_id() | atom() | output_fun(). +-type output_fun_args() :: map(). +-type output() :: #{ + type := output_type(), + target := output_target(), + args => output_fun_args() +}. +-type output_fun() :: fun((selected_data(), envs(), output_fun_args()) -> any()). --type(descr() :: #{en := binary(), zh => binary()}). - --type(mf() :: {Module::atom(), Fun::atom()}). - --type(hook() :: atom() | 'any'). - --type(topic() :: binary()). - --type(resource_status() :: #{ alive := boolean() - , atom() => binary() | atom() | list(binary()|atom()) - }). - --define(descr, #{en => <<>>, zh => <<>>}). - --record(action, - { name :: action_name() - , category :: category() - , for :: hook() - , app :: atom() - , types = [] :: list(resource_type_name()) - , module :: module() - , on_create :: mf() - , on_destroy :: maybe(mf()) - , hidden = false :: boolean() - , params_spec :: #{atom() => term()} %% params specs - , title = ?descr :: descr() - , description = ?descr :: descr() - }). - --record(action_instance, - { id :: action_instance_id() - , name :: action_name() - , fallbacks :: list(#action_instance{}) - , args :: #{binary() => term()} %% the args got from API for initializing action_instance - }). +-type rule_info() :: + #{ from := list(topic()) + , outputs := [output()] + , sql := binary() + , is_foreach := boolean() + , fields := list() + , doeach := term() + , incase := term() + , conditions := tuple() + , enabled := boolean() + , description => binary() + }. -record(rule, { id :: rule_id() - , for :: list(topic()) - , rawsql :: binary() - , is_foreach :: boolean() - , fields :: list() - , doeach :: term() - , incase :: list() - , conditions :: tuple() - , on_action_failed :: continue | stop - , actions :: list(#action_instance{}) - , enabled :: boolean() , created_at :: integer() %% epoch in millisecond precision - , description :: binary() - , state = normal :: atom() - }). - --record(resource, - { id :: resource_id() - , type :: resource_type_name() - , config :: #{} %% the configs got from API for initializing resource - , created_at :: integer() | undefined %% epoch in millisecond precision - , description :: binary() - }). - --record(resource_type, - { name :: resource_type_name() - , provider :: atom() - , params_spec :: #{atom() => term()} %% params specs - , on_create :: mf() - , on_status :: mf() - , on_destroy :: mf() - , title = ?descr :: descr() - , description = ?descr :: descr() - }). - --record(rule_hooks, - { hook :: atom() - , rule_id :: rule_id() - }). - --record(resource_params, - { id :: resource_id() - , params :: #{} %% the params got after initializing the resource - , status = #{is_alive => false} :: #{is_alive := boolean(), atom() => term()} - }). - --record(action_instance_params, - { id :: action_instance_id() - %% the params got after initializing the action - , params :: #{} - %% the Func/Bindings got after initializing the action - , apply :: fun((Data::map(), Envs::map()) -> any()) - | #{mod := module(), bindings := #{atom() => term()}} + , info :: rule_info() }). %% Arithmetic operators @@ -157,9 +94,5 @@ %% Tables -define(RULE_TAB, emqx_rule). --define(ACTION_TAB, emqx_rule_action). --define(ACTION_INST_PARAMS_TAB, emqx_action_instance_params). --define(RES_TAB, emqx_resource). --define(RES_PARAMS_TAB, emqx_resource_params). --define(RULE_HOOKS, emqx_rule_hooks). --define(RES_TYPE_TAB, emqx_resource_type). + +-define(RULE_ENGINE_SHARD, emqx_rule_engine_shard). diff --git a/apps/emqx_rule_engine/src/emqx_rule_actions.erl b/apps/emqx_rule_engine/src/emqx_rule_actions.erl deleted file mode 100644 index 7ac45633c..000000000 --- a/apps/emqx_rule_engine/src/emqx_rule_actions.erl +++ /dev/null @@ -1,208 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - -%% Define the default actions. --module(emqx_rule_actions). - --include("rule_engine.hrl"). --include("rule_actions.hrl"). --include_lib("emqx/include/emqx.hrl"). --include_lib("emqx/include/logger.hrl"). - --define(REPUBLISH_PARAMS_SPEC, #{ - target_topic => #{ - order => 1, - type => string, - required => true, - default => <<"repub/to/${clientid}">>, - title => #{en => <<"Target Topic">>, - zh => <<"目的主题"/utf8>>}, - description => #{en => <<"To which topic the message will be republished">>, - zh => <<"重新发布消息到哪个主题"/utf8>>} - }, - target_qos => #{ - order => 2, - type => number, - enum => [-1, 0, 1, 2], - required => true, - default => 0, - title => #{en => <<"Target QoS">>, - zh => <<"目的 QoS"/utf8>>}, - description => #{en => <<"The QoS Level to be uses when republishing the message. Set to -1 to use the original QoS">>, - zh => <<"重新发布消息时用的 QoS 级别, 设置为 -1 以使用原消息中的 QoS"/utf8>>} - }, - payload_tmpl => #{ - order => 3, - type => string, - input => textarea, - required => false, - default => <<"${payload}">>, - title => #{en => <<"Payload Template">>, - zh => <<"消息内容模板"/utf8>>}, - description => #{en => <<"The payload template, variable interpolation is supported">>, - zh => <<"消息内容模板，支持变量"/utf8>>} - } - }). - --rule_action(#{name => inspect, - category => debug, - for => '$any', - types => [], - create => on_action_create_inspect, - params => #{}, - title => #{en => <<"Inspect (debug)">>, - zh => <<"检查 (调试)"/utf8>>}, - description => #{en => <<"Inspect the details of action params for debug purpose">>, - zh => <<"检查动作参数 (用以调试)"/utf8>>} - }). - --rule_action(#{name => republish, - category => data_forward, - for => '$any', - types => [], - create => on_action_create_republish, - params => ?REPUBLISH_PARAMS_SPEC, - title => #{en => <<"Republish">>, - zh => <<"消息重新发布"/utf8>>}, - description => #{en => <<"Republish a MQTT message to another topic">>, - zh => <<"重新发布消息到另一个主题"/utf8>>} - }). - --rule_action(#{name => do_nothing, - category => debug, - for => '$any', - types => [], - create => on_action_create_do_nothing, - params => #{}, - title => #{en => <<"Do Nothing (debug)">>, - zh => <<"空动作 (调试)"/utf8>>}, - description => #{en => <<"This action does nothing and never fails. It's for debug purpose">>, - zh => <<"此动作什么都不做，并且不会失败 (用以调试)"/utf8>>} - }). - --export([on_resource_create/2]). - -%% callbacks for rule engine --export([ on_action_create_inspect/2 - , on_action_create_republish/2 - , on_action_create_do_nothing/2 - ]). - --export([ on_action_inspect/2 - , on_action_republish/2 - , on_action_do_nothing/2 - ]). - --spec(on_resource_create(binary(), map()) -> map()). -on_resource_create(_Name, Conf) -> - Conf. - -%%------------------------------------------------------------------------------ -%% Action 'inspect' -%%------------------------------------------------------------------------------ --spec on_action_create_inspect(Id :: action_instance_id(), Params :: map()) -> {bindings(), NewParams :: map()}. -on_action_create_inspect(Id, Params) -> - Params. - --spec on_action_inspect(selected_data(), env_vars()) -> any(). -on_action_inspect(Selected, Envs) -> - ?ULOG("[inspect]~n" - "\tSelected Data: ~p~n" - "\tEnvs: ~p~n" - "\tAction Init Params: ~p~n", [Selected, Envs, ?bound_v('Params', Envs)]), - emqx_rule_metrics:inc_actions_success(?bound_v('Id', Envs)). - - -%%------------------------------------------------------------------------------ -%% Action 'republish' -%%------------------------------------------------------------------------------ --spec on_action_create_republish(action_instance_id(), Params :: map()) -> {bindings(), NewParams :: map()}. -on_action_create_republish(Id, Params = #{ - <<"target_topic">> := TargetTopic, - <<"target_qos">> := TargetQoS, - <<"payload_tmpl">> := PayloadTmpl - }) -> - TopicTks = emqx_plugin_libs_rule:preproc_tmpl(TargetTopic), - PayloadTks = emqx_plugin_libs_rule:preproc_tmpl(PayloadTmpl), - Params. - --spec on_action_republish(selected_data(), env_vars()) -> any(). -on_action_republish(_Selected, Envs = #{ - topic := Topic, - headers := #{republish_by := ActId}, - ?BINDING_KEYS := #{'Id' := ActId} - }) -> - ?LOG(error, "[republish] recursively republish detected, msg topic: ~p, target topic: ~p", - [Topic, ?bound_v('TargetTopic', Envs)]), - emqx_rule_metrics:inc_actions_error(?bound_v('Id', Envs)); - -on_action_republish(Selected, _Envs = #{ - qos := QoS, flags := Flags, timestamp := Timestamp, - ?BINDING_KEYS := #{ - 'Id' := ActId, - 'TargetTopic' := TargetTopic, - 'TargetQoS' := TargetQoS, - 'TopicTks' := TopicTks, - 'PayloadTks' := PayloadTks - }}) -> - ?LOG(debug, "[republish] republish to: ~p, Payload: ~p", - [TargetTopic, Selected]), - increase_and_publish(ActId, - #message{ - id = emqx_guid:gen(), - qos = if TargetQoS =:= -1 -> QoS; true -> TargetQoS end, - from = ActId, - flags = Flags, - headers = #{republish_by => ActId}, - topic = emqx_plugin_libs_rule:proc_tmpl(TopicTks, Selected), - payload = emqx_plugin_libs_rule:proc_tmpl(PayloadTks, Selected), - timestamp = Timestamp - }); - -%% in case this is not a "message.publish" request -on_action_republish(Selected, _Envs = #{ - ?BINDING_KEYS := #{ - 'Id' := ActId, - 'TargetTopic' := TargetTopic, - 'TargetQoS' := TargetQoS, - 'TopicTks' := TopicTks, - 'PayloadTks' := PayloadTks - }}) -> - ?LOG(debug, "[republish] republish to: ~p, Payload: ~p", - [TargetTopic, Selected]), - increase_and_publish(ActId, - #message{ - id = emqx_guid:gen(), - qos = if TargetQoS =:= -1 -> 0; true -> TargetQoS end, - from = ActId, - flags = #{dup => false, retain => false}, - headers = #{republish_by => ActId}, - topic = emqx_plugin_libs_rule:proc_tmpl(TopicTks, Selected), - payload = emqx_plugin_libs_rule:proc_tmpl(PayloadTks, Selected), - timestamp = erlang:system_time(millisecond) - }). - -increase_and_publish(ActId, Msg) -> - _ = emqx_broker:safe_publish(Msg), - emqx_rule_metrics:inc_actions_success(ActId), - emqx_metrics:inc_msg(Msg). - --spec on_action_create_do_nothing(action_instance_id(), Params :: map()) -> {bindings(), NewParams :: map()}. -on_action_create_do_nothing(ActId, Params) when is_binary(ActId) -> - Params. - -on_action_do_nothing(Selected, Envs) when is_map(Selected) -> - emqx_rule_metrics:inc_actions_success(?bound_v('ActId', Envs)). diff --git a/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl new file mode 100644 index 000000000..c96e82ecb --- /dev/null +++ b/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl @@ -0,0 +1,188 @@ +-module(emqx_rule_api_schema). + +-behaviour(hocon_schema). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("emqx/include/logger.hrl"). + +-export([ check_params/2 + ]). + +-export([roots/0, fields/1]). + +-type tag() :: rule_creation | rule_test. + +-spec check_params(map(), tag()) -> {ok, map()} | {error, term()}. +check_params(Params, Tag) -> + BTag = atom_to_binary(Tag), + try hocon_schema:check_plain(?MODULE, #{BTag => Params}, + #{atom_key => true, nullable => true}, [BTag]) of + #{Tag := Checked} -> {ok, Checked} + catch + Error:Reason:ST -> + ?SLOG(error, #{msg => "check_rule_params_failed", + exception => Error, + reason => Reason, + stacktrace => ST}), + {error, {Reason, ST}} + end. + +%%====================================================================================== +%% Hocon Schema Definitions + +roots() -> + [ {"rule_creation", sc(ref("rule_creation"), #{desc => "Schema for creating rules"})} + , {"rule_test", sc(ref("rule_test"), #{desc => "Schema for testing rules"})} + ]. + +fields("rule_creation") -> + [ {"id", sc(binary(), #{desc => "The Id of the rule", nullable => false})} + , {"sql", sc(binary(), #{desc => "The SQL of the rule", nullable => false})} + , {"outputs", sc(hoconsc:array(hoconsc:union( + [ ref("bridge_output") + , ref("builtin_output") + ])), + #{desc => "The outputs of the rule", + default => []})} + , {"enable", sc(boolean(), #{desc => "Enable or disable the rule", default => true})} + , {"description", sc(binary(), #{desc => "The description of the rule", default => <<>>})} + ]; + +fields("rule_test") -> + [ {"context", sc(hoconsc:union([ ref("ctx_pub") + , ref("ctx_sub") + , ref("ctx_delivered") + , ref("ctx_acked") + , ref("ctx_dropped") + , ref("ctx_connected") + , ref("ctx_disconnected") + ]), + #{desc => "The context of the event for testing", + default => #{}})} + , {"sql", sc(binary(), #{desc => "The SQL of the rule for testing", nullable => false})} + ]; + +fields("bridge_output") -> + [ {type, bridge} + , {target, sc(binary(), #{desc => "The Channel ID of the bridge"})} + ]; + +fields("builtin_output") -> + [ {type, builtin} + , {target, sc(binary(), #{desc => "The Name of the built-on output"})} + , {args, sc(map(), #{desc => "The arguments of the built-in output", + default => #{}})} + ]; + +%% TODO: how to use this in "builtin_output".args ? +fields("republish_args") -> + [ {topic, sc(binary(), + #{desc => "The target topic of the re-published message." + " Template with with variables is allowed.", + nullable => false})} + , {qos, sc(binary(), + #{desc => "The qos of the re-published message." + " Template with with variables is allowed. Defaults to ${qos}.", + default => <<"${qos}">> })} + , {retain, sc(binary(), + #{desc => "The retain of the re-published message." + " Template with with variables is allowed. Defaults to ${retain}.", + default => <<"${retain}">> })} + , {payload, sc(binary(), + #{desc => "The payload of the re-published message." + " Template with with variables is allowed. Defaults to ${payload}.", + default => <<"${payload}">>})} + ]; + +fields("ctx_pub") -> + [ {"event_type", sc(message_publish, #{desc => "Event Type", nullable => false})} + , {"id", sc(binary(), #{desc => "Message ID"})} + , {"clientid", sc(binary(), #{desc => "The Client ID"})} + , {"username", sc(binary(), #{desc => "The User Name"})} + , {"payload", sc(binary(), #{desc => "The Message Payload"})} + , {"peerhost", sc(binary(), #{desc => "The IP Address of the Peer Client"})} + , {"topic", sc(binary(), #{desc => "Message Topic"})} + , {"publish_received_at", sc(integer(), #{ + desc => "The Time that this Message is Received"})} + ] ++ [qos()]; + +fields("ctx_sub") -> + [ {"event_type", sc(session_subscribed, #{desc => "Event Type", nullable => false})} + , {"clientid", sc(binary(), #{desc => "The Client ID"})} + , {"username", sc(binary(), #{desc => "The User Name"})} + , {"payload", sc(binary(), #{desc => "The Message Payload"})} + , {"peerhost", sc(binary(), #{desc => "The IP Address of the Peer Client"})} + , {"topic", sc(binary(), #{desc => "Message Topic"})} + , {"publish_received_at", sc(integer(), #{ + desc => "The Time that this Message is Received"})} + ] ++ [qos()]; + +fields("ctx_unsub") -> + [{"event_type", sc(session_unsubscribed, #{desc => "Event Type", nullable => false})}] ++ + proplists:delete("event_type", fields("ctx_sub")); + +fields("ctx_delivered") -> + [ {"event_type", sc(message_delivered, #{desc => "Event Type", nullable => false})} + , {"id", sc(binary(), #{desc => "Message ID"})} + , {"from_clientid", sc(binary(), #{desc => "The Client ID"})} + , {"from_username", sc(binary(), #{desc => "The User Name"})} + , {"clientid", sc(binary(), #{desc => "The Client ID"})} + , {"username", sc(binary(), #{desc => "The User Name"})} + , {"payload", sc(binary(), #{desc => "The Message Payload"})} + , {"peerhost", sc(binary(), #{desc => "The IP Address of the Peer Client"})} + , {"topic", sc(binary(), #{desc => "Message Topic"})} + , {"publish_received_at", sc(integer(), #{ + desc => "The Time that this Message is Received"})} + ] ++ [qos()]; + +fields("ctx_acked") -> + [{"event_type", sc(message_acked, #{desc => "Event Type", nullable => false})}] ++ + proplists:delete("event_type", fields("ctx_delivered")); + +fields("ctx_dropped") -> + [ {"event_type", sc(message_dropped, #{desc => "Event Type", nullable => false})} + , {"id", sc(binary(), #{desc => "Message ID"})} + , {"reason", sc(binary(), #{desc => "The Reason for Dropping"})} + , {"clientid", sc(binary(), #{desc => "The Client ID"})} + , {"username", sc(binary(), #{desc => "The User Name"})} + , {"payload", sc(binary(), #{desc => "The Message Payload"})} + , {"peerhost", sc(binary(), #{desc => "The IP Address of the Peer Client"})} + , {"topic", sc(binary(), #{desc => "Message Topic"})} + , {"publish_received_at", sc(integer(), #{ + desc => "The Time that this Message is Received"})} + ] ++ [qos()]; + +fields("ctx_connected") -> + [ {"event_type", sc(client_connected, #{desc => "Event Type", nullable => false})} + , {"clientid", sc(binary(), #{desc => "The Client ID"})} + , {"username", sc(binary(), #{desc => "The User Name"})} + , {"mountpoint", sc(binary(), #{desc => "The Mountpoint"})} + , {"peername", sc(binary(), #{desc => "The IP Address and Port of the Peer Client"})} + , {"sockname", sc(binary(), #{desc => "The IP Address and Port of the Local Listener"})} + , {"proto_name", sc(binary(), #{desc => "Protocol Name"})} + , {"proto_ver", sc(binary(), #{desc => "Protocol Version"})} + , {"keepalive", sc(integer(), #{desc => "KeepAlive"})} + , {"clean_start", sc(boolean(), #{desc => "Clean Start", default => true})} + , {"expiry_interval", sc(integer(), #{desc => "Expiry Interval"})} + , {"is_bridge", sc(boolean(), #{desc => "Is Bridge", default => false})} + , {"connected_at", sc(integer(), #{ + desc => "The Time that this Client is Connected"})} + ]; + +fields("ctx_disconnected") -> + [ {"event_type", sc(client_disconnected, #{desc => "Event Type", nullable => false})} + , {"clientid", sc(binary(), #{desc => "The Client ID"})} + , {"username", sc(binary(), #{desc => "The User Name"})} + , {"reason", sc(binary(), #{desc => "The Reason for Disconnect"})} + , {"peername", sc(binary(), #{desc => "The IP Address and Port of the Peer Client"})} + , {"sockname", sc(binary(), #{desc => "The IP Address and Port of the Local Listener"})} + , {"disconnected_at", sc(integer(), #{ + desc => "The Time that this Client is Disconnected"})} + ]. + +qos() -> + {"qos", sc(hoconsc:union([typerefl:integer(0), typerefl:integer(1), typerefl:integer(2)]), + #{desc => "The Message QoS"})}. + +sc(Type, Meta) -> hoconsc:mk(Type, Meta). +ref(Field) -> hoconsc:ref(?MODULE, Field). diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.erl b/apps/emqx_rule_engine/src/emqx_rule_engine.erl index bf0eb06e8..04d35931a 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.erl @@ -19,627 +19,102 @@ -include("rule_engine.hrl"). -include_lib("emqx/include/logger.hrl"). --export([ load_providers/0 - , unload_providers/0 - , refresh_resources/0 - , refresh_resource/1 - , refresh_rule/1 - , refresh_rules/0 - , refresh_actions/1 - , refresh_actions/2 - , refresh_resource_status/0 - ]). - -export([ create_rule/1 , update_rule/1 , delete_rule/1 - , create_resource/1 - , test_resource/1 - , start_resource/1 - , get_resource_status/1 - , get_resource_params/1 - , delete_resource/1 - , update_resource/2 ]). --export([ init_resource/4 - , init_action/4 - , clear_resource/3 - , clear_rule/1 - , clear_actions/1 - , clear_action/3 - ]). +-export_type([rule/0]). --type(rule() :: #rule{}). --type(action() :: #action{}). --type(resource() :: #resource{}). --type(resource_type() :: #resource_type{}). --type(resource_params() :: #resource_params{}). --type(action_instance_params() :: #action_instance_params{}). - --export_type([ rule/0 - , action/0 - , resource/0 - , resource_type/0 - , resource_params/0 - , action_instance_params/0 - ]). +-type rule() :: #rule{}. -define(T_RETRY, 60000). -%%------------------------------------------------------------------------------ -%% Load resource/action providers from all available applications -%%------------------------------------------------------------------------------ - -%% Load all providers . --spec(load_providers() -> ok). -load_providers() -> - lists:foreach(fun(App) -> - load_provider(App) - end, ignore_lib_apps(application:loaded_applications())). - --spec(load_provider(App :: atom()) -> ok). -load_provider(App) when is_atom(App) -> - ok = load_actions(App), - ok = load_resource_types(App). - -%%------------------------------------------------------------------------------ -%% Unload providers -%%------------------------------------------------------------------------------ -%% Load all providers . --spec(unload_providers() -> ok). -unload_providers() -> - lists:foreach(fun(App) -> - unload_provider(App) - end, ignore_lib_apps(application:loaded_applications())). - -%% @doc Unload a provider. --spec(unload_provider(App :: atom()) -> ok). -unload_provider(App) -> - ok = emqx_rule_registry:remove_actions_of(App), - ok = emqx_rule_registry:unregister_resource_types_of(App). - -load_actions(App) -> - Actions = find_actions(App), - emqx_rule_registry:add_actions(Actions). - -load_resource_types(App) -> - ResourceTypes = find_resource_types(App), - emqx_rule_registry:register_resource_types(ResourceTypes). - --spec(find_actions(App :: atom()) -> list(action())). -find_actions(App) -> - lists:map(fun new_action/1, find_attrs(App, rule_action)). - --spec(find_resource_types(App :: atom()) -> list(resource_type())). -find_resource_types(App) -> - lists:map(fun new_resource_type/1, find_attrs(App, resource_type)). - -new_action({App, Mod, #{name := Name, - for := Hook, - types := Types, - create := Create, - params := ParamsSpec} = Params}) -> - ok = emqx_rule_validator:validate_spec(ParamsSpec), - #action{name = Name, for = Hook, app = App, types = Types, - category = maps:get(category, Params, other), - module = Mod, on_create = Create, - hidden = maps:get(hidden, Params, false), - on_destroy = maps:get(destroy, Params, undefined), - params_spec = ParamsSpec, - title = maps:get(title, Params, ?descr), - description = maps:get(description, Params, ?descr)}. - -new_resource_type({App, Mod, #{name := Name, - params := ParamsSpec, - create := Create} = Params}) -> - ok = emqx_rule_validator:validate_spec(ParamsSpec), - #resource_type{name = Name, provider = App, - params_spec = ParamsSpec, - on_create = {Mod, Create}, - on_status = {Mod, maps:get(status, Params, undefined)}, - on_destroy = {Mod, maps:get(destroy, Params, undefined)}, - title = maps:get(title, Params, ?descr), - description = maps:get(description, Params, ?descr)}. - -find_attrs(App, Def) -> - [{App, Mod, Attr} || {ok, Modules} <- [application:get_key(App, modules)], - Mod <- Modules, - {Name, Attrs} <- module_attributes(Mod), Name =:= Def, - Attr <- Attrs]. - -module_attributes(Module) -> - try Module:module_info(attributes) - catch - error:undef -> [] - end. - %%------------------------------------------------------------------------------ %% APIs for rules and resources %%------------------------------------------------------------------------------ --dialyzer([{nowarn_function, [create_rule/1, rule_id/0]}]). -spec create_rule(map()) -> {ok, rule()} | {error, term()}. -create_rule(Params = #{rawsql := Sql, actions := ActArgs}) -> - case emqx_rule_sqlparser:parse_select(Sql) of - {ok, Select} -> - RuleId = maps:get(id, Params, rule_id()), - Enabled = maps:get(enabled, Params, true), - try prepare_actions(ActArgs, Enabled) of - Actions -> - Rule = #rule{ - id = RuleId, - rawsql = Sql, - for = emqx_rule_sqlparser:select_from(Select), - is_foreach = emqx_rule_sqlparser:select_is_foreach(Select), - fields = emqx_rule_sqlparser:select_fields(Select), - doeach = emqx_rule_sqlparser:select_doeach(Select), - incase = emqx_rule_sqlparser:select_incase(Select), - conditions = emqx_rule_sqlparser:select_where(Select), - on_action_failed = maps:get(on_action_failed, Params, continue), - actions = Actions, - enabled = Enabled, - created_at = erlang:system_time(millisecond), - description = maps:get(description, Params, ""), - state = normal - }, - ok = emqx_rule_registry:add_rule(Rule), - ok = emqx_rule_metrics:create_rule_metrics(RuleId), - {ok, Rule} - catch - throw:{action_not_found, ActionName} -> - {error, {action_not_found, ActionName}}; - throw:Reason -> - {error, Reason} - end; - Reason -> {error, Reason} - end. - --spec(update_rule(#{id := binary(), _=>_}) -> {ok, rule()} | {error, {not_found, rule_id()}}). -update_rule(Params = #{id := RuleId}) -> +create_rule(Params = #{id := RuleId}) -> case emqx_rule_registry:get_rule(RuleId) of - {ok, Rule0} -> - try may_update_rule_params(Rule0, Params) of - Rule -> - ok = emqx_rule_registry:add_rule(Rule), - {ok, Rule} - catch - throw:Reason -> - {error, Reason} - end; - not_found -> - {error, {not_found, RuleId}} + not_found -> do_create_rule(Params); + {ok, _} -> {error, {already_exists, RuleId}} end. --spec(delete_rule(RuleId :: rule_id()) -> ok). +-spec update_rule(map()) -> {ok, rule()} | {error, term()}. +update_rule(Params = #{id := RuleId}) -> + case delete_rule(RuleId) of + ok -> do_create_rule(Params); + Error -> Error + end. + +-spec(delete_rule(RuleId :: rule_id()) -> ok | {error, term()}). delete_rule(RuleId) -> case emqx_rule_registry:get_rule(RuleId) of - {ok, Rule = #rule{actions = Actions}} -> - try - _ = emqx_plugin_libs_rule:cluster_call(?MODULE, clear_rule, [Rule]), - ok = emqx_rule_registry:remove_rule(Rule) - catch - Error:Reason:ST -> - ?LOG(error, "clear_rule ~p failed: ~p", [RuleId, {Error, Reason, ST}]), - refresh_actions(Actions) - end; - not_found -> - ok - end. - --spec(create_resource(#{type := _, config := _, _ => _}) -> {ok, resource()} | {error, Reason :: term()}). -create_resource(#{type := Type, config := Config0} = Params) -> - case emqx_rule_registry:find_resource_type(Type) of - {ok, #resource_type{on_create = {M, F}, params_spec = ParamSpec}} -> - Config = emqx_rule_validator:validate_params(Config0, ParamSpec), - ResId = maps:get(id, Params, resource_id()), - Resource = #resource{id = ResId, - type = Type, - config = Config, - description = iolist_to_binary(maps:get(description, Params, "")), - created_at = erlang:system_time(millisecond) - }, - ok = emqx_rule_registry:add_resource(Resource), - %% Note that we will return OK in case of resource creation failure, - %% A timer is started to re-start the resource later. - catch _ = emqx_plugin_libs_rule:cluster_call(?MODULE, init_resource, [M, F, ResId, Config]), - {ok, Resource}; - not_found -> - {error, {resource_type_not_found, Type}} - end. - --spec(update_resource(resource_id(), map()) -> ok | {error, Reason :: term()}). -update_resource(ResId, NewParams) -> - case emqx_rule_registry:find_enabled_rules_depends_on_resource(ResId) of - [] -> check_and_update_resource(ResId, NewParams); - Rules -> - {error, {dependent_rules_exists, [Id || #rule{id = Id} <- Rules]}} - end. - -check_and_update_resource(Id, NewParams) -> - case emqx_rule_registry:find_resource(Id) of - {ok, #resource{id = Id, type = Type, config = OldConfig, description = OldDescr}} -> - try - Conifg = maps:get(<<"config">>, NewParams, OldConfig), - Descr = maps:get(<<"description">>, NewParams, OldDescr), - do_check_and_update_resource(#{id => Id, config => Conifg, type => Type, - description => Descr}) - catch Error:Reason:ST -> - ?LOG(error, "check_and_update_resource failed: ~0p", [{Error, Reason, ST}]), - {error, Reason} - end; - _Other -> - {error, not_found} - end. - -do_check_and_update_resource(#{id := Id, type := Type, description := NewDescription, - config := NewConfig}) -> - case emqx_rule_registry:find_resource_type(Type) of - {ok, #resource_type{on_create = {Module, Create}, - params_spec = ParamSpec}} -> - Config = emqx_rule_validator:validate_params(NewConfig, ParamSpec), - case test_resource(#{type => Type, config => NewConfig}) of - ok -> - _ = emqx_plugin_libs_rule:cluster_call(?MODULE, init_resource, [Module, Create, Id, Config]), - emqx_rule_registry:add_resource(#resource{ - id = Id, - type = Type, - config = Config, - description = NewDescription, - created_at = erlang:system_time(millisecond) - }), - ok; - {error, Reason} -> - error({error, Reason}) - end - end. - --spec(start_resource(resource_id()) -> ok | {error, Reason :: term()}). -start_resource(ResId) -> - case emqx_rule_registry:find_resource(ResId) of - {ok, #resource{type = ResType, config = Config}} -> - {ok, #resource_type{on_create = {Mod, Create}}} - = emqx_rule_registry:find_resource_type(ResType), - try - init_resource(Mod, Create, ResId, Config), - refresh_actions_of_a_resource(ResId) - catch - throw:Reason -> {error, Reason} - end; - not_found -> - {error, {resource_not_found, ResId}} - end. - --spec(test_resource(#{type := _, config := _, _ => _}) -> ok | {error, Reason :: term()}). -test_resource(#{type := Type, config := Config0}) -> - case emqx_rule_registry:find_resource_type(Type) of - {ok, #resource_type{on_create = {ModC, Create}, - on_destroy = {ModD, Destroy}, - params_spec = ParamSpec}} -> - Config = emqx_rule_validator:validate_params(Config0, ParamSpec), - ResId = resource_id(), - try - _ = emqx_plugin_libs_rule:cluster_call(?MODULE, init_resource, [ModC, Create, ResId, Config]), - _ = emqx_plugin_libs_rule:cluster_call(?MODULE, clear_resource, [ModD, Destroy, ResId]), - ok - catch - throw:Reason -> {error, Reason} - end; - not_found -> - {error, {resource_type_not_found, Type}} - end. - --spec(get_resource_status(resource_id()) -> {ok, resource_status()} | {error, Reason :: term()}). -get_resource_status(ResId) -> - case emqx_rule_registry:find_resource(ResId) of - {ok, #resource{type = ResType}} -> - {ok, #resource_type{on_status = {Mod, OnStatus}}} - = emqx_rule_registry:find_resource_type(ResType), - Status = fetch_resource_status(Mod, OnStatus, ResId), - {ok, Status}; - not_found -> - {error, {resource_not_found, ResId}} - end. - --spec(get_resource_params(resource_id()) -> {ok, map()} | {error, Reason :: term()}). -get_resource_params(ResId) -> - case emqx_rule_registry:find_resource_params(ResId) of - {ok, #resource_params{params = Params}} -> - {ok, Params}; - not_found -> - {error, resource_not_initialized} - end. - --spec(delete_resource(resource_id()) -> ok | {error, Reason :: term()}). -delete_resource(ResId) -> - case emqx_rule_registry:find_resource(ResId) of - {ok, #resource{type = ResType}} -> - {ok, #resource_type{on_destroy = {ModD, Destroy}}} - = emqx_rule_registry:find_resource_type(ResType), - try - case emqx_rule_registry:remove_resource(ResId) of - ok -> - _ = emqx_plugin_libs_rule:cluster_call(?MODULE, clear_resource, [ModD, Destroy, ResId]), - ok; - {error, _} = R -> R - end - catch - throw:Reason -> {error, Reason} - end; + {ok, Rule} -> + ok = emqx_rule_registry:remove_rule(Rule), + _ = emqx_plugin_libs_rule:cluster_call(emqx_rule_metrics, clear_rule_metrics, [RuleId]), + ok; not_found -> {error, not_found} end. -%%------------------------------------------------------------------------------ -%% Re-establish resources -%%------------------------------------------------------------------------------ - --spec(refresh_resources() -> ok). -refresh_resources() -> - lists:foreach(fun refresh_resource/1, - emqx_rule_registry:get_resources()). - -refresh_resource(Type) when is_atom(Type) -> - lists:foreach(fun refresh_resource/1, - emqx_rule_registry:get_resources_by_type(Type)); - -refresh_resource(#resource{id = ResId}) -> - emqx_rule_monitor:ensure_resource_retrier(ResId, ?T_RETRY). - --spec(refresh_rules() -> ok). -refresh_rules() -> - lists:foreach(fun - (#rule{enabled = true} = Rule) -> - try refresh_rule(Rule) - catch _:_ -> - emqx_rule_registry:add_rule(Rule#rule{enabled = false, state = refresh_failed_at_bootup}) - end; - (_) -> ok - end, emqx_rule_registry:get_rules()). - -refresh_rule(#rule{id = RuleId, for = Topics, actions = Actions}) -> - ok = emqx_rule_metrics:create_rule_metrics(RuleId), - lists:foreach(fun emqx_rule_events:load/1, Topics), - refresh_actions(Actions). - --spec(refresh_resource_status() -> ok). -refresh_resource_status() -> - lists:foreach( - fun(#resource{id = ResId, type = ResType}) -> - case emqx_rule_registry:find_resource_type(ResType) of - {ok, #resource_type{on_status = {Mod, OnStatus}}} -> - _ = fetch_resource_status(Mod, OnStatus, ResId); - _ -> ok - end - end, emqx_rule_registry:get_resources()). - %%------------------------------------------------------------------------------ %% Internal Functions %%------------------------------------------------------------------------------ -prepare_actions(Actions, NeedInit) -> - [prepare_action(Action, NeedInit) || Action <- Actions]. -prepare_action(#{name := Name, args := Args0} = Action, NeedInit) -> - case emqx_rule_registry:find_action(Name) of - {ok, #action{module = Mod, on_create = Create, params_spec = ParamSpec}} -> - Args = emqx_rule_validator:validate_params(Args0, ParamSpec), - ActionInstId = maps:get(id, Action, action_instance_id(Name)), - case NeedInit of - true -> - _ = emqx_plugin_libs_rule:cluster_call(?MODULE, init_action, [Mod, Create, ActionInstId, - with_resource_params(Args)]), - ok; - false -> ok - end, - #action_instance{ - id = ActionInstId, name = Name, args = Args, - fallbacks = prepare_actions(maps:get(fallbacks, Action, []), NeedInit) - }; - not_found -> - throw({action_not_found, Name}) - end. - -with_resource_params(Args = #{<<"$resource">> := ResId}) -> - case emqx_rule_registry:find_resource_params(ResId) of - {ok, #resource_params{params = Params}} -> - maps:merge(Args, Params); - not_found -> - throw({resource_not_initialized, ResId}) - end; -with_resource_params(Args) -> Args. - --dialyzer([{nowarn_function, may_update_rule_params/2}]). -may_update_rule_params(Rule, Params = #{rawsql := SQL}) -> - case emqx_rule_sqlparser:parse_select(SQL) of +do_create_rule(Params = #{id := RuleId, sql := Sql, outputs := Outputs}) -> + case emqx_rule_sqlparser:parse(Sql) of {ok, Select} -> - may_update_rule_params( - Rule#rule{ - rawsql = SQL, - for = emqx_rule_sqlparser:select_from(Select), - is_foreach = emqx_rule_sqlparser:select_is_foreach(Select), - fields = emqx_rule_sqlparser:select_fields(Select), - doeach = emqx_rule_sqlparser:select_doeach(Select), - incase = emqx_rule_sqlparser:select_incase(Select), - conditions = emqx_rule_sqlparser:select_where(Select) - }, - maps:remove(rawsql, Params)); - Reason -> throw(Reason) - end; -may_update_rule_params(Rule = #rule{enabled = OldEnb, actions = Actions, state = OldState}, - Params = #{enabled := NewEnb}) -> - State = case {OldEnb, NewEnb} of - {false, true} -> - refresh_rule(Rule), - force_changed; - {true, false} -> - clear_actions(Actions), - force_changed; - _NoChange -> OldState - end, - may_update_rule_params(Rule#rule{enabled = NewEnb, state = State}, maps:remove(enabled, Params)); -may_update_rule_params(Rule, Params = #{description := Descr}) -> - may_update_rule_params(Rule#rule{description = Descr}, maps:remove(description, Params)); -may_update_rule_params(Rule, Params = #{on_action_failed := OnFailed}) -> - may_update_rule_params(Rule#rule{on_action_failed = OnFailed}, - maps:remove(on_action_failed, Params)); -may_update_rule_params(Rule = #rule{actions = OldActions}, Params = #{actions := Actions}) -> - %% prepare new actions before removing old ones - NewActions = prepare_actions(Actions, maps:get(enabled, Params, true)), - _ = emqx_plugin_libs_rule:cluster_call(?MODULE, clear_actions, [OldActions]), - may_update_rule_params(Rule#rule{actions = NewActions}, maps:remove(actions, Params)); -may_update_rule_params(Rule, _Params) -> %% ignore all the unsupported params - Rule. - -ignore_lib_apps(Apps) -> - LibApps = [kernel, stdlib, sasl, appmon, eldap, erts, - syntax_tools, ssl, crypto, mnesia, os_mon, - inets, goldrush, gproc, runtime_tools, - snmp, otp_mibs, public_key, asn1, ssh, hipe, - common_test, observer, webtool, xmerl, tools, - test_server, compiler, debugger, eunit, et, - wx], - [AppName || {AppName, _, _} <- Apps, not lists:member(AppName, LibApps)]. - -resource_id() -> - gen_id("resource:", fun emqx_rule_registry:find_resource/1). - -rule_id() -> - gen_id("rule:", fun emqx_rule_registry:get_rule/1). - -gen_id(Prefix, TestFun) -> - Id = iolist_to_binary([Prefix, emqx_misc:gen_id()]), - case TestFun(Id) of - not_found -> Id; - _Res -> gen_id(Prefix, TestFun) + Rule = #rule{ + id = RuleId, + created_at = erlang:system_time(millisecond), + info = #{ + enabled => maps:get(enabled, Params, true), + sql => Sql, + from => emqx_rule_sqlparser:select_from(Select), + outputs => parse_outputs(Outputs), + description => maps:get(description, Params, ""), + %% -- calculated fields: + is_foreach => emqx_rule_sqlparser:select_is_foreach(Select), + fields => emqx_rule_sqlparser:select_fields(Select), + doeach => emqx_rule_sqlparser:select_doeach(Select), + incase => emqx_rule_sqlparser:select_incase(Select), + conditions => emqx_rule_sqlparser:select_where(Select) + %% -- calculated fields end + } + }, + ok = emqx_rule_registry:add_rule(Rule), + _ = emqx_plugin_libs_rule:cluster_call(emqx_rule_metrics, create_rule_metrics, [RuleId]), + {ok, Rule}; + {error, Reason} -> {error, Reason} end. -action_instance_id(ActionName) -> - iolist_to_binary([atom_to_list(ActionName), "_", integer_to_list(erlang:system_time())]). +parse_outputs(Outputs) -> + [do_parse_outputs(Out) || Out <- Outputs]. -init_resource(Module, OnCreate, ResId, Config) -> - Params = ?RAISE(Module:OnCreate(ResId, Config), - {{Module, OnCreate}, {_EXCLASS_, _EXCPTION_, _ST_}}), - ResParams = #resource_params{id = ResId, - params = Params, - status = #{is_alive => true}}, - emqx_rule_registry:add_resource_params(ResParams). +do_parse_outputs(#{type := bridge, target := ChId}) -> + #{type => bridge, target => ChId}; +do_parse_outputs(#{type := builtin, target := Repub, args := Args}) + when Repub == republish; Repub == <<"republish">> -> + #{type => builtin, target => republish, args => pre_process_repub_args(Args)}; +do_parse_outputs(#{type := Type, target := Name} = Output) + when Type == func; Type == builtin -> + #{type => Type, target => Name, args => maps:get(args, Output, #{})}. -init_action(Module, OnCreate, ActionInstId, Params) -> - ok = emqx_rule_metrics:create_metrics(ActionInstId), - case ?RAISE(Module:OnCreate(ActionInstId, Params), - {{init_action_failure, node()}, - {{Module, OnCreate}, {_EXCLASS_, _EXCPTION_, _ST_}}}) of - {Apply, NewParams} when is_function(Apply) -> %% BACKW: =< e4.2.2 - ok = emqx_rule_registry:add_action_instance_params( - #action_instance_params{id = ActionInstId, params = NewParams, apply = Apply}); - {Bindings, NewParams} when is_list(Bindings) -> - ok = emqx_rule_registry:add_action_instance_params( - #action_instance_params{ - id = ActionInstId, params = NewParams, - apply = #{mod => Module, bindings => maps:from_list(Bindings)}}); - Apply when is_function(Apply) -> %% BACKW: =< e4.2.2 - ok = emqx_rule_registry:add_action_instance_params( - #action_instance_params{id = ActionInstId, params = Params, apply = Apply}) - end. +pre_process_repub_args(#{<<"topic">> := Topic} = Args) -> + QoS = maps:get(<<"qos">>, Args, <<"${qos}">>), + Retain = maps:get(<<"retain">>, Args, <<"${retain}">>), + Payload = maps:get(<<"payload">>, Args, <<"${payload}">>), + #{topic => Topic, qos => QoS, payload => Payload, retain => Retain, + preprocessed_tmpl => #{ + topic => emqx_plugin_libs_rule:preproc_tmpl(Topic), + qos => preproc_vars(QoS), + retain => preproc_vars(Retain), + payload => emqx_plugin_libs_rule:preproc_tmpl(Payload) + }}. -clear_resource(_Module, undefined, ResId) -> - ok = emqx_rule_registry:remove_resource_params(ResId); -clear_resource(Module, Destroy, ResId) -> - case emqx_rule_registry:find_resource_params(ResId) of - {ok, #resource_params{params = Params}} -> - ?RAISE(Module:Destroy(ResId, Params), - {{destroy_resource_failure, node()}, {{Module, Destroy}, {_EXCLASS_,_EXCPTION_,_ST_}}}), - ok = emqx_rule_registry:remove_resource_params(ResId); - not_found -> - ok - end. - -clear_rule(#rule{id = RuleId, actions = Actions}) -> - clear_actions(Actions), - emqx_rule_metrics:clear_rule_metrics(RuleId), - ok. - -clear_actions(Actions) -> - lists:foreach( - fun(#action_instance{id = Id, name = ActName, fallbacks = Fallbacks}) -> - {ok, #action{module = Mod, on_destroy = Destory}} = emqx_rule_registry:find_action(ActName), - clear_action(Mod, Destory, Id), - clear_actions(Fallbacks) - end, Actions). - -clear_action(_Module, undefined, ActionInstId) -> - emqx_rule_metrics:clear_metrics(ActionInstId), - ok = emqx_rule_registry:remove_action_instance_params(ActionInstId); -clear_action(Module, Destroy, ActionInstId) -> - case erlang:function_exported(Module, Destroy, 2) of - true -> - emqx_rule_metrics:clear_metrics(ActionInstId), - case emqx_rule_registry:get_action_instance_params(ActionInstId) of - {ok, #action_instance_params{params = Params}} -> - ?RAISE(Module:Destroy(ActionInstId, Params),{{destroy_action_failure, node()}, - {{Module, Destroy}, {_EXCLASS_,_EXCPTION_,_ST_}}}), - ok = emqx_rule_registry:remove_action_instance_params(ActionInstId); - not_found -> - ok - end; - false -> ok - end. - -fetch_resource_status(Module, OnStatus, ResId) -> - case emqx_rule_registry:find_resource_params(ResId) of - {ok, ResParams = #resource_params{params = Params, status = #{is_alive := LastIsAlive}}} -> - NewStatus = try - case Module:OnStatus(ResId, Params) of - #{is_alive := LastIsAlive} = Status -> Status; - #{is_alive := true} = Status -> - {ok, Type} = find_type(ResId), - Name = alarm_name_of_resource_down(Type, ResId), - emqx_alarm:deactivate(Name), - Status; - #{is_alive := false} = Status -> - {ok, Type} = find_type(ResId), - Name = alarm_name_of_resource_down(Type, ResId), - emqx_alarm:activate(Name, #{id => ResId, type => Type}), - Status - end - catch _Error:Reason:STrace -> - ?LOG(error, "get resource status for ~p failed: ~0p", [ResId, {Reason, STrace}]), - #{is_alive => false} - end, - emqx_rule_registry:add_resource_params(ResParams#resource_params{status = NewStatus}), - NewStatus; - not_found -> - #{is_alive => false} - end. - -refresh_actions_of_a_resource(ResId) -> - R = fun (#action_instance{args = #{<<"$resource">> := ResId0}}) - when ResId0 =:= ResId -> true; - (_) -> false - end, - F = fun(#rule{actions = Actions}) -> refresh_actions(Actions, R) end, - lists:foreach(F, emqx_rule_registry:get_rules()). - -refresh_actions(Actions) -> - refresh_actions(Actions, fun(_) -> true end). -refresh_actions(Actions, Pred) -> - lists:foreach( - fun(#action_instance{args = Args, - id = Id, name = ActName, - fallbacks = Fallbacks} = ActionInst) -> - case Pred(ActionInst) of - true -> - {ok, #action{module = Mod, on_create = Create}} - = emqx_rule_registry:find_action(ActName), - _ = emqx_plugin_libs_rule:cluster_call(?MODULE, init_action, [Mod, Create, Id, with_resource_params(Args)]), - refresh_actions(Fallbacks, Pred); - false -> ok - end - end, Actions). - -find_type(ResId) -> - {ok, #resource{type = Type}} = emqx_rule_registry:find_resource(ResId), - {ok, Type}. - -alarm_name_of_resource_down(Type, ResId) -> - list_to_binary(io_lib:format("resource/~s/~s/down", [Type, ResId])). +preproc_vars(Data) when is_binary(Data) -> + emqx_plugin_libs_rule:preproc_tmpl(Data); +preproc_vars(Data) -> + Data. diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index 122ae7705..b097c7169 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -19,536 +19,312 @@ -include("rule_engine.hrl"). -include_lib("emqx/include/logger.hrl"). +-behaviour(minirest_api). --rest_api(#{name => create_rule, - method => 'POST', - path => "/rules/", - func => create_rule, - descr => "Create a rule" - }). +-export([api_spec/0]). --rest_api(#{name => update_rule, - method => 'PUT', - path => "/rules/:bin:id", - func => update_rule, - descr => "Update a rule" - }). - --rest_api(#{name => list_rules, - method => 'GET', - path => "/rules/", - func => list_rules, - descr => "A list of all rules" - }). - --rest_api(#{name => show_rule, - method => 'GET', - path => "/rules/:bin:id", - func => show_rule, - descr => "Show a rule" - }). - --rest_api(#{name => delete_rule, - method => 'DELETE', - path => "/rules/:bin:id", - func => delete_rule, - descr => "Delete a rule" - }). - --rest_api(#{name => list_actions, - method => 'GET', - path => "/actions/", - func => list_actions, - descr => "A list of all actions" - }). - --rest_api(#{name => show_action, - method => 'GET', - path => "/actions/:atom:name", - func => show_action, - descr => "Show an action" - }). - --rest_api(#{name => list_resources, - method => 'GET', - path => "/resources/", - func => list_resources, - descr => "A list of all resources" - }). - --rest_api(#{name => create_resource, - method => 'POST', - path => "/resources/", - func => create_resource, - descr => "Create a resource" - }). - --rest_api(#{name => update_resource, - method => 'PUT', - path => "/resources/:bin:id", - func => update_resource, - descr => "Update a resource" - }). - --rest_api(#{name => show_resource, - method => 'GET', - path => "/resources/:bin:id", - func => show_resource, - descr => "Show a resource" - }). - --rest_api(#{name => get_resource_status, - method => 'GET', - path => "/resource_status/:bin:id", - func => get_resource_status, - descr => "Get status of a resource" - }). - --rest_api(#{name => start_resource, - method => 'POST', - path => "/resources/:bin:id", - func => start_resource, - descr => "Start a resource" - }). - --rest_api(#{name => delete_resource, - method => 'DELETE', - path => "/resources/:bin:id", - func => delete_resource, - descr => "Delete a resource" - }). - --rest_api(#{name => list_resource_types, - method => 'GET', - path => "/resource_types/", - func => list_resource_types, - descr => "List all resource types" - }). - --rest_api(#{name => show_resource_type, - method => 'GET', - path => "/resource_types/:atom:name", - func => show_resource_type, - descr => "Show a resource type" - }). - --rest_api(#{name => list_resources_by_type, - method => 'GET', - path => "/resource_types/:atom:type/resources", - func => list_resources_by_type, - descr => "List all resources of a resource type" - }). - --rest_api(#{name => list_events, - method => 'GET', - path => "/rule_events/", - func => list_events, - descr => "List all events with detailed info" - }). - --export([ create_rule/2 - , update_rule/2 - , list_rules/2 - , show_rule/2 - , delete_rule/2 +-export([ crud_rules/2 + , list_events/2 + , crud_rules_by_id/2 + , rule_test/2 ]). --export([ list_actions/2 - , show_action/2 - ]). - --export([ create_resource/2 - , list_resources/2 - , show_resource/2 - , get_resource_status/2 - , start_resource/2 - , delete_resource/2 - , update_resource/2 - ]). - --export([ list_resource_types/2 - , list_resources_by_type/2 - , show_resource_type/2 - ]). - --export([list_events/2]). - -define(ERR_NO_RULE(ID), list_to_binary(io_lib:format("Rule ~s Not Found", [(ID)]))). --define(ERR_NO_ACTION(NAME), list_to_binary(io_lib:format("Action ~s Not Found", [(NAME)]))). --define(ERR_NO_RESOURCE(RESID), list_to_binary(io_lib:format("Resource ~s Not Found", [(RESID)]))). --define(ERR_NO_RESOURCE_TYPE(TYPE), list_to_binary(io_lib:format("Resource Type ~s Not Found", [(TYPE)]))). --define(ERR_DEP_RULES_EXISTS(RULEIDS), list_to_binary(io_lib:format("Found rules ~0p depends on this resource, disable them first", [(RULEIDS)]))). -define(ERR_BADARGS(REASON), begin - R0 = list_to_binary(io_lib:format("~0p", [REASON])), + R0 = err_msg(REASON), <<"Bad Arguments: ", R0/binary>> end). +-define(CHECK_PARAMS(PARAMS, TAG, EXPR), + case emqx_rule_api_schema:check_params(PARAMS, TAG) of + {ok, CheckedParams} -> + EXPR; + {error, REASON} -> + {400, #{code => 'BAD_ARGS', message => ?ERR_BADARGS(REASON)}} + end). --dialyzer({nowarn_function, [create_rule/2, - test_rule_sql/1, - do_create_rule/1, - update_rule/2 - ]}). +api_spec() -> + { + [ api_rules_list_create() + , api_rules_crud() + , api_rule_test() + , api_events_list() + ], + [] + }. + +api_rules_list_create() -> + Metadata = #{ + get => #{ + description => <<"List all rules">>, + responses => #{ + <<"200">> => + emqx_mgmt_util:array_schema(resp_schema(), <<"List rules successfully">>)}}, + post => #{ + description => <<"Create a new rule using given Id to all nodes in the cluster">>, + requestBody => emqx_mgmt_util:schema(post_req_schema(), <<"Rule parameters">>), + responses => #{ + <<"400">> => + emqx_mgmt_util:error_schema(<<"Invalid Parameters">>, ['BAD_ARGS']), + <<"201">> => + emqx_mgmt_util:schema(resp_schema(), <<"Create rule successfully">>)}} + }, + {"/rules", Metadata, crud_rules}. + +api_events_list() -> + Metadata = #{ + get => #{ + description => <<"List all events can be used in rules">>, + responses => #{ + <<"200">> => + emqx_mgmt_util:array_schema(resp_schema(), <<"List events successfully">>)}} + }, + {"/rule_events", Metadata, list_events}. + +api_rules_crud() -> + Metadata = #{ + get => #{ + description => <<"Get a rule by given Id">>, + parameters => [param_path_id()], + responses => #{ + <<"404">> => + emqx_mgmt_util:error_schema(<<"Rule not found">>, ['NOT_FOUND']), + <<"200">> => + emqx_mgmt_util:schema(resp_schema(), <<"Get rule successfully">>)}}, + put => #{ + description => <<"Create or update a rule by given Id to all nodes in the cluster">>, + parameters => [param_path_id()], + requestBody => emqx_mgmt_util:schema(put_req_schema(), <<"Rule parameters">>), + responses => #{ + <<"400">> => + emqx_mgmt_util:error_schema(<<"Invalid Parameters">>, ['BAD_ARGS']), + <<"200">> => + emqx_mgmt_util:schema(resp_schema(), <<"Create or update rule successfully">>)}}, + delete => #{ + description => <<"Delete a rule by given Id from all nodes in the cluster">>, + parameters => [param_path_id()], + responses => #{ + <<"200">> => + emqx_mgmt_util:schema(<<"Delete rule successfully">>)}} + }, + {"/rules/:id", Metadata, crud_rules_by_id}. + +api_rule_test() -> + Metadata = #{ + post => #{ + description => <<"Test a rule">>, + requestBody => emqx_mgmt_util:schema(rule_test_req_schema(), <<"Rule parameters">>), + responses => #{ + <<"400">> => + emqx_mgmt_util:error_schema(<<"Invalid Parameters">>, ['BAD_ARGS']), + <<"412">> => + emqx_mgmt_util:error_schema(<<"SQL Not Match">>, ['NOT_MATCH']), + <<"200">> => + emqx_mgmt_util:schema(rule_test_resp_schema(), <<"Rule Test Pass">>)}} + }, + {"/rule_test", Metadata, rule_test}. + +put_req_schema() -> + #{type => object, + properties => #{ + sql => #{ + description => <<"The SQL">>, + type => string, + example => <<"SELECT * from \"t/1\"">> + }, + enable => #{ + description => <<"Enable or disable the rule">>, + type => boolean, + example => true + }, + outputs => #{ + description => <<"The outputs of the rule">>, + type => array, + items => #{ + type => object, + properties => #{ + type => #{ + type => string, + enum => [<<"bridge">>, <<"builtin">>], + example => <<"builtin">> + }, + target => #{ + type => string, + example => <<"console">> + }, + args => #{ + type => object + } + } + } + }, + description => #{ + description => <<"The description for the rule">>, + type => string, + example => <<"A simple rule that handles MQTT messages from topic \"t/1\"">> + } + } + }. + +post_req_schema() -> + Req = #{properties := Prop} = put_req_schema(), + Req#{properties => Prop#{ + id => #{ + description => <<"The Id for the rule">>, + example => <<"my_rule">>, + type => string + } + }}. + +resp_schema() -> + Req = #{properties := Prop} = put_req_schema(), + Req#{properties => Prop#{ + id => #{ + description => <<"The Id for the rule">>, + type => string + }, + created_at => #{ + description => <<"The time that this rule was created, in rfc3339 format">>, + type => string, + example => <<"2021-09-18T13:57:29+08:00">> + } + }}. + +rule_test_req_schema() -> + #{type => object, properties => #{ + sql => #{ + description => <<"The SQL">>, + type => string, + example => <<"SELECT * from \"t/1\"">> + }, + context => #{ + type => object, + properties => #{ + event_type => #{ + description => <<"Event Type">>, + type => string, + enum => [<<"message_publish">>, <<"message_acked">>, <<"message_delivered">>, + <<"message_dropped">>, <<"session_subscribed">>, <<"session_unsubscribed">>, + <<"client_connected">>, <<"client_disconnected">>], + example => <<"message_publish">> + }, + clientid => #{ + description => <<"The Client ID">>, + type => string, + example => <<"\"c_emqx\"">> + }, + topic => #{ + description => <<"The Topic">>, + type => string, + example => <<"t/1">> + } + } + } + }}. + +rule_test_resp_schema() -> + #{type => object}. + +param_path_id() -> + #{ + name => id, + in => path, + schema => #{type => string}, + required => true + }. %%------------------------------------------------------------------------------ %% Rules API %%------------------------------------------------------------------------------ -create_rule(_Bindings, Params) -> - if_test(fun() -> test_rule_sql(Params) end, - fun() -> do_create_rule(Params) end, - Params). - -test_rule_sql(Params) -> - case emqx_rule_sqltester:test(emqx_json:decode(emqx_json:encode(Params), [return_maps])) of - {ok, Result} -> return({ok, Result}); - {error, nomatch} -> return({error, 404, <<"SQL Not Match">>}); - {error, Reason} -> - ?LOG(error, "~p failed: ~0p", [?FUNCTION_NAME, Reason]), - return({error, 400, ?ERR_BADARGS(Reason)}) - end. - -do_create_rule(Params) -> - case emqx_rule_engine:create_rule(parse_rule_params(Params)) of - {ok, Rule} -> return({ok, record_to_map(Rule)}); - {error, {action_not_found, ActionName}} -> - return({error, 400, ?ERR_NO_ACTION(ActionName)}); - {error, Reason} -> - ?LOG(error, "~p failed: ~0p", [?FUNCTION_NAME, Reason]), - return({error, 400, ?ERR_BADARGS(Reason)}) - end. - -update_rule(#{id := Id}, Params) -> - case emqx_rule_engine:update_rule(parse_rule_params(Params, #{id => Id})) of - {ok, Rule} -> return({ok, record_to_map(Rule)}); - {error, {not_found, RuleId}} -> - return({error, 400, ?ERR_NO_RULE(RuleId)}); - {error, Reason} -> - ?LOG(error, "~p failed: ~0p", [?FUNCTION_NAME, Reason]), - return({error, 400, ?ERR_BADARGS(Reason)}) - end. - -list_rules(_Bindings, _Params) -> - return_all(emqx_rule_registry:get_rules_ordered_by_ts()). - -show_rule(#{id := Id}, _Params) -> - reply_with(fun emqx_rule_registry:get_rule/1, Id). - -delete_rule(#{id := Id}, _Params) -> - ok = emqx_rule_engine:delete_rule(Id), - return(ok). - -%%------------------------------------------------------------------------------ -%% Actions API -%%------------------------------------------------------------------------------ - -list_actions(#{}, _Params) -> - return_all( - sort_by_title(action, - emqx_rule_registry:get_actions())). - -show_action(#{name := Name}, _Params) -> - reply_with(fun emqx_rule_registry:find_action/1, Name). - -%%------------------------------------------------------------------------------ -%% Resources API -%%------------------------------------------------------------------------------ -create_resource(#{}, Params) -> - case parse_resource_params(Params) of - {ok, ParsedParams} -> - if_test(fun() -> do_create_resource(test_resource, ParsedParams) end, - fun() -> do_create_resource(create_resource, ParsedParams) end, - Params); - {error, Reason} -> - ?LOG(error, "~p failed: ~0p", [?FUNCTION_NAME, Reason]), - return({error, 400, ?ERR_BADARGS(Reason)}) - end. - -do_create_resource(Create, ParsedParams) -> - case emqx_rule_engine:Create(ParsedParams) of - ok -> - return(ok); - {ok, Resource} -> - return({ok, record_to_map(Resource)}); - {error, {resource_type_not_found, Type}} -> - return({error, 400, ?ERR_NO_RESOURCE_TYPE(Type)}); - {error, {init_resource, _}} -> - return({error, 500, <<"Init resource failure!">>}); - {error, Reason} -> - ?LOG(error, "~p failed: ~0p", [?FUNCTION_NAME, Reason]), - return({error, 400, ?ERR_BADARGS(Reason)}) - end. - -list_resources(#{}, _Params) -> - Data0 = lists:foldr(fun maybe_record_to_map/2, [], emqx_rule_registry:get_resources()), - Data = lists:map(fun(Res = #{id := Id}) -> - Status = lists:all(fun(Node) -> - case rpc:call(Node, emqx_rule_registry, find_resource_params, [Id]) of - {ok, #resource_params{status = #{is_alive := true}}} -> true; - _ -> false - end - end, ekka_mnesia:running_nodes()), - maps:put(status, Status, Res) - end, Data0), - return({ok, Data}). - -list_resources_by_type(#{type := Type}, _Params) -> - return_all(emqx_rule_registry:get_resources_by_type(Type)). - -show_resource(#{id := Id}, _Params) -> - case emqx_rule_registry:find_resource(Id) of - {ok, R} -> - Status = - [begin - {ok, St} = rpc:call(Node, emqx_rule_engine, get_resource_status, [Id]), - maps:put(node, Node, St) - end || Node <- ekka_mnesia:running_nodes()], - return({ok, maps:put(status, Status, record_to_map(R))}); - not_found -> - return({error, 404, <<"Not Found">>}) - end. - -get_resource_status(#{id := Id}, _Params) -> - case emqx_rule_engine:get_resource_status(Id) of - {ok, Status} -> - return({ok, Status}); - {error, {resource_not_found, ResId}} -> - return({error, 400, ?ERR_NO_RESOURCE(ResId)}) - end. - -start_resource(#{id := Id}, _Params) -> - case emqx_rule_engine:start_resource(Id) of - ok -> - return(ok); - {error, {resource_not_found, ResId}} -> - return({error, 400, ?ERR_NO_RESOURCE(ResId)}); - {error, Reason} -> - ?LOG(error, "~p failed: ~0p", [?FUNCTION_NAME, Reason]), - return({error, 400, ?ERR_BADARGS(Reason)}) - end. - -update_resource(#{id := Id}, NewParams) -> - P1 = case proplists:get_value(<<"description">>, NewParams) of - undefined -> #{}; - Value -> #{<<"description">> => Value} - end, - P2 = case proplists:get_value(<<"config">>, NewParams) of - undefined -> #{}; - [{}] -> #{}; - Config -> #{<<"config">> => ?RAISE(json_term_to_map(Config), {invalid_config, Config})} - end, - case emqx_rule_engine:update_resource(Id, maps:merge(P1, P2)) of - ok -> - return(ok); - {error, not_found} -> - return({error, 400, <<"Resource not found:", Id/binary>>}); - {error, {init_resource, _}} -> - return({error, 500, <<"Init resource failure:", Id/binary>>}); - {error, {dependent_rules_exists, RuleIds}} -> - return({error, 400, ?ERR_DEP_RULES_EXISTS(RuleIds)}); - {error, Reason} -> - ?LOG(error, "Resource update failed: ~0p", [Reason]), - return({error, 400, ?ERR_BADARGS(Reason)}) - end. - -delete_resource(#{id := Id}, _Params) -> - case emqx_rule_engine:delete_resource(Id) of - ok -> return(ok); - {error, not_found} -> return(ok); - {error, {dependent_rules_exists, RuleIds}} -> - return({error, 400, ?ERR_DEP_RULES_EXISTS(RuleIds)}); - {error, Reason} -> - return({error, 400, ?ERR_BADARGS(Reason)}) - end. - -%%------------------------------------------------------------------------------ -%% Resource Types API -%%------------------------------------------------------------------------------ - -list_resource_types(#{}, _Params) -> - return_all( - sort_by_title(resource_type, - emqx_rule_registry:get_resource_types())). - -show_resource_type(#{name := Name}, _Params) -> - reply_with(fun emqx_rule_registry:find_resource_type/1, Name). - - -%%------------------------------------------------------------------------------ -%% Events API -%%------------------------------------------------------------------------------ list_events(#{}, _Params) -> - return({ok, emqx_rule_events:event_info()}). + {200, emqx_rule_events:event_info()}. + +crud_rules(get, _Params) -> + Records = emqx_rule_registry:get_rules_ordered_by_ts(), + {200, format_rule_resp(Records)}; + +crud_rules(post, #{body := Params}) -> + ?CHECK_PARAMS(Params, rule_creation, case emqx_rule_engine:create_rule(CheckedParams) of + {ok, Rule} -> {201, format_rule_resp(Rule)}; + {error, Reason} -> + ?SLOG(error, #{msg => "create_rule_failed", reason => Reason}), + {400, #{code => 'BAD_ARGS', message => ?ERR_BADARGS(Reason)}} + end). + +rule_test(post, #{body := Params}) -> + ?CHECK_PARAMS(Params, rule_test, case emqx_rule_sqltester:test(CheckedParams) of + {ok, Result} -> {200, Result}; + {error, nomatch} -> {412, #{code => 'NOT_MATCH', message => <<"SQL Not Match">>}} + end). + +crud_rules_by_id(get, #{bindings := #{id := Id}}) -> + case emqx_rule_registry:get_rule(Id) of + {ok, Rule} -> + {200, format_rule_resp(Rule)}; + not_found -> + {404, #{code => 'NOT_FOUND', message => <<"Rule Id Not Found">>}} + end; + +crud_rules_by_id(put, #{bindings := #{id := Id}, body := Params0}) -> + Params = maps:merge(Params0, #{id => Id}), + ?CHECK_PARAMS(Params, rule_creation, case emqx_rule_engine:update_rule(CheckedParams) of + {ok, Rule} -> {200, format_rule_resp(Rule)}; + {error, not_found} -> + {404, #{code => 'NOT_FOUND', message => <<"Rule Id Not Found">>}}; + {error, Reason} -> + ?SLOG(error, #{msg => "update_rule_failed", + id => Id, + reason => Reason}), + {400, #{code => 'BAD_ARGS', message => ?ERR_BADARGS(Reason)}} + end); + +crud_rules_by_id(delete, #{bindings := #{id := Id}}) -> + case emqx_rule_engine:delete_rule(Id) of + ok -> {200}; + {error, not_found} -> {200} + end. %%------------------------------------------------------------------------------ %% Internal functions %%------------------------------------------------------------------------------ +err_msg(Msg) -> + list_to_binary(io_lib:format("~0p", [Msg])). -if_test(True, False, Params) -> - case proplists:get_value(<<"test">>, Params) of - Test when Test =:= true; Test =:= <<"true">> -> - True(); - _ -> - False() - end. -return_all(Records) -> - Data = lists:foldr(fun maybe_record_to_map/2, [], Records), - return({ok, Data}). +format_rule_resp(Rules) when is_list(Rules) -> + [format_rule_resp(R) || R <- Rules]; -maybe_record_to_map(Rec, Acc) -> - case record_to_map(Rec) of - ignore -> Acc; - Map -> [Map | Acc] - end. - -reply_with(Find, Key) -> - case Find(Key) of - {ok, R} -> - return({ok, record_to_map(R)}); - not_found -> - return({error, 404, <<"Not Found">>}) - end. - -record_to_map(#rule{id = Id, - for = Hook, - rawsql = RawSQL, - actions = Actions, - on_action_failed = OnFailed, - enabled = Enabled, - description = Descr}) -> +format_rule_resp(#rule{id = Id, created_at = CreatedAt, + info = #{ + from := Topics, + outputs := Output, + sql := SQL, + enabled := Enabled, + description := Descr}}) -> #{id => Id, - for => Hook, - rawsql => RawSQL, - actions => printable_actions(Actions), - on_action_failed => OnFailed, + from => Topics, + outputs => format_output(Output), + sql => SQL, metrics => get_rule_metrics(Id), enabled => Enabled, - description => Descr - }; - -record_to_map(#action{hidden = true}) -> - ignore; -record_to_map(#action{name = Name, - category = Category, - app = App, - for = Hook, - types = Types, - params_spec = Params, - title = Title, - description = Descr}) -> - #{name => Name, - category => Category, - app => App, - for => Hook, - types => Types, - params => Params, - title => Title, - description => Descr - }; - -record_to_map(#resource{id = Id, - type = Type, - config = Config, - description = Descr}) -> - #{id => Id, - type => Type, - config => Config, - description => Descr - }; - -record_to_map(#resource_type{name = Name, - provider = Provider, - params_spec = Params, - title = Title, - description = Descr}) -> - #{name => Name, - provider => Provider, - params => Params, - title => Title, + created_at => format_datetime(CreatedAt, millisecond), description => Descr }. -printable_actions(Actions) -> - [#{id => Id, name => Name, params => Args, - metrics => get_action_metrics(Id), - fallbacks => printable_actions(Fallbacks)} - || #action_instance{id = Id, name = Name, args = Args, fallbacks = Fallbacks} <- Actions]. +format_datetime(Timestamp, Unit) -> + list_to_binary(calendar:system_time_to_rfc3339(Timestamp, [{unit, Unit}])). -parse_rule_params(Params) -> - parse_rule_params(Params, #{description => <<"">>}). -parse_rule_params([], Rule) -> - Rule; -parse_rule_params([{<<"id">>, Id} | Params], Rule) -> - parse_rule_params(Params, Rule#{id => Id}); -parse_rule_params([{<<"rawsql">>, RawSQL} | Params], Rule) -> - parse_rule_params(Params, Rule#{rawsql => RawSQL}); -parse_rule_params([{<<"enabled">>, Enabled} | Params], Rule) -> - parse_rule_params(Params, Rule#{enabled => enabled(Enabled)}); -parse_rule_params([{<<"on_action_failed">>, OnFailed} | Params], Rule) -> - parse_rule_params(Params, Rule#{on_action_failed => on_failed(OnFailed)}); -parse_rule_params([{<<"actions">>, Actions} | Params], Rule) -> - parse_rule_params(Params, Rule#{actions => parse_actions(Actions)}); -parse_rule_params([{<<"description">>, Descr} | Params], Rule) -> - parse_rule_params(Params, Rule#{description => Descr}); -parse_rule_params([_ | Params], Rule) -> - parse_rule_params(Params, Rule). +format_output(Outputs) -> + [do_format_output(Out) || Out <- Outputs]. -on_failed(<<"continue">>) -> continue; -on_failed(<<"stop">>) -> stop; -on_failed(OnFailed) -> error({invalid_on_failed, OnFailed}). - -enabled(Enabled) when is_boolean(Enabled) -> Enabled; -enabled(Enabled) -> error({invalid_enabled, Enabled}). - -parse_actions(Actions) -> - [parse_action(json_term_to_map(A)) || A <- Actions]. - -parse_action(Action) -> - #{name => binary_to_existing_atom(maps:get(<<"name">>, Action), utf8), - args => maps:get(<<"params">>, Action, #{}), - fallbacks => parse_actions(maps:get(<<"fallbacks">>, Action, []))}. - -parse_resource_params(Params) -> - parse_resource_params(Params, #{config => #{}, description => <<"">>}). -parse_resource_params([], Res) -> - {ok, Res}; -parse_resource_params([{<<"id">>, Id} | Params], Res) -> - parse_resource_params(Params, Res#{id => Id}); -parse_resource_params([{<<"type">>, ResourceType} | Params], Res) -> - try parse_resource_params(Params, Res#{type => binary_to_existing_atom(ResourceType, utf8)}) - catch error:badarg -> - {error, {resource_type_not_found, ResourceType}} - end; -parse_resource_params([{<<"config">>, Config} | Params], Res) -> - parse_resource_params(Params, Res#{config => json_term_to_map(Config)}); -parse_resource_params([{<<"description">>, Descr} | Params], Res) -> - parse_resource_params(Params, Res#{description => Descr}); -parse_resource_params([_ | Params], Res) -> - parse_resource_params(Params, Res). - -json_term_to_map(List) -> - emqx_json:decode(emqx_json:encode(List), [return_maps]). - -sort_by_title(action, Actions) -> - sort_by(#action.title, Actions); -sort_by_title(resource_type, ResourceTypes) -> - sort_by(#resource_type.title, ResourceTypes). - -sort_by(Pos, TplList) -> - lists:sort( - fun(RecA, RecB) -> - maps:get(en, element(Pos, RecA), 0) - =< maps:get(en, element(Pos, RecB), 0) - end, TplList). +do_format_output(#{type := func}) -> + #{type => func, target => <<"internal_function">>}; +do_format_output(#{type := builtin, target := Name, args := Args}) -> + #{type => builtin, target => Name, args => maps:remove(preprocessed_tmpl, Args)}; +do_format_output(#{type := bridge, target := Name}) -> + #{type => bridge, target => Name}. get_rule_metrics(Id) -> [maps:put(node, Node, rpc:call(Node, emqx_rule_metrics, get_rule_metrics, [Id])) || Node <- ekka_mnesia:running_nodes()]. - -get_action_metrics(Id) -> - [maps:put(node, Node, rpc:call(Node, emqx_rule_metrics, get_action_metrics, [Id])) - || Node <- ekka_mnesia:running_nodes()]. - -%% TODO: V5 API -return(_) -> ok. \ No newline at end of file diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_app.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_app.erl index 5893f9827..e3e959222 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_app.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_app.erl @@ -16,6 +16,8 @@ -module(emqx_rule_engine_app). +-include("rule_engine.hrl"). + -behaviour(application). -export([start/2]). @@ -23,14 +25,9 @@ -export([stop/1]). start(_Type, _Args) -> - {ok, Sup} = emqx_rule_engine_sup:start_link(), - _ = emqx_rule_engine_sup:start_locker(), - ok = emqx_rule_engine:load_providers(), - ok = emqx_rule_engine:refresh_resources(), - ok = emqx_rule_engine:refresh_rules(), - ok = emqx_rule_engine_cli:load(), - {ok, Sup}. + ok = ekka_rlog:wait_for_shards([?RULE_ENGINE_SHARD], infinity), + ok = emqx_rule_events:reload(), + emqx_rule_engine_sup:start_link(). stop(_State) -> - ok = emqx_rule_events:unload(), - ok = emqx_rule_engine_cli:unload(). + ok = emqx_rule_events:unload(). diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_cli.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_cli.erl deleted file mode 100644 index bcb869aec..000000000 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_cli.erl +++ /dev/null @@ -1,387 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_rule_engine_cli). - --include("rule_engine.hrl"). - --export([ load/0 - , commands/0 - , unload/0 - ]). - --export([ rules/1 - , actions/1 - , resources/1 - , resource_types/1 - ]). - --import(proplists, [get_value/2]). - --define(OPTSPEC_RESOURCE_TYPE, - [{type, $t, "type", {atom, undefined}, "Resource Type"}]). --define(OPTSPEC_ACTION_TYPE, - [ {eventype, $k, "eventype", {atom, undefined}, "Event Type"} - ]). - --define(OPTSPEC_RESOURCES_CREATE, - [ {type, undefined, undefined, atom, "Resource Type"} - , {id, $i, "id", {binary, <<"">>}, "The resource id. A random resource id will be used if not provided"} - , {config, $c, "config", {binary, <<"{}">>}, "Config"} - , {descr, $d, "descr", {binary, <<"">>}, "Description"} - ]). - --define(OPTSPEC_RESOURCES_UPDATE, - [ {id, undefined, undefined, binary, "The resource id"} - , {config, $c, "config", {binary, undefined}, "Config"} - , {description, $d, "descr", {binary, undefined}, "Description"} - ]). - --define(OPTSPEC_RULES_CREATE, - [ {sql, undefined, undefined, binary, "Filter Condition SQL"} - , {actions, undefined, undefined, binary, "Action List in JSON format: [{\"name\": , \"params\": {: }}]"} - , {id, $i, "id", {binary, <<"">>}, "The rule id. A random rule id will be used if not provided"} - , {enabled, $e, "enabled", {atom, true}, "'true' or 'false' to enable or disable the rule"} - , {on_action_failed, $g, "on_action_failed", {atom, continue}, "'continue' or 'stop' when an action in the rule fails"} - , {descr, $d, "descr", {binary, <<"">>}, "Description"} - ]). - --define(OPTSPEC_RULES_UPDATE, - [ {id, undefined, undefined, binary, "Rule ID"} - , {sql, $s, "sql", {binary, undefined}, "Filter Condition SQL"} - , {actions, $a, "actions", {binary, undefined}, "Action List in JSON format: [{\"name\": , \"params\": {: }}]"} - , {enabled, $e, "enabled", {atom, undefined}, "'true' or 'false' to enable or disable the rule"} - , {on_action_failed, $g, "on_action_failed", {atom, undefined}, "'continue' or 'stop' when an action in the rule fails"} - , {descr, $d, "descr", {binary, undefined}, "Description"} - ]). -%%----------------------------------------------------------------------------- -%% Load/Unload Commands -%%----------------------------------------------------------------------------- - --spec(load() -> ok). -load() -> - lists:foreach( - fun({Cmd, Func}) -> - emqx_ctl:register_command(Cmd, {?MODULE, Func}, []); - (Cmd) -> - emqx_ctl:register_command(Cmd, {?MODULE, Cmd}, []) - end, commands()). - --spec(commands() -> list(atom())). -commands() -> - [rules, {'rule-actions', actions}, resources, {'resource-types', resource_types}]. - --spec(unload() -> ok). -unload() -> - lists:foreach( - fun({Cmd, _Func}) -> - emqx_ctl:unregister_command(Cmd); - (Cmd) -> - emqx_ctl:unregister_command(Cmd) - end, commands()). - -%%----------------------------------------------------------------------------- -%% 'rules' command -%%----------------------------------------------------------------------------- --dialyzer([{nowarn_function, [rules/1]}]). -rules(["list"]) -> - print_all(emqx_rule_registry:get_rules_ordered_by_ts()); - -rules(["show", RuleId]) -> - print_with(fun emqx_rule_registry:get_rule/1, list_to_binary(RuleId)); - -rules(["create" | Params]) -> - with_opts(fun({Opts, _}) -> - case emqx_rule_engine:create_rule(make_rule(Opts)) of - {ok, #rule{id = RuleId}} -> - emqx_ctl:print("Rule ~s created~n", [RuleId]); - {error, Reason} -> - emqx_ctl:print("Invalid options: ~0p~n", [Reason]) - end - end, Params, ?OPTSPEC_RULES_CREATE, {?FUNCTION_NAME, create}); - -rules(["update" | Params]) -> - with_opts(fun({Opts, _}) -> - case emqx_rule_engine:update_rule(make_updated_rule(Opts)) of - {ok, #rule{id = RuleId}} -> - emqx_ctl:print("Rule ~s updated~n", [RuleId]); - {error, Reason} -> - emqx_ctl:print("Invalid options: ~0p~n", [Reason]) - end - end, Params, ?OPTSPEC_RULES_UPDATE, {?FUNCTION_NAME, update}); - -rules(["delete", RuleId]) -> - ok = emqx_rule_engine:delete_rule(list_to_binary(RuleId)), - emqx_ctl:print("ok~n"); - -rules(_Usage) -> - emqx_ctl:usage([{"rules list", "List all rules"}, - {"rules show ", "Show a rule"}, - {"rules create", "Create a rule"}, - {"rules delete ", "Delete a rule"} - ]). - -%%----------------------------------------------------------------------------- -%% 'rule-actions' command -%%----------------------------------------------------------------------------- - -actions(["list"]) -> - print_all(get_actions()); - -actions(["show", ActionId]) -> - print_with(fun emqx_rule_registry:find_action/1, - ?RAISE(list_to_existing_atom(ActionId), {not_found, ActionId})); - -actions(_Usage) -> - emqx_ctl:usage([{"rule-actions list", "List actions"}, - {"rule-actions show ", "Show a rule action"} - ]). - -%%------------------------------------------------------------------------------ -%% 'resources' command -%%------------------------------------------------------------------------------ - -resources(["create" | Params]) -> - with_opts(fun({Opts, _}) -> - case emqx_rule_engine:create_resource(make_resource(Opts)) of - {ok, #resource{id = ResId}} -> - emqx_ctl:print("Resource ~s created~n", [ResId]); - {error, Reason} -> - emqx_ctl:print("Invalid options: ~0p~n", [Reason]) - end - end, Params, ?OPTSPEC_RESOURCES_CREATE, {?FUNCTION_NAME, create}); - - -resources(["update" | Params]) -> - with_opts(fun({Opts, _}) -> - Id = proplists:get_value(id, Opts), - Maps = make_updated_resource(Opts), - case emqx_rule_engine:update_resource(Id, Maps) of - ok -> - emqx_ctl:print("Resource update successfully~n"); - {error, Reason} -> - emqx_ctl:print("Resource update failed: ~0p~n", [Reason]) - end - end, Params, ?OPTSPEC_RESOURCES_UPDATE, {?FUNCTION_NAME, update}); - -resources(["test" | Params]) -> - with_opts(fun({Opts, _}) -> - case emqx_rule_engine:test_resource(make_resource(Opts)) of - ok -> - emqx_ctl:print("Test creating resource successfully (dry-run)~n"); - {error, Reason} -> - emqx_ctl:print("Test creating resource failed: ~0p~n", [Reason]) - end - end, Params, ?OPTSPEC_RESOURCES_CREATE, {?FUNCTION_NAME, test}); - -resources(["list"]) -> - print_all(emqx_rule_registry:get_resources()); - -resources(["list" | Params]) -> - with_opts(fun({Opts, _}) -> - print_all(emqx_rule_registry:get_resources_by_type( - get_value(type, Opts))) - end, Params, ?OPTSPEC_RESOURCE_TYPE, {?FUNCTION_NAME, list}); - -resources(["show", ResourceId]) -> - print_with(fun emqx_rule_registry:find_resource/1, list_to_binary(ResourceId)); - -resources(["delete", ResourceId]) -> - case emqx_rule_engine:delete_resource(list_to_binary(ResourceId)) of - ok -> emqx_ctl:print("ok~n"); - {error, not_found} -> emqx_ctl:print("ok~n"); - {error, Reason} -> - emqx_ctl:print("Cannot delete resource as ~0p~n", [Reason]) - end; - -resources(_Usage) -> - emqx_ctl:usage([{"resources create", "Create a resource"}, - {"resources list [-t ]", "List resources"}, - {"resources show ", "Show a resource"}, - {"resources delete ", "Delete a resource"}, - {"resources update [-c ] [-d ]", "Update a resource"} - ]). - -%%------------------------------------------------------------------------------ -%% 'resource-types' command -%%------------------------------------------------------------------------------ -resource_types(["list"]) -> - print_all(emqx_rule_registry:get_resource_types()); - -resource_types(["show", Name]) -> - print_with(fun emqx_rule_registry:find_resource_type/1, list_to_atom(Name)); - -resource_types(_Usage) -> - emqx_ctl:usage([{"resource-types list", "List all resource-types"}, - {"resource-types show ", "Show a resource-type"} - ]). - -%%------------------------------------------------------------------------------ -%% Internal functions -%%------------------------------------------------------------------------------ - -print(Data) -> - emqx_ctl:print(untilde(format(Data))). - -print_all(DataList) -> - lists:map(fun(Data) -> - print(Data) - end, DataList). - -print_with(FindFun, Key) -> - case FindFun(Key) of - {ok, R} -> - print(R); - not_found -> - emqx_ctl:print("Cannot found ~s~n", [Key]) - end. - -format(#rule{id = Id, - for = Hook, - rawsql = Sql, - actions = Actions, - on_action_failed = OnFailed, - enabled = Enabled, - description = Descr}) -> - lists:flatten(io_lib:format("rule(id='~s', for='~0p', rawsql='~s', actions=~0p, on_action_failed='~s', metrics=~0p, enabled='~s', description='~s')~n", [Id, Hook, rmlf(Sql), printable_actions(Actions), OnFailed, get_rule_metrics(Id), Enabled, Descr])); - -format(#action{hidden = true}) -> - ok; -format(#action{name = Name, - for = Hook, - app = App, - types = Types, - title = #{en := Title}, - description = #{en := Descr}}) -> - lists:flatten(io_lib:format("action(name='~s', app='~s', for='~s', types=~0p, title ='~s', description='~s')~n", [Name, App, Hook, Types, Title, Descr])); - -format(#resource{id = Id, - type = Type, - config = Config, - description = Descr}) -> - Status = - [begin - {ok, St} = rpc:call(Node, emqx_rule_engine, get_resource_status, [Id]), - maps:put(node, Node, St) - end || Node <- [node()| nodes()]], - lists:flatten(io_lib:format("resource(id='~s', type='~s', config=~0p, status=~0p, description='~s')~n", [Id, Type, Config, Status, Descr])); - -format(#resource_type{name = Name, - provider = Provider, - title = #{en := Title}, - description = #{en := Descr}}) -> - lists:flatten(io_lib:format("resource_type(name='~s', provider='~s', title ='~s', description='~s')~n", [Name, Provider, Title, Descr])). - -make_rule(Opts) -> - Actions = get_value(actions, Opts), - may_with_opt( - #{rawsql => get_value(sql, Opts), - enabled => get_value(enabled, Opts), - actions => parse_actions(emqx_json:decode(Actions, [return_maps])), - on_action_failed => on_failed(get_value(on_action_failed, Opts)), - description => get_value(descr, Opts)}, id, <<"">>, Opts). - -make_updated_rule(Opts) -> - KeyNameParsers = [{sql, rawsql, fun(SQL) -> SQL end}, - enabled, - {actions, actions, fun(Actions) -> - parse_actions(emqx_json:decode(Actions, [return_maps])) - end}, - on_action_failed, - {descr, description, fun(Descr) -> Descr end}], - lists:foldl(fun - ({Key, Name, Parser}, ParamsAcc) -> - case get_value(Key, Opts) of - undefined -> ParamsAcc; - Val -> ParamsAcc#{Name => Parser(Val)} - end; - (Key, ParamsAcc) -> - case get_value(Key, Opts) of - undefined -> ParamsAcc; - Val -> ParamsAcc#{Key => Val} - end - end, #{id => get_value(id, Opts)}, KeyNameParsers). - -make_resource(Opts) -> - Config = get_value(config, Opts), - may_with_opt( - #{type => get_value(type, Opts), - config => ?RAISE(emqx_json:decode(Config, [return_maps]), {invalid_config, Config}), - description => get_value(descr, Opts)}, id, <<"">>, Opts). - -make_updated_resource(Opts) -> - P1 = case proplists:get_value(description, Opts) of - undefined -> #{}; - Value -> #{<<"description">> => Value} - end, - P2 = case proplists:get_value(config, Opts) of - undefined -> #{}; - Map -> #{<<"config">> => ?RAISE((emqx_json:decode(Map, [return_maps])), {invalid_config, Map})} - end, - maps:merge(P1, P2). - -printable_actions(Actions) when is_list(Actions) -> - emqx_json:encode([#{id => Id, name => Name, params => Args, - metrics => get_action_metrics(Id), - fallbacks => printable_actions(Fallbacks)} - || #action_instance{id = Id, name = Name, args = Args, fallbacks = Fallbacks} <- Actions]). - -may_with_opt(Params, OptName, DefaultVal, Options) when is_map(Params) -> - case get_value(OptName, Options) of - DefaultVal -> Params; - Val -> Params#{OptName => Val} - end. - -with_opts(Action, RawParams, OptSpecList, {CmdObject, CmdName}) -> - case getopt:parse_and_check(OptSpecList, RawParams) of - {ok, Params} -> - Action(Params); - {error, Reason} -> - getopt:usage(OptSpecList, - io_lib:format("emqx_ctl ~s ~s", [CmdObject, CmdName]), standard_io), - emqx_ctl:print("~0p~n", [Reason]) - end. - -parse_actions(Actions) -> - ?RAISE([parse_action(Action) || Action <- Actions], - {invalid_action_params, {_EXCLASS_,_EXCPTION_,_ST_}}). - -parse_action(Action) -> - ActName = maps:get(<<"name">>, Action), - #{name => ?RAISE(binary_to_existing_atom(ActName, utf8), {action_not_found, ActName}), - args => maps:get(<<"params">>, Action, #{}), - fallbacks => parse_actions(maps:get(<<"fallbacks">>, Action, []))}. - -get_actions() -> - emqx_rule_registry:get_actions(). - -get_rule_metrics(Id) -> - [maps:put(node, Node, rpc:call(Node, emqx_rule_metrics, get_rule_metrics, [Id])) - || Node <- [node()| nodes()]]. - -get_action_metrics(Id) -> - [maps:put(node, Node, rpc:call(Node, emqx_rule_metrics, get_action_metrics, [Id])) - || Node <- [node()| nodes()]]. - -on_failed(continue) -> continue; -on_failed(stop) -> stop; -on_failed(OnFailed) -> error({invalid_on_failed, OnFailed}). - -rmlf(Str) -> - re:replace(Str, "\n", "", [global]). - -untilde(Str) -> - re:replace(Str, "~", "&&", [{return, list}, global]). diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_sup.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_sup.erl index 9ff5ce741..4fad54a4b 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_sup.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_sup.erl @@ -22,17 +22,12 @@ -export([start_link/0]). --export([start_locker/0]). - -export([init/1]). start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). init([]) -> - Opts = [public, named_table, set, {read_concurrency, true}], - _ = ets:new(?ACTION_INST_PARAMS_TAB, [{keypos, #action_instance_params.id}|Opts]), - _ = ets:new(?RES_PARAMS_TAB, [{keypos, #resource_params.id}|Opts]), Registry = #{id => emqx_rule_registry, start => {emqx_rule_registry, start_link, []}, restart => permanent, @@ -45,19 +40,4 @@ init([]) -> shutdown => 5000, type => worker, modules => [emqx_rule_metrics]}, - Monitor = #{id => emqx_rule_monitor, - start => {emqx_rule_monitor, start_link, []}, - restart => permanent, - shutdown => 5000, - type => worker, - modules => [emqx_rule_monitor]}, - {ok, {{one_for_one, 10, 10}, [Registry, Metrics, Monitor]}}. - -start_locker() -> - Locker = #{id => emqx_rule_locker, - start => {emqx_rule_locker, start_link, []}, - restart => permanent, - shutdown => 5000, - type => worker, - modules => [emqx_rule_locker]}, - supervisor:start_child(?MODULE, Locker). + {ok, {{one_for_one, 10, 10}, [Registry, Metrics]}}. diff --git a/apps/emqx_rule_engine/src/emqx_rule_events.erl b/apps/emqx_rule_engine/src/emqx_rule_events.erl index a0960df25..d030917ef 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_events.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_events.erl @@ -21,7 +21,8 @@ -include_lib("emqx/include/logger.hrl"). --export([ load/1 +-export([ reload/0 + , load/1 , unload/0 , unload/1 , event_name/1 @@ -36,6 +37,7 @@ , on_message_dropped/4 , on_message_delivered/3 , on_message_acked/3 + , on_bridge_message_received/2 ]). -export([ event_info/0 @@ -61,6 +63,12 @@ ]). -endif. +reload() -> + emqx_rule_registry:load_hooks_for_rule(emqx_rule_registry:get_rules()). + +load(<<"$bridges/", _ChannelId/binary>> = BridgeTopic) -> + emqx_hooks:put(BridgeTopic, {?MODULE, on_bridge_message_received, + [#{bridge_topic => BridgeTopic}]}); load(Topic) -> HookPoint = event_name(Topic), emqx_hooks:put(HookPoint, {?MODULE, hook_fun(HookPoint), [[]]}). @@ -77,12 +85,17 @@ unload(Topic) -> %%-------------------------------------------------------------------- %% Callbacks %%-------------------------------------------------------------------- +on_bridge_message_received(Message, #{bridge_topic := BridgeTopic}) -> + case emqx_rule_registry:get_rules_for_topic(BridgeTopic) of + [] -> ok; + Rules -> emqx_rule_runtime:apply_rules(Rules, Message) + end. + on_message_publish(Message = #message{topic = Topic}, _Env) -> case ignore_sys_message(Message) of - true -> - ok; + true -> ok; false -> - case emqx_rule_registry:get_rules_for(Topic) of + case emqx_rule_registry:get_rules_for_topic(Topic) of [] -> ok; Rules -> emqx_rule_runtime:apply_rules(Rules, eventmsg_publish(Message)) end @@ -297,7 +310,7 @@ with_basic_columns(EventName, Data) when is_map(Data) -> %%-------------------------------------------------------------------- apply_event(EventName, GenEventMsg, _Env) -> EventTopic = event_topic(EventName), - case emqx_rule_registry:get_rules_for(EventTopic) of + case emqx_rule_registry:get_rules_for_topic(EventTopic) of [] -> ok; Rules -> emqx_rule_runtime:apply_rules(Rules, GenEventMsg()) end. diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index f73858f32..fd922db86 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -17,6 +17,8 @@ -module(emqx_rule_funcs). -include("rule_engine.hrl"). +-include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/logger.hrl"). %% IoT Funcs -export([ msgid/0 @@ -313,8 +315,10 @@ null() -> '+'(X, Y) when is_number(X), is_number(Y) -> X + Y; -%% concat 2 strings -'+'(X, Y) when is_binary(X), is_binary(Y) -> +%% string concatenation +%% this requires one of the arguments is string, the other argument will be converted +%% to string automatically (implicit conversion) +'+'(X, Y) when is_binary(X); is_binary(Y) -> concat(X, Y). '-'(X, Y) when is_number(X), is_number(Y) -> @@ -615,8 +619,9 @@ tokens(S, Separators) -> tokens(S, Separators, <<"nocrlf">>) -> [list_to_binary(R) || R <- string:lexemes(binary_to_list(S), binary_to_list(Separators) ++ [$\r,$\n,[$\r,$\n]])]. -concat(S1, S2) when is_binary(S1), is_binary(S2) -> - unicode:characters_to_binary([S1, S2], unicode). +%% implicit convert args to strings, and then do concatenation +concat(S1, S2) -> + unicode:characters_to_binary([str(S1), str(S2)], unicode). sprintf_s(Format, Args) when is_list(Args) -> erlang:iolist_to_binary(io_lib:format(binary_to_list(Format), Args)). diff --git a/apps/emqx_rule_engine/src/emqx_rule_metrics.erl b/apps/emqx_rule_engine/src/emqx_rule_metrics.erl index 8db444a7c..990911801 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_metrics.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_metrics.erl @@ -26,42 +26,17 @@ ]). -export([ get_rules_matched/1 - , get_actions_taken/1 - , get_actions_success/1 - , get_actions_error/1 - , get_actions_exception/1 - , get_actions_retry/1 - ]). - --export([ inc_rules_matched/1 - , inc_rules_matched/2 - , inc_actions_taken/1 - , inc_actions_taken/2 - , inc_actions_success/1 - , inc_actions_success/2 - , inc_actions_error/1 - , inc_actions_error/2 - , inc_actions_exception/1 - , inc_actions_exception/2 - , inc_actions_retry/1 - , inc_actions_retry/2 ]). -export([ inc/2 , inc/3 , get/2 - , get_overall/1 , get_rule_speed/1 - , get_overall_rule_speed/0 , create_rule_metrics/1 - , create_metrics/1 , clear_rule_metrics/1 - , clear_metrics/1 - , overall_metrics/0 ]). -export([ get_rule_metrics/1 - , get_action_metrics/1 ]). %% gen_server callbacks @@ -82,7 +57,7 @@ -define(SAMPLING, 1). -endif. --define(CRefID(ID), {?MODULE, ID}). +-define(CntrRef, ?MODULE). -define(SAMPCOUNT_5M, (?SECS_5M div ?SAMPLING)). -record(rule_speed, { @@ -99,48 +74,32 @@ -record(state, { metric_ids = sets:new(), - rule_speeds :: undefined | #{rule_id() => #rule_speed{}}, - overall_rule_speed :: #rule_speed{} + rule_speeds :: undefined | #{rule_id() => #rule_speed{}} }). %%------------------------------------------------------------------------------ %% APIs %%------------------------------------------------------------------------------ + -spec(create_rule_metrics(rule_id()) -> ok). create_rule_metrics(Id) -> gen_server:call(?MODULE, {create_rule_metrics, Id}). --spec(create_metrics(rule_id()) -> ok). -create_metrics(Id) -> - gen_server:call(?MODULE, {create_metrics, Id}). - -spec(clear_rule_metrics(rule_id()) -> ok). clear_rule_metrics(Id) -> gen_server:call(?MODULE, {delete_rule_metrics, Id}). --spec(clear_metrics(rule_id()) -> ok). -clear_metrics(Id) -> - gen_server:call(?MODULE, {delete_metrics, Id}). - -spec(get(rule_id(), atom()) -> number()). get(Id, Metric) -> - case couters_ref(Id) of + case get_couters_ref(Id) of not_found -> 0; Ref -> counters:get(Ref, metrics_idx(Metric)) end. --spec(get_overall(atom()) -> number()). -get_overall(Metric) -> - emqx_metrics:val(Metric). - -spec(get_rule_speed(rule_id()) -> map()). get_rule_speed(Id) -> gen_server:call(?MODULE, {get_rule_speed, Id}). --spec(get_overall_rule_speed() -> map()). -get_overall_rule_speed() -> - gen_server:call(?MODULE, get_overall_rule_speed). - -spec(get_rule_metrics(rule_id()) -> map()). get_rule_metrics(Id) -> #{max := Max, current := Current, last5m := Last5M} = get_rule_speed(Id), @@ -150,95 +109,34 @@ get_rule_metrics(Id) -> speed_last5m => Last5M }. --spec(get_action_metrics(action_instance_id()) -> map()). -get_action_metrics(Id) -> - #{success => get_actions_success(Id), - failed => get_actions_error(Id) + get_actions_exception(Id), - taken => get_actions_taken(Id) - }. - -spec inc(rule_id(), atom()) -> ok. inc(Id, Metric) -> inc(Id, Metric, 1). -spec inc(rule_id(), atom(), pos_integer()) -> ok. inc(Id, Metric, Val) -> - case couters_ref(Id) of + case get_couters_ref(Id) of not_found -> %% this may occur when increasing a counter for %% a rule that was created from a remove node. - case atom_to_list(Metric) of - "rules." ++ _ -> create_rule_metrics(Id); - _ -> create_metrics(Id) - end, - counters:add(couters_ref(Id), metrics_idx(Metric), Val); + create_rule_metrics(Id), + counters:add(get_couters_ref(Id), metrics_idx(Metric), Val); Ref -> counters:add(Ref, metrics_idx(Metric), Val) - end, - inc_overall(Metric, Val). - --spec(inc_overall(atom(), pos_integer()) -> ok). -inc_overall(Metric, Val) -> - emqx_metrics:inc(Metric, Val). - -inc_rules_matched(Id) -> - inc_rules_matched(Id, 1). -inc_rules_matched(Id, Val) -> - inc(Id, 'rules.matched', Val). - -inc_actions_taken(Id) -> - inc_actions_taken(Id, 1). -inc_actions_taken(Id, Val) -> - inc(Id, 'actions.taken', Val). - -inc_actions_success(Id) -> - inc_actions_success(Id, 1). -inc_actions_success(Id, Val) -> - inc(Id, 'actions.success', Val). - -inc_actions_error(Id) -> - inc_actions_error(Id, 1). -inc_actions_error(Id, Val) -> - inc(Id, 'actions.error', Val). - -inc_actions_exception(Id) -> - inc_actions_exception(Id, 1). -inc_actions_exception(Id, Val) -> - inc(Id, 'actions.exception', Val). - -inc_actions_retry(Id) -> - inc_actions_retry(Id, 1). -inc_actions_retry(Id, Val) -> - inc(Id, 'actions.retry', Val). + end. get_rules_matched(Id) -> get(Id, 'rules.matched'). -get_actions_taken(Id) -> - get(Id, 'actions.taken'). - -get_actions_success(Id) -> - get(Id, 'actions.success'). - -get_actions_error(Id) -> - get(Id, 'actions.error'). - -get_actions_exception(Id) -> - get(Id, 'actions.exception'). - -get_actions_retry(Id) -> - get(Id, 'actions.retry'). - start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). init([]) -> erlang:process_flag(trap_exit, true), - %% the overall counters - [ok = emqx_metrics:ensure(Metric)|| Metric <- overall_metrics()], %% the speed metrics erlang:send_after(timer:seconds(?SAMPLING), self(), ticking), - {ok, #state{overall_rule_speed = #rule_speed{}}}. + persistent_term:put(?CntrRef, #{}), + {ok, #state{}}. handle_call({get_rule_speed, _Id}, _From, State = #state{rule_speeds = undefined}) -> {reply, format_rule_speed(#rule_speed{}), State}; @@ -248,12 +146,6 @@ handle_call({get_rule_speed, Id}, _From, State = #state{rule_speeds = RuleSpeeds Speed -> format_rule_speed(Speed) end, State}; -handle_call(get_overall_rule_speed, _From, State = #state{overall_rule_speed = RuleSpeed}) -> - {reply, format_rule_speed(RuleSpeed), State}; - -handle_call({create_metrics, Id}, _From, State = #state{metric_ids = MIDs}) -> - {reply, create_counters(Id), State#state{metric_ids = sets:add_element(Id, MIDs)}}; - handle_call({create_rule_metrics, Id}, _From, State = #state{metric_ids = MIDs, rule_speeds = RuleSpeeds}) -> {reply, create_counters(Id), @@ -263,10 +155,6 @@ handle_call({create_rule_metrics, Id}, _From, _ -> RuleSpeeds#{Id => #rule_speed{}} end}}; -handle_call({delete_metrics, Id}, _From, - State = #state{metric_ids = MIDs, rule_speeds = undefined}) -> - {reply, delete_counters(Id), State#state{metric_ids = sets:del_element(Id, MIDs)}}; - handle_call({delete_rule_metrics, Id}, _From, State = #state{metric_ids = MIDs, rule_speeds = RuleSpeeds}) -> {reply, delete_counters(Id), @@ -283,21 +171,16 @@ handle_cast(_Msg, State) -> {noreply, State}. handle_info(ticking, State = #state{rule_speeds = undefined}) -> - async_refresh_resource_status(), erlang:send_after(timer:seconds(?SAMPLING), self(), ticking), {noreply, State}; -handle_info(ticking, State = #state{rule_speeds = RuleSpeeds0, - overall_rule_speed = OverallRuleSpeed0}) -> +handle_info(ticking, State = #state{rule_speeds = RuleSpeeds0}) -> RuleSpeeds = maps:map( fun(Id, RuleSpeed) -> calculate_speed(get_rules_matched(Id), RuleSpeed) end, RuleSpeeds0), - OverallRuleSpeed = calculate_speed(get_overall('rules.matched'), OverallRuleSpeed0), - async_refresh_resource_status(), erlang:send_after(timer:seconds(?SAMPLING), self(), ticking), - {noreply, State#state{rule_speeds = RuleSpeeds, - overall_rule_speed = OverallRuleSpeed}}; + {noreply, State#state{rule_speeds = RuleSpeeds}}; handle_info(_Info, State) -> {noreply, State}. @@ -307,7 +190,7 @@ code_change(_OldVsn, State, _Extra) -> terminate(_Reason, #state{metric_ids = MIDs}) -> [delete_counters(Id) || Id <- sets:to_list(MIDs)], - persistent_term:erase(?MODULE). + persistent_term:erase(?CntrRef). stop() -> gen_server:stop(?MODULE). @@ -316,26 +199,23 @@ stop() -> %% Internal Functions %%------------------------------------------------------------------------------ -async_refresh_resource_status() -> - spawn(emqx_rule_engine, refresh_resource_status, []). - create_counters(Id) -> - case couters_ref(Id) of + case get_couters_ref(Id) of not_found -> - ok = persistent_term:put(?CRefID(Id), - counters:new(max_counters_size(), [write_concurrency])); + Counters = get_all_counters(), + CntrRef = counters:new(max_counters_size(), [write_concurrency]), + persistent_term:put(?CntrRef, Counters#{Id => CntrRef}); _Ref -> ok end. delete_counters(Id) -> - persistent_term:erase(?CRefID(Id)), - ok. + persistent_term:put(?CntrRef, maps:remove(Id, get_all_counters())). -couters_ref(Id) -> - try persistent_term:get(?CRefID(Id)) - catch - error:badarg -> not_found - end. +get_couters_ref(Id) -> + maps:get(Id, get_all_counters(), not_found). + +get_all_counters() -> + persistent_term:get(?CntrRef, #{}). calculate_speed(_CurrVal, undefined) -> undefined; @@ -379,21 +259,7 @@ precision(Float, N) -> %% Metrics Definitions %%------------------------------------------------------------------------------ -max_counters_size() -> 7. - +max_counters_size() -> 2. metrics_idx('rules.matched') -> 1; -metrics_idx('actions.success') -> 2; -metrics_idx('actions.error') -> 3; -metrics_idx('actions.taken') -> 4; -metrics_idx('actions.exception') -> 5; -metrics_idx('actions.retry') -> 6; -metrics_idx(_) -> 7. +metrics_idx(_) -> 2. -overall_metrics() -> - [ 'rules.matched' - , 'actions.success' - , 'actions.error' - , 'actions.taken' - , 'actions.exception' - , 'actions.retry' - ]. diff --git a/apps/emqx_rule_engine/src/emqx_rule_monitor.erl b/apps/emqx_rule_engine/src/emqx_rule_monitor.erl deleted file mode 100644 index dd2f6237c..000000000 --- a/apps/emqx_rule_engine/src/emqx_rule_monitor.erl +++ /dev/null @@ -1,126 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_rule_monitor). - --behavior(gen_server). - --include("rule_engine.hrl"). --include_lib("emqx/include/logger.hrl"). - --export([init/1, - handle_call/3, - handle_cast/2, - handle_info/2, - terminate/2, - code_change/3]). - --export([ start_link/0 - , stop/0 - , ensure_resource_retrier/2 - , retry_loop/3 - ]). - -start_link() -> - gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). - -stop() -> - gen_server:stop(?MODULE). - -init([]) -> - _ = erlang:process_flag(trap_exit, true), - {ok, #{retryers => #{}}}. - -ensure_resource_retrier(ResId, Interval) -> - gen_server:cast(?MODULE, {create_restart_handler, resource, ResId, Interval}). - -handle_call(_Msg, _From, State) -> - {reply, ok, State}. - -handle_cast({create_restart_handler, Tag, Obj, Interval}, State) -> - Objects = maps:get(Tag, State, #{}), - NewState = case maps:find(Obj, Objects) of - error -> - update_object(Tag, Obj, - create_restart_handler(Tag, Obj, Interval), State); - {ok, _Pid} -> - State - end, - {noreply, NewState}; - -handle_cast(_Msg, State) -> - {noreply, State}. - -handle_info({'EXIT', Pid, Reason}, State = #{retryers := Retryers}) -> - case maps:take(Pid, Retryers) of - {{Tag, Obj}, Retryers2} -> - Objects = maps:get(Tag, State, #{}), - {noreply, State#{Tag => maps:remove(Obj, Objects), - retryers => Retryers2}}; - error -> - ?LOG(error, "got unexpected proc down: ~p ~p", [Pid, Reason]), - {noreply, State} - end; - -handle_info(_Info, State) -> - {noreply, State}. - -terminate(_Reason, _State) -> - ok. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -update_object(Tag, Obj, Retryer, State) -> - Objects = maps:get(Tag, State, #{}), - Retryers = maps:get(retryers, State, #{}), - State#{ - Tag => Objects#{Obj => Retryer}, - retryers => Retryers#{Retryer => {Tag, Obj}} - }. - -create_restart_handler(Tag, Obj, Interval) -> - ?LOG(info, "keep restarting ~p ~p, interval: ~p", [Tag, Obj, Interval]), - %% spawn a dedicated process to handle the restarting asynchronously - spawn_link(?MODULE, retry_loop, [Tag, Obj, Interval]). - -retry_loop(resource, ResId, Interval) -> - case emqx_rule_registry:find_resource(ResId) of - {ok, #resource{type = Type, config = Config}} -> - try - {ok, #resource_type{on_create = {M, F}}} = - emqx_rule_registry:find_resource_type(Type), - ok = emqx_rule_engine:init_resource(M, F, ResId, Config), - refresh_and_enable_rules_of_resource(ResId) - catch - Err:Reason:ST -> - ?LOG(warning, "init_resource failed: ~p, ~0p", - [{Err, Reason}, ST]), - timer:sleep(Interval), - ?MODULE:retry_loop(resource, ResId, Interval) - end; - not_found -> - ok - end. - -refresh_and_enable_rules_of_resource(ResId) -> - lists:foreach( - fun (#rule{id = Id, enabled = false, state = refresh_failed_at_bootup} = Rule) -> - emqx_rule_engine:refresh_rule(Rule), - emqx_rule_registry:add_rule(Rule#rule{enabled = true, state = normal}), - ?LOG(info, "rule ~s is refreshed and re-enabled", [Id]); - (_) -> ok - end, emqx_rule_registry:find_rules_depends_on_resource(ResId)). diff --git a/apps/emqx_rule_engine/src/emqx_rule_outputs.erl b/apps/emqx_rule_engine/src/emqx_rule_outputs.erl new file mode 100644 index 000000000..cd59d3fa5 --- /dev/null +++ b/apps/emqx_rule_engine/src/emqx_rule_outputs.erl @@ -0,0 +1,82 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% Define the default actions. +-module(emqx_rule_outputs). +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/emqx.hrl"). + +-export([ console/3 + , republish/3 + ]). + +-spec console(map(), map(), map()) -> any(). +console(Selected, #{metadata := #{rule_id := RuleId}} = Envs, _Args) -> + ?ULOG("[rule output] ~s~n" + "\tOutput Data: ~p~n" + "\tEnvs: ~p~n", [RuleId, Selected, Envs]). + +republish(_Selected, #{topic := Topic, headers := #{republish_by := RuleId}, + metadata := #{rule_id := RuleId}}, _Args) -> + ?SLOG(error, #{msg => "recursive_republish_detected", topic => Topic}); + +%% republish a PUBLISH message +republish(Selected, #{flags := Flags, metadata := #{rule_id := RuleId}}, + #{preprocessed_tmpl := #{ + qos := QoSTks, + retain := RetainTks, + topic := TopicTks, + payload := PayloadTks}}) -> + Topic = emqx_plugin_libs_rule:proc_tmpl(TopicTks, Selected), + Payload = emqx_plugin_libs_rule:proc_tmpl(PayloadTks, Selected), + QoS = replace_simple_var(QoSTks, Selected), + Retain = replace_simple_var(RetainTks, Selected), + ?SLOG(debug, #{msg => "republish", topic => Topic, payload => Payload}), + safe_publish(RuleId, Topic, QoS, Flags#{retain => Retain}, Payload); + +%% in case this is a "$events/" event +republish(Selected, #{metadata := #{rule_id := RuleId}}, + #{preprocessed_tmpl := #{ + qos := QoSTks, + retain := RetainTks, + topic := TopicTks, + payload := PayloadTks}}) -> + Topic = emqx_plugin_libs_rule:proc_tmpl(TopicTks, Selected), + Payload = emqx_plugin_libs_rule:proc_tmpl(PayloadTks, Selected), + QoS = replace_simple_var(QoSTks, Selected), + Retain = replace_simple_var(RetainTks, Selected), + ?SLOG(debug, #{msg => "republish", topic => Topic, payload => Payload}), + safe_publish(RuleId, Topic, QoS, #{retain => Retain}, Payload). + +safe_publish(RuleId, Topic, QoS, Flags, Payload) -> + Msg = #message{ + id = emqx_guid:gen(), + qos = QoS, + from = RuleId, + flags = Flags, + headers = #{republish_by => RuleId}, + topic = Topic, + payload = Payload, + timestamp = erlang:system_time(millisecond) + }, + _ = emqx_broker:safe_publish(Msg), + emqx_metrics:inc_msg(Msg). + +replace_simple_var(Tokens, Data) when is_list(Tokens) -> + [Var] = emqx_plugin_libs_rule:proc_tmpl(Tokens, Data, #{return => rawlist}), + Var; +replace_simple_var(Val, _Data) -> + Val. diff --git a/apps/emqx_rule_engine/src/emqx_rule_registry.erl b/apps/emqx_rule_engine/src/emqx_rule_registry.erl index a0b8b48d5..aa2c97c76 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_registry.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_registry.erl @@ -19,7 +19,6 @@ -behaviour(gen_server). -include("rule_engine.hrl"). --include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("stdlib/include/qlc.hrl"). @@ -27,7 +26,7 @@ %% Rule Management -export([ get_rules/0 - , get_rules_for/1 + , get_rules_for_topic/1 , get_rules_with_same_event/1 , get_rules_ordered_by_ts/0 , get_rule/1 @@ -37,39 +36,6 @@ , remove_rules/1 ]). -%% Action Management --export([ add_action/1 - , add_actions/1 - , get_actions/0 - , find_action/1 - , remove_action/1 - , remove_actions/1 - , remove_actions_of/1 - , add_action_instance_params/1 - , get_action_instance_params/1 - , remove_action_instance_params/1 - ]). - -%% Resource Management --export([ get_resources/0 - , add_resource/1 - , add_resource_params/1 - , find_resource/1 - , find_resource_params/1 - , get_resources_by_type/1 - , remove_resource/1 - , remove_resource_params/1 - ]). - -%% Resource Types --export([ get_resource_types/0 - , find_resource_type/1 - , find_rules_depends_on_resource/1 - , find_enabled_rules_depends_on_resource/1 - , register_resource_types/1 - , unregister_resource_types_of/1 - ]). - -export([ load_hooks_for_rule/1 , unload_hooks_for_rule/1 ]). @@ -110,53 +76,14 @@ mnesia(boot) -> {rlog_shard, ?RULE_ENGINE_SHARD}, {disc_copies, [node()]}, {record_name, rule}, - {index, [#rule.for]}, {attributes, record_info(fields, rule)}, - {storage_properties, StoreProps}]), - %% Rule action table - ok = ekka_mnesia:create_table(?ACTION_TAB, [ - {rlog_shard, ?RULE_ENGINE_SHARD}, - {ram_copies, [node()]}, - {record_name, action}, - {index, [#action.for, #action.app]}, - {attributes, record_info(fields, action)}, - {storage_properties, StoreProps}]), - %% Resource table - ok = ekka_mnesia:create_table(?RES_TAB, [ - {rlog_shard, ?RULE_ENGINE_SHARD}, - {disc_copies, [node()]}, - {record_name, resource}, - {index, [#resource.type]}, - {attributes, record_info(fields, resource)}, - {storage_properties, StoreProps}]), - %% Resource type table - ok = ekka_mnesia:create_table(?RES_TYPE_TAB, [ - {rlog_shard, ?RULE_ENGINE_SHARD}, - {ram_copies, [node()]}, - {record_name, resource_type}, - {index, [#resource_type.provider]}, - {attributes, record_info(fields, resource_type)}, {storage_properties, StoreProps}]); mnesia(copy) -> - %% Copy rule table - ok = ekka_mnesia:copy_table(?RULE_TAB, disc_copies), - %% Copy rule action table - ok = ekka_mnesia:copy_table(?ACTION_TAB, ram_copies), - %% Copy resource table - ok = ekka_mnesia:copy_table(?RES_TAB, disc_copies), - %% Copy resource type table - ok = ekka_mnesia:copy_table(?RES_TYPE_TAB, ram_copies). + ok = ekka_mnesia:copy_table(?RULE_TAB, disc_copies). dump() -> - ?ULOG("Rules: ~p~n" - "ActionInstParams: ~p~n" - "Resources: ~p~n" - "ResourceParams: ~p~n", - [ets:tab2list(?RULE_TAB), - ets:tab2list(?ACTION_INST_PARAMS_TAB), - ets:tab2list(?RES_TAB), - ets:tab2list(?RES_PARAMS_TAB)]). + ?ULOG("Rules: ~p~n", [ets:tab2list(?RULE_TAB)]). %%------------------------------------------------------------------------------ %% Start the registry @@ -182,16 +109,16 @@ get_rules_ordered_by_ts() -> {atomic, List} = ekka_mnesia:transaction(?RULE_ENGINE_SHARD, F), List. --spec(get_rules_for(Topic :: binary()) -> list(emqx_rule_engine:rule())). -get_rules_for(Topic) -> - [Rule || Rule = #rule{for = For} <- get_rules(), - emqx_plugin_libs_rule:can_topic_match_oneof(Topic, For)]. +-spec(get_rules_for_topic(Topic :: binary()) -> list(emqx_rule_engine:rule())). +get_rules_for_topic(Topic) -> + [Rule || Rule = #rule{info = #{from := From}} <- get_rules(), + emqx_plugin_libs_rule:can_topic_match_oneof(Topic, From)]. -spec(get_rules_with_same_event(Topic :: binary()) -> list(emqx_rule_engine:rule())). get_rules_with_same_event(Topic) -> EventName = emqx_rule_events:event_name(Topic), - [Rule || Rule = #rule{for = For} <- get_rules(), - lists:any(fun(T) -> is_of_event_name(EventName, T) end, For)]. + [Rule || Rule = #rule{info = #{from := From}} <- get_rules(), + lists:any(fun(T) -> is_of_event_name(EventName, T) end, From)]. is_of_event_name(EventName, Topic) -> EventName =:= emqx_rule_events:event_name(Topic). @@ -223,7 +150,7 @@ remove_rules(Rules) -> insert_rules([]) -> ok; insert_rules(Rules) -> - _ = emqx_plugin_libs_rule:cluster_call(?MODULE, load_hooks_for_rule, [Rules]), + _ = emqx_plugin_libs_rule:cluster_call(?MODULE, load_hooks_for_rule, [Rules]), [mnesia:write(?RULE_TAB, Rule, write) ||Rule <- Rules]. %% @private @@ -235,7 +162,7 @@ delete_rules(Rules = [R|_]) when is_binary(R) -> {ok, Rule} -> [Rule|Acc]; not_found -> Acc end - end, [], Rules), + end, [], Rules), delete_rules_unload_hooks(RuleRecs); delete_rules(Rules = [Rule|_]) when is_record(Rule, rule) -> delete_rules_unload_hooks(Rules). @@ -245,209 +172,20 @@ delete_rules_unload_hooks(Rules) -> [mnesia:delete_object(?RULE_TAB, Rule, write) ||Rule <- Rules]. load_hooks_for_rule(Rules) -> - lists:foreach(fun(#rule{for = Topics}) -> - lists:foreach(fun emqx_rule_events:load/1, Topics) - end, Rules). + lists:foreach(fun(#rule{info = #{from := Topics}}) -> + lists:foreach(fun emqx_rule_events:load/1, Topics) + end, Rules). unload_hooks_for_rule(Rules) -> - lists:foreach(fun(#rule{id = Id, for = Topics}) -> + lists:foreach(fun(#rule{id = Id, info = #{from := Topics}}) -> lists:foreach(fun(Topic) -> case get_rules_with_same_event(Topic) of - [#rule{id = Id}] -> %% we are now deleting the last rule + [#rule{id = Id0}] when Id0 == Id -> %% we are now deleting the last rule emqx_rule_events:unload(Topic); _ -> ok end - end, Topics) - end, Rules). - -%%------------------------------------------------------------------------------ -%% Action Management -%%------------------------------------------------------------------------------ - -%% @doc Get all actions. --spec(get_actions() -> list(emqx_rule_engine:action())). -get_actions() -> - get_all_records(?ACTION_TAB). - -%% @doc Find an action by name. --spec(find_action(Name :: action_name()) -> {ok, emqx_rule_engine:action()} | not_found). -find_action(Name) -> - case mnesia:dirty_read(?ACTION_TAB, Name) of - [Action] -> {ok, Action}; - [] -> not_found - end. - -%% @doc Add an action. --spec(add_action(emqx_rule_engine:action()) -> ok). -add_action(Action) when is_record(Action, action) -> - trans(fun insert_action/1, [Action]). - -%% @doc Add actions. --spec(add_actions(list(emqx_rule_engine:action())) -> ok). -add_actions(Actions) when is_list(Actions) -> - trans(fun lists:foreach/2, [fun insert_action/1, Actions]). - -%% @doc Remove an action. --spec(remove_action(emqx_rule_engine:action() | atom()) -> ok). -remove_action(Action) when is_record(Action, action) -> - trans(fun delete_action/1, [Action]); - -remove_action(Name) -> - trans(fun mnesia:delete/1, [{?ACTION_TAB, Name}]). - -%% @doc Remove actions. --spec(remove_actions(list(emqx_rule_engine:action())) -> ok). -remove_actions(Actions) -> - trans(fun lists:foreach/2, [fun delete_action/1, Actions]). - -%% @doc Remove actions of the App. --spec(remove_actions_of(App :: atom()) -> ok). -remove_actions_of(App) -> - trans(fun() -> - lists:foreach(fun delete_action/1, mnesia:index_read(?ACTION_TAB, App, #action.app)) - end). - -%% @private -insert_action(Action) -> - mnesia:write(?ACTION_TAB, Action, write). - -%% @private -delete_action(Action) when is_record(Action, action) -> - mnesia:delete_object(?ACTION_TAB, Action, write); -delete_action(Name) when is_atom(Name) -> - mnesia:delete(?ACTION_TAB, Name, write). - -%% @doc Add an action instance params. --spec(add_action_instance_params(emqx_rule_engine:action_instance_params()) -> ok). -add_action_instance_params(ActionInstParams) when is_record(ActionInstParams, action_instance_params) -> - ets:insert(?ACTION_INST_PARAMS_TAB, ActionInstParams), - ok. - --spec(get_action_instance_params(action_instance_id()) -> {ok, emqx_rule_engine:action_instance_params()} | not_found). -get_action_instance_params(ActionInstId) -> - case ets:lookup(?ACTION_INST_PARAMS_TAB, ActionInstId) of - [ActionInstParams] -> {ok, ActionInstParams}; - [] -> not_found - end. - -%% @doc Delete an action instance params. --spec(remove_action_instance_params(action_instance_id()) -> ok). -remove_action_instance_params(ActionInstId) -> - ets:delete(?ACTION_INST_PARAMS_TAB, ActionInstId), - ok. - -%%------------------------------------------------------------------------------ -%% Resource Management -%%------------------------------------------------------------------------------ - --spec(get_resources() -> list(emqx_rule_engine:resource())). -get_resources() -> - get_all_records(?RES_TAB). - --spec(add_resource(emqx_rule_engine:resource()) -> ok). -add_resource(Resource) when is_record(Resource, resource) -> - trans(fun insert_resource/1, [Resource]). - --spec(add_resource_params(emqx_rule_engine:resource_params()) -> ok). -add_resource_params(ResParams) when is_record(ResParams, resource_params) -> - ets:insert(?RES_PARAMS_TAB, ResParams), - ok. - --spec(find_resource(Id :: resource_id()) -> {ok, emqx_rule_engine:resource()} | not_found). -find_resource(Id) -> - case mnesia:dirty_read(?RES_TAB, Id) of - [Res] -> {ok, Res}; - [] -> not_found - end. - --spec(find_resource_params(Id :: resource_id()) - -> {ok, emqx_rule_engine:resource_params()} | not_found). -find_resource_params(Id) -> - case ets:lookup(?RES_PARAMS_TAB, Id) of - [ResParams] -> {ok, ResParams}; - [] -> not_found - end. - --spec(remove_resource(emqx_rule_engine:resource() | emqx_rule_engine:resource_id()) -> ok | {error, term()}). -remove_resource(Resource) when is_record(Resource, resource) -> - trans(fun delete_resource/1, [Resource#resource.id]); - -remove_resource(ResId) when is_binary(ResId) -> - trans(fun delete_resource/1, [ResId]). - --spec(remove_resource_params(emqx_rule_engine:resource_id()) -> ok). -remove_resource_params(ResId) -> - ets:delete(?RES_PARAMS_TAB, ResId), - ok. - -%% @private -delete_resource(ResId) -> - case find_enabled_rules_depends_on_resource(ResId) of - [] -> mnesia:delete(?RES_TAB, ResId, write); - Rules -> - {error, {dependent_rules_exists, [Id || #rule{id = Id} <- Rules]}} - end. - -%% @private -insert_resource(Resource) -> - mnesia:write(?RES_TAB, Resource, write). - -find_enabled_rules_depends_on_resource(ResId) -> - [R || #rule{enabled = true} = R <- find_rules_depends_on_resource(ResId)]. - -find_rules_depends_on_resource(ResId) -> - lists:foldl(fun(#rule{actions = Actions} = R, Rules) -> - case search_action_despends_on_resource(ResId, Actions) of - false -> Rules; - {value, _} -> [R | Rules] - end - end, [], get_rules()). - -search_action_despends_on_resource(ResId, Actions) -> - lists:search(fun - (#action_instance{args = #{<<"$resource">> := ResId0}}) -> - ResId0 =:= ResId; - (_) -> - false - end, Actions). - -%%------------------------------------------------------------------------------ -%% Resource Type Management -%%------------------------------------------------------------------------------ - --spec(get_resource_types() -> list(emqx_rule_engine:resource_type())). -get_resource_types() -> - get_all_records(?RES_TYPE_TAB). - --spec(find_resource_type(Name :: resource_type_name()) -> {ok, emqx_rule_engine:resource_type()} | not_found). -find_resource_type(Name) -> - case mnesia:dirty_read(?RES_TYPE_TAB, Name) of - [ResType] -> {ok, ResType}; - [] -> not_found - end. - --spec(get_resources_by_type(Type :: resource_type_name()) -> list(emqx_rule_engine:resource())). -get_resources_by_type(Type) -> - mnesia:dirty_index_read(?RES_TAB, Type, #resource.type). - --spec(register_resource_types(list(emqx_rule_engine:resource_type())) -> ok). -register_resource_types(Types) -> - trans(fun lists:foreach/2, [fun insert_resource_type/1, Types]). - -%% @doc Unregister resource types of the App. --spec(unregister_resource_types_of(App :: atom()) -> ok). -unregister_resource_types_of(App) -> - trans(fun() -> - lists:foreach(fun delete_resource_type/1, mnesia:index_read(?RES_TYPE_TAB, App, #resource_type.provider)) - end). - -%% @private -insert_resource_type(Type) -> - mnesia:write(?RES_TYPE_TAB, Type, write). - -%% @private -delete_resource_type(Type) -> - mnesia:delete_object(?RES_TYPE_TAB, Type, write). + end, Topics) + end, Rules). %%------------------------------------------------------------------------------ %% gen_server callbacks @@ -467,15 +205,15 @@ handle_call({remove_rules, Rules}, _From, State) -> {reply, ok, State}; handle_call(Req, _From, State) -> - ?LOG(error, "[RuleRegistry]: unexpected call - ~p", [Req]), + ?SLOG(error, #{msg => "unexpected_call", request => Req}), {reply, ignored, State}. handle_cast(Msg, State) -> - ?LOG(error, "[RuleRegistry]: unexpected cast ~p", [Msg]), + ?SLOG(error, #{msg => "unexpected_cast", request => Msg}), {noreply, State}. handle_info(Info, State) -> - ?LOG(error, "[RuleRegistry]: unexpected info ~p", [Info]), + ?SLOG(error, #{msg => "unexpected_info", request => Info}), {noreply, State}. terminate(_Reason, _State) -> @@ -500,7 +238,6 @@ get_all_records(Tab) -> end), Ret. -trans(Fun) -> trans(Fun, []). trans(Fun, Args) -> case ekka_mnesia:transaction(?RULE_ENGINE_SHARD, Fun, Args) of {atomic, Result} -> Result; diff --git a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl index f9e210ab3..aafc6cddc 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl @@ -33,9 +33,9 @@ -compile({no_auto_import,[alias/1]}). --type(input() :: map()). --type(alias() :: atom()). --type(collection() :: {alias(), [term()]}). +-type input() :: map(). +-type alias() :: atom(). +-type collection() :: {alias(), [term()]}. -define(ephemeral_alias(TYPE, NAME), iolist_to_binary(io_lib:format("_v_~s_~p_~p", [TYPE, NAME, erlang:system_time()]))). @@ -48,27 +48,31 @@ -spec(apply_rules(list(emqx_rule_engine:rule()), input()) -> ok). apply_rules([], _Input) -> ok; -apply_rules([#rule{enabled = false}|More], Input) -> +apply_rules([#rule{info = #{enabled := false}}|More], Input) -> apply_rules(More, Input); apply_rules([Rule = #rule{id = RuleID}|More], Input) -> try apply_rule_discard_result(Rule, Input) catch %% ignore the errors if select or match failed _:{select_and_transform_error, Error} -> - ?LOG(warning, "SELECT clause exception for ~s failed: ~p", - [RuleID, Error]); + ?SLOG(warning, #{msg => "SELECT_clause_exception", + rule_id => RuleID, reason => Error}); _:{match_conditions_error, Error} -> - ?LOG(warning, "WHERE clause exception for ~s failed: ~p", - [RuleID, Error]); + ?SLOG(warning, #{msg => "WHERE_clause_exception", + rule_id => RuleID, reason => Error}); _:{select_and_collect_error, Error} -> - ?LOG(warning, "FOREACH clause exception for ~s failed: ~p", - [RuleID, Error]); + ?SLOG(warning, #{msg => "FOREACH_clause_exception", + rule_id => RuleID, reason => Error}); _:{match_incase_error, Error} -> - ?LOG(warning, "INCASE clause exception for ~s failed: ~p", - [RuleID, Error]); - _:Error:StkTrace -> - ?LOG(error, "Apply rule ~s failed: ~p. Stacktrace:~n~p", - [RuleID, Error, StkTrace]) + ?SLOG(warning, #{msg => "INCASE_clause_exception", + rule_id => RuleID, reason => Error}); + Class:Error:StkTrace -> + ?SLOG(error, #{msg => "apply_rule_failed", + rule_id => RuleID, + exception => Class, + reason => Error, + stacktrace => StkTrace + }) end, apply_rules(More, Input). @@ -80,14 +84,14 @@ apply_rule(Rule = #rule{id = RuleID}, Input) -> clear_rule_payload(), do_apply_rule(Rule, add_metadata(Input, #{rule_id => RuleID})). -do_apply_rule(#rule{id = RuleId, - is_foreach = true, - fields = Fields, - doeach = DoEach, - incase = InCase, - conditions = Conditions, - on_action_failed = OnFailed, - actions = Actions}, Input) -> +do_apply_rule(#rule{id = RuleId, info = #{ + is_foreach := true, + fields := Fields, + doeach := DoEach, + incase := InCase, + conditions := Conditions, + outputs := Outputs + }}, Input) -> {Selected, Collection} = ?RAISE(select_and_collect(Fields, Input), {select_and_collect_error, {_EXCLASS_,_EXCPTION_,_ST_}}), ColumnsAndSelected = maps:merge(Input, Selected), @@ -96,24 +100,24 @@ do_apply_rule(#rule{id = RuleId, true -> ok = emqx_rule_metrics:inc(RuleId, 'rules.matched'), Collection2 = filter_collection(Input, InCase, DoEach, Collection), - {ok, [take_actions(Actions, Coll, Input, OnFailed) || Coll <- Collection2]}; + {ok, [handle_output_list(Outputs, Coll, Input) || Coll <- Collection2]}; false -> {error, nomatch} end; -do_apply_rule(#rule{id = RuleId, - is_foreach = false, - fields = Fields, - conditions = Conditions, - on_action_failed = OnFailed, - actions = Actions}, Input) -> +do_apply_rule(#rule{id = RuleId, info = #{ + is_foreach := false, + fields := Fields, + conditions := Conditions, + outputs := Outputs + }}, Input) -> Selected = ?RAISE(select_and_transform(Fields, Input), {select_and_transform_error, {_EXCLASS_,_EXCPTION_,_ST_}}), case ?RAISE(match_conditions(Conditions, maps:merge(Input, Selected)), {match_conditions_error, {_EXCLASS_,_EXCPTION_,_ST_}}) of true -> ok = emqx_rule_metrics:inc(RuleId, 'rules.matched'), - {ok, take_actions(Actions, Selected, Input, OnFailed)}; + {ok, handle_output_list(Outputs, Selected, Input)}; false -> {error, nomatch} end. @@ -166,7 +170,6 @@ select_and_collect([Field|More], Input, {Output, LastKV}) -> {nested_put(Key, Val, Output), LastKV}). %% Filter each item got from FOREACH --dialyzer({nowarn_function, filter_collection/4}). filter_collection(Input, InCase, DoEach, {CollKey, CollVal}) -> lists:filtermap( fun(Item) -> @@ -198,8 +201,6 @@ match_conditions({'fun', {_, Name}, Args}, Data) -> apply_func(Name, [eval(Arg, Data) || Arg <- Args], Data); match_conditions({Op, L, R}, Data) when ?is_comp(Op) -> compare(Op, eval(L, Data), eval(R, Data)); -%%match_conditions({'like', Var, Pattern}, Data) -> -%% match_like(eval(Var, Data), Pattern); match_conditions({}, _Data) -> true. @@ -229,81 +230,43 @@ number(Bin) -> catch error:badarg -> binary_to_float(Bin) end. -%% Step3 -> Take actions -take_actions(Actions, Selected, Envs, OnFailed) -> - [take_action(ActInst, Selected, Envs, OnFailed, ?ActionMaxRetry) - || ActInst <- Actions]. +handle_output_list(Outputs, Selected, Envs) -> + [handle_output(Out, Selected, Envs) || Out <- Outputs]. -take_action(#action_instance{id = Id, name = ActName, fallbacks = Fallbacks} = ActInst, - Selected, Envs, OnFailed, RetryN) when RetryN >= 0 -> +handle_output(OutId, Selected, Envs) -> try - {ok, #action_instance_params{apply = Apply}} - = emqx_rule_registry:get_action_instance_params(Id), - emqx_rule_metrics:inc_actions_taken(Id), - apply_action_func(Selected, Envs, Apply, ActName) - of - {badact, Reason} -> - handle_action_failure(OnFailed, Id, Fallbacks, Selected, Envs, Reason); - Result -> Result + do_handle_output(OutId, Selected, Envs) catch - error:{badfun, _Func}:_ST -> - %?LOG(warning, "Action ~p maybe outdated, refresh it and try again." - % "Func: ~p~nST:~0p", [Id, Func, ST]), - _ = trans_action_on(Id, fun() -> - emqx_rule_engine:refresh_actions([ActInst]) - end, 5000), - emqx_rule_metrics:inc_actions_retry(Id), - take_action(ActInst, Selected, Envs, OnFailed, RetryN-1); - Error:Reason:Stack -> - emqx_rule_metrics:inc_actions_exception(Id), - handle_action_failure(OnFailed, Id, Fallbacks, Selected, Envs, {Error, Reason, Stack}) - end; - -take_action(#action_instance{id = Id, fallbacks = Fallbacks}, Selected, Envs, OnFailed, _RetryN) -> - emqx_rule_metrics:inc_actions_error(Id), - handle_action_failure(OnFailed, Id, Fallbacks, Selected, Envs, {max_try_reached, ?ActionMaxRetry}). - -apply_action_func(Data, Envs, #{mod := Mod, bindings := Bindings}, Name) -> - %% TODO: Build the Func Name when creating the action - Func = cbk_on_action_triggered(Name), - Mod:Func(Data, Envs#{'__bindings__' => Bindings}); -apply_action_func(Data, Envs, Func, _Name) when is_function(Func) -> - erlang:apply(Func, [Data, Envs]). - -cbk_on_action_triggered(Name) -> - list_to_atom("on_action_" ++ atom_to_list(Name)). - -trans_action_on(Id, Callback, Timeout) -> - case emqx_rule_locker:lock(Id) of - true -> try Callback() after emqx_rule_locker:unlock(Id) end; - _ -> - wait_action_on(Id, Timeout div 10) + Err:Reason:ST -> + ?SLOG(error, #{msg => "output_failed", + output => OutId, + exception => Err, + reason => Reason, + stacktrace => ST + }) end. -wait_action_on(_, 0) -> - {error, timeout}; -wait_action_on(Id, RetryN) -> - timer:sleep(10), - case emqx_rule_registry:get_action_instance_params(Id) of - not_found -> - {error, not_found}; - {ok, #action_instance_params{apply = Apply}} -> - case catch apply_action_func(baddata, #{}, Apply, tryit) of - {'EXIT', {{badfun, _}, _}} -> - wait_action_on(Id, RetryN-1); - _ -> - ok - end +do_handle_output(#{type := bridge, target := ChannelId}, Selected, _Envs) -> + ?SLOG(debug, #{msg => "output to bridge", channel_id => ChannelId}), + emqx_bridge:send_message(ChannelId, Selected); +do_handle_output(#{type := func, target := Func} = Out, Selected, Envs) -> + erlang:apply(Func, [Selected, Envs, maps:get(args, Out, #{})]); +do_handle_output(#{type := builtin, target := Output} = Out, Selected, Envs) + when is_atom(Output) -> + handle_builtin_output(Output, Selected, Envs, maps:get(args, Out, #{})); +do_handle_output(#{type := builtin, target := Output} = Out, Selected, Envs) + when is_binary(Output) -> + try binary_to_existing_atom(Output) of + Func -> handle_builtin_output(Func, Selected, Envs, maps:get(args, Out, #{})) + catch + error:badarg -> error(not_found) end. -handle_action_failure(continue, Id, Fallbacks, Selected, Envs, Reason) -> - ?LOG(error, "Take action ~p failed, continue next action, reason: ~0p", [Id, Reason]), - _ = take_actions(Fallbacks, Selected, Envs, continue), - failed; -handle_action_failure(stop, Id, Fallbacks, Selected, Envs, Reason) -> - ?LOG(error, "Take action ~p failed, skip all actions, reason: ~0p", [Id, Reason]), - _ = take_actions(Fallbacks, Selected, Envs, continue), - error({take_action_failed, {Id, Reason}}). +handle_builtin_output(Func, Selected, Envs, Args) -> + case erlang:function_exported(emqx_rule_outputs, Func, 3) of + true -> erlang:apply(emqx_rule_outputs, Func, [Selected, Envs, Args]); + false -> error(not_found) + end. eval({path, [{key, <<"payload">>} | Path]}, #{payload := Payload}) -> nested_get({path, Path}, may_decode_payload(Payload)); diff --git a/apps/emqx_rule_engine/src/emqx_rule_sqlparser.erl b/apps/emqx_rule_engine/src/emqx_rule_sqlparser.erl index 9a8ce55ea..02c5a02e9 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_sqlparser.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_sqlparser.erl @@ -18,7 +18,7 @@ -include("rule_engine.hrl"). --export([parse_select/1]). +-export([parse/1]). -export([ select_fields/1 , select_is_foreach/1 @@ -36,26 +36,21 @@ -opaque(select() :: #select{}). --type(const() :: {const, number()|binary()}). +-type const() :: {const, number()|binary()}. --type(variable() :: binary() | list(binary())). +-type variable() :: binary() | list(binary()). --type(alias() :: binary() | list(binary())). +-type alias() :: binary() | list(binary()). --type(field() :: const() | variable() +-type field() :: const() | variable() | {as, field(), alias()} - | {'fun', atom(), list(field())}). + | {'fun', atom(), list(field())}. -export_type([select/0]). -%% Dialyzer gives up on the generated code. -%% probably due to stack depth, or inlines. --dialyzer({nowarn_function, [parse_select/1]}). - %% Parse one select statement. --spec(parse_select(string() | binary()) - -> {ok, select()} | {parse_error, term()} | {lex_error, term()}). -parse_select(Sql) -> +-spec(parse(string() | binary()) -> {ok, select()} | {error, term()}). +parse(Sql) -> try case rulesql:parsetree(Sql) of {ok, {select, Clauses}} -> {ok, #select{ @@ -75,11 +70,11 @@ parse_select(Sql) -> from = get_value(from, Clauses), where = get_value(where, Clauses) }}; - Error -> Error + Error -> {error, Error} end catch _Error:Reason:StackTrace -> - {parse_error, {Reason, StackTrace}} + {error, {Reason, StackTrace}} end. -spec(select_fields(select()) -> list(field())). diff --git a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl index 2f1edbeb2..ec263b35a 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl @@ -18,22 +18,14 @@ -include_lib("emqx/include/logger.hrl"). -export([ test/1 + , echo_action/2 + , get_selected_data/3 ]). -%% Dialyzer gives up on the generated code. -%% probably due to stack depth, or inlines. --dialyzer({nowarn_function, [test/1, - test_rule/4, - flatten/1, - sql_test_action/0, - fill_default_values/2, - envs_examp/1 - ]}). - --spec(test(#{}) -> {ok, map() | list()} | {error, term()}). -test(#{<<"rawsql">> := Sql, <<"ctx">> := Context}) -> - {ok, Select} = emqx_rule_sqlparser:parse_select(Sql), - InTopic = maps:get(<<"topic">>, Context, <<>>), +-spec test(#{sql := binary(), context := map()}) -> {ok, map() | list()} | {error, nomatch}. +test(#{sql := Sql, context := Context}) -> + {ok, Select} = emqx_rule_sqlparser:parse(Sql), + InTopic = maps:get(topic, Context, <<>>), EventTopics = emqx_rule_sqlparser:select_from(Select), case lists:all(fun is_publish_topic/1, EventTopics) of true -> @@ -48,40 +40,36 @@ test(#{<<"rawsql">> := Sql, <<"ctx">> := Context}) -> end. test_rule(Sql, Select, Context, EventTopics) -> - RuleId = iolist_to_binary(["test_rule", emqx_misc:gen_id()]), - ActInstId = iolist_to_binary(["test_action", emqx_misc:gen_id()]), + RuleId = iolist_to_binary(["sql_tester:", emqx_misc:gen_id(16)]), ok = emqx_rule_metrics:create_rule_metrics(RuleId), - ok = emqx_rule_metrics:create_metrics(ActInstId), Rule = #rule{ id = RuleId, - rawsql = Sql, - for = EventTopics, - is_foreach = emqx_rule_sqlparser:select_is_foreach(Select), - fields = emqx_rule_sqlparser:select_fields(Select), - doeach = emqx_rule_sqlparser:select_doeach(Select), - incase = emqx_rule_sqlparser:select_incase(Select), - conditions = emqx_rule_sqlparser:select_where(Select), - actions = [#action_instance{ - id = ActInstId, - name = test_rule_sql}] + info = #{ + sql => Sql, + from => EventTopics, + outputs => [#{type => func, target => fun ?MODULE:get_selected_data/3, args => #{}}], + enabled => true, + is_foreach => emqx_rule_sqlparser:select_is_foreach(Select), + fields => emqx_rule_sqlparser:select_fields(Select), + doeach => emqx_rule_sqlparser:select_doeach(Select), + incase => emqx_rule_sqlparser:select_incase(Select), + conditions => emqx_rule_sqlparser:select_where(Select) + }, + created_at = erlang:system_time(millisecond) }, FullContext = fill_default_values(hd(EventTopics), emqx_rule_maps:atom_key_map(Context)), try - ok = emqx_rule_registry:add_action_instance_params( - #action_instance_params{id = ActInstId, - params = #{}, - apply = sql_test_action()}), - R = emqx_rule_runtime:apply_rule(Rule, FullContext), - emqx_rule_metrics:clear_rule_metrics(RuleId), - emqx_rule_metrics:clear_metrics(ActInstId), - R + emqx_rule_runtime:apply_rule(Rule, FullContext) of {ok, Data} -> {ok, flatten(Data)}; {error, nomatch} -> {error, nomatch} after - ok = emqx_rule_registry:remove_action_instance_params(ActInstId) + emqx_rule_metrics:clear_rule_metrics(RuleId) end. +get_selected_data(Selected, _Envs, _Args) -> + Selected. + is_publish_topic(<<"$events/", _/binary>>) -> false; is_publish_topic(_Topic) -> true. @@ -90,10 +78,9 @@ flatten([D1]) -> D1; flatten([D1 | L]) when is_list(D1) -> D1 ++ flatten(L). -sql_test_action() -> - fun(Data, _Envs) -> - ?LOG(info, "Testing Rule SQL OK"), Data - end. +echo_action(Data, Envs) -> + ?SLOG(debug, #{msg => "testing_rule_sql_ok", data => Data, envs => Envs}), + Data. fill_default_values(Event, Context) -> maps:merge(envs_examp(Event), Context). diff --git a/apps/emqx_rule_engine/src/emqx_rule_validator.erl b/apps/emqx_rule_engine/src/emqx_rule_validator.erl deleted file mode 100644 index 8f39d2d1c..000000000 --- a/apps/emqx_rule_engine/src/emqx_rule_validator.erl +++ /dev/null @@ -1,195 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_rule_validator). - --include("rule_engine.hrl"). - --export([ validate_params/2 - , validate_spec/1 - ]). - --type name() :: atom(). - --type spec() :: #{ - type := data_type(), - required => boolean(), - default => term(), - enum => list(term()), - schema => spec() -}. - --type data_type() :: string | password | number | boolean - | object | array | file | cfgselect. - --type params_spec() :: #{name() => spec()} | any. --type params() :: #{binary() => term()}. - --define(DATA_TYPES, - [ string - , password %% TODO: [5.0] remove this, use string instead - , number - , boolean - , object - , array - , file - , cfgselect %% TODO: [5.0] refactor this - ]). - -%%------------------------------------------------------------------------------ -%% APIs -%%------------------------------------------------------------------------------ - -%% Validate the params according to the spec. -%% Some keys will be added into the result if they have default values in spec. -%% Note that this function throws exception in case of validation failure. --spec(validate_params(params(), params_spec()) -> params()). -validate_params(Params, any) -> Params; -validate_params(Params, ParamsSepc) -> - maps:fold(fun(Name, Spec, Params0) -> - IsRequired = maps:get(required, Spec, false), - BinName = bin(Name), - find_field(Name, Params, - fun (not_found) when IsRequired =:= true -> - throw({required_field_missing, BinName}); - (not_found) when IsRequired =:= false -> - case maps:find(default, Spec) of - {ok, Default} -> Params0#{BinName => Default}; - error -> Params0 - end; - (Val) -> - Params0#{BinName => validate_value(Val, Spec)} - end) - end, Params, ParamsSepc). - --spec(validate_spec(params_spec()) -> ok). -validate_spec(any) -> ok; -validate_spec(ParamsSepc) -> - map_foreach(fun do_validate_spec/2, ParamsSepc). - -%%------------------------------------------------------------------------------ -%% Internal Functions -%%------------------------------------------------------------------------------ - -validate_value(Val, #{enum := Enum}) -> - validate_enum(Val, Enum); -validate_value(Val, #{type := object} = Spec) -> - validate_params(Val, maps:get(schema, Spec, any)); -validate_value(Val, #{type := Type} = Spec) -> - validate_type(Val, Type, Spec). - -validate_type(Val, file, _Spec) -> - validate_file(Val); -validate_type(Val, String, Spec) when String =:= string; - String =:= password -> - validate_string(Val, reg_exp(maps:get(format, Spec, any))); -validate_type(Val, number, Spec) -> - validate_number(Val, maps:get(range, Spec, any)); -validate_type(Val, boolean, _Spec) -> - validate_boolean(Val); -validate_type(Val, array, Spec) -> - ItemsSpec = maps:get(items, Spec), - [validate_value(V, ItemsSpec) || V <- Val]; -validate_type(Val, cfgselect, _Spec) -> - %% TODO: [5.0] refactor this. - Val. - -validate_enum(Val, Enum) -> - case lists:member(Val, Enum) of - true -> Val; - false -> throw({invalid_data_type, {enum, {Val, Enum}}}) - end. - -validate_string(Val, RegExp) -> - try re:run(Val, RegExp) of - nomatch -> throw({invalid_data_type, {string, Val}}); - _Match -> Val - catch - _:_ -> throw({invalid_data_type, {string, Val}}) - end. - -validate_number(Val, any) when is_integer(Val); is_float(Val) -> - Val; -validate_number(Val, _Range = [Min, Max]) - when (is_integer(Val) orelse is_float(Val)), - (Val >= Min andalso Val =< Max) -> - Val; -validate_number(Val, Range) -> - throw({invalid_data_type, {number, {Val, Range}}}). - -validate_boolean(true) -> true; -validate_boolean(<<"true">>) -> true; -validate_boolean(false) -> false; -validate_boolean(<<"false">>) -> false; -validate_boolean(Val) -> throw({invalid_data_type, {boolean, Val}}). - -validate_file(Val) when is_map(Val) -> Val; -validate_file(Val) when is_list(Val) -> Val; -validate_file(Val) when is_binary(Val) -> Val; -validate_file(Val) -> throw({invalid_data_type, {file, Val}}). - -reg_exp(url) -> "^https?://\\w+(\.\\w+)*(:[0-9]+)?"; -reg_exp(topic) -> "^/?(\\w|\\#|\\+)+(/?(\\w|\\#|\\+))*/?$"; -reg_exp(resource_type) -> "[a-zA-Z0-9_:-]"; -reg_exp(any) -> ".*"; -reg_exp(RegExp) -> RegExp. - -do_validate_spec(Name, #{type := object} = Spec) -> - find_field(schema, Spec, - fun (not_found) -> throw({required_field_missing, {schema, {in, Name}}}); - (Schema) -> validate_spec(Schema) - end); -do_validate_spec(Name, #{type := array} = Spec) -> - find_field(items, Spec, - fun (not_found) -> throw({required_field_missing, {items, {in, Name}}}); - (Items) -> do_validate_spec(Name, Items) - end); -do_validate_spec(_Name, #{type := Type}) -> - _ = supported_data_type(Type, ?DATA_TYPES); - -do_validate_spec(Name, _Spec) -> - throw({required_field_missing, {type, {in, Name}}}). - -supported_data_type(Type, Supported) -> - case lists:member(Type, Supported) of - false -> throw({unsupported_data_types, Type}); - true -> ok - end. - -map_foreach(Fun, Map) -> - Iterator = maps:iterator(Map), - map_foreach_loop(Fun, maps:next(Iterator)). - -map_foreach_loop(_Fun, none) -> ok; -map_foreach_loop(Fun, {Key, Value, Iterator}) -> - _ = Fun(Key, Value), - map_foreach_loop(Fun, maps:next(Iterator)). - -find_field(Field, Spec, Func) -> - do_find_field([bin(Field), Field], Spec, Func). - -do_find_field([], _Spec, Func) -> - Func(not_found); -do_find_field([F | Fields], Spec, Func) -> - case maps:find(F, Spec) of - {ok, Value} -> Func(Value); - error -> - do_find_field(Fields, Spec, Func) - end. - -bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8); -bin(Str) when is_list(Str) -> iolist_to_binary(Str); -bin(Bin) when is_binary(Bin) -> Bin. diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index 47eb4faad..45b30223a 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -26,17 +26,15 @@ -include_lib("common_test/include/ct.hrl"). %%-define(PROPTEST(M,F), true = proper:quickcheck(M:F())). +-define(TMP_RULEID, atom_to_binary(?FUNCTION_NAME)). all() -> [ {group, engine} - , {group, actions} -%% , {group, api} - , {group, cli} + , {group, api} , {group, funcs} , {group, registry} , {group, runtime} , {group, events} - , {group, multi_actions} , {group, bugs} ]. @@ -45,49 +43,21 @@ suite() -> groups() -> [{engine, [sequence], - [t_register_provider, - t_unregister_provider, - t_create_rule, - t_create_resource - ]}, - {actions, [], - [t_inspect_action - ,t_republish_action - ]}, -%% TODO: V5 API -%% {api, [], -%% [t_crud_rule_api, -%% t_list_actions_api, -%% t_show_action_api, -%% t_crud_resources_api, -%% t_list_resource_types_api, -%% t_show_resource_type_api -%% ]}, - {cli, [], - [t_rules_cli, - t_actions_cli, - t_resources_cli, - t_resource_types_cli + [t_create_rule ]}, + {api, [], + [t_crud_rule_api + ]}, {funcs, [], - [t_topic_func, - t_kv_store + [t_kv_store ]}, {registry, [sequence], [t_add_get_remove_rule, t_add_get_remove_rules, t_create_existing_rule, - t_update_rule, - t_disable_rule, - t_get_rules_for, - t_get_rules_for_2, - t_get_rules_with_same_event, - t_add_get_remove_action, - t_add_get_remove_actions, - t_remove_actions_of, - t_get_resources, - t_add_get_remove_resource, - t_resource_types + t_get_rules_for_topic, + t_get_rules_for_topic_2, + t_get_rules_with_same_event ]}, {runtime, [], [t_match_atom_and_binary, @@ -97,9 +67,6 @@ groups() -> t_sqlselect_02, t_sqlselect_1, t_sqlselect_2, - t_sqlselect_2_1, - t_sqlselect_2_2, - t_sqlselect_2_3, t_sqlselect_3, t_sqlparse_event_1, t_sqlparse_event_2, @@ -132,14 +99,6 @@ groups() -> {bugs, [], [t_sqlparse_payload_as, t_sqlparse_nested_get - ]}, - {multi_actions, [], - [t_sqlselect_multi_actoins_1, - t_sqlselect_multi_actoins_1_1, - t_sqlselect_multi_actoins_2, - t_sqlselect_multi_actoins_3, - t_sqlselect_multi_actoins_3_1, - t_sqlselect_multi_actoins_4 ]} ]. @@ -151,7 +110,7 @@ init_per_suite(Config) -> application:load(emqx_machine), ok = ekka_mnesia:start(), ok = emqx_rule_registry:mnesia(boot), - ok = emqx_ct_helpers:start_apps([emqx_rule_engine], fun set_special_configs/1), + ok = emqx_ct_helpers:start_apps([emqx_rule_engine]), Config. end_per_suite(_Config) -> @@ -182,16 +141,8 @@ end_per_group(_Groupname, _Config) -> %%------------------------------------------------------------------------------ init_per_testcase(t_events, Config) -> - {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), - ok = emqx_rule_engine:load_providers(), init_events_counters(), - ok = emqx_rule_registry:register_resource_types([make_simple_resource_type(simple_resource_type)]), - ok = emqx_rule_registry:add_action( - #action{name = 'hook-metrics-action', app = ?APP, - module = ?MODULE, on_create = hook_metrics_action, - types=[], params_spec = #{}, - title = #{en => <<"Hook metrics action">>}, - description = #{en => <<"Hook metrics action">>}}), + {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), SQL = "SELECT * FROM \"$events/client_connected\", " "\"$events/client_disconnected\", " "\"$events/session_subscribed\", " @@ -201,411 +152,87 @@ init_per_testcase(t_events, Config) -> "\"$events/message_dropped\", " "\"t1\"", {ok, Rule} = emqx_rule_engine:create_rule( - #{id => <<"rule:t_events">>, - rawsql => SQL, - actions => [#{id => <<"action:inspect">>, name => 'inspect', args => #{}}, - #{id => <<"action:hook-metrics-action">>, name => 'hook-metrics-action', args => #{}}], - description => <<"Debug rule">>}), + #{id => <<"rule:t_events">>, + sql => SQL, + outputs => [ + #{type => builtin, target => console}, + #{type => func, target => fun ?MODULE:output_record_triggered_events/3, + args => #{}} + ], + description => <<"to console and record triggered events">>}), ?assertMatch(#rule{id = <<"rule:t_events">>}, Rule), [{hook_points_rules, Rule} | Config]; -init_per_testcase(Test, Config) - when Test =:= t_sqlselect_multi_actoins_1 - ;Test =:= t_sqlselect_multi_actoins_1_1 - ;Test =:= t_sqlselect_multi_actoins_2 - ;Test =:= t_sqlselect_multi_actoins_3 - ;Test =:= t_sqlselect_multi_actoins_3_1 - ;Test =:= t_sqlselect_multi_actoins_4 - -> - emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), - ok = emqx_rule_engine:load_providers(), - ok = emqx_rule_registry:add_action( - #action{name = 'crash_action', app = ?APP, - module = ?MODULE, on_create = crash_action, - types=[], params_spec = #{}, - title = #{en => <<"Crash Action">>}, - description = #{en => <<"This action will always fail!">>}}), - ok = emqx_rule_registry:add_action( - #action{name = 'failure_action', app = ?APP, - module = ?MODULE, on_create = failure_action, - types=[], params_spec = #{}, - title = #{en => <<"Crash Action">>}, - description = #{en => <<"This action will always fail!">>}}), - ok = emqx_rule_registry:add_action( - #action{name = 'plus_by_one', app = ?APP, - module = ?MODULE, on_create = plus_by_one_action, - types=[], params_spec = #{}, - title = #{en => <<"Plus an integer by 1">>} - }), - init_plus_by_one_action(), - SQL = "SELECT * " - "FROM \"$events/client_connected\" " - "WHERE username = 'emqx1'", - {ok, SubClient} = emqtt:start_link([{clientid, <<"emqx0">>}, {username, <<"emqx0">>}]), - {ok, _} = emqtt:connect(SubClient), - {ok, _, _} = emqtt:subscribe(SubClient, <<"t2">>, 0), - ct:sleep(100), - - {ok, ConnClient} = emqtt:start_link([{clientid, <<"c_emqx1">>}, {username, <<"emqx1">>}]), - TriggerConnEvent = fun() -> - {ok, _} = emqtt:connect(ConnClient) - end, - [{subclient, SubClient}, - {connclient, ConnClient}, - {conn_event, TriggerConnEvent}, - {connsql, SQL} - | Config]; init_per_testcase(_TestCase, Config) -> emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), - ok = emqx_rule_registry:register_resource_types( - [#resource_type{ - name = built_in, - provider = ?APP, - params_spec = #{}, - on_create = {?MODULE, on_resource_create}, - on_destroy = {?MODULE, on_resource_destroy}, - on_status = {?MODULE, on_get_resource_status}, - title = #{en => <<"Built-In Resource Type (debug)">>}, - description = #{en => <<"The built in resource type for debug purpose">>}}]), - %ct:pal("============ ~p", [ets:tab2list(emqx_resource_type)]), Config. - end_per_testcase(t_events, Config) -> ets:delete(events_record_tab), - ok = emqx_rule_registry:remove_rule(?config(hook_points_rules, Config)), - ok = emqx_rule_registry:remove_action('hook-metrics-action'); -end_per_testcase(Test, Config) - when Test =:= t_sqlselect_multi_actoins_1, - Test =:= t_sqlselect_multi_actoins_2 - -> - emqtt:stop(?config(subclient, Config)), - emqtt:stop(?config(connclient, Config)), - Config; + ok = emqx_rule_registry:remove_rule(?config(hook_points_rules, Config)); end_per_testcase(_TestCase, _Config) -> ok. %%------------------------------------------------------------------------------ %% Test cases for rule engine %%------------------------------------------------------------------------------ - -t_register_provider(_Config) -> - ok = emqx_rule_engine:load_providers(), - ?assert(length(emqx_rule_registry:get_actions()) >= 2), - ok. - -t_unregister_provider(_Config) -> - ok = emqx_rule_engine:unload_providers(), - ?assert(length(emqx_rule_registry:get_actions()) == 0), - ok. - t_create_rule(_Config) -> - ok = emqx_rule_engine:load_providers(), {ok, #rule{id = Id}} = emqx_rule_engine:create_rule( - #{rawsql => <<"select * from \"t/a\"">>, - actions => [#{name => 'inspect', args => #{arg1 => 1}}], + #{sql => <<"select * from \"t/a\"">>, + id => <<"t_create_rule">>, + outputs => [#{type => builtin, target => console}], description => <<"debug rule">>}), - %ct:pal("======== emqx_rule_registry:get_rules :~p", [emqx_rule_registry:get_rules()]), - ?assertMatch({ok,#rule{id = Id, for = [<<"t/a">>]}}, emqx_rule_registry:get_rule(Id)), - ok = emqx_rule_engine:unload_providers(), + ct:pal("======== emqx_rule_registry:get_rules :~p", [emqx_rule_registry:get_rules()]), + ?assertMatch({ok, #rule{id = Id, info = #{from := [<<"t/a">>]}}}, + emqx_rule_registry:get_rule(Id)), emqx_rule_registry:remove_rule(Id), ok. -t_create_resource(_Config) -> - ok = emqx_rule_engine:load_providers(), - {ok, #resource{id = ResId}} = emqx_rule_engine:create_resource( - #{type => built_in, - config => #{}, - description => <<"debug resource">>}), - ?assert(true, is_binary(ResId)), - ok = emqx_rule_engine:unload_providers(), - emqx_rule_registry:remove_resource(ResId), - ok. - -%%------------------------------------------------------------------------------ -%% Test cases for rule actions -%%------------------------------------------------------------------------------ - -t_inspect_action(_Config) -> - ok = emqx_rule_engine:load_providers(), - {ok, #resource{id = ResId}} = emqx_rule_engine:create_resource( - #{type => built_in, - config => #{}, - description => <<"debug resource">>}), - {ok, #rule{id = Id}} = emqx_rule_engine:create_rule( - #{rawsql => "select clientid as c, username as u " - "from \"t1\" ", - actions => [#{name => 'inspect', - args => #{'$resource' => ResId, a=>1, b=>2}}], - type => built_in, - description => <<"Inspect rule">> - }), - {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), - {ok, _} = emqtt:connect(Client), - emqtt:publish(Client, <<"t1">>, <<"{\"id\": 1, \"name\": \"ha\"}">>, 0), - emqtt:stop(Client), - emqx_rule_registry:remove_rule(Id), - emqx_rule_registry:remove_resource(ResId), - ok. - -t_republish_action(_Config) -> - Qos0Received = emqx_metrics:val('messages.qos0.received'), - Received = emqx_metrics:val('messages.received'), - ok = emqx_rule_engine:load_providers(), - {ok, #rule{id = Id, for = [<<"t1">>]}} = - emqx_rule_engine:create_rule( - #{rawsql => <<"select topic, payload, qos from \"t1\"">>, - actions => [#{name => 'republish', - args => #{<<"target_topic">> => <<"t2">>, - <<"target_qos">> => -1, - <<"payload_tmpl">> => <<"${payload}">>}}], - description => <<"builtin-republish-rule">>}), - {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), - {ok, _} = emqtt:connect(Client), - {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), - - Msg = <<"{\"id\": 1, \"name\": \"ha\"}">>, - emqtt:publish(Client, <<"t1">>, Msg, 0), - receive {publish, #{topic := <<"t2">>, payload := Payload}} -> - ?assertEqual(Msg, Payload) - after 1000 -> - ct:fail(wait_for_t2) - end, - emqtt:stop(Client), - emqx_rule_registry:remove_rule(Id), - ?assertEqual(2, emqx_metrics:val('messages.qos0.received') - Qos0Received), - ?assertEqual(2, emqx_metrics:val('messages.received') - Received), - ok. - %%------------------------------------------------------------------------------ %% Test cases for rule engine api %%------------------------------------------------------------------------------ t_crud_rule_api(_Config) -> - {ok, #{code := 0, data := Rule}} = - emqx_rule_engine_api:create_rule(#{}, - [{<<"name">>, <<"debug-rule">>}, - {<<"rawsql">>, <<"select * from \"t/a\"">>}, - {<<"actions">>, [[{<<"name">>,<<"inspect">>}, - {<<"params">>,[{<<"arg1">>,1}]}]]}, - {<<"description">>, <<"debug rule">>}]), - RuleID = maps:get(id, Rule), - %ct:pal("RCreated : ~p", [Rule]), + RuleID = <<"my_rule">>, + Params0 = #{ + <<"description">> => <<"A simple rule">>, + <<"enable">> => true, + <<"id">> => RuleID, + <<"outputs">> => [#{<<"type">> => <<"builtin">>, <<"target">> => <<"console">>}], + <<"sql">> => <<"SELECT * from \"t/1\"">> + }, + {201, Rule} = emqx_rule_engine_api:crud_rules(post, #{body => Params0}), - {ok, #{code := 0, data := Rules}} = emqx_rule_engine_api:list_rules(#{}, []), - %ct:pal("RList : ~p", [Rules]), + ?assertEqual(RuleID, maps:get(id, Rule)), + {200, Rules} = emqx_rule_engine_api:crud_rules(get, #{}), + ct:pal("RList : ~p", [Rules]), ?assert(length(Rules) > 0), - {ok, #{code := 0, data := Rule1}} = emqx_rule_engine_api:show_rule(#{id => RuleID}, []), - %ct:pal("RShow : ~p", [Rule1]), + {200, Rule1} = emqx_rule_engine_api:crud_rules_by_id(get, #{bindings => #{id => RuleID}}), + ct:pal("RShow : ~p", [Rule1]), ?assertEqual(Rule, Rule1), - {ok, #{code := 0, data := Rule2}} = emqx_rule_engine_api:update_rule(#{id => RuleID}, - [{<<"rawsql">>, <<"select * from \"t/b\"">>}]), + {200, Rule2} = emqx_rule_engine_api:crud_rules_by_id(put, #{ + bindings => #{id => RuleID}, + body => Params0#{<<"sql">> => <<"select * from \"t/b\"">>} + }), - {ok, #{code := 0, data := Rule3}} = emqx_rule_engine_api:show_rule(#{id => RuleID}, []), - %ct:pal("RShow : ~p", [Rule1]), + {200, Rule3} = emqx_rule_engine_api:crud_rules_by_id(get, #{bindings => #{id => RuleID}}), + %ct:pal("RShow : ~p", [Rule3]), ?assertEqual(Rule3, Rule2), - ?assertEqual(<<"select * from \"t/b\"">>, maps:get(rawsql, Rule3)), + ?assertEqual(<<"select * from \"t/b\"">>, maps:get(sql, Rule3)), - {ok, #{code := 0, data := Rule4}} = emqx_rule_engine_api:update_rule(#{id => RuleID}, - [{<<"actions">>, - [[ - {<<"name">>,<<"republish">>}, - {<<"params">>,[ - {<<"arg1">>,1}, - {<<"target_topic">>, <<"t2">>}, - {<<"target_qos">>, -1}, - {<<"payload_tmpl">>, <<"${payload}">>} - ]} - ]] - }]), + ?assertMatch({200}, emqx_rule_engine_api:crud_rules_by_id(delete, + #{bindings => #{id => RuleID}})), - {ok, #{code := 0, data := Rule5}} = emqx_rule_engine_api:show_rule(#{id => RuleID}, []), - %ct:pal("RShow : ~p", [Rule1]), - ?assertEqual(Rule5, Rule4), - ?assertMatch([#{name := republish }], maps:get(actions, Rule5)), - - ?assertMatch({ok, #{code := 0}}, emqx_rule_engine_api:delete_rule(#{id => RuleID}, [])), - - NotFound = emqx_rule_engine_api:show_rule(#{id => RuleID}, []), %ct:pal("Show After Deleted: ~p", [NotFound]), - ?assertMatch({ok, #{code := 404, message := _Message}}, NotFound), - ok. - -t_list_actions_api(_Config) -> - {ok, #{code := 0, data := Actions}} = emqx_rule_engine_api:list_actions(#{}, []), - %ct:pal("RList : ~p", [Actions]), - ?assert(length(Actions) > 0), - ok. - -t_show_action_api(_Config) -> - {ok, #{code := 0, data := Actions}} = emqx_rule_engine_api:show_action(#{name => 'inspect'}, []), - ?assertEqual('inspect', maps:get(name, Actions)), - ok. - -t_crud_resources_api(_Config) -> - {ok, #{code := 0, data := Resources1}} = - emqx_rule_engine_api:create_resource(#{}, - [{<<"name">>, <<"Simple Resource">>}, - {<<"type">>, <<"built_in">>}, - {<<"config">>, [{<<"a">>, 1}]}, - {<<"description">>, <<"Simple Resource">>}]), - ResId = maps:get(id, Resources1), - {ok, #{code := 0, data := Resources}} = emqx_rule_engine_api:list_resources(#{}, []), - ?assert(length(Resources) > 0), - {ok, #{code := 0, data := Resources2}} = emqx_rule_engine_api:show_resource(#{id => ResId}, []), - ?assertEqual(ResId, maps:get(id, Resources2)), - % - {ok, #{code := 0}} = emqx_rule_engine_api:update_resource(#{id => ResId}, - [{<<"config">>, [{<<"a">>, 2}]}, - {<<"description">>, <<"2">>}]), - {ok, #{code := 0, data := Resources3}} = emqx_rule_engine_api:show_resource(#{id => ResId}, []), - ?assertEqual(ResId, maps:get(id, Resources3)), - ?assertEqual(#{<<"a">> => 2}, maps:get(config, Resources3)), - ?assertEqual(<<"2">>, maps:get(description, Resources3)), - % - {ok, #{code := 0}} = emqx_rule_engine_api:update_resource(#{id => ResId}, - [{<<"config">>, [{<<"a">>, 3}]}]), - {ok, #{code := 0, data := Resources4}} = emqx_rule_engine_api:show_resource(#{id => ResId}, []), - ?assertEqual(ResId, maps:get(id, Resources4)), - ?assertEqual(#{<<"a">> => 3}, maps:get(config, Resources4)), - ?assertEqual(<<"2">>, maps:get(description, Resources4)), - % Only config - {ok, #{code := 0}} = emqx_rule_engine_api:update_resource(#{id => ResId}, - [{<<"config">>, [{<<"a">>, 1}, - {<<"b">>, 2}, - {<<"c">>, 3}]}]), - {ok, #{code := 0, data := Resources5}} = emqx_rule_engine_api:show_resource(#{id => ResId}, []), - ?assertEqual(ResId, maps:get(id, Resources5)), - ?assertEqual(#{<<"a">> => 1, <<"b">> => 2, <<"c">> => 3}, maps:get(config, Resources5)), - ?assertEqual(<<"2">>, maps:get(description, Resources5)), - % Only description - {ok, #{code := 0}} = emqx_rule_engine_api:update_resource(#{id => ResId}, - [{<<"description">>, <<"new5">>}]), - {ok, #{code := 0, data := Resources6}} = emqx_rule_engine_api:show_resource(#{id => ResId}, []), - ?assertEqual(ResId, maps:get(id, Resources6)), - ?assertEqual(#{<<"a">> => 1, <<"b">> => 2, <<"c">> => 3}, maps:get(config, Resources6)), - ?assertEqual(<<"new5">>, maps:get(description, Resources6)), - % None - {ok, #{code := 0}} = emqx_rule_engine_api:update_resource(#{id => ResId}, []), - {ok, #{code := 0, data := Resources7}} = emqx_rule_engine_api:show_resource(#{id => ResId}, []), - ?assertEqual(ResId, maps:get(id, Resources7)), - ?assertEqual(#{<<"a">> => 1, <<"b">> => 2, <<"c">> => 3}, maps:get(config, Resources7)), - ?assertEqual(<<"new5">>, maps:get(description, Resources7)), - % - ?assertMatch({ok, #{code := 0}}, emqx_rule_engine_api:delete_resource(#{id => ResId},#{})), - ?assertMatch({ok, #{code := 404}}, emqx_rule_engine_api:show_resource(#{id => ResId}, [])), - ok. - -t_list_resource_types_api(_Config) -> - {ok, #{code := 0, data := ResourceTypes}} = emqx_rule_engine_api:list_resource_types(#{}, []), - ?assert(length(ResourceTypes) > 0), - ok. - -t_show_resource_type_api(_Config) -> - {ok, #{code := 0, data := RShow}} = emqx_rule_engine_api:show_resource_type(#{name => 'built_in'}, []), - %ct:pal("RShow : ~p", [RShow]), - ?assertEqual(built_in, maps:get(name, RShow)), - ok. - -%%------------------------------------------------------------------------------ -%% Test cases for rule engine cli -%%------------------------------------------------------------------------------ - -t_rules_cli(_Config) -> - mock_print(), - RCreate = emqx_rule_engine_cli:rules(["create", - "select * from \"t1\" where topic='t1'", - "[{\"name\":\"inspect\", \"params\": {\"arg1\": 1}}]", - "-d", "Debug Rule"]), - %ct:pal("Result : ~p", [RCreate]), - ?assertMatch({match, _}, re:run(RCreate, "created")), - - RuleId = re:replace(re:replace(RCreate, "Rule\s", "", [{return, list}]), "\screated\n", "", [{return, list}]), - - RList = emqx_rule_engine_cli:rules(["list"]), - ?assertMatch({match, _}, re:run(RList, RuleId)), - %ct:pal("RList : ~p", [RList]), - %ct:pal("table action params: ~p", [ets:tab2list(emqx_action_instance_params)]), - - RShow = emqx_rule_engine_cli:rules(["show", RuleId]), - ?assertMatch({match, _}, re:run(RShow, RuleId)), - %ct:pal("RShow : ~p", [RShow]), - - RUpdate = emqx_rule_engine_cli:rules(["update", - RuleId, - "-s", "select * from \"t2\""]), - ?assertMatch({match, _}, re:run(RUpdate, "updated")), - - RDelete = emqx_rule_engine_cli:rules(["delete", RuleId]), - ?assertEqual("ok~n", RDelete), - %ct:pal("RDelete : ~p", [RDelete]), - %ct:pal("table action params after deleted: ~p", [ets:tab2list(emqx_action_instance_params)]), - - RShow2 = emqx_rule_engine_cli:rules(["show", RuleId]), - ?assertMatch({match, _}, re:run(RShow2, "Cannot found")), - %ct:pal("RShow2 : ~p", [RShow2]), - unmock_print(), - ok. - -t_actions_cli(_Config) -> - mock_print(), - RList = emqx_rule_engine_cli:actions(["list"]), - ?assertMatch({match, _}, re:run(RList, "inspect")), - %ct:pal("RList : ~p", [RList]), - - RShow = emqx_rule_engine_cli:actions(["show", "inspect"]), - ?assertMatch({match, _}, re:run(RShow, "inspect")), - %ct:pal("RShow : ~p", [RShow]), - unmock_print(), - ok. - -t_resources_cli(_Config) -> - mock_print(), - RCreate = emqx_rule_engine_cli:resources(["create", "built_in", "{\"a\" : 1}", "-d", "test resource"]), - ResId = re:replace(re:replace(RCreate, "Resource\s", "", [{return, list}]), "\screated\n", "", [{return, list}]), - - RList = emqx_rule_engine_cli:resources(["list"]), - ?assertMatch({match, _}, re:run(RList, "test resource")), - %ct:pal("RList : ~p", [RList]), - - RListT = emqx_rule_engine_cli:resources(["list", "-t", "built_in"]), - ?assertMatch({match, _}, re:run(RListT, "test resource")), - %ct:pal("RListT : ~p", [RListT]), - - RShow = emqx_rule_engine_cli:resources(["show", ResId]), - ?assertMatch({match, _}, re:run(RShow, "test resource")), - %ct:pal("RShow : ~p", [RShow]), - - RDelete = emqx_rule_engine_cli:resources(["delete", ResId]), - ?assertEqual("ok~n", RDelete), - - RShow2 = emqx_rule_engine_cli:resources(["show", ResId]), - ?assertMatch({match, _}, re:run(RShow2, "Cannot found")), - %ct:pal("RShow2 : ~p", [RShow2]), - unmock_print(), - ok. - -t_resource_types_cli(_Config) -> - mock_print(), - RList = emqx_rule_engine_cli:resource_types(["list"]), - ?assertMatch({match, _}, re:run(RList, "built_in")), - %ct:pal("RList : ~p", [RList]), - - RShow = emqx_rule_engine_cli:resource_types(["show", "inspect"]), - ?assertMatch({match, _}, re:run(RShow, "inspect")), - %ct:pal("RShow : ~p", [RShow]), - unmock_print(), + ?assertMatch({404, #{code := _, message := _Message}}, + emqx_rule_engine_api:crud_rules_by_id(get, #{bindings => #{id => RuleID}})), ok. %%------------------------------------------------------------------------------ %% Test cases for rule funcs %%------------------------------------------------------------------------------ -t_topic_func(_Config) -> - %%TODO: - ok. - t_kv_store(_) -> undefined = emqx_rule_funcs:kv_store_get(<<"abc">>), <<"not_found">> = emqx_rule_funcs:kv_store_get(<<"abc">>, <<"not_found">>), @@ -619,7 +246,6 @@ t_kv_store(_) -> %%------------------------------------------------------------------------------ t_add_get_remove_rule(_Config) -> - mock_print(), RuleId0 = <<"rule-debug-0">>, ok = emqx_rule_registry:add_rule(make_simple_rule(RuleId0)), ?assertMatch({ok, #rule{id = RuleId0}}, emqx_rule_registry:get_rule(RuleId0)), @@ -632,7 +258,6 @@ t_add_get_remove_rule(_Config) -> ?assertMatch({ok, #rule{id = RuleId1}}, emqx_rule_registry:get_rule(RuleId1)), ok = emqx_rule_registry:remove_rule(Rule1), ?assertEqual(not_found, emqx_rule_registry:get_rule(RuleId1)), - unmock_print(), ok. t_add_get_remove_rules(_Config) -> @@ -656,96 +281,22 @@ t_create_existing_rule(_Config) -> %% create a rule using given rule id {ok, _} = emqx_rule_engine:create_rule( #{id => <<"an_existing_rule">>, - rawsql => <<"select * from \"t/#\"">>, - actions => [ - #{name => 'inspect', args => #{}} - ] + sql => <<"select * from \"t/#\"">>, + outputs => [#{type => builtin, target => console}] }), - {ok, #rule{rawsql = SQL}} = emqx_rule_registry:get_rule(<<"an_existing_rule">>), + {ok, #rule{info = #{sql := SQL}}} = emqx_rule_registry:get_rule(<<"an_existing_rule">>), ?assertEqual(<<"select * from \"t/#\"">>, SQL), ok = emqx_rule_engine:delete_rule(<<"an_existing_rule">>), ?assertEqual(not_found, emqx_rule_registry:get_rule(<<"an_existing_rule">>)), ok. -t_update_rule(_Config) -> - {ok, #rule{actions = [#action_instance{id = ActInsId0}]}} = emqx_rule_engine:create_rule( - #{id => <<"an_existing_rule">>, - rawsql => <<"select * from \"t/#\"">>, - actions => [ - #{name => 'inspect', args => #{}} - ] - }), - ?assertMatch({ok, #action_instance_params{apply = _}}, - emqx_rule_registry:get_action_instance_params(ActInsId0)), - %% update the rule and verify the old action instances has been cleared - %% and the new action instances has been created. - emqx_rule_engine:update_rule(#{ id => <<"an_existing_rule">>, - actions => [ - #{name => 'do_nothing', args => #{}} - ]}), - - {ok, #rule{actions = [#action_instance{id = ActInsId1}]}} - = emqx_rule_registry:get_rule(<<"an_existing_rule">>), - - ?assertMatch(not_found, - emqx_rule_registry:get_action_instance_params(ActInsId0)), - - ?assertMatch({ok, #action_instance_params{apply = _}}, - emqx_rule_registry:get_action_instance_params(ActInsId1)), - - ok = emqx_rule_engine:delete_rule(<<"an_existing_rule">>), - ?assertEqual(not_found, emqx_rule_registry:get_rule(<<"an_existing_rule">>)), - ok. - -t_disable_rule(_Config) -> - ets:new(simple_action_2, [named_table, set, public]), - ets:insert(simple_action_2, {created, 0}), - ets:insert(simple_action_2, {destroyed, 0}), - Now = erlang:timestamp(), - emqx_rule_registry:add_action( - #action{name = 'simple_action_2', app = ?APP, - module = ?MODULE, - on_create = simple_action_2_create, - on_destroy = simple_action_2_destroy, - types=[], params_spec = #{}, - title = #{en => <<"Simple Action">>}, - description = #{en => <<"Simple Action">>}}), - {ok, #rule{actions = [#action_instance{}]}} = emqx_rule_engine:create_rule( - #{id => <<"simple_rule_2">>, - rawsql => <<"select * from \"t/#\"">>, - actions => [#{name => 'simple_action_2', args => #{}}] - }), - [{_, CAt}] = ets:lookup(simple_action_2, created), - ?assert(CAt > Now), - [{_, DAt}] = ets:lookup(simple_action_2, destroyed), - ?assert(DAt < Now), - - %% disable the rule and verify the old action instances has been cleared - Now2 = erlang:timestamp(), - emqx_rule_engine:update_rule(#{ id => <<"simple_rule_2">>, - enabled => false}), - [{_, CAt2}] = ets:lookup(simple_action_2, created), - ?assert(CAt2 < Now2), - [{_, DAt2}] = ets:lookup(simple_action_2, destroyed), - ?assert(DAt2 > Now2), - - %% enable the rule again and verify the action instances has been created - Now3 = erlang:timestamp(), - emqx_rule_engine:update_rule(#{ id => <<"simple_rule_2">>, - enabled => true}), - [{_, CAt3}] = ets:lookup(simple_action_2, created), - ?assert(CAt3 > Now3), - [{_, DAt3}] = ets:lookup(simple_action_2, destroyed), - ?assert(DAt3 < Now3), - ok = emqx_rule_engine:delete_rule(<<"simple_rule_2">>). - -t_get_rules_for(_Config) -> - Len0 = length(emqx_rule_registry:get_rules_for(<<"simple/topic">>)), +t_get_rules_for_topic(_Config) -> + Len0 = length(emqx_rule_registry:get_rules_for_topic(<<"simple/topic">>)), ok = emqx_rule_registry:add_rules( [make_simple_rule(<<"rule-debug-1">>), make_simple_rule(<<"rule-debug-2">>)]), - ?assertEqual(Len0+2, length(emqx_rule_registry:get_rules_for(<<"simple/topic">>))), + ?assertEqual(Len0+2, length(emqx_rule_registry:get_rules_for_topic(<<"simple/topic">>))), ok = emqx_rule_registry:remove_rules([<<"rule-debug-1">>, <<"rule-debug-2">>]), ok. @@ -762,8 +313,8 @@ t_get_rules_ordered_by_ts(_Config) -> #rule{id = <<"rule-debug-2">>} ], emqx_rule_registry:get_rules_ordered_by_ts()). -t_get_rules_for_2(_Config) -> - Len0 = length(emqx_rule_registry:get_rules_for(<<"simple/1">>)), +t_get_rules_for_topic_2(_Config) -> + Len0 = length(emqx_rule_registry:get_rules_for_topic(<<"simple/1">>)), ok = emqx_rule_registry:add_rules( [make_simple_rule(<<"rule-debug-1">>, <<"select * from \"simple/#\"">>, [<<"simple/#">>]), make_simple_rule(<<"rule-debug-2">>, <<"select * from \"simple/+\"">>, [<<"simple/+">>]), @@ -772,7 +323,7 @@ t_get_rules_for_2(_Config) -> make_simple_rule(<<"rule-debug-5">>, <<"select * from \"simple/2,simple/+,simple/3\"">>, [<<"simple/2">>,<<"simple/+">>, <<"simple/3">>]), make_simple_rule(<<"rule-debug-6">>, <<"select * from \"simple/2,simple/3,simple/4\"">>, [<<"simple/2">>,<<"simple/3">>, <<"simple/4">>]) ]), - ?assertEqual(Len0+4, length(emqx_rule_registry:get_rules_for(<<"simple/1">>))), + ?assertEqual(Len0+4, length(emqx_rule_registry:get_rules_for_topic(<<"simple/1">>))), ok = emqx_rule_registry:remove_rules([<<"rule-debug-1">>, <<"rule-debug-2">>,<<"rule-debug-3">>, <<"rule-debug-4">>,<<"rule-debug-5">>, <<"rule-debug-6">>]), ok. @@ -809,93 +360,6 @@ t_get_rules_with_same_event(_Config) -> ok = emqx_rule_registry:remove_rules([<<"r1">>, <<"r2">>,<<"r3">>, <<"r4">>,<<"r5">>, <<"r6">>, <<"r7">>, <<"r8">>, <<"r9">>, <<"r10">>]), ok. -t_add_get_remove_action(_Config) -> - ActionName0 = 'action-debug-0', - Action0 = make_simple_action(ActionName0), - ok = emqx_rule_registry:add_action(Action0), - ?assertMatch({ok, #action{name = ActionName0}}, emqx_rule_registry:find_action(ActionName0)), - ok = emqx_rule_registry:remove_action(ActionName0), - ?assertMatch(not_found, emqx_rule_registry:find_action(ActionName0)), - - ok = emqx_rule_registry:add_action(Action0), - ?assertMatch({ok, #action{name = ActionName0}}, emqx_rule_registry:find_action(ActionName0)), - ok = emqx_rule_registry:remove_action(Action0), - ?assertMatch(not_found, emqx_rule_registry:find_action(ActionName0)), - ok. - -t_add_get_remove_actions(_Config) -> - InitActionLen = length(emqx_rule_registry:get_actions()), - ActionName1 = 'action-debug-1', - ActionName2 = 'action-debug-2', - Action1 = make_simple_action(ActionName1), - Action2 = make_simple_action(ActionName2), - ok = emqx_rule_registry:add_actions([Action1, Action2]), - ?assertMatch(2, length(emqx_rule_registry:get_actions()) - InitActionLen), - ok = emqx_rule_registry:remove_actions([ActionName1, ActionName2]), - ?assertMatch(InitActionLen, length(emqx_rule_registry:get_actions())), - - ok = emqx_rule_registry:add_actions([Action1, Action2]), - ?assertMatch(2, length(emqx_rule_registry:get_actions()) - InitActionLen), - ok = emqx_rule_registry:remove_actions([Action1, Action2]), - ?assertMatch(InitActionLen, length(emqx_rule_registry:get_actions())), - ok. - -t_remove_actions_of(_Config) -> - ok = emqx_rule_registry:add_actions([make_simple_action('action-debug-1'), - make_simple_action('action-debug-2')]), - Len1 = length(emqx_rule_registry:get_actions()), - ?assert(Len1 >= 2), - ok = emqx_rule_registry:remove_actions_of(?APP), - ?assert((Len1 - length(emqx_rule_registry:get_actions())) >= 2), - ok. - -t_add_get_remove_resource(_Config) -> - ResId = <<"resource-debug">>, - Res = make_simple_resource(ResId), - ok = emqx_rule_registry:add_resource(Res), - ?assertMatch({ok, #resource{id = ResId}}, emqx_rule_registry:find_resource(ResId)), - ok = emqx_rule_registry:remove_resource(ResId), - ?assertEqual(not_found, emqx_rule_registry:find_resource(ResId)), - ok = emqx_rule_registry:add_resource(Res), - ?assertMatch({ok, #resource{id = ResId}}, emqx_rule_registry:find_resource(ResId)), - ok = emqx_rule_registry:remove_resource(Res), - ?assertEqual(not_found, emqx_rule_registry:find_resource(ResId)), - ok. -t_get_resources(_Config) -> - Len0 = length(emqx_rule_registry:get_resources()), - Res1 = make_simple_resource(<<"resource-debug-1">>), - Res2 = make_simple_resource(<<"resource-debug-2">>), - ok = emqx_rule_registry:add_resource(Res1), - ok = emqx_rule_registry:add_resource(Res2), - ?assertEqual(Len0+2, length(emqx_rule_registry:get_resources())), - ok. - -t_resource_types(_Config) -> - register_resource_types(), - get_resource_type(), - get_resource_types(), - unregister_resource_types_of(). - -register_resource_types() -> - ResType1 = make_simple_resource_type(<<"resource-type-debug-1">>), - ResType2 = make_simple_resource_type(<<"resource-type-debug-2">>), - emqx_rule_registry:register_resource_types([ResType1,ResType2]), - ok. -get_resource_type() -> - ?assertMatch({ok, #resource_type{name = <<"resource-type-debug-1">>}}, emqx_rule_registry:find_resource_type(<<"resource-type-debug-1">>)), - ok. -get_resource_types() -> - ResTypes = emqx_rule_registry:get_resource_types(), - ct:pal("resource types now: ~p", [ResTypes]), - ?assert(length(ResTypes) > 0), - ok. -unregister_resource_types_of() -> - NumOld = length(emqx_rule_registry:get_resource_types()), - ok = emqx_rule_registry:unregister_resource_types_of(?APP), - NumNow = length(emqx_rule_registry:get_resource_types()), - ?assert((NumOld - NumNow) >= 2), - ok. - %%------------------------------------------------------------------------------ %% Test cases for rule runtime %%------------------------------------------------------------------------------ @@ -966,37 +430,14 @@ message_acked(_Client) -> verify_event('message.acked'), ok. -t_mfa_action(_Config) -> - ok = emqx_rule_registry:add_action( - #action{name = 'mfa-action', app = ?APP, - module = ?MODULE, on_create = mfa_action, - types=[], params_spec = #{}, - title = #{en => <<"MFA callback action">>}, - description = #{en => <<"MFA callback action">>}}), - SQL = "SELECT * FROM \"t1\"", - {ok, #rule{id = Id}} = emqx_rule_engine:create_rule( - #{id => <<"rule:t_mfa_action">>, - rawsql => SQL, - actions => [#{id => <<"action:mfa-test">>, name => 'mfa-action', args => #{}}], - description => <<"Debug rule">>}), - {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), - {ok, _} = emqtt:connect(Client), - emqtt:publish(Client, <<"t1">>, <<"{\"id\": 1, \"name\": \"ha\"}">>, 0), - emqtt:stop(Client), - ct:sleep(500), - ?assertEqual(1, persistent_term:get(<<"action:mfa-test">>, 0)), - emqx_rule_registry:remove_rule(Id), - emqx_rule_registry:remove_action('mfa-action'), - ok. - t_match_atom_and_binary(_Config) -> - ok = emqx_rule_engine:load_providers(), - TopicRule = create_simple_repub_rule( - <<"t2">>, - "SELECT connected_at as ts, * " - "FROM \"$events/client_connected\" " - "WHERE username = 'emqx2' ", - <<"user:${ts}">>), + SQL = "SELECT connected_at as ts, * " + "FROM \"$events/client_connected\" " + "WHERE username = 'emqx2' ", + Repub = republish_output(<<"t2">>, <<"user:${ts}">>), + {ok, TopicRule} = emqx_rule_engine:create_rule( + #{sql => SQL, id => ?TMP_RULEID, + outputs => [Repub]}), {ok, Client} = emqtt:start_link([{username, <<"emqx1">>}]), {ok, _} = emqtt:connect(Client), {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), @@ -1021,32 +462,32 @@ t_sqlselect_0(_Config) -> "where payload.cmd.info = 'tt'", ?assertMatch({ok,#{payload := <<"{\"cmd\": {\"info\":\"tt\"}}">>}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => + #{sql => Sql, + context => + #{payload => <<"{\"cmd\": {\"info\":\"tt\"}}">>, - <<"topic">> => <<"t/a">>}})), + topic => <<"t/a">>}})), Sql2 = "select payload.cmd as cmd " "from \"t/#\" " "where cmd.info = 'tt'", ?assertMatch({ok,#{<<"cmd">> := #{<<"info">> := <<"tt">>}}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql2, - <<"ctx">> => - #{<<"payload">> => + #{sql => Sql2, + context => + #{payload => <<"{\"cmd\": {\"info\":\"tt\"}}">>, - <<"topic">> => <<"t/a">>}})), + topic => <<"t/a">>}})), Sql3 = "select payload.cmd as cmd, cmd.info as info " "from \"t/#\" " "where cmd.info = 'tt' and info = 'tt'", ?assertMatch({ok,#{<<"cmd">> := #{<<"info">> := <<"tt">>}, <<"info">> := <<"tt">>}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql3, - <<"ctx">> => - #{<<"payload">> => + #{sql => Sql3, + context => + #{payload => <<"{\"cmd\": {\"info\":\"tt\"}}">>, - <<"topic">> => <<"t/a">>}})), + topic => <<"t/a">>}})), %% cascaded as Sql4 = "select payload.cmd as cmd, cmd.info as meta.info " "from \"t/#\" " @@ -1054,11 +495,11 @@ t_sqlselect_0(_Config) -> ?assertMatch({ok,#{<<"cmd">> := #{<<"info">> := <<"tt">>}, <<"meta">> := #{<<"info">> := <<"tt">>}}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql4, - <<"ctx">> => - #{<<"payload">> => + #{sql => Sql4, + context => + #{payload => <<"{\"cmd\": {\"info\":\"tt\"}}">>, - <<"topic">> => <<"t/a">>}})). + topic => <<"t/a">>}})). t_sqlselect_00(_Config) -> %% Verify plus/subtract and unary_add_or_subtract @@ -1066,42 +507,43 @@ t_sqlselect_00(_Config) -> "from \"t/#\" ", ?assertMatch({ok,#{<<"a">> := 0}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => <<"">>, - <<"topic">> => <<"t/a">>}})), + #{sql => Sql, + context => + #{payload => <<"">>, + topic => <<"t/a">>}})), Sql1 = "select -1 + 1 as a " "from \"t/#\" ", ?assertMatch({ok,#{<<"a">> := 0}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql1, - <<"ctx">> => - #{<<"payload">> => <<"">>, - <<"topic">> => <<"t/a">>}})), + #{sql => Sql1, + context => + #{payload => <<"">>, + topic => <<"t/a">>}})), Sql2 = "select 1 + 1 as a " "from \"t/#\" ", ?assertMatch({ok,#{<<"a">> := 2}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql2, - <<"ctx">> => - #{<<"payload">> => <<"">>, - <<"topic">> => <<"t/a">>}})), + #{sql => Sql2, + context => + #{payload => <<"">>, + topic => <<"t/a">>}})), Sql3 = "select +1 as a " "from \"t/#\" ", ?assertMatch({ok,#{<<"a">> := 1}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql3, - <<"ctx">> => - #{<<"payload">> => <<"">>, - <<"topic">> => <<"t/a">>}})). + #{sql => Sql3, + context => + #{payload => <<"">>, + topic => <<"t/a">>}})). t_sqlselect_01(_Config) -> - ok = emqx_rule_engine:load_providers(), - TopicRule1 = create_simple_repub_rule( - <<"t2">>, - "SELECT json_decode(payload) as p, payload " - "FROM \"t3/#\", \"t1\" " - "WHERE p.x = 1"), + SQL = "SELECT json_decode(payload) as p, payload " + "FROM \"t3/#\", \"t1\" " + "WHERE p.x = 1", + Repub = republish_output(<<"t2">>), + {ok, TopicRule1} = emqx_rule_engine:create_rule( + #{sql => SQL, id => ?TMP_RULEID, + outputs => [Repub]}), {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), {ok, _} = emqtt:connect(Client), {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), @@ -1133,12 +575,13 @@ t_sqlselect_01(_Config) -> emqx_rule_registry:remove_rule(TopicRule1). t_sqlselect_02(_Config) -> - ok = emqx_rule_engine:load_providers(), - TopicRule1 = create_simple_repub_rule( - <<"t2">>, - "SELECT * " - "FROM \"t3/#\", \"t1\" " - "WHERE payload.x = 1"), + SQL = "SELECT * " + "FROM \"t3/#\", \"t1\" " + "WHERE payload.x = 1", + Repub = republish_output(<<"t2">>), + {ok, TopicRule1} = emqx_rule_engine:create_rule( + #{sql => SQL, id => ?TMP_RULEID, + outputs => [Repub]}), {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), {ok, _} = emqtt:connect(Client), {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), @@ -1170,12 +613,13 @@ t_sqlselect_02(_Config) -> emqx_rule_registry:remove_rule(TopicRule1). t_sqlselect_1(_Config) -> - ok = emqx_rule_engine:load_providers(), - TopicRule = create_simple_repub_rule( - <<"t2">>, - "SELECT json_decode(payload) as p, payload " - "FROM \"t1\" " - "WHERE p.x = 1 and p.y = 2"), + SQL = "SELECT json_decode(payload) as p, payload " + "FROM \"t1\" " + "WHERE p.x = 1 and p.y = 2", + Repub = republish_output(<<"t2">>), + {ok, TopicRule} = emqx_rule_engine:create_rule( + #{sql => SQL, id => ?TMP_RULEID, + outputs => [Repub]}), {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), {ok, _} = emqtt:connect(Client), {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), @@ -1199,12 +643,12 @@ t_sqlselect_1(_Config) -> emqx_rule_registry:remove_rule(TopicRule). t_sqlselect_2(_Config) -> - ok = emqx_rule_engine:load_providers(), %% recursively republish to t2 - TopicRule = create_simple_repub_rule( - <<"t2">>, - "SELECT * " - "FROM \"t2\" "), + SQL = "SELECT * FROM \"t2\" ", + Repub = republish_output(<<"t2">>), + {ok, TopicRule} = emqx_rule_engine:create_rule( + #{sql => SQL, id => ?TMP_RULEID, + outputs => [Repub]}), {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), {ok, _} = emqtt:connect(Client), {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), @@ -1224,91 +668,15 @@ t_sqlselect_2(_Config) -> emqtt:stop(Client), emqx_rule_registry:remove_rule(TopicRule). -t_sqlselect_2_1(_Config) -> - ok = emqx_rule_engine:load_providers(), - %% recursively republish to t2, if the msg dropped - TopicRule = create_simple_repub_rule( - <<"t2">>, - "SELECT * " - "FROM \"$events/message_dropped\" "), - {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), - {ok, _} = emqtt:connect(Client), - emqtt:publish(Client, <<"t2">>, <<"{\"x\":1,\"y\":144}">>, 0), - Fun = fun() -> - receive {publish, #{topic := <<"t2">>, payload := _}} -> - received_t2 - after 500 -> - received_nothing - end - end, - received_nothing = Fun(), - - %% it should not keep republishing "t2" - {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), - received_nothing = Fun(), - - emqtt:stop(Client), - emqx_rule_registry:remove_rule(TopicRule). - -t_sqlselect_2_2(_Config) -> - ok = emqx_rule_engine:load_providers(), - %% recursively republish to t2, if the msg acked - TopicRule = create_simple_repub_rule( - <<"t2">>, - "SELECT * " - "FROM \"$events/message_acked\" "), - {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), - {ok, _} = emqtt:connect(Client), - {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 1), - emqtt:publish(Client, <<"t2">>, <<"{\"x\":1,\"y\":144}">>, 1), - Fun = fun() -> - receive {publish, #{topic := <<"t2">>, payload := _}} -> - received_t2 - after 500 -> - received_nothing - end - end, - received_t2 = Fun(), - received_t2 = Fun(), - received_nothing = Fun(), - - emqtt:stop(Client), - emqx_rule_registry:remove_rule(TopicRule). - -t_sqlselect_2_3(_Config) -> - ok = emqx_rule_engine:load_providers(), - %% recursively republish to t2, if the msg delivered - TopicRule = create_simple_repub_rule( - <<"t2">>, - "SELECT * " - "FROM \"$events/message_delivered\" "), - {ok, Client} = emqtt:start_link([{username, <<"emqx">>}]), - {ok, _} = emqtt:connect(Client), - {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), - emqtt:publish(Client, <<"t2">>, <<"{\"x\":1,\"y\":144}">>, 0), - Fun = fun() -> - receive {publish, #{topic := <<"t2">>, payload := _}} -> - received_t2 - after 500 -> - received_nothing - end - end, - received_t2 = Fun(), - received_t2 = Fun(), - received_nothing = Fun(), - - emqtt:stop(Client), - emqx_rule_registry:remove_rule(TopicRule). - t_sqlselect_3(_Config) -> - ok = emqx_rule_engine:load_providers(), %% republish the client.connected msg - TopicRule = create_simple_repub_rule( - <<"t2">>, - "SELECT * " - "FROM \"$events/client_connected\" " - "WHERE username = 'emqx1'", - <<"clientid=${clientid}">>), + SQL = "SELECT * " + "FROM \"$events/client_connected\" " + "WHERE username = 'emqx1'", + Repub = republish_output(<<"t2">>, <<"clientid=${clientid}">>), + {ok, TopicRule} = emqx_rule_engine:create_rule( + #{sql => SQL, id => ?TMP_RULEID, + outputs => [Repub]}), {ok, Client} = emqtt:start_link([{clientid, <<"emqx0">>}, {username, <<"emqx0">>}]), {ok, _} = emqtt:connect(Client), {ok, _, _} = emqtt:subscribe(Client, <<"t2">>, 0), @@ -1332,226 +700,29 @@ t_sqlselect_3(_Config) -> emqtt:stop(Client), emqx_rule_registry:remove_rule(TopicRule). -t_sqlselect_multi_actoins_1(Config) -> - %% We create 2 actions in the same rule: - %% The first will fail and we need to make sure the - %% second one can still execute as the on_action_failed - %% defaults to 'continue' - {ok, Rule} = emqx_rule_engine:create_rule( - #{rawsql => ?config(connsql, Config), - actions => [ - #{name => 'crash_action', args => #{}, fallbacks => []}, - #{name => 'republish', - args => #{<<"target_topic">> => <<"t2">>, - <<"target_qos">> => -1, - <<"payload_tmpl">> => <<"clientid=${clientid}">> - }, - fallbacks => []} - ] - }), - - (?config(conn_event, Config))(), - receive {publish, #{topic := T, payload := Payload}} -> - ?assertEqual(<<"t2">>, T), - ?assertEqual(<<"clientid=c_emqx1">>, Payload) - after 1000 -> - ct:fail(wait_for_t2) - end, - - emqx_rule_registry:remove_rule(Rule). - -t_sqlselect_multi_actoins_1_1(Config) -> - %% Try again but set on_action_failed = 'continue' explicitly - {ok, Rule2} = emqx_rule_engine:create_rule( - #{rawsql => ?config(connsql, Config), - on_action_failed => 'continue', - actions => [ - #{name => 'crash_action', args => #{}, fallbacks => []}, - #{name => 'republish', - args => #{<<"target_topic">> => <<"t2">>, - <<"target_qos">> => -1, - <<"payload_tmpl">> => <<"clientid=${clientid}">> - }, - fallbacks => []} - ] - }), - - (?config(conn_event, Config))(), - receive {publish, #{topic := T2, payload := Payload2}} -> - ?assertEqual(<<"t2">>, T2), - ?assertEqual(<<"clientid=c_emqx1">>, Payload2) - after 1000 -> - ct:fail(wait_for_t2) - end, - - emqx_rule_registry:remove_rule(Rule2). - -t_sqlselect_multi_actoins_2(Config) -> - %% We create 2 actions in the same rule: - %% The first will fail and we need to make sure the - %% second one cannot execute as we've set the on_action_failed = 'stop' - {ok, Rule} = emqx_rule_engine:create_rule( - #{rawsql => ?config(connsql, Config), - on_action_failed => stop, - actions => [ - #{name => 'crash_action', args => #{}, fallbacks => []}, - #{name => 'republish', - args => #{<<"target_topic">> => <<"t2">>, - <<"target_qos">> => -1, - <<"payload_tmpl">> => <<"clientid=${clientid}">> - }, - fallbacks => []} - ] - }), - - (?config(conn_event, Config))(), - receive {publish, #{topic := <<"t2">>}} -> - ct:fail(unexpected_t2) - after 1000 -> - ok - end, - - emqx_rule_registry:remove_rule(Rule). - -t_sqlselect_multi_actoins_3(Config) -> - %% We create 2 actions in the same rule (on_action_failed = continue): - %% The first will fail and we need to make sure the - %% fallback actions can be executed, and the next actoins - %% will be run without influence - {ok, Rule} = emqx_rule_engine:create_rule( - #{rawsql => ?config(connsql, Config), - on_action_failed => continue, - actions => [ - #{name => 'crash_action', args => #{}, fallbacks =>[ - #{name => 'plus_by_one', args => #{}, fallbacks =>[]}, - #{name => 'plus_by_one', args => #{}, fallbacks =>[]} - ]}, - #{name => 'republish', - args => #{<<"target_topic">> => <<"t2">>, - <<"target_qos">> => -1, - <<"payload_tmpl">> => <<"clientid=${clientid}">> - }, - fallbacks => []} - ] - }), - - (?config(conn_event, Config))(), - timer:sleep(100), - - %% verfiy the fallback actions has been run - ?assertEqual(2, ets:lookup_element(plus_by_one_action, num, 2)), - - %% verfiy the next actions can be run - receive {publish, #{topic := T, payload := Payload}} -> - ?assertEqual(<<"t2">>, T), - ?assertEqual(<<"clientid=c_emqx1">>, Payload) - after 1000 -> - ct:fail(wait_for_t2) - end, - - emqx_rule_registry:remove_rule(Rule). - -t_sqlselect_multi_actoins_3_1(Config) -> - %% We create 2 actions in the same rule (on_action_failed = continue): - %% The first will fail (with a 'badact' return) and we need to make sure the - %% fallback actions can be executed, and the next actoins - %% will be run without influence - {ok, Rule} = emqx_rule_engine:create_rule( - #{rawsql => ?config(connsql, Config), - on_action_failed => continue, - actions => [ - #{name => 'failure_action', args => #{}, fallbacks =>[ - #{name => 'plus_by_one', args => #{}, fallbacks =>[]}, - #{name => 'plus_by_one', args => #{}, fallbacks =>[]} - ]}, - #{name => 'republish', - args => #{<<"target_topic">> => <<"t2">>, - <<"target_qos">> => -1, - <<"payload_tmpl">> => <<"clientid=${clientid}">> - }, - fallbacks => []} - ] - }), - - (?config(conn_event, Config))(), - timer:sleep(100), - - %% verfiy the fallback actions has been run - ?assertEqual(2, ets:lookup_element(plus_by_one_action, num, 2)), - - %% verfiy the next actions can be run - receive {publish, #{topic := T, payload := Payload}} -> - ?assertEqual(<<"t2">>, T), - ?assertEqual(<<"clientid=c_emqx1">>, Payload) - after 1000 -> - ct:fail(wait_for_t2) - end, - - emqx_rule_registry:remove_rule(Rule). - -t_sqlselect_multi_actoins_4(Config) -> - %% We create 2 actions in the same rule (on_action_failed = continue): - %% The first will fail and we need to make sure the - %% fallback actions can be executed, and the next actoins - %% will be run without influence - {ok, Rule} = emqx_rule_engine:create_rule( - #{rawsql => ?config(connsql, Config), - on_action_failed => continue, - actions => [ - #{name => 'crash_action', args => #{}, fallbacks => [ - #{name =>'plus_by_one', args => #{}, fallbacks =>[]}, - #{name =>'crash_action', args => #{}, fallbacks =>[]}, - #{name =>'plus_by_one', args => #{}, fallbacks =>[]} - ]}, - #{name => 'republish', - args => #{<<"target_topic">> => <<"t2">>, - <<"target_qos">> => -1, - <<"payload_tmpl">> => <<"clientid=${clientid}">> - }, - fallbacks => []} - ] - }), - - (?config(conn_event, Config))(), - timer:sleep(100), - - %% verfiy all the fallback actions were run, even if the second - %% fallback action crashed - ?assertEqual(2, ets:lookup_element(plus_by_one_action, num, 2)), - - %% verfiy the next actions can be run - receive {publish, #{topic := T, payload := Payload}} -> - ?assertEqual(<<"t2">>, T), - ?assertEqual(<<"clientid=c_emqx1">>, Payload) - after 1000 -> - ct:fail(wait_for_t2) - end, - - emqx_rule_registry:remove_rule(Rule). - t_sqlparse_event_1(_Config) -> Sql = "select topic as tp " "from \"$events/session_subscribed\" ", ?assertMatch({ok,#{<<"tp">> := <<"t/tt">>}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"topic">> => <<"t/tt">>}})). + #{sql => Sql, + context => #{topic => <<"t/tt">>}})). t_sqlparse_event_2(_Config) -> Sql = "select clientid " "from \"$events/client_connected\" ", ?assertMatch({ok,#{<<"clientid">> := <<"abc">>}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"clientid">> => <<"abc">>}})). + #{sql => Sql, + context => #{clientid => <<"abc">>}})). t_sqlparse_event_3(_Config) -> Sql = "select clientid, topic as tp " "from \"t/tt\", \"$events/client_connected\" ", ?assertMatch({ok,#{<<"clientid">> := <<"abc">>, <<"tp">> := <<"t/tt">>}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"clientid">> => <<"abc">>, <<"topic">> => <<"t/tt">>}})). + #{sql => Sql, + context => #{clientid => <<"abc">>, topic => <<"t/tt">>}})). t_sqlparse_foreach_1(_Config) -> %% Verify foreach with and without 'AS' @@ -1559,34 +730,35 @@ t_sqlparse_foreach_1(_Config) -> "from \"t/#\" ", ?assertMatch({ok,[#{<<"s">> := 1}, #{<<"s">> := 2}]}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"sensors\": [1, 2]}">>, - <<"topic">> => <<"t/a">>}})), + #{sql => Sql, + context => #{payload => <<"{\"sensors\": [1, 2]}">>, + topic => <<"t/a">>}})), Sql2 = "foreach payload.sensors " "from \"t/#\" ", ?assertMatch({ok,[#{item := 1}, #{item := 2}]}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql2, - <<"ctx">> => #{<<"payload">> => <<"{\"sensors\": [1, 2]}">>, - <<"topic">> => <<"t/a">>}})), + #{sql => Sql2, + context => #{payload => <<"{\"sensors\": [1, 2]}">>, + topic => <<"t/a">>}})), Sql3 = "foreach payload.sensors " "from \"t/#\" ", ?assertMatch({ok,[#{item := #{<<"cmd">> := <<"1">>}, clientid := <<"c_a">>}, #{item := #{<<"cmd">> := <<"2">>, <<"name">> := <<"ct">>}, clientid := <<"c_a">>}]}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql3, - <<"ctx">> => #{ - <<"payload">> => <<"{\"sensors\": [{\"cmd\":\"1\"}, {\"cmd\":\"2\",\"name\":\"ct\"}]}">>, <<"clientid">> => <<"c_a">>, - <<"topic">> => <<"t/a">>}})), + #{sql => Sql3, + context => #{ + payload => <<"{\"sensors\": [{\"cmd\":\"1\"}, {\"cmd\":\"2\",\"name\":\"ct\"}]}">>, + clientid => <<"c_a">>, + topic => <<"t/a">>}})), Sql4 = "foreach payload.sensors " "from \"t/#\" ", {ok,[#{metadata := #{rule_id := TRuleId}}, #{metadata := #{rule_id := TRuleId}}]} = emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql4, - <<"ctx">> => #{ - <<"payload">> => <<"{\"sensors\": [1, 2]}">>, - <<"topic">> => <<"t/a">>}}), + #{sql => Sql4, + context => #{ + payload => <<"{\"sensors\": [1, 2]}">>, + topic => <<"t/a">>}}), ?assert(is_binary(TRuleId)). t_sqlparse_foreach_2(_Config) -> @@ -1596,31 +768,31 @@ t_sqlparse_foreach_2(_Config) -> "from \"t/#\" ", ?assertMatch({ok,[#{<<"msg_type">> := <<"1">>},#{<<"msg_type">> := <<"2">>}]}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => + #{sql => Sql, + context => + #{payload => <<"{\"sensors\": [{\"cmd\":\"1\"}, {\"cmd\":\"2\"}]}">>, - <<"topic">> => <<"t/a">>}})), + topic => <<"t/a">>}})), Sql2 = "foreach payload.sensors " "do item.cmd as msg_type " "from \"t/#\" ", ?assertMatch({ok,[#{<<"msg_type">> := <<"1">>},#{<<"msg_type">> := <<"2">>}]}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql2, - <<"ctx">> => - #{<<"payload">> => + #{sql => Sql2, + context => + #{payload => <<"{\"sensors\": [{\"cmd\":\"1\"}, {\"cmd\":\"2\"}]}">>, - <<"topic">> => <<"t/a">>}})), + topic => <<"t/a">>}})), Sql3 = "foreach payload.sensors " "do item as item " "from \"t/#\" ", ?assertMatch({ok,[#{<<"item">> := 1},#{<<"item">> := 2}]}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql3, - <<"ctx">> => - #{<<"payload">> => + #{sql => Sql3, + context => + #{payload => <<"{\"sensors\": [1, 2]}">>, - <<"topic">> => <<"t/a">>}})). + topic => <<"t/a">>}})). t_sqlparse_foreach_3(_Config) -> %% Verify foreach-incase with and without 'AS' @@ -1631,11 +803,11 @@ t_sqlparse_foreach_3(_Config) -> #{<<"s">> := #{<<"cmd">> := 3}} ]}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => + #{sql => Sql, + context => + #{payload => <<"{\"sensors\": [{\"cmd\":1}, {\"cmd\":2}, {\"cmd\":3}]}">>, - <<"topic">> => <<"t/a">>}})), + topic => <<"t/a">>}})), Sql2 = "foreach payload.sensors " "incase item.cmd != 1 " "from \"t/#\" ", @@ -1643,11 +815,11 @@ t_sqlparse_foreach_3(_Config) -> #{item := #{<<"cmd">> := 3}} ]}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql2, - <<"ctx">> => - #{<<"payload">> => + #{sql => Sql2, + context => + #{payload => <<"{\"sensors\": [{\"cmd\":1}, {\"cmd\":2}, {\"cmd\":3}]}">>, - <<"topic">> => <<"t/a">>}})). + topic => <<"t/a">>}})). t_sqlparse_foreach_4(_Config) -> %% Verify foreach-do-incase @@ -1657,24 +829,24 @@ t_sqlparse_foreach_4(_Config) -> "from \"t/#\" ", ?assertMatch({ok,[#{<<"msg_type">> := <<"1">>},#{<<"msg_type">> := <<"2">>}]}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => + #{sql => Sql, + context => + #{payload => <<"{\"sensors\": [{\"cmd\":\"1\"}, {\"cmd\":\"2\"}]}">>, - <<"topic">> => <<"t/a">>}})), + topic => <<"t/a">>}})), ?assertMatch({ok,[#{<<"msg_type">> := <<"1">>, <<"name">> := <<"n1">>}, #{<<"msg_type">> := <<"2">>}]}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => + #{sql => Sql, + context => + #{payload => <<"{\"sensors\": [{\"cmd\":\"1\", \"name\":\"n1\"}, {\"cmd\":\"2\"}, {\"name\":\"n3\"}]}">>, - <<"topic">> => <<"t/a">>}})), + topic => <<"t/a">>}})), ?assertMatch({ok,[]}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => <<"{\"sensors\": [1, 2]}">>, - <<"topic">> => <<"t/a">>}})). + #{sql => Sql, + context => + #{payload => <<"{\"sensors\": [1, 2]}">>, + topic => <<"t/a">>}})). t_sqlparse_foreach_5(_Config) -> %% Verify foreach on a empty-list or non-list variable @@ -1682,23 +854,23 @@ t_sqlparse_foreach_5(_Config) -> "do s.cmd as msg_type, s.name as name " "from \"t/#\" ", ?assertMatch({ok,[]}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => <<"{\"sensors\": 1}">>, - <<"topic">> => <<"t/a">>}})), + #{sql => Sql, + context => + #{payload => <<"{\"sensors\": 1}">>, + topic => <<"t/a">>}})), ?assertMatch({ok,[]}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => <<"{\"sensors\": []}">>, - <<"topic">> => <<"t/a">>}})), + #{sql => Sql, + context => + #{payload => <<"{\"sensors\": []}">>, + topic => <<"t/a">>}})), Sql2 = "foreach payload.sensors " "from \"t/#\" ", ?assertMatch({ok,[]}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql2, - <<"ctx">> => - #{<<"payload">> => <<"{\"sensors\": 1}">>, - <<"topic">> => <<"t/a">>}})). + #{sql => Sql2, + context => + #{payload => <<"{\"sensors\": 1}">>, + topic => <<"t/a">>}})). t_sqlparse_foreach_6(_Config) -> %% Verify foreach on a empty-list or non-list variable @@ -1706,10 +878,10 @@ t_sqlparse_foreach_6(_Config) -> "do item.id as zid, timestamp as t " "from \"t/#\" ", {ok, Res} = emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => <<"[{\"id\": 5},{\"id\": 15}]">>, - <<"topic">> => <<"t/a">>}}), + #{sql => Sql, + context => + #{payload => <<"[{\"id\": 5},{\"id\": 15}]">>, + topic => <<"t/a">>}}), [#{<<"t">> := Ts1, <<"zid">> := Zid1}, #{<<"t">> := Ts2, <<"zid">> := Zid2}] = Res, ?assertEqual(true, is_integer(Ts1)), @@ -1727,10 +899,10 @@ t_sqlparse_foreach_7(_Config) -> Payload = <<"{\"sensors\": {\"page\": 2, \"collection\": {\"info\":[{\"name\":\"cmd1\", \"cmd\":\"1\"}, {\"cmd\":\"2\"}]} } }">>, ?assertMatch({ok,[#{<<"name">> := <<"cmd1">>, <<"msg_type">> := <<"1">>}, #{<<"msg_type">> := <<"2">>}]}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => Payload, - <<"topic">> => <<"t/a">>}})), + #{sql => Sql, + context => + #{payload => Payload, + topic => <<"t/a">>}})), Sql2 = "foreach json_decode(payload) as p, p.sensors as s, s.collection as c, c.info as info " "do info.cmd as msg_type, info.name as name " "incase is_not_null(info.cmd) " @@ -1738,10 +910,10 @@ t_sqlparse_foreach_7(_Config) -> "where s.page = '3' ", ?assertMatch({error, nomatch}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql2, - <<"ctx">> => - #{<<"payload">> => Payload, - <<"topic">> => <<"t/a">>}})). + #{sql => Sql2, + context => + #{payload => Payload, + topic => <<"t/a">>}})). t_sqlparse_foreach_8(_Config) -> %% Verify foreach-do-incase and cascaded AS @@ -1753,10 +925,10 @@ t_sqlparse_foreach_8(_Config) -> Payload = <<"{\"sensors\": {\"page\": 2, \"collection\": {\"info\":[\"haha\", {\"name\":\"cmd1\", \"cmd\":\"1\"}]} } }">>, ?assertMatch({ok,[#{<<"name">> := <<"cmd1">>, <<"msg_type">> := <<"1">>}]}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => - #{<<"payload">> => Payload, - <<"topic">> => <<"t/a">>}})), + #{sql => Sql, + context => + #{payload => Payload, + topic => <<"t/a">>}})), Sql3 = "foreach json_decode(payload) as p, p.sensors as s, s.collection as c, sublist(2,1,c.info) as info " "do info.cmd as msg_type, info.name as name " @@ -1764,10 +936,10 @@ t_sqlparse_foreach_8(_Config) -> "where s.page = '2' ", [?assertMatch({ok,[#{<<"name">> := <<"cmd1">>, <<"msg_type">> := <<"1">>}]}, emqx_rule_sqltester:test( - #{<<"rawsql">> => SqlN, - <<"ctx">> => - #{<<"payload">> => Payload, - <<"topic">> => <<"t/a">>}})) + #{sql => SqlN, + context => + #{payload => Payload, + topic => <<"t/a">>}})) || SqlN <- [Sql3]]. t_sqlparse_case_when_1(_Config) -> @@ -1779,25 +951,25 @@ t_sqlparse_case_when_1(_Config) -> " end as y " "from \"t/#\" ", ?assertMatch({ok, #{<<"y">> := 1}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 1}">>, - <<"topic">> => <<"t/a">>}})), + #{sql => Sql, + context => #{payload => <<"{\"x\": 1}">>, + topic => <<"t/a">>}})), ?assertMatch({ok, #{<<"y">> := 0}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 0}">>, - <<"topic">> => <<"t/a">>}})), + #{sql => Sql, + context => #{payload => <<"{\"x\": 0}">>, + topic => <<"t/a">>}})), ?assertMatch({ok, #{<<"y">> := 0}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": -1}">>, - <<"topic">> => <<"t/a">>}})), + #{sql => Sql, + context => #{payload => <<"{\"x\": -1}">>, + topic => <<"t/a">>}})), ?assertMatch({ok, #{<<"y">> := 7}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 7}">>, - <<"topic">> => <<"t/a">>}})), + #{sql => Sql, + context => #{payload => <<"{\"x\": 7}">>, + topic => <<"t/a">>}})), ?assertMatch({ok, #{<<"y">> := 7}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 8}">>, - <<"topic">> => <<"t/a">>}})), + #{sql => Sql, + context => #{payload => <<"{\"x\": 8}">>, + topic => <<"t/a">>}})), ok. t_sqlparse_case_when_2(_Config) -> @@ -1809,25 +981,25 @@ t_sqlparse_case_when_2(_Config) -> " end as y " "from \"t/#\" ", ?assertMatch({ok, #{<<"y">> := 2}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 1}">>, - <<"topic">> => <<"t/a">>}})), + #{sql => Sql, + context => #{payload => <<"{\"x\": 1}">>, + topic => <<"t/a">>}})), ?assertMatch({ok, #{<<"y">> := 3}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 2}">>, - <<"topic">> => <<"t/a">>}})), + #{sql => Sql, + context => #{payload => <<"{\"x\": 2}">>, + topic => <<"t/a">>}})), ?assertMatch({ok, #{<<"y">> := 4}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 4}">>, - <<"topic">> => <<"t/a">>}})), + #{sql => Sql, + context => #{payload => <<"{\"x\": 4}">>, + topic => <<"t/a">>}})), ?assertMatch({ok, #{<<"y">> := 4}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 7}">>, - <<"topic">> => <<"t/a">>}})), + #{sql => Sql, + context => #{payload => <<"{\"x\": 7}">>, + topic => <<"t/a">>}})), ?assertMatch({ok, #{<<"y">> := 4}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 8}">>, - <<"topic">> => <<"t/a">>}})). + #{sql => Sql, + context => #{payload => <<"{\"x\": 8}">>, + topic => <<"t/a">>}})). t_sqlparse_case_when_3(_Config) -> %% case-when clause @@ -1837,29 +1009,29 @@ t_sqlparse_case_when_3(_Config) -> " end as y " "from \"t/#\" ", ?assertMatch({ok, #{}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 1}">>, - <<"topic">> => <<"t/a">>}})), + #{sql => Sql, + context => #{payload => <<"{\"x\": 1}">>, + topic => <<"t/a">>}})), ?assertMatch({ok, #{}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 5}">>, - <<"topic">> => <<"t/a">>}})), + #{sql => Sql, + context => #{payload => <<"{\"x\": 5}">>, + topic => <<"t/a">>}})), ?assertMatch({ok, #{}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 0}">>, - <<"topic">> => <<"t/a">>}})), + #{sql => Sql, + context => #{payload => <<"{\"x\": 0}">>, + topic => <<"t/a">>}})), ?assertMatch({ok, #{<<"y">> := 0}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": -1}">>, - <<"topic">> => <<"t/a">>}})), + #{sql => Sql, + context => #{payload => <<"{\"x\": -1}">>, + topic => <<"t/a">>}})), ?assertMatch({ok, #{}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 7}">>, - <<"topic">> => <<"t/a">>}})), + #{sql => Sql, + context => #{payload => <<"{\"x\": 7}">>, + topic => <<"t/a">>}})), ?assertMatch({ok, #{<<"y">> := 7}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 8}">>, - <<"topic">> => <<"t/a">>}})), + #{sql => Sql, + context => #{payload => <<"{\"x\": 8}">>, + topic => <<"t/a">>}})), ok. t_sqlparse_array_index_1(_Config) -> @@ -1869,38 +1041,38 @@ t_sqlparse_array_index_1(_Config) -> " p[1] as a " "from \"t/#\" ", ?assertMatch({ok, #{<<"a">> := #{<<"x">> := 1}}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"[{\"x\": 1}]">>, - <<"topic">> => <<"t/a">>}})), + #{sql => Sql, + context => #{payload => <<"[{\"x\": 1}]">>, + topic => <<"t/a">>}})), ?assertMatch({ok, #{}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": 1}">>, - <<"topic">> => <<"t/a">>}})), + #{sql => Sql, + context => #{payload => <<"{\"x\": 1}">>, + topic => <<"t/a">>}})), %% index get without 'as' Sql2 = "select " " payload.x[2] " "from \"t/#\" ", ?assertMatch({ok, #{<<"payload">> := #{<<"x">> := [3]}}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql2, - <<"ctx">> => #{<<"payload">> => #{<<"x">> => [1,3,4]}, - <<"topic">> => <<"t/a">>}})), + #{sql => Sql2, + context => #{payload => #{<<"x">> => [1,3,4]}, + topic => <<"t/a">>}})), %% index get without 'as' again Sql3 = "select " " payload.x[2].y " "from \"t/#\" ", ?assertMatch({ok, #{<<"payload">> := #{<<"x">> := [#{<<"y">> := 3}]}}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql3, - <<"ctx">> => #{<<"payload">> => #{<<"x">> => [1,#{y => 3},4]}, - <<"topic">> => <<"t/a">>}})), + #{sql => Sql3, + context => #{payload => #{<<"x">> => [1,#{y => 3},4]}, + topic => <<"t/a">>}})), %% index get with 'as' Sql4 = "select " " payload.x[2].y as b " "from \"t/#\" ", ?assertMatch({ok, #{<<"b">> := 3}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql4, - <<"ctx">> => #{<<"payload">> => #{<<"x">> => [1,#{y => 3},4]}, - <<"topic">> => <<"t/a">>}})). + #{sql => Sql4, + context => #{payload => #{<<"x">> => [1,#{y => 3},4]}, + topic => <<"t/a">>}})). t_sqlparse_array_index_2(_Config) -> %% array get with negative index @@ -1908,9 +1080,9 @@ t_sqlparse_array_index_2(_Config) -> " payload.x[-2].y as b " "from \"t/#\" ", ?assertMatch({ok, #{<<"b">> := 3}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql1, - <<"ctx">> => #{<<"payload">> => #{<<"x">> => [1,#{y => 3},4]}, - <<"topic">> => <<"t/a">>}})), + #{sql => Sql1, + context => #{payload => #{<<"x">> => [1,#{y => 3},4]}, + topic => <<"t/a">>}})), %% array append to head or tail of a list: Sql2 = "select " " payload.x as b, " @@ -1919,9 +1091,9 @@ t_sqlparse_array_index_2(_Config) -> " b as c[0] " "from \"t/#\" ", ?assertMatch({ok, #{<<"b">> := 0, <<"c">> := [0,1,2]}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql2, - <<"ctx">> => #{<<"payload">> => #{<<"x">> => 0}, - <<"topic">> => <<"t/a">>}})), + #{sql => Sql2, + context => #{payload => #{<<"x">> => 0}, + topic => <<"t/a">>}})), %% construct an empty list: Sql3 = "select " " [] as c, " @@ -1930,9 +1102,9 @@ t_sqlparse_array_index_2(_Config) -> " 0 as c[0] " "from \"t/#\" ", ?assertMatch({ok, #{<<"c">> := [0,1,2]}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql3, - <<"ctx">> => #{<<"payload">> => <<"">>, - <<"topic">> => <<"t/a">>}})), + #{sql => Sql3, + context => #{payload => <<"">>, + topic => <<"t/a">>}})), %% construct a list: Sql4 = "select " " [payload.a, \"topic\", 'c'] as c, " @@ -1941,9 +1113,9 @@ t_sqlparse_array_index_2(_Config) -> " 0 as c[0] " "from \"t/#\" ", ?assertMatch({ok, #{<<"c">> := [0,11,<<"t/a">>,<<"c">>,1,2]}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql4, - <<"ctx">> => #{<<"payload">> => <<"{\"a\":11}">>, - <<"topic">> => <<"t/a">> + #{sql => Sql4, + context => #{payload => <<"{\"a\":11}">>, + topic => <<"t/a">> }})). t_sqlparse_array_index_3(_Config) -> @@ -1954,25 +1126,25 @@ t_sqlparse_array_index_3(_Config) -> "from \"t/#\" ", ?assertMatch({ok, #{<<"payload">> := #{<<"x">> := [1, #{<<"y">> := [1,2]}, 3]}}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql0, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": [1,{\"y\": [1,2]},3]}">>, - <<"topic">> => <<"t/a">>}})), + #{sql => Sql0, + context => #{payload => <<"{\"x\": [1,{\"y\": [1,2]},3]}">>, + topic => <<"t/a">>}})), %% same as above but don't select payload: Sql1 = "select " "payload.x[2].y as b " "from \"t/#\" ", ?assertMatch({ok, #{<<"b">> := [1,2]}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql1, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": [1,{\"y\": [1,2]},3]}">>, - <<"topic">> => <<"t/a">>}})), + #{sql => Sql1, + context => #{payload => <<"{\"x\": [1,{\"y\": [1,2]},3]}">>, + topic => <<"t/a">>}})), %% same as above but add 'as' clause: Sql2 = "select " "payload.x[2].y as b.c " "from \"t/#\" ", ?assertMatch({ok, #{<<"b">> := #{<<"c">> := [1,2]}}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql2, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": [1,{\"y\": [1,2]},3]}">>, - <<"topic">> => <<"t/a">>}})). + #{sql => Sql2, + context => #{payload => <<"{\"x\": [1,{\"y\": [1,2]},3]}">>, + topic => <<"t/a">>}})). t_sqlparse_array_index_4(_Config) -> %% array with json string payload: @@ -1981,9 +1153,9 @@ t_sqlparse_array_index_4(_Config) -> "from \"t/#\" ", ?assertMatch({ok, #{<<"payload">> := #{<<"x">> := [#{<<"y">> := 0}]}}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql0, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": [1,{\"y\": [1,2]},3]}">>, - <<"topic">> => <<"t/a">>}})), + #{sql => Sql0, + context => #{payload => <<"{\"x\": [1,{\"y\": [1,2]},3]}">>, + topic => <<"t/a">>}})), %% array with json string payload, and also select payload.x: Sql1 = "select " "payload.x, " @@ -1991,9 +1163,9 @@ t_sqlparse_array_index_4(_Config) -> "from \"t/#\" ", ?assertMatch({ok, #{<<"payload">> := #{<<"x">> := [1, #{<<"y">> := 0}, 3]}}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql1, - <<"ctx">> => #{<<"payload">> => <<"{\"x\": [1,{\"y\": [1,2]},3]}">>, - <<"topic">> => <<"t/a">>}})). + #{sql => Sql1, + context => #{payload => <<"{\"x\": [1,{\"y\": [1,2]},3]}">>, + topic => <<"t/a">>}})). t_sqlparse_array_index_5(_Config) -> Sql00 = "select " @@ -2001,9 +1173,9 @@ t_sqlparse_array_index_5(_Config) -> "from \"t/#\" ", {ok, Res00} = emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql00, - <<"ctx">> => #{<<"payload">> => <<"">>, - <<"topic">> => <<"t/a">>}}), + #{sql => Sql00, + context => #{payload => <<"">>, + topic => <<"t/a">>}}), ?assert(lists:any(fun({_K, V}) -> V =:= [1,2,3,4] end, maps:to_list(Res00))). @@ -2015,17 +1187,17 @@ t_sqlparse_select_matadata_1(_Config) -> "from \"t/#\" ", ?assertNotMatch({ok, #{<<"payload">> := <<"abc">>, metadata := _}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql0, - <<"ctx">> => #{<<"payload">> => <<"abc">>, - <<"topic">> => <<"t/a">>}})), + #{sql => Sql0, + context => #{payload => <<"abc">>, + topic => <<"t/a">>}})), Sql1 = "select " "payload, metadata " "from \"t/#\" ", ?assertMatch({ok, #{<<"payload">> := <<"abc">>, <<"metadata">> := _}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql1, - <<"ctx">> => #{<<"payload">> => <<"abc">>, - <<"topic">> => <<"t/a">>}})). + #{sql => Sql1, + context => #{payload => <<"abc">>, + topic => <<"t/a">>}})). t_sqlparse_array_range_1(_Config) -> %% get a range of list @@ -2033,19 +1205,19 @@ t_sqlparse_array_range_1(_Config) -> " payload.a[1..4] as c " "from \"t/#\" ", ?assertMatch({ok, #{<<"c">> := [0,1,2,3]}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql0, - <<"ctx">> => #{<<"payload">> => <<"{\"a\":[0,1,2,3,4,5]}">>, - <<"topic">> => <<"t/a">>}})), + #{sql => Sql0, + context => #{payload => <<"{\"a\":[0,1,2,3,4,5]}">>, + topic => <<"t/a">>}})), %% get a range from non-list data Sql02 = "select " " payload.a[1..4] as c " "from \"t/#\" ", ?assertThrow({select_and_transform_error, {error,{range_get,non_list_data},_}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql02, - <<"ctx">> => - #{<<"payload">> => <<"{\"x\":[0,1,2,3,4,5]}">>, - <<"topic">> => <<"t/a">>}})), + #{sql => Sql02, + context => + #{payload => <<"{\"x\":[0,1,2,3,4,5]}">>, + topic => <<"t/a">>}})), %% construct a range: Sql1 = "select " " [1..4] as c, " @@ -2054,9 +1226,9 @@ t_sqlparse_array_range_1(_Config) -> " 0 as c[0] " "from \"t/#\" ", ?assertMatch({ok, #{<<"c">> := [0,1,2,3,4,5,6]}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql1, - <<"ctx">> => #{<<"payload">> => <<"">>, - <<"topic">> => <<"t/a">>}})). + #{sql => Sql1, + context => #{payload => <<"">>, + topic => <<"t/a">>}})). t_sqlparse_array_range_2(_Config) -> %% construct a range without 'as' @@ -2065,9 +1237,9 @@ t_sqlparse_array_range_2(_Config) -> "from \"t/#\" ", {ok, Res00} = emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql00, - <<"ctx">> => #{<<"payload">> => <<"">>, - <<"topic">> => <<"t/a">>}}), + #{sql => Sql00, + context => #{payload => <<"">>, + topic => <<"t/a">>}}), ?assert(lists:any(fun({_K, V}) -> V =:= [1,2,3,4] end, maps:to_list(Res00))), @@ -2077,17 +1249,17 @@ t_sqlparse_array_range_2(_Config) -> "from \"t/#\" ", ?assertMatch({ok, #{<<"a">> := [2,3,4]}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql01, - <<"ctx">> => #{<<"a">> => [1,2,3,4,5], - <<"topic">> => <<"t/a">>}})), + #{sql => Sql01, + context => #{<<"a">> => [1,2,3,4,5], + topic => <<"t/a">>}})), %% get a range of list without 'as' Sql02 = "select " " payload.a[1..4] " "from \"t/#\" ", ?assertMatch({ok, #{<<"payload">> := #{<<"a">> := [0,1,2,3]}}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql02, - <<"ctx">> => #{<<"payload">> => <<"{\"a\":[0,1,2,3,4,5]}">>, - <<"topic">> => <<"t/a">>}})). + #{sql => Sql02, + context => #{payload => <<"{\"a\":[0,1,2,3,4,5]}">>, + topic => <<"t/a">>}})). t_sqlparse_true_false(_Config) -> %% construct a range without 'as' @@ -2097,9 +1269,9 @@ t_sqlparse_true_false(_Config) -> "from \"t/#\" ", {ok, Res00} = emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql00, - <<"ctx">> => #{<<"payload">> => <<"">>, - <<"topic">> => <<"t/a">>}}), + #{sql => Sql00, + context => #{payload => <<"">>, + topic => <<"t/a">>}}), ?assertMatch(#{<<"a">> := true, <<"b">> := false, <<"x">> := #{<<"y">> := false}, <<"c">> := [true] @@ -2113,9 +1285,9 @@ t_sqlparse_new_map(_Config) -> "from \"t/#\" ", {ok, Res00} = emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql00, - <<"ctx">> => #{<<"payload">> => <<"">>, - <<"topic">> => <<"t/a">>}}), + #{sql => Sql00, + context => #{payload => <<"">>, + topic => <<"t/a">>}}), ?assertMatch(#{<<"a">> := #{}, <<"b">> := #{}, <<"x">> := #{<<"y">> := #{}}, <<"c">> := [#{}] @@ -2129,9 +1301,9 @@ t_sqlparse_payload_as(_Config) -> "FROM \"t/#\" ", Payload1 = <<"{ \"msgId\": 1002, \"params\": { \"convertTemp\": 20, \"engineSpeed\": 42, \"hydOilTem\": 30 } }">>, {ok, Res01} = emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql00, - <<"ctx">> => #{<<"payload">> => Payload1, - <<"topic">> => <<"t/a">>}}), + #{sql => Sql00, + context => #{payload => Payload1, + topic => <<"t/a">>}}), ?assertMatch(#{ <<"payload">> := #{ <<"params">> := #{ @@ -2145,9 +1317,9 @@ t_sqlparse_payload_as(_Config) -> Payload2 = <<"{ \"msgId\": 1002, \"params\": { \"convertTemp\": 20, \"engineSpeed\": 42 } }">>, {ok, Res02} = emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql00, - <<"ctx">> => #{<<"payload">> => Payload2, - <<"topic">> => <<"t/a">>}}), + #{sql => Sql00, + context => #{payload => Payload2, + topic => <<"t/a">>}}), ?assertMatch(#{ <<"payload">> := #{ <<"params">> := #{ @@ -2164,139 +1336,53 @@ t_sqlparse_nested_get(_Config) -> "from \"t/#\" ", ?assertMatch({ok,#{<<"c">> := 0}}, emqx_rule_sqltester:test( - #{<<"rawsql">> => Sql, - <<"ctx">> => #{ - <<"topic">> => <<"t/1">>, - <<"payload">> => <<"{\"a\": {\"b\": 0}}">> + #{sql => Sql, + context => #{ + topic => <<"t/1">>, + payload => <<"{\"a\": {\"b\": 0}}">> }})). %%------------------------------------------------------------------------------ %% Internal helpers %%------------------------------------------------------------------------------ -make_simple_rule(RuleId) when is_binary(RuleId) -> - #rule{id = RuleId, - rawsql = <<"select * from \"simple/topic\"">>, - for = [<<"simple/topic">>], - fields = [<<"*">>], - is_foreach = false, - conditions = {}, - actions = [{'inspect', #{}}], - description = <<"simple rule">>}. +republish_output(Topic) -> + republish_output(Topic, <<"${payload}">>). +republish_output(Topic, Payload) -> + #{type => builtin, target => republish, + args => #{<<"payload">> => Payload, <<"topic">> => Topic, <<"qos">> => 0}}. make_simple_rule_with_ts(RuleId, Ts) when is_binary(RuleId) -> - #rule{id = RuleId, - rawsql = <<"select * from \"simple/topic\"">>, - for = [<<"simple/topic">>], - fields = [<<"*">>], - is_foreach = false, - conditions = {}, - actions = [{'inspect', #{}}], - created_at = Ts, - description = <<"simple rule">>}. + SQL = <<"select * from \"simple/topic\"">>, + Topics = [<<"simple/topic">>], + make_simple_rule(RuleId, SQL, Topics, Ts). -make_simple_rule(RuleId, SQL, ForTopics) when is_binary(RuleId) -> - #rule{id = RuleId, - rawsql = SQL, - for = ForTopics, - fields = [<<"*">>], - is_foreach = false, - conditions = {}, - actions = [{'inspect', #{}}], - description = <<"simple rule">>}. +make_simple_rule(RuleId) when is_binary(RuleId) -> + SQL = <<"select * from \"simple/topic\"">>, + Topics = [<<"simple/topic">>], + make_simple_rule(RuleId, SQL, Topics). -create_simple_repub_rule(TargetTopic, SQL) -> - create_simple_repub_rule(TargetTopic, SQL, <<"${payload}">>). +make_simple_rule(RuleId, SQL, Topics) when is_binary(RuleId) -> + make_simple_rule(RuleId, SQL, Topics, erlang:system_time(millisecond)). -create_simple_repub_rule(TargetTopic, SQL, Template) -> - {ok, Rule} = emqx_rule_engine:create_rule( - #{rawsql => SQL, - actions => [#{name => 'republish', - args => #{<<"target_topic">> => TargetTopic, - <<"target_qos">> => -1, - <<"payload_tmpl">> => Template} - }], - description => <<"simple repub rule">>}), - Rule. +make_simple_rule(RuleId, SQL, Topics, Ts) when is_binary(RuleId) -> + #rule{ + id = RuleId, + info = #{ + sql => SQL, + from => Topics, + fields => [<<"*">>], + is_foreach => false, + conditions => {}, + ouputs => [#{type => builtin, target => console}], + description => <<"simple rule">> + }, + created_at = Ts + }. -make_simple_action(ActionName) when is_atom(ActionName) -> - #action{name = ActionName, app = ?APP, - module = ?MODULE, on_create = simple_action_inspect, params_spec = #{}, - title = #{en => <<"Simple inspect action">>}, - description = #{en => <<"Simple inspect action">>}}. -make_simple_action(ActionName, Hook) when is_atom(ActionName) -> - #action{name = ActionName, app = ?APP, for = Hook, - module = ?MODULE, on_create = simple_action_inspect, params_spec = #{}, - title = #{en => <<"Simple inspect action">>}, - description = #{en => <<"Simple inspect action with hook">>}}. - -simple_action_inspect(Params) -> - fun(Data) -> - io:format("Action InputData: ~p, Action InitParams: ~p~n", [Data, Params]) - end. - -make_simple_resource(ResId) -> - #resource{id = ResId, - type = simple_resource_type, - config = #{}, - description = <<"Simple Resource">>}. - -make_simple_resource_type(ResTypeName) -> - #resource_type{name = ResTypeName, provider = ?APP, - params_spec = #{}, - on_create = {?MODULE, on_simple_resource_type_create}, - on_destroy = {?MODULE, on_simple_resource_type_destroy}, - on_status = {?MODULE, on_simple_resource_type_status}, - title = #{en => <<"Simple Resource Type">>}, - description = #{en => <<"Simple Resource Type">>}}. - -on_simple_resource_type_create(_Id, #{}) -> #{}. -on_simple_resource_type_destroy(_Id, #{}) -> ok. -on_simple_resource_type_status(_Id, #{}, #{}) -> #{is_alive => true}. - -hook_metrics_action(_Id, _Params) -> - fun(Data = #{event := EventName}, _Envs) -> - ct:pal("applying hook_metrics_action: ~p", [Data]), - ets:insert(events_record_tab, {EventName, Data}) - end. - -mfa_action(Id, _Params) -> - persistent_term:put(Id, 0), - {?MODULE, mfa_action_do, [Id]}. - -mfa_action_do(_Data, _Envs, K) -> - persistent_term:put(K, 1). - -failure_action(_Id, _Params) -> - fun(Data, _Envs) -> - ct:pal("applying crash action, Data: ~p", [Data]), - {badact, intentional_failure} - end. - -crash_action(_Id, _Params) -> - fun(Data, _Envs) -> - ct:pal("applying crash action, Data: ~p", [Data]), - error(crash) - end. - -simple_action_2_create(_Id, _Params) -> - ets:insert(simple_action_2, {created, erlang:timestamp()}), - fun(_Data, _Envs) -> ok end. - -simple_action_2_destroy(_Id, _Params) -> - ets:insert(simple_action_2, {destroyed, erlang:timestamp()}), - fun(_Data, _Envs) -> ok end. - -init_plus_by_one_action() -> - ets:new(plus_by_one_action, [named_table, set, public]), - ets:insert(plus_by_one_action, {num, 0}). - -plus_by_one_action(_Id, #{}) -> - fun(Data, _Envs) -> - ct:pal("applying plus_by_one_action, Data: ~p", [Data]), - Num = ets:lookup_element(plus_by_one_action, num, 2), - ets:insert(plus_by_one_action, {num, Num + 1}) - end. +output_record_triggered_events(Data = #{event := EventName}, _Envs, _Args) -> + ct:pal("applying output_record_triggered_events: ~p", [Data]), + ets:insert(events_record_tab, {EventName, Data}). verify_event(EventName) -> ct:sleep(50), @@ -2561,74 +1647,3 @@ deps_path(App, RelativePath) -> local_path(RelativePath) -> deps_path(emqx_rule_engine, RelativePath). -set_special_configs(emqx_rule_engine) -> - application:set_env(emqx_rule_engine, ignore_sys_message, true), - application:set_env(emqx_rule_engine, events, - [{'client.connected',on,1}, - {'client.disconnected',on,1}, - {'session.subscribed',on,1}, - {'session.unsubscribed',on,1}, - {'message.acked',on,1}, - {'message.dropped',on,1}, - {'message.delivered',on,1} - ]), - ok; -set_special_configs(_App) -> - ok. - -mock_print() -> - catch meck:unload(emqx_ctl), - meck:new(emqx_ctl, [non_strict, passthrough]), - meck:expect(emqx_ctl, print, fun(Arg) -> emqx_ctl:format(Arg) end), - meck:expect(emqx_ctl, print, fun(Msg, Arg) -> emqx_ctl:format(Msg, Arg) end), - meck:expect(emqx_ctl, usage, fun(Usages) -> emqx_ctl:format_usage(Usages) end), - meck:expect(emqx_ctl, usage, fun(Cmd, Descr) -> emqx_ctl:format_usage(Cmd, Descr) end). - -unmock_print() -> - meck:unload(emqx_ctl). - -t_load_providers(_) -> - error('TODO'). - -t_unload_providers(_) -> - error('TODO'). - -t_delete_rule(_) -> - error('TODO'). - -t_start_resource(_) -> - error('TODO'). - -t_test_resource(_) -> - error('TODO'). - -t_get_resource_status(_) -> - error('TODO'). - -t_get_resource_params(_) -> - error('TODO'). - -t_delete_resource(_) -> - error('TODO'). - -t_refresh_resources(_) -> - error('TODO'). - -t_refresh_rules(_) -> - error('TODO'). - -t_refresh_resource_status(_) -> - error('TODO'). - -t_init_resource(_) -> - error('TODO'). - -t_init_action(_) -> - error('TODO'). - -t_clear_resource(_) -> - error('TODO'). - -t_clear_action(_) -> - error('TODO'). - diff --git a/apps/emqx_rule_engine/test/emqx_rule_metrics_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_metrics_SUITE.erl index e7c543c91..ff654ba94 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_metrics_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_metrics_SUITE.erl @@ -31,11 +31,8 @@ suite() -> groups() -> [{metrics, [sequence], - [ t_action - , t_rule - , t_clear + [ t_rule , t_no_creation_1 - , t_no_creation_2 ]}, {speed, [sequence], [ rule_speed @@ -55,59 +52,27 @@ end_per_suite(_Config) -> init_per_testcase(_, Config) -> catch emqx_rule_metrics:stop(), {ok, _} = emqx_rule_metrics:start_link(), - [emqx_metrics:set(M, 0) || M <- emqx_rule_metrics:overall_metrics()], Config. end_per_testcase(_, _Config) -> ok. t_no_creation_1(_) -> - ?assertEqual(ok, emqx_rule_metrics:inc_rules_matched(<<"rule1">>)). - -t_no_creation_2(_) -> - ?assertEqual(ok, emqx_rule_metrics:inc_actions_taken(<<"action:0">>)). - -t_action(_) -> - ?assertEqual(0, emqx_rule_metrics:get_actions_taken(<<"action:1">>)), - ?assertEqual(0, emqx_rule_metrics:get_actions_exception(<<"action:1">>)), - ?assertEqual(0, emqx_rule_metrics:get_actions_taken(<<"action:2">>)), - ok = emqx_rule_metrics:create_metrics(<<"action:1">>), - ok = emqx_rule_metrics:create_metrics(<<"action:2">>), - ok = emqx_rule_metrics:inc_actions_taken(<<"action:1">>), - ok = emqx_rule_metrics:inc_actions_exception(<<"action:1">>), - ok = emqx_rule_metrics:inc_actions_taken(<<"action:2">>), - ok = emqx_rule_metrics:inc_actions_taken(<<"action:2">>), - ?assertEqual(1, emqx_rule_metrics:get_actions_taken(<<"action:1">>)), - ?assertEqual(1, emqx_rule_metrics:get_actions_exception(<<"action:1">>)), - ?assertEqual(2, emqx_rule_metrics:get_actions_taken(<<"action:2">>)), - ?assertEqual(0, emqx_rule_metrics:get_actions_taken(<<"action:3">>)), - ?assertEqual(3, emqx_rule_metrics:get_overall('actions.taken')), - ?assertEqual(1, emqx_rule_metrics:get_overall('actions.exception')), - ok = emqx_rule_metrics:clear_metrics(<<"action:1">>), - ok = emqx_rule_metrics:clear_metrics(<<"action:2">>), - ?assertEqual(0, emqx_rule_metrics:get_actions_taken(<<"action:1">>)), - ?assertEqual(0, emqx_rule_metrics:get_actions_taken(<<"action:2">>)). + ?assertEqual(ok, emqx_rule_metrics:inc(<<"rule1">>, 'rules.matched')). t_rule(_) -> - ok = emqx_rule_metrics:create_rule_metrics(<<"rule:1">>), + ok = emqx_rule_metrics:create_rule_metrics(<<"rule1">>), ok = emqx_rule_metrics:create_rule_metrics(<<"rule2">>), - ok = emqx_rule_metrics:inc(<<"rule:1">>, 'rules.matched'), + ok = emqx_rule_metrics:inc(<<"rule1">>, 'rules.matched'), ok = emqx_rule_metrics:inc(<<"rule2">>, 'rules.matched'), ok = emqx_rule_metrics:inc(<<"rule2">>, 'rules.matched'), - ?assertEqual(1, emqx_rule_metrics:get(<<"rule:1">>, 'rules.matched')), + ct:pal("----couters: ---~p", [persistent_term:get(emqx_rule_metrics)]), + ?assertEqual(1, emqx_rule_metrics:get(<<"rule1">>, 'rules.matched')), ?assertEqual(2, emqx_rule_metrics:get(<<"rule2">>, 'rules.matched')), ?assertEqual(0, emqx_rule_metrics:get(<<"rule3">>, 'rules.matched')), - ?assertEqual(3, emqx_rule_metrics:get_overall('rules.matched')), - ok = emqx_rule_metrics:clear_rule_metrics(<<"rule:1">>), + ok = emqx_rule_metrics:clear_rule_metrics(<<"rule1">>), ok = emqx_rule_metrics:clear_rule_metrics(<<"rule2">>). -t_clear(_) -> - ok = emqx_rule_metrics:create_metrics(<<"action:1">>), - ok = emqx_rule_metrics:inc_actions_taken(<<"action:1">>), - ?assertEqual(1, emqx_rule_metrics:get_actions_taken(<<"action:1">>)), - ok = emqx_rule_metrics:clear_metrics(<<"action:1">>), - ?assertEqual(0, emqx_rule_metrics:get_actions_taken(<<"action:1">>)). - rule_speed(_) -> ok = emqx_rule_metrics:create_rule_metrics(<<"rule1">>), ok = emqx_rule_metrics:create_rule_metrics(<<"rule:2">>), @@ -119,51 +84,11 @@ rule_speed(_) -> ?LET(#{max := Max, current := Current}, emqx_rule_metrics:get_rule_speed(<<"rule1">>), {?assert(Max =< 2), ?assert(Current =< 2)}), - ct:pal("===== Speed: ~p~n", [emqx_rule_metrics:get_overall_rule_speed()]), - ?LET(#{max := Max, current := Current}, emqx_rule_metrics:get_overall_rule_speed(), - {?assert(Max =< 3), - ?assert(Current =< 3)}), ct:sleep(2100), ?LET(#{max := Max, current := Current, last5m := Last5Min}, emqx_rule_metrics:get_rule_speed(<<"rule1">>), {?assert(Max =< 2), ?assert(Current == 0), ?assert(Last5Min =< 0.67)}), - ?LET(#{max := Max, current := Current, last5m := Last5Min}, emqx_rule_metrics:get_overall_rule_speed(), - {?assert(Max =< 3), - ?assert(Current == 0), - ?assert(Last5Min =< 1)}), ct:sleep(3000), - ?LET(#{max := Max, current := Current, last5m := Last5Min}, emqx_rule_metrics:get_overall_rule_speed(), - {?assert(Max =< 3), - ?assert(Current == 0), - ?assert(Last5Min == 0)}), ok = emqx_rule_metrics:clear_rule_metrics(<<"rule1">>), ok = emqx_rule_metrics:clear_rule_metrics(<<"rule:2">>). - -% t_create(_) -> -% error('TODO'). - -% t_get(_) -> -% error('TODO'). - -% t_get_overall(_) -> -% error('TODO'). - -% t_get_rule_speed(_) -> -% error('TODO'). - -% t_get_overall_rule_speed(_) -> -% error('TODO'). - -% t_get_rule_metrics(_) -> -% error('TODO'). - -% t_get_action_metrics(_) -> -% error('TODO'). - -% t_inc(_) -> -% error('TODO'). - -% t_overall_metrics(_) -> -% error('TODO'). - diff --git a/apps/emqx_rule_engine/test/emqx_rule_monitor_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_monitor_SUITE.erl deleted file mode 100644 index 62f538f43..000000000 --- a/apps/emqx_rule_engine/test/emqx_rule_monitor_SUITE.erl +++ /dev/null @@ -1,109 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_rule_monitor_SUITE). - --compile(export_all). --compile(nowarn_export_all). - --include_lib("emqx_rule_engine/include/rule_engine.hrl"). --include_lib("emqx/include/emqx.hrl"). - --include_lib("eunit/include/eunit.hrl"). --include_lib("common_test/include/ct.hrl"). - -all() -> - [ {group, resource} - ]. - -suite() -> - [{ct_hooks, [cth_surefire]}, {timetrap, {seconds, 30}}]. - -groups() -> - [{resource, [sequence], - [ t_restart_resource - ]} - ]. - -init_per_suite(Config) -> - application:load(emqx_machine), - ok = ekka_mnesia:start(), - ok = emqx_rule_registry:mnesia(boot), - Config. - -end_per_suite(_Config) -> - ok. - -init_per_testcase(t_restart_resource, Config) -> - Opts = [public, named_table, set, {read_concurrency, true}], - _ = ets:new(?RES_PARAMS_TAB, [{keypos, #resource_params.id}|Opts]), - ets:new(t_restart_resource, [named_table, public]), - ets:insert(t_restart_resource, {failed_count, 0}), - ets:insert(t_restart_resource, {succ_count, 0}), - Config; - -init_per_testcase(_, Config) -> - Config. - -end_per_testcase(t_restart_resource, Config) -> - ets:delete(t_restart_resource), - Config; -end_per_testcase(_, Config) -> - Config. - -t_restart_resource(_) -> - {ok, _} = emqx_rule_monitor:start_link(), - emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc,1000), - ok = emqx_rule_registry:register_resource_types( - [#resource_type{ - name = test_res_1, - provider = ?APP, - params_spec = #{}, - on_create = {?MODULE, on_resource_create}, - on_destroy = {?MODULE, on_resource_destroy}, - on_status = {?MODULE, on_get_resource_status}, - title = #{en => <<"Test Resource">>}, - description = #{en => <<"Test Resource">>}}]), - ok = emqx_rule_engine:load_providers(), - {ok, #resource{id = ResId}} = emqx_rule_engine:create_resource( - #{type => test_res_1, - config => #{}, - description => <<"debug resource">>}), - [{_, 1}] = ets:lookup(t_restart_resource, failed_count), - [{_, 0}] = ets:lookup(t_restart_resource, succ_count), - ct:pal("monitor: ~p", [whereis(emqx_rule_monitor)]), - emqx_rule_monitor:ensure_resource_retrier(ResId, 100), - timer:sleep(1000), - [{_, 5}] = ets:lookup(t_restart_resource, failed_count), - [{_, 1}] = ets:lookup(t_restart_resource, succ_count), - #{retryers := Pids} = sys:get_state(whereis(emqx_rule_monitor)), - ?assertEqual(0, map_size(Pids)), - ok = emqx_rule_engine:unload_providers(), - emqx_rule_registry:remove_resource(ResId), - emqx_rule_monitor:stop(), - ok. - -on_resource_create(Id, _) -> - case ets:lookup(t_restart_resource, failed_count) of - [{_, 5}] -> - ets:insert(t_restart_resource, {succ_count, 1}), - #{}; - [{_, N}] -> - ets:insert(t_restart_resource, {failed_count, N+1}), - error({incorrect_params, Id}) - end. -on_resource_destroy(_Id, _) -> ok. -on_get_resource_status(_Id, _) -> #{}. diff --git a/apps/emqx_rule_engine/test/emqx_rule_registry_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_registry_SUITE.erl deleted file mode 100644 index cbd69c878..000000000 --- a/apps/emqx_rule_engine/test/emqx_rule_registry_SUITE.erl +++ /dev/null @@ -1,148 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_rule_registry_SUITE). - --compile(export_all). --compile(nowarn_export_all). - --include_lib("eunit/include/eunit.hrl"). - -all() -> emqx_ct:all(?MODULE). - -init_per_testcase(_TestCase, Config) -> - Config. - -end_per_testcase(_TestCase, Config) -> - Config. - -% t_mnesia(_) -> -% error('TODO'). - -% t_dump(_) -> -% error('TODO'). - -% t_start_link(_) -> -% error('TODO'). - -% t_get_rules_for(_) -> -% error('TODO'). - -% t_add_rules(_) -> -% error('TODO'). - -% t_remove_rules(_) -> -% error('TODO'). - -% t_add_action(_) -> -% error('TODO'). - -% t_remove_action(_) -> -% error('TODO'). - -% t_remove_actions(_) -> -% error('TODO'). - -% t_init(_) -> -% error('TODO'). - -% t_handle_call(_) -> -% error('TODO'). - -% t_handle_cast(_) -> -% error('TODO'). - -% t_handle_info(_) -> -% error('TODO'). - -% t_terminate(_) -> -% error('TODO'). - -% t_code_change(_) -> -% error('TODO'). - -% t_get_resource_types(_) -> -% error('TODO'). - -% t_get_resources_by_type(_) -> -% error('TODO'). - -% t_get_actions_for(_) -> -% error('TODO'). - -% t_get_actions(_) -> -% error('TODO'). - -% t_get_action_instance_params(_) -> -% error('TODO'). - -% t_remove_action_instance_params(_) -> -% error('TODO'). - -% t_remove_resource_params(_) -> -% error('TODO'). - -% t_add_action_instance_params(_) -> -% error('TODO'). - -% t_add_resource_params(_) -> -% error('TODO'). - -% t_find_action(_) -> -% error('TODO'). - -% t_get_rules(_) -> -% error('TODO'). - -% t_get_resources(_) -> -% error('TODO'). - -% t_remove_resource(_) -> -% error('TODO'). - -% t_find_resource_params(_) -> -% error('TODO'). - -% t_add_resource(_) -> -% error('TODO'). - -% t_find_resource_type(_) -> -% error('TODO'). - -% t_remove_rule(_) -> -% error('TODO'). - -% t_add_rule(_) -> -% error('TODO'). - -% t_register_resource_types(_) -> -% error('TODO'). - -% t_add_actions(_) -> -% error('TODO'). - -% t_unregister_resource_types_of(_) -> -% error('TODO'). - -% t_remove_actions_of(_) -> -% error('TODO'). - -% t_get_rule(_) -> -% error('TODO'). - -% t_find_resource(_) -> -% error('TODO'). - diff --git a/apps/emqx_rule_engine/test/emqx_rule_validator_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_validator_SUITE.erl deleted file mode 100644 index fdd7857d4..000000000 --- a/apps/emqx_rule_engine/test/emqx_rule_validator_SUITE.erl +++ /dev/null @@ -1,191 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_rule_validator_SUITE). - --compile(nowarn_export_all). --compile(export_all). - --include_lib("eunit/include/eunit.hrl"). - --define(VALID_SPEC, - #{ - string_required => #{ - type => string, - required => true - }, - string_optional_with_default => #{ - type => string, - required => false, - default => <<"a/b">> - }, - string_optional_without_default_0 => #{ - type => string, - required => false - }, - string_optional_without_default_1 => #{ - type => string - }, - type_number => #{ - type => number, - required => true - }, - type_boolean => #{ - type => boolean, - required => true - }, - type_enum_number => #{ - type => number, - enum => [-1, 0, 1, 2], - required => true - }, - type_file => #{ - type => file, - required => true - }, - type_object => #{ - type => object, - required => true, - schema => #{ - string_required => #{ - type => string, - required => true - }, - type_number => #{ - type => number, - required => true - } - } - }, - type_array => #{ - type => array, - required => true, - items => #{ - type => string, - required => true - } - } - }). - -all() -> emqx_ct:all(?MODULE). - -t_validate_spec_the_complex(_) -> - ok = emqx_rule_validator:validate_spec(?VALID_SPEC). - -t_validate_spec_invalid_1(_) -> - ?assertThrow({required_field_missing, {type, _}}, - emqx_rule_validator:validate_spec(#{ - type_enum_number => #{ - required => true - } - })). - -t_validate_spec_invalid_2(_) -> - ?assertThrow({required_field_missing, {schema, _}}, - emqx_rule_validator:validate_spec(#{ - type_enum_number => #{ - type => object - } - })). - -t_validate_spec_invalid_3(_) -> - ?assertThrow({required_field_missing, {items, _}}, - emqx_rule_validator:validate_spec(#{ - type_enum_number => #{ - type => array - } - })). - -t_validate_params_0(_) -> - Params = #{<<"eee">> => <<"eee">>}, - Specs = #{<<"eee">> => #{ - type => string, - required => true - }}, - ?assertEqual(Params, - emqx_rule_validator:validate_params(Params, Specs)). - -t_validate_params_1(_) -> - Params = #{<<"eee">> => 1}, - Specs = #{<<"eee">> => #{ - type => string, - required => true - }}, - ?assertThrow({invalid_data_type, {string, 1}}, - emqx_rule_validator:validate_params(Params, Specs)). - -t_validate_params_2(_) -> - ?assertThrow({required_field_missing, <<"eee">>}, - emqx_rule_validator:validate_params( - #{<<"abc">> => 1}, - #{<<"eee">> => #{ - type => string, - required => true - }})). - -t_validate_params_format(_) -> - Params = #{<<"eee">> => <<"abc">>}, - Params1 = #{<<"eee">> => <<"http://abc:8080">>}, - Params2 = #{<<"eee">> => <<"http://abc">>}, - Specs = #{<<"eee">> => #{ - type => string, - format => url, - required => true - }}, - ?assertThrow({invalid_data_type, {string, <<"abc">>}}, - emqx_rule_validator:validate_params(Params, Specs)), - ?assertEqual(Params1, - emqx_rule_validator:validate_params(Params1, Specs)), - ?assertEqual(Params2, - emqx_rule_validator:validate_params(Params2, Specs)). - -t_validate_params_fill_default(_) -> - Params = #{<<"abc">> => 1}, - Specs = #{<<"eee">> => #{ - type => string, - required => false, - default => <<"hello">> - }}, - ?assertMatch(#{<<"abc">> := 1, <<"eee">> := <<"hello">>}, - emqx_rule_validator:validate_params(Params, Specs)). - -t_validate_params_the_complex(_) -> - Params = #{ - <<"string_required">> => <<"hello">>, - <<"type_number">> => 1, - <<"type_boolean">> => true, - <<"type_enum_number">> => 2, - <<"type_file">> => <<"">>, - <<"type_object">> => #{ - <<"string_required">> => <<"hello2">>, - <<"type_number">> => 1.3 - }, - <<"type_array">> => [<<"ok">>, <<"no">>] - }, - ?assertMatch( - #{ <<"string_required">> := <<"hello">>, - <<"string_optional_with_default">> := <<"a/b">>, - <<"type_number">> := 1, - <<"type_boolean">> := true, - <<"type_enum_number">> := 2, - <<"type_file">> := <<"">>, - <<"type_object">> := #{ - <<"string_required">> := <<"hello2">>, - <<"type_number">> := 1.3 - }, - <<"type_array">> := [<<"ok">>, <<"no">>] - }, - emqx_rule_validator:validate_params(Params, ?VALID_SPEC)). diff --git a/bin/emqx b/bin/emqx index 6c77afd1b..1be7996d1 100755 --- a/bin/emqx +++ b/bin/emqx @@ -488,6 +488,8 @@ case "$1" in logger -t "${REL_NAME}[${PID}]" "STOP: $msg" # log to user console echoerr "stop failed, $msg" + echo "ERROR: $PID is still around" + ps -p "$PID" exit 1 fi logger -t "${REL_NAME}[${PID}]" "STOP: OK" diff --git a/bin/emqx.cmd b/bin/emqx.cmd index 768e30d2c..fe0d474c9 100644 --- a/bin/emqx.cmd +++ b/bin/emqx.cmd @@ -22,14 +22,19 @@ @set script=%~n0 +@set EPMD_ARG=-start_epmd false -epmd_module ekka_epmd -proto_dist ekka +@set ERL_FLAGS=%EPMD_ARG% + :: Discover the release root directory from the directory :: of this script @set script_dir=%~dp0 @for %%A in ("%script_dir%\..") do @( set rel_root_dir=%%~fA ) + @set rel_dir=%rel_root_dir%\releases\%rel_vsn% @set RUNNER_ROOT_DIR=%rel_root_dir% +@set RUNNER_ETC_DIR=%rel_root_dir%\etc @set etc_dir=%rel_root_dir%\etc @set lib_dir=%rel_root_dir%\lib @@ -46,22 +51,22 @@ @set progname=erl.exe @set clean_boot_script=%rel_root_dir%\bin\start_clean @set erlsrv="%bindir%\erlsrv.exe" -@set epmd="%bindir%\epmd.exe" @set escript="%bindir%\escript.exe" @set werl="%bindir%\werl.exe" @set erl_exe="%bindir%\erl.exe" @set nodetool="%rel_root_dir%\bin\nodetool" @set cuttlefish="%rel_root_dir%\bin\cuttlefish" @set node_type="-name" +@set schema_mod="emqx_machine_schema" :: Extract node name from emqx.conf -@for /f "usebackq delims=\= tokens=2" %%I in (`findstr /b node\.name "%emqx_conf%"`) do @( +@for /f "usebackq delims=" %%I in (`"%escript% %nodetool% hocon -s %schema_mod% -c %etc_dir%\emqx.conf get node.name"`) do @( @call :set_trim node_name %%I ) :: Extract node cookie from emqx.conf -@for /f "usebackq delims=\= tokens=2" %%I in (`findstr /b node\.cookie "%emqx_conf%"`) do @( - @call :set_trim node_cookie= %%I +@for /f "usebackq delims=" %%I in (`"%escript% %nodetool% hocon -s %schema_mod% -c %etc_dir%\emqx.conf get node.cookie"`) do @( + @call :set_trim node_cookie %%I ) :: Write the erl.ini file to set up paths relative to this script @@ -139,13 +144,23 @@ ) @goto :eof -:generate_app_config -@set gen_config_cmd=%escript% %cuttlefish% -i %rel_dir%\emqx.schema -c %etc_dir%\emqx.conf -d %data_dir%\configs generate -@for /f "delims=" %%A in ('%%gen_config_cmd%%') do @( - set generated_config_args=%%A +:: get the current time with hocon +:get_cur_time +@for /f "usebackq tokens=1-6 delims=." %%a in (`"%escript% %nodetool% hocon now_time"`) do @( + set now_time=%%a.%%b.%%c.%%d.%%e.%%f ) @goto :eof +:generate_app_config +@call :get_cur_time +%escript% %nodetool% hocon -v -t %now_time% -s %schema_mod% -c "%etc_dir%\emqx.conf" -d "%data_dir%\configs" generate +@set generated_config_args=-config %data_dir%\configs\app.%now_time%.config -args_file %data_dir%\configs\vm.%now_time%.args +:: create one new line +@echo.>>%data_dir%\configs\vm.%now_time%.args +:: write the node type and node name in to vm args file +@echo %node_type% %node_name%>>%data_dir%\configs\vm.%now_time%.args +@goto :eof + :: set boot_script variable :set_boot_script_var @if exist "%rel_dir%\%rel_name%.boot" ( @@ -188,13 +203,11 @@ :: relup and reldown goto relup ) - @goto :eof :: Uninstall the Windows service :uninstall @%erlsrv% remove %service_name% -@%epmd% -kill @goto :eof :: Start the Windows service @@ -207,7 +220,7 @@ @echo off cd /d %rel_root_dir% @echo on -@start "%rel_name%" %werl% -boot "%boot_script%" %args% +@start "%rel_name%" %werl% -boot "%boot_script%" -mode embedded %args% @goto :eof :: Stop the Windows service @@ -237,7 +250,7 @@ cd /d %rel_root_dir% @echo off cd /d %rel_root_dir% @echo on -@start "bin\%rel_name% console" %werl% -boot "%boot_script%" %args% +@start "bin\%rel_name% console" %werl% -boot "%boot_script%" -mode embedded %args% @echo emqx is started! @goto :eof @@ -262,4 +275,3 @@ cd /d %rel_root_dir% :set_trim @set %1=%2 @goto :eof - diff --git a/build b/build index 36eb4d129..22d1f8960 100755 --- a/build +++ b/build @@ -16,7 +16,9 @@ PKG_VSN="$(./pkg-vsn.sh)" export PKG_VSN if [ "$(uname -s)" = 'Darwin' ]; then - SYSTEM=macos + DIST='macos' + VERSION_ID=$(sw_vers | gsed -n '/^ProductVersion:/p' | gsed -r 's/ProductVersion:(.*)/\1/g' | gsed -r 's/([0-9]+).*/\1/g' | gsed 's/^[ \t]*//g') + SYSTEM="$(echo "${DIST}${VERSION_ID}" | gsed -r 's/([a-zA-Z]*)-.*/\1/g')" elif [ "$(uname -s)" = 'Linux' ]; then if grep -q -i 'centos' /etc/*-release; then DIST='centos' diff --git a/rebar.config b/rebar.config index 54e6d23f8..ca8dd3e22 100644 --- a/rebar.config +++ b/rebar.config @@ -18,8 +18,7 @@ %% Check for the mnesia calls forbidden by Ekka: {xref_queries, - [ {"E || \"mnesia\":\"dirty_write\"/\".*\" : Fun", [{{emqx_dashboard_collection,flush,2},{mnesia,dirty_write,2}}]} - , {"E || \"mnesia\":\"dirty_delete.*\"/\".*\" : Fun", []} + [ {"E || \"mnesia\":\"dirty_delete.*\"/\".*\" : Fun", []} , {"E || \"mnesia\":\"transaction\"/\".*\" : Fun", []} , {"E || \"mnesia\":\"async_dirty\"/\".*\" : Fun", []} , {"E || \"mnesia\":\"clear_table\"/\".*\" : Fun", []} @@ -44,24 +43,24 @@ {deps, [ {gpb, "4.11.2"} %% gpb only used to build, but not for release, pin it here to avoid fetching a wrong version due to rebar plugins scattered in all the deps - , {typerefl, {git, "https://github.com/k32/typerefl", {tag, "0.8.4"}}} + , {typerefl, {git, "https://github.com/k32/typerefl", {tag, "0.8.5"}}} , {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.1.9"}}} , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.8.3"}}} - , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.2"}}} + , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.3"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.10.8"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}} - , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.2.4"}}} + , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.2.5"}}} , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.1"}}} , {replayq, "0.3.3"} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.4.3"}}} - , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.2"}}} + , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.4"}}} , {observer_cli, "1.7.1"} % NOTE: depends on recon 2.5.x , {getopt, "1.0.2"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.14.1"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.19.0"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.19.5"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.4.1"}}} , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.1"}}} diff --git a/rebar.config.erl b/rebar.config.erl index 4888fafe0..6f56ce1d2 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -272,13 +272,13 @@ relx_apps(ReleaseType) -> , emqx_exhook , emqx_bridge , emqx_rule_engine - , emqx_rule_actions , emqx_modules , emqx_management , emqx_dashboard , emqx_retainer , emqx_statsd , emqx_prometheus + , emqx_psk ] ++ [quicer || is_quicer_supported()] ++ [emqx_license || is_enterprise()] diff --git a/scripts/split-config.escript b/scripts/split-config.escript deleted file mode 100755 index 0a26d6fd2..000000000 --- a/scripts/split-config.escript +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env escript - -%% This script reads up emqx.conf and split the sections -%% and dump sections to separate files. -%% Sections are grouped between CONFIG_SECTION_BGN and -%% CONFIG_SECTION_END pairs -%% -%% NOTE: this feature is so far not used in opensource -%% edition due to backward-compatibility reasons. - --mode(compile). - --define(BASE, <<"emqx">>). - -main(_) -> - {ok, Bin} = file:read_file(conf_file()), - Lines = binary:split(Bin, <<"\n">>, [global]), - Sections0 = parse_sections(Lines), - {value, _, Sections1} = lists:keytake(<<"modules">>, 1, Sections0), - {value, {N, Base}, Sections2} = lists:keytake(<<"emqx">>, 1, Sections1), - IncludeNames = proplists:get_keys(Sections2), - Includes = lists:map(fun(Name) -> - iolist_to_binary(["include {{ platform_etc_dir }}/", Name, ".conf"]) - end, IncludeNames), - ok = dump_sections([{N, Base ++ Includes}| Sections2]). - -etc_dir() -> filename:join(["apps", "emqx", "etc"]). - -conf_file() -> filename:join([etc_dir(), "emqx.conf"]). - -parse_sections(Lines) -> - {ok, P} = re:compile("#+\s*CONFIG_SECTION_(BGN|END)\s*=\s*([^\s-]+)\s*="), - Parser = - fun(Line) -> - case re:run(Line, P, [{capture, all_but_first, binary}]) of - {match, [<<"BGN">>, Name]} -> {section_bgn, Name}; - {match, [<<"END">>, Name]} -> {section_end, Name}; - nomatch -> continue - end - end, - parse_sections(Lines, Parser, ?BASE, #{?BASE => []}). - -parse_sections([], _Parse, _Section, Sections) -> - lists:map(fun({N, Lines}) -> {N, lists:reverse(Lines)} end, - maps:to_list(Sections)); -parse_sections([Line | Lines], Parse, Section, Sections) -> - case Parse(Line) of - {section_bgn, Name} -> - ?BASE = Section, %% assert - true = (Name =/= ?BASE), %% assert - false = maps:is_key(Name, Sections), %% assert - NewSections = Sections#{?BASE := maps:get(?BASE, Sections), Name => []}, - parse_sections(Lines, Parse, Name, NewSections); - {section_end, Name} -> - true = (Name =:= Section), %% assert - parse_sections(Lines, Parse, ?BASE, Sections); - continue -> - Acc = maps:get(Section, Sections), - parse_sections(Lines, Parse, Section, Sections#{Section => [Line | Acc]}) - end. - -dump_sections([]) -> ok; -dump_sections([{Name, Lines0} | Rest]) -> - Filename = filename:join([etc_dir(), iolist_to_binary([Name, ".conf.seg"])]), - Lines = [[L, "\n"] || L <- Lines0], - ok = file:write_file(Filename, Lines), - dump_sections(Rest).