diff --git a/.ci/build_packages/Dockerfile b/.ci/build_packages/Dockerfile index 92450440d..3e6fa83c6 100644 --- a/.ci/build_packages/Dockerfile +++ b/.ci/build_packages/Dockerfile @@ -1,4 +1,4 @@ -ARG BUILD_FROM=emqx/build-env:erl23.2.7-ubuntu20.04 +ARG BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-1-ubuntu20.04 FROM ${BUILD_FROM} ARG EMQX_NAME=emqx diff --git a/.ci/docker-compose-file/.env b/.ci/docker-compose-file/.env index d474e2637..8c9d056cc 100644 --- a/.ci/docker-compose-file/.env +++ b/.ci/docker-compose-file/.env @@ -3,3 +3,6 @@ REDIS_TAG=6 MONGO_TAG=4 PGSQL_TAG=13 LDAP_TAG=2.4.50 + +TARGET=emqx/emqx +EMQX_TAG=build-alpine-amd64 diff --git a/.ci/docker-compose-file/conf.cluster.env b/.ci/docker-compose-file/conf.cluster.env new file mode 100644 index 000000000..16fb4733e --- /dev/null +++ b/.ci/docker-compose-file/conf.cluster.env @@ -0,0 +1,6 @@ +EMQX_NAME=emqx +EMQX_CLUSTER__DISCOVERY=static +EMQX_CLUSTER__STATIC__SEEDS="emqx@node1.emqx.io, emqx@node2.emqx.io" +EMQX_LISTENER__TCP__EXTERNAL__PROXY_PROTOCOL=on +EMQX_LISTENER__WS__EXTERNAL__PROXY_PROTOCOL=on +EMQX_LOG__LEVEL=debug diff --git a/.ci/fvt_tests/docker-compose.yaml b/.ci/docker-compose-file/docker-compose-emqx-cluster.yaml similarity index 53% rename from .ci/fvt_tests/docker-compose.yaml rename to .ci/docker-compose-file/docker-compose-emqx-cluster.yaml index 6f2ad1be2..18e1bb6cc 100644 --- a/.ci/fvt_tests/docker-compose.yaml +++ b/.ci/docker-compose-file/docker-compose-emqx-cluster.yaml @@ -1,17 +1,38 @@ -version: '3' +version: '3.9' services: + haproxy: + container_name: haproxy + image: haproxy:2.3 + depends_on: + - emqx1 + - emqx2 + volumes: + - ./haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg + - ../../etc/certs:/usr/local/etc/haproxy/certs + ports: + - "18083:18083" +# - "1883:1883" +# - "8883:8883" +# - "8083:8083" +# - "8084:8084" + networks: + - emqx_bridge + working_dir: /usr/local/etc/haproxy + command: + - bash + - -c + - | + cat /usr/local/etc/haproxy/certs/cert.pem /usr/local/etc/haproxy/certs/key.pem > /usr/local/etc/haproxy/certs/emqx.pem + haproxy -f /usr/local/etc/haproxy/haproxy.cfg + emqx1: container_name: node1.emqx.io - image: ${TARGET}:build-alpine-amd64 + image: $TARGET:$EMQX_TAG + env_file: + - conf.cluster.env environment: - - "EMQX_NAME=emqx" - - "EMQX_HOST=node1.emqx.io" - - "EMQX_CLUSTER__DISCOVERY=static" - - "EMQX_CLUSTER__STATIC__SEEDS=emqx@node1.emqx.io, emqx@node2.emqx.io" - - "EMQX_ZONE__EXTERNAL__RETRY_INTERVAL=2s" - - "EMQX_MQTT__MAX_TOPIC_ALIAS=10" - - "EMQX_LOG__LEVEL=debug" + - "EMQX_HOST=node1.emqx.io" command: - /bin/sh - -c @@ -25,21 +46,17 @@ services: timeout: 25s retries: 5 networks: - emqx-bridge: + emqx_bridge: aliases: - node1.emqx.io emqx2: container_name: node2.emqx.io - image: ${TARGET}:build-alpine-amd64 + image: $TARGET:$EMQX_TAG + env_file: + - conf.cluster.env environment: - - "EMQX_NAME=emqx" - - "EMQX_HOST=node2.emqx.io" - - "EMQX_CLUSTER__DISCOVERY=static" - - "EMQX_CLUSTER__STATIC__SEEDS=emqx@node1.emqx.io, emqx@node2.emqx.io" - - "EMQX_ZONE__EXTERNAL__RETRY_INTERVAL=2s" - - "EMQX_MQTT__MAX_TOPIC_ALIAS=10" - - "EMQX_LOG__LEVEL=debug" + - "EMQX_HOST=node2.emqx.io" command: - /bin/sh - -c @@ -53,22 +70,16 @@ services: timeout: 25s retries: 5 networks: - emqx-bridge: + emqx_bridge: aliases: - node2.emqx.io - client: - container_name: paho_client - image: python:3.7.2-alpine3.9 - depends_on: - - emqx1 - - emqx2 - tty: true - networks: - emqx-bridge: - volumes: - - ./scripts:/scripts - networks: - emqx-bridge: + emqx_bridge: driver: bridge + name: emqx_bridge + ipam: + driver: default + config: + - subnet: 172.100.239.0/24 + gateway: 172.100.239.1 diff --git a/.ci/docker-compose-file/docker-compose-python.yaml b/.ci/docker-compose-file/docker-compose-python.yaml new file mode 100644 index 000000000..0b9af4517 --- /dev/null +++ b/.ci/docker-compose-file/docker-compose-python.yaml @@ -0,0 +1,15 @@ +version: '3.9' + +services: + python: + container_name: python + image: python:3.7.2-alpine3.9 + depends_on: + - emqx1 + - emqx2 + tty: true + networks: + emqx_bridge: + volumes: + - ./python:/scripts + diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml index 1f9345bb7..869153d86 100644 --- a/.ci/docker-compose-file/docker-compose.yaml +++ b/.ci/docker-compose-file/docker-compose.yaml @@ -3,7 +3,7 @@ version: '3.9' services: erlang: container_name: erlang - image: emqx/build-env:erl23.2.7-ubuntu20.04 + image: emqx/build-env:erl23.2.7.2-emqx-1-ubuntu20.04 env_file: - conf.env environment: diff --git a/.ci/docker-compose-file/haproxy/haproxy.cfg b/.ci/docker-compose-file/haproxy/haproxy.cfg new file mode 100644 index 000000000..73c219d55 --- /dev/null +++ b/.ci/docker-compose-file/haproxy/haproxy.cfg @@ -0,0 +1,109 @@ +##---------------------------------------------------------------- +## global 2021/04/05 +##---------------------------------------------------------------- +global + log stdout format raw daemon debug + # Replace 1024000 with deployment connections + maxconn 1000 + nbproc 1 + nbthread 2 + cpu-map auto:1/1-2 0-1 + tune.ssl.default-dh-param 2048 + ssl-default-bind-ciphers ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:ECDHE-RSA-DES-CBC3-SHA:ECDHE-ECDSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:DES-CBC3-SHA:HIGH:SEED:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!RSAPSK:!aDH:!aECDH:!EDH-DSS-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA:!SRP + # Enable the HAProxy Runtime API + stats socket :9999 level admin expose-fd listeners + +##---------------------------------------------------------------- +## defaults +##---------------------------------------------------------------- +defaults + log global + mode tcp + option tcplog + # Replace 1024000 with deployment connections + maxconn 1000 + timeout connect 30000 + timeout client 600s + timeout server 600s + +##---------------------------------------------------------------- +## API +##---------------------------------------------------------------- +frontend emqx_mgmt + mode tcp + option tcplog + bind *:8081 + default_backend emqx_mgmt_back + +frontend emqx_dashboard + mode tcp + option tcplog + bind *:18083 + default_backend emqx_dashboard_back + +backend emqx_mgmt_back + mode http + # balance static-rr + server emqx-1 node1.emqx.io:8081 + server emqx-2 node2.emqx.io:8081 + +backend emqx_dashboard_back + mode http + # balance static-rr + server emqx-1 node1.emqx.io:18083 + server emqx-2 node2.emqx.io:18083 + + +##---------------------------------------------------------------- +## public +##---------------------------------------------------------------- +frontend emqx_tcp + mode tcp + option tcplog + bind *:1883 + default_backend emqx_tcp_back + +frontend emqx_ws + mode tcp + option tcplog + bind *:8083 + default_backend emqx_ws_back + +backend emqx_tcp_back + mode tcp + balance static-rr + server emqx-1 node1.emqx.io:1883 check-send-proxy send-proxy-v2 + server emqx-2 node2.emqx.io:1883 check-send-proxy send-proxy-v2 + +backend emqx_ws_back + mode tcp + balance static-rr + server emqx-1 node1.emqx.io:8083 check-send-proxy send-proxy-v2 + server emqx-2 node2.emqx.io:8083 check-send-proxy send-proxy-v2 + +##---------------------------------------------------------------- +## TLS +##---------------------------------------------------------------- +frontend emqx_ssl + mode tcp + option tcplog + bind *:8883 ssl crt /usr/local/etc/haproxy/certs/emqx.pem ca-file /usr/local/etc/haproxy/certs/cacert.pem verify required no-sslv3 + default_backend emqx_ssl_back + +frontend emqx_wss + mode tcp + option tcplog + bind *:8084 ssl crt /usr/local/etc/haproxy/certs/emqx.pem ca-file /usr/local/etc/haproxy/certs/cacert.pem verify required no-sslv3 + default_backend emqx_wss_back + +backend emqx_ssl_back + mode tcp + balance static-rr + server emqx-1 node1.emqx.io:1883 check-send-proxy send-proxy-v2-ssl-cn + server emqx-2 node2.emqx.io:1883 check-send-proxy send-proxy-v2-ssl-cn + +backend emqx_wss_back + mode tcp + balance static-rr + server emqx-1 node1.emqx.io:8083 check-send-proxy send-proxy-v2-ssl-cn + server emqx-2 node2.emqx.io:8083 check-send-proxy send-proxy-v2-ssl-cn diff --git a/.ci/fvt_tests/scripts/pytest.sh b/.ci/docker-compose-file/python/pytest.sh similarity index 68% rename from .ci/fvt_tests/scripts/pytest.sh rename to .ci/docker-compose-file/python/pytest.sh index c93c4a769..eacbecc3b 100755 --- a/.ci/fvt_tests/scripts/pytest.sh +++ b/.ci/docker-compose-file/python/pytest.sh @@ -6,17 +6,19 @@ set -x set +e -NODE1="node1.emqx.io" -NODE2="node2.emqx.io" +LB="haproxy" apk update && apk add git curl git clone -b develop-4.0 https://github.com/emqx/paho.mqtt.testing.git /paho.mqtt.testing pip install pytest -pytest -v /paho.mqtt.testing/interoperability/test_client/V5/test_connect.py -k test_basic --host "$NODE1" + +pytest -v /paho.mqtt.testing/interoperability/test_client/V5/test_connect.py -k test_basic --host "$LB" RESULT=$? -pytest -v /paho.mqtt.testing/interoperability/test_cluster --host1 "$NODE1" --host2 "$NODE2" -RESULT=$(( RESULT + $? )) -pytest -v /paho.mqtt.testing/interoperability/test_client --host "$NODE1" + +pytest -v /paho.mqtt.testing/interoperability/test_client --host "$LB" RESULT=$(( RESULT + $? )) +# pytest -v /paho.mqtt.testing/interoperability/test_cluster --host1 "node1.emqx.io" --host2 "node2.emqx.io" +# RESULT=$(( RESULT + $? )) + exit $RESULT diff --git a/.ci/docker-compose-file/redis/redis-tls.conf b/.ci/docker-compose-file/redis/redis-tls.conf index 584399a29..325c200c3 100644 --- a/.ci/docker-compose-file/redis/redis-tls.conf +++ b/.ci/docker-compose-file/redis/redis-tls.conf @@ -5,6 +5,7 @@ tls-cert-file /tls/redis.crt tls-key-file /tls/redis.key tls-ca-cert-file /tls/ca.crt tls-replication yes +tls-cluster yes protected-mode no requirepass public masterauth public diff --git a/.ci/docker-compose-file/redis/redis.sh b/.ci/docker-compose-file/redis/redis.sh index ceca7e2c3..272a5b443 100755 --- a/.ci/docker-compose-file/redis/redis.sh +++ b/.ci/docker-compose-file/redis/redis.sh @@ -90,7 +90,11 @@ do continue; fi if [ "${node}" = "cluster" ] ; then - yes "yes" | redis-cli --cluster create "$LOCAL_IP:7000" "$LOCAL_IP:7001" "$LOCAL_IP:7002" --pass public --no-auth-warning; + if $tls ; then + yes "yes" | redis-cli --cluster create "$LOCAL_IP:8000" "$LOCAL_IP:8001" "$LOCAL_IP:8002" --pass public --no-auth-warning --tls true --cacert /tls/ca.crt --cert /tls/redis.crt --key /tls/redis.key; + else + yes "yes" | redis-cli --cluster create "$LOCAL_IP:7000" "$LOCAL_IP:7001" "$LOCAL_IP:7002" --pass public --no-auth-warning; + fi elif [ "${node}" = "sentinel" ] ; then tee /_sentinel.conf>/dev/null << EOF port 26379 diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index a6e16a8e8..e833f0d61 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -15,7 +15,7 @@ on: jobs: prepare: runs-on: ubuntu-20.04 - container: emqx/build-env:erl23.2.7-ubuntu20.04 + container: emqx/build-env:erl23.2.7.2-emqx-1-ubuntu20.04 outputs: profiles: ${{ steps.set_profile.outputs.profiles}} @@ -83,7 +83,7 @@ jobs: $version = $( "${{ github.ref }}" -replace "^(.*)/(.*)/" ) if ($version -match "^v[0-9]+\.[0-9]+(\.[0-9]+)?") { - $regex = "[0-9]+\.[0-9]+(-alpha|-beta|-rc)?\.[0-9]" + $regex = "[0-9]+\.[0-9]+(-alpha|-beta|-rc)?\.[0-9]+" $pkg_name = "${{ matrix.profile }}-windows-$([regex]::matches($version, $regex).value).zip" } else { @@ -267,7 +267,7 @@ jobs: cd - - name: build emqx packages env: - ERL_OTP: erl23.2.7 + ERL_OTP: erl23.2.7.2-emqx-1 PROFILE: ${{ matrix.profile }} ARCH: ${{ matrix.arch }} SYSTEM: ${{ matrix.os }} diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 175a08d05..7dad88eed 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -11,7 +11,7 @@ jobs: strategy: matrix: erl_otp: - - erl23.2.7 + - erl23.2.7.2-emqx-1 os: - ubuntu20.04 - centos7 diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml index 4eeefe46c..9b23f27d0 100644 --- a/.github/workflows/check_deps_integrity.yaml +++ b/.github/workflows/check_deps_integrity.yaml @@ -5,7 +5,7 @@ on: [pull_request] jobs: check_deps_integrity: runs-on: ubuntu-20.04 - container: emqx/build-env:erl23.2.7-ubuntu20.04 + container: emqx/build-env:erl23.2.7.2-emqx-1-ubuntu20.04 steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/run_cts_tests.yaml b/.github/workflows/run_cts_tests.yaml index d47b9e407..c8c642af7 100644 --- a/.github/workflows/run_cts_tests.yaml +++ b/.github/workflows/run_cts_tests.yaml @@ -357,13 +357,20 @@ jobs: EMQX_AUTH__REDIS__SERVER=${redis_${{ matrix.network_type }}_address}:6380 EOF - name: setup - if: matrix.node_type == 'sentinel' + if: matrix.node_type == 'sentinel' && matrix.connect_type == 'tcp' run: | cat <<-EOF >> "$GITHUB_ENV" EMQX_AUTH__REDIS__TYPE=sentinel EMQX_AUTH__REDIS__SERVER=${redis_${{ matrix.network_type }}_address}:26379 EMQX_AUTH__REDIS__SENTINEL=mymaster - EMQX_AUTH__REDIS__POOL=1 + EOF + - name: setup + if: matrix.node_type == 'sentinel' && matrix.connect_type == 'tls' + run: | + cat <<-EOF >> "$GITHUB_ENV" + EMQX_AUTH__REDIS__TYPE=sentinel + EMQX_AUTH__REDIS__SERVER=${redis_${{ matrix.network_type }}_address}:26380 + EMQX_AUTH__REDIS__SENTINEL=mymaster EOF - name: setup if: matrix.node_type == 'cluster' && matrix.connect_type == 'tcp' diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 815a5f7ae..8360d638d 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -24,8 +24,10 @@ jobs: echo "${{ secrets.CI_GIT_TOKEN }}" >> scripts/git-token make deps-emqx-ee echo "TARGET=emqx/emqx-ee" >> $GITHUB_ENV + echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV else echo "TARGET=emqx/emqx" >> $GITHUB_ENV + echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV fi - name: make emqx image run: make docker @@ -33,19 +35,20 @@ jobs: timeout-minutes: 5 run: | set -e -u -x - docker-compose -f .ci/fvt_tests/docker-compose.yaml up -d - while [ "$(docker inspect -f '{{ .State.Health.Status}}' node1.emqx.io)" != "healthy" ] || [ "$(docker inspect -f '{{ .State.Health.Status}}' node2.emqx.io)" != "healthy" ]; do - if [ $(docker ps -a -f name=fvt_tests_emqx -f status=exited -q | wc -l) -ne 0 ]; then - echo "['$(date -u +"%Y-%m-%dT%H:%M:%SZ")']:emqx stop"; - exit; - else - echo "['$(date -u +"%Y-%m-%dT%H:%M:%SZ")']:waiting emqx"; - sleep 5; - fi; + echo "CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_" >> .ci/docker-compose-file/conf.cluster.env + echo "EMQX_ZONE__EXTERNAL__RETRY_INTERVAL=2s" >> .ci/docker-compose-file/conf.cluster.env + echo "EMQX_MQTT__MAX_TOPIC_ALIAS=10" >> .ci/docker-compose-file/conf.cluster.env + docker-compose \ + -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \ + -f .ci/docker-compose-file/docker-compose-python.yaml \ + up -d + while ! docker exec -i node1.emqx.io bash -c "emqx eval \"['emqx@node1.emqx.io','emqx@node2.emqx.io'] = maps:get(running_nodes, ekka_cluster:info()).\"" > /dev/null 2>&1; do + echo "['$(date -u +"%Y-%m-%dT%H:%M:%SZ")']:waiting emqx"; + sleep 5; done - name: make paho tests run: | - if ! docker exec -i paho_client /scripts/pytest.sh; then + if ! docker exec -i python /scripts/pytest.sh; then docker logs node1.emqx.io docker logs node2.emqx.io exit 1 @@ -149,7 +152,7 @@ jobs: relup_test: runs-on: ubuntu-20.04 - container: emqx/build-env:erl23.2.7-ubuntu20.04 + container: emqx/build-env:erl23.2.7.2-emqx-1-ubuntu20.04 defaults: run: shell: bash diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index aba4ce7ad..0b3f8da13 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -13,7 +13,7 @@ on: jobs: run_static_analysis: runs-on: ubuntu-20.04 - container: emqx/build-env:erl23.2.7-ubuntu20.04 + container: emqx/build-env:erl23.2.7.2-emqx-1-ubuntu20.04 steps: - uses: actions/checkout@v2 @@ -28,6 +28,21 @@ jobs: - name: dialyzer run: make dialyzer + run_proper_test: + runs-on: ubuntu-20.04 + container: emqx/build-env:erl23.2.7.2-emqx-1-ubuntu20.04 + + steps: + - uses: actions/checkout@v2 + - name: set git credentials + run: | + if make emqx-ee --dry-run > /dev/null 2>&1; then + echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials + git config --global credential.helper store + fi + - name: proper + run: make proper + run_common_test: runs-on: ubuntu-20.04 @@ -133,4 +148,4 @@ jobs: run: | curl -v -k https://coveralls.io/webhook \ --header "Content-Type: application/json" \ - --data "{\"repo_name\":\"$GITHUB_REPOSITORY\",\"repo_token\":\"$GITHUB_TOKEN\",\"payload\":{\"build_num\":$GITHUB_RUN_ID,\"status\":\"done\"}}" + --data "{\"repo_name\":\"$GITHUB_REPOSITORY\",\"repo_token\":\"$GITHUB_TOKEN\",\"payload\":{\"build_num\":$GITHUB_RUN_ID,\"status\":\"done\"}}" || true diff --git a/.tool-versions b/.tool-versions index 6ece25a0e..e7e734a7b 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1 +1 @@ -erlang 23.2.2 +erlang 23.2.7.2-emqx-1 diff --git a/Makefile b/Makefile index 88406c794..f0c5d0833 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ BUILD = $(CURDIR)/build SCRIPTS = $(CURDIR)/scripts export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh) export EMQX_DESC ?= EMQ X -export EMQX_CE_DASHBOARD_VERSION ?= v4.3.0-rc.1 +export EMQX_CE_DASHBOARD_VERSION ?= v4.3.0 ifeq ($(OS),Windows_NT) export REBAR_COLOR=none endif @@ -40,7 +40,7 @@ eunit: $(REBAR) .PHONY: proper proper: $(REBAR) - @ENABLE_COVER_COMPILE=1 $(REBAR) as test proper -d test/props -c + @ENABLE_COVER_COMPILE=1 $(REBAR) proper -d test/props -c .PHONY: ct ct: $(REBAR) @@ -56,6 +56,14 @@ $1-ct: endef $(foreach app,$(APPS),$(eval $(call gen-app-ct-target,$(app)))) +## apps/name-prop targets +.PHONY: $(APPS:%=%-prop) +define gen-app-prop-target +$1-prop: + $(REBAR) proper -d test/props -v -m $(shell $(CURDIR)/scripts/find-props.sh $1) +endef +$(foreach app,$(APPS),$(eval $(call gen-app-prop-target,$(app)))) + .PHONY: cover cover: $(REBAR) @ENABLE_COVER_COMPILE=1 $(REBAR) cover diff --git a/README-CN.md b/README-CN.md index 6abea87b8..f6c5faf1e 100644 --- a/README-CN.md +++ b/README-CN.md @@ -5,12 +5,12 @@ [![Coverage Status](https://coveralls.io/repos/github/emqx/emqx/badge.svg)](https://coveralls.io/github/emqx/emqx) [![Docker Pulls](https://img.shields.io/docker/pulls/emqx/emqx)](https://hub.docker.com/r/emqx/emqx) [![Slack Invite]()](https://slack-invite.emqx.io) -[![Twitter](https://img.shields.io/badge/Twitter-EMQ%20X-1DA1F2?logo=twitter)](https://twitter.com/emqtt) +[![Twitter](https://img.shields.io/badge/Twitter-EMQ-1DA1F2?logo=twitter)](https://twitter.com/EMQTech) [![Community](https://img.shields.io/badge/Community-EMQ%20X-yellow)](https://askemq.com) [![最棒的物联网 MQTT 开源团队期待您的加入](https://www.emqx.io/static/img/github_readme_cn_bg.png)](https://careers.emqx.cn/) -[English](./README.md) | 简体中文 | [日本語](./README-JP.md) +[English](./README.md) | 简体中文 | [日本語](./README-JP.md) | [русский](./README-RU.md) *EMQ X* 是一款完全开源，高度可伸缩，高可用的分布式 MQTT 消息服务器，适用于 IoT、M2M 和移动应用程序，可处理千万级别的并发客户端。 @@ -128,7 +128,7 @@ DIALYZER_ANALYSE_APP=emqx_lwm2m,emqx_auth_jwt,emqx_auth_ldap make dialyzer 你可通过以下途径与 EMQ 社区及开发者联系: - [Slack](https://slack-invite.emqx.io) -- [Twitter](https://twitter.com/emqtt) +- [Twitter](https://twitter.com/EMQTech) - [Facebook](https://www.facebook.com/emqxmqtt) - [Reddit](https://www.reddit.com/r/emqx/) - [Weibo](https://weibo.com/emqtt) diff --git a/README-JP.md b/README-JP.md index bfb9611c5..0219eb25e 100644 --- a/README-JP.md +++ b/README-JP.md @@ -5,11 +5,11 @@ [![Coverage Status](https://coveralls.io/repos/github/emqx/emqx/badge.svg)](https://coveralls.io/github/emqx/emqx) [![Docker Pulls](https://img.shields.io/docker/pulls/emqx/emqx)](https://hub.docker.com/r/emqx/emqx) [![Slack Invite]()](https://slack-invite.emqx.io) -[![Twitter](https://img.shields.io/badge/Twitter-EMQ%20X-1DA1F2?logo=twitter)](https://twitter.com/emqtt) +[![Twitter](https://img.shields.io/badge/Twitter-EMQ-1DA1F2?logo=twitter)](https://twitter.com/EMQTech) [![The best IoT MQTT open source team looks forward to your joining](https://www.emqx.io/static/img/github_readme_en_bg.png)](https://www.emqx.io/careers) -[English](./README.md) | [简体中文](./README-CN.md) | 日本語 +[English](./README.md) | [简体中文](./README-CN.md) | 日本語 | [русский](./README-RU.md) *EMQ X* は、高い拡張性と可用性をもつ、分散型のMQTTブローカーです。数千万のクライアントを同時に処理するIoT、M2M、モバイルアプリケーション向けです。 @@ -55,7 +55,7 @@ _build/emqx/rel/emqx/bin/emqx console ## クイックスタート emqx をソースコードからビルドした場合は、 -`cd _buid/emqx/rel/emqx`でリリースビルドのディレクトリに移動してください。 +`cd _build/emqx/rel/emqx`でリリースビルドのディレクトリに移動してください。 リリースパッケージからインストールした場合は、インストール先のルートディレクトリに移動してください。 diff --git a/README-RU.md b/README-RU.md new file mode 100644 index 000000000..2dc5a6287 --- /dev/null +++ b/README-RU.md @@ -0,0 +1,141 @@ +# Брокер EMQ X + +[![GitHub Release](https://img.shields.io/github/release/emqx/emqx?color=brightgreen)](https://github.com/emqx/emqx/releases) +[![Build Status](https://travis-ci.org/emqx/emqx.svg)](https://travis-ci.org/emqx/emqx) +[![Coverage Status](https://coveralls.io/repos/github/emqx/emqx/badge.svg?branch=master)](https://coveralls.io/github/emqx/emqx?branch=master) +[![Docker Pulls](https://img.shields.io/docker/pulls/emqx/emqx)](https://hub.docker.com/r/emqx/emqx) +[![Slack Invite]()](https://slack-invite.emqx.io) +[![Twitter](https://img.shields.io/badge/Follow-EMQ-1DA1F2?logo=twitter)](https://twitter.com/EMQTech) +[![Community](https://img.shields.io/badge/Community-EMQ%20X-yellow?logo=github)](https://github.com/emqx/emqx/discussions) + +[![The best IoT MQTT open source team looks forward to your joining](https://www.emqx.io/static/img/github_readme_en_bg.png)](https://www.emqx.io/careers) + +[English](./README.md) | [简体中文](./README-CN.md) | [日本語](./README-JP.md) | русский + +*EMQ X* — это масштабируемый, высоко доступный, распределённый MQTT брокер с полностью открытым кодом для интернета вещей, межмашинного взаимодействия и мобильных приложений, который поддерживает миллионы одновременных подключений. + +Начиная с релиза 3.0, брокер *EMQ X* полностью поддерживает протокол MQTT версии 5.0, и обратно совместим с версиями 3.1 и 3.1.1, а также протоколами MQTT-SN, CoAP, LwM2M, WebSocket и STOMP. Начиная с релиза 3.0, брокер *EMQ X* может масштабироваться до более чем 10 миллионов одновременных MQTT соединений на один кластер. + +- Полный список возможностей доступен по ссылке: [EMQ X Release Notes](https://github.com/emqx/emqx/releases). +- Более подробная информация доступна на нашем сайте: [EMQ X homepage](https://www.emqx.io). + +## Установка + +Брокер *EMQ X* кросплатформенный, и поддерживает Linux, Unix, macOS и Windows. Он может работать на серверах с архитектурой x86_64 и устройствах на архитектуре ARM, таких как Raspberry Pi. + +Более подробная информация о запуске на Windows по ссылке: [Windows.md](./Windows.md) + +#### Установка EMQ X с помощью Docker-образа + +``` +docker run -d --name emqx -p 1883:1883 -p 8081:8081 -p 8083:8083 -p 8883:8883 -p 8084:8084 -p 18083:18083 emqx/emqx +``` + +#### Установка бинарного пакета + +Сборки для различных операционных систем: [Загрузить EMQ X](https://www.emqx.io/downloads). + +- [Установка на одном сервере](https://docs.emqx.io/en/broker/latest/getting-started/install.html) +- [Установка на кластере](https://docs.emqx.io/en/broker/latest/advanced/cluster.html) + + +## Сборка из исходного кода + +Начиная с релиза 3.0, для сборки требуется Erlang/OTP R21 или выше. + +Инструкция для сборки версии 4.3 и выше: + +```bash +git clone https://github.com/emqx/emqx.git +cd emqx +make +_build/emqx/rel/emqx/bin console +``` + +Более ранние релизы могут быть собраны с помощью другого репозитория: + +```bash +git clone https://github.com/emqx/emqx-rel.git +cd emqx-rel +make +_build/emqx/rel/emqx/bin/emqx console +``` + +## Первый запуск + +Если emqx был собран из исходников: `cd _build/emqx/rel/emqx`. +Или перейдите в директорию, куда emqx был установлен из бинарного пакета. + +```bash +# Запуск: +./bin/emqx start + +# Проверка статуса: +./bin/emqx_ctl status + +# Остановка: +./bin/emqx stop +``` + +Веб-интерфейс брокера будет доступен по ссылке: http://localhost:18083 + +## Тесты + +### Полное тестирование + +``` +make eunit ct +``` + +### Запуск части тестов + +Пример: + +```bash +make apps/emqx_bridge_mqtt-ct +``` + +### Dialyzer +##### Статический анализ всех приложений +``` +make dialyzer +``` + +##### Статический анализ части приложений (список через запятую) +``` +DIALYZER_ANALYSE_APP=emqx_lwm2m,emqx_auth_jwt,emqx_auth_ldap make dialyzer +``` + +## Сообщество + +### FAQ + +Наиболее частые проблемы разобраны в [EMQ X FAQ](https://docs.emqx.io/en/broker/latest/faq/faq.html). + + +### Вопросы + +Задать вопрос или поделиться идеей можно в [GitHub Discussions](https://github.com/emqx/emqx/discussions). + +### Предложения + +Более масштабные предложения можно присылать в виде pull request в репозиторий [EIP](https://github.com/emqx/eip). + +### Разработка плагинов + +Инструкция по разработке собственных плагинов доступна по ссылке: [lib-extra/README.md](./lib-extra/README.md) + + +## Спецификации стандарта MQTT + +Следующие ссылки содержат спецификации стандартов: + +[MQTT Version 3.1.1](https://docs.oasis-open.org/mqtt/mqtt/v3.1.1/os/mqtt-v3.1.1-os.html) + +[MQTT Version 5.0](https://docs.oasis-open.org/mqtt/mqtt/v5.0/cs02/mqtt-v5.0-cs02.html) + +[MQTT SN](http://mqtt.org/new/wp-content/uploads/2009/06/MQTT-SN_spec_v1.2.pdf) + +## Лицензия + +Apache License 2.0, см. [LICENSE](./LICENSE). diff --git a/README.md b/README.md index aba873046..34366c2ea 100644 --- a/README.md +++ b/README.md @@ -5,12 +5,12 @@ [![Coverage Status](https://coveralls.io/repos/github/emqx/emqx/badge.svg?branch=master)](https://coveralls.io/github/emqx/emqx?branch=master) [![Docker Pulls](https://img.shields.io/docker/pulls/emqx/emqx)](https://hub.docker.com/r/emqx/emqx) [![Slack Invite]()](https://slack-invite.emqx.io) -[![Twitter](https://img.shields.io/badge/Follow-EMQ%20X-1DA1F2?logo=twitter)](https://twitter.com/emqtt) +[![Twitter](https://img.shields.io/badge/Follow-EMQ-1DA1F2?logo=twitter)](https://twitter.com/EMQTech) [![Community](https://img.shields.io/badge/Community-EMQ%20X-yellow?logo=github)](https://github.com/emqx/emqx/discussions) [![The best IoT MQTT open source team looks forward to your joining](https://www.emqx.io/static/img/github_readme_en_bg.png)](https://www.emqx.io/careers) -English | [简体中文](./README-CN.md) | [日本語](./README-JP.md) +English | [简体中文](./README-CN.md) | [日本語](./README-JP.md) | [русский](./README-RU.md) *EMQ X* broker is a fully open source, highly scalable, highly available distributed MQTT messaging broker for IoT, M2M and Mobile applications that can handle tens of millions of concurrent clients. @@ -63,7 +63,7 @@ _build/emqx/rel/emqx/bin/emqx console ## Quick Start -If emqx is built from source, `cd _buid/emqx/rel/emqx`. +If emqx is built from source, `cd _build/emqx/rel/emqx`. Or change to the installation root directory if emqx is installed from a release package. ```bash diff --git a/apps/emqx_auth_http/etc/emqx_auth_http.conf b/apps/emqx_auth_http/etc/emqx_auth_http.conf index c9077b7e0..e8e28116b 100644 --- a/apps/emqx_auth_http/etc/emqx_auth_http.conf +++ b/apps/emqx_auth_http/etc/emqx_auth_http.conf @@ -73,6 +73,7 @@ auth.http.super_req.headers.content-type = "application/x-www-form-urlencoded" auth.http.super_req.params = "clientid=%c,username=%u" ## HTTP URL API path for ACL Request +## Comment out this config to disable ACL checks ## ## Value: URL ## diff --git a/apps/emqx_auth_http/priv/emqx_auth_http.schema b/apps/emqx_auth_http/priv/emqx_auth_http.schema index e1f02ef49..b248c7dc7 100644 --- a/apps/emqx_auth_http/priv/emqx_auth_http.schema +++ b/apps/emqx_auth_http/priv/emqx_auth_http.schema @@ -60,6 +60,10 @@ end}. end end}. +%% @doc URL for ACL checks. Example: http://127.0.0.1:80/mqtt/acl +%% ACL checks are disabled for this plugin if this config is +%% commented out from the config file, or when the overriding +%% environment variable is set to empty string. {mapping, "auth.http.acl_req.url", "emqx_auth_http.acl_req", [ {datatype, string} ]}. @@ -124,4 +128,4 @@ end}. {mapping, "auth.http.ssl.server_name_indication", "emqx_auth_http.server_name_indication", [ {datatype, string} -]}. \ No newline at end of file +]}. diff --git a/apps/emqx_auth_http/rebar.config b/apps/emqx_auth_http/rebar.config index a53379e99..01c0f4209 100644 --- a/apps/emqx_auth_http/rebar.config +++ b/apps/emqx_auth_http/rebar.config @@ -1,6 +1,4 @@ -{deps, - [{ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.1.2"}}} - ]}. +{deps, []}. {edoc_opts, [{preprocess, true}]}. {erl_opts, [warn_unused_vars, diff --git a/apps/emqx_auth_http/src/emqx_acl_http.erl b/apps/emqx_auth_http/src/emqx_acl_http.erl index 23f6ff62b..aa98759b0 100644 --- a/apps/emqx_auth_http/src/emqx_acl_http.erl +++ b/apps/emqx_auth_http/src/emqx_acl_http.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_auth_http/src/emqx_auth_http.erl b/apps/emqx_auth_http/src/emqx_auth_http.erl index e82a37625..aba0a5d8d 100644 --- a/apps/emqx_auth_http/src/emqx_auth_http.erl +++ b/apps/emqx_auth_http/src/emqx_auth_http.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_auth_http/src/emqx_auth_http_app.erl b/apps/emqx_auth_http/src/emqx_auth_http_app.erl index 79b34effb..c5a7f2a77 100644 --- a/apps/emqx_auth_http/src/emqx_auth_http_app.erl +++ b/apps/emqx_auth_http/src/emqx_auth_http_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -53,15 +53,14 @@ translate_env(EnvName) -> {ok, PoolSize} = application:get_env(?APP, pool_size), {ok, ConnectTimeout} = application:get_env(?APP, connect_timeout), URL = proplists:get_value(url, Req), - {ok, #{host := Host0, + {ok, #{host := Host, path := Path0, port := Port, scheme := Scheme}} = emqx_http_lib:uri_parse(URL), Path = path(Path0), - {Inet, Host} = parse_host(Host0), MoreOpts = case Scheme of http -> - [{transport_opts, [Inet]}]; + [{transport_opts, emqx_misc:ipv6_probe([])}]; https -> CACertFile = application:get_env(?APP, cacertfile, undefined), CertFile = application:get_env(?APP, certfile, undefined), @@ -86,7 +85,7 @@ translate_env(EnvName) -> , {ciphers, emqx_tls_lib:default_ciphers()} | TLSOpts ], - [{transport, ssl}, {transport_opts, [Inet | NTLSOpts]}] + [{transport, ssl}, {transport_opts, emqx_misc:ipv6_probe(NTLSOpts)}] end, PoolOpts = [{host, Host}, {port, Port}, @@ -146,17 +145,6 @@ unload_hooks() -> _ = ehttpc_sup:stop_pool('emqx_auth_http/acl_req'), ok. -parse_host(Host) -> - case inet:parse_address(Host) of - {ok, Addr} when size(Addr) =:= 4 -> {inet, Addr}; - {ok, Addr} when size(Addr) =:= 8 -> {inet6, Addr}; - {error, einval} -> - case inet:getaddr(Host, inet6) of - {ok, _} -> {inet6, Host}; - {error, _} -> {inet, Host} - end - end. - to_lower(Headers) -> [{string:to_lower(K), V} || {K, V} <- Headers]. diff --git a/apps/emqx_auth_http/src/emqx_auth_http_cli.erl b/apps/emqx_auth_http/src/emqx_auth_http_cli.erl index 02fdd9862..979ac475d 100644 --- a/apps/emqx_auth_http/src/emqx_auth_http_cli.erl +++ b/apps/emqx_auth_http/src/emqx_auth_http_cli.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_auth_http/src/emqx_auth_http_sup.erl b/apps/emqx_auth_http/src/emqx_auth_http_sup.erl index 36b61a224..798a8a92e 100644 --- a/apps/emqx_auth_http/src/emqx_auth_http_sup.erl +++ b/apps/emqx_auth_http/src/emqx_auth_http_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_auth_http/test/emqx_auth_http_SUITE.erl b/apps/emqx_auth_http/test/emqx_auth_http_SUITE.erl index 5ac5c18e8..ef692e886 100644 --- a/apps/emqx_auth_http/test/emqx_auth_http_SUITE.erl +++ b/apps/emqx_auth_http/test/emqx_auth_http_SUITE.erl @@ -1,4 +1,4 @@ -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -36,35 +36,75 @@ %%-------------------------------------------------------------------- all() -> - [{group, http_inet}, + [ + {group, http_inet}, {group, http_inet6}, {group, https_inet}, - {group, https_inet6}]. + {group, https_inet6}, + pub_sub_no_acl, + no_hook_if_config_unset + ]. groups() -> Cases = emqx_ct:all(?MODULE), [{Name, Cases} || Name <- [http_inet, http_inet6, https_inet, https_inet6]]. init_per_group(GrpName, Cfg) -> - [Schema, Inet] = [list_to_atom(X) || X <- string:tokens(atom_to_list(GrpName), "_")], - http_auth_server:start(Schema, Inet), - Fun = fun(App) -> set_special_configs(App, Schema, Inet) end, - emqx_ct_helpers:start_apps([emqx_auth_http], Fun), + [Scheme, Inet] = [list_to_atom(X) || X <- string:tokens(atom_to_list(GrpName), "_")], + ok = setup(Scheme, Inet), Cfg. end_per_group(_GrpName, _Cfg) -> - http_auth_server:stop(), - emqx_ct_helpers:stop_apps([emqx_auth_http, emqx]). + teardown(). -set_special_configs(emqx, _Schmea, _Inet) -> +init_per_testcase(pub_sub_no_acl, Cfg) -> + Scheme = http, + Inet = inet, + http_auth_server:start(Scheme, Inet), + Fun = fun(App) -> set_special_configs(App, Scheme, Inet, no_acl) end, + emqx_ct_helpers:start_apps([emqx_auth_http], Fun), + ?assert(is_hooked('client.authenticate')), + ?assertNot(is_hooked('client.check_acl')), + Cfg; +init_per_testcase(no_hook_if_config_unset, Cfg) -> + setup(http, inet), + Cfg; +init_per_testcase(_, Cfg) -> + %% init per group + Cfg. + +end_per_testcase(pub_sub_no_acl, _Cfg) -> + teardown(); +end_per_testcase(no_hook_if_config_unset, _Cfg) -> + teardown(); +end_per_testcase(_, _Cfg) -> + %% teardown per group + ok. + +setup(Scheme, Inet) -> + http_auth_server:start(Scheme, Inet), + Fun = fun(App) -> set_special_configs(App, Scheme, Inet, normal) end, + emqx_ct_helpers:start_apps([emqx_auth_http], Fun), + ?assert(is_hooked('client.authenticate')), + ?assert(is_hooked('client.check_acl')). + +teardown() -> + http_auth_server:stop(), + application:stop(emqx_auth_http), + ?assertNot(is_hooked('client.authenticate')), + ?assertNot(is_hooked('client.check_acl')), + emqx_ct_helpers:stop_apps([emqx]). + +set_special_configs(emqx, _Scheme, _Inet, _AuthConfig) -> application:set_env(emqx, allow_anonymous, true), application:set_env(emqx, enable_acl_cache, false), LoadedPluginPath = filename:join(["test", "emqx_SUITE_data", "loaded_plugins"]), application:set_env(emqx, plugins_loaded_file, emqx_ct_helpers:deps_path(emqx, LoadedPluginPath)); -set_special_configs(emqx_auth_http, Schema, Inet) -> - ServerAddr = http_server(Schema, Inet), +set_special_configs(emqx_auth_http, Scheme, Inet, PluginConfig) -> + [application:unset_env(?APP, Par) || Par <- [acl_req, auth_req]], + ServerAddr = http_server(Scheme, Inet), AuthReq = #{method => get, url => ServerAddr ++ "/mqtt/auth", @@ -79,11 +119,14 @@ set_special_configs(emqx_auth_http, Schema, Inet) -> headers => [{"content-type", "application/json"}], params => [{"access", "%A"}, {"username", "%u"}, {"clientid", "%c"}, {"ipaddr", "%a"}, {"topic", "%t"}, {"mountpoint", "%m"}]}, - Schema =:= https andalso set_https_client_opts(), + Scheme =:= https andalso set_https_client_opts(), application:set_env(emqx_auth_http, auth_req, maps:to_list(AuthReq)), application:set_env(emqx_auth_http, super_req, maps:to_list(SuperReq)), - application:set_env(emqx_auth_http, acl_req, maps:to_list(AclReq)). + case PluginConfig of + normal -> ok = application:set_env(emqx_auth_http, acl_req, maps:to_list(AclReq)); + no_acl -> ok + end. %% @private set_https_client_opts() -> @@ -95,16 +138,16 @@ set_https_client_opts() -> application:set_env(emqx_auth_http, server_name_indication, "disable"). %% @private -http_server(http, inet) -> "http://127.0.0.1:8991"; -http_server(http, inet6) -> "http://[::1]:8991"; -http_server(https, inet) -> "https://127.0.0.1:8991"; -http_server(https, inet6) -> "https://[::1]:8991". +http_server(http, inet) -> "http://127.0.0.1:8991"; % ipv4 +http_server(http, inet6) -> "http://localhost:8991"; % test hostname resolution +http_server(https, inet) -> "https://localhost:8991"; % test hostname resolution +http_server(https, inet6) -> "https://[::1]:8991". % ipv6 %%------------------------------------------------------------------------------ %% Testcases %%------------------------------------------------------------------------------ -t_check_acl(_) -> +t_check_acl(Cfg) when is_list(Cfg) -> SuperUser = ?USER(<<"superclient">>, <<"superuser">>, mqtt, {127,0,0,1}, external), deny = emqx_access_control:check_acl(SuperUser, subscribe, <<"users/testuser/1">>), deny = emqx_access_control:check_acl(SuperUser, publish, <<"anytopic">>), @@ -125,7 +168,7 @@ t_check_acl(_) -> deny = emqx_access_control:check_acl(User2, publish, <<"a/b/c">>), deny = emqx_access_control:check_acl(User2, subscribe, <<"$SYS/testuser/1">>). -t_check_auth(_) -> +t_check_auth(Cfg) when is_list(Cfg) -> User1 = ?USER(<<"client1">>, <<"testuser1">>, mqtt, {127,0,0,1}, external, undefined), User2 = ?USER(<<"client2">>, <<"testuser2">>, mqtt, {127,0,0,1}, exteneral, undefined), User3 = ?USER(<<"client3">>, undefined, mqtt, {127,0,0,1}, exteneral, undefined), @@ -142,8 +185,7 @@ t_check_auth(_) -> {error, bad_username_or_password} = emqx_access_control:authenticate(User3#{password => <<"pwd">>}). -t_sub_pub(_) -> - ct:pal("start client"), +pub_sub_no_acl(Cfg) when is_list(Cfg) -> {ok, T1} = emqtt:start_link([{host, "localhost"}, {clientid, <<"client1">>}, {username, <<"testuser1">>}, @@ -164,12 +206,52 @@ t_sub_pub(_) -> emqtt:disconnect(T1), emqtt:disconnect(T2). -t_comment_config(_) -> - AuthCount = length(emqx_hooks:lookup('client.authenticate')), - AclCount = length(emqx_hooks:lookup('client.check_acl')), +t_pub_sub(Cfg) when is_list(Cfg) -> + {ok, T1} = emqtt:start_link([{host, "localhost"}, + {clientid, <<"client1">>}, + {username, <<"testuser1">>}, + {password, <<"pass1">>}]), + {ok, _} = emqtt:connect(T1), + emqtt:publish(T1, <<"topic">>, <<"body">>, [{qos, 0}, {retain, true}]), + timer:sleep(1000), + {ok, T2} = emqtt:start_link([{host, "localhost"}, + {clientid, <<"client2">>}, + {username, <<"testuser2">>}, + {password, <<"pass2">>}]), + {ok, _} = emqtt:connect(T2), + emqtt:subscribe(T2, <<"topic">>), + receive + {publish, _Topic, Payload} -> + ?assertEqual(<<"body">>, Payload) + after 1000 -> false end, + emqtt:disconnect(T1), + emqtt:disconnect(T2). + +no_hook_if_config_unset(Cfg) when is_list(Cfg) -> + ?assert(is_hooked('client.authenticate')), + ?assert(is_hooked('client.check_acl')), application:stop(?APP), [application:unset_env(?APP, Par) || Par <- [acl_req, auth_req]], application:start(?APP), ?assertEqual([], emqx_hooks:lookup('client.authenticate')), - ?assertEqual(AuthCount - 1, length(emqx_hooks:lookup('client.authenticate'))), - ?assertEqual(AclCount - 1, length(emqx_hooks:lookup('client.check_acl'))). + ?assertNot(is_hooked('client.authenticate')), + ?assertNot(is_hooked('client.check_acl')). + +is_hooked(HookName) -> + Callbacks = emqx_hooks:lookup(HookName), + F = fun(Callback) -> + case emqx_hooks:callback_action(Callback) of + {emqx_auth_http, check, _} -> + 'client.authenticate' = HookName, % assert + true; + {emqx_acl_http, check_acl, _} -> + 'client.check_acl' = HookName, % assert + true; + _ -> + false + end + end, + case lists:filter(F, Callbacks) of + [_] -> true; + [] -> false + end. diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt.erl b/apps/emqx_auth_jwt/src/emqx_auth_jwt.erl index 6be726dc9..ba37eac2b 100644 --- a/apps/emqx_auth_jwt/src/emqx_auth_jwt.erl +++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt_app.erl b/apps/emqx_auth_jwt/src/emqx_auth_jwt_app.erl index 0926bf697..e501b0af4 100644 --- a/apps/emqx_auth_jwt/src/emqx_auth_jwt_app.erl +++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt_svr.erl b/apps/emqx_auth_jwt/src/emqx_auth_jwt_svr.erl index 00cc590e9..2d00903e3 100644 --- a/apps/emqx_auth_jwt/src/emqx_auth_jwt_svr.erl +++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt_svr.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_auth_jwt/test/emqx_auth_jwt_SUITE.erl b/apps/emqx_auth_jwt/test/emqx_auth_jwt_SUITE.erl index 12f307b2a..f123d1037 100644 --- a/apps/emqx_auth_jwt/test/emqx_auth_jwt_SUITE.erl +++ b/apps/emqx_auth_jwt/test/emqx_auth_jwt_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_auth_ldap/src/emqx_acl_ldap.erl b/apps/emqx_auth_ldap/src/emqx_acl_ldap.erl index cfd51164a..25287052c 100644 --- a/apps/emqx_auth_ldap/src/emqx_acl_ldap.erl +++ b/apps/emqx_auth_ldap/src/emqx_acl_ldap.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_auth_ldap/src/emqx_auth_ldap.erl b/apps/emqx_auth_ldap/src/emqx_auth_ldap.erl index 01cbb0ecb..9163362c7 100644 --- a/apps/emqx_auth_ldap/src/emqx_auth_ldap.erl +++ b/apps/emqx_auth_ldap/src/emqx_auth_ldap.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_auth_ldap/src/emqx_auth_ldap_app.erl b/apps/emqx_auth_ldap/src/emqx_auth_ldap_app.erl index 8a1343870..c999d5c95 100644 --- a/apps/emqx_auth_ldap/src/emqx_auth_ldap_app.erl +++ b/apps/emqx_auth_ldap/src/emqx_auth_ldap_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_auth_ldap/src/emqx_auth_ldap_cli.erl b/apps/emqx_auth_ldap/src/emqx_auth_ldap_cli.erl index 2f6e8099c..412754664 100644 --- a/apps/emqx_auth_ldap/src/emqx_auth_ldap_cli.erl +++ b/apps/emqx_auth_ldap/src/emqx_auth_ldap_cli.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_auth_ldap/src/emqx_auth_ldap_sup.erl b/apps/emqx_auth_ldap/src/emqx_auth_ldap_sup.erl index ca4440f1a..56b3eea9d 100644 --- a/apps/emqx_auth_ldap/src/emqx_auth_ldap_sup.erl +++ b/apps/emqx_auth_ldap/src/emqx_auth_ldap_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_auth_ldap/test/emqx_auth_ldap_SUITE.erl b/apps/emqx_auth_ldap/test/emqx_auth_ldap_SUITE.erl index c9c38f610..52bed9cf4 100644 --- a/apps/emqx_auth_ldap/test/emqx_auth_ldap_SUITE.erl +++ b/apps/emqx_auth_ldap/test/emqx_auth_ldap_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_auth_ldap/test/emqx_auth_ldap_bind_as_user_SUITE.erl b/apps/emqx_auth_ldap/test/emqx_auth_ldap_bind_as_user_SUITE.erl index 6a5e7bf47..24c03fdaf 100644 --- a/apps/emqx_auth_ldap/test/emqx_auth_ldap_bind_as_user_SUITE.erl +++ b/apps/emqx_auth_ldap/test/emqx_auth_ldap_bind_as_user_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_auth_mnesia/src/emqx_acl_mnesia.erl b/apps/emqx_auth_mnesia/src/emqx_acl_mnesia.erl index ec8670a83..c21955182 100644 --- a/apps/emqx_auth_mnesia/src/emqx_acl_mnesia.erl +++ b/apps/emqx_auth_mnesia/src/emqx_acl_mnesia.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_auth_mnesia/src/emqx_acl_mnesia_api.erl b/apps/emqx_auth_mnesia/src/emqx_acl_mnesia_api.erl index efc2d2fbb..63e8fedd1 100644 --- a/apps/emqx_auth_mnesia/src/emqx_acl_mnesia_api.erl +++ b/apps/emqx_auth_mnesia/src/emqx_acl_mnesia_api.erl @@ -1,5 +1,5 @@ %c%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -222,10 +222,5 @@ format_msg(Message) format_msg(Message) when is_tuple(Message) -> iolist_to_binary(io_lib:format("~p", [Message])). --if(?OTP_RELEASE >= 23). urldecode(S) -> - [{R, _}] = uri_string:dissect_query(S), R. --else. -urldecode(S) -> - http_uri:decode(S). --endif. + emqx_http_lib:uri_decode(S). diff --git a/apps/emqx_auth_mnesia/src/emqx_acl_mnesia_cli.erl b/apps/emqx_auth_mnesia/src/emqx_acl_mnesia_cli.erl index ca1be1676..302a81637 100644 --- a/apps/emqx_auth_mnesia/src/emqx_acl_mnesia_cli.erl +++ b/apps/emqx_auth_mnesia/src/emqx_acl_mnesia_cli.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.erl b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.erl index 45dbd4573..905bcaaf0 100644 --- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.erl +++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_api.erl b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_api.erl index 049af79ad..07ff3bdf5 100644 --- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_api.erl +++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_api.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -301,10 +301,5 @@ format_msg(Message) format_msg(Message) when is_tuple(Message) -> iolist_to_binary(io_lib:format("~p", [Message])). --if(?OTP_RELEASE >= 23). urldecode(S) -> - [{R, _}] = uri_string:dissect_query(S), R. --else. -urldecode(S) -> - http_uri:decode(S). --endif. + emqx_http_lib:uri_decode(S). diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_app.erl b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_app.erl index e5941ea07..09de5640d 100644 --- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_app.erl +++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -58,7 +58,7 @@ load_auth_hook() -> ok = emqx_auth_mnesia:init(#{clientid_list => ClientidList, username_list => UsernameList}), ok = emqx_auth_mnesia:register_metrics(), Params = #{ - hash_type => application:get_env(emqx_auth_mnesia, hash_type, sha256) + hash_type => application:get_env(emqx_auth_mnesia, password_hash, sha256) }, emqx:hook('client.authenticate', fun emqx_auth_mnesia:check/3, [Params]). diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_cli.erl b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_cli.erl index ef78b1c3c..d89e6836c 100644 --- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_cli.erl +++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_cli.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_sup.erl b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_sup.erl index ed32b31e7..3784eaaf6 100644 --- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_sup.erl +++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_auth_mnesia/test/emqx_acl_mnesia_SUITE.erl b/apps/emqx_auth_mnesia/test/emqx_acl_mnesia_SUITE.erl index e5e48cc93..8ace680da 100644 --- a/apps/emqx_auth_mnesia/test/emqx_acl_mnesia_SUITE.erl +++ b/apps/emqx_auth_mnesia/test/emqx_acl_mnesia_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl b/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl index 8ac942257..c5c0eb727 100644 --- a/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl +++ b/apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -47,33 +47,13 @@ groups() -> []. init_per_suite(Config) -> - ok = emqx_ct_helpers:start_apps([emqx_modules, emqx_management, emqx_auth_mnesia], fun set_special_configs/1), + ok = emqx_ct_helpers:start_apps([emqx_management, emqx_auth_mnesia], fun set_special_configs/1), create_default_app(), Config. end_per_suite(_Config) -> delete_default_app(), - emqx_ct_helpers:stop_apps([emqx_modules, emqx_management, emqx_auth_mnesia]). - -init_per_testcase(t_check_as_clientid, Config) -> - Params = #{ - hash_type => application:get_env(emqx_auth_mnesia, hash_type, sha256), - key_as => clientid - }, - emqx:hook('client.authenticate', fun emqx_auth_mnesia:check/3, [Params]), - Config; - -init_per_testcase(_, Config) -> - Params = #{ - hash_type => application:get_env(emqx_auth_mnesia, hash_type, sha256), - key_as => username - }, - emqx:hook('client.authenticate', fun emqx_auth_mnesia:check/3, [Params]), - Config. - -end_per_suite(_, Config) -> - emqx:unhook('client.authenticate', fun emqx_auth_mnesia:check/3), - Config. + emqx_ct_helpers:stop_apps([emqx_management, emqx_auth_mnesia]). set_special_configs(emqx) -> application:set_env(emqx, allow_anonymous, true), @@ -275,6 +255,30 @@ t_username_rest_api(_Config) -> {ok, Result5} = request_http_rest_lookup([Path]), ?assertMatch(#{}, get_http_data(Result5)). +t_password_hash(_) -> + clean_all_users(), + {ok, Default} = application:get_env(emqx_auth_mnesia, password_hash), + application:set_env(emqx_auth_mnesia, password_hash, plain), + + %% change the password_hash to 'plain' + application:stop(emqx_auth_mnesia), + ok = application:start(emqx_auth_mnesia), + + Params = #{<<"username">> => ?USERNAME, <<"password">> => ?PASSWORD}, + {ok, _} = request_http_rest_add(["auth_username"], Params), + + %% check + User = #{username => ?USERNAME, + clientid => undefined, + password => ?PASSWORD, + zone => external}, + {ok, #{auth_result := success, + anonymous := false}} = emqx_access_control:authenticate(User), + + application:set_env(emqx_auth_mnesia, password_hash, Default), + application:stop(emqx_auth_mnesia), + ok = application:start(emqx_auth_mnesia). + %%------------------------------------------------------------------------------ %% Helpers %%------------------------------------------------------------------------------ diff --git a/apps/emqx_auth_mongo/src/emqx_acl_mongo.erl b/apps/emqx_auth_mongo/src/emqx_acl_mongo.erl index c0ff5f8ac..653600395 100644 --- a/apps/emqx_auth_mongo/src/emqx_acl_mongo.erl +++ b/apps/emqx_auth_mongo/src/emqx_acl_mongo.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_auth_mongo/src/emqx_auth_mongo.erl b/apps/emqx_auth_mongo/src/emqx_auth_mongo.erl index 2ca4a6f54..cd1d21b42 100644 --- a/apps/emqx_auth_mongo/src/emqx_auth_mongo.erl +++ b/apps/emqx_auth_mongo/src/emqx_auth_mongo.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_auth_mongo/src/emqx_auth_mongo_app.erl b/apps/emqx_auth_mongo/src/emqx_auth_mongo_app.erl index e13fe30b7..bff779cf0 100644 --- a/apps/emqx_auth_mongo/src/emqx_auth_mongo_app.erl +++ b/apps/emqx_auth_mongo/src/emqx_auth_mongo_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_auth_mongo/src/emqx_auth_mongo_sup.erl b/apps/emqx_auth_mongo/src/emqx_auth_mongo_sup.erl index dc2c37fd4..3f27cb1dd 100644 --- a/apps/emqx_auth_mongo/src/emqx_auth_mongo_sup.erl +++ b/apps/emqx_auth_mongo/src/emqx_auth_mongo_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl b/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl index 03e1fa33e..a988e87bb 100644 --- a/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl +++ b/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_auth_mysql/src/emqx_acl_mysql.erl b/apps/emqx_auth_mysql/src/emqx_acl_mysql.erl index 174e3cd27..ef4acea94 100644 --- a/apps/emqx_auth_mysql/src/emqx_acl_mysql.erl +++ b/apps/emqx_auth_mysql/src/emqx_acl_mysql.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_auth_mysql/src/emqx_auth_mysql.erl b/apps/emqx_auth_mysql/src/emqx_auth_mysql.erl index 9cd9d5a25..eba7ef081 100644 --- a/apps/emqx_auth_mysql/src/emqx_auth_mysql.erl +++ b/apps/emqx_auth_mysql/src/emqx_auth_mysql.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_auth_mysql/src/emqx_auth_mysql_app.erl b/apps/emqx_auth_mysql/src/emqx_auth_mysql_app.erl index 3936f6c4c..7d696ed52 100644 --- a/apps/emqx_auth_mysql/src/emqx_auth_mysql_app.erl +++ b/apps/emqx_auth_mysql/src/emqx_auth_mysql_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_auth_mysql/src/emqx_auth_mysql_cli.erl b/apps/emqx_auth_mysql/src/emqx_auth_mysql_cli.erl index c3ee3b02a..cf3be3426 100644 --- a/apps/emqx_auth_mysql/src/emqx_auth_mysql_cli.erl +++ b/apps/emqx_auth_mysql/src/emqx_auth_mysql_cli.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_auth_mysql/src/emqx_auth_mysql_sup.erl b/apps/emqx_auth_mysql/src/emqx_auth_mysql_sup.erl index 0b2bf2c38..70f4987a3 100644 --- a/apps/emqx_auth_mysql/src/emqx_auth_mysql_sup.erl +++ b/apps/emqx_auth_mysql/src/emqx_auth_mysql_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE.erl b/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE.erl index 044655ac1..168d544cb 100644 --- a/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE.erl +++ b/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_auth_pgsql/src/emqx_acl_pgsql.erl b/apps/emqx_auth_pgsql/src/emqx_acl_pgsql.erl index c1792f1e2..099ce4438 100644 --- a/apps/emqx_auth_pgsql/src/emqx_acl_pgsql.erl +++ b/apps/emqx_auth_pgsql/src/emqx_acl_pgsql.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.erl b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.erl index 2dee5ef50..f8f365cd7 100644 --- a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.erl +++ b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql_app.erl b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql_app.erl index 1d05f6b8a..e08b990b4 100644 --- a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql_app.erl +++ b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql_cli.erl b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql_cli.erl index 5b9dbd24a..7dde566a2 100644 --- a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql_cli.erl +++ b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql_cli.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql_sup.erl b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql_sup.erl index 162d04747..21d005dc2 100644 --- a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql_sup.erl +++ b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE.erl b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE.erl index fa11d6792..6c4cd2eb3 100644 --- a/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE.erl +++ b/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_auth_redis/rebar.config b/apps/emqx_auth_redis/rebar.config index 528ffbeb4..750f07809 100644 --- a/apps/emqx_auth_redis/rebar.config +++ b/apps/emqx_auth_redis/rebar.config @@ -1,7 +1,7 @@ {deps, %% NOTE: mind poolboy version when updating eredis_cluster version %% poolboy version may clash with emqx_auth_mongo - [{eredis_cluster, {git, "https://github.com/emqx/eredis_cluster", {tag, "0.6.4"}}}, + [ {poolboy, {git, "https://github.com/emqx/poolboy.git", {tag, "1.5.2"}}} ]}. diff --git a/apps/emqx_auth_redis/src/emqx_acl_redis.erl b/apps/emqx_auth_redis/src/emqx_acl_redis.erl index 096523487..47f5acbba 100644 --- a/apps/emqx_auth_redis/src/emqx_acl_redis.erl +++ b/apps/emqx_auth_redis/src/emqx_acl_redis.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_auth_redis/src/emqx_auth_redis.erl b/apps/emqx_auth_redis/src/emqx_auth_redis.erl index 65f4d9735..318a8b23a 100644 --- a/apps/emqx_auth_redis/src/emqx_auth_redis.erl +++ b/apps/emqx_auth_redis/src/emqx_auth_redis.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_auth_redis/src/emqx_auth_redis_app.erl b/apps/emqx_auth_redis/src/emqx_auth_redis_app.erl index 619b3f78d..8f8ffb751 100644 --- a/apps/emqx_auth_redis/src/emqx_auth_redis_app.erl +++ b/apps/emqx_auth_redis/src/emqx_auth_redis_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_auth_redis/src/emqx_auth_redis_cli.erl b/apps/emqx_auth_redis/src/emqx_auth_redis_cli.erl index 26550dff4..52ac39a7b 100644 --- a/apps/emqx_auth_redis/src/emqx_auth_redis_cli.erl +++ b/apps/emqx_auth_redis/src/emqx_auth_redis_cli.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -38,7 +38,7 @@ connect(Opts) -> Host = case Sentinel =:= "" of true -> get_value(host, Opts); false -> - _ = eredis_sentinel:start_link(get_value(servers, Opts)), + _ = eredis_sentinel:start_link(get_value(servers, Opts), get_value(options, Opts, [])), "sentinel:" ++ Sentinel end, case eredis:start_link(Host, diff --git a/apps/emqx_auth_redis/src/emqx_auth_redis_sup.erl b/apps/emqx_auth_redis/src/emqx_auth_redis_sup.erl index 83112976d..ef81eef86 100644 --- a/apps/emqx_auth_redis/src/emqx_auth_redis_sup.erl +++ b/apps/emqx_auth_redis/src/emqx_auth_redis_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE.erl b/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE.erl index 3fee03a70..24d3b20bd 100644 --- a/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE.erl +++ b/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_bridge_mqtt/include/emqx_bridge_mqtt.hrl b/apps/emqx_bridge_mqtt/include/emqx_bridge_mqtt.hrl index 4bc9ede14..531518668 100644 --- a/apps/emqx_bridge_mqtt/include/emqx_bridge_mqtt.hrl +++ b/apps/emqx_bridge_mqtt/include/emqx_bridge_mqtt.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_connect.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_connect.erl index 5086ca574..ece6002a7 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_connect.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_connect.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.erl index b3fd1316f..d612af668 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl index 8f5ad16ef..3f685a72a 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_app.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_app.erl index 2c0a0b9a9..2382b3a42 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_app.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_cli.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_cli.erl index c6d6e2378..a76ea3a8c 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_cli.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_cli.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_sup.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_sup.erl index 92d8e4083..80a11c1c0 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_sup.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_msg.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_msg.erl index 19f5fa139..91fd18bf4 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_msg.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_msg.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_rpc.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_rpc.erl index 4226137a1..0cf4b5bc5 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_rpc.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_rpc.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_worker.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_worker.erl index 38454581d..e7414683c 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_worker.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_worker.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -337,7 +337,7 @@ connecting(#{reconnect_delay_ms := ReconnectDelayMs} = State) -> end. connected(state_timeout, connected, #{inflight := Inflight} = State) -> - case retry_inflight(State, Inflight) of + case retry_inflight(State#{inflight := []}, Inflight) of {ok, NewState} -> {keep_state, NewState, {next_event, internal, maybe_send}}; {error, NewState} -> @@ -348,10 +348,10 @@ connected(internal, maybe_send, State) -> {keep_state, NewState}; connected(info, {disconnected, Conn, Reason}, - #{connection := Connection, name := Name, reconnect_delay_ms := ReconnectDelayMs} = State) -> + #{connection := Connection, name := Name, reconnect_delay_ms := ReconnectDelayMs} = State) -> + ?tp(info, disconnected, #{name => Name, reason => Reason}), case Conn =:= maps:get(client_pid, Connection, undefined) of true -> - ?LOG(info, "Bridge ~p diconnected~nreason=~p", [Name, Reason]), {next_state, idle, State#{connection => undefined}, {state_timeout, ReconnectDelayMs, reconnect}}; false -> keep_state_and_data @@ -434,12 +434,14 @@ do_connect(#{forwards := Forwards, subscriptions := Subs, connect_module := ConnectModule, connect_cfg := ConnectCfg, + inflight := Inflight, name := Name} = State) -> ok = subscribe_local_topics(Forwards, Name), case emqx_bridge_connect:start(ConnectModule, ConnectCfg#{subscriptions => Subs}) of {ok, Conn} -> - ?LOG(info, "Bridge ~p is connecting......", [Name]), - {ok, eval_bridge_handler(State#{connection => Conn}, connected)}; + Res = eval_bridge_handler(State#{connection => Conn}, connected), + ?tp(info, connected, #{name => Name, inflight => length(Inflight)}), + {ok, Res}; {error, Reason} -> {error, Reason, State} end. @@ -475,10 +477,12 @@ collect(Acc) -> %% Retry all inflight (previously sent but not acked) batches. retry_inflight(State, []) -> {ok, State}; -retry_inflight(State, [#{q_ack_ref := QAckRef, batch := Batch} | Inflight]) -> - case do_send(State#{inflight := Inflight}, QAckRef, Batch) of - {ok, State1} -> retry_inflight(State1, Inflight); - {error, State1} -> {error, State1} +retry_inflight(State, [#{q_ack_ref := QAckRef, batch := Batch} | Rest] = OldInf) -> + case do_send(State, QAckRef, Batch) of + {ok, State1} -> + retry_inflight(State1, Rest); + {error, #{inflight := NewInf} = State1} -> + {error, State1#{inflight := NewInf ++ OldInf}} end. pop_and_send(#{inflight := Inflight, max_inflight := Max } = State) -> diff --git a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_tests.erl b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_tests.erl index 392fa8e17..830fb1fe0 100644 --- a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_tests.erl +++ b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_tests.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_bridge_mqtt/test/emqx_bridge_rpc_tests.erl b/apps/emqx_bridge_mqtt/test/emqx_bridge_rpc_tests.erl index f79d12dfb..fdcc25d5f 100644 --- a/apps/emqx_bridge_mqtt/test/emqx_bridge_rpc_tests.erl +++ b/apps/emqx_bridge_mqtt/test/emqx_bridge_rpc_tests.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_bridge_mqtt/test/emqx_bridge_stub_conn.erl b/apps/emqx_bridge_mqtt/test/emqx_bridge_stub_conn.erl index 6b8db31ea..d38663fcd 100644 --- a/apps/emqx_bridge_mqtt/test/emqx_bridge_stub_conn.erl +++ b/apps/emqx_bridge_mqtt/test/emqx_bridge_stub_conn.erl @@ -27,14 +27,15 @@ -type ack_ref() :: emqx_bridge_worker:ack_ref(). -type batch() :: emqx_bridge_worker:batch(). -start(Cfg) -> +start(#{client_pid := Pid} = Cfg) -> + Pid ! {self(), ?MODULE, ready}, {ok, Cfg}. stop(_) -> ok. %% @doc Callback for `emqx_bridge_connect' behaviour -spec send(_, batch()) -> {ok, ack_ref()} | {error, any()}. -send(#{stub_pid := Pid}, Batch) -> +send(#{client_pid := Pid}, Batch) -> Ref = make_ref(), Pid ! {stub_message, self(), Ref, Batch}, {ok, Ref}. diff --git a/apps/emqx_bridge_mqtt/test/emqx_bridge_worker_SUITE.erl b/apps/emqx_bridge_mqtt/test/emqx_bridge_worker_SUITE.erl index a39f5bd5b..680756742 100644 --- a/apps/emqx_bridge_mqtt/test/emqx_bridge_worker_SUITE.erl +++ b/apps/emqx_bridge_mqtt/test/emqx_bridge_worker_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -191,9 +191,15 @@ t_stub_normal(Config) when is_list(Config) -> connect_module => emqx_bridge_stub_conn, forward_mountpoint => <<"forwarded">>, start_type => auto, - stub_pid => self() + client_pid => self() }, {ok, Pid} = emqx_bridge_worker:start_link(?FUNCTION_NAME, Cfg), + receive + {Pid, emqx_bridge_stub_conn, ready} -> ok + after + 5000 -> + error(timeout) + end, ClientId = <<"ClientId">>, try {ok, ConnPid} = emqtt:start_link([{clientid, ClientId}]), @@ -203,6 +209,9 @@ t_stub_normal(Config) when is_list(Config) -> {stub_message, WorkerPid, BatchRef, _Batch} -> WorkerPid ! {batch_ack, BatchRef}, ok + after + 5000 -> + error(timeout) end, ?SNK_WAIT(inflight_drained), ?SNK_WAIT(replayq_drained), @@ -218,7 +227,7 @@ t_stub_overflow(Config) when is_list(Config) -> connect_module => emqx_bridge_stub_conn, forward_mountpoint => <<"forwarded">>, start_type => auto, - stub_pid => self(), + client_pid => self(), max_inflight => MaxInflight }, {ok, Worker} = emqx_bridge_worker:start_link(?FUNCTION_NAME, Cfg), @@ -250,7 +259,7 @@ t_stub_random_order(Config) when is_list(Config) -> connect_module => emqx_bridge_stub_conn, forward_mountpoint => <<"forwarded">>, start_type => auto, - stub_pid => self(), + client_pid => self(), max_inflight => MaxInflight }, {ok, Worker} = emqx_bridge_worker:start_link(?FUNCTION_NAME, Cfg), @@ -273,6 +282,53 @@ t_stub_random_order(Config) when is_list(Config) -> ok = emqx_bridge_worker:stop(Worker) end. +t_stub_retry_inflight(Config) when is_list(Config) -> + Topic = <<"to_stub_retry_inflight/a">>, + MaxInflight = 10, + Cfg = #{forwards => [Topic], + connect_module => emqx_bridge_stub_conn, + forward_mountpoint => <<"forwarded">>, + reconnect_delay_ms => 10, + start_type => auto, + client_pid => self(), + max_inflight => MaxInflight + }, + {ok, Worker} = emqx_bridge_worker:start_link(?FUNCTION_NAME, Cfg), + ClientId = <<"ClientId2">>, + try + case ?block_until(#{?snk_kind := connected, inflight := 0}, 2000, 1000) of + {ok, #{inflight := 0}} -> ok; + Other -> ct:fail("~p", [Other]) + end, + {ok, ConnPid} = emqtt:start_link([{clientid, ClientId}]), + {ok, _} = emqtt:connect(ConnPid), + lists:foreach( + fun(I) -> + Data = integer_to_binary(I), + _ = emqtt:publish(ConnPid, Topic, Data, ?QOS_1) + end, lists:seq(1, MaxInflight)), + %% receive acks but do not ack + Acks1 = stub_receive(MaxInflight), + ?assertEqual(MaxInflight, length(Acks1)), + %% simulate a disconnect + Worker ! {disconnected, self(), test}, + ?SNK_WAIT(disconnected), + case ?block_until(#{?snk_kind := connected, inflight := MaxInflight}, 2000, 20) of + {ok, _} -> ok; + Error -> ct:fail("~p", [Error]) + end, + %% expect worker to retry inflight, so to receive acks again + Acks2 = stub_receive(MaxInflight), + ?assertEqual(MaxInflight, length(Acks2)), + lists:foreach(fun({Pid, Ref}) -> Pid ! {batch_ack, Ref} end, + lists:reverse(Acks2)), + ?SNK_WAIT(inflight_drained), + ?SNK_WAIT(replayq_drained), + emqtt:disconnect(ConnPid) + after + ok = emqx_bridge_worker:stop(Worker) + end. + stub_receive(N) -> stub_receive(N, []). diff --git a/apps/emqx_bridge_mqtt/test/emqx_bridge_worker_tests.erl b/apps/emqx_bridge_mqtt/test/emqx_bridge_worker_tests.erl index 1e2dfda1b..69ff87356 100644 --- a/apps/emqx_bridge_mqtt/test/emqx_bridge_worker_tests.erl +++ b/apps/emqx_bridge_mqtt/test/emqx_bridge_worker_tests.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_coap/README.md b/apps/emqx_coap/README.md index 927666358..1a9ee802c 100644 --- a/apps/emqx_coap/README.md +++ b/apps/emqx_coap/README.md @@ -151,7 +151,7 @@ To subscribe any topic, issue following command: - if clientid is absent, a "bad_request" will be returned. - {topicname} in URI should be percent-encoded to prevent special characters, such as + and #. - {username} and {password} are optional. -- if {username} or {password} is incorrect, the error code `uauthorized` will be returned. +- if {username} or {password} is incorrect, the error code `unauthorized` will be returned. - topic is subscribed with qos1. - if the subscription failed due to ACL deny, the error code `forbidden` will be returned. @@ -169,7 +169,7 @@ To cancel observation, issue following command: - if clientid is absent, a "bad_request" will be returned. - {topicname} in URI should be percent-encoded to prevent special characters, such as + and #. - {username} and {password} are optional. -- if {username} or {password} is incorrect, the error code `uauthorized` will be returned. +- if {username} or {password} is incorrect, the error code `unauthorized` will be returned. CoAP Client Notification Operation (subscribed Message) ------------------------------------------------------- @@ -192,7 +192,7 @@ Issue a coap put command to publish messages. For example: - if clientid is absent, a "bad_request" will be returned. - {topicname} in URI should be percent-encoded to prevent special characters, such as + and #. - {username} and {password} are optional. -- if {username} or {password} is incorrect, the error code `uauthorized` will be returned. +- if {username} or {password} is incorrect, the error code `unauthorized` will be returned. - payload could be any binary data. - payload data type is "application/octet-stream". - publish message will be sent with qos0. @@ -211,7 +211,7 @@ Device should issue a get command periodically, serve as a ping to keep mqtt ses - {any_topicname} is optional, and should be percent-encoded to prevent special characters. - {clientid} is mandatory. If clientid is absent, a "bad_request" will be returned. - {username} and {password} are optional. -- if {username} or {password} is incorrect, the error code `uauthorized` will be returned. +- if {username} or {password} is incorrect, the error code `unauthorized` will be returned. - coap client should do keepalive work periodically to keep mqtt session online, especially those devices in a NAT network. @@ -231,7 +231,7 @@ ClientId, Username, Password and Topic ClientId/username/password/topic in the coap URI are the concepts in mqtt. That is to say, emqx-coap is trying to fit coap message into mqtt system, by borrowing the client/username/password/topic from mqtt. The Auth/ACL/Hook features in mqtt also applies on coap stuff. For example: -- If username/password is not authorized, coap client will get an uauthorized error. +- If username/password is not authorized, coap client will get an unauthorized error. - If username or clientid is not allowed to published specific topic, coap message will be dropped in fact, although coap client will get an acknoledgement from emqx-coap. - If a coap message is published, a 'message.publish' hook is able to capture this message as well. diff --git a/apps/emqx_coap/include/emqx_coap.hrl b/apps/emqx_coap/include/emqx_coap.hrl index 8204dc98c..963feca6b 100644 --- a/apps/emqx_coap/include/emqx_coap.hrl +++ b/apps/emqx_coap/include/emqx_coap.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_coap/src/emqx_coap_app.erl b/apps/emqx_coap/src/emqx_coap_app.erl index 029d78071..16f74faf7 100644 --- a/apps/emqx_coap/src/emqx_coap_app.erl +++ b/apps/emqx_coap/src/emqx_coap_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_coap/src/emqx_coap_mqtt_adapter.erl b/apps/emqx_coap/src/emqx_coap_mqtt_adapter.erl index ac981b87c..d465f9ca3 100644 --- a/apps/emqx_coap/src/emqx_coap_mqtt_adapter.erl +++ b/apps/emqx_coap/src/emqx_coap_mqtt_adapter.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -37,7 +37,9 @@ , stop/1 ]). --export([call/2]). +-export([ call/2 + , call/3 + ]). %% gen_server. -export([ init/1 @@ -93,6 +95,9 @@ publish(Pid, Topic, Payload) -> %% For emqx_management plugin call(Pid, Msg) -> + call(Pid, Msg, infinity). + +call(Pid, Msg, _) -> Pid ! Msg, ok. %%-------------------------------------------------------------------- @@ -100,8 +105,8 @@ call(Pid, Msg) -> %%-------------------------------------------------------------------- init({ClientId, Username, Password, Channel}) -> - ?LOG(debug, "try to start adapter ClientId=~p, Username=~p, Password=~p, Channel=~p", - [ClientId, Username, Password, Channel]), + ?LOG(debug, "try to start adapter ClientId=~p, Username=~p, Password=~p, " + "Channel=~0p", [ClientId, Username, Password, Channel]), State0 = #state{peername = Channel, clientid = ClientId, username = Username, diff --git a/apps/emqx_coap/src/emqx_coap_pubsub_resource.erl b/apps/emqx_coap/src/emqx_coap_pubsub_resource.erl index da066bb36..d87a26173 100644 --- a/apps/emqx_coap/src/emqx_coap_pubsub_resource.erl +++ b/apps/emqx_coap/src/emqx_coap_pubsub_resource.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -65,7 +65,7 @@ coap_get(ChId, ?PS_PREFIX, TopicPath, Query, Content=#coap_content{format = Form end; {error, auth_failure} -> put(mqtt_client_pid, undefined), - {error, uauthorized}; + {error, unauthorized}; {error, bad_request} -> put(mqtt_client_pid, undefined), {error, bad_request}; diff --git a/apps/emqx_coap/src/emqx_coap_pubsub_topics.erl b/apps/emqx_coap/src/emqx_coap_pubsub_topics.erl index 79b707e6d..57d0c3ae6 100644 --- a/apps/emqx_coap/src/emqx_coap_pubsub_topics.erl +++ b/apps/emqx_coap/src/emqx_coap_pubsub_topics.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_coap/src/emqx_coap_registry.erl b/apps/emqx_coap/src/emqx_coap_registry.erl index 8e1936a98..066d37f1e 100644 --- a/apps/emqx_coap/src/emqx_coap_registry.erl +++ b/apps/emqx_coap/src/emqx_coap_registry.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_coap/src/emqx_coap_resource.erl b/apps/emqx_coap/src/emqx_coap_resource.erl index e46317347..739037a42 100644 --- a/apps/emqx_coap/src/emqx_coap_resource.erl +++ b/apps/emqx_coap/src/emqx_coap_resource.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -56,7 +56,7 @@ coap_get(ChId, ?MQTT_PREFIX, Path, Query, _Content) -> #coap_content{}; {error, auth_failure} -> put(mqtt_client_pid, undefined), - {error, forbidden}; + {error, unauthorized}; {error, bad_request} -> put(mqtt_client_pid, undefined), {error, bad_request}; diff --git a/apps/emqx_coap/src/emqx_coap_server.erl b/apps/emqx_coap/src/emqx_coap_server.erl index 4774bd310..ebdc1a0fe 100644 --- a/apps/emqx_coap/src/emqx_coap_server.erl +++ b/apps/emqx_coap/src/emqx_coap_server.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_coap/src/emqx_coap_sup.erl b/apps/emqx_coap/src/emqx_coap_sup.erl index f61604ef5..94e9a1c77 100644 --- a/apps/emqx_coap/src/emqx_coap_sup.erl +++ b/apps/emqx_coap/src/emqx_coap_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_coap/src/emqx_coap_timer.erl b/apps/emqx_coap/src/emqx_coap_timer.erl index 3924ba239..92b0ddb2f 100644 --- a/apps/emqx_coap/src/emqx_coap_timer.erl +++ b/apps/emqx_coap/src/emqx_coap_timer.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_coap/test/emqx_coap_SUITE.erl b/apps/emqx_coap/test/emqx_coap_SUITE.erl index 0faa4965c..ed59b309c 100644 --- a/apps/emqx_coap/test/emqx_coap_SUITE.erl +++ b/apps/emqx_coap/test/emqx_coap_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -130,7 +130,7 @@ t_observe_wildcard(_Config) -> ?assert(is_pid(SubPid)), %% Publish a message - emqx:publish(emqx_message:make(Topic, Payload)), + emqx:publish(emqx_message:make(<<"a/b">>, Payload)), Notif = receive_notification(), ?LOGT("observer get Notif=~p", [Notif]), diff --git a/apps/emqx_coap/test/emqx_coap_pubsub_SUITE.erl b/apps/emqx_coap/test/emqx_coap_pubsub_SUITE.erl index 886d5c782..7f49ece7b 100644 --- a/apps/emqx_coap/test/emqx_coap_pubsub_SUITE.erl +++ b/apps/emqx_coap/test/emqx_coap_pubsub_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_exhook/include/emqx_exhook.hrl b/apps/emqx_exhook/include/emqx_exhook.hrl index 301488619..7301fdcbb 100644 --- a/apps/emqx_exhook/include/emqx_exhook.hrl +++ b/apps/emqx_exhook/include/emqx_exhook.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -20,25 +20,25 @@ -define(APP, emqx_exhook). -define(ENABLED_HOOKS, - [ {'client.connect', {?MODULE, on_client_connect, []}} - , {'client.connack', {?MODULE, on_client_connack, []}} - , {'client.connected', {?MODULE, on_client_connected, []}} - , {'client.disconnected', {?MODULE, on_client_disconnected, []}} - , {'client.authenticate', {?MODULE, on_client_authenticate, []}} - , {'client.check_acl', {?MODULE, on_client_check_acl, []}} - , {'client.subscribe', {?MODULE, on_client_subscribe, []}} - , {'client.unsubscribe', {?MODULE, on_client_unsubscribe, []}} - , {'session.created', {?MODULE, on_session_created, []}} - , {'session.subscribed', {?MODULE, on_session_subscribed, []}} - , {'session.unsubscribed',{?MODULE, on_session_unsubscribed, []}} - , {'session.resumed', {?MODULE, on_session_resumed, []}} - , {'session.discarded', {?MODULE, on_session_discarded, []}} - , {'session.takeovered', {?MODULE, on_session_takeovered, []}} - , {'session.terminated', {?MODULE, on_session_terminated, []}} - , {'message.publish', {?MODULE, on_message_publish, []}} - , {'message.delivered', {?MODULE, on_message_delivered, []}} - , {'message.acked', {?MODULE, on_message_acked, []}} - , {'message.dropped', {?MODULE, on_message_dropped, []}} + [ {'client.connect', {emqx_exhook_handler, on_client_connect, []}} + , {'client.connack', {emqx_exhook_handler, on_client_connack, []}} + , {'client.connected', {emqx_exhook_handler, on_client_connected, []}} + , {'client.disconnected', {emqx_exhook_handler, on_client_disconnected, []}} + , {'client.authenticate', {emqx_exhook_handler, on_client_authenticate, []}} + , {'client.check_acl', {emqx_exhook_handler, on_client_check_acl, []}} + , {'client.subscribe', {emqx_exhook_handler, on_client_subscribe, []}} + , {'client.unsubscribe', {emqx_exhook_handler, on_client_unsubscribe, []}} + , {'session.created', {emqx_exhook_handler, on_session_created, []}} + , {'session.subscribed', {emqx_exhook_handler, on_session_subscribed, []}} + , {'session.unsubscribed',{emqx_exhook_handler, on_session_unsubscribed, []}} + , {'session.resumed', {emqx_exhook_handler, on_session_resumed, []}} + , {'session.discarded', {emqx_exhook_handler, on_session_discarded, []}} + , {'session.takeovered', {emqx_exhook_handler, on_session_takeovered, []}} + , {'session.terminated', {emqx_exhook_handler, on_session_terminated, []}} + , {'message.publish', {emqx_exhook_handler, on_message_publish, []}} + , {'message.delivered', {emqx_exhook_handler, on_message_delivered, []}} + , {'message.acked', {emqx_exhook_handler, on_message_acked, []}} + , {'message.dropped', {emqx_exhook_handler, on_message_dropped, []}} ]). -endif. diff --git a/apps/emqx_exhook/priv/emqx_exhook.schema b/apps/emqx_exhook/priv/emqx_exhook.schema index 2a926b968..e5481a3dd 100644 --- a/apps/emqx_exhook/priv/emqx_exhook.schema +++ b/apps/emqx_exhook/priv/emqx_exhook.schema @@ -26,9 +26,9 @@ [{scheme, https}, {host, Host}, {port, Port}, {ssl_options, Filter([{ssl, true}, - {certfile, cuttlefish:conf_get(Prefix ++ ".ssl.certfile", Conf)}, - {keyfile, cuttlefish:conf_get(Prefix ++ ".ssl.keyfile", Conf)}, - {cacertfile, cuttlefish:conf_get(Prefix ++ ".ssl.cacertfile", Conf)} + {certfile, cuttlefish:conf_get(Prefix ++ ".ssl.certfile", Conf, undefined)}, + {keyfile, cuttlefish:conf_get(Prefix ++ ".ssl.keyfile", Conf, undefined)}, + {cacertfile, cuttlefish:conf_get(Prefix ++ ".ssl.cacertfile", Conf, undefined)} ])}]; _ -> error(invalid_server_options) end diff --git a/apps/emqx_exhook/priv/protos/exhook.proto b/apps/emqx_exhook/priv/protos/exhook.proto index 41f471d14..612b5151f 100644 --- a/apps/emqx_exhook/priv/protos/exhook.proto +++ b/apps/emqx_exhook/priv/protos/exhook.proto @@ -1,5 +1,5 @@ //------------------------------------------------------------------------------ -// Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +// Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -335,6 +335,12 @@ message ClientInfo { bool is_superuser = 9; bool anonymous = 10; + + // common name of client TLS cert + string cn = 11; + + // subject of client TLS cert + string dn = 12; } message Message { diff --git a/apps/emqx_exhook/src/emqx_exhook.erl b/apps/emqx_exhook/src/emqx_exhook.erl index 9f6d27b26..032d7f91a 100644 --- a/apps/emqx_exhook/src/emqx_exhook.erl +++ b/apps/emqx_exhook/src/emqx_exhook.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -36,6 +36,7 @@ %% Mgmt APIs %%-------------------------------------------------------------------- +%% XXX: Only return the running servers -spec list() -> [emqx_exhook_server:server()]. list() -> [server(Name) || Name <- running()]. @@ -96,7 +97,8 @@ call_fold(Hookpoint, Req, AccFun, [ServiceName|More]) -> {ok, Resp} -> case AccFun(Req, Resp) of {stop, NReq} -> {stop, NReq}; - {ok, NReq} -> call_fold(Hookpoint, NReq, AccFun, More) + {ok, NReq} -> call_fold(Hookpoint, NReq, AccFun, More); + _ -> call_fold(Hookpoint, Req, AccFun, More) end; _ -> call_fold(Hookpoint, Req, AccFun, More) diff --git a/apps/emqx_exhook/src/emqx_exhook_app.erl b/apps/emqx_exhook/src/emqx_exhook_app.erl index 62c1903bd..4e00340d8 100644 --- a/apps/emqx_exhook/src/emqx_exhook_app.erl +++ b/apps/emqx_exhook/src/emqx_exhook_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -34,7 +34,6 @@ , unload_server/1 , unload_exhooks/0 , init_hooks_cnter/0 - , deinit_hooks_cnter/0 ]). %%-------------------------------------------------------------------- @@ -57,7 +56,6 @@ start(_StartType, _StartArgs) -> prep_stop(State) -> emqx_ctl:unregister_command(exhook), _ = unload_exhooks(), - _ = deinit_hooks_cnter(), ok = unload_all_servers(), State. @@ -94,5 +92,3 @@ init_hooks_cnter() -> ok end. -deinit_hooks_cnter() -> - ets:delete(?CNTER). diff --git a/apps/emqx_exhook/src/emqx_exhook_cli.erl b/apps/emqx_exhook/src/emqx_exhook_cli.erl index 9fea1f50d..a8dc43b16 100644 --- a/apps/emqx_exhook/src/emqx_exhook_cli.erl +++ b/apps/emqx_exhook/src/emqx_exhook_cli.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_exhook/src/emqx_exhook_handler.erl b/apps/emqx_exhook/src/emqx_exhook_handler.erl index 13c41ce6a..f3964dc42 100644 --- a/apps/emqx_exhook/src/emqx_exhook_handler.erl +++ b/apps/emqx_exhook/src/emqx_exhook_handler.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -89,6 +89,12 @@ on_client_disconnected(ClientInfo, Reason, _ConnInfo) -> cast('client.disconnected', Req). on_client_authenticate(ClientInfo, AuthResult) -> + %% XXX: Bool is missing more information about the atom of the result + %% So, the `Req` has missed detailed info too. + %% + %% The return value of `call_fold` just a bool, that has missed + %% detailed info too. + %% Bool = maps:get(auth_result, AuthResult, undefined) == success, Req = #{clientinfo => clientinfo(ClientInfo), result => Bool @@ -96,8 +102,8 @@ on_client_authenticate(ClientInfo, AuthResult) -> case call_fold('client.authenticate', Req, fun merge_responsed_bool/2) of - {StopOrOk, #{result := Bool}} when is_boolean(Bool) -> - Result = case Bool of true -> success; _ -> not_authorized end, + {StopOrOk, #{result := Result0}} when is_boolean(Result0) -> + Result = case Result0 of true -> success; _ -> not_authorized end, {StopOrOk, AuthResult#{auth_result => Result, anonymous => false}}; _ -> {ok, AuthResult} @@ -116,8 +122,8 @@ on_client_check_acl(ClientInfo, PubSub, Topic, Result) -> }, case call_fold('client.check_acl', Req, fun merge_responsed_bool/2) of - {StopOrOk, #{result := Bool}} when is_boolean(Bool) -> - NResult = case Bool of true -> allow; _ -> deny end, + {StopOrOk, #{result := Result0}} when is_boolean(Result0) -> + NResult = case Result0 of true -> allow; _ -> deny end, {StopOrOk, NResult}; _ -> {ok, Result} end. @@ -248,7 +254,9 @@ clientinfo(ClientInfo = protocol => stringfy(Protocol), mountpoint => maybe(Mountpoiont), is_superuser => maps:get(is_superuser, ClientInfo, false), - anonymous => maps:get(anonymous, ClientInfo, true)}. + anonymous => maps:get(anonymous, ClientInfo, true), + cn => maybe(maps:get(cn, ClientInfo, undefined)), + dn => maybe(maps:get(dn, ClientInfo, undefined))}. message(#message{id = Id, qos = Qos, from = From, topic = Topic, payload = Payload, timestamp = Ts}) -> #{node => stringfy(node()), @@ -287,8 +295,8 @@ stringfy(Term) -> %% Acc funcs %% see exhook.proto -merge_responsed_bool(Req, #{type := 'IGNORE'}) -> - {ok, Req}; +merge_responsed_bool(_Req, #{type := 'IGNORE'}) -> + ignore; merge_responsed_bool(Req, #{type := Type, value := {bool_result, NewBool}}) when is_boolean(NewBool) -> NReq = Req#{result => NewBool}, @@ -296,18 +304,18 @@ merge_responsed_bool(Req, #{type := Type, value := {bool_result, NewBool}}) 'CONTINUE' -> {ok, NReq}; 'STOP_AND_RETURN' -> {stop, NReq} end; -merge_responsed_bool(Req, Resp) -> +merge_responsed_bool(_Req, Resp) -> ?LOG(warning, "Unknown responsed value ~0p to merge to callback chain", [Resp]), - {ok, Req}. + ignore. -merge_responsed_message(Req, #{type := 'IGNORE'}) -> - {ok, Req}; +merge_responsed_message(_Req, #{type := 'IGNORE'}) -> + ignore; merge_responsed_message(Req, #{type := Type, value := {message, NMessage}}) -> NReq = Req#{message => NMessage}, case Type of 'CONTINUE' -> {ok, NReq}; 'STOP_AND_RETURN' -> {stop, NReq} end; -merge_responsed_message(Req, Resp) -> +merge_responsed_message(_Req, Resp) -> ?LOG(warning, "Unknown responsed value ~0p to merge to callback chain", [Resp]), - {ok, Req}. + ignore. diff --git a/apps/emqx_exhook/src/emqx_exhook_server.erl b/apps/emqx_exhook/src/emqx_exhook_server.erl index 5a353b61b..848a3f59d 100644 --- a/apps/emqx_exhook/src/emqx_exhook_server.erl +++ b/apps/emqx_exhook/src/emqx_exhook_server.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -83,7 +83,7 @@ -spec load(atom(), list()) -> {ok, server()} | {error, term()} . load(Name0, Opts0) -> - Name = prefix(Name0), + Name = to_list(Name0), {SvrAddr, ClientOpts} = channel_opts(Opts0), case emqx_exhook_sup:start_grpc_client_channel( Name, @@ -110,12 +110,12 @@ load(Name0, Opts0) -> end. %% @private -prefix(Name) when is_atom(Name) -> - "exhook:" ++ atom_to_list(Name); -prefix(Name) when is_binary(Name) -> - "exhook:" ++ binary_to_list(Name); -prefix(Name) when is_list(Name) -> - "exhook:" ++ Name. +to_list(Name) when is_atom(Name) -> + atom_to_list(Name); +to_list(Name) when is_binary(Name) -> + binary_to_list(Name); +to_list(Name) when is_list(Name) -> + Name. %% @private channel_opts(Opts) -> diff --git a/apps/emqx_exhook/src/emqx_exhook_sup.erl b/apps/emqx_exhook/src/emqx_exhook_sup.erl index c8d2ecf35..c3ca811bd 100644 --- a/apps/emqx_exhook/src/emqx_exhook_sup.erl +++ b/apps/emqx_exhook/src/emqx_exhook_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_exhook/test/emqx_exhook_SUITE.erl b/apps/emqx_exhook/test/emqx_exhook_SUITE.erl index b66950215..5d5a396a5 100644 --- a/apps/emqx_exhook/test/emqx_exhook_SUITE.erl +++ b/apps/emqx_exhook/test/emqx_exhook_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -40,8 +40,8 @@ end_per_suite(_Cfg) -> set_special_cfgs(emqx) -> application:set_env(emqx, allow_anonymous, false), application:set_env(emqx, enable_acl_cache, false), - application:set_env(emqx, plugins_loaded_file, - emqx_ct_helpers:deps_path(emqx, "test/emqx_SUITE_data/loaded_plugins")); + application:set_env(emqx, plugins_loaded_file, undefined), + application:set_env(emqx, modules_loaded_file, undefined); set_special_cfgs(emqx_exhook) -> ok. @@ -49,5 +49,48 @@ set_special_cfgs(emqx_exhook) -> %% Test cases %%-------------------------------------------------------------------- -t_hooks(_Cfg) -> - ok. +t_noserver_nohook(_) -> + emqx_exhook:disable(default), + ?assertEqual([], ets:tab2list(emqx_hooks)), + + Opts = proplists:get_value( + default, + application:get_env(emqx_exhook, servers, []) + ), + ok = emqx_exhook:enable(default, Opts), + ?assertNotEqual([], ets:tab2list(emqx_hooks)). + +t_cli_list(_) -> + meck_print(), + ?assertEqual( [[emqx_exhook_server:format(Svr) || Svr <- emqx_exhook:list()]] + , emqx_exhook_cli:cli(["server", "list"]) + ), + unmeck_print(). + +t_cli_enable_disable(_) -> + meck_print(), + ?assertEqual([already_started], emqx_exhook_cli:cli(["server", "enable", "default"])), + ?assertEqual(ok, emqx_exhook_cli:cli(["server", "disable", "default"])), + ?assertEqual([], emqx_exhook_cli:cli(["server", "list"])), + + ?assertEqual([not_running], emqx_exhook_cli:cli(["server", "disable", "default"])), + ?assertEqual(ok, emqx_exhook_cli:cli(["server", "enable", "default"])), + unmeck_print(). + +t_cli_stats(_) -> + meck_print(), + _ = emqx_exhook_cli:cli(["server", "stats"]), + _ = emqx_exhook_cli:cli(x), + unmeck_print(). + +%%-------------------------------------------------------------------- +%% Utils +%%-------------------------------------------------------------------- + +meck_print() -> + meck:new(emqx_ctl, [passthrough, no_history, no_link]), + meck:expect(emqx_ctl, print, fun(_) -> ok end), + meck:expect(emqx_ctl, print, fun(_, Args) -> Args end). + +unmeck_print() -> + meck:unload(emqx_ctl). diff --git a/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl b/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl index 05fa07465..c2db04dd4 100644 --- a/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl +++ b/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -179,18 +179,44 @@ on_client_disconnected(Req, Md) -> -spec on_client_authenticate(emqx_exhook_pb:client_authenticate_request(), grpc:metadata()) -> {ok, emqx_exhook_pb:valued_response(), grpc:metadata()} | {error, grpc_cowboy_h:error_response()}. -on_client_authenticate(Req, Md) -> +on_client_authenticate(#{clientinfo := #{username := Username}} = Req, Md) -> ?MODULE:in({?FUNCTION_NAME, Req}), %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), - {ok, #{type => 'IGNORE'}, Md}. + %% some cases for testing + case Username of + <<"baduser">> -> + {ok, #{type => 'STOP_AND_RETURN', + value => {bool_result, false}}, Md}; + <<"gooduser">> -> + {ok, #{type => 'STOP_AND_RETURN', + value => {bool_result, true}}, Md}; + <<"normaluser">> -> + {ok, #{type => 'CONTINUE', + value => {bool_result, true}}, Md}; + _ -> + {ok, #{type => 'IGNORE'}, Md} + end. -spec on_client_check_acl(emqx_exhook_pb:client_check_acl_request(), grpc:metadata()) -> {ok, emqx_exhook_pb:valued_response(), grpc:metadata()} | {error, grpc_cowboy_h:error_response()}. -on_client_check_acl(Req, Md) -> +on_client_check_acl(#{clientinfo := #{username := Username}} = Req, Md) -> ?MODULE:in({?FUNCTION_NAME, Req}), %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), - {ok, #{type => 'STOP_AND_RETURN', value => {bool_result, true}}, Md}. + %% some cases for testing + case Username of + <<"baduser">> -> + {ok, #{type => 'STOP_AND_RETURN', + value => {bool_result, false}}, Md}; + <<"gooduser">> -> + {ok, #{type => 'STOP_AND_RETURN', + value => {bool_result, true}}, Md}; + <<"normaluser">> -> + {ok, #{type => 'CONTINUE', + value => {bool_result, true}}, Md}; + _ -> + {ok, #{type => 'IGNORE'}, Md} + end. -spec on_client_subscribe(emqx_exhook_pb:client_subscribe_request(), grpc:metadata()) -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()} @@ -267,10 +293,26 @@ on_session_terminated(Req, Md) -> -spec on_message_publish(emqx_exhook_pb:message_publish_request(), grpc:metadata()) -> {ok, emqx_exhook_pb:valued_response(), grpc:metadata()} | {error, grpc_cowboy_h:error_response()}. -on_message_publish(Req, Md) -> +on_message_publish(#{message := #{from := From} = Msg} = Req, Md) -> ?MODULE:in({?FUNCTION_NAME, Req}), %io:format("fun: ~p, req: ~0p~n", [?FUNCTION_NAME, Req]), - {ok, #{}, Md}. + %% some cases for testing + case From of + <<"baduser">> -> + NMsg = Msg#{qos => 0, + topic => <<"">>, + payload => <<"">> + }, + {ok, #{type => 'STOP_AND_RETURN', + value => {message, NMsg}}, Md}; + <<"gooduser">> -> + NMsg = Msg#{topic => From, + payload => From}, + {ok, #{type => 'STOP_AND_RETURN', + value => {message, NMsg}}, Md}; + _ -> + {ok, #{type => 'IGNORE'}, Md} + end. -spec on_message_delivered(emqx_exhook_pb:message_delivered_request(), grpc:metadata()) -> {ok, emqx_exhook_pb:empty_success(), grpc:metadata()} diff --git a/apps/emqx_exhook/test/props/prop_exhook_hooks.erl b/apps/emqx_exhook/test/props/prop_exhook_hooks.erl index e4c11dd3d..24f45c8b0 100644 --- a/apps/emqx_exhook/test/props/prop_exhook_hooks.erl +++ b/apps/emqx_exhook/test/props/prop_exhook_hooks.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -44,20 +44,11 @@ prop_client_connect() -> ?ALL({ConnInfo, ConnProps}, {conninfo(), conn_properties()}, begin - _OutConnProps = emqx_hooks:run_fold('client.connect', [ConnInfo], ConnProps), + ok = emqx_hooks:run('client.connect', [ConnInfo, ConnProps]), {'on_client_connect', Resp} = emqx_exhook_demo_svr:take(), Expected = #{props => properties(ConnProps), - conninfo => - #{node => nodestr(), - clientid => maps:get(clientid, ConnInfo), - username => maybe(maps:get(username, ConnInfo, <<>>)), - peerhost => peerhost(ConnInfo), - sockport => sockport(ConnInfo), - proto_name => maps:get(proto_name, ConnInfo), - proto_ver => stringfy(maps:get(proto_ver, ConnInfo)), - keepalive => maps:get(keepalive, ConnInfo) - } + conninfo => from_conninfo(ConnInfo) }, ?assertEqual(Expected, Resp), true @@ -67,78 +58,86 @@ prop_client_connack() -> ?ALL({ConnInfo, Rc, AckProps}, {conninfo(), connack_return_code(), ack_properties()}, begin - _OutAckProps = emqx_hooks:run_fold('client.connack', [ConnInfo, Rc], AckProps), + ok = emqx_hooks:run('client.connack', [ConnInfo, Rc, AckProps]), {'on_client_connack', Resp} = emqx_exhook_demo_svr:take(), Expected = #{props => properties(AckProps), result_code => atom_to_binary(Rc, utf8), - conninfo => - #{node => nodestr(), - clientid => maps:get(clientid, ConnInfo), - username => maybe(maps:get(username, ConnInfo, <<>>)), - peerhost => peerhost(ConnInfo), - sockport => sockport(ConnInfo), - proto_name => maps:get(proto_name, ConnInfo), - proto_ver => stringfy(maps:get(proto_ver, ConnInfo)), - keepalive => maps:get(keepalive, ConnInfo) - } + conninfo => from_conninfo(ConnInfo) }, ?assertEqual(Expected, Resp), true end). prop_client_authenticate() -> - ?ALL({ClientInfo, AuthResult}, {clientinfo(), authresult()}, + ?ALL({ClientInfo0, AuthResult}, + {clientinfo(), authresult()}, begin - _OutAuthResult = emqx_hooks:run_fold('client.authenticate', [ClientInfo], AuthResult), + ClientInfo = inject_magic_into(username, ClientInfo0), + OutAuthResult = emqx_hooks:run_fold('client.authenticate', [ClientInfo], AuthResult), + ExpectedAuthResult = case maps:get(username, ClientInfo) of + <<"baduser">> -> + AuthResult#{ + auth_result => not_authorized, + anonymous => false}; + <<"gooduser">> -> + AuthResult#{ + auth_result => success, + anonymous => false}; + <<"normaluser">> -> + AuthResult#{ + auth_result => success, + anonymous => false}; + _ -> + case maps:get(auth_result, AuthResult) of + success -> + #{auth_result => success, + anonymous => false}; + _ -> + #{auth_result => not_authorized, + anonymous => false} + end + end, + ?assertEqual(ExpectedAuthResult, OutAuthResult), + {'on_client_authenticate', Resp} = emqx_exhook_demo_svr:take(), Expected = #{result => authresult_to_bool(AuthResult), - clientinfo => - #{node => nodestr(), - clientid => maps:get(clientid, ClientInfo), - username => maybe(maps:get(username, ClientInfo, <<>>)), - password => maybe(maps:get(password, ClientInfo, <<>>)), - peerhost => ntoa(maps:get(peerhost, ClientInfo)), - sockport => maps:get(sockport, ClientInfo), - protocol => stringfy(maps:get(protocol, ClientInfo)), - mountpoint => maybe(maps:get(mountpoint, ClientInfo, <<>>)), - is_superuser => maps:get(is_superuser, ClientInfo, false), - anonymous => maps:get(anonymous, ClientInfo, true) - } + clientinfo => from_clientinfo(ClientInfo) }, ?assertEqual(Expected, Resp), true end). prop_client_check_acl() -> - ?ALL({ClientInfo, PubSub, Topic, Result}, - {clientinfo(), oneof([publish, subscribe]), topic(), oneof([allow, deny])}, + ?ALL({ClientInfo0, PubSub, Topic, Result}, + {clientinfo(), oneof([publish, subscribe]), + topic(), oneof([allow, deny])}, begin - _OutResult = emqx_hooks:run_fold('client.check_acl', [ClientInfo, PubSub, Topic], Result), + ClientInfo = inject_magic_into(username, ClientInfo0), + OutResult = emqx_hooks:run_fold( + 'client.check_acl', + [ClientInfo, PubSub, Topic], + Result), + ExpectedOutResult = case maps:get(username, ClientInfo) of + <<"baduser">> -> deny; + <<"gooduser">> -> allow; + <<"normaluser">> -> allow; + _ -> Result + end, + ?assertEqual(ExpectedOutResult, OutResult), + {'on_client_check_acl', Resp} = emqx_exhook_demo_svr:take(), Expected = #{result => aclresult_to_bool(Result), type => pubsub_to_enum(PubSub), topic => Topic, - clientinfo => - #{node => nodestr(), - clientid => maps:get(clientid, ClientInfo), - username => maybe(maps:get(username, ClientInfo, <<>>)), - password => maybe(maps:get(password, ClientInfo, <<>>)), - peerhost => ntoa(maps:get(peerhost, ClientInfo)), - sockport => maps:get(sockport, ClientInfo), - protocol => stringfy(maps:get(protocol, ClientInfo)), - mountpoint => maybe(maps:get(mountpoint, ClientInfo, <<>>)), - is_superuser => maps:get(is_superuser, ClientInfo, false), - anonymous => maps:get(anonymous, ClientInfo, true) - } + clientinfo => from_clientinfo(ClientInfo) }, ?assertEqual(Expected, Resp), true end). - prop_client_connected() -> ?ALL({ClientInfo, ConnInfo}, {clientinfo(), conninfo()}, @@ -146,18 +145,7 @@ prop_client_connected() -> ok = emqx_hooks:run('client.connected', [ClientInfo, ConnInfo]), {'on_client_connected', Resp} = emqx_exhook_demo_svr:take(), Expected = - #{clientinfo => - #{node => nodestr(), - clientid => maps:get(clientid, ClientInfo), - username => maybe(maps:get(username, ClientInfo, <<>>)), - password => maybe(maps:get(password, ClientInfo, <<>>)), - peerhost => ntoa(maps:get(peerhost, ClientInfo)), - sockport => maps:get(sockport, ClientInfo), - protocol => stringfy(maps:get(protocol, ClientInfo)), - mountpoint => maybe(maps:get(mountpoint, ClientInfo, <<>>)), - is_superuser => maps:get(is_superuser, ClientInfo, false), - anonymous => maps:get(anonymous, ClientInfo, true) - } + #{clientinfo => from_clientinfo(ClientInfo) }, ?assertEqual(Expected, Resp), true @@ -171,18 +159,7 @@ prop_client_disconnected() -> {'on_client_disconnected', Resp} = emqx_exhook_demo_svr:take(), Expected = #{reason => stringfy(Reason), - clientinfo => - #{node => nodestr(), - clientid => maps:get(clientid, ClientInfo), - username => maybe(maps:get(username, ClientInfo, <<>>)), - password => maybe(maps:get(password, ClientInfo, <<>>)), - peerhost => ntoa(maps:get(peerhost, ClientInfo)), - sockport => maps:get(sockport, ClientInfo), - protocol => stringfy(maps:get(protocol, ClientInfo)), - mountpoint => maybe(maps:get(mountpoint, ClientInfo, <<>>)), - is_superuser => maps:get(is_superuser, ClientInfo, false), - anonymous => maps:get(anonymous, ClientInfo, true) - } + clientinfo => from_clientinfo(ClientInfo) }, ?assertEqual(Expected, Resp), true @@ -192,23 +169,12 @@ prop_client_subscribe() -> ?ALL({ClientInfo, SubProps, TopicTab}, {clientinfo(), sub_properties(), topictab()}, begin - _OutTopicTab = emqx_hooks:run_fold('client.subscribe', [ClientInfo, SubProps], TopicTab), + ok = emqx_hooks:run('client.subscribe', [ClientInfo, SubProps, TopicTab]), {'on_client_subscribe', Resp} = emqx_exhook_demo_svr:take(), Expected = #{props => properties(SubProps), topic_filters => topicfilters(TopicTab), - clientinfo => - #{node => nodestr(), - clientid => maps:get(clientid, ClientInfo), - username => maybe(maps:get(username, ClientInfo, <<>>)), - password => maybe(maps:get(password, ClientInfo, <<>>)), - peerhost => ntoa(maps:get(peerhost, ClientInfo)), - sockport => maps:get(sockport, ClientInfo), - protocol => stringfy(maps:get(protocol, ClientInfo)), - mountpoint => maybe(maps:get(mountpoint, ClientInfo, <<>>)), - is_superuser => maps:get(is_superuser, ClientInfo, false), - anonymous => maps:get(anonymous, ClientInfo, true) - } + clientinfo => from_clientinfo(ClientInfo) }, ?assertEqual(Expected, Resp), true @@ -218,23 +184,12 @@ prop_client_unsubscribe() -> ?ALL({ClientInfo, UnSubProps, TopicTab}, {clientinfo(), unsub_properties(), topictab()}, begin - _OutTopicTab = emqx_hooks:run_fold('client.unsubscribe', [ClientInfo, UnSubProps], TopicTab), + ok = emqx_hooks:run('client.unsubscribe', [ClientInfo, UnSubProps, TopicTab]), {'on_client_unsubscribe', Resp} = emqx_exhook_demo_svr:take(), Expected = #{props => properties(UnSubProps), topic_filters => topicfilters(TopicTab), - clientinfo => - #{node => nodestr(), - clientid => maps:get(clientid, ClientInfo), - username => maybe(maps:get(username, ClientInfo, <<>>)), - password => maybe(maps:get(password, ClientInfo, <<>>)), - peerhost => ntoa(maps:get(peerhost, ClientInfo)), - sockport => maps:get(sockport, ClientInfo), - protocol => stringfy(maps:get(protocol, ClientInfo)), - mountpoint => maybe(maps:get(mountpoint, ClientInfo, <<>>)), - is_superuser => maps:get(is_superuser, ClientInfo, false), - anonymous => maps:get(anonymous, ClientInfo, true) - } + clientinfo => from_clientinfo(ClientInfo) }, ?assertEqual(Expected, Resp), true @@ -246,18 +201,7 @@ prop_session_created() -> ok = emqx_hooks:run('session.created', [ClientInfo, SessInfo]), {'on_session_created', Resp} = emqx_exhook_demo_svr:take(), Expected = - #{clientinfo => - #{node => nodestr(), - clientid => maps:get(clientid, ClientInfo), - username => maybe(maps:get(username, ClientInfo, <<>>)), - password => maybe(maps:get(password, ClientInfo, <<>>)), - peerhost => ntoa(maps:get(peerhost, ClientInfo)), - sockport => maps:get(sockport, ClientInfo), - protocol => stringfy(maps:get(protocol, ClientInfo)), - mountpoint => maybe(maps:get(mountpoint, ClientInfo, <<>>)), - is_superuser => maps:get(is_superuser, ClientInfo, false), - anonymous => maps:get(anonymous, ClientInfo, true) - } + #{clientinfo => from_clientinfo(ClientInfo) }, ?assertEqual(Expected, Resp), true @@ -272,18 +216,7 @@ prop_session_subscribed() -> Expected = #{topic => Topic, subopts => subopts(SubOpts), - clientinfo => - #{node => nodestr(), - clientid => maps:get(clientid, ClientInfo), - username => maybe(maps:get(username, ClientInfo, <<>>)), - password => maybe(maps:get(password, ClientInfo, <<>>)), - peerhost => ntoa(maps:get(peerhost, ClientInfo)), - sockport => maps:get(sockport, ClientInfo), - protocol => stringfy(maps:get(protocol, ClientInfo)), - mountpoint => maybe(maps:get(mountpoint, ClientInfo, <<>>)), - is_superuser => maps:get(is_superuser, ClientInfo, false), - anonymous => maps:get(anonymous, ClientInfo, true) - } + clientinfo => from_clientinfo(ClientInfo) }, ?assertEqual(Expected, Resp), true @@ -297,18 +230,7 @@ prop_session_unsubscribed() -> {'on_session_unsubscribed', Resp} = emqx_exhook_demo_svr:take(), Expected = #{topic => Topic, - clientinfo => - #{node => nodestr(), - clientid => maps:get(clientid, ClientInfo), - username => maybe(maps:get(username, ClientInfo, <<>>)), - password => maybe(maps:get(password, ClientInfo, <<>>)), - peerhost => ntoa(maps:get(peerhost, ClientInfo)), - sockport => maps:get(sockport, ClientInfo), - protocol => stringfy(maps:get(protocol, ClientInfo)), - mountpoint => maybe(maps:get(mountpoint, ClientInfo, <<>>)), - is_superuser => maps:get(is_superuser, ClientInfo, false), - anonymous => maps:get(anonymous, ClientInfo, true) - } + clientinfo => from_clientinfo(ClientInfo) }, ?assertEqual(Expected, Resp), true @@ -320,18 +242,7 @@ prop_session_resumed() -> ok = emqx_hooks:run('session.resumed', [ClientInfo, SessInfo]), {'on_session_resumed', Resp} = emqx_exhook_demo_svr:take(), Expected = - #{clientinfo => - #{node => nodestr(), - clientid => maps:get(clientid, ClientInfo), - username => maybe(maps:get(username, ClientInfo, <<>>)), - password => maybe(maps:get(password, ClientInfo, <<>>)), - peerhost => ntoa(maps:get(peerhost, ClientInfo)), - sockport => maps:get(sockport, ClientInfo), - protocol => stringfy(maps:get(protocol, ClientInfo)), - mountpoint => maybe(maps:get(mountpoint, ClientInfo, <<>>)), - is_superuser => maps:get(is_superuser, ClientInfo, false), - anonymous => maps:get(anonymous, ClientInfo, true) - } + #{clientinfo => from_clientinfo(ClientInfo) }, ?assertEqual(Expected, Resp), true @@ -343,18 +254,7 @@ prop_session_discared() -> ok = emqx_hooks:run('session.discarded', [ClientInfo, SessInfo]), {'on_session_discarded', Resp} = emqx_exhook_demo_svr:take(), Expected = - #{clientinfo => - #{node => nodestr(), - clientid => maps:get(clientid, ClientInfo), - username => maybe(maps:get(username, ClientInfo, <<>>)), - password => maybe(maps:get(password, ClientInfo, <<>>)), - peerhost => ntoa(maps:get(peerhost, ClientInfo)), - sockport => maps:get(sockport, ClientInfo), - protocol => stringfy(maps:get(protocol, ClientInfo)), - mountpoint => maybe(maps:get(mountpoint, ClientInfo, <<>>)), - is_superuser => maps:get(is_superuser, ClientInfo, false), - anonymous => maps:get(anonymous, ClientInfo, true) - } + #{clientinfo => from_clientinfo(ClientInfo) }, ?assertEqual(Expected, Resp), true @@ -366,18 +266,7 @@ prop_session_takeovered() -> ok = emqx_hooks:run('session.takeovered', [ClientInfo, SessInfo]), {'on_session_takeovered', Resp} = emqx_exhook_demo_svr:take(), Expected = - #{clientinfo => - #{node => nodestr(), - clientid => maps:get(clientid, ClientInfo), - username => maybe(maps:get(username, ClientInfo, <<>>)), - password => maybe(maps:get(password, ClientInfo, <<>>)), - peerhost => ntoa(maps:get(peerhost, ClientInfo)), - sockport => maps:get(sockport, ClientInfo), - protocol => stringfy(maps:get(protocol, ClientInfo)), - mountpoint => maybe(maps:get(mountpoint, ClientInfo, <<>>)), - is_superuser => maps:get(is_superuser, ClientInfo, false), - anonymous => maps:get(anonymous, ClientInfo, true) - } + #{clientinfo => from_clientinfo(ClientInfo) }, ?assertEqual(Expected, Resp), true @@ -391,21 +280,98 @@ prop_session_terminated() -> {'on_session_terminated', Resp} = emqx_exhook_demo_svr:take(), Expected = #{reason => stringfy(Reason), - clientinfo => - #{node => nodestr(), - clientid => maps:get(clientid, ClientInfo), - username => maybe(maps:get(username, ClientInfo, <<>>)), - password => maybe(maps:get(password, ClientInfo, <<>>)), - peerhost => ntoa(maps:get(peerhost, ClientInfo)), - sockport => maps:get(sockport, ClientInfo), - protocol => stringfy(maps:get(protocol, ClientInfo)), - mountpoint => maybe(maps:get(mountpoint, ClientInfo, <<>>)), - is_superuser => maps:get(is_superuser, ClientInfo, false), - anonymous => maps:get(anonymous, ClientInfo, true) - } + clientinfo => from_clientinfo(ClientInfo) }, ?assertEqual(Expected, Resp), + true + end). +prop_message_publish() -> + ?ALL(Msg0, message(), + begin + Msg = emqx_message:from_map( + inject_magic_into(from, emqx_message:to_map(Msg0))), + OutMsg= emqx_hooks:run_fold('message.publish', [], Msg), + case emqx_topic:match(emqx_message:topic(Msg), <<"$SYS/#">>) of + true -> + ?assertEqual(Msg, OutMsg), + skip; + _ -> + ExpectedOutMsg = case emqx_message:from(Msg) of + <<"baduser">> -> + MsgMap = emqx_message:to_map(Msg), + emqx_message:from_map( + MsgMap#{qos => 0, + topic => <<"">>, + payload => <<"">> + }); + <<"gooduser">> = From -> + MsgMap = emqx_message:to_map(Msg), + emqx_message:from_map( + MsgMap#{topic => From, + payload => From + }); + _ -> Msg + end, + ?assertEqual(ExpectedOutMsg, OutMsg), + + {'on_message_publish', Resp} = emqx_exhook_demo_svr:take(), + Expected = + #{message => from_message(Msg) + }, + ?assertEqual(Expected, Resp) + end, + true + end). + +prop_message_dropped() -> + ?ALL({Msg, By, Reason}, {message(), hardcoded, shutdown_reason()}, + begin + ok = emqx_hooks:run('message.dropped', [Msg, By, Reason]), + case emqx_topic:match(emqx_message:topic(Msg), <<"$SYS/#">>) of + true -> skip; + _ -> + {'on_message_dropped', Resp} = emqx_exhook_demo_svr:take(), + Expected = + #{reason => stringfy(Reason), + message => from_message(Msg) + }, + ?assertEqual(Expected, Resp) + end, + true + end). + +prop_message_delivered() -> + ?ALL({ClientInfo, Msg}, {clientinfo(), message()}, + begin + ok = emqx_hooks:run('message.delivered', [ClientInfo, Msg]), + case emqx_topic:match(emqx_message:topic(Msg), <<"$SYS/#">>) of + true -> skip; + _ -> + {'on_message_delivered', Resp} = emqx_exhook_demo_svr:take(), + Expected = + #{clientinfo => from_clientinfo(ClientInfo), + message => from_message(Msg) + }, + ?assertEqual(Expected, Resp) + end, + true + end). + +prop_message_acked() -> + ?ALL({ClientInfo, Msg}, {clientinfo(), message()}, + begin + ok = emqx_hooks:run('message.acked', [ClientInfo, Msg]), + case emqx_topic:match(emqx_message:topic(Msg), <<"$SYS/#">>) of + true -> skip; + _ -> + {'on_message_acked', Resp} = emqx_exhook_demo_svr:take(), + Expected = + #{clientinfo => from_clientinfo(ClientInfo), + message => from_message(Msg) + }, + ?assertEqual(Expected, Resp) + end, true end). @@ -465,32 +431,50 @@ aclresult_to_bool(Result) -> pubsub_to_enum(publish) -> 'PUBLISH'; pubsub_to_enum(subscribe) -> 'SUBSCRIBE'. -%prop_message_publish() -> -% ?ALL({Msg, Env, Encode}, {message(), topic_filter_env()}, -% begin -% true -% end). -% -%prop_message_delivered() -> -% ?ALL({ClientInfo, Msg, Env, Encode}, {clientinfo(), message(), topic_filter_env()}, -% begin -% true -% end). -% -%prop_message_acked() -> -% ?ALL({ClientInfo, Msg, Env, Encode}, {clientinfo(), message()}, -% begin -% true -% end). +from_conninfo(ConnInfo) -> + #{node => nodestr(), + clientid => maps:get(clientid, ConnInfo), + username => maybe(maps:get(username, ConnInfo, <<>>)), + peerhost => peerhost(ConnInfo), + sockport => sockport(ConnInfo), + proto_name => maps:get(proto_name, ConnInfo), + proto_ver => stringfy(maps:get(proto_ver, ConnInfo)), + keepalive => maps:get(keepalive, ConnInfo) + }. + +from_clientinfo(ClientInfo) -> + #{node => nodestr(), + clientid => maps:get(clientid, ClientInfo), + username => maybe(maps:get(username, ClientInfo, <<>>)), + password => maybe(maps:get(password, ClientInfo, <<>>)), + peerhost => ntoa(maps:get(peerhost, ClientInfo)), + sockport => maps:get(sockport, ClientInfo), + protocol => stringfy(maps:get(protocol, ClientInfo)), + mountpoint => maybe(maps:get(mountpoint, ClientInfo, <<>>)), + is_superuser => maps:get(is_superuser, ClientInfo, false), + anonymous => maps:get(anonymous, ClientInfo, true), + cn => maybe(maps:get(cn, ClientInfo, <<>>)), + dn => maybe(maps:get(dn, ClientInfo, <<>>)) + }. + +from_message(Msg) -> + #{node => nodestr(), + id => emqx_guid:to_hexstr(emqx_message:id(Msg)), + qos => emqx_message:qos(Msg), + from => stringfy(emqx_message:from(Msg)), + topic => emqx_message:topic(Msg), + payload => emqx_message:payload(Msg), + timestamp => emqx_message:timestamp(Msg) + }. %%-------------------------------------------------------------------- %% Helper %%-------------------------------------------------------------------- do_setup() -> + logger:set_primary_config(#{level => warning}), _ = emqx_exhook_demo_svr:start(), emqx_ct_helpers:start_apps([emqx_exhook], fun set_special_cfgs/1), - emqx_logger:set_log_level(warning), %% waiting first loaded event {'on_provider_loaded', _} = emqx_exhook_demo_svr:take(), ok. @@ -500,12 +484,14 @@ do_teardown(_) -> %% waiting last unloaded event {'on_provider_unloaded', _} = emqx_exhook_demo_svr:take(), _ = emqx_exhook_demo_svr:stop(), + logger:set_primary_config(#{level => notice}), timer:sleep(2000), ok. set_special_cfgs(emqx) -> application:set_env(emqx, allow_anonymous, false), application:set_env(emqx, enable_acl_cache, false), + application:set_env(emqx, modules_loaded_file, undefined), application:set_env(emqx, plugins_loaded_file, emqx_ct_helpers:deps_path(emqx, "test/emqx_SUITE_data/loaded_plugins")); set_special_cfgs(emqx_exhook) -> @@ -528,10 +514,18 @@ unsub_properties() -> #{}. shutdown_reason() -> - oneof([utf8(), {shutdown, atom()}]). + oneof([utf8(), {shutdown, emqx_ct_proper_types:limited_atom()}]). authresult() -> - #{auth_result => connack_return_code()}. + ?LET(RC, connack_return_code(), #{auth_result => RC}). -%topic_filter_env() -> -% oneof([{<<"#">>}, {undefined}, {topic()}]). +inject_magic_into(Key, Object) -> + case castspell() of + muggles -> Object; + Spell -> + Object#{Key => Spell} + end. + +castspell() -> + L = [<<"baduser">>, <<"gooduser">>, <<"normaluser">>, muggles], + lists:nth(rand:uniform(length(L)), L). diff --git a/apps/emqx_exproto/include/emqx_exproto.hrl b/apps/emqx_exproto/include/emqx_exproto.hrl index 079a1e60f..a599b3039 100644 --- a/apps/emqx_exproto/include/emqx_exproto.hrl +++ b/apps/emqx_exproto/include/emqx_exproto.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_exproto/priv/protos/exproto.proto b/apps/emqx_exproto/priv/protos/exproto.proto index 4b567693c..bbc10073c 100644 --- a/apps/emqx_exproto/priv/protos/exproto.proto +++ b/apps/emqx_exproto/priv/protos/exproto.proto @@ -1,5 +1,5 @@ //------------------------------------------------------------------------------ -// Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +// Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/apps/emqx_exproto/src/emqx_exproto.erl b/apps/emqx_exproto/src/emqx_exproto.erl index 07f56ef1b..453b4989b 100644 --- a/apps/emqx_exproto/src/emqx_exproto.erl +++ b/apps/emqx_exproto/src/emqx_exproto.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_exproto/src/emqx_exproto_app.erl b/apps/emqx_exproto/src/emqx_exproto_app.erl index 73e8a65bc..d9598bcd9 100644 --- a/apps/emqx_exproto/src/emqx_exproto_app.erl +++ b/apps/emqx_exproto/src/emqx_exproto_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_exproto/src/emqx_exproto_channel.erl b/apps/emqx_exproto/src/emqx_exproto_channel.erl index 2966cbba9..229e6f930 100644 --- a/apps/emqx_exproto/src/emqx_exproto_channel.erl +++ b/apps/emqx_exproto/src/emqx_exproto_channel.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_exproto/src/emqx_exproto_conn.erl b/apps/emqx_exproto/src/emqx_exproto_conn.erl index efe4b7f22..da655bcb4 100644 --- a/apps/emqx_exproto/src/emqx_exproto_conn.erl +++ b/apps/emqx_exproto/src/emqx_exproto_conn.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -32,6 +32,7 @@ ]). -export([ call/2 + , call/3 , cast/2 ]). @@ -154,7 +155,10 @@ stats(#state{socket = Socket, lists:append([SockStats, ConnStats, ChanStats, ProcStats]). call(Pid, Req) -> - gen_server:call(Pid, Req, infinity). + call(Pid, Req, infinity). + +call(Pid, Req, Timeout) -> + gen_server:call(Pid, Req, Timeout). cast(Pid, Req) -> gen_server:cast(Pid, Req). diff --git a/apps/emqx_exproto/src/emqx_exproto_gcli.erl b/apps/emqx_exproto/src/emqx_exproto_gcli.erl index 90e5d75a3..650922c4b 100644 --- a/apps/emqx_exproto/src/emqx_exproto_gcli.erl +++ b/apps/emqx_exproto/src/emqx_exproto_gcli.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_exproto/src/emqx_exproto_gsvr.erl b/apps/emqx_exproto/src/emqx_exproto_gsvr.erl index c1007ee1d..a4ad5b2e4 100644 --- a/apps/emqx_exproto/src/emqx_exproto_gsvr.erl +++ b/apps/emqx_exproto/src/emqx_exproto_gsvr.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_exproto/src/emqx_exproto_sup.erl b/apps/emqx_exproto/src/emqx_exproto_sup.erl index d4bde316a..fc70b8131 100644 --- a/apps/emqx_exproto/src/emqx_exproto_sup.erl +++ b/apps/emqx_exproto/src/emqx_exproto_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_exproto/test/emqx_exproto_SUITE.erl b/apps/emqx_exproto/test/emqx_exproto_SUITE.erl index c5cabe154..f1fdfa9f8 100644 --- a/apps/emqx_exproto/test/emqx_exproto_SUITE.erl +++ b/apps/emqx_exproto/test/emqx_exproto_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_exproto/test/emqx_exproto_echo_svr.erl b/apps/emqx_exproto/test/emqx_exproto_echo_svr.erl index 031093b50..69c64d4ca 100644 --- a/apps/emqx_exproto/test/emqx_exproto_echo_svr.erl +++ b/apps/emqx_exproto/test/emqx_exproto_echo_svr.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_lua_hook/include/emqx_lua_hook.hrl b/apps/emqx_lua_hook/include/emqx_lua_hook.hrl index 94d7b7623..fb7010c2b 100644 --- a/apps/emqx_lua_hook/include/emqx_lua_hook.hrl +++ b/apps/emqx_lua_hook/include/emqx_lua_hook.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_lua_hook/src/emqx_lua_hook.erl b/apps/emqx_lua_hook/src/emqx_lua_hook.erl index 40fd26ab7..6e7810827 100644 --- a/apps/emqx_lua_hook/src/emqx_lua_hook.erl +++ b/apps/emqx_lua_hook/src/emqx_lua_hook.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_lua_hook/src/emqx_lua_hook_app.erl b/apps/emqx_lua_hook/src/emqx_lua_hook_app.erl index 076c74031..6b0ec3574 100644 --- a/apps/emqx_lua_hook/src/emqx_lua_hook_app.erl +++ b/apps/emqx_lua_hook/src/emqx_lua_hook_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_lua_hook/src/emqx_lua_hook_cli.erl b/apps/emqx_lua_hook/src/emqx_lua_hook_cli.erl index 3839b01c9..83c6fc5ef 100644 --- a/apps/emqx_lua_hook/src/emqx_lua_hook_cli.erl +++ b/apps/emqx_lua_hook/src/emqx_lua_hook_cli.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_lua_hook/src/emqx_lua_hook_sup.erl b/apps/emqx_lua_hook/src/emqx_lua_hook_sup.erl index 374cb033d..a10e5731c 100644 --- a/apps/emqx_lua_hook/src/emqx_lua_hook_sup.erl +++ b/apps/emqx_lua_hook/src/emqx_lua_hook_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_lua_hook/src/emqx_lua_script.erl b/apps/emqx_lua_hook/src/emqx_lua_script.erl index 9b86be247..ffc53fc8d 100644 --- a/apps/emqx_lua_hook/src/emqx_lua_script.erl +++ b/apps/emqx_lua_hook/src/emqx_lua_script.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_lua_hook/test/emqx_lua_hook_SUITE.erl b/apps/emqx_lua_hook/test/emqx_lua_hook_SUITE.erl index 1f58dcb22..5e6681aed 100644 --- a/apps/emqx_lua_hook/test/emqx_lua_hook_SUITE.erl +++ b/apps/emqx_lua_hook/test/emqx_lua_hook_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_lwm2m/include/emqx_lwm2m.hrl b/apps/emqx_lwm2m/include/emqx_lwm2m.hrl index 9a992c6c0..5462f489d 100644 --- a/apps/emqx_lwm2m/include/emqx_lwm2m.hrl +++ b/apps/emqx_lwm2m/include/emqx_lwm2m.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_app.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_app.erl index 62d3adbbd..14d042681 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_app.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_cmd_handler.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_cmd_handler.erl index cd22797fa..2d208bdb4 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_cmd_handler.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_cmd_handler.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_coap_resource.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_coap_resource.erl index 62c0e0904..de83643a8 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_coap_resource.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_coap_resource.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_coap_server.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_coap_server.erl index 12755dcd4..60d5f4b85 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_coap_server.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_coap_server.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_json.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_json.erl index 3da69c6c4..641cf7d97 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_json.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_json.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_message.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_message.erl index 908f94f83..2dc9fa08a 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_message.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_message.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl index d71f15e5e..bcc2267cd 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -36,7 +36,9 @@ ]). %% For Mgmt --export([call/2]). +-export([ call/2 + , call/3 + ]). -record(lwm2m_state, { peername , endpoint_name @@ -63,7 +65,10 @@ %%-------------------------------------------------------------------- call(Pid, Msg) -> - case catch gen_server:call(Pid, Msg) of + call(Pid, Msg, 5000). + +call(Pid, Msg, Timeout) -> + case catch gen_server:call(Pid, Msg, Timeout) of ok -> ok; {'EXIT', {{shutdown, kick},_}} -> ok; Error -> {error, Error} diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_sup.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_sup.erl index ffafbfdf2..d28e00874 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_sup.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_timer.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_timer.erl index 1aaaf5fdb..75ab2d42a 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_timer.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_timer.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_tlv.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_tlv.erl index 9288f89fb..8576595f8 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_tlv.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_tlv.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object.erl index cf4c51c49..5931ae75e 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object_db.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object_db.erl index 873a9f27b..b974c148c 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object_db.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object_db.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_lwm2m/test/emqx_lwm2m_SUITE.erl b/apps/emqx_lwm2m/test/emqx_lwm2m_SUITE.erl index a77a44dfe..2cfb745bf 100644 --- a/apps/emqx_lwm2m/test/emqx_lwm2m_SUITE.erl +++ b/apps/emqx_lwm2m/test/emqx_lwm2m_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_lwm2m/test/emqx_tlv_SUITE.erl b/apps/emqx_lwm2m/test/emqx_tlv_SUITE.erl index aee6f14ad..bfb85d832 100644 --- a/apps/emqx_lwm2m/test/emqx_tlv_SUITE.erl +++ b/apps/emqx_lwm2m/test/emqx_tlv_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_lwm2m/test/test_mqtt_broker.erl b/apps/emqx_lwm2m/test/test_mqtt_broker.erl index eab2ec5e5..dd85340b6 100644 --- a/apps/emqx_lwm2m/test/test_mqtt_broker.erl +++ b/apps/emqx_lwm2m/test/test_mqtt_broker.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/include/emqx_mgmt.hrl b/apps/emqx_management/include/emqx_mgmt.hrl index e3e1f9fce..6d510ed0c 100644 --- a/apps/emqx_management/include/emqx_mgmt.hrl +++ b/apps/emqx_management/include/emqx_mgmt.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -32,4 +32,4 @@ -define(ERROR14, 114). %% OldPassword error -define(ERROR15, 115). %% bad topic --define(VERSIONS, ["1", "3.2", "3.4", "4.0", "4.1", "4.2", "4.3"]). \ No newline at end of file +-define(VERSIONS, ["4.0", "4.1", "4.2", "4.3"]). \ No newline at end of file diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl index 19582907e..8c196f682 100644 --- a/apps/emqx_management/src/emqx_mgmt.erl +++ b/apps/emqx_management/src/emqx_mgmt.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -553,7 +553,7 @@ rpc_call(Node, Fun, Args) -> end. otp_rel() -> - lists:concat(["R", erlang:system_info(otp_release), "/", erlang:system_info(version)]). + lists:concat([emqx_vm:get_otp_version(), "/", erlang:system_info(version)]). check_row_limit(Tables) -> check_row_limit(Tables, max_row_limit()). diff --git a/apps/emqx_management/src/emqx_mgmt_api.erl b/apps/emqx_management/src/emqx_mgmt_api.erl index 38b9d4db0..5dedc91be 100644 --- a/apps/emqx_management/src/emqx_mgmt_api.erl +++ b/apps/emqx_management/src/emqx_mgmt_api.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/src/emqx_mgmt_api_alarms.erl b/apps/emqx_management/src/emqx_mgmt_api_alarms.erl index a274b899c..d8a0f25dc 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_alarms.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_alarms.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/src/emqx_mgmt_api_apps.erl b/apps/emqx_management/src/emqx_mgmt_api_apps.erl index e9a4e0997..cca0b41f0 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_apps.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_apps.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/src/emqx_mgmt_api_banned.erl b/apps/emqx_management/src/emqx_mgmt_api_banned.erl index 4d5856fd0..b92875d9e 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_banned.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_banned.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/src/emqx_mgmt_api_brokers.erl b/apps/emqx_management/src/emqx_mgmt_api_brokers.erl index 89707f49c..bd901a3fe 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_brokers.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_brokers.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl index 23a70f6eb..90f0d3466 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/src/emqx_mgmt_api_data.erl b/apps/emqx_management/src/emqx_mgmt_api_data.erl index 855e09525..83b8d0e28 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_data.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_data.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -112,8 +112,10 @@ import(_Bindings, Params) -> undefined -> minirest:return({error, missing_required_params}); Filename -> - Result = case proplists:get_value(<<"node">>, Params) of - undefined -> do_import(Filename); + case proplists:get_value(<<"node">>, Params) of + undefined -> + Result = do_import(Filename), + minirest:return(Result); Node -> case lists:member(Node, [ erlang:atom_to_binary(N, utf8) || N <- ekka_mnesia:running_nodes() ] @@ -121,8 +123,7 @@ import(_Bindings, Params) -> true -> minirest:return(rpc:call(erlang:binary_to_atom(Node, utf8), ?MODULE, do_import, [Filename])); false -> minirest:return({error, no_existent_node}) end - end, - minirest:return(Result) + end end. do_import(Filename) -> diff --git a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl index 5425d82b4..7cccbd2ac 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/src/emqx_mgmt_api_metrics.erl b/apps/emqx_management/src/emqx_mgmt_api_metrics.erl index c265fd20e..b59aa0ac5 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_metrics.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_metrics.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/src/emqx_mgmt_api_nodes.erl b/apps/emqx_management/src/emqx_mgmt_api_nodes.erl index 89325193f..2be151aaa 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_nodes.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_nodes.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl index c50a806ea..e4908aeff 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/src/emqx_mgmt_api_pubsub.erl b/apps/emqx_management/src/emqx_mgmt_api_pubsub.erl index 693f71a3e..e5a3e9d77 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_pubsub.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_pubsub.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/src/emqx_mgmt_api_routes.erl b/apps/emqx_management/src/emqx_mgmt_api_routes.erl index ed173436f..380c9f0f6 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_routes.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_routes.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/src/emqx_mgmt_api_stats.erl b/apps/emqx_management/src/emqx_mgmt_api_stats.erl index 97d80410b..95d54b775 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_stats.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_stats.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl index f61ebfd97..4165ca51a 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/src/emqx_mgmt_app.erl b/apps/emqx_management/src/emqx_mgmt_app.erl index e2ec6385c..33d5b2d0e 100644 --- a/apps/emqx_management/src/emqx_mgmt_app.erl +++ b/apps/emqx_management/src/emqx_mgmt_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/src/emqx_mgmt_auth.erl b/apps/emqx_management/src/emqx_mgmt_auth.erl index 3c3576578..c05cbf581 100644 --- a/apps/emqx_management/src/emqx_mgmt_auth.erl +++ b/apps/emqx_management/src/emqx_mgmt_auth.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/src/emqx_mgmt_cli.erl b/apps/emqx_management/src/emqx_mgmt_cli.erl index a7c78d4b8..252ac3857 100644 --- a/apps/emqx_management/src/emqx_mgmt_cli.erl +++ b/apps/emqx_management/src/emqx_mgmt_cli.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/src/emqx_mgmt_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_data_backup.erl index c7beb8a07..75b929d08 100644 --- a/apps/emqx_management/src/emqx_mgmt_data_backup.erl +++ b/apps/emqx_management/src/emqx_mgmt_data_backup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -44,7 +44,7 @@ , import_blacklist/1 , import_applications/1 , import_users/1 - , import_auth_clientid/2 %% BACKW: 4.1.x + , import_auth_clientid/1 %% BACKW: 4.1.x , import_auth_username/1 %% BACKW: 4.1.x , import_auth_mnesia/2 , import_acl_mnesia/2 @@ -234,13 +234,6 @@ import_resource(#{<<"id">> := Id, config => Config, created_at => NCreatedAt, description => Desc}). - --ifdef(EMQX_ENTERPRISE). -import_resources_and_rules(Resources, Rules, _FromVersion) -> - import_resources(Resources), - import_rules(Rules). --else. - import_resources_and_rules(Resources, Rules, FromVersion) when FromVersion =:= "4.0" orelse FromVersion =:= "4.1" orelse @@ -329,6 +322,8 @@ apply_new_config(Actions, Configs) -> apply_new_config([], _Configs, Acc) -> Acc; +apply_new_config(Actions, [], []) -> + Actions; apply_new_config([Action = #{<<"name">> := <<"data_to_webserver">>, <<"args">> := #{<<"$resource">> := ID, <<"path">> := Path, @@ -356,7 +351,6 @@ apply_new_config([Action = #{<<"args">> := #{<<"$resource">> := ResourceId, <<"forward_topic">> => ForwardTopic}, apply_new_config(More, Configs, [Action#{<<"args">> := Args} | Acc]). --endif. actions_to_prop_list(Actions) -> [action_to_prop_list(Act) || Act <- Actions]. @@ -405,18 +399,14 @@ import_users(Users) -> emqx_dashboard_admin:force_add_user(Username, NPassword, Tags) end, Users). -import_auth_clientid(Lists, Version) -> +import_auth_clientid(Lists) -> case ets:info(emqx_user) of undefined -> ok; _ -> - lists:foreach(fun(#{<<"clientid">> := Clientid, <<"password">> := Password0}) -> - Password = case Version of - "4.1" -> base64:decode(Password0); - _ -> ensure_binary(Password0) - end, + lists:foreach(fun(#{<<"clientid">> := Clientid, <<"password">> := Password}) -> mnesia:dirty_write({emqx_user, {clientid, Clientid} - , Password - , erlang:system_time(millisecond)}) + , base64:decode(Password) + , erlang:system_time(millisecond)}) end, Lists) end. @@ -443,33 +433,16 @@ import_acl_mnesia(Acls, FromVersion) when FromVersion =:= "4.0" orelse import_acl_mnesia(Acls, _) -> do_import_acl_mnesia(Acls). -else. -import_auth_mnesia(Auths, FromVersion) when FromVersion =:= "4.0" orelse - FromVersion =:= "4.1" -> - do_import_auth_mnesia_by_old_data(Auths); -import_auth_mnesia(Auths, "4.2") -> - %% 4.2 contains a bug where password is not base64-encoded - do_import_auth_mnesia_4_2(Auths); -import_auth_mnesia(Auths, _) -> - do_import_auth_mnesia(Auths). +import_auth_mnesia(Auths, FromVersion) when FromVersion =:= "4.3" -> + do_import_auth_mnesia(Auths); +import_auth_mnesia(Auths, _FromVersion) -> + do_import_auth_mnesia_by_old_data(Auths). -import_acl_mnesia(Acls, FromVersion) when FromVersion =:= "4.0" orelse - FromVersion =:= "4.1" orelse - FromVersion =:= "4.2" -> - do_import_acl_mnesia_by_old_data(Acls); +import_acl_mnesia(Acls, FromVersion) when FromVersion =:= "4.3" -> + do_import_acl_mnesia(Acls); +import_acl_mnesia(Acls, _FromVersion) -> + do_import_acl_mnesia_by_old_data(Acls). -import_acl_mnesia(Acls, _) -> - do_import_acl_mnesia(Acls). - -do_import_auth_mnesia_4_2(Auths) -> - case ets:info(emqx_user) of - undefined -> ok; - _ -> - CreatedAt = erlang:system_time(millisecond), - lists:foreach(fun(#{<<"login">> := Login, - <<"password">> := Password}) -> - mnesia:dirty_write({emqx_user, {get_old_type(), Login}, Password, CreatedAt}) - end, Auths) - end. -endif. do_import_auth_mnesia_by_old_data(Auths) -> @@ -617,6 +590,7 @@ do_export_extra_data() -> do_export_extra_data() -> []. -endif. +-ifdef(EMQX_ENTERPRISE). import(Filename, OverridesJson) -> case file:read_file(Filename) of {ok, Json} -> @@ -624,8 +598,27 @@ import(Filename, OverridesJson) -> Overrides = emqx_json:decode(OverridesJson, [return_maps]), Data = maps:merge(Imported, Overrides), Version = to_version(maps:get(<<"version">>, Data)), - read_global_auth_type(Data, Version), - case lists:member(Version, ?VERSIONS) of + read_global_auth_type(Data), + try + do_import_data(Data, Version), + logger:debug("The emqx data has been imported successfully"), + ok + catch Class:Reason:Stack -> + logger:error("The emqx data import failed: ~0p", [{Class, Reason, Stack}]), + {error, import_failed} + end; + Error -> Error + end. +-else. +import(Filename, OverridesJson) -> + case file:read_file(Filename) of + {ok, Json} -> + Imported = emqx_json:decode(Json, [return_maps]), + Overrides = emqx_json:decode(OverridesJson, [return_maps]), + Data = maps:merge(Imported, Overrides), + Version = to_version(maps:get(<<"version">>, Data)), + read_global_auth_type(Data), + case is_version_supported(Data, Version) of true -> try do_import_data(Data, Version), @@ -637,10 +630,11 @@ import(Filename, OverridesJson) -> end; false -> logger:error("Unsupported version: ~p", [Version]), - {error, unsupported_version} + {error, unsupported_version, Version} end; Error -> Error end. +-endif. do_import_data(Data, Version) -> do_import_extra_data(Data, Version), @@ -648,7 +642,7 @@ do_import_data(Data, Version) -> import_blacklist(maps:get(<<"blacklist">>, Data, [])), import_applications(maps:get(<<"apps">>, Data, [])), import_users(maps:get(<<"users">>, Data, [])), - import_auth_clientid(maps:get(<<"auth_clientid">>, Data, []), Version), + import_auth_clientid(maps:get(<<"auth_clientid">>, Data, [])), import_auth_username(maps:get(<<"auth_username">>, Data, [])), import_auth_mnesia(maps:get(<<"auth_mnesia">>, Data, []), Version), import_acl_mnesia(maps:get(<<"acl_mnesia">>, Data, []), Version). @@ -663,19 +657,63 @@ do_import_extra_data(Data, _Version) -> do_import_extra_data(_Data, _Version) -> ok. -endif. --ifndef(EMQX_ENTERPRISE). covert_empty_headers([]) -> #{}; covert_empty_headers(Other) -> Other. flag_to_boolean(<<"on">>) -> true; flag_to_boolean(<<"off">>) -> false; flag_to_boolean(Other) -> Other. + +-ifndef(EMQX_ENTERPRISE). +is_version_supported(Data, Version) -> + case { maps:get(<<"auth_clientid">>, Data, []) + , maps:get(<<"auth_username">>, Data, []) + , maps:get(<<"auth_mnesia">>, Data, [])} of + {[], [], []} -> lists:member(Version, ?VERSIONS); + _ -> is_version_supported2(Version) + end. + +is_version_supported2("4.1") -> + true; +is_version_supported2("4.3") -> + true; +is_version_supported2(Version) -> + case re:run(Version, "^4.[02].\\d+$", [{capture, none}]) of + match -> + try lists:map(fun erlang:list_to_integer/1, string:tokens(Version, ".")) of + [4, 2, N] -> N >= 11; + [4, 0, N] -> N >= 13; + _ -> false + catch + _ : _ -> false + end; + nomatch -> + false + end. -endif. -read_global_auth_type(Data, Version) when Version =:= "4.0" orelse - Version =:= "4.1" orelse - Version =:= "4.2" -> - ct:print("|>=> :~p~n", [Data]), +read_global_auth_type(Data) -> + case {maps:get(<<"auth_mnesia">>, Data, []), maps:get(<<"acl_mnesia">>, Data, [])} of + {[], []} -> + %% Auth mnesia plugin is not used: + ok; + _ -> + do_read_global_auth_type(Data) + end. + +-ifdef(EMQX_ENTERPRISE). +do_read_global_auth_type(Data) -> + case Data of + #{<<"auth.mnesia.as">> := <<"username">>} -> + application:set_env(emqx_auth_mnesia, as, username); + #{<<"auth.mnesia.as">> := <<"clientid">>} -> + application:set_env(emqx_auth_mnesia, as, clientid); + _ -> + ok + end. + +-else. +do_read_global_auth_type(Data) -> case Data of #{<<"auth.mnesia.as">> := <<"username">>} -> application:set_env(emqx_auth_mnesia, as, username); @@ -691,15 +729,9 @@ read_global_auth_type(Data, Version) when Version =:= "4.0" orelse " $ emqx_ctl data import --env '{\"auth.mnesia.as\":\"clientid\"}'", []), error(import_failed) - end; -read_global_auth_type(_Data, _Version) -> - ok. + end. +-endif. get_old_type() -> {ok, Type} = application:get_env(emqx_auth_mnesia, as), Type. - -ensure_binary(A) when is_binary(A) -> - A; -ensure_binary(A) -> - list_to_binary(A). diff --git a/apps/emqx_management/src/emqx_mgmt_http.erl b/apps/emqx_management/src/emqx_mgmt_http.erl index 82b45b368..a037da37f 100644 --- a/apps/emqx_management/src/emqx_mgmt_http.erl +++ b/apps/emqx_management/src/emqx_mgmt_http.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/src/emqx_mgmt_sup.erl b/apps/emqx_management/src/emqx_mgmt_sup.erl index 8a8e9ad1c..f3f5545f2 100644 --- a/apps/emqx_management/src/emqx_mgmt_sup.erl +++ b/apps/emqx_management/src/emqx_mgmt_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/src/emqx_mgmt_util.erl b/apps/emqx_management/src/emqx_mgmt_util.erl index 7c6ce4ef9..132bbc83f 100644 --- a/apps/emqx_management/src/emqx_mgmt_util.erl +++ b/apps/emqx_management/src/emqx_mgmt_util.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -74,11 +74,6 @@ merge_maps(Default, New) -> end end, Default, New). --if(?OTP_RELEASE >= 23). urldecode(S) -> - [{R, _}] = uri_string:dissect_query(S), R. --else. -urldecode(S) -> - http_uri:decode(S). --endif. + emqx_http_lib:uri_decode(S). diff --git a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl index ad98150b3..838529f03 100644 --- a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl +++ b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl @@ -25,27 +25,12 @@ -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx_auth_mnesia/include/emqx_auth_mnesia.hrl"). --ifdef(EMQX_ENTERPRISE). - -matrix() -> - [ {username, "e4.2.9"} - , {clientid, "e4.1.1"} - , {username, "e4.1.1"} - ]. - -all() -> - [t_matrix]. - --else. %% ! EMQX_ENTERPRISE - matrix() -> [{ImportAs, Version} || ImportAs <- [clientid, username] - , Version <- ["v4.2.9", "v4.1.5"]]. + , Version <- ["v4.2.10", "v4.1.5"]]. all() -> - [t_matrix, t_import_4_0]. - --endif. %% EMQX_ENTERPRISE + [t_import_4_0, t_import_4_1, t_import_4_2]. groups() -> [{username, [], cases()}, {clientid, [], cases()}]. @@ -60,45 +45,99 @@ init_per_suite(Config) -> Config. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([emqx_modules, emqx_management, emqx_dashboard, emqx_management, emqx_auth_mnesia]), + emqx_ct_helpers:stop_apps([emqx_modules, emqx_management, emqx_dashboard, emqx_auth_mnesia]), ekka_mnesia:ensure_stopped(). init_per_testcase(_, Config) -> Config. end_per_testcase(_, _Config) -> - mnesia:clear_table(emqx_acl), - mnesia:clear_table(emqx_user), + {atomic,ok} = mnesia:clear_table(emqx_acl), + {atomic,ok} = mnesia:clear_table(emqx_user), ok. - -t_matrix(Config) -> - [begin - ct:pal("Testing import of ~p from ~p", [ImportAs, FromVersion]), - do_import(Config, ImportAs, FromVersion), - test_clientid_import(), - ct:pal("ok") - end - || {ImportAs, FromVersion} <- matrix()]. - -%% This version is special, since it doesn't have mnesia ACL plugin +-ifdef(EMQX_ENTERPRISE). t_import_4_0(Config) -> - mnesia:clear_table(emqx_acl), - mnesia:clear_table(emqx_user), - Filename = filename:join(proplists:get_value(data_dir, Config), "v4.0.7.json"), Overrides = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(clientid)}), - ?assertMatch(ok, emqx_mgmt_data_backup:import(Filename, Overrides)), + ?assertMatch(ok, do_import("e4.0.10.json", Config, Overrides)), timer:sleep(100), - test_clientid_import(). + ct:pal("---~p~n", [ets:tab2list(emqx_user)]), + test_import(username, {<<"emqx_username">>, <<"public">>}), + test_import(clientid, {<<"emqx_c">>, <<"public">>}), + + Overrides1 = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(username)}), + ?assertMatch(ok, do_import("e4.0.10.json", Config, Overrides1)), + timer:sleep(100), + test_import(username, {<<"emqx_c">>, <<"public">>}), + test_import(username, {<<"emqx_username">>, <<"public">>}). +t_import_4_1(Config) -> + Overrides = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(clientid)}), + ?assertMatch(ok, do_import("e4.1.1.json", Config, Overrides)), + timer:sleep(100), + test_import(clientid, {<<"emqx_c">>, <<"public">>}), + test_import(clientid, {<<"emqx_c">>, <<"public">>}), + + Overrides1 = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(username)}), + ?assertMatch(ok, do_import("e4.1.1.json", Config, Overrides1)), + timer:sleep(100), + test_import(username, {<<"emqx_c">>, <<"public">>}), + test_import(clientid, {<<"emqx_clientid">>, <<"public">>}). + +t_import_4_2(Config) -> + ?assertMatch(ok, do_import("e4.2.9.json", Config, "{}")), + timer:sleep(100), + test_import(username, {<<"emqx_c">>, <<"public">>}), + test_import(clientid, {<<"emqx_clientid">>, <<"public">>}). + +-else. +t_import_4_0(Config) -> + ?assertMatch(ok, do_import("v4.0.11-no-auth.json", Config)), + timer:sleep(100), + ?assertMatch(0, ets:info(emqx_user, size)), + + ?assertMatch({error, unsupported_version, "4.0"}, do_import("v4.0.11.json", Config)), + + ?assertMatch(ok, do_import("v4.0.13.json", Config)), + timer:sleep(100), + test_import(clientid, {<<"client_for_test">>, <<"public">>}), + test_import(username, {<<"user_for_test">>, <<"public">>}). + +t_import_4_1(Config) -> + Overrides = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(clientid)}), + ?assertMatch(ok, do_import("v4.1.5.json", Config, Overrides)), + timer:sleep(100), + test_import(clientid, {<<"user_mnesia">>, <<"public">>}), + test_import(clientid, {<<"client_for_test">>, <<"public">>}), + test_import(username, {<<"user_for_test">>, <<"public">>}), + + Overrides1 = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(username)}), + ?assertMatch(ok, do_import("v4.1.5.json", Config, Overrides1)), + timer:sleep(100), + test_import(username, {<<"user_mnesia">>, <<"public">>}), + test_import(clientid, {<<"client_for_test">>, <<"public">>}), + test_import(username, {<<"user_for_test">>, <<"public">>}). + +t_import_4_2(Config) -> + ?assertMatch(ok, do_import("v4.2.10-no-auth.json", Config)), + timer:sleep(100), + ?assertMatch(0, ets:info(emqx_user, size)), + + Overrides = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(clientid)}), + ?assertMatch({error, unsupported_version, "4.2"}, do_import("v4.2.10.json", Config, Overrides)), + + Overrides1 = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(clientid)}), + ?assertMatch(ok, do_import("v4.2.11.json", Config, Overrides1)), + timer:sleep(100), + test_import(clientid, {<<"user_mnesia">>, <<"public">>}), + test_import(clientid, {<<"client_for_test">>, <<"public">>}), + test_import(username, {<<"user_for_test">>, <<"public">>}), + + Overrides2 = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(username)}), + ?assertMatch(ok, do_import("v4.2.11.json", Config, Overrides2)), + timer:sleep(100), + test_import(username, {<<"user_mnesia">>, <<"public">>}), + test_import(clientid, {<<"client_for_test">>, <<"public">>}), + test_import(username, {<<"user_for_test">>, <<"public">>}), -do_import(Config, Type, V) -> - File = V ++ ".json", - mnesia:clear_table(emqx_acl), - mnesia:clear_table(emqx_user), - Filename = filename:join(proplists:get_value(data_dir, Config), File), - Overrides = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(Type)}), - ?assertMatch(ok, emqx_mgmt_data_backup:import(Filename, Overrides)), - Records = lists:sort(ets:tab2list(emqx_acl)), - %% Check importing of records related to emqx_auth_mnesia ?assertMatch([#emqx_acl{ filter = {{Type,<<"emqx_c">>}, <<"Topic/A">>}, action = pub, @@ -109,21 +148,28 @@ do_import(Config, Type, V) -> action = sub, access = allow }], - lists:sort(Records)), - ?assertMatch([_, _], ets:tab2list(emqx_user)), - ?assertMatch([_], ets:lookup(emqx_user, {Type, <<"emqx_c">>})), - Req = #{clientid => <<"blah">>} - #{Type => <<"emqx_c">>, - password => <<"emqx_p">> - }, - ?assertMatch({stop, #{auth_result := success}}, - emqx_auth_mnesia:check(Req, #{}, #{hash_type => sha256})). + lists:sort(ets:tab2list(emqx_acl))). +-endif. -test_clientid_import() -> - [#emqx_user{password = _Pass}] = ets:lookup(emqx_user, {clientid, <<"emqx_clientid">>}), - %% Req = #{clientid => <<"emqx_clientid">>, - %% password => <<"emqx_p">> - %% }, - %% ?assertMatch({stop, #{auth_result := success}}, - %% emqx_auth_mnesia:check(Req, #{}, #{hash_type => sha256})), - ok. +do_import(File, Config) -> + do_import(File, Config, "{}"). + +do_import(File, Config, Overrides) -> + mnesia:clear_table(emqx_acl), + mnesia:clear_table(emqx_user), + Filename = filename:join(proplists:get_value(data_dir, Config), File), + emqx_mgmt_data_backup:import(Filename, Overrides). + +test_import(username, {Username, Password}) -> + [#emqx_user{password = _}] = ets:lookup(emqx_user, {username, Username}), + Req = #{clientid => <<"anyname">>, + username => Username, + password => Password}, + ?assertMatch({stop, #{auth_result := success}}, + emqx_auth_mnesia:check(Req, #{}, #{hash_type => sha256})); +test_import(clientid, {ClientID, Password}) -> + [#emqx_user{password = _}] = ets:lookup(emqx_user, {clientid, ClientID}), + Req = #{clientid => ClientID, + password => Password}, + ?assertMatch({stop, #{auth_result := success}}, + emqx_auth_mnesia:check(Req, #{}, #{hash_type => sha256})). \ No newline at end of file diff --git a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/make_data.sh b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/make_data.sh index ab1743cb1..e31729784 100755 --- a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/make_data.sh +++ b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/make_data.sh @@ -56,17 +56,28 @@ export_data() { cat "${filename}.json" } + +collect_4_2_no_mnesia_auth () { + container "4.2.10" + + # Add clientid + docker exec emqx emqx_ctl clientid add emqx_clientid emqx_p + + export_data "v4.2.10-no-auth" +} + collect_4_2 () { - container "4.2.9" + container "4.2.10" create_acls "api/v4/mqtt_acl" create_user mqtt_user # Add clientid docker exec emqx emqx_ctl clientid add emqx_clientid emqx_p - export_data "v4.2.9" + export_data "v4.2.10" } + collect_e4_2 () { container "4.2.5" "ee" # Add ACLs: @@ -77,7 +88,7 @@ collect_e4_2 () { # Add clientid docker exec emqx emqx_ctl clientid add emqx_clientid emqx_p - export_data "e4.2.9" + export_data "e4.2.5" } collect_e4_1 () { @@ -105,17 +116,18 @@ collect_4_1 () { } collect_4_0 () { - container "v4.0.7" + container "v4.0.11" # Add clientid docker exec emqx emqx_ctl clientid add emqx_clientid emqx_p - export_data "v4.0.7" + export_data "v4.0.11" } collect_4_0 collect_4_1 collect_4_2 +collect_4_2_no_mnesia_auth collect_e4_2 collect_e4_1 diff --git a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.11-no-auth.json b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.11-no-auth.json new file mode 100644 index 000000000..8dad197dd --- /dev/null +++ b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.11-no-auth.json @@ -0,0 +1,23 @@ +{ + "version": "4.0", + "users": [], + "schemas": [], + "rules": [], + "resources": [], + "date": "2021-04-10 11:45:26", + "blacklist": [], + "auth_username": [], + "auth_mnesia": [], + "auth_clientid": [], + "apps": [ + { + "status": true, + "secret": "public", + "name": "Default", + "id": "admin", + "expired": "undefined", + "desc": "Application user" + } + ], + "acl_mnesia": [] +} diff --git a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.7.json b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.11.json similarity index 74% rename from apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.7.json rename to apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.11.json index 6de74dc48..a701d0944 100644 --- a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.7.json +++ b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.11.json @@ -4,13 +4,13 @@ "schemas": [], "rules": [], "resources": [], - "date": "2021-04-07 14:28:49", + "date": "2021-04-10 11:45:26", "blacklist": [], "auth_username": [], "auth_mnesia": [], "auth_clientid": [ { - "password": "��Pd56c0fcdcd7636dcf8ed1ea48cd3d58acab74030157551f7f7f8684804b9239e", + "password": "9Sv2tzJlNDlmNWZhYWQ5Yzc4MWUwNmFhZWI4NjFlMDM2OWEzYmE1OTkxOTBhOGQ4N2Y3MzExY2ZiZmIxNTFkMTdkZmY=", "clientid": "emqx_clientid" } ], diff --git a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.13.json b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.13.json new file mode 100644 index 000000000..9c602a0e3 --- /dev/null +++ b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.13.json @@ -0,0 +1,37 @@ +{ + "version":"4.0.13", + "users":[ + { + "username":"admin", + "tags":"administrator", + "password":"p6C65OF0BQhvPmCziM2yRa8JN5o=" + } + ], + "schemas":[], + "rules":[], + "resources":[], + "date":"2021-04-16 11:20:00", + "blacklist":[], + "auth_username":[ + { + "username":"user_for_test", + "password":"ARLrzTRhZTI1MzgxNjdjMDU5ODFhZDU3ZTdmNzJiOWM5MWUwMTFkNDk4OGUyZWUyYmU0ZTE2ZTg2OWNhMGQyYWQ5ZmU=" + } + ], + "auth_clientid":[ + { + "password":"JBgSnzIxOWNiMDU1ZWFiNDAwMjVhOTQzZThlZjkxN2JlZWE4MGE4YzlmM2I5MjQ4OGI1NjllY2Q4NGQ4NjhjYzQ1NDM=", + "clientid":"client_for_test" + } + ], + "apps":[ + { + "status":true, + "secret":"public", + "name":"Default", + "id":"admin", + "expired":"undefined", + "desc":"Application user" + } + ] +} \ No newline at end of file diff --git a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.1.5.json b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.1.5.json index 324d8188a..3d3b9aa7e 100644 --- a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.1.5.json +++ b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.1.5.json @@ -12,18 +12,23 @@ "resources": [], "date": "2021-04-07 14:28:58", "blacklist": [], - "auth_username": [], + "auth_username": [ + { + "username":"user_for_test", + "password": "ARLrzTRhZTI1MzgxNjdjMDU5ODFhZDU3ZTdmNzJiOWM5MWUwMTFkNDk4OGUyZWUyYmU0ZTE2ZTg2OWNhMGQyYWQ5ZmU=" + } + ], "auth_mnesia": [ { - "password": "Y2ViNWU5MTdmNzkzMGFlOGYwZGMzY2ViNDk2YTQyOGY3ZTY0NDczNmVlYmNhMzZhMmI4ZjZiYmFjNzU2MTcxYQ==", - "login": "emqx_c", + "password": "ARLrzTRhZTI1MzgxNjdjMDU5ODFhZDU3ZTdmNzJiOWM5MWUwMTFkNDk4OGUyZWUyYmU0ZTE2ZTg2OWNhMGQyYWQ5ZmU=", + "login": "user_mnesia", "is_superuser": true } ], "auth_clientid": [ { - "password": "MctXdjZkYzRhMDUwMTc4MDM0OWY4YTg1NTg4Y2ZlOThjYWIyMDk3M2UzNjgzYzYyZWYwOTAzMTk2N2E4OWVjZDk4Mjk=", - "clientid": "emqx_clientid" + "password": "ARLrzTRhZTI1MzgxNjdjMDU5ODFhZDU3ZTdmNzJiOWM5MWUwMTFkNDk4OGUyZWUyYmU0ZTE2ZTg2OWNhMGQyYWQ5ZmU=", + "clientid": "client_for_test" } ], "apps": [ diff --git a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.10-no-auth.json b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.10-no-auth.json new file mode 100644 index 000000000..446a46f07 --- /dev/null +++ b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.10-no-auth.json @@ -0,0 +1,29 @@ +{ + "version": "4.2", + "date": "2021-04-12 10:41:10", + "rules": [], + "resources": [], + "blacklist": [], + "apps": [ + { + "id": "admin", + "secret": "public", + "name": "Default", + "desc": "Application user", + "status": true, + "expired": "undefined" + } + ], + "users": [ + { + "username": "admin", + "password": "e5M8oWEwQVqjdqceQIthC+3cPoY=", + "tags": "administrator" + } + ], + "auth_clientid": [], + "auth_username": [], + "auth_mnesia": [], + "acl_mnesia": [], + "schemas": [] +} diff --git a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.9.json b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.10.json similarity index 81% rename from apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.9.json rename to apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.10.json index f1ae8c9f0..1ccc6ce9d 100644 --- a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.9.json +++ b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.10.json @@ -1,6 +1,6 @@ { "version": "4.2", - "date": "2021-04-07 14:29:08", + "date": "2021-04-12 10:40:58", "rules": [], "resources": [], "blacklist": [], @@ -17,14 +17,14 @@ "users": [ { "username": "admin", - "password": "gCBXISkivpaeKetFcPSm+Eaxyxs=", + "password": "8Vd7+gVg2J3nE1Xjyxqd59sA5mo=", "tags": "administrator" } ], "auth_clientid": [ { "clientid": "emqx_clientid", - "password": "z�7d413fee461607065c161072f3707dc0a01bd1fc8476eb7be703a74a66701bb14" + "password": "UNb0e2RhNDc3NWIyNjg5Yjg4ZDExOTVhNWFkY2MzNGFmNzY2OTNmNmRlYzE4Y2ZiZjRjNzIyMWZlZTljZmEyZDE5Yzc=" } ], "auth_username": [], diff --git a/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.11.json b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.11.json new file mode 100644 index 000000000..52a91ac68 --- /dev/null +++ b/apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.11.json @@ -0,0 +1,58 @@ +{ + "version": "4.2.11", + "date": "2021-04-12 10:40:58", + "rules": [], + "resources": [], + "blacklist": [], + "apps": [ + { + "id": "admin", + "secret": "public", + "name": "Default", + "desc": "Application user", + "status": true, + "expired": "undefined" + } + ], + "users": [ + { + "username": "test", + "password": "8Vd7+gVg2J3nE1Xjyxqd59sA5mo=", + "tags": "administrator" + } + ], + "auth_clientid": [ + { + "clientid": "client_for_test", + "password": "ARLrzTRhZTI1MzgxNjdjMDU5ODFhZDU3ZTdmNzJiOWM5MWUwMTFkNDk4OGUyZWUyYmU0ZTE2ZTg2OWNhMGQyYWQ5ZmU=" + } + ], + "auth_username": [ + { + "username": "user_for_test", + "password": "ARLrzTRhZTI1MzgxNjdjMDU5ODFhZDU3ZTdmNzJiOWM5MWUwMTFkNDk4OGUyZWUyYmU0ZTE2ZTg2OWNhMGQyYWQ5ZmU=" + } + ], + "auth_mnesia": [ + { + "login": "user_mnesia", + "password": "ARLrzTRhZTI1MzgxNjdjMDU5ODFhZDU3ZTdmNzJiOWM5MWUwMTFkNDk4OGUyZWUyYmU0ZTE2ZTg2OWNhMGQyYWQ5ZmU=", + "is_superuser": true + } + ], + "acl_mnesia": [ + { + "login": "emqx_c", + "topic": "Topic/A", + "action": "sub", + "allow": true + }, + { + "login": "emqx_c", + "topic": "Topic/A", + "action": "pub", + "allow": true + } + ], + "schemas": [] +} diff --git a/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl b/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl index ce4c1807e..2609e1006 100644 --- a/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl +++ b/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -18,6 +18,9 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("emqx_rule_engine/include/rule_engine.hrl"). -compile([export_all, nowarn_export_all]). + +% -define(EMQX_ENTERPRISE, true). + %%-------------------------------------------------------------------- %% Setups %%-------------------------------------------------------------------- @@ -47,59 +50,27 @@ end_per_suite(Cfg) -> get_data_path() -> emqx_ct_helpers:deps_path(emqx_management, "test/emqx_bridge_mqtt_data_export_import_SUITE_data/"). + +import(FilePath, Version) -> + ok = emqx_mgmt_data_backup:import(get_data_path() ++ "/" ++ FilePath, <<"{}">>), + lists:foreach(fun(#resource{id = Id, config = Config} = _Resource) -> + timer:sleep(2000), + case Id of + <<"bridge">> -> + test_utils:resource_is_alive(Id), + handle_config(Config, Version, bridge); + <<"rpc">> -> + test_utils:resource_is_alive(Id), + handle_config(Config, Version, rpc); + _ -> ok + end + end, emqx_rule_registry:get_resources()). + %%-------------------------------------------------------------------- %% Cases %%-------------------------------------------------------------------- -handle_config(Config, 420, brigde) -> - ?assertEqual(<<"off">>, maps:get(<<"ssl">>, Config)); - -handle_config(Config, 430, brigde) -> - ?assertEqual(false, maps:get(<<"ssl">>, Config)); - -handle_config(Config, 420, rpc) -> - handle_config(Config, 430, rpc); - -handle_config(Config, 409, rpc) -> - handle_config(Config, 420, rpc); - -handle_config(Config, 415, rpc) -> - handle_config(Config, 420, rpc); - -handle_config(Config, 409, brigde) -> - handle_config(Config, 420, brigde); - -handle_config(Config, 415, brigde) -> - handle_config(Config, 420, brigde); - -handle_config(Config, 430, rpc) -> - ?assertEqual(<<"emqx@127.0.0.1">>, maps:get(<<"address">>, Config)), - ?assertEqual(32, maps:get(<<"batch_size">>, Config)), - ?assertEqual(<<"off">>, maps:get(<<"disk_cache">>, Config)), - ?assertEqual(<<"bridge/emqx/${node}/">>, maps:get(<<"mountpoint">>, Config)), - ?assertEqual(<<"30s">>, maps:get(<<"reconnect_interval">>, Config)), - ?assertEqual(8, maps:get(<<"pool_size">>, Config)); - -handle_config(_, _, _) -> ok. - -remove_resource(Id) -> - emqx_rule_registry:remove_resource(Id), - emqx_rule_registry:remove_resource_params(Id). - -import(FilePath, Version) -> - Overrides = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(clientid)}), - ok = emqx_mgmt_data_backup:import(get_data_path() ++ "/" ++ FilePath, Overrides), - lists:foreach(fun(#resource{id = Id, config = Config} = _Resource) -> - case Id of - <<"brigde">> -> - handle_config(Config, Version, brigde), - remove_resource(Id); - <<"rpc">> -> - handle_config(Config, Version, rpc), - remove_resource(Id); - _ -> ok - end - end, emqx_rule_registry:get_resources()). +-ifndef(EMQX_ENTERPRISE). t_import420(_) -> import("420.json", 420), @@ -116,3 +87,92 @@ t_import409(_) -> t_import415(_) -> import("415.json", 415), {ok, _} = emqx_mgmt_data_backup:export(). + + +handle_config(Config, 420, bridge) -> + ?assertEqual(false, maps:get(<<"ssl">>, Config)); + +handle_config(Config, 430, bridge) -> + ?assertEqual(false, maps:get(<<"ssl">>, Config)); + +handle_config(Config, 420, rpc) -> + handle_config(Config, 430, rpc); + +handle_config(Config, 409, rpc) -> + handle_config(Config, 420, rpc); + +handle_config(Config, 415, rpc) -> + handle_config(Config, 420, rpc); + +handle_config(Config, 409, bridge) -> + handle_config(Config, 420, bridge); + +handle_config(Config, 415, bridge) -> + handle_config(Config, 420, bridge); + +handle_config(Config, 430, rpc) -> + ?assertEqual(<<"test@127.0.0.1">>, maps:get(<<"address">>, Config)), + ?assertEqual(32, maps:get(<<"batch_size">>, Config)), + ?assertEqual(<<"off">>, maps:get(<<"disk_cache">>, Config)), + ?assertEqual(<<"bridge/emqx/${node}/">>, maps:get(<<"mountpoint">>, Config)), + ?assertEqual(<<"30s">>, maps:get(<<"reconnect_interval">>, Config)), + ?assertEqual(8, maps:get(<<"pool_size">>, Config)); + +handle_config(_, _, _) -> ok. + +-endif. + +-ifdef(EMQX_ENTERPRISE). + +t_importee4010(_) -> + import("ee4010.json", ee4010), + {ok, _} = emqx_mgmt_data_backup:export(). + +t_importee410(_) -> + import("ee410.json", ee410), + {ok, _} = emqx_mgmt_data_backup:export(). + +t_importee411(_) -> + import("ee411.json", ee411), + {ok, _} = emqx_mgmt_data_backup:export(). + +t_importee420(_) -> + import("ee420.json", ee420), + {ok, _} = emqx_mgmt_data_backup:export(). + +t_importee425(_) -> + import("ee425.json", ee425), + {ok, _} = emqx_mgmt_data_backup:export(). + +t_importee430(_) -> + import("ee430.json", ee430), + {ok, _} = emqx_mgmt_data_backup:export(). + +%%-------------------------------------------------------------------- +%% handle_config +%%-------------------------------------------------------------------- + +handle_config(Config, ee4010, Id) -> + handle_config(Config, ee430, Id); + +handle_config(Config, ee410, Id) -> + handle_config(Config, ee430, Id); + +handle_config(Config, ee411, Id) -> + handle_config(Config, ee430, Id); + +handle_config(Config, ee420, Id) -> + handle_config(Config, ee430, Id); + +handle_config(Config, ee425, Id) -> + handle_config(Config, ee430, Id); + +handle_config(Config, ee430, bridge) -> + ?assertEqual(false, maps:get(<<"ssl">>, Config)); + +handle_config(Config, ee430, rpc) -> + ?assertEqual(<<"off">>, maps:get(<<"disk_cache">>, Config)); + +handle_config(Config, ee435, Id) -> + handle_config(Config, ee430, Id). +-endif. diff --git a/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE_data/409.json b/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE_data/409.json index bfd92db78..cb50a31af 100644 --- a/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE_data/409.json +++ b/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE_data/409.json @@ -45,7 +45,7 @@ "mountpoint": "bridge/emqx/${node}/", "disk_cache": "off", "batch_size": 32, - "address": "emqx@127.0.0.1" + "address": "test@127.0.0.1" } } ], diff --git a/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE_data/415.json b/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE_data/415.json index 6b7f5d20f..176ac1f71 100644 --- a/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE_data/415.json +++ b/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE_data/415.json @@ -21,7 +21,7 @@ "mountpoint": "bridge/emqx/${node}/", "disk_cache": "off", "batch_size": 32, - "address": "emqx@127.0.0.1" + "address": "test@127.0.0.1" } }, { diff --git a/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE_data/420.json b/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE_data/420.json index 2bd5d0748..9922e459f 100644 --- a/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE_data/420.json +++ b/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE_data/420.json @@ -7,7 +7,7 @@ "id": "rpc", "type": "bridge_rpc", "config": { - "address": "emqx@127.0.0.1", + "address": "test@127.0.0.1", "batch_size": 32, "disk_cache": "off", "mountpoint": "bridge/emqx/${node}/", @@ -31,7 +31,7 @@ "disk_cache": "off", "keepalive": "60s", "keyfile": "etc/certs/client-key.pem", - "mountpoint": "bridge/aws/${node}/", + "mountpoint": "bridge/emqx/${node}/", "password": "", "pool_size": 8, "proto_ver": "mqttv4", diff --git a/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE_data/430.json b/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE_data/430.json index 778776778..4f535bfb7 100644 --- a/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE_data/430.json +++ b/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE_data/430.json @@ -25,7 +25,7 @@ "filename": "etc/certs/client-key.pem", "file": "" }, - "mountpoint": "bridge/aws/${node}/", + "mountpoint": "bridge/emqx/${node}/", "password": "", "pool_size": 8, "proto_ver": "mqttv4", @@ -41,7 +41,7 @@ "id": "rpc", "type": "bridge_rpc", "config": { - "address": "emqx@127.0.0.1", + "address": "test@127.0.0.1", "batch_size": 32, "disk_cache": "off", "mountpoint": "bridge/emqx/${node}/", diff --git a/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE_data/ee4010.json b/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE_data/ee4010.json new file mode 100644 index 000000000..fedd35884 --- /dev/null +++ b/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE_data/ee4010.json @@ -0,0 +1,68 @@ +{ + "version": "4.0", + "users": [ + { + "username": "admin", + "tags": "administrator", + "password": "GCX5nvOMK0hbiMB4AUyc25wI8fU=" + } + ], + "schemas": [], + "rules": [], + "resources": [ + { + "type": "bridge_mqtt", + "id": "bridge", + "description": "bridge", + "created_at": null, + "config": { + "username": "user", + "ssl": "off", + "retry_interval": "20s", + "reconnect_interval": "30s", + "proto_ver": "mqttv4", + "password": "passwd", + "mountpoint": "bridge/emqx/${node}/", + "keyfile": "etc/certs/client-key.pem", + "keepalive": "60s", + "disk_cache": "off", + "clientid": "bridge_aws", + "ciphers": "ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384", + "certfile": "etc/certs/client-cert.pem", + "cacertfile": "etc/certs/cacert.pem", + "bridge_mode": true, + "address": "127.0.0.1:1883" + } + }, + { + "type": "bridge_rpc", + "id": "rpc", + "description": "rpc", + "created_at": null, + "config": { + "reconnect_interval": "30s", + "pool_size": 8, + "mountpoint": "bridge/emqx/${node}/", + "disk_cache": "off", + "batch_size": 32, + "address": "test@127.0.0.1" + } + } + ], + "date": "2021-04-13 13:57:23", + "blacklist": [], + "auth_username": [], + "auth_mnesia": [], + "auth_clientid": [], + "apps": [ + { + "status": true, + "secret": "public", + "name": "Default", + "id": "admin", + "expired": "undefined", + "desc": "Application user" + } + ], + "acl_mnesia": [] +} \ No newline at end of file diff --git a/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE_data/ee410.json b/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE_data/ee410.json new file mode 100644 index 000000000..8a6f7129b --- /dev/null +++ b/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE_data/ee410.json @@ -0,0 +1,70 @@ +{ + "version": "4.1", + "users": [ + { + "username": "admin", + "tags": "administrator", + "password": "atdwlByxL9/9P3CoFJ60drhodkY=" + } + ], + "schemas": [], + "rules": [], + "resources": [ + { + "type": "bridge_rpc", + "id": "rpc", + "description": "rpc", + "created_at": null, + "config": { + "reconnect_interval": "30s", + "pool_size": 8, + "mountpoint": "bridge/emqx/${node}/", + "disk_cache": "off", + "batch_size": 32, + "address": "test@127.0.0.1" + } + }, + { + "type": "bridge_mqtt", + "id": "bridge", + "description": "bridge", + "created_at": null, + "config": { + "username": "", + "ssl": "off", + "retry_interval": "20s", + "reconnect_interval": "30s", + "proto_ver": "mqttv4", + "pool_size": 8, + "password": "", + "mountpoint": "bridge/emqx/${node}/", + "keyfile": "etc/certs/client-key.pem", + "keepalive": "60s", + "disk_cache": "off", + "clientid": "client", + "ciphers": "ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA", + "certfile": "etc/certs/client-cert.pem", + "cacertfile": "etc/certs/cacert.pem", + "bridge_mode": false, + "append": true, + "address": "127.0.0.1:1883" + } + } + ], + "date": "2021-04-13 11:30:21", + "blacklist": [], + "auth_username": [], + "auth_mnesia": [], + "auth_clientid": [], + "apps": [ + { + "status": true, + "secret": "public", + "name": "Default", + "id": "admin", + "expired": "undefined", + "desc": "Application user" + } + ], + "acl_mnesia": [] +} \ No newline at end of file diff --git a/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE_data/ee411.json b/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE_data/ee411.json new file mode 100644 index 000000000..6bbd3ac7b --- /dev/null +++ b/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE_data/ee411.json @@ -0,0 +1,70 @@ +{ + "version": "4.1", + "users": [ + { + "username": "admin", + "tags": "administrator", + "password": "tsYtP3TylchkM7J7YTc46Di0kPk=" + } + ], + "schemas": [], + "rules": [], + "resources": [ + { + "type": "bridge_rpc", + "id": "rpc", + "description": "rpc", + "created_at": null, + "config": { + "reconnect_interval": "30s", + "pool_size": 8, + "mountpoint": "bridge/emqx/${node}/", + "disk_cache": "off", + "batch_size": 32, + "address": "test@127.0.0.1" + } + }, + { + "type": "bridge_mqtt", + "id": "bridge", + "description": "bridge", + "created_at": null, + "config": { + "username": "", + "ssl": "off", + "retry_interval": "20s", + "reconnect_interval": "30s", + "proto_ver": "mqttv4", + "pool_size": 8, + "password": "", + "mountpoint": "bridge/emqx/${node}/", + "keyfile": "etc/certs/client-key.pem", + "keepalive": "60s", + "disk_cache": "off", + "clientid": "client", + "ciphers": "ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA", + "certfile": "etc/certs/client-cert.pem", + "cacertfile": "etc/certs/cacert.pem", + "bridge_mode": false, + "append": true, + "address": "127.0.0.1:1883" + } + } + ], + "date": "2021-04-13 16:37:18", + "blacklist": [], + "auth_username": [], + "auth_mnesia": [], + "auth_clientid": [], + "apps": [ + { + "status": true, + "secret": "public", + "name": "Default", + "id": "admin", + "expired": "undefined", + "desc": "Application user" + } + ], + "acl_mnesia": [] +} \ No newline at end of file diff --git a/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE_data/ee420.json b/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE_data/ee420.json new file mode 100644 index 000000000..3b56495d1 --- /dev/null +++ b/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE_data/ee420.json @@ -0,0 +1,119 @@ +{ + "version": "4.2", + "date": "2021-04-13 11:35:13", + "modules": [ + { + "id": "module:b9294d70", + "type": "recon", + "config": {}, + "enabled": true, + "created_at": "undefined", + "description": "" + }, + { + "id": "module:c7c7b692", + "type": "presence", + "config": { + "qos": 0 + }, + "enabled": true, + "created_at": "undefined", + "description": "" + }, + { + "id": "module:486adc4b", + "type": "internal_acl", + "config": { + "acl_rule_file": "etc/acl.conf" + }, + "enabled": true, + "created_at": "undefined", + "description": "" + }, + { + "id": "module:411cf85d", + "type": "retainer", + "config": { + "storage_type": "ram", + "max_retained_messages": 0, + "max_payload_size": "1MB", + "expiry_interval": 0 + }, + "enabled": true, + "created_at": "undefined", + "description": "" + }, + { + "id": "module:127b92c3", + "type": "hot_confs", + "config": {}, + "enabled": true, + "created_at": "undefined", + "description": "" + } + ], + "rules": [], + "resources": [ + { + "id": "bridge", + "type": "bridge_mqtt", + "config": { + "address": "127.0.0.1:1883", + "append": true, + "bridge_mode": false, + "cacertfile": "etc/certs/cacert.pem", + "certfile": "etc/certs/client-cert.pem", + "ciphers": "ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA", + "clientid": "client", + "disk_cache": "off", + "keepalive": "60s", + "keyfile": "etc/certs/client-key.pem", + "mountpoint": "bridge/emqx/${node}/", + "password": "", + "pool_size": 8, + "proto_ver": "mqttv4", + "reconnect_interval": "30s", + "retry_interval": "20s", + "ssl": "off", + "username": "" + }, + "created_at": null, + "description": "bridge" + }, + { + "id": "rpc", + "type": "bridge_rpc", + "config": { + "address": "test@127.0.0.1", + "batch_size": 32, + "disk_cache": "off", + "mountpoint": "bridge/emqx/${node}/", + "pool_size": 8, + "reconnect_interval": "30s" + }, + "created_at": null, + "description": "rpc" + } + ], + "blacklist": [], + "apps": [ + { + "id": "admin", + "secret": "public", + "name": "Default", + "desc": "Application user", + "status": true, + "expired": "undefined" + } + ], + "users": [ + { + "username": "admin", + "password": "bx1P63qGDhKvZYdltxX4NVY2kS4=", + "tags": "administrator" + } + ], + "auth_mnesia": [], + "acl_mnesia": [], + "schemas": [] +} \ No newline at end of file diff --git a/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE_data/ee425.json b/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE_data/ee425.json new file mode 100644 index 000000000..c6a80b9f6 --- /dev/null +++ b/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE_data/ee425.json @@ -0,0 +1,123 @@ +{ + "version": "4.2", + "date": "2021-04-13 16:42:34", + "modules": [ + { + "id": "module:3dc04a9c", + "type": "retainer", + "config": { + "storage_type": "ram", + "max_retained_messages": 0, + "max_payload_size": "1MB", + "expiry_interval": 0 + }, + "enabled": true, + "created_at": "undefined", + "description": "" + }, + { + "id": "module:5128901f", + "type": "recon", + "config": {}, + "enabled": true, + "created_at": "undefined", + "description": "" + }, + { + "id": "module:9d1596c8", + "type": "presence", + "config": { + "qos": 0 + }, + "enabled": true, + "created_at": "undefined", + "description": "" + }, + { + "id": "module:43d43410", + "type": "internal_acl", + "config": { + "acl_rule_file": "etc/acl.conf" + }, + "enabled": true, + "created_at": "undefined", + "description": "" + } + ], + "rules": [], + "resources": [ + { + "id": "bridge", + "type": "bridge_mqtt", + "config": { + "address": "127.0.0.1:1883", + "append": true, + "bridge_mode": false, + "cacertfile": { + "filename": "", + "file": "" + }, + "certfile": { + "filename": "", + "file": "" + }, + "ciphers": "ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA", + "clientid": "client", + "disk_cache": "off", + "keepalive": "60s", + "keyfile": { + "filename": "", + "file": "" + }, + "mountpoint": "bridge/emqx/${node}/", + "password": "", + "pool_size": 8, + "proto_ver": "mqttv4", + "reconnect_interval": "30s", + "retry_interval": "20s", + "ssl": false, + "username": "", + "verify": false + }, + "created_at": null, + "description": "bridge" + }, + { + "id": "rpc", + "type": "bridge_rpc", + "config": { + "address": "test@127.0.0.1", + "batch_size": 32, + "disk_cache": "off", + "mountpoint": "bridge/emqx/${node}/", + "pool_size": 8, + "reconnect_interval": "30s" + }, + "created_at": null, + "description": "rpc" + } + ], + "blacklist": [], + "apps": [ + { + "id": "admin", + "secret": "public", + "name": "Default", + "desc": "Application user", + "status": true, + "expired": "undefined" + } + ], + "users": [ + { + "username": "admin", + "password": "Hd8AMmbFs+LsqQXQxaV/WqLoGEk=", + "tags": "administrator" + } + ], + "auth_mnesia": [], + "acl_mnesia": [], + "schemas": [], + "configs": [], + "listeners_state": [] +} \ No newline at end of file diff --git a/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE_data/ee430.json b/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE_data/ee430.json new file mode 100644 index 000000000..2b7e66013 --- /dev/null +++ b/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE_data/ee430.json @@ -0,0 +1,123 @@ +{ + "version": "4.3", + "rules": [], + "resources": [ + { + "id": "bridge", + "type": "bridge_mqtt", + "config": { + "address": "127.0.0.1:1883", + "append": true, + "bridge_mode": false, + "cacertfile": { + "filename": "", + "file": "" + }, + "certfile": { + "filename": "", + "file": "" + }, + "ciphers": "ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA", + "clientid": "client", + "disk_cache": "off", + "keepalive": "60s", + "keyfile": { + "filename": "", + "file": "" + }, + "mountpoint": "bridge/emqx/${node}/", + "password": "", + "pool_size": 8, + "proto_ver": "mqttv4", + "reconnect_interval": "30s", + "retry_interval": "20s", + "ssl": false, + "username": "", + "verify": false + }, + "created_at": 1618304391051, + "description": "bridge" + }, + { + "id": "rpc", + "type": "bridge_rpc", + "config": { + "address": "test@127.0.0.1", + "batch_size": 32, + "disk_cache": "off", + "mountpoint": "bridge/emqx/${node}/", + "pool_size": 8, + "reconnect_interval": "30s" + }, + "created_at": 1618304406842, + "description": "rpc" + } + ], + "blacklist": [], + "apps": [ + { + "id": "admin", + "secret": "public", + "name": "Default", + "desc": "Application user", + "status": true, + "expired": "undefined" + } + ], + "users": [ + { + "username": "admin", + "password": "qq8hg9pOkmYiHqzi3+bcUaK2CGA=", + "tags": "administrator" + } + ], + "auth_mnesia": [], + "acl_mnesia": [], + "modules": [ + { + "id": "module:aabeddbf", + "type": "recon", + "config": {}, + "enabled": true, + "created_at": 1618304311061, + "description": "" + }, + { + "id": "module:cbe6d976", + "type": "internal_acl", + "config": { + "acl_rule_file": "etc/acl.conf" + }, + "enabled": true, + "created_at": 1618304311061, + "description": "" + }, + { + "id": "module:46375e06", + "type": "retainer", + "config": { + "storage_type": "ram", + "max_retained_messages": 0, + "max_payload_size": "1MB", + "expiry_interval": 0 + }, + "enabled": true, + "created_at": 1618304311061, + "description": "" + }, + { + "id": "module:091eb7c3", + "type": "presence", + "config": { + "qos": 0 + }, + "enabled": true, + "created_at": 1618304311061, + "description": "" + } + ], + "schemas": [], + "configs": [], + "listeners_state": [], + "date": "2021-04-13 17:59:52" +} \ No newline at end of file diff --git a/apps/emqx_management/test/emqx_mgmt_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_SUITE.erl index c9aea26d6..42578baf7 100644 --- a/apps/emqx_management/test/emqx_mgmt_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl index b04daf359..86b65939c 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_management/test/emqx_webhook_data_export_import_SUITE.erl b/apps/emqx_management/test/emqx_webhook_data_export_import_SUITE.erl index 261796afd..03d348d2a 100644 --- a/apps/emqx_management/test/emqx_webhook_data_export_import_SUITE.erl +++ b/apps/emqx_management/test/emqx_webhook_data_export_import_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -18,6 +18,9 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("emqx_rule_engine/include/rule_engine.hrl"). -compile([export_all, nowarn_export_all]). + +% -define(EMQX_ENTERPRISE, true). + %%-------------------------------------------------------------------- %% Setups %%-------------------------------------------------------------------- @@ -47,10 +50,55 @@ end_per_suite(Cfg) -> get_data_path() -> emqx_ct_helpers:deps_path(emqx_management, "test/emqx_webhook_data_export_import_SUITE_data/"). + +remove_resource(Id) -> + emqx_rule_registry:remove_resource(Id), + emqx_rule_registry:remove_resource_params(Id). + +import(FilePath, Version) -> + ok = emqx_mgmt_data_backup:import(get_data_path() ++ "/" ++ FilePath, <<"{}">>), + lists:foreach(fun(#resource{id = Id, config = Config} = _Resource) -> + case Id of + <<"webhook">> -> + test_utils:resource_is_alive(Id), + handle_config(Config, Version), + remove_resource(Id); + _ -> ok + end + end, emqx_rule_registry:get_resources()). + %%-------------------------------------------------------------------- %% Cases %%-------------------------------------------------------------------- +-ifdef(EMQX_ENTERPRISE). +t_importee4010(_) -> + import("ee4010.json", ee4010), + {ok, _} = emqx_mgmt_data_backup:export(). + +t_importee410(_) -> + import("ee410.json", ee410), + {ok, _} = emqx_mgmt_data_backup:export(). + +t_importee411(_) -> + import("ee411.json", ee411), + {ok, _} = emqx_mgmt_data_backup:export(). + +t_importee420(_) -> + import("ee420.json", ee420), + {ok, _} = emqx_mgmt_data_backup:export(). + +t_importee425(_) -> + import("ee425.json", ee425), + {ok, _} = emqx_mgmt_data_backup:export(). + +t_importee430(_) -> + import("ee430.json", ee430), + {ok, _} = emqx_mgmt_data_backup:export(). + +%%-------------------------------------------------------------------- +%% handle_config +%%-------------------------------------------------------------------- handle_config(Config, 409) -> handle_config(Config, 422); @@ -87,24 +135,10 @@ handle_config(Config, 430) -> ?assertEqual(true, is_map(maps:get(<<"certfile">>, Config))), ?assertEqual(true, is_map(maps:get(<<"keyfile">>, Config))), ?assertEqual(8, maps:get(<<"pool_size">>, Config)); - handle_config(_, _) -> ok. +-endif. -remove_resource(Id) -> - emqx_rule_registry:remove_resource(Id), - emqx_rule_registry:remove_resource_params(Id). - -import(FilePath, Version) -> - Overrides = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(clientid)}), - ok = emqx_mgmt_data_backup:import(get_data_path() ++ "/" ++ FilePath, Overrides), - lists:foreach(fun(#resource{id = Id, config = Config} = _Resource) -> - case Id of - "webhook" -> - handle_config(Config, Version), - remove_resource(Id); - _ -> ok - end - end, emqx_rule_registry:get_resources()). +-ifndef(EMQX_ENTERPRISE). t_import422(_) -> import("422.json", 422), @@ -129,3 +163,41 @@ t_import409(_) -> t_import415(_) -> import("415.json", 415), {ok, _} = emqx_mgmt_data_backup:export(). + +%%-------------------------------------------------------------------- +%% handle_config +%%-------------------------------------------------------------------- + +handle_config(Config, ee4010) -> + ?assertEqual(<<"http://www.emqx.io">>, maps:get(<<"url">>, Config)); + +handle_config(Config, ee410) -> + ?assertEqual(<<"http://www.emqx.io">>, maps:get(<<"url">>, Config)); + +handle_config(Config, ee411) -> + ?assertEqual(<<"http://www.emqx.io">>, maps:get(<<"url">>, Config)); + +handle_config(Config, ee420) -> + ?assertEqual(<<"http://www.emqx.io">>, maps:get(<<"url">>, Config)); + +handle_config(Config, ee425) -> + ?assertEqual(<<"http://www.emqx.io">>, maps:get(<<"url">>, Config)), + ?assertEqual(<<"5s">>, maps:get(<<"connect_timeout">>, Config)), + ?assertEqual(<<"5s">>, maps:get(<<"request_timeout">>, Config)), + ?assertEqual(false, maps:get(<<"verify">>, Config)), + ?assertEqual(8, maps:get(<<"pool_size">>, Config)); + +handle_config(Config, ee435) -> + handle_config(Config, ee430); + +handle_config(Config, ee430) -> + ?assertEqual(<<"http://www.emqx.io">>, maps:get(<<"url">>, Config)), + ?assertEqual(<<"5s">>, maps:get(<<"connect_timeout">>, Config)), + ?assertEqual(<<"5s">>, maps:get(<<"request_timeout">>, Config)), + ?assertEqual(false, maps:get(<<"verify">>, Config)), + ?assertEqual(8, maps:get(<<"pool_size">>, Config)); + +handle_config(Config, _) -> + io:format("|>=> :~p~n", [Config]). + +-endif. diff --git a/apps/emqx_management/test/emqx_webhook_data_export_import_SUITE_data/ee4010.json b/apps/emqx_management/test/emqx_webhook_data_export_import_SUITE_data/ee4010.json new file mode 100644 index 000000000..d979d1d0d --- /dev/null +++ b/apps/emqx_management/test/emqx_webhook_data_export_import_SUITE_data/ee4010.json @@ -0,0 +1,43 @@ +{ + "version": "4.0", + "users": [ + { + "username": "admin", + "tags": "administrator", + "password": "GCX5nvOMK0hbiMB4AUyc25wI8fU=" + } + ], + "schemas": [], + "rules": [], + "resources": [ + { + "type": "web_hook", + "id": "web_hook", + "description": "webhook", + "created_at": null, + "config": { + "url": "http://www.emqx.io", + "method": "POST", + "headers": { + "k": "v" + } + } + } + ], + "date": "2021-04-13 13:57:23", + "blacklist": [], + "auth_username": [], + "auth_mnesia": [], + "auth_clientid": [], + "apps": [ + { + "status": true, + "secret": "public", + "name": "Default", + "id": "admin", + "expired": "undefined", + "desc": "Application user" + } + ], + "acl_mnesia": [] +} \ No newline at end of file diff --git a/apps/emqx_management/test/emqx_webhook_data_export_import_SUITE_data/ee410.json b/apps/emqx_management/test/emqx_webhook_data_export_import_SUITE_data/ee410.json new file mode 100644 index 000000000..e97427e01 --- /dev/null +++ b/apps/emqx_management/test/emqx_webhook_data_export_import_SUITE_data/ee410.json @@ -0,0 +1,43 @@ +{ + "version": "4.1", + "users": [ + { + "username": "admin", + "tags": "administrator", + "password": "atdwlByxL9/9P3CoFJ60drhodkY=" + } + ], + "schemas": [], + "rules": [], + "resources": [ + { + "type": "web_hook", + "id": "webhook", + "description": "webhook", + "created_at": null, + "config": { + "url": "http://www.emqx.io", + "method": "POST", + "headers": { + "k": "v" + } + } + } + ], + "date": "2021-04-13 11:30:21", + "blacklist": [], + "auth_username": [], + "auth_mnesia": [], + "auth_clientid": [], + "apps": [ + { + "status": true, + "secret": "public", + "name": "Default", + "id": "admin", + "expired": "undefined", + "desc": "Application user" + } + ], + "acl_mnesia": [] +} \ No newline at end of file diff --git a/apps/emqx_management/test/emqx_webhook_data_export_import_SUITE_data/ee411.json b/apps/emqx_management/test/emqx_webhook_data_export_import_SUITE_data/ee411.json new file mode 100644 index 000000000..ed7fab854 --- /dev/null +++ b/apps/emqx_management/test/emqx_webhook_data_export_import_SUITE_data/ee411.json @@ -0,0 +1,43 @@ +{ + "version": "4.1", + "users": [ + { + "username": "admin", + "tags": "administrator", + "password": "tsYtP3TylchkM7J7YTc46Di0kPk=" + } + ], + "schemas": [], + "rules": [], + "resources": [ + { + "type": "web_hook", + "id": "web_hook", + "description": "webhook", + "created_at": null, + "config": { + "url": "http://www.emqx.io", + "method": "POST", + "headers": { + "k": "v" + } + } + } + ], + "date": "2021-04-13 16:37:18", + "blacklist": [], + "auth_username": [], + "auth_mnesia": [], + "auth_clientid": [], + "apps": [ + { + "status": true, + "secret": "public", + "name": "Default", + "id": "admin", + "expired": "undefined", + "desc": "Application user" + } + ], + "acl_mnesia": [] +} \ No newline at end of file diff --git a/apps/emqx_management/test/emqx_webhook_data_export_import_SUITE_data/ee420.json b/apps/emqx_management/test/emqx_webhook_data_export_import_SUITE_data/ee420.json new file mode 100644 index 000000000..52ad0c83f --- /dev/null +++ b/apps/emqx_management/test/emqx_webhook_data_export_import_SUITE_data/ee420.json @@ -0,0 +1,92 @@ +{ + "version": "4.2", + "date": "2021-04-13 11:35:13", + "modules": [ + { + "id": "module:b9294d70", + "type": "recon", + "config": {}, + "enabled": true, + "created_at": "undefined", + "description": "" + }, + { + "id": "module:c7c7b692", + "type": "presence", + "config": { + "qos": 0 + }, + "enabled": true, + "created_at": "undefined", + "description": "" + }, + { + "id": "module:486adc4b", + "type": "internal_acl", + "config": { + "acl_rule_file": "etc/acl.conf" + }, + "enabled": true, + "created_at": "undefined", + "description": "" + }, + { + "id": "module:411cf85d", + "type": "retainer", + "config": { + "storage_type": "ram", + "max_retained_messages": 0, + "max_payload_size": "1MB", + "expiry_interval": 0 + }, + "enabled": true, + "created_at": "undefined", + "description": "" + }, + { + "id": "module:127b92c3", + "type": "hot_confs", + "config": {}, + "enabled": true, + "created_at": "undefined", + "description": "" + } + ], + "rules": [], + "resources": [ + { + "id": "webhook", + "type": "web_hook", + "config": { + "headers": { + "k": "v" + }, + "method": "POST", + "url": "http://www.emqx.io" + }, + "created_at": null, + "description": "webhook" + } + ], + "blacklist": [], + "apps": [ + { + "id": "admin", + "secret": "public", + "name": "Default", + "desc": "Application user", + "status": true, + "expired": "undefined" + } + ], + "users": [ + { + "username": "admin", + "password": "bx1P63qGDhKvZYdltxX4NVY2kS4=", + "tags": "administrator" + } + ], + "auth_mnesia": [], + "acl_mnesia": [], + "schemas": [] +} \ No newline at end of file diff --git a/apps/emqx_management/test/emqx_webhook_data_export_import_SUITE_data/ee425.json b/apps/emqx_management/test/emqx_webhook_data_export_import_SUITE_data/ee425.json new file mode 100644 index 000000000..46a1870bf --- /dev/null +++ b/apps/emqx_management/test/emqx_webhook_data_export_import_SUITE_data/ee425.json @@ -0,0 +1,102 @@ +{ + "version": "4.2", + "date": "2021-04-13 16:42:34", + "modules": [ + { + "id": "module:3dc04a9c", + "type": "retainer", + "config": { + "storage_type": "ram", + "max_retained_messages": 0, + "max_payload_size": "1MB", + "expiry_interval": 0 + }, + "enabled": true, + "created_at": "undefined", + "description": "" + }, + { + "id": "module:5128901f", + "type": "recon", + "config": {}, + "enabled": true, + "created_at": "undefined", + "description": "" + }, + { + "id": "module:9d1596c8", + "type": "presence", + "config": { + "qos": 0 + }, + "enabled": true, + "created_at": "undefined", + "description": "" + }, + { + "id": "module:43d43410", + "type": "internal_acl", + "config": { + "acl_rule_file": "etc/acl.conf" + }, + "enabled": true, + "created_at": "undefined", + "description": "" + } + ], + "rules": [], + "resources": [ + { + "id": "webhook", + "type": "web_hook", + "config": { + "cacertfile": { + "filename": "", + "file": "" + }, + "certfile": { + "filename": "", + "file": "" + }, + "connect_timeout": "5s", + "headers": { + "k": "v" + }, + "keyfile": { + "filename": "", + "file": "" + }, + "method": "POST", + "pool_size": 8, + "request_timeout": "5s", + "url": "http://www.emqx.io", + "verify": false + }, + "created_at": null, + "description": "webhook" + } + ], + "blacklist": [], + "apps": [ + { + "id": "admin", + "secret": "public", + "name": "Default", + "desc": "Application user", + "status": true, + "expired": "undefined" + } + ], + "users": [ + { + "username": "admin", + "password": "Hd8AMmbFs+LsqQXQxaV/WqLoGEk=", + "tags": "administrator" + } + ], + "auth_mnesia": [], + "acl_mnesia": [], + "schemas": [], + "configs": [], + "listeners_state": [] +} \ No newline at end of file diff --git a/apps/emqx_management/test/emqx_webhook_data_export_import_SUITE_data/ee430.json b/apps/emqx_management/test/emqx_webhook_data_export_import_SUITE_data/ee430.json new file mode 100644 index 000000000..36986f24b --- /dev/null +++ b/apps/emqx_management/test/emqx_webhook_data_export_import_SUITE_data/ee430.json @@ -0,0 +1,98 @@ +{ + "version": "4.3", + "rules": [], + "resources": [ + { + "id": "webhook", + "type": "web_hook", + "config": { + "cacertfile": { + "filename": "", + "file": "" + }, + "certfile": { + "filename": "", + "file": "" + }, + "connect_timeout": "5s", + "keyfile": { + "filename": "", + "file": "" + }, + "pool_size": 8, + "request_timeout": "5s", + "url": "http://www.emqx.io", + "verify": false + }, + "created_at": 1618304340172, + "description": "webhook" + } + ], + "blacklist": [], + "apps": [ + { + "id": "admin", + "secret": "public", + "name": "Default", + "desc": "Application user", + "status": true, + "expired": "undefined" + } + ], + "users": [ + { + "username": "admin", + "password": "qq8hg9pOkmYiHqzi3+bcUaK2CGA=", + "tags": "administrator" + } + ], + "auth_mnesia": [], + "acl_mnesia": [], + "modules": [ + { + "id": "module:aabeddbf", + "type": "recon", + "config": {}, + "enabled": true, + "created_at": 1618304311061, + "description": "" + }, + { + "id": "module:cbe6d976", + "type": "internal_acl", + "config": { + "acl_rule_file": "etc/acl.conf" + }, + "enabled": true, + "created_at": 1618304311061, + "description": "" + }, + { + "id": "module:46375e06", + "type": "retainer", + "config": { + "storage_type": "ram", + "max_retained_messages": 0, + "max_payload_size": "1MB", + "expiry_interval": 0 + }, + "enabled": true, + "created_at": 1618304311061, + "description": "" + }, + { + "id": "module:091eb7c3", + "type": "presence", + "config": { + "qos": 0 + }, + "enabled": true, + "created_at": 1618304311061, + "description": "" + } + ], + "schemas": [], + "configs": [], + "listeners_state": [], + "date": "2021-04-13 17:59:52" +} \ No newline at end of file diff --git a/apps/emqx_management/test/test_utils.erl b/apps/emqx_management/test/test_utils.erl new file mode 100644 index 000000000..337a9499b --- /dev/null +++ b/apps/emqx_management/test/test_utils.erl @@ -0,0 +1,19 @@ +%% @author: +%% @description: +-module(test_utils). +%% ==================================================================== +%% API functions +%% ==================================================================== +-include_lib("eunit/include/eunit.hrl"). +-include_lib("emqx_rule_engine/include/rule_engine.hrl"). + +-compile([export_all, nowarn_export_all]). + +%% ==================================================================== +%% Internal functions +%% ==================================================================== +resource_is_alive(Id) -> + {ok, #resource_params{status = #{is_alive := Alive}} = Params} = emqx_rule_registry:find_resource_params(Id), + ct:pal("Id: ~p, Alive: ~p, Resource ===> :~p~n", [Id, Alive, Params]), + ?assertEqual(true, Alive), + Alive. diff --git a/apps/emqx_prometheus/src/emqx_prometheus.erl b/apps/emqx_prometheus/src/emqx_prometheus.erl index 4527997f7..c11c8f0d7 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -124,8 +124,10 @@ collect_mf(_Registry, Callback) -> Metrics = emqx_metrics:all(), Stats = emqx_stats:getstats(), VMData = emqx_vm_data(), + ClusterData = emqx_cluster_data(), _ = [add_collect_family(Name, Stats, Callback, gauge) || Name <- emqx_stats()], _ = [add_collect_family(Name, VMData, Callback, gauge) || Name <- emqx_vm()], + _ = [add_collect_family(Name, ClusterData, Callback, gauge) || Name <- emqx_cluster()], _ = [add_collect_family(Name, Metrics, Callback, counter) || Name <- emqx_metrics_packets()], _ = [add_collect_family(Name, Metrics, Callback, counter) || Name <- emqx_metrics_messages()], _ = [add_collect_family(Name, Metrics, Callback, counter) || Name <- emqx_metrics_delivery()], @@ -454,7 +456,13 @@ emqx_collect(emqx_vm_total_memory, VMData) -> gauge_metric(?C(total_memory, VMData)); emqx_collect(emqx_vm_used_memory, VMData) -> - gauge_metric(?C(used_memory, VMData)). + gauge_metric(?C(used_memory, VMData)); + +emqx_collect(emqx_cluster_nodes_running, ClusterData) -> + gauge_metric(?C(nodes_running, ClusterData)); + +emqx_collect(emqx_cluster_nodes_stopped, ClusterData) -> + gauge_metric(?C(nodes_stopped, ClusterData)). %%-------------------------------------------------------------------- %% Indicators @@ -592,3 +600,13 @@ emqx_vm_data() -> {process_total_messages, 0}, %% XXX: Plan removed at v5.0 {cpu_idle, Idle}, {cpu_use, 100 - Idle}] ++ emqx_vm:mem_info(). + +emqx_cluster() -> + [ emqx_cluster_nodes_running + , emqx_cluster_nodes_stopped + ]. + +emqx_cluster_data() -> + #{running_nodes := Running, stopped_nodes := Stopped} = ekka_mnesia:cluster_info(), + [{nodes_running, length(Running)}, + {nodes_stopped, length(Stopped)}]. diff --git a/apps/emqx_prometheus/src/emqx_prometheus_app.erl b/apps/emqx_prometheus/src/emqx_prometheus_app.erl index 8fdcd2f02..df0701551 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_app.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_prometheus/src/emqx_prometheus_sup.erl b/apps/emqx_prometheus/src/emqx_prometheus_sup.erl index aa0aeb047..8ebbb02ae 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_sup.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl b/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl index 74ea4f217..cb2ae4e8e 100644 --- a/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl +++ b/apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_psk_file/src/emqx_psk_file.erl b/apps/emqx_psk_file/src/emqx_psk_file.erl index ca9bc1292..ced11b0da 100644 --- a/apps/emqx_psk_file/src/emqx_psk_file.erl +++ b/apps/emqx_psk_file/src/emqx_psk_file.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_psk_file/src/emqx_psk_file_app.erl b/apps/emqx_psk_file/src/emqx_psk_file_app.erl index be7440150..934ffe49e 100644 --- a/apps/emqx_psk_file/src/emqx_psk_file_app.erl +++ b/apps/emqx_psk_file/src/emqx_psk_file_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_psk_file/src/emqx_psk_file_sup.erl b/apps/emqx_psk_file/src/emqx_psk_file_sup.erl index cf4ea8317..041eecdb6 100644 --- a/apps/emqx_psk_file/src/emqx_psk_file_sup.erl +++ b/apps/emqx_psk_file/src/emqx_psk_file_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_psk_file/test/emqx_psk_file_SUITE.erl b/apps/emqx_psk_file/test/emqx_psk_file_SUITE.erl index 9773da2b9..d0083247d 100644 --- a/apps/emqx_psk_file/test/emqx_psk_file_SUITE.erl +++ b/apps/emqx_psk_file/test/emqx_psk_file_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_recon/src/emqx_recon.erl b/apps/emqx_recon/src/emqx_recon.erl index f3e303f23..67039c3ac 100644 --- a/apps/emqx_recon/src/emqx_recon.erl +++ b/apps/emqx_recon/src/emqx_recon.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_recon/src/emqx_recon_cli.erl b/apps/emqx_recon/src/emqx_recon_cli.erl index 587616ba7..b4e3a52ab 100644 --- a/apps/emqx_recon/src/emqx_recon_cli.erl +++ b/apps/emqx_recon/src/emqx_recon_cli.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_recon/test/emqx_recon_SUITE.erl b/apps/emqx_recon/test/emqx_recon_SUITE.erl index 7e9b796d6..2abf8f709 100644 --- a/apps/emqx_recon/test/emqx_recon_SUITE.erl +++ b/apps/emqx_recon/test/emqx_recon_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_retainer/include/emqx_retainer.hrl b/apps/emqx_retainer/include/emqx_retainer.hrl index d6ac62de2..a1e229cfb 100644 --- a/apps/emqx_retainer/include/emqx_retainer.hrl +++ b/apps/emqx_retainer/include/emqx_retainer.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_retainer/src/emqx_retainer.erl b/apps/emqx_retainer/src/emqx_retainer.erl index 28812ebfa..c1523b7a1 100644 --- a/apps/emqx_retainer/src/emqx_retainer.erl +++ b/apps/emqx_retainer/src/emqx_retainer.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_retainer/src/emqx_retainer_app.erl b/apps/emqx_retainer/src/emqx_retainer_app.erl index 5c9175a71..adca5ae7a 100644 --- a/apps/emqx_retainer/src/emqx_retainer_app.erl +++ b/apps/emqx_retainer/src/emqx_retainer_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_retainer/src/emqx_retainer_cli.erl b/apps/emqx_retainer/src/emqx_retainer_cli.erl index 6e83b8641..fe8fa9578 100644 --- a/apps/emqx_retainer/src/emqx_retainer_cli.erl +++ b/apps/emqx_retainer/src/emqx_retainer_cli.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_retainer/src/emqx_retainer_sup.erl b/apps/emqx_retainer/src/emqx_retainer_sup.erl index 2562814ed..fef245489 100644 --- a/apps/emqx_retainer/src/emqx_retainer_sup.erl +++ b/apps/emqx_retainer/src/emqx_retainer_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_retainer/test/emqx_retainer_SUITE.erl b/apps/emqx_retainer/test/emqx_retainer_SUITE.erl index 0c0e2ce7a..1df042dd9 100644 --- a/apps/emqx_retainer/test/emqx_retainer_SUITE.erl +++ b/apps/emqx_retainer/test/emqx_retainer_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_retainer/test/emqx_retainer_cli_SUITE.erl b/apps/emqx_retainer/test/emqx_retainer_cli_SUITE.erl index caeddeaef..f9f657483 100644 --- a/apps/emqx_retainer/test/emqx_retainer_cli_SUITE.erl +++ b/apps/emqx_retainer/test/emqx_retainer_cli_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_retainer/test/mqtt_protocol_v5_SUITE.erl b/apps/emqx_retainer/test/mqtt_protocol_v5_SUITE.erl index 1803f3574..cec492c6a 100644 --- a/apps/emqx_retainer/test/mqtt_protocol_v5_SUITE.erl +++ b/apps/emqx_retainer/test/mqtt_protocol_v5_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_rule_engine/README.md b/apps/emqx_rule_engine/README.md index ad0b637cf..0b1ddd848 100644 --- a/apps/emqx_rule_engine/README.md +++ b/apps/emqx_rule_engine/README.md @@ -39,7 +39,7 @@ select id, time, temperature as t from "topic/a" where t > 50; ## License -Copyright (c) 2019 [EMQ Technologies Co., Ltd](https://emqx.io). All Rights Reserved. +Copyright (c) 2019-2021 [EMQ Technologies Co., Ltd](https://emqx.io). All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License");you may not use this file except in compliance with the License.You may obtain a copy of the License at diff --git a/apps/emqx_rule_engine/include/rule_engine.hrl b/apps/emqx_rule_engine/include/rule_engine.hrl index a54afb7a7..568724263 100644 --- a/apps/emqx_rule_engine/include/rule_engine.hrl +++ b/apps/emqx_rule_engine/include/rule_engine.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -81,6 +81,7 @@ , enabled :: boolean() , created_at :: integer() %% epoch in millisecond precision , description :: binary() + , state = normal :: atom() }). -record(resource, diff --git a/apps/emqx_rule_engine/src/emqx_rule_actions.erl b/apps/emqx_rule_engine/src/emqx_rule_actions.erl index 378ed2684..61dc36b35 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_actions.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_actions.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.erl b/apps/emqx_rule_engine/src/emqx_rule_engine.erl index c301367fd..07827ed1f 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -180,7 +180,8 @@ create_rule(Params = #{rawsql := Sql, actions := ActArgs}) -> actions = Actions, enabled = Enabled, created_at = erlang:system_time(millisecond), - description = maps:get(description, Params, "") + description = maps:get(description, Params, ""), + state = normal }, ok = emqx_rule_registry:add_rule(Rule), ok = emqx_rule_metrics:create_rule_metrics(RuleId), @@ -249,40 +250,30 @@ create_resource(#{type := Type, config := Config0} = Params) -> -spec(update_resource(resource_id(), map()) -> ok | {error, Reason :: term()}). update_resource(ResId, NewParams) -> - try - lists:foreach(fun(#rule{id = RuleId, enabled = Enabled, actions = Actions}) -> - lists:foreach( - fun (#action_instance{args = #{<<"$resource">> := ResId1}}) - when ResId =:= ResId1, Enabled =:= true -> - throw({dependency_exists, RuleId}); - (_) -> ok - end, Actions) - end, ets:tab2list(?RULE_TAB)), - do_update_resource_check(ResId, NewParams) - catch _ : Reason -> - {error, Reason} + case emqx_rule_registry:find_enabled_rules_depends_on_resource(ResId) of + [] -> check_and_update_resource(ResId, NewParams); + Rules -> + {error, {dependent_rules_exists, [Id || #rule{id = Id} <- Rules]}} end. -do_update_resource_check(Id, NewParams) -> +check_and_update_resource(Id, NewParams) -> case emqx_rule_registry:find_resource(Id) of - {ok, #resource{id = Id, - type = Type, - config = OldConfig, - description = OldDescription} = _OldResource} -> + {ok, #resource{id = Id, type = Type, config = OldConfig, description = OldDescr}} -> try Conifg = maps:get(<<"config">>, NewParams, OldConfig), - Descr = maps:get(<<"description">>, NewParams, OldDescription), - do_update_resource(#{id => Id, config => Conifg, type => Type, - description => Descr}), - ok - catch _ : Reason -> + Descr = maps:get(<<"description">>, NewParams, OldDescr), + do_check_and_update_resource(#{id => Id, config => Conifg, type => Type, + description => Descr}) + catch Error:Reason:ST -> + ?LOG(error, "check_and_update_resource failed: ~0p", [{Error, Reason, ST}]), {error, Reason} end; _Other -> {error, not_found} end. -do_update_resource(#{id := Id, type := Type, description := NewDescription, config := NewConfig}) -> +do_check_and_update_resource(#{id := Id, type := Type, description := NewDescription, + config := NewConfig}) -> case emqx_rule_registry:find_resource_type(Type) of {ok, #resource_type{on_create = {Module, Create}, params_spec = ParamSpec}} -> @@ -296,7 +287,8 @@ do_update_resource(#{id := Id, type := Type, description := NewDescription, conf config = Config, description = NewDescription, created_at = erlang:system_time(millisecond) - }); + }), + ok; {error, Reason} -> error({error, Reason}) end @@ -365,9 +357,12 @@ delete_resource(ResId) -> {ok, #resource_type{on_destroy = {ModD, Destroy}}} = emqx_rule_registry:find_resource_type(ResType), try - ok = emqx_rule_registry:remove_resource(ResId), - _ = ?CLUSTER_CALL(clear_resource, [ModD, Destroy, ResId]), - ok + case emqx_rule_registry:remove_resource(ResId) of + ok -> + _ = ?CLUSTER_CALL(clear_resource, [ModD, Destroy, ResId]), + ok; + {error, _} = R -> R + end catch throw:Reason -> {error, Reason} end; @@ -393,15 +388,13 @@ refresh_resource(#resource{id = ResId}) -> -spec(refresh_rules() -> ok). refresh_rules() -> - lists:foreach(fun(#rule{id = RuleId} = Rule) -> - try refresh_rule(Rule) - catch Error:Reason:ST -> - logger:critical( - "Can not re-build rule ~p: ~0p. The rule is disabled." - "Fix the issue and enable it manually.\n" - "Stacktrace: ~0p", - [RuleId, {Error,Reason}, ST]) - end + lists:foreach(fun + (#rule{enabled = true} = Rule) -> + try refresh_rule(Rule) + catch _:_ -> + emqx_rule_registry:add_rule(Rule#rule{enabled = false, state = refresh_failed_at_bootup}) + end; + (_) -> ok end, emqx_rule_registry:get_rules()). refresh_rule(#rule{id = RuleId, for = Topics, actions = Actions}) -> @@ -472,14 +465,18 @@ may_update_rule_params(Rule, Params = #{rawsql := SQL}) -> maps:remove(rawsql, Params)); Reason -> throw(Reason) end; -may_update_rule_params(Rule = #rule{enabled = OldEnb, actions = Actions}, +may_update_rule_params(Rule = #rule{enabled = OldEnb, actions = Actions, state = OldState}, Params = #{enabled := NewEnb}) -> - case {OldEnb, NewEnb} of - {false, true} -> refresh_rule(Rule); - {true, false} -> clear_actions(Actions); - _ -> ok - end, - may_update_rule_params(Rule#rule{enabled = NewEnb}, maps:remove(enabled, Params)); + State = case {OldEnb, NewEnb} of + {false, true} -> + refresh_rule(Rule), + force_changed; + {true, false} -> + clear_actions(Actions), + force_changed; + _NoChange -> OldState + end, + may_update_rule_params(Rule#rule{enabled = NewEnb, state = State}, maps:remove(enabled, Params)); may_update_rule_params(Rule, Params = #{description := Descr}) -> may_update_rule_params(Rule#rule{description = Descr}, maps:remove(description, Params)); may_update_rule_params(Rule, Params = #{on_action_failed := OnFailed}) -> diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index 82b4c2682..4fa3b8aa3 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -180,6 +180,7 @@ -define(ERR_NO_ACTION(NAME), list_to_binary(io_lib:format("Action ~s Not Found", [(NAME)]))). -define(ERR_NO_RESOURCE(RESID), list_to_binary(io_lib:format("Resource ~s Not Found", [(RESID)]))). -define(ERR_NO_RESOURCE_TYPE(TYPE), list_to_binary(io_lib:format("Resource Type ~s Not Found", [(TYPE)]))). +-define(ERR_DEP_RULES_EXISTS(RULEIDS), list_to_binary(io_lib:format("Found rules ~0p depends on this resource, disable them first", [(RULEIDS)]))). -define(ERR_BADARGS(REASON), begin R0 = list_to_binary(io_lib:format("~0p", [REASON])), @@ -342,23 +343,22 @@ update_resource(#{id := Id}, NewParams) -> ok -> return(ok); {error, not_found} -> - ?LOG(error, "Resource not found: ~0p", [Id]), return({error, 400, <<"Resource not found:", Id/binary>>}); {error, {init_resource, _}} -> - ?LOG(error, "Init resource failure: ~0p", [Id]), return({error, 500, <<"Init resource failure:", Id/binary>>}); - {error, {dependency_exists, RuleId}} -> - ?LOG(error, "Dependency exists: ~0p", [RuleId]), - return({error, 500, <<"Dependency exists:", RuleId/binary>>}); + {error, {dependent_rules_exists, RuleIds}} -> + return({error, 400, ?ERR_DEP_RULES_EXISTS(RuleIds)}); {error, Reason} -> ?LOG(error, "Resource update failed: ~0p", [Reason]), - return({error, 500, <<"Resource update failed!">>}) + return({error, 400, ?ERR_BADARGS(Reason)}) end. delete_resource(#{id := Id}, _Params) -> case emqx_rule_engine:delete_resource(Id) of ok -> return(ok); {error, not_found} -> return(ok); + {error, {dependent_rules_exists, RuleIds}} -> + return({error, 400, ?ERR_DEP_RULES_EXISTS(RuleIds)}); {error, Reason} -> return({error, 400, ?ERR_BADARGS(Reason)}) end. diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_app.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_app.erl index ad9e5ba37..e00d717d1 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_app.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_cli.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_cli.erl index dbcf3e0e5..bcb869aec 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_cli.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_cli.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_sup.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_sup.erl index d561340bb..9ff5ce741 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_sup.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_rule_engine/src/emqx_rule_events.erl b/apps/emqx_rule_engine/src/emqx_rule_events.erl index fc3096f6a..97e40439d 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_events.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_events.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index 4d91b2fe1..4f8851d40 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -180,6 +180,10 @@ %% Date functions -export([ now_rfc3339/0 , now_rfc3339/1 + , unix_ts_to_rfc3339/1 + , unix_ts_to_rfc3339/2 + , rfc3339_to_unix_ts/1 + , rfc3339_to_unix_ts/2 , now_timestamp/0 , now_timestamp/1 ]). @@ -834,9 +838,22 @@ now_rfc3339() -> now_rfc3339(<<"second">>). now_rfc3339(Unit) -> + unix_ts_to_rfc3339(now_timestamp(Unit), Unit). + +unix_ts_to_rfc3339(Epoch) -> + unix_ts_to_rfc3339(Epoch, <<"second">>). + +unix_ts_to_rfc3339(Epoch, Unit) when is_integer(Epoch) -> emqx_rule_utils:bin( calendar:system_time_to_rfc3339( - now_timestamp(Unit), [{unit, time_unit(Unit)}])). + Epoch, [{unit, time_unit(Unit)}])). + +rfc3339_to_unix_ts(DateTime) -> + rfc3339_to_unix_ts(DateTime, <<"second">>). + +rfc3339_to_unix_ts(DateTime, Unit) when is_binary(DateTime) -> + calendar:rfc3339_to_system_time(binary_to_list(DateTime), + [{unit, time_unit(Unit)}]). now_timestamp() -> erlang:system_time(second). diff --git a/apps/emqx_rule_engine/src/emqx_rule_id.erl b/apps/emqx_rule_engine/src/emqx_rule_id.erl index 749fe8551..3f9c1fc6d 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_id.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_id.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_rule_engine/src/emqx_rule_locker.erl b/apps/emqx_rule_engine/src/emqx_rule_locker.erl index 40451a0d8..9e45b8c09 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_locker.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_locker.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_rule_engine/src/emqx_rule_maps.erl b/apps/emqx_rule_engine/src/emqx_rule_maps.erl index e826d108b..512ae5c74 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_maps.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_maps.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_rule_engine/src/emqx_rule_metrics.erl b/apps/emqx_rule_engine/src/emqx_rule_metrics.erl index 90554d982..bc2b04c07 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_metrics.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_metrics.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_rule_engine/src/emqx_rule_monitor.erl b/apps/emqx_rule_engine/src/emqx_rule_monitor.erl index b4c1ff2fc..ed3b23395 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_monitor.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_monitor.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -104,7 +104,8 @@ retry_loop(resource, ResId, Interval) -> try {ok, #resource_type{on_create = {M, F}}} = emqx_rule_registry:find_resource_type(Type), - emqx_rule_engine:init_resource(M, F, ResId, Config) + ok = emqx_rule_engine:init_resource(M, F, ResId, Config), + refresh_and_enable_rules_of_resource(ResId) catch Err:Reason:ST -> ?LOG(warning, "init_resource failed: ~p, ~0p", @@ -115,3 +116,12 @@ retry_loop(resource, ResId, Interval) -> not_found -> ok end. + +refresh_and_enable_rules_of_resource(ResId) -> + lists:foreach( + fun (#rule{id = Id, enabled = false, state = refresh_failed_at_bootup} = Rule) -> + emqx_rule_engine:refresh_rule(Rule), + emqx_rule_registry:add_rule(Rule#rule{enabled = true, state = normal}), + ?LOG(info, "rule ~s is refreshed and re-enabled", [Id]); + (_) -> ok + end, emqx_rule_registry:find_rules_depends_on_resource(ResId)). diff --git a/apps/emqx_rule_engine/src/emqx_rule_registry.erl b/apps/emqx_rule_engine/src/emqx_rule_registry.erl index 9285b0ad5..4ec5d1cb7 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_registry.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_registry.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -63,6 +63,8 @@ %% Resource Types -export([ get_resource_types/0 , find_resource_type/1 + , find_rules_depends_on_resource/1 + , find_enabled_rules_depends_on_resource/1 , register_resource_types/1 , unregister_resource_types_of/1 ]). @@ -354,7 +356,7 @@ find_resource_params(Id) -> [] -> not_found end. --spec(remove_resource(emqx_rule_engine:resource() | emqx_rule_engine:resource_id()) -> ok). +-spec(remove_resource(emqx_rule_engine:resource() | emqx_rule_engine:resource_id()) -> ok | {error, term()}). remove_resource(Resource) when is_record(Resource, resource) -> trans(fun delete_resource/1, [Resource#resource.id]); @@ -368,20 +370,32 @@ remove_resource_params(ResId) -> %% @private delete_resource(ResId) -> - lists:foreach(fun(#rule{id = Id, actions = Actions}) -> - lists:foreach( - fun (#action_instance{args = #{<<"$resource">> := ResId1}}) - when ResId =:= ResId1 -> - throw({dependency_exists, {rule, Id}}); - (_) -> ok - end, Actions) - end, get_rules()), - mnesia:delete(?RES_TAB, ResId, write). + case find_enabled_rules_depends_on_resource(ResId) of + [] -> mnesia:delete(?RES_TAB, ResId, write); + Rules -> + {error, {dependent_rules_exists, [Id || #rule{id = Id} <- Rules]}} + end. %% @private insert_resource(Resource) -> mnesia:write(?RES_TAB, Resource, write). +find_enabled_rules_depends_on_resource(ResId) -> + [R || #rule{enabled = true} = R <- find_rules_depends_on_resource(ResId)]. + +find_rules_depends_on_resource(ResId) -> + lists:foldl(fun(#rule{actions = Actions} = R, Rules) -> + case search_action_despends_on_resource(ResId, Actions) of + false -> Rules; + {value, _} -> [R | Rules] + end + end, [], get_rules()). + +search_action_despends_on_resource(ResId, Actions) -> + lists:search(fun(#action_instance{args = #{<<"$resource">> := ResId0}}) -> + ResId0 =:= ResId + end, Actions). + %%------------------------------------------------------------------------------ %% Resource Type Management %%------------------------------------------------------------------------------ @@ -475,6 +489,6 @@ get_all_records(Tab) -> trans(Fun) -> trans(Fun, []). trans(Fun, Args) -> case mnesia:transaction(Fun, Args) of - {atomic, ok} -> ok; + {atomic, Result} -> Result; {aborted, Reason} -> error(Reason) end. diff --git a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl index c5c42b277..4c2b661ea 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_rule_engine/src/emqx_rule_sqlparser.erl b/apps/emqx_rule_engine/src/emqx_rule_sqlparser.erl index ed2142ea4..9a8ce55ea 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_sqlparser.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_sqlparser.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl index 90eb623ac..760205d62 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl @@ -1,4 +1,4 @@ -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_rule_engine/src/emqx_rule_utils.erl b/apps/emqx_rule_engine/src/emqx_rule_utils.erl index e178fc9d2..ff7ee0304 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_utils.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_utils.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -204,7 +204,7 @@ http_connectivity(Url) -> http_connectivity(Url, Timeout) -> case emqx_http_lib:uri_parse(Url) of {ok, #{host := Host, port := Port}} -> - tcp_connectivity(str(Host), Port, Timeout); + tcp_connectivity(Host, Port, Timeout); {error, Reason} -> {error, Reason} end. @@ -220,7 +220,7 @@ tcp_connectivity(Host, Port) -> Timeout :: integer()) -> ok | {error, Reason :: term()}). tcp_connectivity(Host, Port, Timeout) -> - case gen_tcp:connect(Host, Port, [], Timeout) of + case gen_tcp:connect(Host, Port, emqx_misc:ipv6_probe([]), Timeout) of {ok, Sock} -> gen_tcp:close(Sock), ok; {error, Reason} -> {error, Reason} end. diff --git a/apps/emqx_rule_engine/src/emqx_rule_validator.erl b/apps/emqx_rule_engine/src/emqx_rule_validator.erl index 252145b84..8f39d2d1c 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_validator.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_validator.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index dae6f5edb..2e82ef0cd 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -706,7 +706,7 @@ t_disable_rule(_Config) -> types=[], params_spec = #{}, title = #{en => <<"Simple Action">>}, description = #{en => <<"Simple Action">>}}), - {ok, #rule{actions = [#action_instance{id = ActInsId0}]}} = emqx_rule_engine:create_rule( + {ok, #rule{actions = [#action_instance{}]}} = emqx_rule_engine:create_rule( #{id => <<"simple_rule_2">>, rawsql => <<"select * from \"t/#\"">>, actions => [#{name => 'simple_action_2', args => #{}}] diff --git a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl index 4060fb49d..32421df32 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -647,6 +647,23 @@ t_now_timestamp_1(_) -> apply_func(now_timestamp, [atom_to_binary(Unit, utf8)]))) || Unit <- [second,millisecond,microsecond,nanosecond]]. +t_unix_ts_to_rfc3339(_) -> + [begin + BUnit = atom_to_binary(Unit, utf8), + Epoch = apply_func(now_timestamp, [BUnit]), + DateTime = apply_func(unix_ts_to_rfc3339, [Epoch, BUnit]), + ?assertEqual(Epoch, + calendar:rfc3339_to_system_time(binary_to_list(DateTime), [{unit, Unit}])) + end || Unit <- [second,millisecond,microsecond,nanosecond]]. + +t_rfc3339_to_unix_ts(_) -> + [begin + BUnit = atom_to_binary(Unit, utf8), + Epoch = apply_func(now_timestamp, [BUnit]), + DateTime = apply_func(unix_ts_to_rfc3339, [Epoch, BUnit]), + ?assertEqual(Epoch, emqx_rule_funcs:rfc3339_to_unix_ts(DateTime, BUnit)) + end || Unit <- [second,millisecond,microsecond,nanosecond]]. + %%------------------------------------------------------------------------------ %% Utility functions %%------------------------------------------------------------------------------ diff --git a/apps/emqx_rule_engine/test/emqx_rule_id_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_id_SUITE.erl index d8445e4b5..a1e1c4f8c 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_id_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_id_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_rule_engine/test/emqx_rule_maps_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_maps_SUITE.erl index 3d4adb364..ffb1440d8 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_maps_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_maps_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_rule_engine/test/emqx_rule_metrics_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_metrics_SUITE.erl index 0fa8b6625..e7c543c91 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_metrics_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_metrics_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_rule_engine/test/emqx_rule_monitor_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_monitor_SUITE.erl index fe44d8df7..67c59e26c 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_monitor_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_monitor_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_rule_engine/test/emqx_rule_registry_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_registry_SUITE.erl index 5fb1505b1..cbd69c878 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_registry_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_registry_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_rule_engine/test/emqx_rule_utils_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_utils_SUITE.erl index 90b6ff86f..002d47c8c 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_utils_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_utils_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_rule_engine/test/emqx_rule_validator_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_validator_SUITE.erl index 8d442a5a6..fdd7857d4 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_validator_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_validator_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_sasl/include/emqx_sasl.hrl b/apps/emqx_sasl/include/emqx_sasl.hrl index 04504fd0a..a1658f2b8 100644 --- a/apps/emqx_sasl/include/emqx_sasl.hrl +++ b/apps/emqx_sasl/include/emqx_sasl.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_sasl/src/emqx_sasl.erl b/apps/emqx_sasl/src/emqx_sasl.erl index 711518ce2..7010ae902 100644 --- a/apps/emqx_sasl/src/emqx_sasl.erl +++ b/apps/emqx_sasl/src/emqx_sasl.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_sasl/src/emqx_sasl_api.erl b/apps/emqx_sasl/src/emqx_sasl_api.erl index 1f77cd9e7..a69ac557d 100644 --- a/apps/emqx_sasl/src/emqx_sasl_api.erl +++ b/apps/emqx_sasl/src/emqx_sasl_api.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -223,11 +223,5 @@ pipeline([Fun | More], Params) -> {error, Reason} end. --if(?OTP_RELEASE >= 23). urldecode(S) -> - [{R, _}] = uri_string:dissect_query(S), R. --else. -urldecode(S) -> - http_uri:decode(S). --endif. - + emqx_http_lib:uri_decode(S). diff --git a/apps/emqx_sasl/src/emqx_sasl_app.erl b/apps/emqx_sasl/src/emqx_sasl_app.erl index 9d0b5955c..0ce3dc076 100644 --- a/apps/emqx_sasl/src/emqx_sasl_app.erl +++ b/apps/emqx_sasl/src/emqx_sasl_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_sasl/src/emqx_sasl_cli.erl b/apps/emqx_sasl/src/emqx_sasl_cli.erl index 375fc60ec..01e862373 100644 --- a/apps/emqx_sasl/src/emqx_sasl_cli.erl +++ b/apps/emqx_sasl/src/emqx_sasl_cli.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_sasl/src/emqx_sasl_scram.erl b/apps/emqx_sasl/src/emqx_sasl_scram.erl index ba27f7784..beb415568 100644 --- a/apps/emqx_sasl/src/emqx_sasl_scram.erl +++ b/apps/emqx_sasl/src/emqx_sasl_scram.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_sasl/test/emqx_sasl_scram_SUITE.erl b/apps/emqx_sasl/test/emqx_sasl_scram_SUITE.erl index 5d54aa921..202b0da2a 100644 --- a/apps/emqx_sasl/test/emqx_sasl_scram_SUITE.erl +++ b/apps/emqx_sasl/test/emqx_sasl_scram_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_sn/include/emqx_sn.hrl b/apps/emqx_sn/include/emqx_sn.hrl index afeda100a..29c5b2c86 100644 --- a/apps/emqx_sn/include/emqx_sn.hrl +++ b/apps/emqx_sn/include/emqx_sn.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_sn/src/emqx_sn_app.erl b/apps/emqx_sn/src/emqx_sn_app.erl index 8a2f15865..d39e8b34f 100644 --- a/apps/emqx_sn/src/emqx_sn_app.erl +++ b/apps/emqx_sn/src/emqx_sn_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_sn/src/emqx_sn_asleep_timer.erl b/apps/emqx_sn/src/emqx_sn_asleep_timer.erl index de56cd585..56a63ee2f 100644 --- a/apps/emqx_sn/src/emqx_sn_asleep_timer.erl +++ b/apps/emqx_sn/src/emqx_sn_asleep_timer.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_sn/src/emqx_sn_broadcast.erl b/apps/emqx_sn/src/emqx_sn_broadcast.erl index 747a34515..a1630b844 100644 --- a/apps/emqx_sn/src/emqx_sn_broadcast.erl +++ b/apps/emqx_sn/src/emqx_sn_broadcast.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_sn/src/emqx_sn_frame.erl b/apps/emqx_sn/src/emqx_sn_frame.erl index 5583a246e..eed32803d 100644 --- a/apps/emqx_sn/src/emqx_sn_frame.erl +++ b/apps/emqx_sn/src/emqx_sn_frame.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% %% Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/apps/emqx_sn/src/emqx_sn_gateway.erl b/apps/emqx_sn/src/emqx_sn_gateway.erl index bd343e038..f24860e6b 100644 --- a/apps/emqx_sn/src/emqx_sn_gateway.erl +++ b/apps/emqx_sn/src/emqx_sn_gateway.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -32,7 +32,9 @@ , stats/1 ]). --export([call/2]). +-export([ call/2 + , call/3 + ]). %% SUB/UNSUB Asynchronously, called by plugins. -export([ subscribe/2 @@ -65,6 +67,8 @@ -type(maybe(T) :: T | undefined). +-type(pending_msgs() :: #{integer() => [#mqtt_sn_message{}]}). + -record(will_msg, {retain = false :: boolean(), qos = ?QOS_0 :: emqx_mqtt_types:qos(), topic :: maybe(binary()), @@ -90,7 +94,8 @@ stats_timer :: maybe(reference()), idle_timeout :: integer(), enable_qos3 = false :: boolean(), - has_pending_pingresp = false :: boolean() + has_pending_pingresp = false :: boolean(), + pending_topic_ids = #{} :: pending_msgs() }). -define(INFO_KEYS, [socktype, peername, sockname, sockstate]). %, active_n]). @@ -178,8 +183,8 @@ init([{_, SockPid, Sock}, Peername, Options]) -> callback_mode() -> state_functions. idle(cast, {incoming, ?SN_SEARCHGW_MSG(_Radius)}, State = #state{gwid = GwId}) -> - send_message(?SN_GWINFO_MSG(GwId, <<>>), State), - {keep_state, State, State#state.idle_timeout}; + State0 = send_message(?SN_GWINFO_MSG(GwId, <<>>), State), + {keep_state, State0, State0#state.idle_timeout}; idle(cast, {incoming, ?SN_CONNECT_MSG(Flags, _ProtoId, Duration, ClientId)}, State) -> #mqtt_sn_flags{will = Will, clean_start = CleanStart} = Flags, @@ -202,7 +207,7 @@ idle(cast, {incoming, ?SN_PUBLISH_MSG(#mqtt_sn_flags{qos = ?QOS_NEG1, }, TopicId, _MsgId, Data)}, State = #state{clientid = ClientId, registry = Registry}) -> TopicName = case (TopicIdType =:= ?SN_SHORT_TOPIC) of - false -> emqx_sn_registry:lookup_topic(Registry, ClientId, TopicId); + false -> emqx_sn_registry:lookup_topic(Registry, self(), TopicId); true -> <> end, _ = case TopicName =/= undefined of @@ -219,12 +224,10 @@ idle(cast, {incoming, PingReq = ?SN_PINGREQ_MSG(_ClientId)}, State) -> handle_ping(PingReq, State); idle(cast, {outgoing, Packet}, State) -> - ok = handle_outgoing(Packet, State), - {keep_state, State}; + {keep_state, handle_outgoing(Packet, State)}; idle(cast, {connack, ConnAck}, State) -> - ok = handle_outgoing(ConnAck, State), - {next_state, connected, State}; + {next_state, connected, handle_outgoing(ConnAck, State)}; idle(timeout, _Timeout, State) -> stop(idle_timeout, State); @@ -243,8 +246,8 @@ wait_for_will_topic(cast, {incoming, ?SN_WILLTOPIC_EMPTY_MSG}, State = #state{co wait_for_will_topic(cast, {incoming, ?SN_WILLTOPIC_MSG(Flags, Topic)}, State) -> #mqtt_sn_flags{qos = QoS, retain = Retain} = Flags, WillMsg = #will_msg{retain = Retain, qos = QoS, topic = Topic}, - send_message(?SN_WILLMSGREQ_MSG(), State), - {next_state, wait_for_will_msg, State#state{will_msg = WillMsg}}; + State0 = send_message(?SN_WILLMSGREQ_MSG(), State), + {next_state, wait_for_will_msg, State0#state{will_msg = WillMsg}}; wait_for_will_topic(cast, {incoming, ?SN_ADVERTISE_MSG(_GwId, _Radius)}, _State) -> % ignore @@ -254,12 +257,10 @@ wait_for_will_topic(cast, {incoming, ?SN_CONNECT_MSG(Flags, _ProtoId, Duration, do_2nd_connect(Flags, Duration, ClientId, State); wait_for_will_topic(cast, {outgoing, Packet}, State) -> - ok = handle_outgoing(Packet, State), - {keep_state, State}; + {keep_state, handle_outgoing(Packet, State)}; wait_for_will_topic(cast, {connack, ConnAck}, State) -> - ok = handle_outgoing(ConnAck, State), - {next_state, connected, State}; + {next_state, connected, handle_outgoing(ConnAck, State)}; wait_for_will_topic(cast, Event, _State) -> ?LOG(error, "wait_for_will_topic UNEXPECTED Event: ~p", [Event]), @@ -282,19 +283,18 @@ wait_for_will_msg(cast, {incoming, ?SN_CONNECT_MSG(Flags, _ProtoId, Duration, Cl do_2nd_connect(Flags, Duration, ClientId, State); wait_for_will_msg(cast, {outgoing, Packet}, State) -> - ok = handle_outgoing(Packet, State), - {keep_state, State}; + {keep_state, handle_outgoing(Packet, State)}; wait_for_will_msg(cast, {connack, ConnAck}, State) -> - ok = handle_outgoing(ConnAck, State), - {next_state, connected, State}; + {next_state, connected, handle_outgoing(ConnAck, State)}; wait_for_will_msg(EventType, EventContent, State) -> handle_event(EventType, EventContent, wait_for_will_msg, State). connected(cast, {incoming, ?SN_REGISTER_MSG(_TopicId, MsgId, TopicName)}, State = #state{clientid = ClientId, registry = Registry}) -> - case emqx_sn_registry:register_topic(Registry, ClientId, TopicName) of + State0 = + case emqx_sn_registry:register_topic(Registry, self(), TopicName) of TopicId when is_integer(TopicId) -> ?LOG(debug, "register ClientId=~p, TopicName=~p, TopicId=~p", [ClientId, TopicName, TopicId]), send_message(?SN_REGACK_MSG(TopicId, MsgId, ?SN_RC_ACCEPTED), State); @@ -305,7 +305,7 @@ connected(cast, {incoming, ?SN_REGISTER_MSG(_TopicId, MsgId, TopicName)}, ?LOG(error, "wildcard topic can not be registered! ClientId=~p, TopicName=~p", [ClientId, TopicName]), send_message(?SN_REGACK_MSG(?SN_INVALID_TOPIC_ID, MsgId, ?SN_RC_NOT_SUPPORTED), State) end, - {keep_state, State}; + {keep_state, State0}; connected(cast, {incoming, ?SN_PUBLISH_MSG(Flags, TopicId, MsgId, Data)}, State = #state{enable_qos3 = EnableQoS3}) -> @@ -337,19 +337,19 @@ connected(cast, {incoming, ?SN_UNSUBSCRIBE_MSG(Flags, MsgId, TopicId)}, State) - connected(cast, {incoming, PingReq = ?SN_PINGREQ_MSG(_ClientId)}, State) -> handle_ping(PingReq, State); -connected(cast, {incoming, ?SN_REGACK_MSG(_TopicId, _MsgId, ?SN_RC_ACCEPTED)}, State) -> - {keep_state, State}; +connected(cast, {incoming, ?SN_REGACK_MSG(TopicId, _MsgId, ?SN_RC_ACCEPTED)}, State) -> + {keep_state, replay_no_reg_pending_publishes(TopicId, State)}; connected(cast, {incoming, ?SN_REGACK_MSG(TopicId, MsgId, ReturnCode)}, State) -> ?LOG(error, "client does not accept register TopicId=~p, MsgId=~p, ReturnCode=~p", [TopicId, MsgId, ReturnCode]), {keep_state, State}; connected(cast, {incoming, ?SN_DISCONNECT_MSG(Duration)}, State) -> - ok = send_message(?SN_DISCONNECT_MSG(undefined), State), + State0 = send_message(?SN_DISCONNECT_MSG(undefined), State), case Duration of undefined -> - handle_incoming(?DISCONNECT_PACKET(), State); - _Other -> goto_asleep_state(Duration, State) + handle_incoming(?DISCONNECT_PACKET(), State0); + _Other -> goto_asleep_state(Duration, State0) end; connected(cast, {incoming, ?SN_WILLTOPICUPD_MSG(Flags, Topic)}, State = #state{will_msg = WillMsg}) -> @@ -357,12 +357,12 @@ connected(cast, {incoming, ?SN_WILLTOPICUPD_MSG(Flags, Topic)}, State = #state{w undefined -> undefined; _ -> update_will_topic(WillMsg, Flags, Topic) end, - send_message(?SN_WILLTOPICRESP_MSG(0), State), - {keep_state, State#state{will_msg = WillMsg1}}; + State0 = send_message(?SN_WILLTOPICRESP_MSG(0), State), + {keep_state, State0#state{will_msg = WillMsg1}}; connected(cast, {incoming, ?SN_WILLMSGUPD_MSG(Payload)}, State = #state{will_msg = WillMsg}) -> - ok = send_message(?SN_WILLMSGRESP_MSG(0), State), - {keep_state, State#state{will_msg = update_will_msg(WillMsg, Payload)}}; + State0 = send_message(?SN_WILLMSGRESP_MSG(0), State), + {keep_state, State0#state{will_msg = update_will_msg(WillMsg, Payload)}}; connected(cast, {incoming, ?SN_ADVERTISE_MSG(_GwId, _Radius)}, State) -> % ignore @@ -372,17 +372,14 @@ connected(cast, {incoming, ?SN_CONNECT_MSG(Flags, _ProtoId, Duration, ClientId)} do_2nd_connect(Flags, Duration, ClientId, State); connected(cast, {outgoing, Packet}, State) -> - ok = handle_outgoing(Packet, State), - {keep_state, State}; + {keep_state, handle_outgoing(Packet, State)}; %% XXX: It's so strange behavoir!!! connected(cast, {connack, ConnAck}, State) -> - ok = handle_outgoing(ConnAck, State), - {keep_state, State}; + {keep_state, handle_outgoing(ConnAck, State)}; connected(cast, {shutdown, Reason, Packet}, State) -> - ok = handle_outgoing(Packet, State), - stop(Reason, State); + stop(Reason, handle_outgoing(Packet, State)); connected(cast, {shutdown, Reason}, State) -> stop(Reason, State); @@ -395,12 +392,12 @@ connected(EventType, EventContent, State) -> handle_event(EventType, EventContent, connected, State). asleep(cast, {incoming, ?SN_DISCONNECT_MSG(Duration)}, State) -> - ok = send_message(?SN_DISCONNECT_MSG(undefined), State), + State0 = send_message(?SN_DISCONNECT_MSG(undefined), State), case Duration of undefined -> - handle_incoming(?DISCONNECT_PACKET(), State); + handle_incoming(?DISCONNECT_PACKET(), State0); _Other -> - goto_asleep_state(Duration, State) + goto_asleep_state(Duration, State0) end; asleep(cast, {incoming, ?SN_PINGREQ_MSG(undefined)}, State) -> @@ -409,13 +406,13 @@ asleep(cast, {incoming, ?SN_PINGREQ_MSG(undefined)}, State) -> asleep(cast, {incoming, ?SN_PINGREQ_MSG(ClientIdPing)}, State = #state{clientid = ClientId, channel = Channel}) -> + inc_ping_counter(), case ClientIdPing of ClientId -> - inc_ping_counter(), case emqx_session:replay(emqx_channel:get_session(Channel)) of {ok, [], Session0} -> - send_message(?SN_PINGRESP_MSG(), State), - {keep_state, State#state{ + State0 = send_message(?SN_PINGRESP_MSG(), State), + {keep_state, State0#state{ channel = emqx_channel:set_session(Session0, Channel)}}; {ok, Publishes, Session0} -> {Packets, Channel1} = emqx_channel:do_deliver(Publishes, @@ -447,14 +444,13 @@ asleep(cast, {incoming, ?SN_CONNECT_MSG(_Flags, _ProtoId, _Duration, _ClientId)} % keepalive timer may timeout in asleep state and delete itself, need to restart keepalive % TODO: Fixme later. %% self() ! {keepalive, start, Interval}, - send_connack(State), - {next_state, connected, State}; + {next_state, connected, send_connack(State)}; asleep(EventType, EventContent, State) -> handle_event(EventType, EventContent, asleep, State). -awake(cast, {incoming, ?SN_REGACK_MSG(_TopicId, _MsgId, ?SN_RC_ACCEPTED)}, State) -> - {keep_state, State}; +awake(cast, {incoming, ?SN_REGACK_MSG(TopicId, _MsgId, ?SN_RC_ACCEPTED)}, State) -> + {keep_state, replay_no_reg_pending_publishes(TopicId, State)}; awake(cast, {incoming, ?SN_REGACK_MSG(TopicId, MsgId, ReturnCode)}, State) -> ?LOG(error, "client does not accept register TopicId=~p, MsgId=~p, ReturnCode=~p", @@ -465,8 +461,7 @@ awake(cast, {incoming, PingReq = ?SN_PINGREQ_MSG(_ClientId)}, State) -> handle_ping(PingReq, State); awake(cast, {outgoing, Packet}, State) -> - ok = handle_outgoing(Packet, State), - {keep_state, State}; + {keep_state, handle_outgoing(Packet, State)}; awake(cast, {incoming, ?SN_PUBACK_MSG(TopicId, MsgId, ReturnCode)}, State) -> do_puback(TopicId, MsgId, ReturnCode, awake, State); @@ -480,8 +475,8 @@ awake(cast, try_goto_asleep, State=#state{channel = Channel, Inflight = emqx_session:info(inflight, emqx_channel:get_session(Channel)), case emqx_inflight:size(Inflight) of 0 when PingPending =:= true -> - send_message(?SN_PINGRESP_MSG(), State), - goto_asleep_state(State#state{has_pending_pingresp = false}); + State0 = send_message(?SN_PINGRESP_MSG(), State), + goto_asleep_state(State0#state{has_pending_pingresp = false}); 0 when PingPending =:= false -> goto_asleep_state(State); _Size -> @@ -497,13 +492,13 @@ handle_event({call, From}, Req, _StateName, State) -> gen_server:reply(From, Reply), {keep_state, NState}; {stop, Reason, Reply, NState} -> - case NState#state.sockstate of + State0 = case NState#state.sockstate of running -> send_message(?SN_DISCONNECT_MSG(undefined), NState); - _ -> ok + _ -> NState end, gen_server:reply(From, Reply), - stop(Reason, NState) + stop(Reason, State0) end; handle_event(info, {datagram, SockPid, Data}, StateName, @@ -524,9 +519,10 @@ handle_event(info, {datagram, SockPid, Data}, StateName, end; handle_event(info, {deliver, _Topic, Msg}, asleep, - State = #state{channel = Channel}) -> + State = #state{channel = Channel, pending_topic_ids = Pendings}) -> % section 6.14, Support of sleeping clients - ?LOG(debug, "enqueue downlink message in asleep state Msg=~0p", [Msg]), + ?LOG(debug, "enqueue downlink message in asleep state, msg: ~0p, pending_topic_ids: ~0p", + [Msg, Pendings]), Session = emqx_session:enqueue(Msg, emqx_channel:get_session(Channel)), {keep_state, State#state{channel = emqx_channel:set_session(Session, Channel)}}; @@ -535,8 +531,7 @@ handle_event(info, Deliver = {deliver, _Topic, _Msg}, _StateName, handle_return(emqx_channel:handle_deliver([Deliver], Channel), State); handle_event(info, {redeliver, {?PUBREL, MsgId}}, _StateName, State) -> - send_message(?SN_PUBREC_MSG(?SN_PUBREL, MsgId), State), - {keep_state, State}; + {keep_state, send_message(?SN_PUBREC_MSG(?SN_PUBREL, MsgId), State)}; %% FIXME: Is not unused in v4.x handle_event(info, {timeout, TRef, emit_stats}, _StateName, @@ -586,10 +581,9 @@ handle_event(EventType, EventContent, StateName, State) -> [StateName, {EventType, EventContent}]), {keep_state, State}. -terminate(Reason, _StateName, #state{clientid = ClientId, - channel = Channel, +terminate(Reason, _StateName, #state{channel = Channel, registry = Registry}) -> - emqx_sn_registry:unregister_topic(Registry, ClientId), + emqx_sn_registry:unregister_topic(Registry, self()), case Channel =:= undefined of true -> ok; false -> emqx_channel:terminate(Reason, Channel) @@ -633,8 +627,7 @@ handle_return({shutdown, Reason, NChannel}, State, _AddEvents) -> stop(Reason, State#state{channel = NChannel}); handle_return({shutdown, Reason, OutPacket, NChannel}, State, _AddEvents) -> NState = State#state{channel = NChannel}, - ok = handle_outgoing(OutPacket, NState), - stop(Reason, NState). + stop(Reason, handle_outgoing(OutPacket, NState)). outgoing_events(Actions) -> lists:map(fun outgoing_event/1, Actions). @@ -692,15 +685,18 @@ stats(#state{socket = Socket, channel = Channel}) -> lists:append([SockStats, ConnStats, ChanStats, ProcStats]). call(Pid, Req) -> - gen_server:call(Pid, Req, infinity). + call(Pid, Req, infinity). + +call(Pid, Req, Timeout) -> + gen_server:call(Pid, Req, Timeout). %%-------------------------------------------------------------------- %% Internal Functions %%-------------------------------------------------------------------- handle_ping(_PingReq, State) -> - ok = send_message(?SN_PINGRESP_MSG(), State), + State0 = send_message(?SN_PINGRESP_MSG(), State), inc_ping_counter(), - {keep_state, State}. + {keep_state, State0}. inc_ping_counter() -> inc_counter(recv_msg, 1). @@ -724,12 +720,10 @@ mqtt2sn(?UNSUBACK_PACKET(MsgId), _State)-> ?SN_UNSUBACK_MSG(MsgId); mqtt2sn(?PUBLISH_PACKET(QoS, Topic, PacketId, Payload), #state{registry = Registry}) -> - NewPacketId = if - QoS =:= ?QOS_0 -> 0; - true -> PacketId + NewPacketId = if QoS =:= ?QOS_0 -> 0; + true -> PacketId end, - ClientId = get(clientid), - {TopicIdType, TopicContent} = case emqx_sn_registry:lookup_topic_id(Registry, ClientId, Topic) of + {TopicIdType, TopicContent} = case emqx_sn_registry:lookup_topic_id(Registry, self(), Topic) of {predef, PredefTopicId} -> {?SN_PREDEFINED_TOPIC, PredefTopicId}; TopicId when is_integer(TopicId) -> @@ -766,13 +760,13 @@ send_connack(State) -> send_message(?SN_CONNACK_MSG(?SN_RC_ACCEPTED), State). send_message(Msg = #mqtt_sn_message{type = Type}, - #state{sockpid = SockPid, peername = Peername}) -> + State = #state{sockpid = SockPid, peername = Peername}) -> ?LOG(debug, "SEND ~s~n", [emqx_sn_frame:format(Msg)]), inc_outgoing_stats(Type), Data = emqx_sn_frame:serialize(Msg), ok = emqx_metrics:inc('bytes.sent', iolist_size(Data)), SockPid ! {datagram, Peername, Data}, - ok. + State. goto_asleep_state(State) -> goto_asleep_state(undefined, State). @@ -790,9 +784,10 @@ stop(Reason, State) -> ?LOG(stop_log_level(Reason), "stop due to ~p", [Reason]), case Reason of %% FIXME: The Will-Msg should publish when a Session terminated! - asleep_timeout -> do_publish_will(State); - keepalive_timeout -> do_publish_will(State); - _ -> ok + Reason when Reason =:= normal -> + ok; + _ -> + do_publish_will(State) end, {stop, {shutdown, Reason}, State}. @@ -830,10 +825,9 @@ do_connect(ClientId, CleanStart, WillFlag, Duration, State) -> keepalive = Duration, properties = OnlyOneInflight }, - put(clientid, ClientId), case WillFlag of - true -> send_message(?SN_WILLTOPICREQ_MSG(), State), - NState = State#state{connpkt = ConnPkt, + true -> State0 = send_message(?SN_WILLTOPICREQ_MSG(), State), + NState = State0#state{connpkt = ConnPkt, clientid = ClientId, keepalive_interval = Duration }, @@ -847,7 +841,6 @@ do_connect(ClientId, CleanStart, WillFlag, Duration, State) -> do_2nd_connect(Flags, Duration, ClientId, State = #state{sockname = Sockname, peername = Peername, - clientid = OldClientId, registry = Registry, channel = Channel}) -> emqx_logger:set_metadata_clientid(ClientId), @@ -855,7 +848,7 @@ do_2nd_connect(Flags, Duration, ClientId, State = #state{sockname = Sockname, NChannel = case CleanStart of true -> emqx_channel:terminate(normal, Channel), - emqx_sn_registry:unregister_topic(Registry, OldClientId), + emqx_sn_registry:unregister_topic(Registry, self()), emqx_channel:init(#{socktype => udp, sockname => Sockname, peername => Peername, @@ -868,14 +861,14 @@ do_2nd_connect(Flags, Duration, ClientId, State = #state{sockname = Sockname, do_connect(ClientId, CleanStart, Will, Duration, NState). handle_subscribe(?SN_NORMAL_TOPIC, TopicName, QoS, MsgId, - State=#state{clientid = ClientId, registry = Registry}) -> - case emqx_sn_registry:register_topic(Registry, ClientId, TopicName) of + State=#state{registry = Registry}) -> + case emqx_sn_registry:register_topic(Registry, self(), TopicName) of {error, too_large} -> - ok = send_message(?SN_SUBACK_MSG(#mqtt_sn_flags{qos = QoS}, + State0 = send_message(?SN_SUBACK_MSG(#mqtt_sn_flags{qos = QoS}, ?SN_INVALID_TOPIC_ID, MsgId, ?SN_RC_INVALID_TOPIC_ID), State), - {keep_state, State}; + {keep_state, State0}; {error, wildcard_topic} -> proto_subscribe(TopicName, QoS, MsgId, ?SN_INVALID_TOPIC_ID, State); NewTopicId when is_integer(NewTopicId) -> @@ -883,14 +876,14 @@ handle_subscribe(?SN_NORMAL_TOPIC, TopicName, QoS, MsgId, end; handle_subscribe(?SN_PREDEFINED_TOPIC, TopicId, QoS, MsgId, - State = #state{clientid = ClientId, registry = Registry}) -> - case emqx_sn_registry:lookup_topic(Registry, ClientId, TopicId) of + State = #state{registry = Registry}) -> + case emqx_sn_registry:lookup_topic(Registry, self(), TopicId) of undefined -> - ok = send_message(?SN_SUBACK_MSG(#mqtt_sn_flags{qos = QoS}, + State0 = send_message(?SN_SUBACK_MSG(#mqtt_sn_flags{qos = QoS}, TopicId, MsgId, ?SN_RC_INVALID_TOPIC_ID), State), - {next_state, connected, State}; + {next_state, connected, State0}; PredefinedTopic -> proto_subscribe(PredefinedTopic, QoS, MsgId, TopicId, State) end; @@ -903,21 +896,20 @@ handle_subscribe(?SN_SHORT_TOPIC, TopicId, QoS, MsgId, State) -> proto_subscribe(TopicName, QoS, MsgId, ?SN_INVALID_TOPIC_ID, State); handle_subscribe(_, _TopicId, QoS, MsgId, State) -> - ok = send_message(?SN_SUBACK_MSG(#mqtt_sn_flags{qos = QoS}, + State0 = send_message(?SN_SUBACK_MSG(#mqtt_sn_flags{qos = QoS}, ?SN_INVALID_TOPIC_ID, MsgId, ?SN_RC_INVALID_TOPIC_ID), State), - {keep_state, State}. + {keep_state, State0}. handle_unsubscribe(?SN_NORMAL_TOPIC, TopicId, MsgId, State) -> proto_unsubscribe(TopicId, MsgId, State); handle_unsubscribe(?SN_PREDEFINED_TOPIC, TopicId, MsgId, - State = #state{clientid = ClientId, registry = Registry}) -> - case emqx_sn_registry:lookup_topic(Registry, ClientId, TopicId) of + State = #state{registry = Registry}) -> + case emqx_sn_registry:lookup_topic(Registry, self(), TopicId) of undefined -> - ok = send_message(?SN_UNSUBACK_MSG(MsgId), State), - {keep_state, State}; + {keep_state, send_message(?SN_UNSUBACK_MSG(MsgId), State)}; PredefinedTopic -> proto_unsubscribe(PredefinedTopic, MsgId, State) end; @@ -930,38 +922,38 @@ handle_unsubscribe(?SN_SHORT_TOPIC, TopicId, MsgId, State) -> proto_unsubscribe(TopicName, MsgId, State); handle_unsubscribe(_, _TopicId, MsgId, State) -> - send_message(?SN_UNSUBACK_MSG(MsgId), State), - {keep_state, State}. + {keep_state, send_message(?SN_UNSUBACK_MSG(MsgId), State)}. do_publish(?SN_NORMAL_TOPIC, TopicName, Data, Flags, MsgId, State) -> %% XXX: Handle normal topic id as predefined topic id, to be compatible with paho mqtt-sn library <> = TopicName, do_publish(?SN_PREDEFINED_TOPIC, TopicId, Data, Flags, MsgId, State); do_publish(?SN_PREDEFINED_TOPIC, TopicId, Data, Flags, MsgId, - State=#state{clientid = ClientId, registry = Registry}) -> + State=#state{registry = Registry}) -> #mqtt_sn_flags{qos = QoS, dup = Dup, retain = Retain} = Flags, NewQoS = get_corrected_qos(QoS), - case emqx_sn_registry:lookup_topic(Registry, ClientId, TopicId) of + case emqx_sn_registry:lookup_topic(Registry, self(), TopicId) of undefined -> - (NewQoS =/= ?QOS_0) andalso send_message(?SN_PUBACK_MSG(TopicId, MsgId, ?SN_RC_INVALID_TOPIC_ID), State), - {keep_state, State}; + {keep_state, maybe_send_puback(NewQoS, TopicId, MsgId, ?SN_RC_INVALID_TOPIC_ID, + State)}; TopicName -> proto_publish(TopicName, Data, Dup, NewQoS, Retain, MsgId, TopicId, State) end; + do_publish(?SN_SHORT_TOPIC, STopicName, Data, Flags, MsgId, State) -> #mqtt_sn_flags{qos = QoS, dup = Dup, retain = Retain} = Flags, NewQoS = get_corrected_qos(QoS), <> = STopicName , case emqx_topic:wildcard(STopicName) of true -> - (NewQoS =/= ?QOS_0) andalso send_message(?SN_PUBACK_MSG(TopicId, MsgId, ?SN_RC_NOT_SUPPORTED), State), - {keep_state, State}; + {keep_state, maybe_send_puback(NewQoS, TopicId, MsgId, ?SN_RC_NOT_SUPPORTED, + State)}; false -> proto_publish(STopicName, Data, Dup, NewQoS, Retain, MsgId, TopicId, State) end; do_publish(_, TopicId, _Data, #mqtt_sn_flags{qos = QoS}, MsgId, State) -> - (QoS =/= ?QOS_0) andalso send_message(?SN_PUBACK_MSG(TopicId, MsgId, ?SN_RC_NOT_SUPPORTED), State), - {keep_state, State}. + {keep_state, maybe_send_puback(QoS, TopicId, MsgId, ?SN_RC_NOT_SUPPORTED, + State)}. do_publish_will(#state{will_msg = undefined}) -> ok; @@ -979,18 +971,17 @@ do_publish_will(#state{will_msg = WillMsg, clientid = ClientId}) -> ok. do_puback(TopicId, MsgId, ReturnCode, StateName, - State=#state{clientid = ClientId, registry = Registry}) -> + State=#state{registry = Registry}) -> case ReturnCode of ?SN_RC_ACCEPTED -> handle_incoming(?PUBACK_PACKET(MsgId), StateName, State); ?SN_RC_INVALID_TOPIC_ID -> - case emqx_sn_registry:lookup_topic(Registry, ClientId, TopicId) of - undefined -> ok; + case emqx_sn_registry:lookup_topic(Registry, self(), TopicId) of + undefined -> {keep_state, State}; TopicName -> %%notice that this TopicName maybe normal or predefined, %% involving the predefined topic name in register to enhance the gateway's robustness even inconsistent with MQTT-SN channels - send_register(TopicName, TopicId, MsgId, State), - {keep_state, State} + {keep_state, send_register(TopicName, TopicId, MsgId, State)} end; _ -> ?LOG(error, "CAN NOT handle PUBACK ReturnCode=~p", [ReturnCode]), @@ -1069,30 +1060,45 @@ channel_handle_in(Packet = ?PACKET(Type), #state{channel = Channel}) -> emqx_channel:handle_in(Packet, Channel). handle_outgoing(Packets, State) when is_list(Packets) -> - lists:foreach(fun(Packet) -> handle_outgoing(Packet, State) end, Packets); + lists:foldl(fun(Packet, State0) -> + handle_outgoing(Packet, State0) + end, State, Packets); -handle_outgoing(PubPkt = ?PUBLISH_PACKET(QoS, TopicName, PacketId, Payload), - State = #state{clientid = ClientId, registry = Registry}) -> - #mqtt_packet{header = #mqtt_packet_header{dup = Dup, retain = Retain}} = PubPkt, - MsgId = message_id(PacketId), - ?LOG(debug, "Handle outgoing: ~0p", [PubPkt]), - - (emqx_sn_registry:lookup_topic_id(Registry, ClientId, TopicName) == undefined) - andalso (byte_size(TopicName) =/= 2) - andalso register_and_notify_client(TopicName, Payload, Dup, QoS, - Retain, MsgId, ClientId, State), - - send_message(mqtt2sn(PubPkt, State), State); +handle_outgoing(PubPkt = ?PUBLISH_PACKET(_, TopicName, _, _), + State = #state{registry = Registry}) -> + ?LOG(debug, "Handle outgoing publish: ~0p", [PubPkt]), + TopicId = emqx_sn_registry:lookup_topic_id(Registry, self(), TopicName), + case (TopicId == undefined) andalso (byte_size(TopicName) =/= 2) of + true -> register_and_notify_client(PubPkt, State); + false -> send_message(mqtt2sn(PubPkt, State), State) + end; handle_outgoing(Packet, State) -> send_message(mqtt2sn(Packet, State), State). -register_and_notify_client(TopicName, Payload, Dup, QoS, Retain, MsgId, ClientId, - State = #state{registry = Registry}) -> - TopicId = emqx_sn_registry:register_topic(Registry, ClientId, TopicName), +cache_no_reg_publish_message(Pendings, TopicId, PubPkt, State) -> + ?LOG(debug, "cache non-registered publish message for topic-id: ~p, msg: ~0p, pendings: ~0p", + [TopicId, PubPkt, Pendings]), + Msgs = maps:get(pending_topic_ids, Pendings, []), + Pendings#{TopicId => Msgs ++ [mqtt2sn(PubPkt, State)]}. + +replay_no_reg_pending_publishes(TopicId, #state{pending_topic_ids = Pendings} = State0) -> + ?LOG(debug, "replay non-registered publish message for topic-id: ~p, pendings: ~0p", + [TopicId, Pendings]), + State = lists:foldl(fun(Msg, State1) -> + send_message(Msg, State1) + end, State0, maps:get(TopicId, Pendings, [])), + State#state{pending_topic_ids = maps:remove(TopicId, Pendings)}. + +register_and_notify_client(?PUBLISH_PACKET(QoS, TopicName, PacketId, Payload) = PubPkt, + State = #state{registry = Registry, pending_topic_ids = Pendings}) -> + MsgId = message_id(PacketId), + #mqtt_packet{header = #mqtt_packet_header{dup = Dup, retain = Retain}} = PubPkt, + TopicId = emqx_sn_registry:register_topic(Registry, self(), TopicName), ?LOG(debug, "Register TopicId=~p, TopicName=~p, Payload=~p, Dup=~p, QoS=~p, " "Retain=~p, MsgId=~p", [TopicId, TopicName, Payload, Dup, QoS, Retain, MsgId]), - send_register(TopicName, TopicId, MsgId, State). + NewPendings = cache_no_reg_publish_message(Pendings, TopicId, PubPkt, State), + send_register(TopicName, TopicId, MsgId, State#state{pending_topic_ids = NewPendings}). message_id(undefined) -> rand:uniform(16#FFFF); @@ -1125,3 +1131,8 @@ append(Replies, AddEvents) when is_list(Replies) -> Replies ++ AddEvents; append(Replies, AddEvents) -> [Replies] ++ AddEvents. + +maybe_send_puback(?QOS_0, _TopicId, _MsgId, _ReasonCode, State) -> + State; +maybe_send_puback(_QoS, TopicId, MsgId, ReasonCode, State) -> + send_message(?SN_PUBACK_MSG(TopicId, MsgId, ReasonCode), State). diff --git a/apps/emqx_sn/src/emqx_sn_registry.erl b/apps/emqx_sn/src/emqx_sn_registry.erl index 78e27ae01..fa17ebc27 100644 --- a/apps/emqx_sn/src/emqx_sn_registry.erl +++ b/apps/emqx_sn/src/emqx_sn_registry.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -60,11 +60,11 @@ start_link(Tab, PredefTopics) -> stop({_Tab, Pid}) -> gen_server:stop(Pid, normal, infinity). --spec(register_topic(registry(), binary(), binary()) -> integer() | {error, term()}). -register_topic({_, Pid}, ClientId, TopicName) when is_binary(TopicName) -> +-spec(register_topic(registry(), pid(), binary()) -> integer() | {error, term()}). +register_topic({_, Pid}, ClientPid, TopicName) when is_binary(TopicName) -> case emqx_topic:wildcard(TopicName) of false -> - gen_server:call(Pid, {register, ClientId, TopicName}); + gen_server:call(Pid, {register, ClientPid, TopicName}); %% TopicId: in case of “accepted” the value that will be used as topic %% id by the gateway when sending PUBLISH messages to the client (not %% relevant in case of subscriptions to a short topic name or to a topic @@ -72,22 +72,22 @@ register_topic({_, Pid}, ClientId, TopicName) when is_binary(TopicName) -> true -> {error, wildcard_topic} end. --spec(lookup_topic(registry(), binary(), pos_integer()) -> undefined | binary()). -lookup_topic({Tab, _Pid}, ClientId, TopicId) when is_integer(TopicId) -> +-spec(lookup_topic(registry(), pid(), pos_integer()) -> undefined | binary()). +lookup_topic({Tab, _Pid}, ClientPid, TopicId) when is_integer(TopicId) -> case lookup_element(Tab, {predef, TopicId}, 2) of undefined -> - lookup_element(Tab, {ClientId, TopicId}, 2); + lookup_element(Tab, {ClientPid, TopicId}, 2); Topic -> Topic end. --spec(lookup_topic_id(registry(), binary(), binary()) +-spec(lookup_topic_id(registry(), pid(), binary()) -> undefined | pos_integer() | {predef, integer()}). -lookup_topic_id({Tab, _Pid}, ClientId, TopicName) when is_binary(TopicName) -> +lookup_topic_id({Tab, _Pid}, ClientPid, TopicName) when is_binary(TopicName) -> case lookup_element(Tab, {predef, TopicName}, 2) of undefined -> - lookup_element(Tab, {ClientId, TopicName}, 2); + lookup_element(Tab, {ClientPid, TopicName}, 2); TopicId -> {predef, TopicId} end. @@ -96,17 +96,17 @@ lookup_topic_id({Tab, _Pid}, ClientId, TopicName) when is_binary(TopicName) -> lookup_element(Tab, Key, Pos) -> try ets:lookup_element(Tab, Key, Pos) catch error:badarg -> undefined end. --spec(unregister_topic(registry(), binary()) -> ok). -unregister_topic({_Tab, Pid}, ClientId) -> - gen_server:call(Pid, {unregister, ClientId}). +-spec(unregister_topic(registry(), pid()) -> ok). +unregister_topic({_Tab, Pid}, ClientPid) -> + gen_server:call(Pid, {unregister, ClientPid}). %%----------------------------------------------------------------------------- init([Tab, PredefTopics]) -> %% {predef, TopicId} -> TopicName %% {predef, TopicName} -> TopicId - %% {ClientId, TopicId} -> TopicName - %% {ClientId, TopicName} -> TopicId + %% {ClientPid, TopicId} -> TopicName + %% {ClientPid, TopicName} -> TopicId _ = ets:new(Tab, [set, public, named_table, {read_concurrency, true}]), MaxPredefId = lists:foldl( fun({TopicId, TopicName}, AccId) -> @@ -116,27 +116,27 @@ init([Tab, PredefTopics]) -> end, 0, PredefTopics), {ok, #state{tab = Tab, max_predef_topic_id = MaxPredefId}}. -handle_call({register, ClientId, TopicName}, _From, +handle_call({register, ClientPid, TopicName}, _From, State = #state{tab = Tab, max_predef_topic_id = PredefId}) -> - case lookup_topic_id({Tab, self()}, ClientId, TopicName) of + case lookup_topic_id({Tab, self()}, ClientPid, TopicName) of {predef, PredefTopicId} when is_integer(PredefTopicId) -> {reply, PredefTopicId, State}; TopicId when is_integer(TopicId) -> {reply, TopicId, State}; undefined -> - case next_topic_id(Tab, PredefId, ClientId) of + case next_topic_id(Tab, PredefId, ClientPid) of TopicId when TopicId >= 16#FFFF -> {reply, {error, too_large}, State}; TopicId -> - _ = ets:insert(Tab, {{ClientId, next_topic_id}, TopicId + 1}), - _ = ets:insert(Tab, {{ClientId, TopicName}, TopicId}), - _ = ets:insert(Tab, {{ClientId, TopicId}, TopicName}), + _ = ets:insert(Tab, {{ClientPid, next_topic_id}, TopicId + 1}), + _ = ets:insert(Tab, {{ClientPid, TopicName}, TopicId}), + _ = ets:insert(Tab, {{ClientPid, TopicId}, TopicName}), {reply, TopicId, State} end end; -handle_call({unregister, ClientId}, _From, State = #state{tab = Tab}) -> - ets:match_delete(Tab, {{ClientId, '_'}, '_'}), +handle_call({unregister, ClientPid}, _From, State = #state{tab = Tab}) -> + ets:match_delete(Tab, {{ClientPid, '_'}, '_'}), {reply, ok, State}; handle_call(Req, _From, State) -> @@ -159,8 +159,8 @@ code_change(_OldVsn, State, _Extra) -> %%----------------------------------------------------------------------------- -next_topic_id(Tab, PredefId, ClientId) -> - case ets:lookup(Tab, {ClientId, next_topic_id}) of +next_topic_id(Tab, PredefId, ClientPid) -> + case ets:lookup(Tab, {ClientPid, next_topic_id}) of [{_, Id}] -> Id; [] -> PredefId + 1 end. diff --git a/apps/emqx_sn/src/emqx_sn_sup.erl b/apps/emqx_sn/src/emqx_sn_sup.erl index b165e2d36..817aa4d06 100644 --- a/apps/emqx_sn/src/emqx_sn_sup.erl +++ b/apps/emqx_sn/src/emqx_sn_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_sn/test/emqx_sn_frame_SUITE.erl b/apps/emqx_sn/test/emqx_sn_frame_SUITE.erl index 607af0acb..85042a4be 100644 --- a/apps/emqx_sn/test/emqx_sn_frame_SUITE.erl +++ b/apps/emqx_sn/test/emqx_sn_frame_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl b/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl index b2999e442..746d0f4a1 100644 --- a/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl +++ b/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -808,25 +808,34 @@ t_publish_qos2_case03(_) -> t_will_case01(_) -> QoS = 1, Duration = 1, + WillMsg = <<10, 11, 12, 13, 14>>, + WillTopic = <<"abc">>, {ok, Socket} = gen_udp:open(0, [binary]), ClientId = <<"test">>, + + ok = emqx_broker:subscribe(WillTopic), + send_connect_msg_with_will(Socket, Duration, ClientId), ?assertEqual(<<2, ?SN_WILLTOPICREQ>>, receive_response(Socket)), - send_willtopic_msg(Socket, <<"abc">>, QoS), + send_willtopic_msg(Socket, WillTopic, QoS), ?assertEqual(<<2, ?SN_WILLMSGREQ>>, receive_response(Socket)), - send_willmsg_msg(Socket, <<10, 11, 12, 13, 14>>), + send_willmsg_msg(Socket, WillMsg), ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_pingreq_msg(Socket, undefined), ?assertEqual(<<2, ?SN_PINGRESP>>, receive_response(Socket)), % wait udp client keepalive timeout - timer:sleep(10000), - - receive_response(Socket), % ignore PUBACK + timer:sleep(2000), + receive + {deliver, WillTopic, #message{payload = WillMsg}} -> ok; + Msg -> ct:print("recevived --- unex: ~p", [Msg]) + after + 1000 -> ct:fail(wait_willmsg_timeout) + end, send_disconnect_msg(Socket, undefined), ?assertEqual(udp_receive_timeout, receive_response(Socket)), @@ -1093,10 +1102,12 @@ t_asleep_test04_to_awake_qos1_dl_msg(_) -> %% send downlink data in asleep state. This message should be send to device once it wake up Payload1 = <<55, 66, 77, 88, 99>>, + Payload2 = <<55, 66, 77, 88, 100>>, {ok, C} = emqtt:start_link(), {ok, _} = emqtt:connect(C), {ok, _} = emqtt:publish(C, <<"a/b/c">>, Payload1, QoS), + {ok, _} = emqtt:publish(C, <<"a/b/c">>, Payload2, QoS), timer:sleep(100), ok = emqtt:disconnect(C), @@ -1105,21 +1116,32 @@ t_asleep_test04_to_awake_qos1_dl_msg(_) -> % goto awake state, receive downlink messages, and go back to asleep send_pingreq_msg(Socket, <<"test">>), - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - %% get REGISTER first, since this topic has never been registered - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - UdpData2 = receive_response(Socket), - {TopicIdNew, MsgId3} = check_register_msg_on_udp(<<"a/b/c">>, UdpData2), + %% 1. get REGISTER first, since this topic has never been registered + UdpData1 = receive_response(Socket), + {TopicIdNew, MsgId3} = check_register_msg_on_udp(<<"a/b/c">>, UdpData1), + + %% 2. but before we reply the REGACK, the sn-gateway should not send any PUBLISH + ?assertError(_, receive_publish(Socket)), + send_regack_msg(Socket, TopicIdNew, MsgId3), - UdpData = receive_response(Socket), - MsgId_udp = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicIdNew, Payload1}, UdpData), - send_puback_msg(Socket, TopicIdNew, MsgId_udp), + UdpData2 = receive_response(Socket), + MsgId_udp2 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicIdNew, Payload1}, UdpData2), + send_puback_msg(Socket, TopicIdNew, MsgId_udp2), + + UdpData3 = receive_response(Socket), + MsgId_udp3 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicIdNew, Payload2}, UdpData3), + send_puback_msg(Socket, TopicIdNew, MsgId_udp3), ?assertEqual(<<2, ?SN_PINGRESP>>, receive_response(Socket)), gen_udp:close(Socket). +receive_publish(Socket) -> + UdpData3 = receive_response(Socket, 1000), + <> = UdpData3, + <<_:8, ?SN_PUBLISH, _/binary>> = HeaderUdp. + t_asleep_test05_to_awake_qos1_dl_msg(_) -> QoS = 1, Duration = 5, diff --git a/apps/emqx_sn/test/emqx_sn_registry_SUITE.erl b/apps/emqx_sn/test/emqx_sn_registry_SUITE.erl index fbd32465d..8cce4592a 100644 --- a/apps/emqx_sn/test/emqx_sn_registry_SUITE.erl +++ b/apps/emqx_sn/test/emqx_sn_registry_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_sn/test/props/emqx_sn_proper_types.erl b/apps/emqx_sn/test/props/emqx_sn_proper_types.erl index f91684c8d..8d4dae357 100644 --- a/apps/emqx_sn/test/props/emqx_sn_proper_types.erl +++ b/apps/emqx_sn/test/props/emqx_sn_proper_types.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_sn/test/props/prop_emqx_sn_frame.erl b/apps/emqx_sn/test/props/prop_emqx_sn_frame.erl index ae82feb8a..0135ebac7 100644 --- a/apps/emqx_sn/test/props/prop_emqx_sn_frame.erl +++ b/apps/emqx_sn/test/props/prop_emqx_sn_frame.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_stomp/include/emqx_stomp.hrl b/apps/emqx_stomp/include/emqx_stomp.hrl index e2eeee7d8..a9cf2cf48 100644 --- a/apps/emqx_stomp/include/emqx_stomp.hrl +++ b/apps/emqx_stomp/include/emqx_stomp.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_stomp/src/emqx_stomp.erl b/apps/emqx_stomp/src/emqx_stomp.erl index 3d6396d43..9eafe3cf7 100644 --- a/apps/emqx_stomp/src/emqx_stomp.erl +++ b/apps/emqx_stomp/src/emqx_stomp.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_stomp/src/emqx_stomp_connection.erl b/apps/emqx_stomp/src/emqx_stomp_connection.erl index 740320489..d4e7f6475 100644 --- a/apps/emqx_stomp/src/emqx_stomp_connection.erl +++ b/apps/emqx_stomp/src/emqx_stomp_connection.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_stomp/src/emqx_stomp_frame.erl b/apps/emqx_stomp/src/emqx_stomp_frame.erl index 011a61f6b..fa9cb63a8 100644 --- a/apps/emqx_stomp/src/emqx_stomp_frame.erl +++ b/apps/emqx_stomp/src/emqx_stomp_frame.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_stomp/src/emqx_stomp_heartbeat.erl b/apps/emqx_stomp/src/emqx_stomp_heartbeat.erl index 79cc8f435..145359e53 100644 --- a/apps/emqx_stomp/src/emqx_stomp_heartbeat.erl +++ b/apps/emqx_stomp/src/emqx_stomp_heartbeat.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_stomp/src/emqx_stomp_protocol.erl b/apps/emqx_stomp/src/emqx_stomp_protocol.erl index 84e5bb8d6..cc5c28ce9 100644 --- a/apps/emqx_stomp/src/emqx_stomp_protocol.erl +++ b/apps/emqx_stomp/src/emqx_stomp_protocol.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_stomp/test/emqx_stomp_SUITE.erl b/apps/emqx_stomp/test/emqx_stomp_SUITE.erl index ca46762ed..9a5d9698e 100644 --- a/apps/emqx_stomp/test/emqx_stomp_SUITE.erl +++ b/apps/emqx_stomp/test/emqx_stomp_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_stomp/test/emqx_stomp_heartbeat_SUITE.erl b/apps/emqx_stomp/test/emqx_stomp_heartbeat_SUITE.erl index b3ea25aa1..cbced5f43 100644 --- a/apps/emqx_stomp/test/emqx_stomp_heartbeat_SUITE.erl +++ b/apps/emqx_stomp/test/emqx_stomp_heartbeat_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_web_hook/rebar.config b/apps/emqx_web_hook/rebar.config index aa44bfdb5..387972c9f 100644 --- a/apps/emqx_web_hook/rebar.config +++ b/apps/emqx_web_hook/rebar.config @@ -1,8 +1,6 @@ {plugins, [rebar3_proper]}. -{deps, - [{ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.1.2"}}} - ]}. +{deps, []}. {edoc_opts, [{preprocess, true}]}. {erl_opts, [warn_unused_vars, diff --git a/apps/emqx_web_hook/src/emqx_web_hook.erl b/apps/emqx_web_hook/src/emqx_web_hook.erl index 6c63d6d49..7af83d749 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook.erl +++ b/apps/emqx_web_hook/src/emqx_web_hook.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -378,8 +378,10 @@ encode_payload(Payload, base62) -> emqx_base62:encode(Payload); encode_payload(Payload, base64) -> base64:encode(Payload); encode_payload(Payload, plain) -> Payload. -stringfy(Term) when is_atom(Term); is_binary(Term) -> +stringfy(Term) when is_binary(Term) -> Term; +stringfy(Term) when is_atom(Term) -> + atom_to_binary(Term, utf8); stringfy(Term) -> unicode:characters_to_binary((io_lib:format("~0p", [Term]))). diff --git a/apps/emqx_web_hook/src/emqx_web_hook_actions.erl b/apps/emqx_web_hook/src/emqx_web_hook_actions.erl index f026434c6..c88f8f39b 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook_actions.erl +++ b/apps/emqx_web_hook/src/emqx_web_hook_actions.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -325,17 +325,18 @@ str(Atom) when is_atom(Atom) -> atom_to_list(Atom); str(Bin) when is_binary(Bin) -> binary_to_list(Bin). pool_opts(Params = #{<<"url">> := URL}, ResId) -> - {ok, #{host := Host0, + {ok, #{host := Host, port := Port, scheme := Scheme}} = emqx_http_lib:uri_parse(URL), PoolSize = maps:get(<<"pool_size">>, Params, 32), ConnectTimeout = cuttlefish_duration:parse(str(maps:get(<<"connect_timeout">>, Params, <<"5s">>))), - {Inet, Host} = parse_host(Host0), - TransportOpts = case Scheme =:= https of - true -> [Inet | get_ssl_opts(Params, ResId)]; - false -> [Inet] - end, + TransportOpts0 = + case Scheme =:= https of + true -> [get_ssl_opts(Params, ResId)]; + false -> [] + end, + TransportOpts = emqx_misc:ipv6_probe(TransportOpts0), Opts = case Scheme =:= https of true -> [{transport_opts, TransportOpts}, {transport, ssl}]; false -> [{transport_opts, TransportOpts}] @@ -354,17 +355,6 @@ pool_name(ResId) -> get_ssl_opts(Opts, ResId) -> [{ssl, true}, {ssl_opts, emqx_plugin_libs_ssl:save_files_return_opts(Opts, "rules", ResId)}]. -parse_host(Host) -> - case inet:parse_address(Host) of - {ok, Addr} when size(Addr) =:= 4 -> {inet, Addr}; - {ok, Addr} when size(Addr) =:= 8 -> {inet6, Addr}; - {error, einval} -> - case inet:getaddr(Host, inet6) of - {ok, _} -> {inet6, Host}; - {error, _} -> {inet, Host} - end - end. - test_http_connect(Conf) -> Url = fun() -> maps:get(<<"url">>, Conf) end, try diff --git a/apps/emqx_web_hook/src/emqx_web_hook_app.erl b/apps/emqx_web_hook/src/emqx_web_hook_app.erl index c75c7cb01..492af628d 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook_app.erl +++ b/apps/emqx_web_hook/src/emqx_web_hook_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -41,16 +41,15 @@ stop(_State) -> translate_env() -> {ok, URL} = application:get_env(?APP, url), - {ok, #{host := Host0, + {ok, #{host := Host, path := Path0, port := Port, scheme := Scheme}} = emqx_http_lib:uri_parse(URL), Path = path(Path0), - {Inet, Host} = parse_host(Host0), PoolSize = application:get_env(?APP, pool_size, 32), MoreOpts = case Scheme of http -> - [{transport_opts, [Inet]}]; + [{transport_opts, emqx_misc:ipv6_probe([])}]; https -> CACertFile = application:get_env(?APP, cacertfile, undefined), CertFile = application:get_env(?APP, certfile, undefined), @@ -75,7 +74,7 @@ translate_env() -> , {ciphers, emqx_tls_lib:default_ciphers()} | TLSOpts ], - [{transport, ssl}, {transport_opts, [Inet | NTLSOpts]}] + [{transport, ssl}, {transport_opts, emqx_misc:ipv6_probe(NTLSOpts)}] end, PoolOpts = [{host, Host}, {port, Port}, @@ -98,14 +97,3 @@ path(Path) -> set_content_type(Headers) -> NHeaders = proplists:delete(<<"Content-Type">>, proplists:delete(<<"content-type">>, Headers)), [{<<"content-type">>, <<"application/json">>} | NHeaders]. - -parse_host(Host) -> - case inet:parse_address(Host) of - {ok, Addr} when size(Addr) =:= 4 -> {inet, Addr}; - {ok, Addr} when size(Addr) =:= 8 -> {inet6, Addr}; - {error, einval} -> - case inet:getaddr(Host, inet6) of - {ok, _} -> {inet6, Host}; - {error, _} -> {inet, Host} - end - end. diff --git a/apps/emqx_web_hook/src/emqx_web_hook_sup.erl b/apps/emqx_web_hook/src/emqx_web_hook_sup.erl index 7663c8ff3..ec46efaa0 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook_sup.erl +++ b/apps/emqx_web_hook/src/emqx_web_hook_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_web_hook/test/emqx_web_hook_SUITE.erl b/apps/emqx_web_hook/test/emqx_web_hook_SUITE.erl index 6c010de9d..cd4fb4869 100644 --- a/apps/emqx_web_hook/test/emqx_web_hook_SUITE.erl +++ b/apps/emqx_web_hook/test/emqx_web_hook_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -31,59 +31,74 @@ %%-------------------------------------------------------------------- all() -> - [{group, http}, - {group, https}, - {group, ipv6http}, - {group, ipv6https}]. + [ {group, http} + , {group, https} + , {group, ipv6http} + , {group, ipv6https} + , {group, all} + ]. groups() -> - Cases = emqx_ct:all(?MODULE), - [{http, [sequence], Cases}, - {https, [sequence], Cases}, - {ipv6http, [sequence], Cases}, - {ipv6https, [sequence], Cases}]. + Cases = [test_full_flow], + [ {http, [sequence], Cases} + , {https, [sequence], Cases} + , {ipv6http, [sequence], Cases} + , {ipv6https, [sequence], Cases} + , {all, [sequence], emqx_ct:all(?MODULE)} + ]. + +start_apps(F) -> emqx_ct_helpers:start_apps(apps(), F). init_per_group(Name, Config) -> application:ensure_all_started(emqx_management), set_special_cfgs(), - case Name of - http -> - emqx_ct_helpers:start_apps([emqx_web_hook, emqx_modules, emqx_management, - emqx_rule_engine], fun set_special_configs_http/1); - https -> - emqx_ct_helpers:start_apps([emqx_web_hook, emqx_modules, emqx_management, - emqx_rule_engine], fun set_special_configs_https/1); - ipv6http -> - emqx_ct_helpers:start_apps([emqx_web_hook, emqx_management, - emqx_rule_engine], fun set_special_configs_ipv6_http/1); - ipv6https -> - emqx_ct_helpers:start_apps([emqx_web_hook, emqx_management, - emqx_rule_engine], fun set_special_configs_ipv6_https/1) - end, - Config. + BasePort = + case Name of + all -> 8801; + http -> 8811; + https -> 8821; + ipv6http -> 8831; + ipv6https -> 8841 + end, + CF = case Name of + all -> fun set_special_configs_http/1; + http -> fun set_special_configs_http/1; + https -> fun set_special_configs_https/1; + ipv6http -> fun set_special_configs_ipv6_http/1; + ipv6https -> fun set_special_configs_ipv6_https/1 + end, + start_apps(fun(_) -> CF(BasePort) end), + Opts = case atom_to_list(Name) of + "ipv6" ++ _ -> [{ip, {0,0,0,0,0,0,0,1}}, inet6]; + _ -> [inet] + end, + [{base_port, BasePort}, {transport_opts, Opts} | Config]. end_per_group(_Name, Config) -> - emqx_ct_helpers:stop_apps([emqx_web_hook]), + emqx_ct_helpers:stop_apps(apps()), Config. -set_special_configs_http(_) -> - application:set_env(emqx_web_hook, url, "http://127.0.0.1:9999"). +set_special_configs_http(Port) -> + application:set_env(emqx_web_hook, url, "http://127.0.0.1:" ++ integer_to_list(Port)). -set_special_configs_https(_) -> +set_special_configs_https(Port) -> + set_ssl_configs(), + application:set_env(emqx_web_hook, url, "https://127.0.0.1:" ++ integer_to_list(Port+1)). + +set_special_configs_ipv6_http(Port) -> + application:set_env(emqx_web_hook, url, "http://[::1]:" ++ integer_to_list(Port)). + +set_special_configs_ipv6_https(Port) -> + set_ssl_configs(), + application:set_env(emqx_web_hook, url, "https://[::1]:" ++ integer_to_list(Port+1)). + +set_ssl_configs() -> Path = emqx_ct_helpers:deps_path(emqx_web_hook, "test/emqx_web_hook_SUITE_data/"), SslOpts = [{keyfile, Path ++ "/client-key.pem"}, {certfile, Path ++ "/client-cert.pem"}, {cacertfile, Path ++ "/ca.pem"}], application:set_env(emqx_web_hook, ssl, true), - application:set_env(emqx_web_hook, ssloptions, SslOpts), - application:set_env(emqx_web_hook, url, "https://127.0.0.1:8888"). - -set_special_configs_ipv6_http(_) -> - application:set_env(emqx_web_hook, url, "http://[::1]:9999"). - -set_special_configs_ipv6_https(N) -> - set_special_configs_https(N), - application:set_env(emqx_web_hook, url, "https://[::1]:8888"). + application:set_env(emqx_web_hook, ssloptions, SslOpts). set_special_cfgs() -> AllRules = [{"message.acked", "{\"action\": \"on_message_acked\"}"}, @@ -99,30 +114,71 @@ set_special_cfgs() -> {"client.connack", "{\"action\": \"on_client_connack\"}"}, {"client.connect", "{\"action\": \"on_client_connect\"}"}], application:set_env(emqx_web_hook, rules, AllRules). + %%-------------------------------------------------------------------- %% Test cases %%-------------------------------------------------------------------- -t_valid(Config) -> - {ok, ServerPid} = http_server:start_link(), +test_full_flow(Config) -> + [_|_] = Opts = proplists:get_value(transport_opts, Config), + BasePort = proplists:get_value(base_port, Config), + Tester = self(), + {ok, ServerPid} = http_server:start_link(Tester, BasePort, Opts), + receive {ServerPid, ready} -> ok + after 1000 -> error(timeout) + end, application:set_env(emqx_web_hook, headers, [{"k1","K1"}, {"k2", "K2"}]), - {ok, C} = emqtt:start_link([ {clientid, <<"simpleClient">>} + ClientId = iolist_to_binary(["client-", integer_to_list(erlang:system_time())]), + {ok, C} = emqtt:start_link([ {clientid, ClientId} , {proto_ver, v5} , {keepalive, 60} ]), + try + do_test_full_flow(C, ClientId) + after + Ref = erlang:monitor(process, ServerPid), + http_server:stop(ServerPid), + receive {'DOWN', Ref, _, _, _} -> ok + after 5000 -> error(timeout) + end + end. + +do_test_full_flow(C, ClientId) -> {ok, _} = emqtt:connect(C), - emqtt:subscribe(C, <<"TopicA">>, qos2), - emqtt:publish(C, <<"TopicA">>, <<"Payload...">>, qos2), - emqtt:unsubscribe(C, <<"TopicA">>), + {ok, _, _} = emqtt:subscribe(C, <<"TopicA">>, qos2), + {ok, _} = emqtt:publish(C, <<"TopicA">>, <<"Payload...">>, qos2), + {ok, _, _} = emqtt:unsubscribe(C, <<"TopicA">>), emqtt:disconnect(C), - [begin - Maps = emqx_json:decode(P, [return_maps]), - validate_hook_resp(Maps), - validate_hook_headers(Headers) - end - || {{P, _Bool}, Headers} <- http_server:get_received_data()], - http_server:stop(ServerPid), - Config. + validate_params_and_headers(undefined, ClientId). + +validate_params_and_headers(ClientState, ClientId) -> + receive + {http_server, {Params0, _Bool}, Headers} -> + Params = emqx_json:decode(Params0, [return_maps]), + try + validate_hook_resp(ClientId, Params), + validate_hook_headers(Headers), + case maps:get(<<"action">>, Params) of + <<"session_terminated">> -> + ok; + <<"client_connect">> -> + validate_params_and_headers(connected, ClientId); + _ -> + validate_params_and_headers(ClientState, ClientId) %% continue looping + end + catch + throw : {unknown_client, Other} -> + ct:pal("ignored_event_from_other_client ~p~n~p~n~p", + [Other, Params, Headers]), + validate_params_and_headers(ClientState, ClientId) %% continue looping + end + after + 5000 -> + case ClientState =:= undefined of + true -> error("client_was_never_connected"); + false -> error("terminate_action_is_not_received_in_time") + end + end. t_check_hooked(_) -> {ok, Rules} = application:get_env(emqx_web_hook, rules), @@ -150,66 +206,69 @@ validate_hook_headers(Headers) -> ?assertEqual(<<"K1">>, maps:get(<<"k1">>, Headers)), ?assertEqual(<<"K2">>, maps:get(<<"k2">>, Headers)). -validate_hook_resp(Body = ?ACTION(<<"client_connect">>)) -> +validate_hook_resp(ClientId, Body = ?ACTION(<<"client_connect">>)) -> + assert_username_clientid(ClientId, Body), ?assertEqual(5, maps:get(<<"proto_ver">>, Body)), ?assertEqual(60, maps:get(<<"keepalive">>, Body)), ?assertEqual(<<"127.0.0.1">>, maps:get(<<"ipaddress">>, Body)), ?assertEqual(<<"test@127.0.0.1">>, maps:get(<<"node">>, Body)), - assert_username_clientid(Body); -validate_hook_resp(Body = ?ACTION(<<"client_connack">>)) -> + ok; +validate_hook_resp(ClientId, Body = ?ACTION(<<"client_connack">>)) -> + assert_username_clientid(ClientId, Body), ?assertEqual(5, maps:get(<<"proto_ver">>, Body)), ?assertEqual(60, maps:get(<<"keepalive">>, Body)), ?assertEqual(<<"success">>, maps:get(<<"conn_ack">>, Body)), ?assertEqual(<<"127.0.0.1">>, maps:get(<<"ipaddress">>, Body)), ?assertEqual(<<"test@127.0.0.1">>, maps:get(<<"node">>, Body)), - assert_username_clientid(Body); -validate_hook_resp(Body = ?ACTION(<<"client_connected">>)) -> + ok; +validate_hook_resp(ClientId, Body = ?ACTION(<<"client_connected">>)) -> + assert_username_clientid(ClientId, Body), _ = maps:get(<<"connected_at">>, Body), ?assertEqual(5, maps:get(<<"proto_ver">>, Body)), ?assertEqual(60, maps:get(<<"keepalive">>, Body)), ?assertEqual(<<"127.0.0.1">>, maps:get(<<"ipaddress">>, Body)), - ?assertEqual(<<"test@127.0.0.1">>, maps:get(<<"node">>, Body)), - assert_username_clientid(Body); -validate_hook_resp(Body = ?ACTION(<<"client_disconnected">>)) -> + ?assertEqual(<<"test@127.0.0.1">>, maps:get(<<"node">>, Body)); +validate_hook_resp(ClientId, Body = ?ACTION(<<"client_disconnected">>)) -> + assert_username_clientid(ClientId, Body), ?assertEqual(<<"normal">>, maps:get(<<"reason">>, Body)), - ?assertEqual(<<"test@127.0.0.1">>, maps:get(<<"node">>, Body)), - assert_username_clientid(Body); -validate_hook_resp(Body = ?ACTION(<<"client_subscribe">>)) -> + ?assertEqual(<<"test@127.0.0.1">>, maps:get(<<"node">>, Body)); +validate_hook_resp(ClientId, Body = ?ACTION(<<"client_subscribe">>)) -> + assert_username_clientid(ClientId, Body), _ = maps:get(<<"opts">>, Body), ?assertEqual(<<"TopicA">>, maps:get(<<"topic">>, Body)), - ?assertEqual(<<"test@127.0.0.1">>, maps:get(<<"node">>, Body)), - assert_username_clientid(Body); -validate_hook_resp(Body = ?ACTION(<<"client_unsubscribe">>)) -> + ?assertEqual(<<"test@127.0.0.1">>, maps:get(<<"node">>, Body)); +validate_hook_resp(ClientId, Body = ?ACTION(<<"client_unsubscribe">>)) -> + assert_username_clientid(ClientId, Body), _ = maps:get(<<"opts">>, Body), ?assertEqual(<<"TopicA">>, maps:get(<<"topic">>, Body)), - ?assertEqual(<<"test@127.0.0.1">>, maps:get(<<"node">>, Body)), - assert_username_clientid(Body); -validate_hook_resp(Body = ?ACTION(<<"session_subscribed">>)) -> + ?assertEqual(<<"test@127.0.0.1">>, maps:get(<<"node">>, Body)); +validate_hook_resp(ClientId, Body = ?ACTION(<<"session_subscribed">>)) -> + assert_username_clientid(ClientId, Body), _ = maps:get(<<"opts">>, Body), ?assertEqual(<<"TopicA">>, maps:get(<<"topic">>, Body)), - ?assertEqual(<<"test@127.0.0.1">>, maps:get(<<"node">>, Body)), - assert_username_clientid(Body); -validate_hook_resp(Body = ?ACTION(<<"session_unsubscribed">>)) -> + ?assertEqual(<<"test@127.0.0.1">>, maps:get(<<"node">>, Body)); +validate_hook_resp(ClientId, Body = ?ACTION(<<"session_unsubscribed">>)) -> + assert_username_clientid(ClientId, Body), ?assertEqual(<<"TopicA">>, maps:get(<<"topic">>, Body)), - ?assertEqual(<<"test@127.0.0.1">>, maps:get(<<"node">>, Body)), - assert_username_clientid(Body); -validate_hook_resp(Body = ?ACTION(<<"session_terminated">>)) -> + ?assertEqual(<<"test@127.0.0.1">>, maps:get(<<"node">>, Body)); +validate_hook_resp(ClientId, Body = ?ACTION(<<"session_terminated">>)) -> + assert_username_clientid(ClientId, Body), ?assertEqual(<<"normal">>, maps:get(<<"reason">>, Body)), - ?assertEqual(<<"test@127.0.0.1">>, maps:get(<<"node">>, Body)), - assert_username_clientid(Body); -validate_hook_resp(Body = ?ACTION(<<"message_publish">>)) -> + ?assertEqual(<<"test@127.0.0.1">>, maps:get(<<"node">>, Body)); +validate_hook_resp(_ClientId, Body = ?ACTION(<<"message_publish">>)) -> ?assertEqual(<<"test@127.0.0.1">>, maps:get(<<"node">>, Body)), assert_messages_attrs(Body); -validate_hook_resp(Body = ?ACTION(<<"message_delivered">>)) -> +validate_hook_resp(_ClientId, Body = ?ACTION(<<"message_delivered">>)) -> ?assertEqual(<<"test@127.0.0.1">>, maps:get(<<"node">>, Body)), assert_messages_attrs(Body); -validate_hook_resp(Body = ?ACTION(<<"message_acked">>)) -> +validate_hook_resp(_ClientId, Body = ?ACTION(<<"message_acked">>)) -> ?assertEqual(<<"test@127.0.0.1">>, maps:get(<<"node">>, Body)), assert_messages_attrs(Body). -assert_username_clientid(#{<<"clientid">> := ClientId, <<"username">> := Username}) -> - ?assertEqual(<<"simpleClient">>, ClientId), - ?assertEqual(null, Username). +assert_username_clientid(ClientId, #{<<"clientid">> := ClientId, <<"username">> := Username}) -> + ?assertEqual(null, Username); +assert_username_clientid(_ClientId, #{<<"clientid">> := Other}) -> + throw({unknown_client, Other}). assert_messages_attrs(#{ <<"ts">> := _ , <<"qos">> := _ @@ -220,3 +279,6 @@ assert_messages_attrs(#{ <<"ts">> := _ , <<"from_client_id">> := _ }) -> ok. + +apps() -> + [emqx_web_hook, emqx_modules, emqx_management, emqx_rule_engine]. diff --git a/apps/emqx_web_hook/test/http_server.erl b/apps/emqx_web_hook/test/http_server.erl index 6a23e1035..791f725d1 100644 --- a/apps/emqx_web_hook/test/http_server.erl +++ b/apps/emqx_web_hook/test/http_server.erl @@ -9,27 +9,25 @@ -module(http_server). -behaviour(gen_server). --export([start_link/0]). --export([get_received_data/0]). +-export([start_link/3]). -export([stop/1]). -export([code_change/3, handle_call/3, handle_cast/2, handle_info/2, init/1, init/2, terminate/2]). --define(HTTP_PORT, 9999). --define(HTTPS_PORT, 8888). --record(state, {}). +-record(state, {parent :: pid()}). %%-------------------------------------------------------------------- %% APIs %%-------------------------------------------------------------------- -start_link() -> - gen_server:start_link(?MODULE, [], []). +start_link(Parent, BasePort, Opts) -> + stop_http(), + stop_https(), + timer:sleep(100), + gen_server:start_link(?MODULE, {Parent, BasePort, Opts}, []). -init([]) -> - EtsOptions = [named_table, public, set, {write_concurrency, true}, - {read_concurrency, true}], - emqx_web_hook_http_test = ets:new(emqx_web_hook_http_test, EtsOptions), - ok = start_http(?HTTP_PORT), - ok = start_https(?HTTPS_PORT), - {ok, #state{}}. +init({Parent, BasePort, Opts}) -> + ok = start_http(Parent, [{port, BasePort} | Opts]), + ok = start_https(Parent, [{port, BasePort + 1} | Opts]), + Parent ! {self(), ready}, + {ok, #state{parent = Parent}}. handle_call(_Request, _From, State) -> {reply, ignored, State}. @@ -47,9 +45,6 @@ terminate(_Reason, _State) -> code_change(_OldVsn, State, _Extra) -> {ok, State}. -get_received_data() -> - ets:tab2list(emqx_web_hook_http_test). - stop(Pid) -> ok = gen_server:stop(Pid). @@ -57,37 +52,39 @@ stop(Pid) -> %% Callbacks %%-------------------------------------------------------------------- -start_http(Port) -> - {ok, _Pid1} = cowboy:start_clear(http, [{port, Port}], #{ - env => #{dispatch => compile_router()} +start_http(Parent, Opts) -> + {ok, _Pid1} = cowboy:start_clear(http, Opts, #{ + env => #{dispatch => compile_router(Parent)} }), - io:format(standard_error, "[TEST LOG] Start http server on 9999 successfully!~n", []). + Port = proplists:get_value(port, Opts), + io:format(standard_error, "[TEST LOG] Start http server on ~p successfully!~n", [Port]). -start_https(Port) -> +start_https(Parent, Opts) -> Path = emqx_ct_helpers:deps_path(emqx_web_hook, "test/emqx_web_hook_SUITE_data/"), SslOpts = [{keyfile, Path ++ "/server-key.pem"}, {cacertfile, Path ++ "/ca.pem"}, {certfile, Path ++ "/server-cert.pem"}], - {ok, _Pid2} = cowboy:start_tls(https, [{port, Port}] ++ SslOpts, - #{env => #{dispatch => compile_router()}}), - io:format(standard_error, "[TEST LOG] Start https server on 8888 successfully!~n", []). + {ok, _Pid2} = cowboy:start_tls(https, Opts ++ SslOpts, + #{env => #{dispatch => compile_router(Parent)}}), + Port = proplists:get_value(port, Opts), + io:format(standard_error, "[TEST LOG] Start https server on ~p successfully!~n", [Port]). stop_http() -> - ok = cowboy:stop_listener(http), - io:format("[TEST LOG] Stopped http server on 9999"). + cowboy:stop_listener(http), + io:format("[TEST LOG] Stopped http server"). stop_https() -> - ok = cowboy:stop_listener(https), - io:format("[TEST LOG] Stopped https server on 8888"). + cowboy:stop_listener(https), + io:format("[TEST LOG] Stopped https server"). -compile_router() -> +compile_router(Parent) -> {ok, _} = application:ensure_all_started(cowboy), cowboy_router:compile([ - {'_', [{"/", ?MODULE, #{}}]} + {'_', [{"/", ?MODULE, #{parent => Parent}}]} ]). -init(Req, State) -> +init(Req, #{parent := Parent} = State) -> Method = cowboy_req:method(Req), Headers = cowboy_req:headers(Req), [Params] = case Method of @@ -96,7 +93,7 @@ init(Req, State) -> {ok, PostVals, _} = cowboy_req:read_urlencoded_body(Req), PostVals end, - ets:insert(emqx_web_hook_http_test, {Params, Headers}), + Parent ! {?MODULE, Params, Headers}, {ok, reply(Req, ok), State}. reply(Req, ok) -> diff --git a/apps/emqx_web_hook/test/props/prop_webhook_confs.erl b/apps/emqx_web_hook/test/props/prop_webhook_confs.erl index cc5e7af64..8946ce1d2 100644 --- a/apps/emqx_web_hook/test/props/prop_webhook_confs.erl +++ b/apps/emqx_web_hook/test/props/prop_webhook_confs.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -53,12 +53,13 @@ prop_confs() -> %%-------------------------------------------------------------------- do_setup() -> - application:set_env(kernel, logger_level, error), + logger:set_primary_config(#{level => warning}), emqx_ct_helpers:start_apps([], fun set_special_cfgs/1), ok. do_teardown(_) -> emqx_ct_helpers:stop_apps([]), + logger:set_primary_config(#{level => info}), ok. set_special_cfgs(_) -> @@ -113,7 +114,10 @@ cuttlefish_conf_option(K, V) %%-------------------------------------------------------------------- confs() -> - nof([{"web.hook.encode_payload", oneof(["base64", "base62"])}, + nof([{"web.hook.headers.content-type", + oneof(["application/json"])}, + {"web.hook.body.encoding_of_payload_field", + oneof(["plain", "base64", "base62"])}, {"web.hook.rule.client.connect.1", rule_spec()}, {"web.hook.rule.client.connack.1", rule_spec()}, {"web.hook.rule.client.connected.1", rule_spec()}, diff --git a/apps/emqx_web_hook/test/props/prop_webhook_hooks.erl b/apps/emqx_web_hook/test/props/prop_webhook_hooks.erl index d5fb85955..311585287 100644 --- a/apps/emqx_web_hook/test/props/prop_webhook_hooks.erl +++ b/apps/emqx_web_hook/test/props/prop_webhook_hooks.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -211,9 +211,9 @@ prop_session_terminated() -> prop_message_publish() -> ?ALL({Msg, Env, Encode}, {message(), topic_filter_env(), payload_encode()}, begin - application:set_env(emqx_web_hook, encode_payload, Encode), + application:set_env(emqx_web_hook, encoding_of_payload_field, Encode), {ok, Msg} = emqx_web_hook:on_message_publish(Msg, Env), - application:unset_env(emqx_web_hook, encode_payload), + application:unset_env(emqx_web_hook, encoding_of_payload_field), (not emqx_message:is_sys(Msg)) andalso filter_topic_match(emqx_message:topic(Msg), Env) @@ -237,9 +237,9 @@ prop_message_publish() -> prop_message_delivered() -> ?ALL({ClientInfo, Msg, Env, Encode}, {clientinfo(), message(), topic_filter_env(), payload_encode()}, begin - application:set_env(emqx_web_hook, encode_payload, Encode), + application:set_env(emqx_web_hook, encoding_of_payload_field, Encode), ok = emqx_web_hook:on_message_delivered(ClientInfo, Msg, Env), - application:unset_env(emqx_web_hook, encode_payload), + application:unset_env(emqx_web_hook, encoding_of_payload_field), (not emqx_message:is_sys(Msg)) andalso filter_topic_match(emqx_message:topic(Msg), Env) @@ -265,9 +265,9 @@ prop_message_delivered() -> prop_message_acked() -> ?ALL({ClientInfo, Msg, Env, Encode}, {clientinfo(), message(), empty_env(), payload_encode()}, begin - application:set_env(emqx_web_hook, encode_payload, Encode), + application:set_env(emqx_web_hook, encoding_of_payload_field, Encode), ok = emqx_web_hook:on_message_acked(ClientInfo, Msg, Env), - application:unset_env(emqx_web_hook, encode_payload), + application:unset_env(emqx_web_hook, encoding_of_payload_field), (not emqx_message:is_sys(Msg)) andalso filter_topic_match(emqx_message:topic(Msg), Env) @@ -305,7 +305,7 @@ do_setup() -> meck:new(ehttpc, [passthrough, no_history]), meck:expect(ehttpc, request, fun(_ClientId, Method, {Path, Headers, Body}) -> - Self ! {Method, Path, Headers, Body}, {ok, ok, ok} + Self ! {Method, Path, Headers, Body}, {ok, 200, ok} end), meck:new(emqx_metrics, [passthrough, no_history]), @@ -327,8 +327,10 @@ peer2addr(Host) -> stringfy({shutdown, Reason}) -> stringfy(Reason); -stringfy(Term) when is_atom(Term); is_binary(Term) -> +stringfy(Term) when is_binary(Term) -> Term; +stringfy(Term) when is_atom(Term) -> + atom_to_binary(Term, utf8); stringfy(Term) -> unicode:characters_to_binary(io_lib:format("~0p", [Term])). @@ -374,7 +376,10 @@ unsub_properties() -> #{}. shutdown_reason() -> - oneof([any(), {shutdown, atom()}]). + oneof([disconnected, not_autherised, + "list_reason", <<"binary_reason">>, + {tuple, reason}, + {shutdown, emqx_ct_proper_types:limited_atom()}]). empty_env() -> {undefined}. @@ -383,7 +388,7 @@ topic_filter_env() -> oneof([{<<"#">>}, {undefined}, {topic()}]). payload_encode() -> - oneof([base62, base64, undefined]). + oneof([base62, base64, plain]). disconnected_conninfo() -> ?LET(Info, conninfo(), diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index 288c79cec..1891c39a0 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -1,4 +1,4 @@ -ARG BUILD_FROM=emqx/build-env:erl23.2.7-alpine-amd64 +ARG BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-1-alpine-amd64 ARG RUN_FROM=alpine:3.12 FROM ${BUILD_FROM} AS builder diff --git a/deploy/docker/docker-entrypoint.sh b/deploy/docker/docker-entrypoint.sh index 91e474818..a95aa50ee 100755 --- a/deploy/docker/docker-entrypoint.sh +++ b/deploy/docker/docker-entrypoint.sh @@ -129,4 +129,10 @@ if [[ -n "$EMQX_LOADED_MODULES" ]]; then fill_tuples "$LOADED_MODULES" "$EMQX_LOADED_MODULES" fi +# The default rpc port discovery 'stateless' is mostly for clusters +# having static node names. So it's troulbe-free for multiple emqx nodes +# running on the same host. +# When start emqx in docker, it's mostly one emqx node in one container +export EMQX_RPC__PORT_DISCOVERY="${EMQX_RPC__PORT_DISCOVERY:-manual}" + exec "$@" diff --git a/docker.mk b/docker.mk index d2333b1b0..8178e0938 100644 --- a/docker.mk +++ b/docker.mk @@ -62,7 +62,7 @@ docker-build: @docker build --no-cache \ --build-arg PKG_VSN=$(PKG_VSN) \ - --build-arg BUILD_FROM=emqx/build-env:erl23.2.7-alpine-$(ARCH) \ + --build-arg BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-1-alpine-$(ARCH) \ --build-arg RUN_FROM=$(ARCH)/alpine:3.12 \ --build-arg EMQX_NAME=$(EMQX_NAME) \ --build-arg QEMU_ARCH=$(QEMU_ARCH) \ diff --git a/etc/emqx.conf b/etc/emqx.conf index a3aaee1f9..3e9d394b1 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -408,8 +408,7 @@ rpc.socket_buffer = 1MB ## Where to emit the logs. ## Enable the console (standard output) logs. ## -## Value: off | file | console | both -## - off: disable logs entirely +## Value: file | console | both ## - file: write logs only to file ## - console: write logs only to standard I/O ## - both: write logs both to file and standard I/O @@ -452,6 +451,10 @@ log.file = emqx.log ## Default: No Limit #log.chars_limit = 8192 +## Log to single line +## Value: boolean +#log.single_line = false + ## Enables the log rotation. ## With this enabled, new log files will be created when the current ## log file is full, max to `log.rotation.size` files will be created. @@ -1120,14 +1123,14 @@ listener.tcp.external.access.1 = "allow all" ## Enable the option for X.509 certificate based authentication. ## EMQX will use the common name of certificate as MQTT username. -## The proxy-protocol protocol can get the certificate CN through tcp +## Only support Proxy Protocol V2, the CN is available in Proxy Protocol V2 additional info ## ## Value: cn ## listener.tcp.external.peer_cert_as_username = cn ## Enable the option for X.509 certificate based authentication. ## EMQX will use the common name of certificate as MQTT clientid. -## The proxy-protocol protocol can get the certificate CN through tcp +## Only support Proxy Protocol V2, the CN is available in Proxy Protocol V2 additional info ## ## Value: cn ## listener.tcp.external.peer_cert_as_clientid = cn @@ -1614,6 +1617,18 @@ listener.ws.external.access.1 = "allow all" ## Default: mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5 ## listener.ws.external.supported_subprotocols = "mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5" +## Specify which HTTP header for real source IP if the EMQ X cluster is +## deployed behind NGINX or HAProxy. +## +## Default: X-Forwarded-For +## listener.ws.external.proxy_address_header = X-Forwarded-For + +## Specify which HTTP header for real source port if the EMQ X cluster is +## deployed behind NGINX or HAProxy. +## +## Default: X-Forwarded-Port +## listener.ws.external.proxy_port_header = X-Forwarded-Port + ## Enable the Proxy Protocol V1/2 if the EMQ cluster is deployed behind ## HAProxy or Nginx. ## @@ -1629,6 +1644,20 @@ listener.ws.external.access.1 = "allow all" ## Value: Duration ## listener.ws.external.proxy_protocol_timeout = 3s +## Enable the option for X.509 certificate based authentication. +## EMQX will use the common name of certificate as MQTT username. +## Only support Proxy Protocol V2, the CN is available in Proxy Protocol V2 additional info +## +## Value: cn +## listener.ws.external.peer_cert_as_username = cn + +## Enable the option for X.509 certificate based authentication. +## EMQX will use the common name of certificate as MQTT clientid. +## Only support Proxy Protocol V2, the CN is available in Proxy Protocol V2 additional info +## +## Value: cn +## listener.ws.external.peer_cert_as_clientid = cn + ## The TCP backlog of external MQTT/WebSocket Listener. ## ## See: listener.ws.$name.backlog @@ -1857,6 +1886,18 @@ listener.wss.external.access.1 = "allow all" ## Default: mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5 ## listener.wss.external.supported_subprotocols = "mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5" +## Specify which HTTP header for real source IP if the EMQ X cluster is +## deployed behind NGINX or HAProxy. +## +## Default: X-Forwarded-For +## listener.wss.external.proxy_address_header = X-Forwarded-For + +## Specify which HTTP header for real source port if the EMQ X cluster is +## deployed behind NGINX or HAProxy. +## +## Default: X-Forwarded-Port +## listener.wss.external.proxy_port_header = X-Forwarded-Port + ## Enable the Proxy Protocol V1/2 support. ## ## See: listener.tcp.$name.proxy_protocol @@ -2233,6 +2274,32 @@ broker.shared_dispatch_ack_enabled = false ## Value: Flag broker.route_batch_clean = off +## Performance toggle for subscribe/unsubscribe wildcard topic. +## Change this toggle only when there are many wildcard topics. +## Value: Enum +## - key: mnesia translational updates with per-key locks. recommended for single node setup. +## - tab: mnesia translational updates with table lock. recommended for multi-nodes setup. +## - global: global lock protected updates. recommended for larger cluster. +## NOTE: when changing from/to 'global' lock, it requires all nodes in the cluster +## to be stopped before the change. +# broker.perf.route_lock_type = key + +## Enable trie path compaction. +## Enabling it significantly improves wildcard topic subscribe rate, +## if wildcard topics have unique prefixes like: 'sensor/{{id}}/+/', +## where ID is unique per subscriber. +## +## Topic match performance (when publishing) may degrade if messages +## are mostly published to topics with large number of levels. +## +## NOTE: This is a cluster-wide configuration. +## It rquires all nodes to be stopped before changing it. +## +## Value: Enum +## - true: enable trie path compaction +## - false: disable trie path compaction +# broker.perf.trie_compaction = true + ## CONFIG_SECTION_BGN=sys_mon ================================================== ## Enable Long GC monitoring. Disable if the value is 0. diff --git a/include/emqx.hrl b/include/emqx.hrl index c1e701f12..8d0b61c21 100644 --- a/include/emqx.hrl +++ b/include/emqx.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -89,31 +89,6 @@ dest :: node() | {binary(), node()} }). -%%-------------------------------------------------------------------- -%% Trie -%%-------------------------------------------------------------------- - --type(trie_node_id() :: binary() | atom()). - --record(trie_node, { - node_id :: trie_node_id(), - edge_count = 0 :: non_neg_integer(), - topic :: binary() | undefined, - flags :: list(atom()) | undefined - }). - --record(trie_edge, { - node_id :: trie_node_id(), - word :: binary() | atom() - }). - --record(trie, { - edge :: #trie_edge{}, - node_id :: trie_node_id() - }). - --type(trie_node() :: #trie_node{}). - %%-------------------------------------------------------------------- %% Plugin %%-------------------------------------------------------------------- diff --git a/include/emqx_mqtt.hrl b/include/emqx_mqtt.hrl index 220fbbb1c..3e6bf6e3e 100644 --- a/include/emqx_mqtt.hrl +++ b/include/emqx_mqtt.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index 2689b75b8..2369dbd5b 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.3-rc.3"}). +-define(EMQX_RELEASE, {opensource, "4.3-rc.5"}). -else. diff --git a/include/logger.hrl b/include/logger.hrl index eb669a3fa..4881a6413 100644 --- a/include/logger.hrl +++ b/include/logger.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/include/types.hrl b/include/types.hrl index e92f3f2a2..7b44521d1 100644 --- a/include/types.hrl +++ b/include/types.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib-ce/emqx_dashboard/include/emqx_dashboard.hrl b/lib-ce/emqx_dashboard/include/emqx_dashboard.hrl index 73b64de77..6cb759c9b 100644 --- a/lib-ce/emqx_dashboard/include/emqx_dashboard.hrl +++ b/lib-ce/emqx_dashboard/include/emqx_dashboard.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard.erl b/lib-ce/emqx_dashboard/src/emqx_dashboard.erl index b7b6ca42e..0390339d3 100644 --- a/lib-ce/emqx_dashboard/src/emqx_dashboard.erl +++ b/lib-ce/emqx_dashboard/src/emqx_dashboard.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard_admin.erl b/lib-ce/emqx_dashboard/src/emqx_dashboard_admin.erl index 0e5616c64..c70308744 100644 --- a/lib-ce/emqx_dashboard/src/emqx_dashboard_admin.erl +++ b/lib-ce/emqx_dashboard/src/emqx_dashboard_admin.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard_api.erl b/lib-ce/emqx_dashboard/src/emqx_dashboard_api.erl index 8663b13ae..e1c89efbb 100644 --- a/lib-ce/emqx_dashboard/src/emqx_dashboard_api.erl +++ b/lib-ce/emqx_dashboard/src/emqx_dashboard_api.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard_app.erl b/lib-ce/emqx_dashboard/src/emqx_dashboard_app.erl index 76228ec8b..12a829490 100644 --- a/lib-ce/emqx_dashboard/src/emqx_dashboard_app.erl +++ b/lib-ce/emqx_dashboard/src/emqx_dashboard_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard_cli.erl b/lib-ce/emqx_dashboard/src/emqx_dashboard_cli.erl index 3977e292e..14c5009ff 100644 --- a/lib-ce/emqx_dashboard/src/emqx_dashboard_cli.erl +++ b/lib-ce/emqx_dashboard/src/emqx_dashboard_cli.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard_sup.erl b/lib-ce/emqx_dashboard/src/emqx_dashboard_sup.erl index 141465e40..b132ebaf4 100644 --- a/lib-ce/emqx_dashboard/src/emqx_dashboard_sup.erl +++ b/lib-ce/emqx_dashboard/src/emqx_dashboard_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib-ce/emqx_dashboard/test/emqx_dashboard_SUITE.erl b/lib-ce/emqx_dashboard/test/emqx_dashboard_SUITE.erl index 60714a639..4a8ca7311 100644 --- a/lib-ce/emqx_dashboard/test/emqx_dashboard_SUITE.erl +++ b/lib-ce/emqx_dashboard/test/emqx_dashboard_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib-ce/emqx_modules/src/emqx_mod_acl_internal.erl b/lib-ce/emqx_modules/src/emqx_mod_acl_internal.erl index 1701de69d..8956229ea 100644 --- a/lib-ce/emqx_modules/src/emqx_mod_acl_internal.erl +++ b/lib-ce/emqx_modules/src/emqx_mod_acl_internal.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib-ce/emqx_modules/src/emqx_mod_api_topic_metrics.erl b/lib-ce/emqx_modules/src/emqx_mod_api_topic_metrics.erl index 1588cf0c8..150dcb151 100644 --- a/lib-ce/emqx_modules/src/emqx_mod_api_topic_metrics.erl +++ b/lib-ce/emqx_modules/src/emqx_mod_api_topic_metrics.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib-ce/emqx_modules/src/emqx_mod_delayed.erl b/lib-ce/emqx_modules/src/emqx_mod_delayed.erl index 208da68ca..852fc4717 100644 --- a/lib-ce/emqx_modules/src/emqx_mod_delayed.erl +++ b/lib-ce/emqx_modules/src/emqx_mod_delayed.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib-ce/emqx_modules/src/emqx_mod_presence.erl b/lib-ce/emqx_modules/src/emqx_mod_presence.erl index a64934160..7ba147c9a 100644 --- a/lib-ce/emqx_modules/src/emqx_mod_presence.erl +++ b/lib-ce/emqx_modules/src/emqx_mod_presence.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib-ce/emqx_modules/src/emqx_mod_rewrite.erl b/lib-ce/emqx_modules/src/emqx_mod_rewrite.erl index de1eb6d79..c3a550692 100644 --- a/lib-ce/emqx_modules/src/emqx_mod_rewrite.erl +++ b/lib-ce/emqx_modules/src/emqx_mod_rewrite.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib-ce/emqx_modules/src/emqx_mod_subscription.erl b/lib-ce/emqx_modules/src/emqx_mod_subscription.erl index 69a85a381..06178aee7 100644 --- a/lib-ce/emqx_modules/src/emqx_mod_subscription.erl +++ b/lib-ce/emqx_modules/src/emqx_mod_subscription.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib-ce/emqx_modules/src/emqx_mod_sup.erl b/lib-ce/emqx_modules/src/emqx_mod_sup.erl index b4e565e56..755e52a60 100644 --- a/lib-ce/emqx_modules/src/emqx_mod_sup.erl +++ b/lib-ce/emqx_modules/src/emqx_mod_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib-ce/emqx_modules/src/emqx_mod_topic_metrics.erl b/lib-ce/emqx_modules/src/emqx_mod_topic_metrics.erl index 635f8b527..0196b9b2a 100644 --- a/lib-ce/emqx_modules/src/emqx_mod_topic_metrics.erl +++ b/lib-ce/emqx_modules/src/emqx_mod_topic_metrics.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib-ce/emqx_modules/src/emqx_modules.erl b/lib-ce/emqx_modules/src/emqx_modules.erl index 74a6c1256..3de9c6ba3 100644 --- a/lib-ce/emqx_modules/src/emqx_modules.erl +++ b/lib-ce/emqx_modules/src/emqx_modules.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib-ce/emqx_modules/src/emqx_modules_api.erl b/lib-ce/emqx_modules/src/emqx_modules_api.erl index 6bd65f766..3490c116c 100644 --- a/lib-ce/emqx_modules/src/emqx_modules_api.erl +++ b/lib-ce/emqx_modules/src/emqx_modules_api.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib-ce/emqx_modules/test/emqx_mod_acl_internal_SUITE.erl b/lib-ce/emqx_modules/test/emqx_mod_acl_internal_SUITE.erl index 5152cb79a..3edcd31fa 100644 --- a/lib-ce/emqx_modules/test/emqx_mod_acl_internal_SUITE.erl +++ b/lib-ce/emqx_modules/test/emqx_mod_acl_internal_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib-ce/emqx_modules/test/emqx_mod_delayed_SUITE.erl b/lib-ce/emqx_modules/test/emqx_mod_delayed_SUITE.erl index f3385d8b4..fcd73bb61 100644 --- a/lib-ce/emqx_modules/test/emqx_mod_delayed_SUITE.erl +++ b/lib-ce/emqx_modules/test/emqx_mod_delayed_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib-ce/emqx_modules/test/emqx_mod_presence_SUITE.erl b/lib-ce/emqx_modules/test/emqx_mod_presence_SUITE.erl index 61f045210..fafcc3c2f 100644 --- a/lib-ce/emqx_modules/test/emqx_mod_presence_SUITE.erl +++ b/lib-ce/emqx_modules/test/emqx_mod_presence_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib-ce/emqx_modules/test/emqx_mod_rewrite_SUITE.erl b/lib-ce/emqx_modules/test/emqx_mod_rewrite_SUITE.erl index f8304b079..997eff1c2 100644 --- a/lib-ce/emqx_modules/test/emqx_mod_rewrite_SUITE.erl +++ b/lib-ce/emqx_modules/test/emqx_mod_rewrite_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib-ce/emqx_modules/test/emqx_mod_subscription_SUITE.erl b/lib-ce/emqx_modules/test/emqx_mod_subscription_SUITE.erl index 1b3d0cb92..c2905754b 100644 --- a/lib-ce/emqx_modules/test/emqx_mod_subscription_SUITE.erl +++ b/lib-ce/emqx_modules/test/emqx_mod_subscription_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib-ce/emqx_modules/test/emqx_mod_sup_SUITE.erl b/lib-ce/emqx_modules/test/emqx_mod_sup_SUITE.erl index 230c5ab1d..59d0ffde2 100644 --- a/lib-ce/emqx_modules/test/emqx_mod_sup_SUITE.erl +++ b/lib-ce/emqx_modules/test/emqx_mod_sup_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -28,7 +28,6 @@ all() -> emqx_ct:all(?MODULE). %%-------------------------------------------------------------------- t_start(_) -> - {ok, _} = emqx_mod_sup:start_link(), ?assertEqual([], supervisor:which_children(emqx_mod_sup)). t_start_child(_) -> @@ -41,7 +40,6 @@ t_start_child(_) -> type => worker, modules => [Mod]}, - {ok, _} = emqx_mod_sup:start_link(), ok = emqx_mod_sup:start_child(Mod, worker), ?assertError({already_started, _}, emqx_mod_sup:start_child(Spec)), diff --git a/lib-ce/emqx_modules/test/emqx_mod_topic_metrics_SUITE.erl b/lib-ce/emqx_modules/test/emqx_mod_topic_metrics_SUITE.erl index bb7c1c44f..d41150e4a 100644 --- a/lib-ce/emqx_modules/test/emqx_mod_topic_metrics_SUITE.erl +++ b/lib-ce/emqx_modules/test/emqx_mod_topic_metrics_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib-ce/emqx_modules/test/emqx_modules_SUITE.erl b/lib-ce/emqx_modules/test/emqx_modules_SUITE.erl index cc4ea6662..48bd0bd5b 100644 --- a/lib-ce/emqx_modules/test/emqx_modules_SUITE.erl +++ b/lib-ce/emqx_modules/test/emqx_modules_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -127,6 +127,19 @@ t_modules_cmd(_) -> "Module emqx_mod_presence unloaded successfully.\n"), unmock_print(). +%% For: https://github.com/emqx/emqx/issues/4511 +t_join_cluster(_) -> + %% Started by emqx application + {error, {already_started, emqx_modules}} = application:start(emqx_modules), + %% After clustered + emqx:shutdown(), + emqx:reboot(), + {error,{already_started,emqx_modules}} = application:start(emqx_modules), + %% After emqx reboot, we should not interfere with other tests + _ = end_per_suite([]), + _ = init_per_suite([]), + ok. + mock_print() -> catch meck:unload(emqx_ctl), meck:new(emqx_ctl, [non_strict, passthrough]), diff --git a/lib-ce/emqx_telemetry/src/emqx_telemetry.erl b/lib-ce/emqx_telemetry/src/emqx_telemetry.erl index b436768a2..3e8d3ffd3 100644 --- a/lib-ce/emqx_telemetry/src/emqx_telemetry.erl +++ b/lib-ce/emqx_telemetry/src/emqx_telemetry.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -21,7 +21,8 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/logger.hrl"). - -include_lib("kernel/include/file.hrl"). +-include_lib("kernel/include/file.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). -logger_header("[Telemetry]"). @@ -159,16 +160,18 @@ init([Opts]) -> end, case official_version(emqx_app:get_release()) of true -> - {ok, ensure_report_timer(NState), {continue, first_report}}; + _ = erlang:send(self(), first_report), + {ok, NState}; false -> {ok, NState#state{enabled = false}} - end. + end. handle_call(enable, _From, State = #state{uuid = UUID}) -> mnesia:dirty_write(?TELEMETRY, #telemetry{id = ?UNIQUE_ID, uuid = UUID, enabled = true}), - {reply, ok, ensure_report_timer(State#state{enabled = true})}; + _ = erlang:send(self(), first_report), + {reply, ok, State#state{enabled = true}}; handle_call(disable, _From, State = #state{uuid = UUID}) -> mnesia:dirty_write(?TELEMETRY, #telemetry{id = ?UNIQUE_ID, @@ -193,14 +196,19 @@ handle_cast(Msg, State) -> ?LOG(error, "Unexpected msg: ~p", [Msg]), {noreply, State}. -handle_continue(first_report, State) -> - report_telemetry(State), - {noreply, State}; - handle_continue(Continue, State) -> ?LOG(error, "Unexpected continue: ~p", [Continue]), {noreply, State}. +handle_info(first_report, State) -> + case is_pid(erlang:whereis(emqx)) of + true -> + report_telemetry(State), + {noreply, ensure_report_timer(State)}; + false -> + _ = erlang:send_after(1000, self(), first_report), + {noreply, State} + end; handle_info({timeout, TRef, time_to_report_telemetry_data}, State = #state{timer = TRef, enabled = false}) -> {noreply, State}; @@ -223,12 +231,8 @@ code_change(_OldVsn, State, _Extra) -> %%------------------------------------------------------------------------------ official_version(Version) -> - case re:run(Version, - "^\\d+\\.\\d+(?:(-(?:alpha|beta|rc)\\.[1-9][0-9]*)|\\.\\d+)$", - [{capture, none}]) of - match -> true; - nomatch -> false - end. + Pt = "^\\d+\\.\\d+(?:(-(?:alpha|beta|rc)\\.[1-9][0-9]*)|\\.\\d+)$", + match =:= re:run(Version, Pt, [{capture, none}]). ensure_report_timer(State = #state{report_interval = ReportInterval}) -> State#state{timer = emqx_misc:start_timer(ReportInterval, time_to_report_telemetry_data)}. @@ -361,9 +365,11 @@ report_telemetry(State = #state{url = URL}) -> Data = get_telemetry(State), case emqx_json:safe_encode(Data) of {ok, Bin} -> - httpc_request(post, URL, [], Bin); + httpc_request(post, URL, [], Bin), + ?tp(debug, telemetry_data_reported, #{}); {error, Reason} -> - ?LOG(debug, "Encode ~p failed due to ~p", [Data, Reason]) + %% debug? why? + ?tp(debug, telemetry_data_encode_error, #{data => Data, reason => Reason}) end. httpc_request(Method, URL, Headers, Body) -> diff --git a/lib-ce/emqx_telemetry/src/emqx_telemetry_api.erl b/lib-ce/emqx_telemetry/src/emqx_telemetry_api.erl index afbc690f4..a6e4f220a 100644 --- a/lib-ce/emqx_telemetry/src/emqx_telemetry_api.erl +++ b/lib-ce/emqx_telemetry/src/emqx_telemetry_api.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib-ce/emqx_telemetry/src/emqx_telemetry_app.erl b/lib-ce/emqx_telemetry/src/emqx_telemetry_app.erl index 846532058..5effededa 100644 --- a/lib-ce/emqx_telemetry/src/emqx_telemetry_app.erl +++ b/lib-ce/emqx_telemetry/src/emqx_telemetry_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib-ce/emqx_telemetry/src/emqx_telemetry_sup.erl b/lib-ce/emqx_telemetry/src/emqx_telemetry_sup.erl index 165812849..da17c7a67 100644 --- a/lib-ce/emqx_telemetry/src/emqx_telemetry_sup.erl +++ b/lib-ce/emqx_telemetry/src/emqx_telemetry_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib-ce/emqx_telemetry/test/emqx_telemetry_SUITE.erl b/lib-ce/emqx_telemetry/test/emqx_telemetry_SUITE.erl index 8c4a2424b..9431cd325 100644 --- a/lib-ce/emqx_telemetry/test/emqx_telemetry_SUITE.erl +++ b/lib-ce/emqx_telemetry/test/emqx_telemetry_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -21,6 +21,7 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("eunit/include/eunit.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). -import(proplists, [get_value/2]). @@ -74,6 +75,16 @@ t_enable(_) -> ok = emqx_telemetry:disable(), ?assertEqual(false, emqx_telemetry:is_enabled()). +t_send_after_enable(_) -> + ok = emqx_telemetry:disable(), + ok = snabbkaffe:start_trace(), + try + ok = emqx_telemetry:enable(), + ?assertMatch({ok, _}, ?block_until(#{?snk_kind := telemetry_data_reported}, 2000, 100)) + after + ok = snabbkaffe:stop() + end. + bin(L) when is_list(L) -> list_to_binary(L); bin(B) when is_binary(B) -> diff --git a/priv/emqx.schema b/priv/emqx.schema index a5010ce8d..a108e6857 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -457,7 +457,7 @@ end}. {mapping, "log.to", "kernel.logger", [ {default, file}, - {datatype, {enum, [off, file, console, both]}} + {datatype, {enum, [file, console, both]}} ]}. {mapping, "log.level", "kernel.logger", [ @@ -479,11 +479,6 @@ end}. {datatype, {enum, [debug, info, notice, warning, error, critical, alert, emergency, all]}} ]}. -{mapping, "log.logger_sasl_compatible", "kernel.logger_sasl_compatible", [ - {default, true}, - {datatype, {enum, [true, false]}} -]}. - {mapping, "log.dir", "kernel.logger", [ {default, "log"}, {datatype, string} @@ -499,6 +494,12 @@ end}. {datatype, integer} ]}. +%% @doc format logs in a single line. +{mapping, "log.single_line", "kernel.logger", [ + {default, false}, + {datatype, {enum, [true, false]}} +]}. + {mapping, "log.rotation.enable", "kernel.logger", [ {default, on}, {datatype, flag} @@ -563,12 +564,6 @@ end}. {datatype, string} ]}. -{mapping, "log.sasl", "sasl.sasl_error_logger", [ - {default, off}, - {datatype, flag}, - hidden -]}. - {mapping, "log.error_logger", "kernel.error_logger", [ {default, silent}, {datatype, {enum, [silent]}}, @@ -621,7 +616,8 @@ end}. -1 -> unlimited; V -> V end, - Formatter = {emqx_logger_formatter, + SingleLine = cuttlefish:conf_get("log.single_line", Conf), + Formatter = {logger_formatter, #{template => [time," [",level,"] ", {clientid, @@ -633,7 +629,7 @@ end}. []}]}, msg,"\n"], chars_limit => CharsLimit, - time_offset => LogTimeoffset + single_line => SingleLine }}, {BustLimitOn, {MaxBurstCount, TimeWindow}} = case string:tokens(cuttlefish:conf_get("log.burst_limit", Conf), ", ") of @@ -671,7 +667,7 @@ end}. %% For the default logger that outputs to console DefaultHandler = if LogTo =:= console orelse LogTo =:= both -> - [{handler, default, logger_std_h, + [{handler, console, logger_std_h, #{level => LogLevel, config => #{type => standard_io}, formatter => Formatter}}]; @@ -1225,11 +1221,6 @@ end}. {datatype, [integer, ip]} ]}. -{mapping, "listener.tcp.external", "emqx.listeners", [ - {datatype, [integer, ip]}, - {override_env, "TCP_PORT"} -]}. - {mapping, "listener.tcp.$name.acceptors", "emqx.listeners", [ {default, 8}, {datatype, integer} @@ -1337,11 +1328,6 @@ end}. {datatype, [integer, ip]} ]}. -{mapping, "listener.ssl.external", "emqx.listeners", [ - {datatype, [integer, ip]}, - {override_env, "SSL_PORT"} -]}. - {mapping, "listener.ssl.$name.acceptors", "emqx.listeners", [ {default, 8}, {datatype, integer} @@ -1510,11 +1496,6 @@ end}. {datatype, [integer, ip]} ]}. -{mapping, "listener.ws.external", "emqx.listeners", [ - {datatype, [integer, ip]}, - {override_env, "WS_PORT"} -]}. - {mapping, "listener.ws.$name.mqtt_path", "emqx.listeners", [ {default, "/mqtt"}, {datatype, string} @@ -1562,6 +1543,16 @@ end}. {datatype, string} ]}. +{mapping, "listener.ws.$name.proxy_address_header", "emqx.listeners", [ + {default, "X-Forwarded-For"}, + {datatype, string} +]}. + +{mapping, "listener.ws.$name.proxy_port_header", "emqx.listeners", [ + {default, "X-Forwarded-Port"}, + {datatype, string} +]}. + {mapping, "listener.ws.$name.proxy_protocol", "emqx.listeners", [ {datatype, flag} ]}. @@ -1668,7 +1659,11 @@ end}. ]}. {mapping, "listener.ws.$name.peer_cert_as_username", "emqx.listeners", [ - {datatype, {enum, [cn, dn, crt]}} + {datatype, {enum, [cn]}} +]}. + +{mapping, "listener.ws.$name.peer_cert_as_clientid", "emqx.listeners", [ + {datatype, {enum, [cn]}} ]}. {mapping, "listener.ws.$name.check_origin_enable", "emqx.listeners", [ @@ -1695,11 +1690,6 @@ end}. {datatype, [integer, ip]} ]}. -{mapping, "listener.wss.external", "emqx.listeners", [ - {datatype, [integer, ip]}, - {override_env, "WSS_PORT"} -]}. - {mapping, "listener.wss.$name.mqtt_path", "emqx.listeners", [ {default, "/mqtt"}, {datatype, string} @@ -1746,6 +1736,16 @@ end}. {datatype, string} ]}. +{mapping, "listener.wss.$name.proxy_address_header", "emqx.listeners", [ + {default, "X-Forwarded-For"}, + {datatype, string} +]}. + +{mapping, "listener.wss.$name.proxy_port_header", "emqx.listeners", [ + {default, "X-Forwarded-Port"}, + {datatype, string} +]}. + {mapping, "listener.wss.$name.proxy_protocol", "emqx.listeners", [ {datatype, flag} ]}. @@ -1997,6 +1997,8 @@ end}. {zone, Atom(cuttlefish:conf_get(Prefix ++ ".zone", Conf, undefined))}, {rate_limit, RateLimit(cuttlefish:conf_get(Prefix ++ ".rate_limit", Conf, undefined))}, {proxy_protocol, cuttlefish:conf_get(Prefix ++ ".proxy_protocol", Conf, undefined)}, + {proxy_address_header, list_to_binary(string:lowercase(cuttlefish:conf_get(Prefix ++ ".proxy_address_header", Conf, "")))}, + {proxy_port_header, list_to_binary(string:lowercase(cuttlefish:conf_get(Prefix ++ ".proxy_port_header", Conf, "")))}, {proxy_protocol_timeout, cuttlefish:conf_get(Prefix ++ ".proxy_protocol_timeout", Conf, undefined)}, {fail_if_no_subprotocol, cuttlefish:conf_get(Prefix ++ ".fail_if_no_subprotocol", Conf, undefined)}, {supported_subprotocols, string:tokens(cuttlefish:conf_get(Prefix ++ ".supported_subprotocols", Conf, ""), ", ")}, @@ -2279,6 +2281,33 @@ end}. {datatype, flag} ]}. +%% @doc Performance toggle for subscribe/unsubscribe wildcard topic. +%% Change this toggle only when there are many wildcard topics. +%% key: mnesia translational updates with per-key locks. recommended for single node setup. +%% tab: mnesia translational updates with table lock. recommended for multi-nodes setup. +%% global: global lock protected updates. recommended for larger cluster. +%% NOTE: when changing from/to 'global' lock, it requires all nodes in the cluster +%% +{mapping, "broker.perf.route_lock_type", "emqx.route_lock_type", [ + {default, key}, + {datatype, {enum, [key, tab, global]}} +]}. + +%% @doc Enable trie path compaction. +%% Enabling it significantly improves wildcard topic subscribe rate, +%% if wildcard topics have unique prefixes like: 'sensor/{{id}}/+/', +%% where ID is unique per subscriber. +%% +%% Topic match performance (when publishing) may degrade if messages +%% are mostly published to topics with large number of levels. +%% +%% NOTE: This is a cluster-wide configuration. +%% It rquires all nodes to be stopped before changing it. +{mapping, "broker.perf.trie_compaction", "emqx.trie_compaction", [ + {default, true}, + {datatype, {enum, [true, false]}} +]}. + %%-------------------------------------------------------------------- %% System Monitor %%-------------------------------------------------------------------- diff --git a/rebar.config b/rebar.config index f3413a770..12bb8eb98 100644 --- a/rebar.config +++ b/rebar.config @@ -36,15 +36,17 @@ {erl_first_files, ["src/emqx_logger.erl", "src/emqx_rule_actions_trans.erl"]}. {deps, - [ {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} + [ {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.1.3"}}} + , {eredis_cluster, {git, "https://github.com/emqx/eredis_cluster", {tag, "0.6.5"}}} + , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.8.2"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.0"}}} - , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.0"}}} - , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.0"}}} + , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.1"}}} + , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}} , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {branch, "hocon"}}} , {minirest, {git, "https://github.com/emqx/minirest", {tag, "0.3.5"}}} - , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.0"}}} + , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.1"}}} , {replayq, {git, "https://github.com/emqx/replayq", {tag, "0.3.2"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {branch, "2.0.4"}}} , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.2.3"}}} @@ -52,7 +54,7 @@ , {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}} , {observer_cli, "1.6.1"} % NOTE: depends on recon 2.5.1 , {getopt, "1.0.1"} - , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.10.0"}}} + , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.12.0"}}} ]}. {xref_ignores, diff --git a/rebar.config.erl b/rebar.config.erl index 31b45d52a..b908f8097 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -106,8 +106,9 @@ test_deps() -> common_compile_opts() -> [ debug_info % alwyas include debug_info , {compile_info, [{emqx_vsn, get_vsn()}]} - | [{d, 'EMQX_ENTERPRISE'} || is_enterprise()] - ]. + ] ++ + [{d, 'EMQX_ENTERPRISE'} || is_enterprise()] ++ + [{d, 'EMQX_BENCHMARK'} || os:getenv("EMQX_BENCHMARK") =:= "1" ]. prod_compile_opts() -> [ compressed diff --git a/scripts/find-apps.sh b/scripts/find-apps.sh index 1f199269c..fabec239e 100755 --- a/scripts/find-apps.sh +++ b/scripts/find-apps.sh @@ -10,6 +10,9 @@ find_app() { find "${appdir}" -mindepth 1 -maxdepth 1 -type d } +# append emqx application first +echo 'emqx' + find_app 'apps' if [ -f 'EMQX_ENTERPRISE' ]; then find_app 'lib-ee' diff --git a/scripts/find-props.sh b/scripts/find-props.sh new file mode 100755 index 000000000..582ef26ae --- /dev/null +++ b/scripts/find-props.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +## this script prints out all test/props/prop_*.erl files of a given app, +## file names are separated by comma for proper's `-m` option + +set -euo pipefail + +# ensure dir +cd -P -- "$(dirname -- "$0")/.." + +BASEDIR="." +if [ "$1" != "emqx" ]; then + BASEDIR="$1" +fi + +find "${BASEDIR}/test/props" -name "prop_*.erl" 2>/dev/null | xargs -I{} basename {} .erl | xargs | tr ' ' ',' diff --git a/scripts/find-suites.sh b/scripts/find-suites.sh index 66296b758..97939d931 100755 --- a/scripts/find-suites.sh +++ b/scripts/find-suites.sh @@ -8,6 +8,8 @@ set -euo pipefail # ensure dir cd -P -- "$(dirname -- "$0")/.." -APPDIR="$1" - -find "${APPDIR}/test" -name "*_SUITE.erl" | tr -d '\r' | tr '\n' ',' +TESTDIR="test" +if [ "$1" != "emqx" ]; then + TESTDIR="$1/test" +fi +find "${TESTDIR}" -name "*_SUITE.erl" 2>/dev/null | xargs | tr ' ' ',' diff --git a/src/emqx.erl b/src/emqx.erl index a8af9d82d..2d2e4eb52 100644 --- a/src/emqx.erl +++ b/src/emqx.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -69,10 +69,6 @@ -define(APP, ?MODULE). --define(COPYRIGHT, "Copyright (c) 2020 EMQ Technologies Co., Ltd"). - --define(LICENSE_MESSAGE, "Licensed under the Apache License, Version 2.0"). - %% @hidden Path to the file which has debug_info encryption secret in it. %% Evaluate this function if there is a need to access encrypted debug_info. %% NOTE: Do not change the API to accept the secret text because it may diff --git a/src/emqx_access_control.erl b/src/emqx_access_control.erl index 32383b3f0..1ef885ed5 100644 --- a/src/emqx_access_control.erl +++ b/src/emqx_access_control.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_access_rule.erl b/src/emqx_access_rule.erl index 2931bc9f2..5a607dd16 100644 --- a/src/emqx_access_rule.erl +++ b/src/emqx_access_rule.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_acl_cache.erl b/src/emqx_acl_cache.erl index 2ba2bf04d..4cbe6b06a 100644 --- a/src/emqx_acl_cache.erl +++ b/src/emqx_acl_cache.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_alarm.erl b/src/emqx_alarm.erl index 7b4ca5989..62ce1af8b 100644 --- a/src/emqx_alarm.erl +++ b/src/emqx_alarm.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_alarm_handler.erl b/src/emqx_alarm_handler.erl index 667dc99e6..a69913afd 100644 --- a/src/emqx_alarm_handler.erl +++ b/src/emqx_alarm_handler.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_app.erl b/src/emqx_app.erl index 2a89f0da1..8501243a8 100644 --- a/src/emqx_app.erl +++ b/src/emqx_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -36,11 +36,14 @@ start(_Type, _Args) -> set_backtrace_depth(), print_otp_version_warning(), print_banner(), + %% Load application first for ekka_mnesia scanner + _ = load_ce_modules(), ekka:start(), {ok, Sup} = emqx_sup:start_link(), ok = start_autocluster(), ok = emqx_plugins:init(), _ = emqx_plugins:load(), + _ = start_ce_modules(), emqx_boot:is_enabled(listeners) andalso (ok = emqx_listeners:start()), register(emqx, self()), ok = emqx_alarm_handler:load(), @@ -58,6 +61,18 @@ set_backtrace_depth() -> _ = erlang:system_flag(backtrace_depth, Depth), ok. +-ifndef(EMQX_ENTERPRISE). +load_ce_modules() -> + application:load(emqx_modules). +start_ce_modules() -> + application:ensure_all_started(emqx_modules). +-else. +load_ce_modules() -> + ok. +start_ce_modules() -> + ok. +-endif. + %%-------------------------------------------------------------------- %% Print Banner %%-------------------------------------------------------------------- @@ -68,19 +83,25 @@ print_otp_version_warning() -> ok. print_otp_version_warning() -> io:format("WARNING: Running on Erlang/OTP version ~p. Recommended: 23~n", [?OTP_RELEASE]). --endif. +-endif. % OTP_RELEASE +-ifndef(TEST). print_banner() -> io:format("Starting ~s on node ~s~n", [?APP, node()]). --ifndef(TEST). print_vsn() -> io:format("~s ~s is running now!~n", [get_description(), get_release()]). --else. + +-else. % TEST + print_vsn() -> ok. --endif. + +print_banner() -> + ok. + +-endif. % TEST get_description() -> {ok, Descr0} = application:get_key(?APP, description), diff --git a/src/emqx_banned.erl b/src/emqx_banned.erl index 5c46a6e74..762a2b61b 100644 --- a/src/emqx_banned.erl +++ b/src/emqx_banned.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_base62.erl b/src/emqx_base62.erl index 52728faab..2ec3bbdf0 100644 --- a/src/emqx_base62.erl +++ b/src/emqx_base62.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_batch.erl b/src/emqx_batch.erl index c6b805e4e..5f9f5f830 100644 --- a/src/emqx_batch.erl +++ b/src/emqx_batch.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_boot.erl b/src/emqx_boot.erl index be27607be..9e14dcc4b 100644 --- a/src/emqx_boot.erl +++ b/src/emqx_boot.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_broker.erl b/src/emqx_broker.erl index c8d9c4b58..d3ad128bb 100644 --- a/src/emqx_broker.erl +++ b/src/emqx_broker.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -419,7 +419,7 @@ safe_update_stats(Tab, Stat, MaxStat) -> -compile({inline, [call/2, cast/2, pick/1]}). call(Broker, Req) -> - gen_server:call(Broker, Req). + gen_server:call(Broker, Req, infinity). cast(Broker, Msg) -> gen_server:cast(Broker, Msg). diff --git a/src/emqx_broker_bench.erl b/src/emqx_broker_bench.erl new file mode 100644 index 000000000..45ef0eab6 --- /dev/null +++ b/src/emqx_broker_bench.erl @@ -0,0 +1,115 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_broker_bench). + +-ifdef(EMQX_BENCHMARK). + +-export([start/1, run1/0, run1/2]). + +run1() -> run1(4, 1000). + +run1(Factor, Limit) -> + start(#{factor => Factor, + limit => Limit, + sub_ptn => <<"device/{{id}}/+/{{num}}/#">>, + pub_ptn => <<"device/{{id}}/xays/{{num}}/foo/bar/baz">>}). + +%% setting fields: +%% - factor: spawn broker-pool-size * factor number of callers +%% - limit: limit the total number of topics for each caller +%% - sub_ptn: subscribe topic pattern like a/+/b/+/c/# +%% or a/+/{{id}}/{{num}}/# to generate pattern with {{id}} +%% replaced by worker id and {{num}} replaced by topic number. +%% - pub_ptn: topic pattern used to benchmark publish (match) performance +%% e.g. a/x/{{id}}/{{num}}/foo/bar +start(#{factor := Factor} = Settings) -> + BrokerPoolSize = emqx_vm:schedulers() * 2, + Pids = start_callers(BrokerPoolSize * Factor, Settings), + R = collect_results(Pids, #{subscribe => 0, match => 0}), + io:format(user, "mnesia table(s) RAM: ~p~n", [ram_bytes()]), + io:format(user, "~p~n", [erlang:memory()]), + io:format(user, "~p~n", [R]), + lists:foreach(fun(Pid) -> Pid ! stop end, Pids). + +ram_bytes() -> + Wordsize = erlang:system_info(wordsize), + mnesia:table_info(emqx_trie, memory) * Wordsize + + case lists:member(emqx_trie_node, ets:all()) of + true -> + %% before 4.3 + mnesia:table_info(emqx_trie_node, memory) * Wordsize; + false -> + 0 + end. + +start_callers(0, _) -> []; +start_callers(N, Settings) -> + [start_caller(Settings#{id => N}) | start_callers(N - 1, Settings)]. + +collect_results([], R) -> R; +collect_results([Pid | Pids], Acc = #{subscribe := Sr, match := Mr}) -> + receive + {Pid, #{subscribe := Srd, match := Mrd}} -> + collect_results(Pids, Acc#{subscribe := Sr + Srd, match := Mr + Mrd}) + end. + +%% ops per second +rps(T, N) -> round(N / (T / 1000000)). + +start_caller(#{id := Id, limit := N, sub_ptn := SubPtn, pub_ptn := PubPtn}) -> + Parent = self(), + proc_lib:spawn_link( + fun() -> + SubTopics = make_topics(SubPtn, Id, N), + {Ts, _} = timer:tc(fun() -> subscribe(SubTopics) end), + PubTopics = make_topics(PubPtn, Id, N), + {Tm, _} = timer:tc(fun() -> match(PubTopics) end), + _ = erlang:send(Parent, {self(), #{subscribe => rps(Ts, N), match => rps(Tm, N)}}), + receive + stop -> + ok + end + end). + +match([]) -> ok; +match([Topic | Topics]) -> + _ = emqx_router:lookup_routes(Topic), + match(Topics). + +subscribe([]) -> ok; +subscribe([Topic | Rest]) -> + ok = emqx_broker:subscribe(Topic), + subscribe(Rest). + +make_topics(SubPtn0, Id, Limit) -> + SubPtn = emqx_topic:words(SubPtn0), + F = fun(N) -> render(Id, N, SubPtn) end, + lists:map(F, lists:seq(1, Limit)). + +render(ID, N, Ptn) -> + render(ID, N, Ptn, []). + +render(_ID, _N, [], Acc) -> + emqx_topic:join(lists:reverse(Acc)); +render(ID, N, [<<"{{id}}">> | T], Acc) -> + render(ID, N, T, [integer_to_binary(ID) | Acc]); +render(ID, N, [<<"{{num}}">> | T], Acc) -> + render(ID, N, T, [integer_to_binary(N) | Acc]); +render(ID, N, [H | T], Acc) -> + render(ID, N, T, [H | Acc]). + +-endif. diff --git a/src/emqx_broker_helper.erl b/src/emqx_broker_helper.erl index 18ebbccf2..705715c6c 100644 --- a/src/emqx_broker_helper.erl +++ b/src/emqx_broker_helper.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_broker_sup.erl b/src/emqx_broker_sup.erl index 8f62e5111..69df72408 100644 --- a/src/emqx_broker_sup.erl +++ b/src/emqx_broker_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_channel.erl b/src/emqx_channel.erl index 229ffa5cb..c8acccce9 100644 --- a/src/emqx_channel.erl +++ b/src/emqx_channel.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_cm.erl b/src/emqx_cm.erl index cb9792adb..7f324e322 100644 --- a/src/emqx_cm.erl +++ b/src/emqx_cm.erl @@ -1,5 +1,5 @@ %%------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -92,6 +92,8 @@ %% Server name -define(CM, ?MODULE). +-define(T_TAKEOVER, 15000). + %% @doc Start the channel manager. -spec(start_link() -> startlink_ret()). start_link() -> @@ -223,7 +225,7 @@ open_session(false, ClientInfo = #{clientid := ClientId}, ConnInfo) -> case takeover_session(ClientId) of {ok, ConnMod, ChanPid, Session} -> ok = emqx_session:resume(ClientInfo, Session), - Pendings = ConnMod:call(ChanPid, {takeover, 'end'}), + Pendings = ConnMod:call(ChanPid, {takeover, 'end'}, ?T_TAKEOVER), register_channel(ClientId, Self, ConnInfo), {ok, #{session => Session, present => true, @@ -265,7 +267,7 @@ takeover_session(ClientId, ChanPid) when node(ChanPid) == node() -> undefined -> {error, not_found}; ConnMod when is_atom(ConnMod) -> - Session = ConnMod:call(ChanPid, {takeover, 'begin'}), + Session = ConnMod:call(ChanPid, {takeover, 'begin'}, ?T_TAKEOVER), {ok, ConnMod, ChanPid, Session} end; @@ -295,7 +297,7 @@ discard_session(ClientId, ChanPid) when node(ChanPid) == node() -> case get_chann_conn_mod(ClientId, ChanPid) of undefined -> ok; ConnMod when is_atom(ConnMod) -> - ConnMod:call(ChanPid, discard) + ConnMod:call(ChanPid, discard, ?T_TAKEOVER) end; discard_session(ClientId, ChanPid) -> @@ -318,7 +320,7 @@ kick_session(ClientId) -> kick_session(ClientId, ChanPid) when node(ChanPid) == node() -> case get_chan_info(ClientId, ChanPid) of #{conninfo := #{conn_mod := ConnMod}} -> - ConnMod:call(ChanPid, kick); + ConnMod:call(ChanPid, kick, ?T_TAKEOVER); undefined -> {error, not_found} end; diff --git a/src/emqx_cm_locker.erl b/src/emqx_cm_locker.erl index b5979c706..4ce6a9279 100644 --- a/src/emqx_cm_locker.erl +++ b/src/emqx_cm_locker.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_cm_registry.erl b/src/emqx_cm_registry.erl index f3b1518cb..50a55c1bf 100644 --- a/src/emqx_cm_registry.erl +++ b/src/emqx_cm_registry.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_cm_sup.erl b/src/emqx_cm_sup.erl index 02c4cd3d8..f332a0868 100644 --- a/src/emqx_cm_sup.erl +++ b/src/emqx_cm_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_congestion.erl b/src/emqx_congestion.erl index b210a2cc3..ea99a63b1 100644 --- a/src/emqx_congestion.erl +++ b/src/emqx_congestion.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_connection.erl b/src/emqx_connection.erl index eee3b9c5a..69ffddf06 100644 --- a/src/emqx_connection.erl +++ b/src/emqx_connection.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -41,7 +41,9 @@ , stats/1 ]). --export([call/2]). +-export([ call/2 + , call/3 + ]). %% Callback -export([init/4]). @@ -54,7 +56,7 @@ ]). %% Internal callback --export([wakeup_from_hib/2]). +-export([wakeup_from_hib/2, recvloop/2]). %% Export for CT -export([set_field/3]). @@ -183,7 +185,9 @@ stats(#state{transport = Transport, lists:append([SockStats, ConnStats, ChanStats, ProcStats]). call(Pid, Req) -> - gen_server:call(Pid, Req, infinity). + call(Pid, Req, infinity). +call(Pid, Req, Timeout) -> + gen_server:call(Pid, Req, Timeout). stop(Pid) -> gen_server:stop(Pid). @@ -280,15 +284,22 @@ recvloop(Parent, State = #state{idle_timeout = IdleTimeout}) -> handle_recv({system, From, Request}, Parent, State) -> sys:handle_system_msg(Request, From, Parent, ?MODULE, [], State); handle_recv({'EXIT', Parent, Reason}, Parent, State) -> + %% FIXME: it's not trapping exit, should never receive an EXIT terminate(Reason, State); handle_recv(Msg, Parent, State = #state{idle_timeout = IdleTimeout}) -> - process_msg([Msg], Parent, ensure_stats_timer(IdleTimeout, State)). + case process_msg([Msg], ensure_stats_timer(IdleTimeout, State)) of + {ok, NewState} -> + ?MODULE:recvloop(Parent, NewState); + {stop, Reason, NewSate} -> + terminate(Reason, NewSate) + end. hibernate(Parent, State) -> proc_lib:hibernate(?MODULE, wakeup_from_hib, [Parent, State]). %% Maybe do something here later. -wakeup_from_hib(Parent, State) -> recvloop(Parent, State). +wakeup_from_hib(Parent, State) -> + ?MODULE:recvloop(Parent, State). %%-------------------------------------------------------------------- %% Ensure/cancel stats timer @@ -307,22 +318,31 @@ cancel_stats_timer(State) -> State. %%-------------------------------------------------------------------- %% Process next Msg -process_msg([], Parent, State) -> recvloop(Parent, State); - -process_msg([Msg|More], Parent, State) -> - case catch handle_msg(Msg, State) of - ok -> - process_msg(More, Parent, State); - {ok, NState} -> - process_msg(More, Parent, NState); - {ok, Msgs, NState} -> - process_msg(append_msg(More, Msgs), Parent, NState); - {stop, Reason} -> - terminate(Reason, State); - {stop, Reason, NState} -> - terminate(Reason, NState); - {'EXIT', Reason} -> - terminate(Reason, State) +process_msg([], State) -> + {ok, State}; +process_msg([Msg|More], State) -> + try + case handle_msg(Msg, State) of + ok -> + process_msg(More, State); + {ok, NState} -> + process_msg(More, NState); + {ok, Msgs, NState} -> + process_msg(append_msg(More, Msgs), NState); + {stop, Reason, NState} -> + {stop, Reason, NState} + end + catch + exit : normal -> + {stop, normal, State}; + exit : shutdown -> + {stop, shutdown, State}; + exit : {shutdown, _} = Shutdown -> + {stop, Shutdown, State}; + Exception : Context : Stack -> + {stop, #{exception => Exception, + context => Context, + stacktrace => Stack}, State} end. -compile({inline, [append_msg/2]}). @@ -446,18 +466,37 @@ handle_msg(Msg, State) -> -spec terminate(any(), state()) -> no_return(). terminate(Reason, State = #state{channel = Channel, transport = Transport, socket = Socket}) -> - ?tp(debug, terminate, #{reason => Reason}), - Channel1 = emqx_channel:set_conn_state(disconnected, Channel), - emqx_congestion:cancel_alarms(Socket, Transport, Channel1), - emqx_channel:terminate(Reason, Channel1), + try + Channel1 = emqx_channel:set_conn_state(disconnected, Channel), + emqx_congestion:cancel_alarms(Socket, Transport, Channel1), + emqx_channel:terminate(Reason, Channel1), + close_socket_ok(State) + catch + E : C : S -> + ?tp(warning, unclean_terminate, #{exception => E, context => C, stacktrace => S}) + end, + ?tp(debug, terminate, #{}), + maybe_raise_excption(Reason). + +%% close socket, discard new state, always return ok. +close_socket_ok(State) -> _ = close_socket(State), + ok. + +%% tell truth about the original exception +maybe_raise_excption(#{exception := Exception, + context := Context, + stacktrace := Stacktrace + }) -> + erlang:raise(Exception, Context, Stacktrace); +maybe_raise_excption(Reason) -> exit(Reason). %%-------------------------------------------------------------------- %% Sys callbacks system_continue(Parent, _Debug, State) -> - recvloop(Parent, State). + ?MODULE:recvloop(Parent, State). system_terminate(Reason, _Parent, _Debug, State) -> terminate(Reason, State). diff --git a/src/emqx_ctl.erl b/src/emqx_ctl.erl index 45cf34543..84aa54de6 100644 --- a/src/emqx_ctl.erl +++ b/src/emqx_ctl.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_flapping.erl b/src/emqx_flapping.erl index 0c1b60ddd..a0eab9c18 100644 --- a/src/emqx_flapping.erl +++ b/src/emqx_flapping.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_frame.erl b/src/emqx_frame.erl index 8dd3fa602..1af2f010a 100644 --- a/src/emqx_frame.erl +++ b/src/emqx_frame.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_gc.erl b/src/emqx_gc.erl index 3bf826f73..8232ed7f6 100644 --- a/src/emqx_gc.erl +++ b/src/emqx_gc.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_gen_mod.erl b/src/emqx_gen_mod.erl index 26a3c13cb..0ebf6b59a 100644 --- a/src/emqx_gen_mod.erl +++ b/src/emqx_gen_mod.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_global_gc.erl b/src/emqx_global_gc.erl index da953e347..51741ab1c 100644 --- a/src/emqx_global_gc.erl +++ b/src/emqx_global_gc.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_guid.erl b/src/emqx_guid.erl index 97e023d8c..3b66f6e92 100644 --- a/src/emqx_guid.erl +++ b/src/emqx_guid.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_hooks.erl b/src/emqx_hooks.erl index 33dca08cb..95ef33fdd 100644 --- a/src/emqx_hooks.erl +++ b/src/emqx_hooks.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -38,6 +38,11 @@ , lookup/1 ]). +-export([ callback_action/1 + , callback_filter/1 + , callback_priority/1 + ]). + %% gen_server Function Exports -export([ init/1 , handle_call/3 @@ -87,6 +92,19 @@ start_link() -> stop() -> gen_server:stop(?SERVER, normal, infinity). +%%-------------------------------------------------------------------- +%% Test APIs +%%-------------------------------------------------------------------- + +%% @doc Get callback action. +callback_action(#callback{action = A}) -> A. + +%% @doc Get callback filter. +callback_filter(#callback{filter = F}) -> F. + +%% @doc Get callback priority. +callback_priority(#callback{priority= P}) -> P. + %%-------------------------------------------------------------------- %% Hooks API %%-------------------------------------------------------------------- diff --git a/src/emqx_http_lib.erl b/src/emqx_http_lib.erl index 195656622..9e86fe394 100644 --- a/src/emqx_http_lib.erl +++ b/src/emqx_http_lib.erl @@ -31,22 +31,40 @@ fragment => unicode:chardata(), userinfo => unicode:chardata()}. +-type hex_uri() :: string() | binary(). +-type maybe_hex_uri() :: string() | binary(). %% A possibly hexadecimal encoded URI. +-type uri() :: string() | binary(). + %% @doc Decode percent-encoded URI. %% This is copied from http_uri.erl which has been deprecated since OTP-23 %% The recommended replacement uri_string function is not quite equivalent %% and not backward compatible. --spec uri_decode(binary()) -> binary(). -uri_decode(<<$%, Hex:2/binary, Rest/bits>>) -> - <<(binary_to_integer(Hex, 16)), (uri_decode(Rest))/binary>>; -uri_decode(<>) -> - <>; -uri_decode(<<>>) -> +-spec uri_decode(maybe_hex_uri()) -> uri(). +uri_decode(String) when is_list(String) -> + do_uri_decode(String); +uri_decode(String) when is_binary(String) -> + do_uri_decode_binary(String). + +do_uri_decode([$%,Hex1,Hex2|Rest]) -> + [hex2dec(Hex1)*16+hex2dec(Hex2)|do_uri_decode(Rest)]; +do_uri_decode([First|Rest]) -> + [First|do_uri_decode(Rest)]; +do_uri_decode([]) -> + []. + +do_uri_decode_binary(<<$%, Hex:2/binary, Rest/bits>>) -> + <<(binary_to_integer(Hex, 16)), (do_uri_decode_binary(Rest))/binary>>; +do_uri_decode_binary(<>) -> + <>; +do_uri_decode_binary(<<>>) -> <<>>. %% @doc Encode URI. --spec uri_encode(binary()) -> binary(). +-spec uri_encode(uri()) -> hex_uri(). +uri_encode(URI) when is_list(URI) -> + lists:append([do_uri_encode(Char) || Char <- URI]); uri_encode(URI) when is_binary(URI) -> - << <<(uri_encode_binary(Char))/binary>> || <> <= URI >>. + << <<(do_uri_encode_binary(Char))/binary>> || <> <= URI >>. %% @doc Parse URI into a map as uri_string:uri_map(), but with two fields %% normalised: (1): port number is never 'undefined', default ports are used @@ -73,7 +91,7 @@ do_parse(URI) -> normalise_parse_result(Map2) end. -normalise_parse_result(#{host := _, scheme := Scheme0} = Map) -> +normalise_parse_result(#{host := Host, scheme := Scheme0} = Map) -> Scheme = atom_scheme(Scheme0), DefaultPort = case https =:= Scheme of true -> 443; @@ -83,7 +101,8 @@ normalise_parse_result(#{host := _, scheme := Scheme0} = Map) -> N when is_number(N) -> N; _ -> DefaultPort end, - Map#{ scheme => Scheme + Map#{ scheme := Scheme + , host := emqx_misc:maybe_parse_ip(Host) , port => Port }. @@ -93,7 +112,15 @@ atom_scheme(<<"https">>) -> https; atom_scheme(<<"http">>) -> http; atom_scheme(Other) -> throw({unsupported_scheme, Other}). -uri_encode_binary(Char) -> +do_uri_encode(Char) -> + case reserved(Char) of + true -> + [ $% | integer_to_hexlist(Char)]; + false -> + [Char] + end. + +do_uri_encode_binary(Char) -> case reserved(Char) of true -> << $%, (integer_to_binary(Char, 16))/binary >>; @@ -125,3 +152,10 @@ reserved($^) -> true; reserved($%) -> true; reserved($\s) -> true; reserved(_) -> false. + +integer_to_hexlist(Int) -> + integer_to_list(Int, 16). + +hex2dec(X) when (X>=$0) andalso (X=<$9) -> X-$0; +hex2dec(X) when (X>=$A) andalso (X=<$F) -> X-$A+10; +hex2dec(X) when (X>=$a) andalso (X=<$f) -> X-$a+10. diff --git a/src/emqx_inflight.erl b/src/emqx_inflight.erl index c94a8cc48..4ad94e38f 100644 --- a/src/emqx_inflight.erl +++ b/src/emqx_inflight.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_json.erl b/src/emqx_json.erl index e79cf09d3..d2cdb83fe 100644 --- a/src/emqx_json.erl +++ b/src/emqx_json.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_keepalive.erl b/src/emqx_keepalive.erl index daa8e5221..8fba00f50 100644 --- a/src/emqx_keepalive.erl +++ b/src/emqx_keepalive.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_kernel_sup.erl b/src/emqx_kernel_sup.erl index 82f196913..042416a40 100644 --- a/src/emqx_kernel_sup.erl +++ b/src/emqx_kernel_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_limiter.erl b/src/emqx_limiter.erl index 447e04fea..181e5c6bf 100644 --- a/src/emqx_limiter.erl +++ b/src/emqx_limiter.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_listeners.erl b/src/emqx_listeners.erl index 01ee753c4..1f3d1776b 100644 --- a/src/emqx_listeners.erl +++ b/src/emqx_listeners.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_logger.erl b/src/emqx_logger.erl index 5c6808840..a733b0d3a 100644 --- a/src/emqx_logger.erl +++ b/src/emqx_logger.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -138,7 +138,15 @@ critical(Metadata, Format, Args) when is_map(Metadata) -> set_metadata_clientid(<<>>) -> ok; set_metadata_clientid(ClientId) -> - set_proc_metadata(#{clientid => ClientId}). + try + %% try put string format client-id metadata so + %% so the log is not like <<"...">> + Id = unicode:characters_to_list(ClientId, utf8), + set_proc_metadata(#{clientid => Id}) + catch + _: _-> + ok + end. -spec(set_metadata_peername(peername_str()) -> ok). set_metadata_peername(Peername) -> diff --git a/src/emqx_logger_formatter.erl b/src/emqx_logger_formatter.erl deleted file mode 100644 index b8642271d..000000000 --- a/src/emqx_logger_formatter.erl +++ /dev/null @@ -1,378 +0,0 @@ -%% -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2013-2019. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%% -%% %CopyrightEnd% -%% - -%% This file is copied from lib/kernel/src/logger_formatter.erl, and -%% modified for a more concise time format other than the default RFC3339. - --module(emqx_logger_formatter). - --export([format/2]). - --export([check_config/1]). - --define(DEFAULT_FORMAT_TEMPLATE_SINGLE, [time," ",level,": ",msg,"\n"]). - --define(FormatP, "~0tp"). - --define(IS_STRING(String), - (is_list(String) orelse is_binary(String))). - -%%-------------------------------------------------------------------- -%% Types - --type(config() :: #{chars_limit => pos_integer() | unlimited, - depth => pos_integer() | unlimited, - max_size => pos_integer() | unlimited, - report_cb => logger:report_cb(), - quit => template()}). - --type(template() :: [metakey() | {metakey(),template(),template()} | string()]). - --type(metakey() :: atom() | [atom()]). - -%%-------------------------------------------------------------------- -%% API - --spec format(LogEvent,Config) -> unicode:chardata() when - LogEvent :: logger:log_event(), - Config :: config(). -format(#{level:=Level,msg:=Msg0,meta:=Meta},Config0) - when is_map(Config0) -> - Config = add_default_config(Config0), - Template = maps:get(template,Config), - {BT,AT0} = lists:splitwith(fun(msg) -> false; (_) -> true end, Template), - {DoMsg,AT} = - case AT0 of - [msg|Rest] -> {true,Rest}; - _ ->{false,AT0} - end, - B = do_format(Level,Meta,BT,Config), - A = do_format(Level,Meta,AT,Config), - MsgStr = - if DoMsg -> - Config1 = - case maps:get(chars_limit,Config) of - unlimited -> - Config; - Size0 -> - Size = - case Size0 - string:length([B,A]) of - S when S>=0 -> S; - _ -> 0 - end, - Config#{chars_limit=>Size} - end, - format_msg(Msg0,Meta,Config1); - true -> - "" - end, - truncate([B,MsgStr,A],maps:get(max_size,Config)). - -do_format(Level,Data,[level|Format],Config) -> - [to_string(level,Level,Config)|do_format(Level,Data,Format,Config)]; -do_format(Level,Data,[{Key,IfExist,Else}|Format],Config) -> - String = - case value(Key,Data) of - {ok,Value} -> do_format(Level,Data#{Key=>Value},IfExist,Config); - error -> do_format(Level,Data,Else,Config) - end, - [String|do_format(Level,Data,Format,Config)]; -do_format(Level,Data,[Key|Format],Config) - when is_atom(Key) orelse - (is_list(Key) andalso is_atom(hd(Key))) -> - String = - case value(Key,Data) of - {ok,Value} -> to_string(Key,Value,Config); - error -> "" - end, - [String|do_format(Level,Data,Format,Config)]; -do_format(Level,Data,[Str|Format],Config) -> - [Str|do_format(Level,Data,Format,Config)]; -do_format(_Level,_Data,[],_Config) -> - []. - -value(Key,Meta) when is_map_key(Key,Meta) -> - {ok,maps:get(Key,Meta)}; -value([Key|Keys],Meta) when is_map_key(Key,Meta) -> - value(Keys,maps:get(Key,Meta)); -value([],Value) -> - {ok,Value}; -value(_,_) -> - error. - -to_string(time,Time,Config) -> - format_time(Time,Config); -to_string(mfa,MFA,Config) -> - format_mfa(MFA,Config); -to_string(_,Value,Config) -> - to_string(Value,Config). - -to_string(X,_) when is_atom(X) -> - atom_to_list(X); -to_string(X,_) when is_integer(X) -> - integer_to_list(X); -to_string(X,_) when is_pid(X) -> - pid_to_list(X); -to_string(X,_) when is_reference(X) -> - ref_to_list(X); -to_string(X,_) when is_list(X) -> - case printable_list(lists:flatten(X)) of - true -> X; - _ -> io_lib:format(?FormatP,[X]) - end; -to_string(X,_) -> - io_lib:format(?FormatP,[X]). - -printable_list([]) -> - false; -printable_list(X) -> - io_lib:printable_list(X). - -format_msg({string,Chardata},Meta,Config) -> - format_msg({"~ts",[Chardata]},Meta,Config); -format_msg({report,_}=Msg,Meta,#{report_cb:=Fun}=Config) - when is_function(Fun,1); is_function(Fun,2) -> - format_msg(Msg,Meta#{report_cb=>Fun},maps:remove(report_cb,Config)); -format_msg({report,Report},#{report_cb:=Fun}=Meta,Config) when is_function(Fun,1) -> - try Fun(Report) of - {Format,Args} when is_list(Format), is_list(Args) -> - format_msg({Format,Args},maps:remove(report_cb,Meta),Config); - Other -> - format_msg({"REPORT_CB/1 ERROR: ~0tp; Returned: ~0tp", - [Report,Other]},Meta,Config) - catch C:R:S -> - format_msg({"REPORT_CB/1 CRASH: ~0tp; Reason: ~0tp", - [Report,{C,R,logger:filter_stacktrace(?MODULE,S)}]},Meta,Config) - end; -format_msg({report,Report},#{report_cb:=Fun}=Meta,Config) when is_function(Fun,2) -> - try Fun(Report,maps:with([depth,chars_limit,single_line],Config)) of - Chardata when ?IS_STRING(Chardata) -> - try chardata_to_list(Chardata) % already size limited by report_cb - catch _:_ -> - format_msg({"REPORT_CB/2 ERROR: ~0tp; Returned: ~0tp", - [Report,Chardata]},Meta,Config) - end; - Other -> - format_msg({"REPORT_CB/2 ERROR: ~0tp; Returned: ~0tp", - [Report,Other]},Meta,Config) - catch C:R:S -> - format_msg({"REPORT_CB/2 CRASH: ~0tp; Reason: ~0tp", - [Report,{C,R,logger:filter_stacktrace(?MODULE,S)}]}, - Meta,Config) - end; -format_msg({report,Report},Meta,Config) -> - format_msg({report,Report}, - Meta#{report_cb=>fun logger:format_report/1}, - Config); -format_msg(Msg,_Meta,#{depth:=Depth,chars_limit:=CharsLimit}) -> - Opts = chars_limit_to_opts(CharsLimit), - do_format_msg(Msg, Depth, Opts). - -chars_limit_to_opts(unlimited) -> []; -chars_limit_to_opts(CharsLimit) -> [{chars_limit,CharsLimit}]. - -do_format_msg({Format0,Args},Depth,Opts) -> - try - Format1 = io_lib:scan_format(Format0, Args), - Format = reformat(Format1, Depth), - io_lib:build_text(Format,Opts) - catch C:R:S -> - FormatError = "FORMAT ERROR: ~0tp - ~0tp", - case Format0 of - FormatError -> - %% already been here - avoid failing cyclically - erlang:raise(C,R,S); - _ -> - do_format_msg({FormatError,[Format0,Args]},Depth,Opts) - end - end. - -reformat(Format,unlimited) -> - Format; -reformat([#{control_char:=C}=M|T], Depth) when C =:= $p -> - [limit_depth(M#{width => 0}, Depth)|reformat(T, Depth)]; -reformat([#{control_char:=C}=M|T], Depth) when C =:= $P -> - [M#{width => 0}|reformat(T, Depth)]; -reformat([#{control_char:=C}=M|T], Depth) when C =:= $p; C =:= $w -> - [limit_depth(M, Depth)|reformat(T, Depth)]; -reformat([H|T], Depth) -> - [H|reformat(T, Depth)]; -reformat([], _) -> - []. - -limit_depth(M0, unlimited) -> - M0; -limit_depth(#{control_char:=C0, args:=Args}=M0, Depth) -> - C = C0 - ($a - $A), %To uppercase. - M0#{control_char:=C,args:=Args++[Depth]}. - -chardata_to_list(Chardata) -> - case unicode:characters_to_list(Chardata,unicode) of - List when is_list(List) -> - List; - Error -> - throw(Error) - end. - -truncate(String,unlimited) -> - String; -truncate(String,Size) -> - Length = string:length(String), - if Length>Size -> - case lists:reverse(lists:flatten(String)) of - [$\n|_] -> - string:slice(String,0,Size-4)++"...\n"; - _ -> - string:slice(String,0,Size-3)++"..." - end; - true -> - String - end. - -format_time(SysTime, Config) when is_integer(SysTime) -> - Offset = maps:get(time_offset, Config, ""), - calendar:system_time_to_rfc3339(SysTime, [{unit,microsecond}, - {offset,Offset}]). - -format_mfa({M,F,A},_) when is_atom(M), is_atom(F), is_integer(A) -> - atom_to_list(M)++":"++atom_to_list(F)++"/"++integer_to_list(A); -format_mfa({M,F,A},Config) when is_atom(M), is_atom(F), is_list(A) -> - format_mfa({M,F,length(A)},Config); -format_mfa(MFA,Config) -> - to_string(MFA,Config). - -%% Ensure that all valid configuration parameters exist in the final -%% configuration map -add_default_config(Config0) -> - Default = - #{chars_limit=>unlimited, - error_logger_notice_header=>info}, - MaxSize = get_max_size(maps:get(max_size,Config0,undefined)), - Depth = get_depth(maps:get(depth,Config0,undefined)), - add_default_template(maps:merge(Default,Config0#{max_size=>MaxSize, - depth=>Depth})). - -add_default_template(#{template:=_}=Config) -> - Config; -add_default_template(Config) -> - Config#{template=>?DEFAULT_FORMAT_TEMPLATE_SINGLE}. - -get_max_size(undefined) -> - unlimited; -get_max_size(S) -> - max(10,S). - -get_depth(undefined) -> - error_logger:get_format_depth(); -get_depth(S) -> - max(5,S). - --spec check_config(Config) -> ok | {error,term()} when - Config :: config(). -check_config(Config) when is_map(Config) -> - do_check_config(maps:to_list(Config)); -check_config(Config) -> - {error,{invalid_formatter_config,?MODULE,Config}}. - -do_check_config([{Type,L}|Config]) when Type == chars_limit; - Type == depth; - Type == max_size -> - case check_limit(L) of - ok -> do_check_config(Config); - error -> {error,{invalid_formatter_config,?MODULE,{Type,L}}} - end; -do_check_config([{error_logger_notice_header,ELNH}|Config]) when ELNH == info; - ELNH == notice -> - do_check_config(Config); -do_check_config([{report_cb,RCB}|Config]) when is_function(RCB,1); - is_function(RCB,2) -> - do_check_config(Config); -do_check_config([{template,T}|Config]) -> - case check_template(T) of - ok -> do_check_config(Config); - error -> {error,{invalid_formatter_template,?MODULE,T}} - end; -do_check_config([{time_offset, Offset} | Config]) -> - case lists:member(Offset, ["", "0", "Z", "z"]) orelse check_time_offset(Offset) of - true -> do_check_config(Config); - error -> {error, {time_offset, ?MODULE, Offset}} - end; -do_check_config([C|_]) -> - {error,{invalid_formatter_config,?MODULE,C}}; -do_check_config([]) -> - ok. - -check_time_offset([S, H1, H2, $:, M1, M2]) -> - (S =:= $+ orelse S =:= $-) andalso - try - begin - H = list_to_integer([H1, H2]), - M = list_to_integer([M1, M2]), - H >=0 andalso H =< 14 andalso - M >= 0 andalso M =< 59 - end - catch - _ : _ -> - error - end; -check_time_offset(_) -> error. - -check_limit(L) when is_integer(L), L>0 -> - ok; -check_limit(unlimited) -> - ok; -check_limit(_) -> - error. - -check_template([Key|T]) when is_atom(Key) -> - check_template(T); -check_template([Key|T]) when is_list(Key), is_atom(hd(Key)) -> - case lists:all(fun(X) when is_atom(X) -> true; - (_) -> false - end, - Key) of - true -> - check_template(T); - false -> - error - end; -check_template([{Key,IfExist,Else}|T]) - when is_atom(Key) orelse - (is_list(Key) andalso is_atom(hd(Key))) -> - case check_template(IfExist) of - ok -> - case check_template(Else) of - ok -> - check_template(T); - error -> - error - end; - error -> - error - end; -check_template([Str|T]) when is_list(Str) -> - case io_lib:printable_unicode_list(Str) of - true -> check_template(T); - false -> error - end; -check_template([]) -> - ok; -check_template(_) -> - error. diff --git a/src/emqx_message.erl b/src/emqx_message.erl index 3050ae517..faae621d8 100644 --- a/src/emqx_message.erl +++ b/src/emqx_message.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_metrics.erl b/src/emqx_metrics.erl index 88acec4e7..44009d4e2 100644 --- a/src/emqx_metrics.erl +++ b/src/emqx_metrics.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_misc.erl b/src/emqx_misc.erl index bf6bcd4a6..04af5f72c 100644 --- a/src/emqx_misc.erl +++ b/src/emqx_misc.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -43,6 +43,8 @@ , now_to_secs/1 , now_to_ms/1 , index_of/2 + , maybe_parse_ip/1 + , ipv6_probe/1 ]). -export([ bin2hexstr_A_F/1 @@ -50,6 +52,30 @@ , hexstr2bin/1 ]). +%% @doc Parse v4 or v6 string format address to tuple. +%% `Host' itself is returned if it's not an ip string. +maybe_parse_ip(Host) -> + case inet:parse_address(Host) of + {ok, Addr} when is_tuple(Addr) -> Addr; + {error, einval} -> Host + end. + +%% @doc Add `ipv6_probe' socket option if it's supported. +ipv6_probe(Opts) -> + case persistent_term:get({?MODULE, ipv6_probe_supported}, unknown) of + unknown -> + %% e.g. 23.2.7.1-emqx-2-x86_64-unknown-linux-gnu-64 + OtpVsn = emqx_vm:get_otp_version(), + Bool = (match =:= re:run(OtpVsn, "emqx", [{capture, none}])), + _ = persistent_term:put({?MODULE, ipv6_probe_supported}, Bool), + ipv6_probe(Bool, Opts); + Bool -> + ipv6_probe(Bool, Opts) + end. + +ipv6_probe(false, Opts) -> Opts; +ipv6_probe(true, Opts) -> [{ipv6_probe, true} | Opts]. + %% @doc Merge options -spec(merge_opts(Opts, Opts) -> Opts when Opts :: proplists:proplist()). merge_opts(Defaults, Options) -> @@ -258,3 +284,10 @@ hexchar2int(I) when I >= $0 andalso I =< $9 -> I - $0; hexchar2int(I) when I >= $A andalso I =< $F -> I - $A + 10; hexchar2int(I) when I >= $a andalso I =< $f -> I - $a + 10. +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +ipv6_probe_test() -> + ?assertEqual([{ipv6_probe, true}], ipv6_probe([])). + +-endif. diff --git a/src/emqx_mountpoint.erl b/src/emqx_mountpoint.erl index 8dfc23b30..f3c7a94de 100644 --- a/src/emqx_mountpoint.erl +++ b/src/emqx_mountpoint.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_mqtt_caps.erl b/src/emqx_mqtt_caps.erl index f29d59915..b1be5d5a5 100644 --- a/src/emqx_mqtt_caps.erl +++ b/src/emqx_mqtt_caps.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_mqtt_props.erl b/src/emqx_mqtt_props.erl index 7acea6761..43ea4c0e1 100644 --- a/src/emqx_mqtt_props.erl +++ b/src/emqx_mqtt_props.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_mqueue.erl b/src/emqx_mqueue.erl index f77a1d98f..d0c6365ff 100644 --- a/src/emqx_mqueue.erl +++ b/src/emqx_mqueue.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_os_mon.erl b/src/emqx_os_mon.erl index 0b057e1f8..d6579cac9 100644 --- a/src/emqx_os_mon.erl +++ b/src/emqx_os_mon.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_packet.erl b/src/emqx_packet.erl index 015f95e0d..7f154b3da 100644 --- a/src/emqx_packet.erl +++ b/src/emqx_packet.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_passwd.erl b/src/emqx_passwd.erl index a60e96a46..f6c382ef9 100644 --- a/src/emqx_passwd.erl +++ b/src/emqx_passwd.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_pd.erl b/src/emqx_pd.erl index 8754ac090..e67489ee0 100644 --- a/src/emqx_pd.erl +++ b/src/emqx_pd.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_plugins.erl b/src/emqx_plugins.erl index 5a074d5b3..bf8de4a59 100644 --- a/src/emqx_plugins.erl +++ b/src/emqx_plugins.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_pmon.erl b/src/emqx_pmon.erl index 99105f6e0..1f9ba7af6 100644 --- a/src/emqx_pmon.erl +++ b/src/emqx_pmon.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_pool.erl b/src/emqx_pool.erl index 49a7c2cac..0a336264d 100644 --- a/src/emqx_pool.erl +++ b/src/emqx_pool.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_pool_sup.erl b/src/emqx_pool_sup.erl index 080cd842b..6587ed7f8 100644 --- a/src/emqx_pool_sup.erl +++ b/src/emqx_pool_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_psk.erl b/src/emqx_psk.erl index f1170d2d9..f8be93056 100644 --- a/src/emqx_psk.erl +++ b/src/emqx_psk.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_reason_codes.erl b/src/emqx_reason_codes.erl index 8e96df165..893084b9d 100644 --- a/src/emqx_reason_codes.erl +++ b/src/emqx_reason_codes.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_router.erl b/src/emqx_router.erl index 754124298..7288d2e19 100644 --- a/src/emqx_router.erl +++ b/src/emqx_router.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -118,7 +118,8 @@ do_add_route(Topic, Dest) when is_binary(Topic) -> false -> ok = emqx_router_helper:monitor(Dest), case emqx_topic:wildcard(Topic) of - true -> trans(fun insert_trie_route/1, [Route]); + true -> + maybe_trans(fun insert_trie_route/1, [Route]); false -> insert_direct_route(Route) end end. @@ -132,12 +133,11 @@ match_routes(Topic) when is_binary(Topic) -> lists:append([lookup_routes(To) || To <- [Topic | Matched]]) end. -%% @private %% Optimize: routing table will be replicated to all router nodes. match_trie(Topic) -> case emqx_trie:empty() of true -> []; - false -> mnesia:ets(fun emqx_trie:match/1, [Topic]) + false -> emqx_trie:match(Topic) end. -spec(lookup_routes(emqx_topic:topic()) -> [emqx_types:route()]). @@ -164,7 +164,8 @@ do_delete_route(Topic) when is_binary(Topic) -> do_delete_route(Topic, Dest) -> Route = #route{topic = Topic, dest = Dest}, case emqx_topic:wildcard(Topic) of - true -> trans(fun delete_trie_route/1, [Route]); + true -> + maybe_trans(fun delete_trie_route/1, [Route]); false -> delete_direct_route(Route) end. @@ -247,10 +248,59 @@ delete_trie_route(Route = #route{topic = Topic}) -> end. %% @private --spec(trans(function(), list(any())) -> ok | {error, term()}). -trans(Fun, Args) -> - case mnesia:transaction(Fun, Args) of - {atomic, Ok} -> Ok; - {aborted, Reason} -> {error, Reason} +-spec(maybe_trans(function(), list(any())) -> ok | {error, term()}). +maybe_trans(Fun, Args) -> + case persistent_term:get(emqx_route_lock_type) of + key -> + trans(Fun, Args); + global -> + lock_router(), + try mnesia:sync_dirty(Fun, Args) + after + unlock_router() + end; + tab -> + trans(fun() -> + emqx_trie:lock_tables(), + apply(Fun, Args) + end, []) end. +-spec(trans(function(), list(any())) -> ok | {error, term()}). +trans(Fun, Args) -> + %% trigger selective receive optimization of compiler, + %% ideal for handling bursty traffic. + Ref = erlang:make_ref(), + Owner = self(), + {WPid, RefMon} = spawn_monitor( + fun() -> + Res = case mnesia:transaction(Fun, Args) of + {atomic, Ok} -> Ok; + {aborted, Reason} -> {error, Reason} + end, + Owner ! {Ref, Res} + end), + receive + {Ref, TransRes} -> + receive + {'DOWN', RefMon, process, WPid, normal} -> ok + end, + TransRes; + {'DOWN', RefMon, process, WPid, Info} -> + {error, {trans_crash, Info}} + end. + +lock_router() -> + %% if Retry is not 0, global:set_lock could sleep a random time up to 8s. + %% Considering we have a limited number of brokers, it is safe to use sleep 1 ms. + case global:set_lock({?MODULE, self()}, [node() | nodes()], 0) of + false -> + %% Force to sleep 1ms instead. + timer:sleep(1), + lock_router(); + true -> + ok + end. + +unlock_router() -> + global:del_lock({?MODULE, self()}). diff --git a/src/emqx_router_helper.erl b/src/emqx_router_helper.erl index 5a7c6479f..f05220791 100644 --- a/src/emqx_router_helper.erl +++ b/src/emqx_router_helper.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_router_sup.erl b/src/emqx_router_sup.erl index 9060e58bf..1105a476b 100644 --- a/src/emqx_router_sup.erl +++ b/src/emqx_router_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -34,6 +34,10 @@ init([]) -> type => worker, modules => [emqx_router_helper]}, + ok = persistent_term:put(emqx_route_lock_type, + application:get_env(emqx, route_lock_type, key) + ), + %% Router pool RouterPool = emqx_pool_sup:spec([router_pool, hash, {emqx_router, start_link, []}]), diff --git a/src/emqx_rpc.erl b/src/emqx_rpc.erl index 727eb458b..a37d67a0a 100644 --- a/src/emqx_rpc.erl +++ b/src/emqx_rpc.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_sequence.erl b/src/emqx_sequence.erl index d0823cd82..caa38a99c 100644 --- a/src/emqx_sequence.erl +++ b/src/emqx_sequence.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_session.erl b/src/emqx_session.erl index 4dd26d3da..9463345d4 100644 --- a/src/emqx_session.erl +++ b/src/emqx_session.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_shared_sub.erl b/src/emqx_shared_sub.erl index ed18f1ba7..4707f63db 100644 --- a/src/emqx_shared_sub.erl +++ b/src/emqx_shared_sub.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_stats.erl b/src/emqx_stats.erl index a6d53cfeb..f53549e65 100644 --- a/src/emqx_stats.erl +++ b/src/emqx_stats.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_sup.erl b/src/emqx_sup.erl index 1782b64a0..3bddac1ed 100644 --- a/src/emqx_sup.erl +++ b/src/emqx_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_sys.erl b/src/emqx_sys.erl index 11f9f5635..2d816569d 100644 --- a/src/emqx_sys.erl +++ b/src/emqx_sys.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_sys_mon.erl b/src/emqx_sys_mon.erl index 632dcf579..152f975eb 100644 --- a/src/emqx_sys_mon.erl +++ b/src/emqx_sys_mon.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_sys_sup.erl b/src/emqx_sys_sup.erl index 0a246e22a..50d086156 100644 --- a/src/emqx_sys_sup.erl +++ b/src/emqx_sys_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_tables.erl b/src/emqx_tables.erl index 82ea7b042..6483c3023 100644 --- a/src/emqx_tables.erl +++ b/src/emqx_tables.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_topic.erl b/src/emqx_topic.erl index e4ba90023..00d26d147 100644 --- a/src/emqx_topic.erl +++ b/src/emqx_topic.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_tracer.erl b/src/emqx_tracer.erl index ef7e73a8d..995712f6c 100644 --- a/src/emqx_tracer.erl +++ b/src/emqx_tracer.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -31,7 +31,7 @@ -type(trace_who() :: {clientid | topic, binary()}). -define(TRACER, ?MODULE). --define(FORMAT, {emqx_logger_formatter, +-define(FORMAT, {logger_formatter, #{template => [time, " [", level, "] ", {clientid, @@ -41,7 +41,9 @@ [{peername, [peername, " "], []}]}, - msg, "\n"]}}). + msg, "\n"], + single_line => false + }}). -define(TOPIC_TRACE_ID(T), "trace_topic_"++T). -define(CLIENT_TRACE_ID(C), "trace_clientid_"++C). -define(TOPIC_TRACE(T), {topic, T}). @@ -143,17 +145,19 @@ handler_id(?TOPIC_TRACE(Topic)) -> handler_id(?CLIENT_TRACE(ClientId)) -> list_to_atom(?CLIENT_TRACE_ID(handler_name(ClientId))). -filter_by_meta_key(#{meta:=Meta}=LogEvent, {MetaKey, MetaValue}) -> - case maps:find(MetaKey, Meta) of - {ok, MetaValue} -> LogEvent; - {ok, Topic} when MetaKey =:= topic -> - case emqx_topic:match(Topic, MetaValue) of - true -> LogEvent; - false -> ignore - end; - _ -> ignore +filter_by_meta_key(#{meta := Meta} = Log, {Key, Value}) -> + case is_meta_match(Key, Value, Meta) of + true -> Log; + false -> ignore end. +is_meta_match(clientid, ClientId, #{clientid := ClientIdStr}) -> + ClientId =:= iolist_to_binary(ClientIdStr); +is_meta_match(topic, TopicFilter, #{topic := TopicMeta}) -> + emqx_topic:match(TopicMeta, TopicFilter); +is_meta_match(_, _, _) -> + false. + handler_name(Bin) -> case byte_size(Bin) of Size when Size =< 200 -> binary_to_list(Bin); diff --git a/src/emqx_trie.erl b/src/emqx_trie.erl index ae85d92c2..e8a51b8ef 100644 --- a/src/emqx_trie.erl +++ b/src/emqx_trie.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -27,189 +27,306 @@ %% Trie APIs -export([ insert/1 , match/1 - , lookup/1 , delete/1 + , put_compaction_flag/1 + , put_default_compaction_flag/0 ]). --export([empty/0]). +-export([ empty/0 + , lock_tables/0 + ]). -ifdef(TEST). -compile(export_all). -compile(nowarn_export_all). -endif. --type(triple() :: {root | binary(), emqx_topic:word(), binary()}). +-define(TRIE, emqx_trie). +-define(PREFIX(Prefix), {Prefix, 0}). +-define(TOPIC(Topic), {Topic, 1}). -%% Mnesia tables --define(TRIE_TAB, emqx_trie). --define(TRIE_NODE_TAB, emqx_trie_node). +-record(?TRIE, + { key :: ?TOPIC(binary()) | ?PREFIX(binary()) + , count = 0 :: non_neg_integer() + }). --elvis([{elvis_style, function_naming_convention, disable}]). +-define(IS_COMPACT, true). %%-------------------------------------------------------------------- %% Mnesia bootstrap %%-------------------------------------------------------------------- -%% @doc Create or replicate trie tables. +put_compaction_flag(Bool) when is_boolean(Bool) -> + _ = persistent_term:put({?MODULE, compaction}, Bool), + ok. + +put_default_compaction_flag() -> + ok = put_compaction_flag(?IS_COMPACT). + +%% @doc Create or replicate topics table. -spec(mnesia(boot | copy) -> ok). mnesia(boot) -> %% Optimize storage StoreProps = [{ets, [{read_concurrency, true}, - {write_concurrency, true}]}], - %% Trie table - ok = ekka_mnesia:create_table(?TRIE_TAB, [ + {write_concurrency, true} + ]}], + ok = ekka_mnesia:create_table(?TRIE, [ {ram_copies, [node()]}, - {record_name, trie}, - {attributes, record_info(fields, trie)}, - {storage_properties, StoreProps}]), - %% Trie node table - ok = ekka_mnesia:create_table(?TRIE_NODE_TAB, [ - {ram_copies, [node()]}, - {record_name, trie_node}, - {attributes, record_info(fields, trie_node)}, + {record_name, ?TRIE}, + {attributes, record_info(fields, ?TRIE)}, + {type, ordered_set}, {storage_properties, StoreProps}]); - mnesia(copy) -> - %% Copy trie table - ok = ekka_mnesia:copy_table(?TRIE_TAB, ram_copies), - %% Copy trie_node table - ok = ekka_mnesia:copy_table(?TRIE_NODE_TAB, ram_copies). + %% Copy topics table + ok = ekka_mnesia:copy_table(?TRIE, ram_copies). %%-------------------------------------------------------------------- -%% Trie APIs +%% Topics APIs %%-------------------------------------------------------------------- %% @doc Insert a topic filter into the trie. -spec(insert(emqx_topic:topic()) -> ok). insert(Topic) when is_binary(Topic) -> - case mnesia:wread({?TRIE_NODE_TAB, Topic}) of - [#trie_node{topic = Topic}] -> - ok; - [TrieNode = #trie_node{topic = undefined}] -> - write_trie_node(TrieNode#trie_node{topic = Topic}); - [] -> - %% Add trie path - ok = lists:foreach(fun add_path/1, triples(Topic)), - %% Add last node - write_trie_node(#trie_node{node_id = Topic, topic = Topic}) + {TopicKey, PrefixKeys} = make_keys(Topic), + case mnesia:wread({?TRIE, TopicKey}) of + [_] -> ok; %% already inserted + [] -> lists:foreach(fun insert_key/1, [TopicKey | PrefixKeys]) end. -%% @doc Find trie nodes that match the topic name. --spec(match(emqx_topic:topic()) -> list(emqx_topic:topic())). -match(Topic) when is_binary(Topic) -> - TrieNodes = match_node(root, emqx_topic:words(Topic)), - [Name || #trie_node{topic = Name} <- TrieNodes, Name =/= undefined]. - -%% @doc Lookup a trie node. --spec(lookup(NodeId :: binary()) -> [trie_node()]). -lookup(NodeId) -> - mnesia:read(?TRIE_NODE_TAB, NodeId). - %% @doc Delete a topic filter from the trie. -spec(delete(emqx_topic:topic()) -> ok). delete(Topic) when is_binary(Topic) -> - case mnesia:wread({?TRIE_NODE_TAB, Topic}) of - [#trie_node{edge_count = 0}] -> - ok = mnesia:delete({?TRIE_NODE_TAB, Topic}), - delete_path(lists:reverse(triples(Topic))); - [TrieNode] -> - write_trie_node(TrieNode#trie_node{topic = undefined}); - [] -> ok + {TopicKey, PrefixKeys} = make_keys(Topic), + case [] =/= mnesia:wread({?TRIE, TopicKey}) of + true -> lists:foreach(fun delete_key/1, [TopicKey | PrefixKeys]); + false -> ok + end. + +%% @doc Find trie nodes that matchs the topic name. +-spec(match(emqx_topic:topic()) -> list(emqx_topic:topic())). +match(Topic) when is_binary(Topic) -> + Words = emqx_topic:words(Topic), + case emqx_topic:wildcard(Words) of + true -> + %% In MQTT spec, clients are not allowed to + %% publish messages to a wildcard topic. + %% Here we refuse to match wildcard topic. + %% + %% NOTE: this does not imply emqx allows clients + %% publishing to wildcard topics. + %% Such clients will get disconnected. + []; + false -> + do_match(Words) end. %% @doc Is the trie empty? -spec(empty() -> boolean()). -empty() -> - ets:info(?TRIE_TAB, size) == 0. +empty() -> ets:info(?TRIE, size) == 0. + +-spec lock_tables() -> ok. +lock_tables() -> + mnesia:write_lock_table(?TRIE). %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- -%% @doc Topic to triples. --spec(triples(emqx_topic:topic()) -> list(triple())). -triples(Topic) when is_binary(Topic) -> - triples(emqx_topic:words(Topic), root, []). +make_keys(Topic) -> + Words = emqx_topic:words(Topic), + {?TOPIC(Topic), [?PREFIX(Prefix) || Prefix <- make_prefixes(Words)]}. -triples([], _Parent, Acc) -> - lists:reverse(Acc); -triples([W|Words], Parent, Acc) -> - Node = join(Parent, W), - triples(Words, Node, [{Parent, W, Node}|Acc]). +compact(Words) -> + case is_compact() of + true -> do_compact(Words); + false -> Words + end. -join(root, W) -> - emqx_topic:join([W]); -join(Parent, W) -> - emqx_topic:join([Parent, W]). +%% join split words into compacted segments +%% each segment ends with one wildcard word +%% e.g. +%% a/b/c/+/d/# => [a/b/c/+, d/#] +%% a/+/+/b => [a/+, +, b] +%% a/+/+/+/+/b => [a/+, +, +, +, b] +do_compact(Words) -> + do_compact(Words, empty, []). -%% @private -%% @doc Add a path to the trie. -add_path({Node, Word, Child}) -> - Edge = #trie_edge{node_id = Node, word = Word}, - case mnesia:wread({?TRIE_NODE_TAB, Node}) of - [TrieNode = #trie_node{edge_count = Count}] -> - case mnesia:wread({?TRIE_TAB, Edge}) of - [] -> - ok = write_trie_node(TrieNode#trie_node{edge_count = Count + 1}), - write_trie(#trie{edge = Edge, node_id = Child}); - [_] -> ok - end; +do_compact([], empty, Acc) -> lists:reverse(Acc); +do_compact([], Seg, Acc) -> lists:reverse([Seg | Acc]); +do_compact([Word | Words], Seg, Acc) when Word =:= '+' orelse Word =:= '#' -> + do_compact(Words, empty, [join(Seg, Word) | Acc]); +do_compact([Word | Words], Seg, Acc) -> + do_compact(Words, join(Seg, Word), Acc). + +join(empty, '+') -> <<"+">>; +join(empty, '#') -> <<"#">>; +join(empty, '') -> <<>>; +join(empty, Word) -> Word; +join(Prefix, Word) -> emqx_topic:join([Prefix, Word]). + +make_prefixes(Words) -> + lists:map(fun emqx_topic:join/1, + make_prefixes(compact(Words), [], [])). + +make_prefixes([_LastWord], _Prefix, Acc) -> + lists:map(fun lists:reverse/1, Acc); +make_prefixes([H | T], Prefix0, Acc0) -> + Prefix = [H | Prefix0], + Acc = [Prefix | Acc0], + make_prefixes(T, Prefix, Acc). + +insert_key(Key) -> + T = case mnesia:wread({?TRIE, Key}) of + [#?TRIE{count = C} = T1] -> + T1#?TRIE{count = C + 1}; + [] -> + #?TRIE{key = Key, count = 1} + end, + ok = mnesia:write(T). + +delete_key(Key) -> + case mnesia:wread({?TRIE, Key}) of + [#?TRIE{count = C} = T] when C > 1 -> + ok = mnesia:write(T#?TRIE{count = C - 1}); + [_] -> + ok = mnesia:delete(?TRIE, Key, write); [] -> - ok = write_trie_node(#trie_node{node_id = Node, edge_count = 1}), - write_trie(#trie{edge = Edge, node_id = Child}) + ok end. -%% @private -%% @doc Match node with word or '+'. -match_node(root, [NodeId = <<$$, _/binary>>|Words]) -> - match_node(NodeId, Words, []); - -match_node(NodeId, Words) -> - match_node(NodeId, Words, []). - -match_node(NodeId, [], ResAcc) -> - mnesia:read(?TRIE_NODE_TAB, NodeId) ++ 'match_#'(NodeId, ResAcc); - -match_node(NodeId, [W|Words], ResAcc) -> - lists:foldl(fun(WArg, Acc) -> - case mnesia:read(?TRIE_TAB, #trie_edge{node_id = NodeId, word = WArg}) of - [#trie{node_id = ChildId}] -> match_node(ChildId, Words, Acc); - [] -> Acc - end - end, 'match_#'(NodeId, ResAcc), [W, '+']). - -%% @private -%% @doc Match node with '#'. -'match_#'(NodeId, ResAcc) -> - case mnesia:read(?TRIE_TAB, #trie_edge{node_id = NodeId, word = '#'}) of - [#trie{node_id = ChildId}] -> - mnesia:read(?TRIE_NODE_TAB, ChildId) ++ ResAcc; - [] -> ResAcc +lookup_topic(Topic) when is_binary(Topic) -> + case ets:lookup(?TRIE, ?TOPIC(Topic)) of + [#?TRIE{count = C}] -> [Topic || C > 0]; + [] -> [] end. -%% @private -%% @doc Delete paths from the trie. -delete_path([]) -> - ok; -delete_path([{NodeId, Word, _} | RestPath]) -> - ok = mnesia:delete({?TRIE_TAB, #trie_edge{node_id = NodeId, word = Word}}), - case mnesia:wread({?TRIE_NODE_TAB, NodeId}) of - [#trie_node{edge_count = 1, topic = undefined}] -> - ok = mnesia:delete({?TRIE_NODE_TAB, NodeId}), - delete_path(RestPath); - [TrieNode = #trie_node{edge_count = 1, topic = _}] -> - write_trie_node(TrieNode#trie_node{edge_count = 0}); - [TrieNode = #trie_node{edge_count = C}] -> - write_trie_node(TrieNode#trie_node{edge_count = C-1}); - [] -> - mnesia:abort({node_not_found, NodeId}) +has_prefix(empty) -> true; %% this is the virtual tree root +has_prefix(Prefix) -> + case ets:lookup(?TRIE, ?PREFIX(Prefix)) of + [#?TRIE{count = C}] -> C > 0; + [] -> false end. -%% @private -write_trie(Trie) -> - mnesia:write(?TRIE_TAB, Trie, write). +do_match([<<"$", _/binary>> = Prefix | Words]) -> + %% For topics having dollar sign prefix, + %% we do not match root level + or #, + %% fast forward to the next level. + case Words =:= [] of + true -> lookup_topic(Prefix); + false -> [] + end ++ do_match(Words, Prefix); +do_match(Words) -> + do_match(Words, empty). -%% @private -write_trie_node(TrieNode) -> - mnesia:write(?TRIE_NODE_TAB, TrieNode, write). +do_match(Words, Prefix) -> + match(is_compact(), Words, Prefix, []). +match(_IsCompact, [], Topic, Acc) -> + 'match_#'(Topic) ++ %% try match foo/bar/# + lookup_topic(Topic) ++ %% try match foo/bar + Acc; +match(IsCompact, [Word | Words], Prefix, Acc0) -> + case {has_prefix(Prefix), IsCompact} of + {false, false} -> + %% non-compact paths in database + %% if there is no prefix matches the current topic prefix + %% we can simpliy return from here + %% e.g. a/b/c/+ results in + %% - a + %% - a/b + %% - a/b/c + %% - a/b/c/+ + %% if the input topic is to match 'a/x/y', + %% then at the second level, we lookup prefix a/x, + %% no such prefix to be found, meaning there is no point + %% searching for 'a/x/y', 'a/x/+' or 'a/x/#' + Acc0; + _ -> + %% compact paths in database + %% we have to enumerate all possible prefixes + %% e.g. a/+/b/# results with below entries in database + %% - a/+ + %% - a/+/b/# + %% when matching a/x/y, we need to enumerate + %% - a + %% - a/x + %% - a/x/y + %% *with '+', '#' replaced at each level + Acc1 = 'match_#'(Prefix) ++ Acc0, + Acc = match(IsCompact, Words, join(Prefix, '+'), Acc1), + match(IsCompact, Words, join(Prefix, Word), Acc) + end. + +'match_#'(Prefix) -> + MlTopic = join(Prefix, '#'), + lookup_topic(MlTopic). + +is_compact() -> + case persistent_term:get({?MODULE, compaction}, undefined) of + undefined -> + Default = ?IS_COMPACT, + FromEnv = emqx:get_env(trie_compaction, Default), + _ = put_compaction_flag(FromEnv), + true = is_boolean(FromEnv), + FromEnv; + Value when is_boolean(Value) -> + Value + end. + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +make_keys_test_() -> + [{"no compact", fun() -> with_compact_flag(false, fun make_keys_no_compact/0) end}, + {"compact", fun() -> with_compact_flag(true, fun make_keys_compact/0) end} + ]. + +make_keys_no_compact() -> + ?assertEqual({?TOPIC(<<"#">>), []}, make_keys(<<"#">>)), + ?assertEqual({?TOPIC(<<"a/+">>), + [?PREFIX(<<"a">>)]}, make_keys(<<"a/+">>)), + ?assertEqual({?TOPIC(<<"+">>), []}, make_keys(<<"+">>)). + +make_keys_compact() -> + ?assertEqual({?TOPIC(<<"#">>), []}, make_keys(<<"#">>)), + ?assertEqual({?TOPIC(<<"a/+">>), []}, make_keys(<<"a/+">>)), + ?assertEqual({?TOPIC(<<"+">>), []}, make_keys(<<"+">>)), + ?assertEqual({?TOPIC(<<"a/+/c">>), + [?PREFIX(<<"a/+">>)]}, make_keys(<<"a/+/c">>)). + +words(T) -> emqx_topic:words(T). + +make_prefixes_t(Topic) -> make_prefixes(words(Topic)). + +with_compact_flag(IsCmopact, F) -> + put_compaction_flag(IsCmopact), + try F() + after put_default_compaction_flag() + end. + +make_prefixes_test_() -> + [{"no compact", fun() -> with_compact_flag(false, fun make_prefixes_no_compact/0) end}, + {"compact", fun() -> with_compact_flag(true, fun make_prefixes_compact/0) end} + ]. + +make_prefixes_no_compact() -> + ?assertEqual([<<"a/b">>, <<"a">>], make_prefixes_t(<<"a/b/+">>)), + ?assertEqual([<<"a/b/+/c">>, <<"a/b/+">>, <<"a/b">>, <<"a">>], + make_prefixes_t(<<"a/b/+/c/#">>)). + +make_prefixes_compact() -> + ?assertEqual([], make_prefixes_t(<<"a/b/+">>)), + ?assertEqual([<<"a/b/+">>], make_prefixes_t(<<"a/b/+/c/#">>)). + +do_compact_test() -> + ?assertEqual([<<"/+">>], do_compact(words(<<"/+">>))), + ?assertEqual([<<"/#">>], do_compact(words(<<"/#">>))), + ?assertEqual([<<"a/b/+">>, <<"c">>], do_compact(words(<<"a/b/+/c">>))), + ?assertEqual([<<"a/+">>, <<"+">>, <<"b">>], do_compact(words(<<"a/+/+/b">>))), + ?assertEqual([<<"a/+">>, <<"+">>, <<"+">>, <<"+">>, <<"b">>], + do_compact(words(<<"a/+/+/+/+/b">>))), + ok. + +clear_tables() -> mnesia:clear_table(?TRIE). + +-endif. % TEST diff --git a/src/emqx_types.erl b/src/emqx_types.erl index b92d457b8..fbe62e4b2 100644 --- a/src/emqx_types.erl +++ b/src/emqx_types.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_vm.erl b/src/emqx_vm.erl index 5094eea03..537330b01 100644 --- a/src/emqx_vm.erl +++ b/src/emqx_vm.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -40,6 +40,7 @@ -export([ get_ets_list/0 , get_ets_info/0 , get_ets_info/1 + , get_otp_version/0 ]). -export([cpu_util/0]). @@ -367,3 +368,30 @@ compat_windows(Fun) -> end end. +%% @doc Return on which Eralng/OTP the current vm is running. +%% NOTE: This API reads a file, do not use it in critical code paths. +get_otp_version() -> + parse_built_on(read_otp_version()). + +read_otp_version() -> + ReleasesDir = filename:join([code:root_dir(), "releases"]), + Filename = filename:join([ReleasesDir, emqx_app:get_release(), "BUILT_ON"]), + case file:read_file(Filename) of + {ok, BuiltOn} -> + %% running on EQM X release + BuiltOn; + {error, enoent} -> + %% running tests etc. + OtpMajor = erlang:system_info(otp_release), + OtpVsnFile = filename:join([ReleasesDir, OtpMajor, "OTP_VERSION"]), + {ok, Vsn} = file:read_file(OtpVsnFile), + Vsn + end. + +parse_built_on(BuiltOn) -> + case binary:split(BuiltOn, <<"-">>, [global]) of + [Vsn, <<"emqx">>, N | _] -> + binary_to_list(Vsn) ++ "-emqx-" ++ binary_to_list(N); + [Vsn | _] -> + string:trim(binary_to_list(Vsn)) + end. diff --git a/src/emqx_vm_mon.erl b/src/emqx_vm_mon.erl index 791e700d1..ce34fff43 100644 --- a/src/emqx_vm_mon.erl +++ b/src/emqx_vm_mon.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/src/emqx_ws_connection.erl b/src/emqx_ws_connection.erl index 01f7b5e2b..389a81e7b 100644 --- a/src/emqx_ws_connection.erl +++ b/src/emqx_ws_connection.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -34,7 +34,9 @@ , stats/1 ]). --export([call/2]). +-export([ call/2 + , call/3 + ]). %% WebSocket callbacks -export([ init/2 @@ -151,7 +153,10 @@ stats(#state{channel = Channel}) -> %% kick|discard|takeover -spec(call(pid(), Req :: term()) -> Reply :: term()). -call(WsPid, Req) when is_pid(WsPid) -> +call(WsPid, Req) -> + call(WsPid, Req, 5000). + +call(WsPid, Req, Timeout) when is_pid(WsPid) -> Mref = erlang:monitor(process, WsPid), WsPid ! {call, {self(), Mref}, Req}, receive @@ -160,7 +165,7 @@ call(WsPid, Req) when is_pid(WsPid) -> Reply; {'DOWN', Mref, _, _, Reason} -> exit(Reason) - after 5000 -> + after Timeout -> erlang:demonitor(Mref, [flush]), exit(timeout) end. @@ -248,15 +253,21 @@ check_origin_header(Req, Opts) -> end. websocket_init([Req, Opts]) -> - Peername = case proplists:get_bool(proxy_protocol, Opts) - andalso maps:get(proxy_header, Req) of - #{src_address := SrcAddr, src_port := SrcPort} -> - {SrcAddr, SrcPort}; - _ -> - cowboy_req:peer(Req) - end, + {Peername, Peercert} = + case proplists:get_bool(proxy_protocol, Opts) + andalso maps:get(proxy_header, Req) of + #{src_address := SrcAddr, src_port := SrcPort, ssl := SSL} -> + ProxyName = {SrcAddr, SrcPort}, + %% Notice: Only CN is available in Proxy Protocol V2 additional info + ProxySSL = case maps:get(cn, SSL, undefined) of + undeined -> nossl; + CN -> [{pp2_ssl_cn, CN}] + end, + {ProxyName, ProxySSL}; + _ -> + {get_peer(Req, Opts), cowboy_req:cert(Req)} + end, Sockname = cowboy_req:sock(Req), - Peercert = cowboy_req:cert(Req), WsCookie = try cowboy_req:parse_cookies(Req) catch error:badarg -> @@ -725,6 +736,34 @@ classify([Event|More], Packets, Cmds, Events) -> trigger(Event) -> erlang:send(self(), Event). +get_peer(Req, Opts) -> + {PeerAddr, PeerPort} = cowboy_req:peer(Req), + AddrHeader = cowboy_req:header(proplists:get_value(proxy_address_header, Opts), Req, <<>>), + ClientAddr = case string:tokens(binary_to_list(AddrHeader), ", ") of + [] -> + undefined; + AddrList -> + hd(AddrList) + end, + Addr = case inet:parse_address(ClientAddr) of + {ok, A} -> + A; + _ -> + PeerAddr + end, + PortHeader = cowboy_req:header(proplists:get_value(proxy_port_header, Opts), Req, <<>>), + ClientPort = case string:tokens(binary_to_list(PortHeader), ", ") of + [] -> + undefined; + PortList -> + hd(PortList) + end, + try + {Addr, list_to_integer(ClientPort)} + catch + _:_ -> {Addr, PeerPort} + end. + %%-------------------------------------------------------------------- %% For CT tests %%-------------------------------------------------------------------- diff --git a/src/emqx_zone.erl b/src/emqx_zone.erl index 6cec6c1ee..459c36764 100644 --- a/src/emqx_zone.erl +++ b/src/emqx_zone.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_SUITE.erl b/test/emqx_SUITE.erl index 56a78e00e..714f4e0a5 100644 --- a/test/emqx_SUITE.erl +++ b/test/emqx_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_access_control_SUITE.erl b/test/emqx_access_control_SUITE.erl index 2681f31ad..b9ae2c2cb 100644 --- a/test/emqx_access_control_SUITE.erl +++ b/test/emqx_access_control_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_access_rule_SUITE.erl b/test/emqx_access_rule_SUITE.erl index d99a1fd78..93c84a958 100644 --- a/test/emqx_access_rule_SUITE.erl +++ b/test/emqx_access_rule_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_acl_cache_SUITE.erl b/test/emqx_acl_cache_SUITE.erl index 8d760c284..be7c29055 100644 --- a/test/emqx_acl_cache_SUITE.erl +++ b/test/emqx_acl_cache_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_acl_test_mod.erl b/test/emqx_acl_test_mod.erl index 5d36cce78..da400f076 100644 --- a/test/emqx_acl_test_mod.erl +++ b/test/emqx_acl_test_mod.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_alarm_SUITE.erl b/test/emqx_alarm_SUITE.erl index 2f3b42fd2..db6cdfe7f 100644 --- a/test/emqx_alarm_SUITE.erl +++ b/test/emqx_alarm_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_banned_SUITE.erl b/test/emqx_banned_SUITE.erl index a4030fc57..6e69a7371 100644 --- a/test/emqx_banned_SUITE.erl +++ b/test/emqx_banned_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_batch_SUITE.erl b/test/emqx_batch_SUITE.erl index cdf893942..856a9b1ad 100644 --- a/test/emqx_batch_SUITE.erl +++ b/test/emqx_batch_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_boot_SUITE.erl b/test/emqx_boot_SUITE.erl index 118aace92..f46647ad5 100644 --- a/test/emqx_boot_SUITE.erl +++ b/test/emqx_boot_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_broker_SUITE.erl b/test/emqx_broker_SUITE.erl index ca87648dd..3313d79a5 100644 --- a/test/emqx_broker_SUITE.erl +++ b/test/emqx_broker_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_broker_helper_SUITE.erl b/test/emqx_broker_helper_SUITE.erl index 0084a4012..31b36ecdc 100644 --- a/test/emqx_broker_helper_SUITE.erl +++ b/test/emqx_broker_helper_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_channel_SUITE.erl b/test/emqx_channel_SUITE.erl index b5f83a0be..9558dfd28 100644 --- a/test/emqx_channel_SUITE.erl +++ b/test/emqx_channel_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_client_SUITE.erl b/test/emqx_client_SUITE.erl index 6476bd569..73a92024b 100644 --- a/test/emqx_client_SUITE.erl +++ b/test/emqx_client_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_cm_SUITE.erl b/test/emqx_cm_SUITE.erl index 4a009d9c6..b8e0f9066 100644 --- a/test/emqx_cm_SUITE.erl +++ b/test/emqx_cm_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -34,6 +34,7 @@ %%-------------------------------------------------------------------- %% CT callbacks %%-------------------------------------------------------------------- +suite() -> [{timetrap, {minutes, 2}}]. all() -> emqx_ct:all(?MODULE). @@ -85,6 +86,7 @@ t_get_set_chan_stats(_) -> t_open_session(_) -> ok = meck:new(emqx_connection, [passthrough, no_history]), ok = meck:expect(emqx_connection, call, fun(_, _) -> ok end), + ok = meck:expect(emqx_connection, call, fun(_, _, _) -> ok end), ClientInfo = #{zone => external, clientid => <<"clientid">>, @@ -164,6 +166,7 @@ t_discard_session(_) -> ok = meck:new(emqx_connection, [passthrough, no_history]), ok = meck:expect(emqx_connection, call, fun(_, _) -> ok end), + ok = meck:expect(emqx_connection, call, fun(_, _, _) -> ok end), ok = emqx_cm:discard_session(<<"clientid">>), ok = emqx_cm:register_channel(<<"clientid">>, self(), ConnInfo), ok = emqx_cm:discard_session(<<"clientid">>), @@ -171,6 +174,7 @@ t_discard_session(_) -> ok = emqx_cm:register_channel(<<"clientid">>, self(), ConnInfo), ok = emqx_cm:discard_session(<<"clientid">>), ok = meck:expect(emqx_connection, call, fun(_, _) -> error(testing) end), + ok = meck:expect(emqx_connection, call, fun(_, _, _) -> error(testing) end), ok = emqx_cm:discard_session(<<"clientid">>), ok = emqx_cm:unregister_channel(<<"clientid">>), ok = meck:unload(emqx_connection). @@ -178,13 +182,13 @@ t_discard_session(_) -> t_takeover_session(_) -> #{conninfo := ConnInfo} = ?ChanInfo, {error, not_found} = emqx_cm:takeover_session(<<"clientid">>), - erlang:spawn(fun() -> - ok = emqx_cm:register_channel(<<"clientid">>, self(), ConnInfo), - receive - {'$gen_call', From, {takeover, 'begin'}} -> - gen_server:reply(From, test), ok - end - end), + erlang:spawn_link(fun() -> + ok = emqx_cm:register_channel(<<"clientid">>, self(), ConnInfo), + receive + {'$gen_call', From, {takeover, 'begin'}} -> + gen_server:reply(From, test), ok + end + end), timer:sleep(100), {ok, emqx_connection, _, test} = emqx_cm:takeover_session(<<"clientid">>), emqx_cm:unregister_channel(<<"clientid">>). @@ -193,16 +197,18 @@ t_kick_session(_) -> Info = #{conninfo := ConnInfo} = ?ChanInfo, ok = meck:new(emqx_connection, [passthrough, no_history]), ok = meck:expect(emqx_connection, call, fun(_, _) -> test end), + ok = meck:expect(emqx_connection, call, fun(_, _, _) -> test end), {error, not_found} = emqx_cm:kick_session(<<"clientid">>), ok = emqx_cm:register_channel(<<"clientid">>, self(), ConnInfo), ok = emqx_cm:insert_channel_info(<<"clientid">>, Info, []), test = emqx_cm:kick_session(<<"clientid">>), - erlang:spawn(fun() -> - ok = emqx_cm:register_channel(<<"clientid">>, self(), ConnInfo), - ok = emqx_cm:insert_channel_info(<<"clientid">>, Info, []), + erlang:spawn_link( + fun() -> + ok = emqx_cm:register_channel(<<"clientid">>, self(), ConnInfo), + ok = emqx_cm:insert_channel_info(<<"clientid">>, Info, []), - timer:sleep(1000) - end), + timer:sleep(1000) + end), ct:sleep(100), test = emqx_cm:kick_session(<<"clientid">>), ok = emqx_cm:unregister_channel(<<"clientid">>), diff --git a/test/emqx_cm_locker_SUITE.erl b/test/emqx_cm_locker_SUITE.erl index 8a745cbf5..ec40a8985 100644 --- a/test/emqx_cm_locker_SUITE.erl +++ b/test/emqx_cm_locker_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_cm_registry_SUITE.erl b/test/emqx_cm_registry_SUITE.erl index c1e78bf3a..097bfc7b4 100644 --- a/test/emqx_cm_registry_SUITE.erl +++ b/test/emqx_cm_registry_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_connection_SUITE.erl b/test/emqx_connection_SUITE.erl index d5a082f7e..a6b2b614a 100644 --- a/test/emqx_connection_SUITE.erl +++ b/test/emqx_connection_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_ctl_SUITE.erl b/test/emqx_ctl_SUITE.erl index ff728bc97..4f8224871 100644 --- a/test/emqx_ctl_SUITE.erl +++ b/test/emqx_ctl_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_flapping_SUITE.erl b/test/emqx_flapping_SUITE.erl index dcd552385..8f069b747 100644 --- a/test/emqx_flapping_SUITE.erl +++ b/test/emqx_flapping_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_frame_SUITE.erl b/test/emqx_frame_SUITE.erl index c1fb1b580..09aa97c3e 100644 --- a/test/emqx_frame_SUITE.erl +++ b/test/emqx_frame_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_gc_SUITE.erl b/test/emqx_gc_SUITE.erl index 4a8fcc0dc..6504234f9 100644 --- a/test/emqx_gc_SUITE.erl +++ b/test/emqx_gc_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_global_gc_SUITE.erl b/test/emqx_global_gc_SUITE.erl index 528189300..92d6a5251 100644 --- a/test/emqx_global_gc_SUITE.erl +++ b/test/emqx_global_gc_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_guid_SUITE.erl b/test/emqx_guid_SUITE.erl index a5da59e9e..b72062067 100644 --- a/test/emqx_guid_SUITE.erl +++ b/test/emqx_guid_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_hooks_SUITE.erl b/test/emqx_hooks_SUITE.erl index f687aa644..d40030122 100644 --- a/test/emqx_hooks_SUITE.erl +++ b/test/emqx_hooks_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_http_lib_tests.erl b/test/emqx_http_lib_tests.erl index b5c50e7c9..393d1ff86 100644 --- a/test/emqx_http_lib_tests.erl +++ b/test/emqx_http_lib_tests.erl @@ -62,7 +62,7 @@ uri_parse_test_() -> end } , {"normalise", - fun() -> ?assertMatch({ok, #{scheme := https}}, + fun() -> ?assertMatch({ok, #{scheme := https, host := {127, 0, 0, 1}}}, emqx_http_lib:uri_parse("HTTPS://127.0.0.1")) end } @@ -71,4 +71,9 @@ uri_parse_test_() -> emqx_http_lib:uri_parse("wss://127.0.0.1")) end } + , {"ipv6 host", + fun() -> ?assertMatch({ok, #{scheme := http, host := T}} when size(T) =:= 8, + emqx_http_lib:uri_parse("http://[::1]:80")) + end + } ]. diff --git a/test/emqx_inflight_SUITE.erl b/test/emqx_inflight_SUITE.erl index 4668a13ce..a819e788b 100644 --- a/test/emqx_inflight_SUITE.erl +++ b/test/emqx_inflight_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_json_SUITE.erl b/test/emqx_json_SUITE.erl index fa544ffbb..8f579f7be 100644 --- a/test/emqx_json_SUITE.erl +++ b/test/emqx_json_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_keepalive_SUITE.erl b/test/emqx_keepalive_SUITE.erl index 5e87f58f2..d8a0c1316 100644 --- a/test/emqx_keepalive_SUITE.erl +++ b/test/emqx_keepalive_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_limiter_SUITE.erl b/test/emqx_limiter_SUITE.erl index a444b3f43..b22840edc 100644 --- a/test/emqx_limiter_SUITE.erl +++ b/test/emqx_limiter_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_listeners_SUITE.erl b/test/emqx_listeners_SUITE.erl index 969ff6af5..673cc80e9 100644 --- a/test/emqx_listeners_SUITE.erl +++ b/test/emqx_listeners_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_logger_SUITE.erl b/test/emqx_logger_SUITE.erl index 948bb813e..a087761b8 100644 --- a/test/emqx_logger_SUITE.erl +++ b/test/emqx_logger_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_logger_formatter_SUITE.erl b/test/emqx_logger_formatter_SUITE.erl deleted file mode 100644 index a495809b5..000000000 --- a/test/emqx_logger_formatter_SUITE.erl +++ /dev/null @@ -1,652 +0,0 @@ -%% -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2018. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%% -%% %CopyrightEnd% -%% --module(emqx_logger_formatter_SUITE). - --compile(export_all). --compile(nowarn_export_all). - --include_lib("eunit/include/eunit.hrl"). --include_lib("common_test/include/ct.hrl"). --include_lib("kernel/include/logger.hrl"). - --define(TRY(X), my_try(fun() -> X end)). - -suite() -> - [{timetrap,{seconds,30}}]. - -init_per_suite(Config) -> - Config. - -end_per_suite(_Config) -> - ok. - -init_per_group(_Group, Config) -> - Config. - -end_per_group(_Group, _Config) -> - ok. - -init_per_testcase(_TestCase, Config) -> - Config. - -end_per_testcase(Case, Config) -> - try apply(?MODULE,Case,[cleanup,Config]) - catch error:undef -> ok - end, - ok. - -groups() -> - []. - -all() -> - [default, - single_line, - template, - format_msg, - report_cb, - max_size, - depth, - chars_limit, - format_mfa, - level_or_msg_in_meta, - faulty_log, - faulty_config, - faulty_msg, - check_config, - update_config]. - -default(_Config) -> - String1 = format(info,{"~p",[term]},#{},#{}), - ct:log(String1), - ?assertMatch([_DateTime,"info:","term\n"], string:lexemes(String1, " ")), - - Time = timestamp(), - ExpectedTimestamp = default_time_format(Time), - String2 = format(info,{"~p",[term]},#{time=>Time},#{}), - ct:log("ExpectedTimestamp: ~p, got: ~p", [ExpectedTimestamp, String2]), - " info: term\n" = string:prefix(String2,ExpectedTimestamp), - ok. - -single_line(_Config) -> - Time = timestamp(), - ExpectedTimestamp = default_time_format(Time), - String1 = format(info,{"~p",[term]},#{time=>Time},#{}), - ct:log(String1), - ?assertMatch(" info: term\n", string:prefix(String1,ExpectedTimestamp)), - - String2 = format(info,{"a:~n~p",[term]},#{time=>Time},#{}), - ct:log(String2), - ?assertMatch(" info: a:\nterm\n", string:prefix(String2,ExpectedTimestamp)), - - - Prefix = - "Some characters to fill the line ------------------------------------- ", - %% There would actually be newlines inside the - %% list and map. - String4 = format(info,{"~s~p~n~s~p~n",[Prefix, - lists:seq(1,10), - Prefix, - #{a=>map,with=>a,few=>accociations}]}, - #{time=>Time}, - #{}), - ct:log(String4), - match = re:run(String4,"\\[1,2,3,\n",[global,{capture,none}]), - {match,Match4} = re:run(String4,"=>\n",[global,{capture,all}]), - 3 = length(Match4), - - %% Test that big metadata fields do not get line breaks - String5 = format(info,"", - #{mymeta=>lists:seq(1,100)}, - #{template=>[mymeta,"\n"]}), - ct:log(String5), - [_] = string:lexemes(String5,"\n"), - ok. - -template(_Config) -> - Time = timestamp(), - - Template1 = [msg], - String1 = format(info,{"~p",[term]},#{time=>Time},#{template=>Template1}), - ct:log(String1), - "term" = String1, - - Template2 = [msg,unknown], - String2 = format(info,{"~p",[term]},#{time=>Time},#{template=>Template2}), - ct:log(String2), - "term" = String2, - - Template3 = ["string"], - String3 = format(info,{"~p",[term]},#{time=>Time},#{template=>Template3}), - ct:log(String3), - "string" = String3, - - Template4 = ["string\nnewline"], - String4 = format(info,{"~p",[term]},#{time=>Time},#{template=>Template4}), - ct:log(String4), - "string\nnewline" = String4, - - Template5 = [], - String5 = format(info,{"~p",[term]},#{time=>Time},#{template=>Template5}), - ct:log(String5), - "" = String5, - - Ref6 = erlang:make_ref(), - Meta6 = #{atom=>some_atom, - integer=>632, - list=>[list,"string",4321,#{},{tuple}], - mfa=>{mod,func,0}, - pid=>self(), - ref=>Ref6, - string=>"some string", - time=>Time, - tuple=>{1,atom,"list"}, - nested=>#{subkey=>subvalue}}, - Template6 = lists:join(";",lists:sort(maps:keys(maps:remove(nested,Meta6))) ++ - [[nested,subkey]]), - String6 = format(info,{"~p",[term]},Meta6,#{template=>Template6}), - ct:log(String6), - SelfStr = pid_to_list(self()), - RefStr6 = ref_to_list(Ref6), - ListStr = "[list,\"string\",4321,#{},{tuple}]", - ExpectedTime6 = default_time_format(Time), - ["some_atom", - "632", - ListStr, - "mod:func/0", - SelfStr, - RefStr6, - "some string", - ExpectedTime6, - "{1,atom,\"list\"}", - "subvalue"] = string:lexemes(String6,";"), - - Meta7 = #{time=>Time, - nested=>#{key1=>#{subkey1=>value1}, - key2=>value2}}, - Template7 = lists:join(";",[nested, - [nested,key1], - [nested,key1,subkey1], - [nested,key2], - [nested,key2,subkey2], - [nested,key3], - [nested,key3,subkey3]]), - String7 = format(info,{"~p",[term]},Meta7,#{template=>Template7}), - ct:log(String7), - [MultipleKeysStr7, - "#{subkey1 => value1}", - "value1", - "value2", - "", - "", - ""] = string:split(String7,";",all), - %% Order of keys is not fixed - case MultipleKeysStr7 of - "#{key2 => value2,key1 => #{subkey1 => value1}}" -> ok; - "#{key1 => #{subkey1 => value1},key2 => value2}" -> ok; - _ -> ct:fail({full_nested_map_unexpected,MultipleKeysStr7}) - end, - - Meta8 = #{time=>Time, - nested=>#{key1=>#{subkey1=>value1}, - key2=>value2}}, - Template8 = - lists:join( - ";", - [{nested,["exist:",nested],["noexist"]}, - {[nested,key1],["exist:",[nested,key1]],["noexist"]}, - {[nested,key1,subkey1],["exist:",[nested,key1,subkey1]],["noexist"]}, - {[nested,key2],["exist:",[nested,key2]],["noexist"]}, - {[nested,key2,subkey2],["exist:",[nested,key2,subkey2]],["noexist"]}, - {[nested,key3],["exist:",[nested,key3]],["noexist"]}, - {[nested,key3,subkey3],["exist:",[nested,key3,subkey3]],["noexist"]}]), - String8 = format(info,{"~p",[term]},Meta8,#{template=>Template8}), - ct:log(String8), - [MultipleKeysStr8, - "exist:#{subkey1 => value1}", - "exist:value1", - "exist:value2", - "noexist", - "noexist", - "noexist"] = string:split(String8,";",all), - %% Order of keys is not fixed - case MultipleKeysStr8 of - "exist:#{key2 => value2,key1 => #{subkey1 => value1}}" -> ok; - "exist:#{key1 => #{subkey1 => value1},key2 => value2}" -> ok; - _ -> ct:fail({full_nested_map_unexpected,MultipleKeysStr8}) - end, - - ok. - -format_msg(_Config) -> - Template = [msg], - - String1 = format(info,{"~p",[term]},#{},#{template=>Template}), - ct:log(String1), - "term" = String1, - - String2 = format(info,{"list",[term]},#{},#{template=>Template}), - ct:log(String2), - "FORMAT ERROR: \"list\" - [term]" = String2, - - String3 = format(info,{report,term},#{},#{template=>Template}), - ct:log(String3), - "term" = String3, - - String4 = format(info,{report,term}, - #{report_cb=>fun(_)-> {"formatted",[]} end}, - #{template=>Template}), - ct:log(String4), - "formatted" = String4, - - String5 = format(info,{report,term}, - #{report_cb=>fun(_)-> faulty_return end}, - #{template=>Template}), - ct:log(String5), - "REPORT_CB/1 ERROR: term; Returned: faulty_return" = String5, - - String6 = format(info,{report,term}, - #{report_cb=>fun(_)-> erlang:error(fun_crashed) end}, - #{template=>Template}), - ct:log(String6), - "REPORT_CB/1 CRASH: term; Reason: {error,fun_crashed,"++_ = String6, - - String7 = format(info,{report,term}, - #{report_cb=>fun(_,_)-> ['not',a,string] end}, - #{template=>Template}), - ct:log(String7), - "REPORT_CB/2 ERROR: term; Returned: ['not',a,string]" = String7, - - String8 = format(info,{report,term}, - #{report_cb=>fun(_,_)-> faulty_return end}, - #{template=>Template}), - ct:log(String8), - "REPORT_CB/2 ERROR: term; Returned: faulty_return" = String8, - - String9 = format(info,{report,term}, - #{report_cb=>fun(_,_)-> erlang:error(fun_crashed) end}, - #{template=>Template}), - ct:log(String9), - "REPORT_CB/2 CRASH: term; Reason: {error,fun_crashed,"++_ = String9, - - %% strings are not formatted - String10 = format(info,{string,"string"}, - #{report_cb=>fun(_)-> {"formatted",[]} end}, - #{template=>Template}), - ct:log(String10), - "string" = String10, - - String11 = format(info,{string,['not',printable,list]}, - #{report_cb=>fun(_)-> {"formatted",[]} end}, - #{template=>Template}), - ct:log("~ts",[String11]), % avoiding ct_log crash - "FORMAT ERROR: \"~ts\" - [['not',printable,list]]" = String11, - - String12 = format(info,{string,"string"},#{},#{template=>Template}), - ct:log(String12), - "string" = String12, - - ok. - -report_cb(_Config) -> - Template = [msg], - MetaFun = fun(_) -> {"meta_rcb",[]} end, - ConfigFun = fun(_) -> {"config_rcb",[]} end, - "term" = format(info,{report,term},#{},#{template=>Template}), - "meta_rcb" = - format(info,{report,term},#{report_cb=>MetaFun},#{template=>Template}), - "config_rcb" = - format(info,{report,term},#{},#{template=>Template, - report_cb=>ConfigFun}), - "config_rcb" = - format(info,{report,term},#{report_cb=>MetaFun},#{template=>Template, - report_cb=>ConfigFun}), - ok. - -max_size(_Config) -> - Cfg = #{template=>[msg]}, - "12345678901234567890" = format(info,{"12345678901234567890",[]},#{},Cfg), - %% application:set_env(kernel,logger_max_size,11), - %% "12345678901234567890" = % min value is 50, so this is not limited - %% format(info,{"12345678901234567890",[]},#{},Cfg), - %% "12345678901234567890123456789012345678901234567..." = % 50 - %% format(info, - %% {"123456789012345678901234567890123456789012345678901234567890", - %% []}, - %% #{}, - %% Cfg), - %% application:set_env(kernel,logger_max_size,53), - %% "12345678901234567890123456789012345678901234567890..." = %53 - %% format(info, - %% {"123456789012345678901234567890123456789012345678901234567890", - %% []}, - %% #{}, - %% Cfg), - "123456789012..." = - format(info,{"12345678901234567890",[]},#{},Cfg#{max_size=>15}), - "12345678901234567890" = - format(info,{"12345678901234567890",[]},#{},Cfg#{max_size=>unlimited}), - %% Check that one newline at the end of the line is kept (if it exists) - "12345678901...\n" = - format(info,{"12345678901234567890\n",[]},#{},Cfg#{max_size=>15}), - "12345678901...\n" = - format(info,{"12345678901234567890",[]},#{},Cfg#{template=>[msg,"\n"], - max_size=>15}), - ok. -max_size(cleanup,_Config) -> - application:unset_env(kernel,logger_max_size), - ok. - -depth(_Config) -> - Template = [msg], - "[1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0]" = - format(info, - {"~p",[[1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0]]}, - #{}, - #{template=>Template}), - application:set_env(kernel,error_logger_format_depth,11), - "[1,2,3,4,5,6,7,8,9,0|...]" = - format(info, - {"~p",[[1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0]]}, - #{}, - #{template=>Template}), - "[1,2,3,4,5,6,7,8,9,0,1,2|...]" = - format(info, - {"~p",[[1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0]]}, - #{}, - #{template=>Template, - depth=>13}), - "[1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0]" = - format(info, - {"~p",[[1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0]]}, - #{}, - #{template=>Template, - depth=>unlimited}), - ok. -depth(cleanup,_Config) -> - application:unset_env(kernel,error_logger_format_depth), - ok. - -chars_limit(_Config) -> - FA = {"LoL: ~p~nL: ~p~nMap: ~p~n", - [lists:duplicate(10,lists:seq(1,100)), - lists:seq(1,100), - maps:from_list(lists:zip(lists:seq(1,100), - lists:duplicate(100,value)))]}, - Meta = #{time=>timestamp()}, - Template = [time," - ", msg, "\n"], - FC = #{template=>Template, - depth=>unlimited, - max_size=>unlimited, - chars_limit=>unlimited}, - CL1 = 80, - String1 = format(info,FA,Meta,FC#{chars_limit=>CL1}), - L1 = string:length(String1), - ct:log("String1: ~p~nLength1: ~p~n",[lists:flatten(String1),L1]), - true = L1 > CL1, - true = L1 < CL1 + 15, - - String2 = format(info,FA,Meta,FC#{chars_limit=>CL1,depth=>10}), - L2 = string:length(String2), - ct:log("String2: ~p~nLength2: ~p~n",[lists:flatten(String2),L2]), - String2 = String1, - - CL3 = 200, - String3 = format(info,FA,Meta,FC#{chars_limit=>CL3}), - L3 = string:length(String3), - ct:log("String3: ~p~nLength3: ~p~n",[lists:flatten(String3),L3]), - true = L3 > CL3, - true = L3 < CL3 + 15, - - String4 = format(info,FA,Meta,FC#{chars_limit=>CL3,depth=>10}), - L4 = string:length(String4), - ct:log("String4: ~p~nLength4: ~p~n",[lists:flatten(String4),L4]), - true = L4 > CL3, - true = L4 < CL3 + 15, - - %% Test that max_size truncates the string which is limited by - %% depth and chars_limit - MS5 = 150, - String5 = format(info,FA,Meta,FC#{chars_limit=>CL3,depth=>10,max_size=>MS5}), - L5 = string:length(String5), - ct:log("String5: ~p~nLength5: ~p~n",[String5,L5]), - L5 = MS5, - true = lists:prefix(lists:sublist(String5,L5-4),String4), - - %% Test that chars_limit limits string also - Str = "123456789012345678901234567890123456789012345678901234567890123456789", - CL6 = 80, - String6 = format(info,{string,Str},Meta,FC#{chars_limit=>CL6}), - L6 = string:length(String6), - ct:log("String6: ~p~nLength6: ~p~n",[String6,L6]), - L6 = CL6, - - ok. - -format_mfa(_Config) -> - Template = [mfa], - - Meta1 = #{mfa=>{mod,func,0}}, - String1 = format(info,{"~p",[term]},Meta1,#{template=>Template}), - ct:log(String1), - "mod:func/0" = String1, - - Meta2 = #{mfa=>{mod,func,[]}}, - String2 = format(info,{"~p",[term]},Meta2,#{template=>Template}), - ct:log(String2), - "mod:func/0" = String2, - - Meta3 = #{mfa=>"mod:func/0"}, - String3 = format(info,{"~p",[term]},Meta3,#{template=>Template}), - ct:log(String3), - "mod:func/0" = String3, - - Meta4 = #{mfa=>othermfa}, - String4 = format(info,{"~p",[term]},Meta4,#{template=>Template}), - ct:log(String4), - "othermfa" = String4, - - ok. - -level_or_msg_in_meta(_Config) -> - %% The template contains atoms to pick out values from meta, - %% or level/msg to add these from the log event. What if you have - %% a key named 'level' or 'msg' in meta and want to display - %% its value? - %% For now we simply ignore Meta on this and display the - %% actual level and msg from the log event. - - Meta = #{level=>mylevel, - msg=>"metamsg"}, - Template = [level,";",msg], - String = format(info,{"~p",[term]},Meta,#{template=>Template}), - ct:log(String), - "info;term" = String, % so mylevel and "metamsg" are ignored - ok. - -faulty_log(_Config) -> - %% Unexpected log (should be type logger:log_event()) - print error - {error, - function_clause, - {emqx_logger_formatter,format,[_,_],_}} = - ?TRY(emqx_logger_formatter:format(unexp_log,#{})), - ok. - -faulty_config(_Config) -> - {error, - function_clause, - {emqx_logger_formatter,format,[_,_],_}} = - ?TRY(emqx_logger_formatter:format(#{level=>info, - msg=>{"~p",[term]}, - meta=>#{time=>timestamp()}}, - unexp_config)), - ok. - -faulty_msg(_Config) -> - {error, - function_clause, - {emqx_logger_formatter,_,_,_}} = - ?TRY(emqx_logger_formatter:format(#{level=>info, - msg=>term, - meta=>#{time=>timestamp()}}, - #{})), - ok. - --define(cfgerr(X), {error,{invalid_formatter_config,emqx_logger_formatter,X}}). -check_config(_Config) -> - ok = emqx_logger_formatter:check_config(#{}), - ?cfgerr(bad) = emqx_logger_formatter:check_config(bad), - - C1 = #{chars_limit => 1, - depth => 1, - max_size => 1, - report_cb => fun(R) -> {"~p",[R]} end, - template => []}, - ok = emqx_logger_formatter:check_config(C1), - - ok = emqx_logger_formatter:check_config(#{chars_limit => unlimited}), - ?cfgerr({chars_limit,bad}) = - emqx_logger_formatter:check_config(#{chars_limit => bad}), - - ok = emqx_logger_formatter:check_config(#{depth => unlimited}), - ?cfgerr({depth,bad}) = - emqx_logger_formatter:check_config(#{depth => bad}), - - ok = emqx_logger_formatter:check_config(#{max_size => unlimited}), - ?cfgerr({max_size,bad}) = - emqx_logger_formatter:check_config(#{max_size => bad}), - - ok = - emqx_logger_formatter:check_config(#{report_cb => fun(_,_) -> "" end}), - ?cfgerr({report_cb,F}) = - emqx_logger_formatter:check_config(#{report_cb => F=fun(_,_,_) -> {"",[]} end}), - ?cfgerr({report_cb,bad}) = - emqx_logger_formatter:check_config(#{report_cb => bad}), - - Ts = [[key], - [[key1,key2]], - [{key,[key],[]}], - [{[key1,key2],[[key1,key2]],["noexist"]}], - ["string"]], - [begin - ct:log("check template: ~p",[T]), - ok = emqx_logger_formatter:check_config(#{template => T}) - end - || T <- Ts], - - ETs = [bad, - [{key,bad}], - [{key,[key],bad}], - [{key,[key],"bad"}], - "bad", - [[key,$a,$b,$c]], - [[$a,$b,$c,key]]], - [begin - ct:log("check template: ~p",[T]), - {error,{invalid_formatter_template,emqx_logger_formatter,T}} = - emqx_logger_formatter:check_config(#{template => T}) - end - || T <- ETs], - ok. - -%% Test that formatter config can be changed, and that the default -%% template is updated accordingly -update_config(_Config) -> - #{level := OldLevel} = logger:get_primary_config(), - logger:set_primary_config(level, debug), - {error,{not_found,?MODULE}} = logger:update_formatter_config(?MODULE,#{}), - - logger:add_handler_filter(default,silence,{fun(_,_) -> stop end,ok}), - ok = logger:add_handler(?MODULE,?MODULE,#{formatter => {emqx_logger_formatter, #{chars_limit => unlimited}}, - config => #{type => standard_io}}), - D = lists:seq(1,1000), - logger:notice("~p~n",[D]), - {Lines1,C1} = check_log(), - ct:log("lines1: ~p", [Lines1]), - ct:log("c1: ~p",[C1]), - [Line1 | _] = Lines1, - [_DateTime1, WithOutDate1] = string:split(Line1, " "), - ["notice:", D1] = string:split(WithOutDate1, " "), - ?assert(length(D1)<1000), - ?assertMatch(#{chars_limit := unlimited}, C1), - - error_logger:error_msg("~p",[D]), - {Lines5,C5} = check_log(), - ct:log("Lines5: ~p", [Lines5]), - ct:log("c5: ~p",[C5]), - [Line5 | _] = Lines5, - [_DateTime2, WithOutDate5] = string:split(Line5, " "), - ["error:", _D5] = string:split(WithOutDate5, " "), - - ?assertMatch({error,{invalid_formatter_config,bad}}, - logger:update_formatter_config(?MODULE,bad)), - ?assertMatch({error,{invalid_formatter_config,emqx_logger_formatter,{depth,bad}}}, - logger:update_formatter_config(?MODULE,depth,bad)), - - logger:set_primary_config(level, OldLevel), - ok. - -update_config(cleanup,_Config) -> - _ = logger:remove_handler(?MODULE), - _ = logger:remove_handler_filter(default,silence), - ok. - -%%%----------------------------------------------------------------- -%%% Internal -format(Level,Msg,Meta,Config) -> - format(#{level=>Level,msg=>Msg,meta=>add_time(Meta)},Config). - -format(Log,Config) -> - lists:flatten(emqx_logger_formatter:format(Log,Config)). - -default_time_format(SysTime) when is_integer(SysTime) -> - calendar:system_time_to_rfc3339(SysTime, [{unit,microsecond}, - {offset,""}]). - -integer(Str) -> - is_integer(list_to_integer(Str)). -integer(Str,Max) -> - integer(Str,0,Max). -integer(Str,Min,Max) -> - Int = list_to_integer(Str), - Int >= Min andalso Int = - try Fun() catch C:R:S -> {C,R,hd(S)} end. - -timestamp() -> - erlang:system_time(microsecond). - -%% necessary? -add_time(#{time:=_}=Meta) -> - Meta; -add_time(Meta) -> - Meta#{time=>timestamp()}. - -%%%----------------------------------------------------------------- -%%% handler callback -log(Log,#{formatter:={M,C}}) -> - put(log,{M:format(Log,C),C}), - ok. - -check_log() -> - {S,C} = erase(log), - {string:lexemes(S,"\n"),C}. diff --git a/test/emqx_message_SUITE.erl b/test/emqx_message_SUITE.erl index bbead8f7e..63db85664 100644 --- a/test/emqx_message_SUITE.erl +++ b/test/emqx_message_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_metrics_SUITE.erl b/test/emqx_metrics_SUITE.erl index e55ce505b..751a26780 100644 --- a/test/emqx_metrics_SUITE.erl +++ b/test/emqx_metrics_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_misc_SUITE.erl b/test/emqx_misc_SUITE.erl index d3b77c384..f933fb498 100644 --- a/test/emqx_misc_SUITE.erl +++ b/test/emqx_misc_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_mountpoint_SUITE.erl b/test/emqx_mountpoint_SUITE.erl index ed37dce2e..cb62db051 100644 --- a/test/emqx_mountpoint_SUITE.erl +++ b/test/emqx_mountpoint_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_mqtt_SUITE.erl b/test/emqx_mqtt_SUITE.erl index 853680edf..cb6174712 100644 --- a/test/emqx_mqtt_SUITE.erl +++ b/test/emqx_mqtt_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_mqtt_caps_SUITE.erl b/test/emqx_mqtt_caps_SUITE.erl index c40f484ae..d6cd5925b 100644 --- a/test/emqx_mqtt_caps_SUITE.erl +++ b/test/emqx_mqtt_caps_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_mqtt_props_SUITE.erl b/test/emqx_mqtt_props_SUITE.erl index ba2064419..2e96182b0 100644 --- a/test/emqx_mqtt_props_SUITE.erl +++ b/test/emqx_mqtt_props_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_mqueue_SUITE.erl b/test/emqx_mqueue_SUITE.erl index 43320be47..34e509145 100644 --- a/test/emqx_mqueue_SUITE.erl +++ b/test/emqx_mqueue_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_os_mon_SUITE.erl b/test/emqx_os_mon_SUITE.erl index cb57b900f..f7abd094b 100644 --- a/test/emqx_os_mon_SUITE.erl +++ b/test/emqx_os_mon_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_packet_SUITE.erl b/test/emqx_packet_SUITE.erl index 468a461fd..4c3221107 100644 --- a/test/emqx_packet_SUITE.erl +++ b/test/emqx_packet_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_passwd_SUITE.erl b/test/emqx_passwd_SUITE.erl index 59d5247bb..fe4694294 100644 --- a/test/emqx_passwd_SUITE.erl +++ b/test/emqx_passwd_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_pd_SUITE.erl b/test/emqx_pd_SUITE.erl index 4ebac5e74..ecd5b9e81 100644 --- a/test/emqx_pd_SUITE.erl +++ b/test/emqx_pd_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_plugins_SUITE.erl b/test/emqx_plugins_SUITE.erl index b3a524848..401cf87dd 100644 --- a/test/emqx_plugins_SUITE.erl +++ b/test/emqx_plugins_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_pmon_SUITE.erl b/test/emqx_pmon_SUITE.erl index 777c6db4d..382f885a9 100644 --- a/test/emqx_pmon_SUITE.erl +++ b/test/emqx_pmon_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_pool_SUITE.erl b/test/emqx_pool_SUITE.erl index 23eec531f..c41900ca9 100644 --- a/test/emqx_pool_SUITE.erl +++ b/test/emqx_pool_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_pqueue_SUITE.erl b/test/emqx_pqueue_SUITE.erl index a3fc64f08..4930f547d 100644 --- a/test/emqx_pqueue_SUITE.erl +++ b/test/emqx_pqueue_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_reason_codes_SUITE.erl b/test/emqx_reason_codes_SUITE.erl index 2b9a57aed..fe261899f 100644 --- a/test/emqx_reason_codes_SUITE.erl +++ b/test/emqx_reason_codes_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_request_handler.erl b/test/emqx_request_handler.erl index 921b1a713..ddd0a563f 100644 --- a/test/emqx_request_handler.erl +++ b/test/emqx_request_handler.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_request_responser_SUITE.erl b/test/emqx_request_responser_SUITE.erl index be6ff623d..91d50d280 100644 --- a/test/emqx_request_responser_SUITE.erl +++ b/test/emqx_request_responser_SUITE.erl @@ -1,4 +1,4 @@ -%% Copyright (c) 2013-2019 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2013-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_request_sender.erl b/test/emqx_request_sender.erl index 1ae78b37c..ac85b2966 100644 --- a/test/emqx_request_sender.erl +++ b/test/emqx_request_sender.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_router_SUITE.erl b/test/emqx_router_SUITE.erl index 731d47fb9..d16659844 100644 --- a/test/emqx_router_SUITE.erl +++ b/test/emqx_router_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_router_helper_SUITE.erl b/test/emqx_router_helper_SUITE.erl index 338a7ba65..553f88b61 100644 --- a/test/emqx_router_helper_SUITE.erl +++ b/test/emqx_router_helper_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_sequence_SUITE.erl b/test/emqx_sequence_SUITE.erl index f91012c0b..4a622f5c8 100644 --- a/test/emqx_sequence_SUITE.erl +++ b/test/emqx_sequence_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_session_SUITE.erl b/test/emqx_session_SUITE.erl index 089087f9e..cb7c10cae 100644 --- a/test/emqx_session_SUITE.erl +++ b/test/emqx_session_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_shared_sub_SUITE.erl b/test/emqx_shared_sub_SUITE.erl index 2e8e624c6..54a0de6d2 100644 --- a/test/emqx_shared_sub_SUITE.erl +++ b/test/emqx_shared_sub_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_stats_SUITE.erl b/test/emqx_stats_SUITE.erl index 64b907fb6..b37f0975b 100644 --- a/test/emqx_stats_SUITE.erl +++ b/test/emqx_stats_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_sup_SUITE.erl b/test/emqx_sup_SUITE.erl index 80b3cdf98..51672e1b6 100644 --- a/test/emqx_sup_SUITE.erl +++ b/test/emqx_sup_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_sys_SUITE.erl b/test/emqx_sys_SUITE.erl index 0814f8b00..29583524f 100644 --- a/test/emqx_sys_SUITE.erl +++ b/test/emqx_sys_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_sys_mon_SUITE.erl b/test/emqx_sys_mon_SUITE.erl index a1f51b2e7..70f518ad5 100644 --- a/test/emqx_sys_mon_SUITE.erl +++ b/test/emqx_sys_mon_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_tables_SUITE.erl b/test/emqx_tables_SUITE.erl index a48475b39..17468cf17 100644 --- a/test/emqx_tables_SUITE.erl +++ b/test/emqx_tables_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_takeover_SUITE.erl b/test/emqx_takeover_SUITE.erl index 3238d2bc9..ebc942e7d 100644 --- a/test/emqx_takeover_SUITE.erl +++ b/test/emqx_takeover_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_topic_SUITE.erl b/test/emqx_topic_SUITE.erl index 782e05320..0cccb74bb 100644 --- a/test/emqx_topic_SUITE.erl +++ b/test/emqx_topic_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_tracer_SUITE.erl b/test/emqx_tracer_SUITE.erl index 94bac7e3d..f5af65440 100644 --- a/test/emqx_tracer_SUITE.erl +++ b/test/emqx_tracer_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -23,7 +23,7 @@ -include_lib("common_test/include/ct.hrl"). -all() -> [t_start_traces]. +all() -> [t_trace_clientid, t_trace_topic]. init_per_suite(Config) -> emqx_ct_helpers:boot_modules(all), @@ -32,9 +32,8 @@ init_per_suite(Config) -> end_per_suite(_Config) -> emqx_ct_helpers:stop_apps([]). - -t_start_traces(_Config) -> +t_trace_clientid(_Config) -> {ok, T} = emqtt:start_link([{host, "localhost"}, {clientid, <<"client">>}, {username, <<"testuser">>}, @@ -51,21 +50,17 @@ t_start_traces(_Config) -> ok = emqx_tracer:start_trace({clientid, <<"client3">>}, all, "tmp/client3.log"), {error, {invalid_log_level, bad_level}} = emqx_tracer:start_trace({clientid, <<"client4">>}, bad_level, "tmp/client4.log"), {error, {handler_not_added, {file_error,".",eisdir}}} = emqx_tracer:start_trace({clientid, <<"client5">>}, debug, "."), - ok = emqx_tracer:start_trace({topic, <<"a/#">>}, all, "tmp/topic_trace.log"), - ok = emqx_tracer:start_trace({topic, <<"b/#">>}, all, "tmp/topic_trace.log"), ct:sleep(100), %% Verify the tracing file exits ?assert(filelib:is_regular("tmp/client.log")), ?assert(filelib:is_regular("tmp/client2.log")), - ?assert(filelib:is_regular("tmp/topic_trace.log")), %% Get current traces ?assertEqual([{{clientid,"client"},{debug,"tmp/client.log"}}, {{clientid,"client2"},{debug,"tmp/client2.log"}}, - {{clientid,"client3"},{debug,"tmp/client3.log"}}, - {{topic,"a/#"},{debug,"tmp/topic_trace.log"}}, - {{topic,"b/#"},{debug,"tmp/topic_trace.log"}}], emqx_tracer:lookup_traces()), + {{clientid,"client3"},{debug,"tmp/client3.log"}} + ], emqx_tracer:lookup_traces()), %% set the overall log level to debug emqx_logger:set_log_level(debug), @@ -74,19 +69,52 @@ t_start_traces(_Config) -> emqtt:publish(T, <<"a/b/c">>, <<"hi">>), ct:sleep(200), - %% Verify messages are logged to "tmp/client.log" and "tmp/topic_trace.log", but not "tmp/client2.log". + %% Verify messages are logged to "tmp/client.log" but not "tmp/client2.log". ?assert(filelib:file_size("tmp/client.log") > 0), - ?assert(filelib:file_size("tmp/topic_trace.log") > 0), ?assert(filelib:file_size("tmp/client2.log") == 0), %% Stop tracing ok = emqx_tracer:stop_trace({clientid, <<"client">>}), ok = emqx_tracer:stop_trace({clientid, <<"client2">>}), ok = emqx_tracer:stop_trace({clientid, <<"client3">>}), - ok = emqx_tracer:stop_trace({topic, <<"a/#">>}), - ok = emqx_tracer:stop_trace({topic, <<"b/#">>}), - {error, _Reason} = emqx_tracer:stop_trace({topic, <<"c/#">>}), emqtt:disconnect(T), emqx_logger:set_log_level(warning). - \ No newline at end of file + +t_trace_topic(_Config) -> + {ok, T} = emqtt:start_link([{host, "localhost"}, + {clientid, <<"client">>}, + {username, <<"testuser">>}, + {password, <<"pass">>} + ]), + emqtt:connect(T), + + %% Start tracing + emqx_logger:set_log_level(debug), + ok = emqx_tracer:start_trace({topic, <<"x/#">>}, all, "tmp/topic_trace.log"), + ok = emqx_tracer:start_trace({topic, <<"y/#">>}, all, "tmp/topic_trace.log"), + ct:sleep(100), + + %% Verify the tracing file exits + ?assert(filelib:is_regular("tmp/topic_trace.log")), + + %% Get current traces + ?assertEqual([{{topic,"x/#"},{debug,"tmp/topic_trace.log"}}, + {{topic,"y/#"},{debug,"tmp/topic_trace.log"}}], emqx_tracer:lookup_traces()), + + %% set the overall log level to debug + emqx_logger:set_log_level(debug), + + %% Client with clientid = "client" publishes a "hi" message to "x/y/z". + emqtt:publish(T, <<"x/y/z">>, <<"hi">>), + ct:sleep(200), + + ?assert(filelib:file_size("tmp/topic_trace.log") > 0), + + %% Stop tracing + ok = emqx_tracer:stop_trace({topic, <<"x/#">>}), + ok = emqx_tracer:stop_trace({topic, <<"y/#">>}), + {error, _Reason} = emqx_tracer:stop_trace({topic, <<"z/#">>}), + emqtt:disconnect(T), + + emqx_logger:set_log_level(warning). diff --git a/test/emqx_trie_SUITE.erl b/test/emqx_trie_SUITE.erl index a489222e3..7516b58a0 100644 --- a/test/emqx_trie_SUITE.erl +++ b/test/emqx_trie_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -23,9 +23,25 @@ -include_lib("eunit/include/eunit.hrl"). -define(TRIE, emqx_trie). --define(TRIE_TABS, [emqx_trie, emqx_trie_node]). -all() -> emqx_ct:all(?MODULE). +all() -> + [{group, compact}, + {group, not_compact} + ]. + +groups() -> + Cases = emqx_ct:all(?MODULE), + [{compact, Cases}, {not_compact, Cases}]. + +init_per_group(compact, Config) -> + emqx_trie:put_compaction_flag(true), + Config; +init_per_group(not_compact, Config) -> + emqx_trie:put_compaction_flag(false), + Config. + +end_per_group(_, _) -> + emqx_trie:put_default_compaction_flag(). init_per_suite(Config) -> application:load(emqx), @@ -38,6 +54,7 @@ end_per_suite(_Config) -> ekka_mnesia:delete_schema(). init_per_testcase(_TestCase, Config) -> + clear_tables(), Config. end_per_testcase(_TestCase, _Config) -> @@ -47,50 +64,56 @@ t_mnesia(_) -> ok = ?TRIE:mnesia(copy). t_insert(_) -> - TN = #trie_node{node_id = <<"sensor">>, - edge_count = 3, - topic = <<"sensor">>, - flags = undefined - }, Fun = fun() -> ?TRIE:insert(<<"sensor/1/metric/2">>), ?TRIE:insert(<<"sensor/+/#">>), - ?TRIE:insert(<<"sensor/#">>), - ?TRIE:insert(<<"sensor">>), - ?TRIE:insert(<<"sensor">>), - ?TRIE:lookup(<<"sensor">>) + ?TRIE:insert(<<"sensor/#">>) end, - ?assertEqual({atomic, [TN]}, trans(Fun)). + ?assertEqual({atomic, ok}, trans(Fun)), + ?assertEqual([<<"sensor/#">>], ?TRIE:match(<<"sensor">>)). t_match(_) -> - Machted = [<<"sensor/+/#">>, <<"sensor/#">>], - Fun = fun() -> + Machted = [<<"sensor/#">>, <<"sensor/+/#">>], + trans(fun() -> ?TRIE:insert(<<"sensor/1/metric/2">>), ?TRIE:insert(<<"sensor/+/#">>), - ?TRIE:insert(<<"sensor/#">>), - ?TRIE:match(<<"sensor/1">>) - end, - ?assertEqual({atomic, Machted}, trans(Fun)). + ?TRIE:insert(<<"sensor/#">>) + end), + ?assertEqual(Machted, lists:sort(?TRIE:match(<<"sensor/1">>))). + +t_match_invalid(_) -> + trans(fun() -> + ?TRIE:insert(<<"sensor/1/metric/2">>), + ?TRIE:insert(<<"sensor/+/#">>), + ?TRIE:insert(<<"sensor/#">>) + end), + ?assertEqual([], lists:sort(?TRIE:match(<<"sensor/+">>))), + ?assertEqual([], lists:sort(?TRIE:match(<<"#">>))). + t_match2(_) -> - Matched = {[<<"+/+/#">>, <<"+/#">>, <<"#">>], []}, - Fun = fun() -> + Matched = [<<"#">>, <<"+/#">>, <<"+/+/#">>], + trans(fun() -> ?TRIE:insert(<<"#">>), ?TRIE:insert(<<"+/#">>), - ?TRIE:insert(<<"+/+/#">>), - {?TRIE:match(<<"a/b/c">>), - ?TRIE:match(<<"$SYS/broker/zenmq">>)} - end, - ?assertEqual({atomic, Matched}, trans(Fun)). + ?TRIE:insert(<<"+/+/#">>) + end), + ?assertEqual(Matched, lists:sort(?TRIE:match(<<"a/b/c">>))), + ?assertEqual([], ?TRIE:match(<<"$SYS/broker/zenmq">>)). t_match3(_) -> Topics = [<<"d/#">>, <<"a/b/c">>, <<"a/b/+">>, <<"a/#">>, <<"#">>, <<"$SYS/#">>], trans(fun() -> [emqx_trie:insert(Topic) || Topic <- Topics] end), Matched = mnesia:async_dirty(fun emqx_trie:match/1, [<<"a/b/c">>]), ?assertEqual(4, length(Matched)), - SysMatched = mnesia:async_dirty(fun emqx_trie:match/1, [<<"$SYS/a/b/c">>]), + SysMatched = emqx_trie:match(<<"$SYS/a/b/c">>), ?assertEqual([<<"$SYS/#">>], SysMatched). +t_match4(_) -> + Topics = [<<"/#">>, <<"/+">>, <<"/+/a/b/c">>], + trans(fun() -> lists:foreach(fun emqx_trie:insert/1, Topics) end), + ?assertEqual([<<"/#">>, <<"/+/a/b/c">>], lists:sort(emqx_trie:match(<<"/0/a/b/c">>))). + t_empty(_) -> ?assert(?TRIE:empty()), trans(fun ?TRIE:insert/1, [<<"topic/x/#">>]), @@ -99,56 +122,50 @@ t_empty(_) -> ?assert(?TRIE:empty()). t_delete(_) -> - TN = #trie_node{node_id = <<"sensor/1">>, - edge_count = 2, - topic = undefined, - flags = undefined}, - Fun = fun() -> + trans(fun() -> ?TRIE:insert(<<"sensor/1/#">>), ?TRIE:insert(<<"sensor/1/metric/2">>), - ?TRIE:insert(<<"sensor/1/metric/3">>), + ?TRIE:insert(<<"sensor/1/metric/3">>) + end), + trans(fun() -> ?TRIE:delete(<<"sensor/1/metric/2">>), ?TRIE:delete(<<"sensor/1/metric">>), - ?TRIE:delete(<<"sensor/1/metric">>), - ?TRIE:lookup(<<"sensor/1">>) - end, - ?assertEqual({atomic, [TN]}, trans(Fun)). + ?TRIE:delete(<<"sensor/1/metric">>) + end), + ?assertEqual([<<"sensor/1/#">>], ?TRIE:match(<<"sensor/1/x">>)). t_delete2(_) -> - Fun = fun() -> + trans(fun() -> ?TRIE:insert(<<"sensor">>), ?TRIE:insert(<<"sensor/1/metric/2">>), - ?TRIE:insert(<<"sensor/+/metric/3">>), + ?TRIE:insert(<<"sensor/+/metric/3">>) + end), + trans(fun() -> ?TRIE:delete(<<"sensor">>), ?TRIE:delete(<<"sensor/1/metric/2">>), ?TRIE:delete(<<"sensor/+/metric/3">>), - ?TRIE:delete(<<"sensor/+/metric/3">>), - {?TRIE:lookup(<<"sensor">>), ?TRIE:lookup(<<"sensor/1">>)} - end, - ?assertEqual({atomic, {[], []}}, trans(Fun)). + ?TRIE:delete(<<"sensor/+/metric/3">>) + end), + ?assertEqual([], ?TRIE:match(<<"sensor">>)), + ?assertEqual([], ?TRIE:match(<<"sensor/1">>)). t_delete3(_) -> - Fun = fun() -> + trans(fun() -> ?TRIE:insert(<<"sensor/+">>), ?TRIE:insert(<<"sensor/+/metric/2">>), - ?TRIE:insert(<<"sensor/+/metric/3">>), + ?TRIE:insert(<<"sensor/+/metric/3">>) + end), + trans(fun() -> ?TRIE:delete(<<"sensor/+/metric/2">>), ?TRIE:delete(<<"sensor/+/metric/3">>), ?TRIE:delete(<<"sensor">>), ?TRIE:delete(<<"sensor/+">>), - ?TRIE:delete(<<"sensor/+/unknown">>), - {?TRIE:lookup(<<"sensor">>), ?TRIE:lookup(<<"sensor/+">>)} - end, - ?assertEqual({atomic, {[], []}}, trans(Fun)). + ?TRIE:delete(<<"sensor/+/unknown">>) + end), + ?assertEqual([], ?TRIE:match(<<"sensor">>)), + ?assertEqual([], ?TRIE:lookup_topic(<<"sensor/+">>)). -t_triples(_) -> - Triples = [{root,<<"a">>,<<"a">>}, - {<<"a">>,<<"b">>,<<"a/b">>}, - {<<"a/b">>,<<"c">>,<<"a/b/c">>}], - ?assertEqual(Triples, emqx_trie:triples(<<"a/b/c">>)). - -clear_tables() -> - lists:foreach(fun mnesia:clear_table/1, ?TRIE_TABS). +clear_tables() -> emqx_trie:clear_tables(). trans(Fun) -> mnesia:transaction(Fun). diff --git a/test/emqx_vm_SUITE.erl b/test/emqx_vm_SUITE.erl index 268bf7c41..51905010a 100644 --- a/test/emqx_vm_SUITE.erl +++ b/test/emqx_vm_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_vm_mon_SUITE.erl b/test/emqx_vm_mon_SUITE.erl index f1a169b49..5f9f4084c 100644 --- a/test/emqx_vm_mon_SUITE.erl +++ b/test/emqx_vm_mon_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/emqx_ws_connection_SUITE.erl b/test/emqx_ws_connection_SUITE.erl index 04018b141..6db831972 100644 --- a/test/emqx_ws_connection_SUITE.erl +++ b/test/emqx_ws_connection_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2019-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -50,6 +50,7 @@ init_per_testcase(TestCase, Config) when -> %% Mock cowboy_req ok = meck:new(cowboy_req, [passthrough, no_history, no_link]), + ok = meck:expect(cowboy_req, header, fun(_, _, _) -> <<>> end), ok = meck:expect(cowboy_req, peer, fun(_) -> {{127,0,0,1}, 3456} end), ok = meck:expect(cowboy_req, sock, fun(_) -> {{127,0,0,1}, 18083} end), ok = meck:expect(cowboy_req, cert, fun(_) -> undefined end), @@ -123,6 +124,22 @@ t_info(_) -> sockstate := running } = SockInfo. +t_header(_) -> + ok = meck:expect(cowboy_req, header, fun(<<"x-forwarded-for">>, _, _) -> <<"100.100.100.100, 99.99.99.99">>; + (<<"x-forwarded-port">>, _, _) -> <<"1000">> end), + {ok, St, _} = ?ws_conn:websocket_init([req, [{zone, external}, + {proxy_address_header, <<"x-forwarded-for">>}, + {proxy_port_header, <<"x-forwarded-port">>}]]), + WsPid = spawn(fun() -> + receive {call, From, info} -> + gen_server:reply(From, ?ws_conn:info(St)) + end end), + #{sockinfo := SockInfo} = ?ws_conn:call(WsPid, info), + #{socktype := ws, + peername := {{100,100,100,100}, 1000}, + sockstate := running + } = SockInfo. + t_info_limiter(_) -> St = st(#{limiter => emqx_limiter:init(external, [])}), ?assertEqual(undefined, ?ws_conn:info(limiter, St)). diff --git a/test/emqx_zone_SUITE.erl b/test/emqx_zone_SUITE.erl index f3d623a56..8294ac0da 100644 --- a/test/emqx_zone_SUITE.erl +++ b/test/emqx_zone_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/mqtt_protocol_v5_SUITE.erl b/test/mqtt_protocol_v5_SUITE.erl index 814162a4a..0956a07fc 100644 --- a/test/mqtt_protocol_v5_SUITE.erl +++ b/test/mqtt_protocol_v5_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/props/prop_emqx_base62.erl b/test/props/prop_emqx_base62.erl index 4bae49524..c6854ed36 100644 --- a/test/props/prop_emqx_base62.erl +++ b/test/props/prop_emqx_base62.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/props/prop_emqx_frame.erl b/test/props/prop_emqx_frame.erl index 8ca847043..5f924a3e5 100644 --- a/test/props/prop_emqx_frame.erl +++ b/test/props/prop_emqx_frame.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/props/prop_emqx_json.erl b/test/props/prop_emqx_json.erl index 86f57a1d3..819b029d2 100644 --- a/test/props/prop_emqx_json.erl +++ b/test/props/prop_emqx_json.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -135,13 +135,7 @@ json_basic() -> oneof([true, false, null, number(), json_string()]). latin_atom() -> - ?LET(L, list(latin_char()), list_to_atom(L)). - -latin_char() -> - L = lists:concat([lists:seq($0, $9), - lists:seq($a, $z), - lists:seq($A, $Z)]), - oneof(L). + emqx_ct_proper_types:limited_latin_atom(). json_string() -> utf8(). diff --git a/test/props/prop_emqx_psk.erl b/test/props/prop_emqx_psk.erl index d853c1b93..106de3fda 100644 --- a/test/props/prop_emqx_psk.erl +++ b/test/props/prop_emqx_psk.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/props/prop_emqx_reason_codes.erl b/test/props/prop_emqx_reason_codes.erl index 5469456ea..141c6b137 100644 --- a/test/props/prop_emqx_reason_codes.erl +++ b/test/props/prop_emqx_reason_codes.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/test/props/prop_emqx_rpc.erl b/test/props/prop_emqx_rpc.erl index ab876db39..18f34ba8d 100644 --- a/test/props/prop_emqx_rpc.erl +++ b/test/props/prop_emqx_rpc.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -19,6 +19,8 @@ -include_lib("proper/include/proper.hrl"). -include_lib("eunit/include/eunit.hrl"). +-define(NODENAME, 'test@127.0.0.1'). + -define(ALL(Vars, Types, Exprs), ?SETUP(fun() -> State = do_setup(), @@ -30,8 +32,9 @@ %%-------------------------------------------------------------------- prop_node() -> - ?ALL(Node, nodename(), + ?ALL(Node0, nodename(), begin + Node = punch(Node0), ?assert(emqx_rpc:cast(Node, erlang, system_time, [])), case emqx_rpc:call(Node, erlang, system_time, []) of {badrpc, _Reason} -> true; @@ -41,8 +44,9 @@ prop_node() -> end). prop_node_with_key() -> - ?ALL({Node, Key}, nodename_with_key(), + ?ALL({Node0, Key}, nodename_with_key(), begin + Node = punch(Node0), ?assert(emqx_rpc:cast(Key, Node, erlang, system_time, [])), case emqx_rpc:call(Key, Node, erlang, system_time, []) of {badrpc, _Reason} -> true; @@ -52,8 +56,9 @@ prop_node_with_key() -> end). prop_nodes() -> - ?ALL(Nodes, nodesname(), + ?ALL(Nodes0, nodesname(), begin + Nodes = punch(Nodes0), case emqx_rpc:multicall(Nodes, erlang, system_time, []) of {badrpc, _Reason} -> true; {RealResults, RealBadNodes} @@ -65,8 +70,9 @@ prop_nodes() -> end). prop_nodes_with_key() -> - ?ALL({Nodes, Key}, nodesname_with_key(), + ?ALL({Nodes0, Key}, nodesname_with_key(), begin + Nodes = punch(Nodes0), case emqx_rpc:multicall(Key, Nodes, erlang, system_time, []) of {badrpc, _Reason} -> true; {RealResults, RealBadNodes} @@ -82,24 +88,42 @@ prop_nodes_with_key() -> %%-------------------------------------------------------------------- do_setup() -> + ensure_distributed_nodename(), + ok = logger:set_primary_config(#{level => warning}), {ok, _Apps} = application:ensure_all_started(gen_rpc), - ok = application:set_env(gen_rpc, call_receive_timeout, 1), - ok = emqx_logger:set_log_level(emergency), + ok = application:set_env(gen_rpc, call_receive_timeout, 100), ok = meck:new(gen_rpc, [passthrough, no_history]), ok = meck:expect(gen_rpc, multicall, fun(Nodes, Mod, Fun, Args) -> - gen_rpc:multicall(Nodes, Mod, Fun, Args, 1) + gen_rpc:multicall(Nodes, Mod, Fun, Args, 100) end). do_teardown(_) -> - ok = emqx_logger:set_log_level(debug), + ok = net_kernel:stop(), ok = application:stop(gen_rpc), - ok = meck:unload(gen_rpc). + ok = meck:unload(gen_rpc), + %% wait for tcp close + timer:sleep(1500). + +ensure_distributed_nodename() -> + case net_kernel:start([?NODENAME]) of + {ok, _} -> + ok; + {error, {already_started, _}} -> + net_kernel:stop(), + net_kernel:start([?NODENAME]); + {error, {{shutdown, {_, _, {'EXIT', nodistribution}}}, _}} -> + %% start epmd first + spawn_link(fun() -> os:cmd("epmd") end), + timer:sleep(100), + net_kernel:start([?NODENAME]) + end. %%-------------------------------------------------------------------- %% Generator %%-------------------------------------------------------------------- + nodename() -> ?LET({NodePrefix, HostName}, {node_prefix(), hostname()}, @@ -126,7 +150,21 @@ node_prefix() -> oneof(["emqxct", text_like()]). text_like() -> - ?SUCHTHAT(Text, list(range($a, $z)), (length(Text) =< 5 andalso length(Text) > 0)). + ?SUCHTHAT(Text, list(range($a, $z)), (length(Text) =< 100 andalso length(Text) > 0)). hostname() -> oneof(["127.0.0.1", "localhost"]). + +%%-------------------------------------------------------------------- +%% Utils +%%-------------------------------------------------------------------- + +%% After running the props, the `node()` () is only able to return an +%% incorrect node name - `nonode@nohost`, But we want a distributed nodename +%% So, just translate the `nonode@nohost` to ?NODENAME +punch(Nodes) when is_list(Nodes) -> + lists:map(fun punch/1, Nodes); +punch('nonode@nohost') -> + node(); %% Equal to ?NODENAME +punch(GoodBoy) -> + GoodBoy. diff --git a/test/props/prop_emqx_sys.erl b/test/props/prop_emqx_sys.erl index d80edbe4e..67718ec37 100644 --- a/test/props/prop_emqx_sys.erl +++ b/test/props/prop_emqx_sys.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License.